From 3a71a53d8d71583259fc099c1f9a3270c703fd73 Mon Sep 17 00:00:00 2001 From: Daniel Povey Date: Tue, 23 May 2023 10:55:33 +0800 Subject: [PATCH] Set lr_factor on to_scores, max_abs=4.0 on balancer --- egs/libriheavy/LM/zipformer1/subformer.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/egs/libriheavy/LM/zipformer1/subformer.py b/egs/libriheavy/LM/zipformer1/subformer.py index c049ab898..6b92b5fc8 100644 --- a/egs/libriheavy/LM/zipformer1/subformer.py +++ b/egs/libriheavy/LM/zipformer1/subformer.py @@ -813,6 +813,7 @@ class LearnedDownsamplingModule(nn.Module): super().__init__() self.to_scores = nn.Linear(embed_dim, 1, bias=False) + self.to_scores.lr_factor = 0.5 # score_balancer is just to keep the magnitudes of the scores in # a fixed range and keep them balanced around zero, to stop # these drifting around. @@ -820,7 +821,8 @@ class LearnedDownsamplingModule(nn.Module): self.score_balancer = Balancer(1, channel_dim=-1, min_positive=1/(2*downsampling_factor), max_positive=0.6, - min_abs=1.0) + min_abs=1.0, + max_abs=4.0) # below are for diagnostics. self.copy_weights1 = nn.Identity()