From 2368968114805135b85677bdb9497caf82ec4ca2 Mon Sep 17 00:00:00 2001 From: Daniel Povey Date: Fri, 25 Nov 2022 18:00:46 +0800 Subject: [PATCH] Make out_proj of feedforward modules be a LinearWithAuxLoss, with nonzero final value at 0.01. --- .../ASR/pruned_transducer_stateless7/zipformer.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/egs/librispeech/ASR/pruned_transducer_stateless7/zipformer.py b/egs/librispeech/ASR/pruned_transducer_stateless7/zipformer.py index 9e2d29ab1..98e36d749 100644 --- a/egs/librispeech/ASR/pruned_transducer_stateless7/zipformer.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless7/zipformer.py @@ -1390,8 +1390,10 @@ class FeedforwardModule(nn.Module): min_prob=0.25) self.activation = DoubleSwish() self.dropout = nn.Dropout(dropout) - self.out_proj = ScaledLinear(feedforward_dim, embed_dim, - initial_scale=0.01) + self.out_proj = LinearWithAuxLoss(feedforward_dim, embed_dim, + initial_scale=0.01, + aux_grad_scale=ScheduledFloat((0.0, 0.1), (1000.0, 0.01)), + ) self.out_whiten = Whiten(num_groups=1, whitening_limit=_whitening_schedule(7.5), prob=(0.025, 0.25),