From c6bad1ee4f0757a0ea65bc497a294578c7b43608 Mon Sep 17 00:00:00 2001 From: Daniel Povey Date: Thu, 29 Dec 2022 18:50:12 +0800 Subject: [PATCH] Start ff modules with larger initial_scale --- egs/librispeech/ASR/pruned_transducer_stateless7/zipformer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/egs/librispeech/ASR/pruned_transducer_stateless7/zipformer.py b/egs/librispeech/ASR/pruned_transducer_stateless7/zipformer.py index 105e7ddea..763183b92 100644 --- a/egs/librispeech/ASR/pruned_transducer_stateless7/zipformer.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless7/zipformer.py @@ -1405,7 +1405,7 @@ class FeedforwardModule(nn.Module): self.activation = SwooshL() self.dropout = Dropout2(dropout) self.out_proj = ScaledLinear(feedforward_dim, embed_dim, - initial_scale=0.01) + initial_scale=0.1) self.out_whiten = Whiten(num_groups=1, whitening_limit=_whitening_schedule(7.5),