diff --git a/egs/librispeech/ASR/pruned_transducer_stateless2/model.py b/egs/librispeech/ASR/pruned_transducer_stateless2/model.py
index ab729a429..0355c4531 100644
--- a/egs/librispeech/ASR/pruned_transducer_stateless2/model.py
+++ b/egs/librispeech/ASR/pruned_transducer_stateless2/model.py
@@ -61,8 +61,10 @@ class Transducer(nn.Module):
         self.decoder = decoder
         self.joiner = joiner
 
-        self.simple_am_proj = ScaledLinear(embedding_dim, vocab_size)
-        self.simple_lm_proj = ScaledLinear(embedding_dim, vocab_size)
+        self.simple_am_proj = ScaledLinear(embedding_dim, vocab_size,
+                                           initial_speed=0.5)
+        self.simple_lm_proj = ScaledLinear(embedding_dim, vocab_size,
+                                           initial_speed=0.5)
         with torch.no_grad():
             # Initialize the two projections to be the same; this will be
             # convenient for the real joiner, which adds the endcoder