diff --git a/egs/librispeech/ASR/pruned_transducer_stateless2/model.py b/egs/librispeech/ASR/pruned_transducer_stateless2/model.py
index f1a3d4d11..9fef48fcc 100644
--- a/egs/librispeech/ASR/pruned_transducer_stateless2/model.py
+++ b/egs/librispeech/ASR/pruned_transducer_stateless2/model.py
@@ -63,7 +63,8 @@ class Transducer(nn.Module):
 
         # could perhaps separate this into 2 linear projections, one
         # for lm and one for am.
-        self.simple_joiner = ScaledLinear(embedding_dim, vocab_size)
+        self.simple_joiner = ScaledLinear(embedding_dim, vocab_size,
+                                          initial_speed=0.5)
 
     def forward(
         self,