diff --git a/egs/libriheavy/LM/zipformer1/subformer.py b/egs/libriheavy/LM/zipformer1/subformer.py
index e55de4a42..0d7f0248b 100644
--- a/egs/libriheavy/LM/zipformer1/subformer.py
+++ b/egs/libriheavy/LM/zipformer1/subformer.py
@@ -430,8 +430,6 @@ class SubformerEncoderLayer(nn.Module):
 
         self.norm = BiasNorm(embed_dim)
 
-        self.bypass_scale = nn.Parameter(torch.full((embed_dim,), 0.5))
-
         self.balancer1 = Balancer(
             embed_dim, channel_dim=-1,
             min_positive=0.45, max_positive=0.55,