diff --git a/egs/libriheavy/ASR/zipformer_prompt_asr/decoder.py b/egs/libriheavy/ASR/zipformer_prompt_asr/decoder.py
index 93e0f9f7e..8a55eb5c8 100644
--- a/egs/libriheavy/ASR/zipformer_prompt_asr/decoder.py
+++ b/egs/libriheavy/ASR/zipformer_prompt_asr/decoder.py
@@ -95,6 +95,10 @@ class Decoder(nn.Module):
                 max_abs=1.0,
                 prob=0.05,
             )
+        else:
+            # To avoid `RuntimeError: Module 'Decoder' has no attribute 'conv'`
+            # when inference with torch.jit.script and context_size == 1
+            self.conv = nn.Identity()
 
     def forward(self, y: torch.Tensor, need_pad: bool = True) -> torch.Tensor:
         """