diff --git a/egs/librispeech/ASR/pruned_transducer_stateless5/.conformer.py.swp b/egs/librispeech/ASR/pruned_transducer_stateless5/.conformer.py.swp
index e8ba22351..6d87c6d76 100644
Binary files a/egs/librispeech/ASR/pruned_transducer_stateless5/.conformer.py.swp and b/egs/librispeech/ASR/pruned_transducer_stateless5/.conformer.py.swp differ
diff --git a/egs/librispeech/ASR/pruned_transducer_stateless5/conformer.py b/egs/librispeech/ASR/pruned_transducer_stateless5/conformer.py
index 2b76b3dbc..e731ded4f 100644
--- a/egs/librispeech/ASR/pruned_transducer_stateless5/conformer.py
+++ b/egs/librispeech/ASR/pruned_transducer_stateless5/conformer.py
@@ -696,7 +696,9 @@ class ConformerEncoder(nn.Module):
         residual = None
 
         for i, mod in enumerate(self.layers):
-            if i in [2,5,8]:
+            if i == 2:
+                residual = output
+            elif i in [2,5,8]:
                 residual = output
                 output = mod(
                     output,