diff --git a/egs/librispeech/ASR/pruned_transducer_stateless5/.conformer.py.swp b/egs/librispeech/ASR/pruned_transducer_stateless5/.conformer.py.swp
index a3dcb4b2f..e8ba22351 100644
Binary files a/egs/librispeech/ASR/pruned_transducer_stateless5/.conformer.py.swp and b/egs/librispeech/ASR/pruned_transducer_stateless5/.conformer.py.swp differ
diff --git a/egs/librispeech/ASR/pruned_transducer_stateless5/conformer.py b/egs/librispeech/ASR/pruned_transducer_stateless5/conformer.py
index 97ded1f54..2b76b3dbc 100644
--- a/egs/librispeech/ASR/pruned_transducer_stateless5/conformer.py
+++ b/egs/librispeech/ASR/pruned_transducer_stateless5/conformer.py
@@ -693,17 +693,27 @@ class ConformerEncoder(nn.Module):
         output = src
 
         outputs = []
-        
+        residual = None
+
         for i, mod in enumerate(self.layers):
-            if random.random() < 0.05:
-                continue
-            output = mod(
-                output,
-                pos_emb,
-                src_mask=mask,
-                src_key_padding_mask=src_key_padding_mask,
-                warmup=warmup,
-            )
+            if i in [2,5,8]:
+                residual = output
+                output = mod(
+                    output,
+                    pos_emb,
+                    src_mask=mask,
+                    src_key_padding_mask=src_key_padding_mask,
+                    warmup=warmup,
+                )
+                output += residual
+            else:
+                output = mod(
+                    output,
+                    pos_emb,
+                    src_mask=mask,
+                    src_key_padding_mask=src_key_padding_mask,
+                    warmup=warmup,
+                )
             #if i in self.aux_layers:
             #    outputs.append(output)