diff --git a/egs/librispeech/ASR/incremental_transf/.conformer.py.swp b/egs/librispeech/ASR/incremental_transf/.conformer.py.swp
index db2caf2b9..d619f0b50 100644
Binary files a/egs/librispeech/ASR/incremental_transf/.conformer.py.swp and b/egs/librispeech/ASR/incremental_transf/.conformer.py.swp differ
diff --git a/egs/librispeech/ASR/incremental_transf/conformer.py b/egs/librispeech/ASR/incremental_transf/conformer.py
index b6ff0a156..b2478e46b 100644
--- a/egs/librispeech/ASR/incremental_transf/conformer.py
+++ b/egs/librispeech/ASR/incremental_transf/conformer.py
@@ -480,27 +480,21 @@ class Tempformer(EncoderInterface):
         self.short_chunk_size = short_chunk_size
         self.num_left_chunks = num_left_chunks
 
-        encoder_layer = ConformerEncoderLayer(
-            d_model=d_model,
-            nhead=nhead,
-            dim_feedforward=dim_feedforward,
-            dropout=dropout,
-            layer_dropout=layer_dropout,
-            cnn_module_kernel=cnn_module_kernel,
-            causal=causal,
-        )
-        # aux_layers from 1/3
-        self.encoder = ConformerEncoder(
-            encoder_layer=encoder_layer,
-            num_layers=num_encoder_layers,
-            aux_layers=list(
-                range(
-                    num_encoder_layers // 3,
-                    num_encoder_layers - 1,
-                    aux_layer_period,
-                )
-            ),
-        )
+        def build_conformer(d_model, nhead, dim_feedforward, dropout, layer_dropout, cnn_module_kernel, causal):
+            encoder_layer = ConformerEncoderLayer(
+                d_model=d_model,
+                nhead=nhead,
+                dim_feedforward=dim_feedforward,
+                dropout=dropout,
+                layer_dropout=layer_dropout,
+                cnn_module_kernel=cnn_module_kernel,
+                causal=causal,
+            )
+            return encoder_layer
+
+
+        self.encoder_layers = nn.ModuleList(
+
         self._init_state: List[torch.Tensor] = [torch.empty(0)]
 
     def forward(