From c20fc3be14c2d729e9f7bc51a2158eed02a1a36f Mon Sep 17 00:00:00 2001 From: Daniel Povey Date: Mon, 3 Oct 2022 13:02:42 +0800 Subject: [PATCH 1/3] Randomize order of some modules --- .../ASR/pruned_transducer_stateless7/conformer.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/egs/librispeech/ASR/pruned_transducer_stateless7/conformer.py b/egs/librispeech/ASR/pruned_transducer_stateless7/conformer.py index 9de413fb4..2edff7889 100644 --- a/egs/librispeech/ASR/pruned_transducer_stateless7/conformer.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless7/conformer.py @@ -366,7 +366,20 @@ class ConformerEncoder(nn.Module): else: feature_mask = None - for i, mod in enumerate(self.layers): + + num_layers = len(self.layers) + indexes = list(range(num_layers)) + if self.training: + num_to_swap = int((num_layers - 1) * 0.1) + for i in range(num_to_swap): + j = random.randint(0, num_layers - 2) + a,b = indexes[j], indexes[j+1] + indexes[j] = b + indexes[j+1] = a + + + for i in indexes: + mod = self.layers[i] output, attn_scores = mod( output, pos_emb, From a0a187441534f914eb5c1360852d5955246b3b39 Mon Sep 17 00:00:00 2001 From: Daniel Povey Date: Mon, 3 Oct 2022 13:23:26 +0800 Subject: [PATCH 2/3] Bug fix --- egs/librispeech/ASR/pruned_transducer_stateless7/conformer.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/egs/librispeech/ASR/pruned_transducer_stateless7/conformer.py b/egs/librispeech/ASR/pruned_transducer_stateless7/conformer.py index 2edff7889..d8b1261a6 100644 --- a/egs/librispeech/ASR/pruned_transducer_stateless7/conformer.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless7/conformer.py @@ -391,10 +391,12 @@ class ConformerEncoder(nn.Module): ) if i in self.aux_layers: outputs.append(output) + if i == num_layers - 1: + final_output = output output = self.combiner(outputs) - output = output + 0.0 * attn_scores.sum() # just ensure attn_scores is used in backprop + output = final_output + 0.0 * attn_scores.sum() # just ensure attn_scores is used in backprop return output From 5a8995328fd5481889a877bb2698ccf89c06480b Mon Sep 17 00:00:00 2001 From: Daniel Povey Date: Mon, 3 Oct 2022 13:33:01 +0800 Subject: [PATCH 3/3] Stop backprop bug --- .../ASR/pruned_transducer_stateless7/conformer.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/egs/librispeech/ASR/pruned_transducer_stateless7/conformer.py b/egs/librispeech/ASR/pruned_transducer_stateless7/conformer.py index d8b1261a6..2d3c9a6a5 100644 --- a/egs/librispeech/ASR/pruned_transducer_stateless7/conformer.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless7/conformer.py @@ -378,8 +378,8 @@ class ConformerEncoder(nn.Module): indexes[j+1] = a - for i in indexes: - mod = self.layers[i] + for i,j in enumerate(indexes): + mod = self.layers[j] output, attn_scores = mod( output, pos_emb, @@ -391,12 +391,10 @@ class ConformerEncoder(nn.Module): ) if i in self.aux_layers: outputs.append(output) - if i == num_layers - 1: - final_output = output output = self.combiner(outputs) - output = final_output + 0.0 * attn_scores.sum() # just ensure attn_scores is used in backprop + output = output + 0.0 * attn_scores.sum() # just ensure attn_scores is used in backprop return output