From 606d3bd2d3e2fa8f4f976ad0f72ae75b8018625a Mon Sep 17 00:00:00 2001 From: Daniel Povey Date: Sat, 8 Oct 2022 12:55:11 +0800 Subject: [PATCH] Do dropout a different way --- .../ASR/pruned_transducer_stateless7/conformer.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/egs/librispeech/ASR/pruned_transducer_stateless7/conformer.py b/egs/librispeech/ASR/pruned_transducer_stateless7/conformer.py index 261a2981a..bbd0bd6e7 100644 --- a/egs/librispeech/ASR/pruned_transducer_stateless7/conformer.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless7/conformer.py @@ -369,9 +369,9 @@ class ConformerEncoderLayer(nn.Module): warmup_value = self.get_warmup_value(warmup_count) if warmup_value < 1.0 and self.training: - delta = torch.nn.functional.dropout(src_orig - src, - p=0.5 * (1. - warmup_value), - training=self.training) + delta = src_orig - src + keep_prob = 0.5 * (1. + warmup_value) + delta = delta * (torch.rand_like(delta) < keep_prob) src = src_orig + delta @@ -481,6 +481,8 @@ class ConformerEncoder(nn.Module): # this seemed to be helpful... output = 0.5 * (next_output + output) + output = output * feature_mask + return output