From 4a425f7eb5f272dfe3d4b9989dae464c683a8dcd Mon Sep 17 00:00:00 2001 From: Daniel Povey Date: Fri, 19 May 2023 17:55:03 +0800 Subject: [PATCH] Half the time, flip weights_discarded --- egs/libriheavy/LM/zipformer1/subformer.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/egs/libriheavy/LM/zipformer1/subformer.py b/egs/libriheavy/LM/zipformer1/subformer.py index bbf1ad889..c489fd561 100644 --- a/egs/libriheavy/LM/zipformer1/subformer.py +++ b/egs/libriheavy/LM/zipformer1/subformer.py @@ -879,6 +879,10 @@ class LearnedDownsamplingModule(nn.Module): # we were getting too many discarded weights before introducing this factor, which was # hurting test-mode performance by creating a mismatch. discarded_weights_factor = 2.0 + if random.random() < 0.5: + # flipping it half the time increases the randomness, so gives an extra incentive + # to avoid nonzero weights in the discarded half + weights_discarded = weights_discarded.flip(dims=(1,)) weights = (weights[:, :seq_len_reduced] - (weights_discarded * discarded_weights_factor)).clamp(min=0.0, max=1.0) else: