mirrors/icefall
1
0
Fork 0
mirror of https://github.com/k2-fsa/icefall.git synced 2025-12-11 06:55:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Half the time, flip weights_discarded

Browse Source
This commit is contained in:
Daniel Povey 2023-05-19 17:55:03 +08:00
parent 7d162bf41e
commit 4a425f7eb5
1 changed files with 4 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
egs/libriheavy/LM/zipformer1/subformer.py
View File

@ -879,6 +879,10 @@ class LearnedDownsamplingModule(nn.Module):
# we were getting too many discarded weights before introducing this factor, which was # we were getting too many discarded weights before introducing this factor, which was
# hurting test-mode performance by creating a mismatch. # hurting test-mode performance by creating a mismatch.
discarded_weights_factor = 2.0 discarded_weights_factor = 2.0
if random.random() < 0.5:
# flipping it half the time increases the randomness, so gives an extra incentive
# to avoid nonzero weights in the discarded half
weights_discarded = weights_discarded.flip(dims=(1,))
weights = (weights[:, :seq_len_reduced] - (weights_discarded * discarded_weights_factor)).clamp(min=0.0, max=1.0) weights = (weights[:, :seq_len_reduced] - (weights_discarded * discarded_weights_factor)).clamp(min=0.0, max=1.0)
else: else: