mirrors/icefall
1
0
Fork 0
mirror of https://github.com/k2-fsa/icefall.git synced 2025-12-11 06:55:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Implement extra_layerdrop

Browse Source
This commit is contained in:
Daniel Povey 2023-01-04 20:59:58 +08:00
parent b973929d7c
commit 22b4a417dd
1 changed files with 12 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
egs/librispeech/ASR/pruned_transducer_stateless7/zipformer.py
View File

@ -199,6 +199,15 @@ class Zipformer(EncoderInterface):
attention_share_layers=attention_share_layers[i], attention_share_layers=attention_share_layers[i],
) )
# modify the layerdrop schedule with an extra schedule that takes longer
# to warm up for the less-downsampled layers; this encourages the more
# heavily downsampled layers to learn something.
extra_layerdrop = ScheduledFloat((0.0, 0.2), (20000.0 / downsampling_factor[i], 0.0))
for layer in encoder.layers:
# we can add objects of type ScheduledFloat.
layer.layer_skip_rate = layer.layer_skip_rate + extra_layerdrop
if downsampling_factor[i] != 1: if downsampling_factor[i] != 1:
encoder = DownsampledZipformerEncoder( encoder = DownsampledZipformerEncoder(
encoder, encoder,
@ -208,6 +217,9 @@ class Zipformer(EncoderInterface):
dropout=dropout, dropout=dropout,
) )
encoders.append(encoder) encoders.append(encoder)
self.encoders = nn.ModuleList(encoders) self.encoders = nn.ModuleList(encoders)
# initializes self.skip_layers and self.skip_modules # initializes self.skip_layers and self.skip_modules