From 867556200f2e80ec2864b424f6ebcdf6f10e52a5 Mon Sep 17 00:00:00 2001 From: Daniel Povey Date: Tue, 15 Nov 2022 11:39:20 +0800 Subject: [PATCH] Have zero dropout in the position embedding, but dropout the entire thing with twice the final prob. --- egs/librispeech/ASR/pruned_transducer_stateless7/zipformer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/egs/librispeech/ASR/pruned_transducer_stateless7/zipformer.py b/egs/librispeech/ASR/pruned_transducer_stateless7/zipformer.py index eeb186a6f..0b2ab23d5 100644 --- a/egs/librispeech/ASR/pruned_transducer_stateless7/zipformer.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless7/zipformer.py @@ -553,7 +553,7 @@ class ZipformerEncoder(nn.Module): final_layerdrop_prob: float = 0.05, ) -> None: super().__init__() - self.encoder_pos = CompactRelPositionalEncoding(pos_dim, dropout_rate=0.15) + self.encoder_pos = CompactRelPositionalEncoding(pos_dim, dropout_rate=0.0) self.layers = nn.ModuleList( [copy.deepcopy(encoder_layer) for i in range(num_layers)] @@ -1031,7 +1031,7 @@ class RelPositionMultiheadAttentionWeights(nn.Module): pos_head_dim: int, dropout: float = 0.0, pos_emb_skip: FloatLike = ScheduledFloat((0.0, 0.5), - (4000.0, 0.025)) + (4000.0, 0.05)) ) -> None: super().__init__() self.embed_dim = embed_dim