mirrors/icefall
1
0
Fork 0
mirror of https://github.com/k2-fsa/icefall.git synced 2025-12-11 06:55:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Have zero dropout in the position embedding, but dropout the entire thing with twice the final prob.

Browse Source
This commit is contained in:
Daniel Povey 2022-11-15 11:39:20 +08:00
parent 380f773069
commit 867556200f
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
egs/librispeech/ASR/pruned_transducer_stateless7/zipformer.py
View File

@ -553,7 +553,7 @@ class ZipformerEncoder(nn.Module):
final_layerdrop_prob: float = 0.05,
) -> None:
super().__init__()
self.encoder_pos = CompactRelPositionalEncoding(pos_dim, dropout_rate=0.15)
self.encoder_pos = CompactRelPositionalEncoding(pos_dim, dropout_rate=0.0)
self.layers = nn.ModuleList(
[copy.deepcopy(encoder_layer) for i in range(num_layers)]
@ -1031,7 +1031,7 @@ class RelPositionMultiheadAttentionWeights(nn.Module):
pos_head_dim: int,
dropout: float = 0.0,
pos_emb_skip: FloatLike = ScheduledFloat((0.0, 0.5),
(4000.0, 0.025))
(4000.0, 0.05))
) -> None:
super().__init__()
self.embed_dim = embed_dim