mirrors/icefall
1
0
Fork 0
mirror of https://github.com/k2-fsa/icefall.git synced 2025-12-11 06:55:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'zlm51' into zlm52

Browse Source
This commit is contained in:
Daniel Povey 2023-05-29 20:01:27 +08:00
commit 09294c0b51
1 changed files with 6 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
egs/libriheavy/LM/zipformer1/subformer.py
View File

@ -1754,7 +1754,11 @@ class NonlinAttention(nn.Module):
self.identity1 = Identity() # for diagnostics.
self.identity2 = Identity() # for diagnostics.
self.identity3 = Identity() # for diagnostics.
# ensure the activations after multiplication don't get too large.
self.hidden_penalty = AbsValuePenalizer(
limit=10.0, penalty=1.0e-04, prob=0.1)
self.out_proj = ScaledLinear(hidden_channels, channels,
bias=True,
@ -1815,7 +1819,7 @@ attn_weights: a Tensor of shape (num_heads, batch_size, seq_len, seq_len)
y = self.identity2(y)
x = x * y
x = self.identity3(x)
x = self.hidden_penalty(x)
x = self.out_proj(x)
x = self.whiten2(x)