mirrors/icefall
1
0
Fork 0
mirror of https://github.com/k2-fsa/icefall.git synced 2025-12-11 06:55:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Change for diagnostic purposes, sigmoid of NonlinAttention.

Browse Source
This commit is contained in:
Daniel Povey 2022-11-25 12:39:16 +08:00
parent dd3826104e
commit ba348169bf
1 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
egs/librispeech/ASR/pruned_transducer_stateless7/zipformer.py
View File

@ -1431,6 +1431,7 @@ class NonlinAttentionModule(nn.Module):
min_abs=0.2, max_abs=10.0, min_abs=0.2, max_abs=10.0,
min_prob=0.05, min_prob=0.05,
) )
self.sigmoid = nn.Sigmoid()
self.activation = Identity() # for diagnostics. self.activation = Identity() # for diagnostics.
self.out_proj = ScaledLinear(channels, channels, self.out_proj = ScaledLinear(channels, channels,
@ -1471,7 +1472,7 @@ attn_weights: a Tensor of shape (num_heads, batch_size, seq_len, seq_len)
v = self.whiten1(v) v = self.whiten1(v)
# GLU mechanism # GLU mechanism
x = s.sigmoid() * v x = self.sigmoid(s) * v
x = self.balancer(x) x = self.balancer(x)
(seq_len, batch_size, embed_dim) = x.shape (seq_len, batch_size, embed_dim) = x.shape