mirrors/icefall
1
0
Fork 0
mirror of https://github.com/k2-fsa/icefall.git synced 2025-12-11 06:55:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Remove in_balancer.

Browse Source
This commit is contained in:
Daniel Povey 2022-10-19 12:35:17 +08:00
parent f4442de1c4
commit 45c38dec61
1 changed files with 1 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
egs/librispeech/ASR/pruned_transducer_stateless7/conformer.py
View File

@ -869,8 +869,6 @@ class RelPositionMultiheadAttention(nn.Module):
self.copy_pos_query = Identity() self.copy_pos_query = Identity()
self.copy_query = Identity() self.copy_query = Identity()
self.in_balancer = ActivationBalancer(3 * attention_dim,
channel_dim=-1, max_abs=5.0)
self.out_proj = ScaledLinear( self.out_proj = ScaledLinear(
attention_dim // 2, embed_dim, bias=True, initial_scale=0.05 attention_dim // 2, embed_dim, bias=True, initial_scale=0.05
) )
@ -930,7 +928,7 @@ class RelPositionMultiheadAttention(nn.Module):
and S is the sequence length. and S is the sequence length.
""" """
x, weights = self.multi_head_attention_forward( x, weights = self.multi_head_attention_forward(
self.in_balancer(self.in_proj(x)), self.in_proj(x),
self.linear_pos(pos_emb), self.linear_pos(pos_emb),
self.attention_dim, self.attention_dim,
self.num_heads, self.num_heads,