mirrors/icefall
1
0
Fork 0
mirror of https://github.com/k2-fsa/icefall.git synced 2025-09-18 21:44:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Adding max_abs=3.0 to ActivationBalancer modules inside feedoforward modules.

Browse Source
This commit is contained in:
Daniel Povey 2022-07-16 07:19:26 +08:00
parent 23e6d2e6d8
commit de1fd91435
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
egs/librispeech/ASR/pruned_transducer_stateless7/conformer.py
View File

@ -176,7 +176,7 @@ class ConformerEncoderLayer(nn.Module):
self.feed_forward = nn.Sequential(
nn.Linear(d_model, dim_feedforward),
ActivationBalancer(channel_dim=-1),
ActivationBalancer(channel_dim=-1, max_abs=3.0),
DoubleSwish(),
nn.Dropout(dropout),
ScaledLinear(dim_feedforward, d_model,
@ -185,7 +185,7 @@ class ConformerEncoderLayer(nn.Module):
self.feed_forward_macaron = nn.Sequential(
nn.Linear(d_model, dim_feedforward),
ActivationBalancer(channel_dim=-1),
ActivationBalancer(channel_dim=-1, max_abs=3.0),
DoubleSwish(),
nn.Dropout(dropout),
ScaledLinear(dim_feedforward, d_model,