From ba31272c929e374d070f7cffd2a24a7c6ffa5b62 Mon Sep 17 00:00:00 2001 From: Daniel Povey Date: Wed, 30 Nov 2022 21:44:45 +0800 Subject: [PATCH] Change sigmoid to tanh in NonlinAttentionModule, and adjust abs limits of balancer to compensate. --- .../ASR/pruned_transducer_stateless7/zipformer.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/egs/librispeech/ASR/pruned_transducer_stateless7/zipformer.py b/egs/librispeech/ASR/pruned_transducer_stateless7/zipformer.py index aa94b9f08..74a7c5094 100644 --- a/egs/librispeech/ASR/pruned_transducer_stateless7/zipformer.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless7/zipformer.py @@ -1464,10 +1464,10 @@ class NonlinAttentionModule(nn.Module): channels // (2 * ratio), channel_dim=-1, min_positive=ScheduledFloat((0.0, 0.1), (8000.0, 0.05)), max_positive=1.0, - min_abs=1.5, - max_abs=ScheduledFloat((0.0, 5.0), (8000.0, 10.0), default=1.0), + min_abs=0.75, + max_abs=ScheduledFloat((0.0, 2.5), (8000.0, 5.0), default=1.0), ) - self.sigmoid = nn.Sigmoid() + self.tanh = nn.Tanh() self.activation = Identity() # for diagnostics. self.out_proj = ScaledLinear(channels // 2, channels, @@ -1501,7 +1501,7 @@ attn_weights: a Tensor of shape (num_heads, batch_size, seq_len, seq_len) x = x[..., :num_channels // 2] s = self.balancer(s) - s = self.sigmoid(s) + s = self.tanh(s) s = s.unsqueeze(-1).expand(-1, -1, -1, self.ratio).reshape(seq_len, batch_size, num_channels // 2)