From d95571eacf86dea90d634025b13d5bf9eee888ad Mon Sep 17 00:00:00 2001 From: Daniel Povey Date: Wed, 23 Nov 2022 14:10:01 +0800 Subject: [PATCH] From 460->461, revert change about balancing output of attention_squeeze module. --- .../ASR/pruned_transducer_stateless7/zipformer.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/egs/librispeech/ASR/pruned_transducer_stateless7/zipformer.py b/egs/librispeech/ASR/pruned_transducer_stateless7/zipformer.py index 9590de27f..7332a9eda 100644 --- a/egs/librispeech/ASR/pruned_transducer_stateless7/zipformer.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless7/zipformer.py @@ -1343,17 +1343,10 @@ class AttentionSqueeze(nn.Module): self.out_proj = ScaledLinear(embed_dim, embed_dim, bias=False, initial_scale=0.05) - self.out_whiten = Whiten(num_groups=1, whitening_limit=10.0, prob=(0.01, 0.1), grad_scale=0.01) - self.out_balancer = ActivationBalancer( - embed_dim, channel_dim=-1, - min_positive=0.45, max_positive=0.55, - min_abs=0.005, max_abs=2.0, - min_prob=0.05, - ) def forward(self, x: Tensor, @@ -1388,7 +1381,6 @@ attn_weights: a Tensor of shape (num_heads, batch_size, seq_len, seq_len) x = self.activation(x) # Identity only. For diagnostics. x = self.out_proj(x) x = self.out_whiten(x) - x = self.out_balancer(x) return x