From f0f204552d1738c4f92a465fb99a7cafaa1f480d Mon Sep 17 00:00:00 2001 From: Daniel Povey Date: Fri, 2 Dec 2022 15:17:41 +0800 Subject: [PATCH] Add -0.05 to DoubleSwish. --- egs/librispeech/ASR/pruned_transducer_stateless7/scaling.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/egs/librispeech/ASR/pruned_transducer_stateless7/scaling.py b/egs/librispeech/ASR/pruned_transducer_stateless7/scaling.py index 11db31dfd..57a0b2f64 100644 --- a/egs/librispeech/ASR/pruned_transducer_stateless7/scaling.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless7/scaling.py @@ -1118,7 +1118,7 @@ class DoubleSwishFunction(torch.autograd.Function): assert d_scaled.max() < 256.0 d_int = d_scaled.to(torch.uint8) ctx.save_for_backward(d_int) - y = y + alpha * x + beta * x.clamp(min=-x_limit, max=x_limit) + y = y + alpha * x + beta * x.clamp(min=-x_limit, max=x_limit) + -0.05 if x.dtype == torch.float16 or torch.is_autocast_enabled(): y = y.to(torch.float16) return y