From 0ac26f4234cd42c70e343eb2b00a3b8790d5cd95 Mon Sep 17 00:00:00 2001 From: Daniel Povey Date: Thu, 24 Nov 2022 15:18:28 +0800 Subject: [PATCH] Increase initial whitening target for self_attn from 2.0 to 3.0. --- egs/librispeech/ASR/pruned_transducer_stateless7/zipformer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/egs/librispeech/ASR/pruned_transducer_stateless7/zipformer.py b/egs/librispeech/ASR/pruned_transducer_stateless7/zipformer.py index f02813c78..26e977692 100644 --- a/egs/librispeech/ASR/pruned_transducer_stateless7/zipformer.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless7/zipformer.py @@ -1055,7 +1055,7 @@ class RelPositionMultiheadAttentionWeights(nn.Module): initial_scale=query_head_dim**-0.25) self.whiten_keys = Whiten(num_groups=num_heads, - whitening_limit=_whitening_schedule(2.0), + whitening_limit=_whitening_schedule(3.0), prob=(0.025, 0.25), grad_scale=0.025)