From 2940d3106f08a06d37618229e872ace7e371fa66 Mon Sep 17 00:00:00 2001 From: Daniel Povey Date: Fri, 11 Mar 2022 14:43:57 +0800 Subject: [PATCH] Fix q*scaling logic --- egs/librispeech/ASR/transducer_stateless/conformer.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/egs/librispeech/ASR/transducer_stateless/conformer.py b/egs/librispeech/ASR/transducer_stateless/conformer.py index d0be5af00..e14c7a02e 100644 --- a/egs/librispeech/ASR/transducer_stateless/conformer.py +++ b/egs/librispeech/ASR/transducer_stateless/conformer.py @@ -616,7 +616,6 @@ class RelPositionMultiheadAttention(nn.Module): ), "embed_dim must be divisible by num_heads" scaling = float(head_dim) ** -0.5 - q = q * scaling if torch.equal(query, key) and torch.equal(key, value): # self-attention @@ -721,7 +720,7 @@ class RelPositionMultiheadAttention(nn.Module): ) key_padding_mask = key_padding_mask.to(torch.bool) - q = q.contiguous().view(tgt_len, bsz, num_heads, head_dim) + q = (q.contiguous() * scaling).view(tgt_len, bsz, num_heads, head_dim) k = k.contiguous().view(-1, bsz, num_heads, head_dim) v = v.contiguous().view(-1, bsz * num_heads, head_dim).transpose(0, 1)