Add bias in interior of SelfAttn module

This commit is contained in:
Daniel Povey 2022-11-12 14:37:23 +08:00
parent 4a5a13b678
commit 6c16d08b4f

View File

@ -1282,7 +1282,7 @@ class SelfAttention(nn.Module):
super().__init__()
self.in_proj = nn.Linear(embed_dim,
num_heads * value_head_dim,
bias=False)
bias=True)
# attempt to make the output of `in_proj` uncorrelated within each head
# and all heads having roughly the same magnitude. the hope is to