Reduce scale on bias from 4 to 2

This commit is contained in:
Daniel Povey 2022-05-21 20:07:41 +08:00
parent dcab1aee4e
commit cbdddbe670

View File

@ -179,7 +179,7 @@ class ScaledLinear(nn.Linear):
with torch.no_grad():
self.weight[:] *= initial_scale
if self.bias is not None:
self.bias[:] *= initial_scale * 4.0
self.bias[:] *= 2.0 * initial_scale
def get_weight(self): # not needed any more but kept for back compatibility
return self.weight