mirrors/icefall
1
0
Fork 0
mirror of https://github.com/k2-fsa/icefall.git synced 2025-09-07 08:04:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Change initial bias scales from 0.1 to 0.2

Browse Source
This commit is contained in:
Daniel Povey 2022-05-22 10:59:51 +08:00
parent 435b073979
commit 5d57dd3930
2 changed files with 5 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
egs/librispeech/ASR/pruned_transducer_stateless4/conformer.py
View File

@ -457,8 +457,8 @@ class RelPositionMultiheadAttention(nn.Module):
return self.pos_bias_v * self.pos_bias_v_scale.exp()
def _reset_parameters(self) -> None:
nn.init.uniform_(self.pos_bias_u, -0.1, 0.1)
nn.init.uniform_(self.pos_bias_v, -0.1, 0.1)
nn.init.uniform_(self.pos_bias_u, -0.2, 0.2)
nn.init.uniform_(self.pos_bias_v, -0.2, 0.2)
def forward(
self,

6
egs/librispeech/ASR/pruned_transducer_stateless4/scaling.py
View File

@ -179,7 +179,7 @@ class ScaledLinear(nn.Linear):
with torch.no_grad():
self.weight[:] *= initial_scale
if self.bias is not None:
torch.nn.init.uniform_(self.bias, -0.1, 0.1)
torch.nn.init.uniform_(self.bias, -0.2, 0.2)
def get_weight(self): # not needed any more but kept for back compatibility
return self.weight
@ -201,7 +201,7 @@ class ScaledConv1d(nn.Conv1d):
with torch.no_grad():
self.weight[:] *= initial_scale
if self.bias is not None:
torch.nn.init.uniform_(self.bias, -0.1, 0.1)
torch.nn.init.uniform_(self.bias, -0.2, 0.2)
def get_weight(self): # TODO: delete
return self.weight
@ -222,7 +222,7 @@ class ScaledConv2d(nn.Conv2d):
with torch.no_grad():
self.weight[:] *= initial_scale
if self.bias is not None:
torch.nn.init.uniform_(self.bias, -0.1, 0.1)
torch.nn.init.uniform_(self.bias, -0.2, 0.2)
def get_weight(self):
return self.weight