mirrors/icefall
1
0
Fork 0
mirror of https://github.com/k2-fsa/icefall.git synced 2025-09-18 21:44:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix bias_correction2, power of 0.5

Browse Source
This commit is contained in:
Daniel Povey 2022-06-17 20:20:16 +08:00
parent 2d13b682fe
commit cbccc1dd91
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
egs/librispeech/ASR/pruned_transducer_stateless7/optim.py
View File

@ -270,7 +270,7 @@ class NeutralGradient(Optimizer):
exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value=1 - beta2)
grad_rms = (exp_avg_sq.sqrt()).add_(grad_eps)
this_delta = grad / grad_rms
alpha = (-(1-beta1) * lr * bias_correction2) * scale
alpha = (-(1-beta1) * lr * (bias_correction2 ** 0.5)) * scale
delta.add_(this_delta, alpha=alpha)
else:
# The full update.
@ -296,7 +296,7 @@ class NeutralGradient(Optimizer):
cur_grad = cur_grad / (exp_avg_sq.sqrt() + grad_eps)
if bias_correction2 < 0.99:
cur_grad *= bias_correction2
cur_grad *= (bias_correction2 ** 0.5)
cur_grad = self._change_coordinates(cur_grad, state, forward=False)