mirrors/icefall
1
0
Fork 0
mirror of https://github.com/k2-fsa/icefall.git synced 2025-09-07 08:04:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

train scalars slower

Browse Source
This commit is contained in:
Daniel Povey 2022-05-15 15:44:56 +08:00
parent 0989aec741
commit a1dc020270
1 changed files with 5 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
egs/librispeech/ASR/pruned_transducer_stateless4b/optim.py
View File

@ -334,6 +334,10 @@ class Abel(Optimizer):
# to standard Adam.
denom = (exp_avg_sq/bias_correction2 + eps*eps).sqrt()
this_delta = grad / denom
# eventually will remove this special case, this is to deal
# with our model that has scaling factors.
lr_eff = lr * (bias_correction2 ** 0.5)
else:
# update that includes factorization-related stuff..
factors_exp_avg_sq, factorization = state["factors_exp_avg_sq"], state["factorization"]
@ -396,8 +400,8 @@ class Abel(Optimizer):
# and x_factor2: again, before taking into account the learning
# rate or momentum.
this_delta = ((grad * factorization / denom) + p * factors_sum)
lr_eff = lr * (bias_correction2 ** 0.5) / target_rms
lr_eff = lr * (bias_correction2 ** 0.5) / target_rms
delta.mul_(beta1).add_(this_delta, alpha=-lr_eff*(1-beta1))
p.add_(delta)