mirrors/icefall
1
0
Fork 0
mirror of https://github.com/k2-fsa/icefall.git synced 2025-09-09 17:14:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

Turn off natural gradient update for biases.

Browse Source
This commit is contained in:
Daniel Povey 2022-06-03 18:40:14 +08:00
parent 3fff0c75bb
commit 8085ed6ef9
1 changed files with 15 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
egs/librispeech/ASR/pruned_transducer_stateless7/optim.py
View File

@ -270,19 +270,19 @@ class Cain(Optimizer):
scale_delta) scale_delta)
delta.add_(p, alpha=scale_delta) delta.add_(p, alpha=scale_delta)
grad = self._change_coordinates(grad, state, forward=True) grad = self._change_coordinates(grad, state, forward=True)
p_transformed = self._change_coordinates(p, state, forward=True, is_grad=False) p_transformed = self._change_coordinates(p, state, forward=True, is_grad=False)
# Decay the second moment running average coefficient # Decay the second moment running average coefficient
exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value=1 - beta2) exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value=1 - beta2)
bias_correction2 = 1 - beta2 ** self._step_for_bias_correction(step) bias_correction2 = 1 - beta2 ** self._step_for_bias_correction(step)
grad_rms = (exp_avg_sq.sqrt()).add_(eps).mul_(1.0 / bias_correction2) grad_rms = (exp_avg_sq.sqrt()).add_(eps).mul_(1.0 / bias_correction2)
this_delta = grad / grad_rms this_delta = grad / grad_rms
if p.ndim() > 1:
smoothed_grad_rms = ng_eps * grad_rms.mean() + grad_rms smoothed_grad_rms = ng_eps * grad_rms.mean() + grad_rms
lr_factor = (smoothed_grad_rms ** ng_pow) lr_factor = (smoothed_grad_rms ** ng_pow)
lr_factor_mean = lr_factor.mean() lr_factor_mean = lr_factor.mean()
@ -290,6 +290,12 @@ class Cain(Optimizer):
# that param_rms is 1.0 for purposes of computing the shrinkage coefficient; actually # that param_rms is 1.0 for purposes of computing the shrinkage coefficient; actually
# it does not matter. # it does not matter.
this_lr = (lr / (target_rms * lr_factor_mean)) * lr_factor this_lr = (lr / (target_rms * lr_factor_mean)) * lr_factor
else:
# turn off this natural gradient for biases, it seems to have
# a bad effect, possibly because we don't decorrelate in this case, or
# because their gradients are not dominated by noise.
this_lr = lr / target_rms
# change sign so it's an actual parameter change, # change sign so it's an actual parameter change,
# incorporate param_rms at this point. # incorporate param_rms at this point.
this_delta *= -this_lr * state["param_rms"].clamp(min=rms_eps) this_delta *= -this_lr * state["param_rms"].clamp(min=rms_eps)
@ -323,8 +329,6 @@ class Cain(Optimizer):
return loss return loss
def _change_coordinates(self, def _change_coordinates(self,
grad: Tensor, grad: Tensor,
state: dict, state: dict,