mirror of
https://github.com/k2-fsa/icefall.git
synced 2025-08-09 10:02:22 +00:00
Add clamping operation in Eve optimizer for all scalar weights to avoid (#550)
non stable training in some scenarios. The clamping range is set to (-10,2). Note that this change may cause unexpected effect if you resume training from a model that is trained without clamping.
This commit is contained in:
parent
0967cf5b38
commit
1e31fbcd7d
@ -164,6 +164,10 @@ class Eve(Optimizer):
|
|||||||
p.mul_(1 - (weight_decay * is_above_target_rms))
|
p.mul_(1 - (weight_decay * is_above_target_rms))
|
||||||
p.addcdiv_(exp_avg, denom, value=-step_size)
|
p.addcdiv_(exp_avg, denom, value=-step_size)
|
||||||
|
|
||||||
|
# Constrain the range of scalar weights
|
||||||
|
if p.numel() == 1:
|
||||||
|
p.clamp_(min=-10, max=2)
|
||||||
|
|
||||||
return loss
|
return loss
|
||||||
|
|
||||||
|
|
||||||
|
Loading…
x
Reference in New Issue
Block a user