From eed864a3dbdf95ac7debc1eb2f12a4f351268fa7 Mon Sep 17 00:00:00 2001 From: Daniel Povey Date: Fri, 27 May 2022 16:45:42 +0800 Subject: [PATCH] Change power from 0.66 to 1.0, like natural gradient. --- egs/librispeech/ASR/pruned_transducer_stateless4/optim.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/egs/librispeech/ASR/pruned_transducer_stateless4/optim.py b/egs/librispeech/ASR/pruned_transducer_stateless4/optim.py index 4e3bf26ba..06115d5a4 100644 --- a/egs/librispeech/ASR/pruned_transducer_stateless4/optim.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless4/optim.py @@ -639,7 +639,7 @@ class Cain(Optimizer): this_delta_ref = grad / denom - this_delta = grad / (denom ** 1.5) + this_delta = grad / (exp_avg_sq + eps) renorm_scale = ((this_delta_ref**2).mean() / ((this_delta**2).mean() + eps)) ** 0.5