From e039441a0f7d7af58193eee1582615f85e0048ea Mon Sep 17 00:00:00 2001
From: dohe0342 <kimdohe1070@gmail.com>
Date: Tue, 13 Dec 2022 13:57:44 +0900
Subject: [PATCH] from local

---
 .../.train.py.swp                             | Bin 73728 -> 73728 bytes
 .../train.py                                  |   7 ++++---
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/egs/librispeech/ASR/pruned_transducer_stateless_d2v_v2/.train.py.swp b/egs/librispeech/ASR/pruned_transducer_stateless_d2v_v2/.train.py.swp
index af98429932daa5c08ead582abb77d64fae716c49..69093f15546294419919a054285fe3fa96126c4e 100644
GIT binary patch
delta 311
zcmXBOEl5LA90l-m<-I3+ANsD02NQJ$VL{kt)8GaJ2MfbyRTK?^MuP@d{lMTAX4?#=
z$waPM9)iW9^M!#XgJsB)mTB<F4*cLB4u^jwG%BGHtY()=<;l5p7K{(Xt@E|FkDH+9
zi-M9eDN=eyMt9z~2_y#lor0Fnjt~hw-ugAwXnSfPH?Q;UhuQs;7*MqPt<?V_^D+#C
zv_sSMIRrFllkya&FAsR6IxUh<kqcbWF>TN~mC2(Y2k6l~T~m`T9At3BV}-^@QkTy>
zQIig6k9KK`inbQUSxnX5UA28_Df>{KMy~F*JX57#EtOfe)NLoHN(&>^#)NRIouml<
E09*A$XaE2J

delta 288
zcmXBOF-SsT6b0aO!Dr}uKHpEE6@(Gl&`^d;%}zl&RYPvMv=jx6LC!VVPdGYdlZgC7
zQ}UJ?0znZHbFn4LqOl<wI^ls2?r=EVq8}*wf!JPZXCphezMKN%0XMJZLjJB1`|%{B
zl}bp`tg3D^(zgJa7<V#8wQZ3ZGu!#K1bVu+pi85;+3wZX&b~uHH9RgR|3{gpC~!fW
zWYePyROp1ZXo*1e2yjGca_G|m9_XHGbU=C9rFn{yOF|zu;M-WlDUV~y(F(cr&W8r{
kOf9P3Haxism4^Ge^5W>j>!1#wtIMuqzB-du`7I$+e~v>wWB>pF

diff --git a/egs/librispeech/ASR/pruned_transducer_stateless_d2v_v2/train.py b/egs/librispeech/ASR/pruned_transducer_stateless_d2v_v2/train.py
index 38caa2b3e..0c299c744 100755
--- a/egs/librispeech/ASR/pruned_transducer_stateless_d2v_v2/train.py
+++ b/egs/librispeech/ASR/pruned_transducer_stateless_d2v_v2/train.py
@@ -1009,9 +1009,10 @@ def train_one_epoch(
 
             # NOTE: We use reduction==sum and loss is computed over utterances
             # in the batch and there is no normalization to it so far.
-            if scaler._scale.item() < 1.0e-05:
-                continue
-            scaler.scale(loss).backward()
+            if scaler._scale.item() >= 1.0e-05:
+                scaler.scale(loss).backward()
+            else:
+                logging.warning(f"Grad scale is small: {cur_grad_scale}")
             if params.multi_optim and batch_idx % params.accum_grads == 0:
                 set_batch_count(model, params.batch_idx_train)
                 scheduler_enc.step_batch(params.batch_idx_train)