mirror of
https://github.com/k2-fsa/icefall.git
synced 2025-12-11 06:55:27 +00:00
from local
This commit is contained in:
parent
e2c5f1b605
commit
9fe0b4cdca
Binary file not shown.
@ -1086,8 +1086,8 @@ def train_one_epoch(
|
|||||||
f"grad_scale is too small, exiting: {cur_grad_scale}"
|
f"grad_scale is too small, exiting: {cur_grad_scale}"
|
||||||
)
|
)
|
||||||
|
|
||||||
if params.batch_idx_train > 4000 and loss > 300:
|
|
||||||
logging.info(loss)
|
logging.info(loss)
|
||||||
|
if params.batch_idx_train > 4000 and loss > 300:
|
||||||
wb.log({"valid/loss": 10000})
|
wb.log({"valid/loss": 10000})
|
||||||
raise RunteimError(
|
raise RunteimError(
|
||||||
f"divergence... exiting: loss={loss}"
|
f"divergence... exiting: loss={loss}"
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user