from local

This commit is contained in:
dohe0342 2022-12-15 17:54:01 +09:00
parent e2c5f1b605
commit 9fe0b4cdca
2 changed files with 1 additions and 1 deletions

View File

@ -1086,8 +1086,8 @@ def train_one_epoch(
f"grad_scale is too small, exiting: {cur_grad_scale}" f"grad_scale is too small, exiting: {cur_grad_scale}"
) )
logging.info(loss)
if params.batch_idx_train > 4000 and loss > 300: if params.batch_idx_train > 4000 and loss > 300:
logging.info(loss)
wb.log({"valid/loss": 10000}) wb.log({"valid/loss": 10000})
raise RunteimError( raise RunteimError(
f"divergence... exiting: loss={loss}" f"divergence... exiting: loss={loss}"