from local

This commit is contained in:
dohe0342 2022-12-15 17:52:13 +09:00
parent 9d65e42008
commit e2c5f1b605
2 changed files with 1 additions and 0 deletions

View File

@ -1087,6 +1087,7 @@ def train_one_epoch(
)
if params.batch_idx_train > 4000 and loss > 300:
logging.info(loss)
wb.log({"valid/loss": 10000})
raise RunteimError(
f"divergence... exiting: loss={loss}"