from local

This commit is contained in:
dohe0342 2023-01-09 15:08:41 +09:00
parent ea1e0c0f67
commit fef78e281f
2 changed files with 5 additions and 5 deletions

View File

@ -1115,11 +1115,11 @@ def train_one_epoch(
f"grad_scale is too small, exiting: {cur_grad_scale}"
)
if params.batch_idx_train > 4000 and loss > 300 and params.wandb:
wb.log({"valid/loss": 10000})
raise RuntimeError(
f"divergence... exiting: loss={loss}"
)
#if params.batch_idx_train > 4000 and loss > 300 and params.wandb:
# wb.log({"valid/loss": 10000})
# raise RuntimeError(
# f"divergence... exiting: loss={loss}"
# )
if batch_idx % (params.log_interval*params.accum_grads) == 0:
#for n, p in model.named_parameters():