from local

This commit is contained in:
dohe0342 2023-01-09 15:08:41 +09:00
parent ea1e0c0f67
commit fef78e281f
2 changed files with 5 additions and 5 deletions

View File

@ -1115,11 +1115,11 @@ def train_one_epoch(
f"grad_scale is too small, exiting: {cur_grad_scale}" f"grad_scale is too small, exiting: {cur_grad_scale}"
) )
if params.batch_idx_train > 4000 and loss > 300 and params.wandb: #if params.batch_idx_train > 4000 and loss > 300 and params.wandb:
wb.log({"valid/loss": 10000}) # wb.log({"valid/loss": 10000})
raise RuntimeError( # raise RuntimeError(
f"divergence... exiting: loss={loss}" # f"divergence... exiting: loss={loss}"
) # )
if batch_idx % (params.log_interval*params.accum_grads) == 0: if batch_idx % (params.log_interval*params.accum_grads) == 0:
#for n, p in model.named_parameters(): #for n, p in model.named_parameters():