Update train.py

This commit is contained in:
jinzr 2024-03-08 02:56:53 +08:00
parent a542a68757
commit 89632bf748

View File

@ -514,9 +514,10 @@ def train_one_epoch(
logging.warning(f"Grad scale is small: {cur_grad_scale}") logging.warning(f"Grad scale is small: {cur_grad_scale}")
if cur_grad_scale < 1.0e-05: if cur_grad_scale < 1.0e-05:
save_bad_model() save_bad_model()
raise RuntimeError( logging.warning(f"Grad scale is small: {cur_grad_scale}")
f"grad_scale is too small, exiting: {cur_grad_scale}" # raise RuntimeError(
) # f"grad_scale is too small, exiting: {cur_grad_scale}"
# )
if params.batch_idx_train % params.log_interval == 0: if params.batch_idx_train % params.log_interval == 0:
cur_lr_g = max(scheduler_g.get_last_lr()) cur_lr_g = max(scheduler_g.get_last_lr())