from local

This commit is contained in:
dohe0342 2022-12-13 11:25:55 +09:00
parent 90cdf0d0c7
commit 42259f2af5
3 changed files with 2 additions and 1 deletions

Binary file not shown.

View File

@ -1074,7 +1074,8 @@ def train_one_epoch(
# behavior depending on the current grad scale.
cur_grad_scale = scaler._scale.item()
if cur_grad_scale < 1.0 or (cur_grad_scale < 8.0 and batch_idx % 400 == 0):
scaler.update(cur_grad_scale * 2.0)
continue
#scaler.update(cur_grad_scale * 2.0)
if cur_grad_scale < 0.01:
logging.warning(f"Grad scale is small: {cur_grad_scale}")
if cur_grad_scale < 1.0e-05: