from local
This commit is contained in:
parent
669053c0d8
commit
5a4f4acd05
Binary file not shown.
@ -1081,6 +1081,7 @@ def train_one_epoch(
|
|||||||
if cur_grad_scale < 0.01:
|
if cur_grad_scale < 0.01:
|
||||||
logging.warning(f"Grad scale is small: {cur_grad_scale}")
|
logging.warning(f"Grad scale is small: {cur_grad_scale}")
|
||||||
if cur_grad_scale < 1.0e-05:
|
if cur_grad_scale < 1.0e-05:
|
||||||
|
wb.log({"valid/loss": 10000})
|
||||||
raise RuntimeError(
|
raise RuntimeError(
|
||||||
f"grad_scale is too small, exiting: {cur_grad_scale}"
|
f"grad_scale is too small, exiting: {cur_grad_scale}"
|
||||||
)
|
)
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user