mirrors/icefall
1
0
Fork 0
mirror of https://github.com/k2-fsa/icefall.git synced 2025-08-08 09:32:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

Warn instead of raising exceptions in inf-check (#1852)

Browse Source
This commit is contained in:
Han Zhu 2024-12-31 16:52:06 +08:00 committed by GitHub
parent a2b0f6057c
commit df46a3eaf9
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 2 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
icefall/hooks.py
View File

@ -40,9 +40,7 @@ def register_inf_check_hooks(model: nn.Module) -> None:
def forward_hook(_module, _input, _output, _name=name):
if isinstance(_output, Tensor):
if not torch.isfinite(_output.to(torch.float32).sum()):
raise ValueError(
f"The sum of {_name}.output is not finite: {_output}"
)
logging.warning(f"The sum of {_name}.output is not finite")
elif isinstance(_output, tuple):
for i, o in enumerate(_output):
if isinstance(o, tuple):
@ -50,9 +48,7 @@ def register_inf_check_hooks(model: nn.Module) -> None:
if not isinstance(o, Tensor):
continue
if not torch.isfinite(o.to(torch.float32).sum()):
raise ValueError(
f"The sum of {_name}.output[{i}] is not finite: {_output}"
)
logging.warning(f"The sum of {_name}.output[{i}] is not finite")
# default param _name is a way to capture the current value of the variable "name".
def backward_hook(_module, _input, _output, _name=name):