mirrors/icefall
1
0
Fork 0
mirror of https://github.com/k2-fsa/icefall.git synced 2025-09-08 16:44:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

Make diagnostics.py more error-tolerant and have wider range of supported torch versions

Browse Source
This commit is contained in:
Daniel Povey 2023-08-31 12:42:53 +08:00
parent 4ab7d61008
commit 4a6035a4aa
1 changed files with 13 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
icefall/diagnostics.py
View File

@ -247,14 +247,22 @@ class TensorDiagnostic(object):
if stats_type == "eigs": if stats_type == "eigs":
try: try:
if hasattr(torch, 'linalg') and hasattr(torch.linalg, 'eigh'):
eigs, _ = torch.linalg.eigh(stats)
else:
eigs, _ = torch.symeig(stats) eigs, _ = torch.symeig(stats)
stats = eigs.abs().sqrt() stats = eigs.abs().sqrt()
except: # noqa except: # noqa
print( print(
"Error getting eigenvalues, trying another method." "Error getting eigenvalues, trying another method."
) )
if hasattr(torch, 'linalg') and hasattr(torch.linalg, 'eig'):
eigs, _ = torch.linalg.eig(stats)
eigs = eigs.abs()
else:
eigs, _ = torch.eig(stats) eigs, _ = torch.eig(stats)
stats = eigs.norm(dim=1).sqrt() eigs = eigs.norm(dim=1)
stats = eigs.sqrt()
# sqrt so it reflects data magnitude, like stddev- not variance # sqrt so it reflects data magnitude, like stddev- not variance
if stats_type in [ "rms", "stddev" ]: if stats_type in [ "rms", "stddev" ]:
@ -556,7 +564,7 @@ def attach_diagnostics(
class_name=get_class_name(_module)) class_name=get_class_name(_module))
elif isinstance(_output, tuple): elif isinstance(_output, tuple):
for i, o in enumerate(_output): for i, o in enumerate(_output):
if o.dtype in ( torch.float32, torch.float16, torch.float64 ): if isinstance(o, Tensor) and o.dtype in ( torch.float32, torch.float16, torch.float64 ):
_model_diagnostic[f"{_name}.output[{i}]"].accumulate(o, _model_diagnostic[f"{_name}.output[{i}]"].accumulate(o,
class_name=get_class_name(_module)) class_name=get_class_name(_module))
@ -570,7 +578,7 @@ def attach_diagnostics(
class_name=get_class_name(_module)) class_name=get_class_name(_module))
elif isinstance(_output, tuple): elif isinstance(_output, tuple):
for i, o in enumerate(_output): for i, o in enumerate(_output):
if o.dtype in ( torch.float32, torch.float16, torch.float64 ): if isinstance(o, Tensor) and o.dtype in ( torch.float32, torch.float16, torch.float64 ):
_model_diagnostic[f"{_name}.grad[{i}]"].accumulate(o, _model_diagnostic[f"{_name}.grad[{i}]"].accumulate(o,
class_name=get_class_name(_module)) class_name=get_class_name(_module))