mirrors/icefall
1
0
Fork 0
mirror of https://github.com/k2-fsa/icefall.git synced 2025-08-09 10:02:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

update diagnostics, print limits in Balancer, merge changes from Dan's branch zlm59 (#1109)

Browse Source
This commit is contained in:
Zengwei Yao 2023-06-01 14:24:19 +08:00 committed by GitHub
parent 03853f1ee5
commit 7a604057f9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 21 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
icefall/diagnostics.py
View File

@ -498,6 +498,22 @@ class ModelDiagnostic(object):
self.diagnostics[k].print_diagnostics() self.diagnostics[k].print_diagnostics()
def get_class_name(module: nn.Module):
ans = type(module).__name__
# we put the below in try blocks in case anyone is using a different version of these modules that
# might have different member names.
if ans == 'Balancer' or ans == 'ActivationBalancer':
try:
ans += f'[{float(module.min_positive)},{float(module.max_positive)},{float(module.min_abs)},{float(module.max_abs)}]'
except:
pass
elif ans == 'AbsValuePenalizer':
try:
ans += f'[{module.limit}]'
except:
pass
return ans
def attach_diagnostics( def attach_diagnostics(
model: nn.Module, opts: Optional[TensorDiagnosticOptions] = None model: nn.Module, opts: Optional[TensorDiagnosticOptions] = None
) -> ModelDiagnostic: ) -> ModelDiagnostic:
@ -537,12 +553,12 @@ def attach_diagnostics(
if isinstance(_output, Tensor) and _output.dtype in ( torch.float32, torch.float16, torch.float64 ): if isinstance(_output, Tensor) and _output.dtype in ( torch.float32, torch.float16, torch.float64 ):
_model_diagnostic[f"{_name}.output"].accumulate(_output, _model_diagnostic[f"{_name}.output"].accumulate(_output,
class_name=type(_module).__name__) class_name=get_class_name(_module))
elif isinstance(_output, tuple): elif isinstance(_output, tuple):
for i, o in enumerate(_output): for i, o in enumerate(_output):
if o.dtype in ( torch.float32, torch.float16, torch.float64 ): if o.dtype in ( torch.float32, torch.float16, torch.float64 ):
_model_diagnostic[f"{_name}.output[{i}]"].accumulate(o, _model_diagnostic[f"{_name}.output[{i}]"].accumulate(o,
class_name=type(_module).__name__) class_name=get_class_name(_module))
def backward_hook( def backward_hook(
_module, _input, _output, _model_diagnostic=ans, _name=name _module, _input, _output, _model_diagnostic=ans, _name=name
@ -551,12 +567,12 @@ def attach_diagnostics(
_output = _output[0] _output = _output[0]
if isinstance(_output, Tensor) and _output.dtype in ( torch.float32, torch.float16, torch.float64 ): if isinstance(_output, Tensor) and _output.dtype in ( torch.float32, torch.float16, torch.float64 ):
_model_diagnostic[f"{_name}.grad"].accumulate(_output, _model_diagnostic[f"{_name}.grad"].accumulate(_output,
class_name=type(_module).__name__) class_name=get_class_name(_module))
elif isinstance(_output, tuple): elif isinstance(_output, tuple):
for i, o in enumerate(_output): for i, o in enumerate(_output):
if o.dtype in ( torch.float32, torch.float16, torch.float64 ): if o.dtype in ( torch.float32, torch.float16, torch.float64 ):
_model_diagnostic[f"{_name}.grad[{i}]"].accumulate(o, _model_diagnostic[f"{_name}.grad[{i}]"].accumulate(o,
class_name=type(_module).__name__) class_name=get_class_name(_module))
module.register_forward_hook(forward_hook) module.register_forward_hook(forward_hook)
@ -574,7 +590,7 @@ def attach_diagnostics(
_input, = _input _input, = _input
assert isinstance(_input, Tensor) assert isinstance(_input, Tensor)
_model_diagnostic[f"{_name}.scalar"].accumulate_input(_input, _model_diagnostic[f"{_name}.scalar"].accumulate_input(_input,
class_name=type(_module).__name__) class_name=get_class_name(_module))
def scalar_backward_hook( def scalar_backward_hook(
_module, _input, _output, _model_diagnostic=ans, _name=name _module, _input, _output, _model_diagnostic=ans, _name=name