mirrors/icefall
1
0
Fork 0
mirror of https://github.com/k2-fsa/icefall.git synced 2025-12-11 06:55:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'scaled_adam_exp564' into scaled_adam_exp573

Browse Source
This commit is contained in:
Daniel Povey 2022-12-01 16:26:12 +08:00
commit 8976e1e43b
1 changed files with 4 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
icefall/diagnostics.py
View File

@ -543,6 +543,10 @@ def attach_diagnostics(
_model_diagnostic[f"{_name}.grad[{i}]"].accumulate(o, _model_diagnostic[f"{_name}.grad[{i}]"].accumulate(o,
class_name=type(_module).__name__) class_name=type(_module).__name__)
module.register_forward_hook(forward_hook)
module.register_backward_hook(backward_hook)
if type(module).__name__ in ["Sigmoid", "Tanh", "ReLU", "TanSwish", "Swish", "DoubleSwish"]: if type(module).__name__ in ["Sigmoid", "Tanh", "ReLU", "TanSwish", "Swish", "DoubleSwish"]:
# For these specific module types, accumulate some additional diagnostics # For these specific module types, accumulate some additional diagnostics
# that can help us improve the activation function. These require a lot of memory, # that can help us improve the activation function. These require a lot of memory,