diff --git a/egs/librispeech/ASR/pruned_transducer_stateless_d2v_v2/.train_adapter.py.swp b/egs/librispeech/ASR/pruned_transducer_stateless_d2v_v2/.train_adapter.py.swp index e3e78daf1..24cf9f319 100644 Binary files a/egs/librispeech/ASR/pruned_transducer_stateless_d2v_v2/.train_adapter.py.swp and b/egs/librispeech/ASR/pruned_transducer_stateless_d2v_v2/.train_adapter.py.swp differ diff --git a/egs/librispeech/ASR/pruned_transducer_stateless_d2v_v2/train_adapter.py b/egs/librispeech/ASR/pruned_transducer_stateless_d2v_v2/train_adapter.py index efa974310..80022ed6a 100755 --- a/egs/librispeech/ASR/pruned_transducer_stateless_d2v_v2/train_adapter.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless_d2v_v2/train_adapter.py @@ -1115,11 +1115,11 @@ def train_one_epoch( f"grad_scale is too small, exiting: {cur_grad_scale}" ) - if params.batch_idx_train > 4000 and loss > 300 and params.wandb: - wb.log({"valid/loss": 10000}) - raise RuntimeError( - f"divergence... exiting: loss={loss}" - ) + #if params.batch_idx_train > 4000 and loss > 300 and params.wandb: + # wb.log({"valid/loss": 10000}) + # raise RuntimeError( + # f"divergence... exiting: loss={loss}" + # ) if batch_idx % (params.log_interval*params.accum_grads) == 0: #for n, p in model.named_parameters():