mirrors/icefall
1
0
Fork 0
mirror of https://github.com/k2-fsa/icefall.git synced 2025-08-11 19:12:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

disable fp16 when computing ctc loss

Browse Source
This commit is contained in:
yaozengwei 2023-02-16 15:27:23 +08:00
parent 20207f0e4e
commit cc74ba574e
2 changed files with 28 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
egs/librispeech/ASR/conformer_ctc3/train.py
View File

@ -601,12 +601,12 @@ def compute_loss(
else: else:
raise ValueError(f"Unsupported type of graph compiler: {type(graph_compiler)}") raise ValueError(f"Unsupported type of graph compiler: {type(graph_compiler)}")
with torch.cuda.amp.autocast(enabled=False):
dense_fsa_vec = k2.DenseFsaVec( dense_fsa_vec = k2.DenseFsaVec(
nnet_output, nnet_output.float(),
supervision_segments, supervision_segments,
allow_truncate=params.subsampling_factor - 1, allow_truncate=params.subsampling_factor - 1,
) )
ctc_loss = k2.ctc_loss( ctc_loss = k2.ctc_loss(
decoding_graph=decoding_graph, decoding_graph=decoding_graph,
dense_fsa_vec=dense_fsa_vec, dense_fsa_vec=dense_fsa_vec,
@ -615,6 +615,7 @@ def compute_loss(
reduction=params.reduction, reduction=params.reduction,
use_double_scores=params.use_double_scores, use_double_scores=params.use_double_scores,
) )
ctc_loss_is_finite = torch.isfinite(ctc_loss) ctc_loss_is_finite = torch.isfinite(ctc_loss)
if not torch.all(ctc_loss_is_finite): if not torch.all(ctc_loss_is_finite):
logging.info("Not all losses are finite!\n" f"ctc_loss: {ctc_loss}") logging.info("Not all losses are finite!\n" f"ctc_loss: {ctc_loss}")

5
egs/librispeech/ASR/pruned_transducer_stateless4_ctc/train.py
View File

@ -710,12 +710,12 @@ def compute_loss(
# Works with a BPE model # Works with a BPE model
decoding_graph = k2.ctc_graph(token_ids, modified=False, device=device) decoding_graph = k2.ctc_graph(token_ids, modified=False, device=device)
with torch.cuda.amp.autocast(enabled=False):
dense_fsa_vec = k2.DenseFsaVec( dense_fsa_vec = k2.DenseFsaVec(
ctc_output, ctc_output.float(),
supervision_segments, supervision_segments,
allow_truncate=params.subsampling_factor - 1, allow_truncate=params.subsampling_factor - 1,
) )
ctc_loss = k2.ctc_loss( ctc_loss = k2.ctc_loss(
decoding_graph=decoding_graph, decoding_graph=decoding_graph,
dense_fsa_vec=dense_fsa_vec, dense_fsa_vec=dense_fsa_vec,
@ -724,6 +724,7 @@ def compute_loss(
reduction="none", reduction="none",
use_double_scores=params.use_double_scores, use_double_scores=params.use_double_scores,
) )
ctc_loss_is_finite = torch.isfinite(ctc_loss) ctc_loss_is_finite = torch.isfinite(ctc_loss)
if not torch.all(ctc_loss_is_finite): if not torch.all(ctc_loss_is_finite):
logging.info("Not all losses are finite!\n" f"ctc_loss: {ctc_loss}") logging.info("Not all losses are finite!\n" f"ctc_loss: {ctc_loss}")