mirrors/icefall
1
0
Fork 0
mirror of https://github.com/k2-fsa/icefall.git synced 2025-08-09 01:52:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

Reformatting

Browse Source
This commit is contained in:
Piotr Żelasko 2021-10-14 21:40:14 -04:00
parent 1c7c79f2fc
commit 060117a9ff
1 changed files with 19 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
egs/librispeech/ASR/conformer_ctc/train.py
View File

@ -607,19 +607,31 @@ def run(rank, world_size, args):
valid_dl = librispeech.valid_dataloaders() valid_dl = librispeech.valid_dataloaders()
from lhotse.dataset import find_pessimistic_batches from lhotse.dataset import find_pessimistic_batches
logging.info('Sanity check -- see if any of the batches in epoch 0 would cause OOM.')
logging.info(
"Sanity check -- see if any of the batches in epoch 0 would cause OOM."
)
batches, crit_values = find_pessimistic_batches(train_dl.sampler) batches, crit_values = find_pessimistic_batches(train_dl.sampler)
for criterion, cuts in batches.items(): for criterion, cuts in batches.items():
logging.info(f'* criterion: {criterion} (={crit_values[criterion]}) ...') logging.info(
f"* criterion: {criterion} (={crit_values[criterion]}) ..."
)
batch = train_dl.dataset[cuts] batch = train_dl.dataset[cuts]
try: try:
compute_loss(params=params, model=model, batch=batch, graph_compiler=graph_compiler, is_training=True) compute_loss(
logging.info('OK!') params=params,
model=model,
batch=batch,
graph_compiler=graph_compiler,
is_training=True,
)
logging.info("OK!")
except RuntimeError as e: except RuntimeError as e:
if 'CUDA out of memory' in str(e): if "CUDA out of memory" in str(e):
logging.error( logging.error(
'Your GPU ran out of memory with the current max_duration setting. ' "Your GPU ran out of memory with the current "
'We recommend decreasing max_duration and trying again.' "max_duration setting. We recommend decreasing "
"max_duration and trying again."
) )
raise raise