mirrors/icefall
1
0
Fork 0
mirror of https://github.com/k2-fsa/icefall.git synced 2025-08-08 09:32:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

Replace ChunkedLilcomHdf5Writer with LilcomChunkyWriter. (#411)

Browse Source
This commit is contained in:
Fangjun Kuang 2022-06-09 11:18:52 +08:00 committed by GitHub
parent 8512aaf585
commit ed66877694
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 25 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
egs/aidatatang_200zh/ASR/local/compute_fbank_aidatatang_200zh.py
View File

@ -29,7 +29,7 @@ import os
from pathlib import Path from pathlib import Path
import torch import torch
from lhotse import ChunkedLilcomHdf5Writer, CutSet, Fbank, FbankConfig from lhotse import CutSet, Fbank, FbankConfig, LilcomChunkyWriter
from lhotse.recipes.utils import read_manifests_if_cached from lhotse.recipes.utils import read_manifests_if_cached
from icefall.utils import get_executor from icefall.utils import get_executor
@ -90,7 +90,7 @@ def compute_fbank_aidatatang_200zh(num_mel_bins: int = 80):
# when an executor is specified, make more partitions # when an executor is specified, make more partitions
num_jobs=num_jobs if ex is None else 80, num_jobs=num_jobs if ex is None else 80,
executor=ex, executor=ex,
storage_type=ChunkedLilcomHdf5Writer, storage_type=LilcomChunkyWriter,
) )
cut_set.to_file(output_dir / f"{prefix}_cuts_{partition}.{suffix}") cut_set.to_file(output_dir / f"{prefix}_cuts_{partition}.{suffix}")

4
egs/aishell/ASR/local/compute_fbank_aidatatang_200zh.py
View File

@ -29,7 +29,7 @@ import os
from pathlib import Path from pathlib import Path
import torch import torch
from lhotse import ChunkedLilcomHdf5Writer, CutSet, Fbank, FbankConfig from lhotse import CutSet, Fbank, FbankConfig, LilcomChunkyWriter
from lhotse.recipes.utils import read_manifests_if_cached from lhotse.recipes.utils import read_manifests_if_cached
from icefall.utils import get_executor from icefall.utils import get_executor
@ -90,7 +90,7 @@ def compute_fbank_aidatatang_200zh(num_mel_bins: int = 80):
# when an executor is specified, make more partitions # when an executor is specified, make more partitions
num_jobs=num_jobs if ex is None else 80, num_jobs=num_jobs if ex is None else 80,
executor=ex, executor=ex,
storage_type=ChunkedLilcomHdf5Writer, storage_type=LilcomChunkyWriter,
) )
cut_set.to_file(output_dir / f"{prefix}_cuts_{partition}.{suffix}") cut_set.to_file(output_dir / f"{prefix}_cuts_{partition}.{suffix}")

4
egs/aishell/ASR/local/compute_fbank_aishell.py
View File

@ -29,7 +29,7 @@ import os
from pathlib import Path from pathlib import Path
import torch import torch
from lhotse import ChunkedLilcomHdf5Writer, CutSet, Fbank, FbankConfig from lhotse import CutSet, Fbank, FbankConfig, LilcomChunkyWriter
from lhotse.recipes.utils import read_manifests_if_cached from lhotse.recipes.utils import read_manifests_if_cached
from icefall.utils import get_executor from icefall.utils import get_executor
@ -86,7 +86,7 @@ def compute_fbank_aishell(num_mel_bins: int = 80):
# when an executor is specified, make more partitions # when an executor is specified, make more partitions
num_jobs=num_jobs if ex is None else 80, num_jobs=num_jobs if ex is None else 80,
executor=ex, executor=ex,
storage_type=ChunkedLilcomHdf5Writer, storage_type=LilcomChunkyWriter,
) )
cut_set.to_file(output_dir / f"{prefix}_cuts_{partition}.{suffix}") cut_set.to_file(output_dir / f"{prefix}_cuts_{partition}.{suffix}")

4
egs/alimeeting/ASR/local/compute_fbank_alimeeting.py
View File

@ -29,7 +29,7 @@ import os
from pathlib import Path from pathlib import Path
import torch import torch
from lhotse import ChunkedLilcomHdf5Writer, CutSet, Fbank, FbankConfig from lhotse import CutSet, Fbank, FbankConfig, LilcomChunkyWriter
from lhotse.recipes.utils import read_manifests_if_cached from lhotse.recipes.utils import read_manifests_if_cached
from icefall.utils import get_executor from icefall.utils import get_executor
@ -90,7 +90,7 @@ def compute_fbank_alimeeting(num_mel_bins: int = 80):
# when an executor is specified, make more partitions # when an executor is specified, make more partitions
num_jobs=cur_num_jobs, num_jobs=cur_num_jobs,
executor=ex, executor=ex,
storage_type=ChunkedLilcomHdf5Writer, storage_type=LilcomChunkyWriter,
) )
logging.info("About splitting cuts into smaller chunks") logging.info("About splitting cuts into smaller chunks")

4
egs/librispeech/ASR/local/compute_fbank_librispeech.py
View File

@ -28,7 +28,7 @@ import os
from pathlib import Path from pathlib import Path
import torch import torch
from lhotse import ChunkedLilcomHdf5Writer, CutSet, Fbank, FbankConfig from lhotse import CutSet, Fbank, FbankConfig, LilcomChunkyWriter
from lhotse.recipes.utils import read_manifests_if_cached from lhotse.recipes.utils import read_manifests_if_cached
from icefall.utils import get_executor from icefall.utils import get_executor
@ -91,7 +91,7 @@ def compute_fbank_librispeech():
# when an executor is specified, make more partitions # when an executor is specified, make more partitions
num_jobs=num_jobs if ex is None else 80, num_jobs=num_jobs if ex is None else 80,
executor=ex, executor=ex,
storage_type=ChunkedLilcomHdf5Writer, storage_type=LilcomChunkyWriter,
) )
cut_set.to_file(output_dir / cuts_filename) cut_set.to_file(output_dir / cuts_filename)

4
egs/librispeech/ASR/local/compute_fbank_musan.py
View File

@ -28,7 +28,7 @@ import os
from pathlib import Path from pathlib import Path
import torch import torch
from lhotse import ChunkedLilcomHdf5Writer, CutSet, Fbank, FbankConfig, combine from lhotse import CutSet, Fbank, FbankConfig, LilcomChunkyWriter, combine
from lhotse.recipes.utils import read_manifests_if_cached from lhotse.recipes.utils import read_manifests_if_cached
from icefall.utils import get_executor from icefall.utils import get_executor
@ -92,7 +92,7 @@ def compute_fbank_musan():
storage_path=f"{output_dir}/musan_feats", storage_path=f"{output_dir}/musan_feats",
num_jobs=num_jobs if ex is None else 80, num_jobs=num_jobs if ex is None else 80,
executor=ex, executor=ex,
storage_type=ChunkedLilcomHdf5Writer, storage_type=LilcomChunkyWriter,
) )
) )
musan_cuts.to_file(musan_cuts_path) musan_cuts.to_file(musan_cuts_path)

4
egs/spgispeech/ASR/local/compute_fbank_musan.py
View File

@ -27,7 +27,7 @@ import logging
from pathlib import Path from pathlib import Path
import torch import torch
from lhotse import ChunkedLilcomHdf5Writer, CutSet, combine from lhotse import CutSet, LilcomChunkyWriter, combine
from lhotse.features.kaldifeat import ( from lhotse.features.kaldifeat import (
KaldifeatFbank, KaldifeatFbank,
KaldifeatFbankConfig, KaldifeatFbankConfig,
@ -91,7 +91,7 @@ def compute_fbank_musan():
storage_path=output_dir / "feats_musan", storage_path=output_dir / "feats_musan",
batch_duration=500, batch_duration=500,
num_workers=4, num_workers=4,
storage_type=ChunkedLilcomHdf5Writer, storage_type=LilcomChunkyWriter,
) )
) )

6
egs/spgispeech/ASR/local/compute_fbank_spgispeech.py
View File

@ -27,7 +27,7 @@ import logging
from pathlib import Path from pathlib import Path
import torch import torch
from lhotse import ChunkedLilcomHdf5Writer, load_manifest_lazy from lhotse import LilcomChunkyWriter, load_manifest_lazy
from lhotse.features.kaldifeat import ( from lhotse.features.kaldifeat import (
KaldifeatFbank, KaldifeatFbank,
KaldifeatFbankConfig, KaldifeatFbankConfig,
@ -118,7 +118,7 @@ def compute_fbank_spgispeech(args):
storage_path=output_dir / f"feats_train_{idx}", storage_path=output_dir / f"feats_train_{idx}",
batch_duration=500, batch_duration=500,
num_workers=4, num_workers=4,
storage_type=ChunkedLilcomHdf5Writer, storage_type=LilcomChunkyWriter,
) )
cs.to_file(cuts_train_idx_path) cs.to_file(cuts_train_idx_path)
@ -137,7 +137,7 @@ def compute_fbank_spgispeech(args):
manifest_path=src_dir / f"cuts_{partition}.jsonl.gz", manifest_path=src_dir / f"cuts_{partition}.jsonl.gz",
batch_duration=500, batch_duration=500,
num_workers=4, num_workers=4,
storage_type=ChunkedLilcomHdf5Writer, storage_type=LilcomChunkyWriter,
) )

4
egs/tedlium3/ASR/local/compute_fbank_tedlium.py
View File

@ -27,7 +27,7 @@ import os
from pathlib import Path from pathlib import Path
import torch import torch
from lhotse import ChunkedLilcomHdf5Writer, CutSet, Fbank, FbankConfig from lhotse import CutSet, Fbank, FbankConfig, LilcomChunkyWriter
from lhotse.recipes.utils import read_manifests_if_cached from lhotse.recipes.utils import read_manifests_if_cached
from icefall.utils import get_executor from icefall.utils import get_executor
@ -89,7 +89,7 @@ def compute_fbank_tedlium():
# when an executor is specified, make more partitions # when an executor is specified, make more partitions
num_jobs=cur_num_jobs, num_jobs=cur_num_jobs,
executor=ex, executor=ex,
storage_type=ChunkedLilcomHdf5Writer, storage_type=LilcomChunkyWriter,
) )
# Split long cuts into many short and un-overlapping cuts # Split long cuts into many short and un-overlapping cuts
cut_set = cut_set.trim_to_supervisions(keep_overlapping=False) cut_set = cut_set.trim_to_supervisions(keep_overlapping=False)

4
egs/timit/ASR/local/compute_fbank_timit.py
View File

@ -29,7 +29,7 @@ import os
from pathlib import Path from pathlib import Path
import torch import torch
from lhotse import ChunkedLilcomHdf5Writer, CutSet, Fbank, FbankConfig from lhotse import CutSet, Fbank, FbankConfig, LilcomChunkyWriter
from lhotse.recipes.utils import read_manifests_if_cached from lhotse.recipes.utils import read_manifests_if_cached
from icefall.utils import get_executor from icefall.utils import get_executor
@ -88,7 +88,7 @@ def compute_fbank_timit():
# when an executor is specified, make more partitions # when an executor is specified, make more partitions
num_jobs=num_jobs if ex is None else 80, num_jobs=num_jobs if ex is None else 80,
executor=ex, executor=ex,
storage_type=ChunkedLilcomHdf5Writer, storage_type=LilcomChunkyWriter,
) )
cut_set.to_file(cuts_file) cut_set.to_file(cuts_file)

4
egs/wenetspeech/ASR/local/compute_fbank_wenetspeech_splits.py
View File

@ -23,10 +23,10 @@ from pathlib import Path
import torch import torch
from lhotse import ( from lhotse import (
ChunkedLilcomHdf5Writer,
CutSet, CutSet,
KaldifeatFbank, KaldifeatFbank,
KaldifeatFbankConfig, KaldifeatFbankConfig,
LilcomChunkyWriter,
set_audio_duration_mismatch_tolerance, set_audio_duration_mismatch_tolerance,
set_caching_enabled, set_caching_enabled,
) )
@ -135,7 +135,7 @@ def compute_fbank_wenetspeech_splits(args):
storage_path=f"{output_dir}/feats_{subset}_{idx}", storage_path=f"{output_dir}/feats_{subset}_{idx}",
num_workers=args.num_workers, num_workers=args.num_workers,
batch_duration=args.batch_duration, batch_duration=args.batch_duration,
storage_type=ChunkedLilcomHdf5Writer, storage_type=LilcomChunkyWriter,
) )
logging.info("About to split cuts into smaller chunks.") logging.info("About to split cuts into smaller chunks.")

4
egs/yesno/ASR/local/compute_fbank_yesno.py
View File

@ -12,7 +12,7 @@ import os
from pathlib import Path from pathlib import Path
import torch import torch
from lhotse import ChunkedLilcomHdf5Writer, CutSet, Fbank, FbankConfig from lhotse import CutSet, Fbank, FbankConfig, LilcomChunkyWriter
from lhotse.recipes.utils import read_manifests_if_cached from lhotse.recipes.utils import read_manifests_if_cached
from icefall.utils import get_executor from icefall.utils import get_executor
@ -74,7 +74,7 @@ def compute_fbank_yesno():
# when an executor is specified, make more partitions # when an executor is specified, make more partitions
num_jobs=num_jobs if ex is None else 1, # use one job num_jobs=num_jobs if ex is None else 1, # use one job
executor=ex, executor=ex,
storage_type=ChunkedLilcomHdf5Writer, storage_type=LilcomChunkyWriter,
) )
cut_set.to_file(cuts_file) cut_set.to_file(cuts_file)