mirrors/icefall
1
0
Fork 0
mirror of https://github.com/k2-fsa/icefall.git synced 2025-08-08 09:32:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

Replace ChunkedLilcomHdf5Writer with LilcomChunkyWriter. (#411)

Browse Source
This commit is contained in:
Fangjun Kuang 2022-06-09 11:18:52 +08:00 committed by GitHub
parent 8512aaf585
commit ed66877694
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 25 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
egs/aidatatang_200zh/ASR/local/compute_fbank_aidatatang_200zh.py
View File

@ -29,7 +29,7 @@ import os
from pathlib import Path
import torch
from lhotse import ChunkedLilcomHdf5Writer, CutSet, Fbank, FbankConfig
from lhotse import CutSet, Fbank, FbankConfig, LilcomChunkyWriter
from lhotse.recipes.utils import read_manifests_if_cached
from icefall.utils import get_executor
@ -90,7 +90,7 @@ def compute_fbank_aidatatang_200zh(num_mel_bins: int = 80):
# when an executor is specified, make more partitions
num_jobs=num_jobs if ex is None else 80,
executor=ex,
storage_type=ChunkedLilcomHdf5Writer,
storage_type=LilcomChunkyWriter,
)
cut_set.to_file(output_dir / f"{prefix}_cuts_{partition}.{suffix}")

4
egs/aishell/ASR/local/compute_fbank_aidatatang_200zh.py
View File

@ -29,7 +29,7 @@ import os
from pathlib import Path
import torch
from lhotse import ChunkedLilcomHdf5Writer, CutSet, Fbank, FbankConfig
from lhotse import CutSet, Fbank, FbankConfig, LilcomChunkyWriter
from lhotse.recipes.utils import read_manifests_if_cached
from icefall.utils import get_executor
@ -90,7 +90,7 @@ def compute_fbank_aidatatang_200zh(num_mel_bins: int = 80):
# when an executor is specified, make more partitions
num_jobs=num_jobs if ex is None else 80,
executor=ex,
storage_type=ChunkedLilcomHdf5Writer,
storage_type=LilcomChunkyWriter,
)
cut_set.to_file(output_dir / f"{prefix}_cuts_{partition}.{suffix}")

4
egs/aishell/ASR/local/compute_fbank_aishell.py
View File

@ -29,7 +29,7 @@ import os
from pathlib import Path
import torch
from lhotse import ChunkedLilcomHdf5Writer, CutSet, Fbank, FbankConfig
from lhotse import CutSet, Fbank, FbankConfig, LilcomChunkyWriter
from lhotse.recipes.utils import read_manifests_if_cached
from icefall.utils import get_executor
@ -86,7 +86,7 @@ def compute_fbank_aishell(num_mel_bins: int = 80):
# when an executor is specified, make more partitions
num_jobs=num_jobs if ex is None else 80,
executor=ex,
storage_type=ChunkedLilcomHdf5Writer,
storage_type=LilcomChunkyWriter,
)
cut_set.to_file(output_dir / f"{prefix}_cuts_{partition}.{suffix}")

4
egs/alimeeting/ASR/local/compute_fbank_alimeeting.py
View File

@ -29,7 +29,7 @@ import os
from pathlib import Path
import torch
from lhotse import ChunkedLilcomHdf5Writer, CutSet, Fbank, FbankConfig
from lhotse import CutSet, Fbank, FbankConfig, LilcomChunkyWriter
from lhotse.recipes.utils import read_manifests_if_cached
from icefall.utils import get_executor
@ -90,7 +90,7 @@ def compute_fbank_alimeeting(num_mel_bins: int = 80):
# when an executor is specified, make more partitions
num_jobs=cur_num_jobs,
executor=ex,
storage_type=ChunkedLilcomHdf5Writer,
storage_type=LilcomChunkyWriter,
)
logging.info("About splitting cuts into smaller chunks")

4
egs/librispeech/ASR/local/compute_fbank_librispeech.py
View File

@ -28,7 +28,7 @@ import os
from pathlib import Path
import torch
from lhotse import ChunkedLilcomHdf5Writer, CutSet, Fbank, FbankConfig
from lhotse import CutSet, Fbank, FbankConfig, LilcomChunkyWriter
from lhotse.recipes.utils import read_manifests_if_cached
from icefall.utils import get_executor
@ -91,7 +91,7 @@ def compute_fbank_librispeech():
# when an executor is specified, make more partitions
num_jobs=num_jobs if ex is None else 80,
executor=ex,
storage_type=ChunkedLilcomHdf5Writer,
storage_type=LilcomChunkyWriter,
)
cut_set.to_file(output_dir / cuts_filename)

4
egs/librispeech/ASR/local/compute_fbank_musan.py
View File

@ -28,7 +28,7 @@ import os
from pathlib import Path
import torch
from lhotse import ChunkedLilcomHdf5Writer, CutSet, Fbank, FbankConfig, combine
from lhotse import CutSet, Fbank, FbankConfig, LilcomChunkyWriter, combine
from lhotse.recipes.utils import read_manifests_if_cached
from icefall.utils import get_executor
@ -92,7 +92,7 @@ def compute_fbank_musan():
storage_path=f"{output_dir}/musan_feats",
num_jobs=num_jobs if ex is None else 80,
executor=ex,
storage_type=ChunkedLilcomHdf5Writer,
storage_type=LilcomChunkyWriter,
)
)
musan_cuts.to_file(musan_cuts_path)

4
egs/spgispeech/ASR/local/compute_fbank_musan.py
View File

@ -27,7 +27,7 @@ import logging
from pathlib import Path
import torch
from lhotse import ChunkedLilcomHdf5Writer, CutSet, combine
from lhotse import CutSet, LilcomChunkyWriter, combine
from lhotse.features.kaldifeat import (
KaldifeatFbank,
KaldifeatFbankConfig,
@ -91,7 +91,7 @@ def compute_fbank_musan():
storage_path=output_dir / "feats_musan",
batch_duration=500,
num_workers=4,
storage_type=ChunkedLilcomHdf5Writer,
storage_type=LilcomChunkyWriter,
)
)

6
egs/spgispeech/ASR/local/compute_fbank_spgispeech.py
View File

@ -27,7 +27,7 @@ import logging
from pathlib import Path
import torch
from lhotse import ChunkedLilcomHdf5Writer, load_manifest_lazy
from lhotse import LilcomChunkyWriter, load_manifest_lazy
from lhotse.features.kaldifeat import (
KaldifeatFbank,
KaldifeatFbankConfig,
@ -118,7 +118,7 @@ def compute_fbank_spgispeech(args):
storage_path=output_dir / f"feats_train_{idx}",
batch_duration=500,
num_workers=4,
storage_type=ChunkedLilcomHdf5Writer,
storage_type=LilcomChunkyWriter,
)
cs.to_file(cuts_train_idx_path)
@ -137,7 +137,7 @@ def compute_fbank_spgispeech(args):
manifest_path=src_dir / f"cuts_{partition}.jsonl.gz",
batch_duration=500,
num_workers=4,
storage_type=ChunkedLilcomHdf5Writer,
storage_type=LilcomChunkyWriter,
)

4
egs/tedlium3/ASR/local/compute_fbank_tedlium.py
View File

@ -27,7 +27,7 @@ import os
from pathlib import Path
import torch
from lhotse import ChunkedLilcomHdf5Writer, CutSet, Fbank, FbankConfig
from lhotse import CutSet, Fbank, FbankConfig, LilcomChunkyWriter
from lhotse.recipes.utils import read_manifests_if_cached
from icefall.utils import get_executor
@ -89,7 +89,7 @@ def compute_fbank_tedlium():
# when an executor is specified, make more partitions
num_jobs=cur_num_jobs,
executor=ex,
storage_type=ChunkedLilcomHdf5Writer,
storage_type=LilcomChunkyWriter,
)
# Split long cuts into many short and un-overlapping cuts
cut_set = cut_set.trim_to_supervisions(keep_overlapping=False)

4
egs/timit/ASR/local/compute_fbank_timit.py
View File

@ -29,7 +29,7 @@ import os
from pathlib import Path
import torch
from lhotse import ChunkedLilcomHdf5Writer, CutSet, Fbank, FbankConfig
from lhotse import CutSet, Fbank, FbankConfig, LilcomChunkyWriter
from lhotse.recipes.utils import read_manifests_if_cached
from icefall.utils import get_executor
@ -88,7 +88,7 @@ def compute_fbank_timit():
# when an executor is specified, make more partitions
num_jobs=num_jobs if ex is None else 80,
executor=ex,
storage_type=ChunkedLilcomHdf5Writer,
storage_type=LilcomChunkyWriter,
)
cut_set.to_file(cuts_file)

4
egs/wenetspeech/ASR/local/compute_fbank_wenetspeech_splits.py
View File

@ -23,10 +23,10 @@ from pathlib import Path
import torch
from lhotse import (
ChunkedLilcomHdf5Writer,
CutSet,
KaldifeatFbank,
KaldifeatFbankConfig,
LilcomChunkyWriter,
set_audio_duration_mismatch_tolerance,
set_caching_enabled,
)
@ -135,7 +135,7 @@ def compute_fbank_wenetspeech_splits(args):
storage_path=f"{output_dir}/feats_{subset}_{idx}",
num_workers=args.num_workers,
batch_duration=args.batch_duration,
storage_type=ChunkedLilcomHdf5Writer,
storage_type=LilcomChunkyWriter,
)
logging.info("About to split cuts into smaller chunks.")

4
egs/yesno/ASR/local/compute_fbank_yesno.py
View File

@ -12,7 +12,7 @@ import os
from pathlib import Path
import torch
from lhotse import ChunkedLilcomHdf5Writer, CutSet, Fbank, FbankConfig
from lhotse import CutSet, Fbank, FbankConfig, LilcomChunkyWriter
from lhotse.recipes.utils import read_manifests_if_cached
from icefall.utils import get_executor
@ -74,7 +74,7 @@ def compute_fbank_yesno():
# when an executor is specified, make more partitions
num_jobs=num_jobs if ex is None else 1, # use one job
executor=ex,
storage_type=ChunkedLilcomHdf5Writer,
storage_type=LilcomChunkyWriter,
)
cut_set.to_file(cuts_file)