mirrors/icefall
1
0
Fork 0
mirror of https://github.com/k2-fsa/icefall.git synced 2025-08-09 01:52:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

Replace LilcomChunkyWriter with ChunkedLilcomHdf5Writer. (#404)

Browse Source
This commit is contained in:
Fangjun Kuang 2022-06-07 18:14:25 +08:00 committed by GitHub
parent 80c46f0abd
commit 1094a3cb37
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 17 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
egs/aishell/ASR/local/compute_fbank_aidatatang_200zh.py
View File

@ -29,7 +29,7 @@ import os
from pathlib import Path from pathlib import Path
import torch import torch
from lhotse import CutSet, Fbank, FbankConfig, LilcomChunkyWriter from lhotse import ChunkedLilcomHdf5Writer, CutSet, Fbank, FbankConfig
from lhotse.recipes.utils import read_manifests_if_cached from lhotse.recipes.utils import read_manifests_if_cached
from icefall.utils import get_executor from icefall.utils import get_executor
@ -90,7 +90,7 @@ def compute_fbank_aidatatang_200zh(num_mel_bins: int = 80):
# when an executor is specified, make more partitions # when an executor is specified, make more partitions
num_jobs=num_jobs if ex is None else 80, num_jobs=num_jobs if ex is None else 80,
executor=ex, executor=ex,
storage_type=LilcomChunkyWriter, storage_type=ChunkedLilcomHdf5Writer,
) )
cut_set.to_file(output_dir / f"{prefix}_cuts_{partition}.{suffix}") cut_set.to_file(output_dir / f"{prefix}_cuts_{partition}.{suffix}")

4
egs/aishell/ASR/local/compute_fbank_aishell.py
View File

@ -29,7 +29,7 @@ import os
from pathlib import Path from pathlib import Path
import torch import torch
from lhotse import CutSet, Fbank, FbankConfig, LilcomChunkyWriter from lhotse import ChunkedLilcomHdf5Writer, CutSet, Fbank, FbankConfig
from lhotse.recipes.utils import read_manifests_if_cached from lhotse.recipes.utils import read_manifests_if_cached
from icefall.utils import get_executor from icefall.utils import get_executor
@ -86,7 +86,7 @@ def compute_fbank_aishell(num_mel_bins: int = 80):
# when an executor is specified, make more partitions # when an executor is specified, make more partitions
num_jobs=num_jobs if ex is None else 80, num_jobs=num_jobs if ex is None else 80,
executor=ex, executor=ex,
storage_type=LilcomChunkyWriter, storage_type=ChunkedLilcomHdf5Writer,
) )
cut_set.to_file(output_dir / f"{prefix}_cuts_{partition}.{suffix}") cut_set.to_file(output_dir / f"{prefix}_cuts_{partition}.{suffix}")

4
egs/librispeech/ASR/local/compute_fbank_librispeech.py
View File

@ -28,7 +28,7 @@ import os
from pathlib import Path from pathlib import Path
import torch import torch
from lhotse import CutSet, Fbank, FbankConfig, LilcomChunkyWriter from lhotse import ChunkedLilcomHdf5Writer, CutSet, Fbank, FbankConfig
from lhotse.recipes.utils import read_manifests_if_cached from lhotse.recipes.utils import read_manifests_if_cached
from icefall.utils import get_executor from icefall.utils import get_executor
@ -91,7 +91,7 @@ def compute_fbank_librispeech():
# when an executor is specified, make more partitions # when an executor is specified, make more partitions
num_jobs=num_jobs if ex is None else 80, num_jobs=num_jobs if ex is None else 80,
executor=ex, executor=ex,
storage_type=LilcomChunkyWriter, storage_type=ChunkedLilcomHdf5Writer,
) )
cut_set.to_file(output_dir / cuts_filename) cut_set.to_file(output_dir / cuts_filename)

4
egs/librispeech/ASR/local/compute_fbank_musan.py
View File

@ -28,7 +28,7 @@ import os
from pathlib import Path from pathlib import Path
import torch import torch
from lhotse import CutSet, Fbank, FbankConfig, LilcomChunkyWriter, combine from lhotse import ChunkedLilcomHdf5Writer, CutSet, Fbank, FbankConfig, combine
from lhotse.recipes.utils import read_manifests_if_cached from lhotse.recipes.utils import read_manifests_if_cached
from icefall.utils import get_executor from icefall.utils import get_executor
@ -92,7 +92,7 @@ def compute_fbank_musan():
storage_path=f"{output_dir}/musan_feats", storage_path=f"{output_dir}/musan_feats",
num_jobs=num_jobs if ex is None else 80, num_jobs=num_jobs if ex is None else 80,
executor=ex, executor=ex,
storage_type=LilcomChunkyWriter, storage_type=ChunkedLilcomHdf5Writer,
) )
) )
musan_cuts.to_file(musan_cuts_path) musan_cuts.to_file(musan_cuts_path)

4
egs/spgispeech/ASR/local/compute_fbank_musan.py
View File

@ -27,7 +27,7 @@ import logging
from pathlib import Path from pathlib import Path
import torch import torch
from lhotse import CutSet, LilcomChunkyWriter, combine from lhotse import ChunkedLilcomHdf5Writer, CutSet, combine
from lhotse.features.kaldifeat import ( from lhotse.features.kaldifeat import (
KaldifeatFbank, KaldifeatFbank,
KaldifeatFbankConfig, KaldifeatFbankConfig,
@ -91,7 +91,7 @@ def compute_fbank_musan():
storage_path=output_dir / "feats_musan", storage_path=output_dir / "feats_musan",
batch_duration=500, batch_duration=500,
num_workers=4, num_workers=4,
storage_type=LilcomChunkyWriter, storage_type=ChunkedLilcomHdf5Writer,
) )
) )

6
egs/spgispeech/ASR/local/compute_fbank_spgispeech.py
View File

@ -27,7 +27,7 @@ import logging
from pathlib import Path from pathlib import Path
import torch import torch
from lhotse import LilcomChunkyWriter, load_manifest_lazy from lhotse import ChunkedLilcomHdf5Writer, load_manifest_lazy
from lhotse.features.kaldifeat import ( from lhotse.features.kaldifeat import (
KaldifeatFbank, KaldifeatFbank,
KaldifeatFbankConfig, KaldifeatFbankConfig,
@ -118,7 +118,7 @@ def compute_fbank_spgispeech(args):
storage_path=output_dir / f"feats_train_{idx}", storage_path=output_dir / f"feats_train_{idx}",
batch_duration=500, batch_duration=500,
num_workers=4, num_workers=4,
storage_type=LilcomChunkyWriter, storage_type=ChunkedLilcomHdf5Writer,
) )
cs.to_file(cuts_train_idx_path) cs.to_file(cuts_train_idx_path)
@ -137,7 +137,7 @@ def compute_fbank_spgispeech(args):
manifest_path=src_dir / f"cuts_{partition}.jsonl.gz", manifest_path=src_dir / f"cuts_{partition}.jsonl.gz",
batch_duration=500, batch_duration=500,
num_workers=4, num_workers=4,
storage_type=LilcomChunkyWriter, storage_type=ChunkedLilcomHdf5Writer,
) )

4
egs/timit/ASR/local/compute_fbank_timit.py
View File

@ -29,7 +29,7 @@ import os
from pathlib import Path from pathlib import Path
import torch import torch
from lhotse import CutSet, Fbank, FbankConfig, LilcomChunkyWriter from lhotse import ChunkedLilcomHdf5Writer, CutSet, Fbank, FbankConfig
from lhotse.recipes.utils import read_manifests_if_cached from lhotse.recipes.utils import read_manifests_if_cached
from icefall.utils import get_executor from icefall.utils import get_executor
@ -88,7 +88,7 @@ def compute_fbank_timit():
# when an executor is specified, make more partitions # when an executor is specified, make more partitions
num_jobs=num_jobs if ex is None else 80, num_jobs=num_jobs if ex is None else 80,
executor=ex, executor=ex,
storage_type=LilcomChunkyWriter, storage_type=ChunkedLilcomHdf5Writer,
) )
cut_set.to_file(cuts_file) cut_set.to_file(cuts_file)

4
egs/yesno/ASR/local/compute_fbank_yesno.py
View File

@ -12,7 +12,7 @@ import os
from pathlib import Path from pathlib import Path
import torch import torch
from lhotse import CutSet, Fbank, FbankConfig, LilcomChunkyWriter from lhotse import ChunkedLilcomHdf5Writer, CutSet, Fbank, FbankConfig
from lhotse.recipes.utils import read_manifests_if_cached from lhotse.recipes.utils import read_manifests_if_cached
from icefall.utils import get_executor from icefall.utils import get_executor
@ -74,7 +74,7 @@ def compute_fbank_yesno():
# when an executor is specified, make more partitions # when an executor is specified, make more partitions
num_jobs=num_jobs if ex is None else 1, # use one job num_jobs=num_jobs if ex is None else 1, # use one job
executor=ex, executor=ex,
storage_type=LilcomChunkyWriter, storage_type=ChunkedLilcomHdf5Writer,
) )
cut_set.to_file(cuts_file) cut_set.to_file(cuts_file)