mirror of
https://github.com/k2-fsa/icefall.git
synced 2025-12-11 06:55:27 +00:00
from local
This commit is contained in:
dohe0342
parent
29b56903d6
commit
07ed265a84
BIN
egs/tedlium3/ASR/.prepare.sh.swp
Normal file
BIN
egs/tedlium3/ASR/.prepare.sh.swp
Normal file
Binary file not shown.
Binary file not shown.
@ -1,5 +1,5 @@
|
|||||||
# Copyright 2021 Piotr Żelasko
|
# Copyright 2021 Piotr Żelasko
|
||||||
# Copyright 2022 Xiaomi Corporation (Author: Mingshuang Luo)
|
# Copyright 2021 Xiaomi Corporation (Author: Mingshuang Luo)
|
||||||
#
|
#
|
||||||
# See ../../../../LICENSE for clarification regarding multiple authors
|
# See ../../../../LICENSE for clarification regarding multiple authors
|
||||||
#
|
#
|
||||||
@ -17,48 +17,32 @@
|
|||||||
|
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
import inspect
|
|
||||||
import logging
|
import logging
|
||||||
from glob import glob
|
|
||||||
from functools import lru_cache
|
from functools import lru_cache
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any, Dict, Optional
|
from typing import Any, Dict, Optional
|
||||||
|
|
||||||
import torch
|
|
||||||
from lhotse import CutSet, Fbank, FbankConfig, load_manifest, load_manifest_lazy
|
from lhotse import CutSet, Fbank, FbankConfig, load_manifest, load_manifest_lazy
|
||||||
from lhotse.dataset import ( # noqa F401 for PrecomputedFeatures
|
from lhotse.dataset import (
|
||||||
CutConcatenate,
|
CutConcatenate,
|
||||||
CutMix,
|
CutMix,
|
||||||
DynamicBucketingSampler,
|
DynamicBucketingSampler,
|
||||||
K2SpeechRecognitionDataset,
|
K2SpeechRecognitionDataset,
|
||||||
PrecomputedFeatures,
|
|
||||||
SingleCutSampler,
|
SingleCutSampler,
|
||||||
SpecAugment,
|
SpecAugment,
|
||||||
)
|
)
|
||||||
from lhotse.dataset.input_strategies import ( # noqa F401 For AudioSamples
|
from lhotse.dataset.input_strategies import OnTheFlyFeatures
|
||||||
AudioSamples,
|
|
||||||
OnTheFlyFeatures,
|
|
||||||
)
|
|
||||||
from lhotse.utils import fix_random_seed
|
|
||||||
from torch.utils.data import DataLoader
|
from torch.utils.data import DataLoader
|
||||||
|
|
||||||
from icefall.utils import str2bool
|
from icefall.utils import str2bool
|
||||||
|
|
||||||
|
|
||||||
class _SeedWorkers:
|
class TedLiumAsrDataModule:
|
||||||
def __init__(self, seed: int):
|
|
||||||
self.seed = seed
|
|
||||||
|
|
||||||
def __call__(self, worker_id: int):
|
|
||||||
fix_random_seed(self.seed + worker_id)
|
|
||||||
|
|
||||||
|
|
||||||
class LibriSpeechAsrDataModule:
|
|
||||||
"""
|
"""
|
||||||
DataModule for k2 ASR experiments.
|
DataModule for k2 ASR experiments.
|
||||||
It assumes there is always one train and valid dataloader,
|
It assumes there is always one train and valid dataloader,
|
||||||
but there can be multiple test dataloaders (e.g. LibriSpeech test-clean
|
but there can be multiple test dataloaders (e.g. TEDLium3 dev
|
||||||
and test-other).
|
and test).
|
||||||
|
|
||||||
It contains all the common data pipeline modules used in ASR
|
It contains all the common data pipeline modules used in ASR
|
||||||
experiments, e.g.:
|
experiments, e.g.:
|
||||||
@ -83,12 +67,6 @@ class LibriSpeechAsrDataModule:
|
|||||||
"effective batch sizes, sampling strategies, applied data "
|
"effective batch sizes, sampling strategies, applied data "
|
||||||
"augmentations, etc.",
|
"augmentations, etc.",
|
||||||
)
|
)
|
||||||
group.add_argument(
|
|
||||||
"--full-libri",
|
|
||||||
type=str2bool,
|
|
||||||
default=False,
|
|
||||||
help="When enabled, use 960h LibriSpeech. Otherwise, use 100h subset.",
|
|
||||||
)
|
|
||||||
group.add_argument(
|
group.add_argument(
|
||||||
"--manifest-dir",
|
"--manifest-dir",
|
||||||
type=Path,
|
type=Path,
|
||||||
@ -98,7 +76,7 @@ class LibriSpeechAsrDataModule:
|
|||||||
group.add_argument(
|
group.add_argument(
|
||||||
"--max-duration",
|
"--max-duration",
|
||||||
type=int,
|
type=int,
|
||||||
default=250.0,
|
default=200.0,
|
||||||
help="Maximum pooled recordings duration (seconds) in a "
|
help="Maximum pooled recordings duration (seconds) in a "
|
||||||
"single batch. You can reduce it if it causes CUDA OOM.",
|
"single batch. You can reduce it if it causes CUDA OOM.",
|
||||||
)
|
)
|
||||||
@ -153,12 +131,6 @@ class LibriSpeechAsrDataModule:
|
|||||||
help="When enabled (=default), the examples will be "
|
help="When enabled (=default), the examples will be "
|
||||||
"shuffled for each epoch.",
|
"shuffled for each epoch.",
|
||||||
)
|
)
|
||||||
group.add_argument(
|
|
||||||
"--drop-last",
|
|
||||||
type=str2bool,
|
|
||||||
default=True,
|
|
||||||
help="Whether to drop last batch. Used by sampler.",
|
|
||||||
)
|
|
||||||
group.add_argument(
|
group.add_argument(
|
||||||
"--return-cuts",
|
"--return-cuts",
|
||||||
type=str2bool,
|
type=str2bool,
|
||||||
@ -167,7 +139,6 @@ class LibriSpeechAsrDataModule:
|
|||||||
"field: batch['supervisions']['cut'] with the cuts that "
|
"field: batch['supervisions']['cut'] with the cuts that "
|
||||||
"were used to construct it.",
|
"were used to construct it.",
|
||||||
)
|
)
|
||||||
|
|
||||||
group.add_argument(
|
group.add_argument(
|
||||||
"--num-workers",
|
"--num-workers",
|
||||||
type=int,
|
type=int,
|
||||||
@ -175,14 +146,12 @@ class LibriSpeechAsrDataModule:
|
|||||||
help="The number of training dataloader workers that "
|
help="The number of training dataloader workers that "
|
||||||
"collect the batches.",
|
"collect the batches.",
|
||||||
)
|
)
|
||||||
|
|
||||||
group.add_argument(
|
group.add_argument(
|
||||||
"--enable-spec-aug",
|
"--enable-spec-aug",
|
||||||
type=str2bool,
|
type=str2bool,
|
||||||
default=False,
|
default=True,
|
||||||
help="When enabled, use SpecAugment for training dataset.",
|
help="When enabled, use SpecAugment for training dataset.",
|
||||||
)
|
)
|
||||||
|
|
||||||
group.add_argument(
|
group.add_argument(
|
||||||
"--spec-aug-time-warp-factor",
|
"--spec-aug-time-warp-factor",
|
||||||
type=int,
|
type=int,
|
||||||
@ -192,38 +161,16 @@ class LibriSpeechAsrDataModule:
|
|||||||
"Larger values mean more warping. "
|
"Larger values mean more warping. "
|
||||||
"A value less than 1 means to disable time warp.",
|
"A value less than 1 means to disable time warp.",
|
||||||
)
|
)
|
||||||
|
|
||||||
group.add_argument(
|
group.add_argument(
|
||||||
"--enable-musan",
|
"--enable-musan",
|
||||||
type=str2bool,
|
type=str2bool,
|
||||||
default=True,
|
default=True,
|
||||||
help="When enabled, select noise from MUSAN and mix it"
|
help="When enabled, select noise from MUSAN and mix it"
|
||||||
"with training dataset. ",
|
"with training dataset.",
|
||||||
)
|
|
||||||
|
|
||||||
group.add_argument(
|
|
||||||
"--input-strategy",
|
|
||||||
type=str,
|
|
||||||
default="AudioSamples",
|
|
||||||
help="AudioSamples or PrecomputedFeatures",
|
|
||||||
)
|
|
||||||
|
|
||||||
group.add_argument(
|
|
||||||
"--spk-id",
|
|
||||||
type=int,
|
|
||||||
default=0,
|
|
||||||
)
|
|
||||||
|
|
||||||
group.add_argument(
|
|
||||||
"--prefix",
|
|
||||||
type=str,
|
|
||||||
default='vox',
|
|
||||||
)
|
)
|
||||||
|
|
||||||
def train_dataloaders(
|
def train_dataloaders(
|
||||||
self,
|
self, cuts_train: CutSet, sampler_state_dict: Optional[Dict[str, Any]] = None
|
||||||
cuts_train: CutSet,
|
|
||||||
sampler_state_dict: Optional[Dict[str, Any]] = None,
|
|
||||||
) -> DataLoader:
|
) -> DataLoader:
|
||||||
"""
|
"""
|
||||||
Args:
|
Args:
|
||||||
@ -232,10 +179,30 @@ class LibriSpeechAsrDataModule:
|
|||||||
sampler_state_dict:
|
sampler_state_dict:
|
||||||
The state dict for the training sampler.
|
The state dict for the training sampler.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
input_transforms = []
|
||||||
|
if self.args.enable_spec_aug:
|
||||||
|
logging.info("Enable SpecAugment")
|
||||||
|
logging.info(f"Time warp factor: {self.args.spec_aug_time_warp_factor}")
|
||||||
|
|
||||||
|
input_transforms.append(
|
||||||
|
SpecAugment(
|
||||||
|
time_warp_factor=self.args.spec_aug_time_warp_factor,
|
||||||
|
num_frame_masks=10,
|
||||||
|
features_mask_size=27,
|
||||||
|
num_feature_masks=2,
|
||||||
|
frames_mask_size=100,
|
||||||
|
max_frames_mask_fraction=0.15,
|
||||||
|
p=0.9,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
logging.info("Disable SpecAugment")
|
||||||
|
|
||||||
|
logging.info("About to get Musan cuts")
|
||||||
transforms = []
|
transforms = []
|
||||||
if self.args.enable_musan:
|
if self.args.enable_musan:
|
||||||
logging.info("Enable MUSAN")
|
logging.info("Enable MUSAN")
|
||||||
logging.info("About to get Musan cuts")
|
|
||||||
cuts_musan = load_manifest(self.args.manifest_dir / "musan_cuts.jsonl.gz")
|
cuts_musan = load_manifest(self.args.manifest_dir / "musan_cuts.jsonl.gz")
|
||||||
transforms.append(
|
transforms.append(
|
||||||
CutMix(cuts=cuts_musan, prob=0.5, snr=(10, 20), preserve_id=True)
|
CutMix(cuts=cuts_musan, prob=0.5, snr=(10, 20), preserve_id=True)
|
||||||
@ -257,40 +224,7 @@ class LibriSpeechAsrDataModule:
|
|||||||
)
|
)
|
||||||
] + transforms
|
] + transforms
|
||||||
|
|
||||||
input_transforms = []
|
|
||||||
if self.args.enable_spec_aug:
|
|
||||||
logging.info("Enable SpecAugment")
|
|
||||||
logging.info(f"Time warp factor: {self.args.spec_aug_time_warp_factor}")
|
|
||||||
# Set the value of num_frame_masks according to Lhotse's version.
|
|
||||||
# In different Lhotse's versions, the default of num_frame_masks is
|
|
||||||
# different.
|
|
||||||
num_frame_masks = 10
|
|
||||||
num_frame_masks_parameter = inspect.signature(
|
|
||||||
SpecAugment.__init__
|
|
||||||
).parameters["num_frame_masks"]
|
|
||||||
if num_frame_masks_parameter.default == 1:
|
|
||||||
num_frame_masks = 2
|
|
||||||
logging.info(f"Num frame mask: {num_frame_masks}")
|
|
||||||
input_transforms.append(
|
|
||||||
SpecAugment(
|
|
||||||
time_warp_factor=self.args.spec_aug_time_warp_factor,
|
|
||||||
num_frame_masks=num_frame_masks,
|
|
||||||
features_mask_size=27,
|
|
||||||
num_feature_masks=2,
|
|
||||||
frames_mask_size=100,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
logging.info("Disable SpecAugment")
|
|
||||||
|
|
||||||
logging.info("About to create train dataset")
|
logging.info("About to create train dataset")
|
||||||
train = K2SpeechRecognitionDataset(
|
|
||||||
input_strategy=eval(self.args.input_strategy)(),
|
|
||||||
cut_transforms=transforms,
|
|
||||||
input_transforms=input_transforms,
|
|
||||||
return_cuts=self.args.return_cuts,
|
|
||||||
)
|
|
||||||
|
|
||||||
if self.args.on_the_fly_feats:
|
if self.args.on_the_fly_feats:
|
||||||
# NOTE: the PerturbSpeed transform should be added only if we
|
# NOTE: the PerturbSpeed transform should be added only if we
|
||||||
# remove it from data prep stage.
|
# remove it from data prep stage.
|
||||||
@ -308,6 +242,12 @@ class LibriSpeechAsrDataModule:
|
|||||||
input_transforms=input_transforms,
|
input_transforms=input_transforms,
|
||||||
return_cuts=self.args.return_cuts,
|
return_cuts=self.args.return_cuts,
|
||||||
)
|
)
|
||||||
|
else:
|
||||||
|
train = K2SpeechRecognitionDataset(
|
||||||
|
cut_transforms=transforms,
|
||||||
|
input_transforms=input_transforms,
|
||||||
|
return_cuts=self.args.return_cuts,
|
||||||
|
)
|
||||||
|
|
||||||
if self.args.bucketing_sampler:
|
if self.args.bucketing_sampler:
|
||||||
logging.info("Using DynamicBucketingSampler.")
|
logging.info("Using DynamicBucketingSampler.")
|
||||||
@ -316,7 +256,7 @@ class LibriSpeechAsrDataModule:
|
|||||||
max_duration=self.args.max_duration,
|
max_duration=self.args.max_duration,
|
||||||
shuffle=self.args.shuffle,
|
shuffle=self.args.shuffle,
|
||||||
num_buckets=self.args.num_buckets,
|
num_buckets=self.args.num_buckets,
|
||||||
drop_last=self.args.drop_last,
|
drop_last=True,
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
logging.info("Using SingleCutSampler.")
|
logging.info("Using SingleCutSampler.")
|
||||||
@ -325,29 +265,24 @@ class LibriSpeechAsrDataModule:
|
|||||||
max_duration=self.args.max_duration,
|
max_duration=self.args.max_duration,
|
||||||
shuffle=self.args.shuffle,
|
shuffle=self.args.shuffle,
|
||||||
)
|
)
|
||||||
logging.info("About to create train dataloader")
|
|
||||||
|
|
||||||
if sampler_state_dict is not None:
|
if sampler_state_dict is not None:
|
||||||
logging.info("Loading sampler state dict")
|
logging.info("Loading sampler state dict")
|
||||||
train_sampler.load_state_dict(sampler_state_dict)
|
train_sampler.load_state_dict(sampler_state_dict)
|
||||||
|
|
||||||
# 'seed' is derived from the current random state, which will have
|
logging.info("About to create train dataloader")
|
||||||
# previously been set in the main process.
|
|
||||||
seed = torch.randint(0, 100000, ()).item()
|
|
||||||
worker_init_fn = _SeedWorkers(seed)
|
|
||||||
|
|
||||||
train_dl = DataLoader(
|
train_dl = DataLoader(
|
||||||
train,
|
train,
|
||||||
sampler=train_sampler,
|
sampler=train_sampler,
|
||||||
batch_size=None,
|
batch_size=None,
|
||||||
num_workers=self.args.num_workers,
|
num_workers=self.args.num_workers,
|
||||||
persistent_workers=False,
|
persistent_workers=False,
|
||||||
worker_init_fn=worker_init_fn,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
return train_dl
|
return train_dl
|
||||||
|
|
||||||
def valid_dataloaders(self, cuts_valid: CutSet) -> DataLoader:
|
def valid_dataloaders(self, cuts_valid: CutSet) -> DataLoader:
|
||||||
|
|
||||||
transforms = []
|
transforms = []
|
||||||
if self.args.concatenate_cuts:
|
if self.args.concatenate_cuts:
|
||||||
transforms = [
|
transforms = [
|
||||||
@ -360,21 +295,21 @@ class LibriSpeechAsrDataModule:
|
|||||||
if self.args.on_the_fly_feats:
|
if self.args.on_the_fly_feats:
|
||||||
validate = K2SpeechRecognitionDataset(
|
validate = K2SpeechRecognitionDataset(
|
||||||
cut_transforms=transforms,
|
cut_transforms=transforms,
|
||||||
input_strategy=eval(self.args.input_strategy)(),
|
input_strategy=OnTheFlyFeatures(Fbank(FbankConfig(num_mel_bins=80))),
|
||||||
#input_strategy=OnTheFlyFeatures(Fbank(FbankConfig(num_mel_bins=80))),
|
|
||||||
return_cuts=self.args.return_cuts,
|
return_cuts=self.args.return_cuts,
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
validate = K2SpeechRecognitionDataset(
|
validate = K2SpeechRecognitionDataset(
|
||||||
cut_transforms=transforms,
|
cut_transforms=transforms,
|
||||||
input_strategy=eval(self.args.input_strategy)(),
|
|
||||||
return_cuts=self.args.return_cuts,
|
return_cuts=self.args.return_cuts,
|
||||||
)
|
)
|
||||||
|
|
||||||
valid_sampler = DynamicBucketingSampler(
|
valid_sampler = DynamicBucketingSampler(
|
||||||
cuts_valid,
|
cuts_valid,
|
||||||
max_duration=self.args.max_duration,
|
max_duration=self.args.max_duration,
|
||||||
shuffle=False,
|
shuffle=False,
|
||||||
)
|
)
|
||||||
|
|
||||||
logging.info("About to create dev dataloader")
|
logging.info("About to create dev dataloader")
|
||||||
valid_dl = DataLoader(
|
valid_dl = DataLoader(
|
||||||
validate,
|
validate,
|
||||||
@ -386,174 +321,48 @@ class LibriSpeechAsrDataModule:
|
|||||||
|
|
||||||
return valid_dl
|
return valid_dl
|
||||||
|
|
||||||
def test_dataloaders(self, cuts: CutSet) -> DataLoader:
|
def test_dataloaders(self, cuts_test: CutSet) -> DataLoader:
|
||||||
|
|
||||||
logging.debug("About to create test dataset")
|
logging.debug("About to create test dataset")
|
||||||
test = K2SpeechRecognitionDataset(
|
if self.args.on_the_fly_feats:
|
||||||
input_strategy=OnTheFlyFeatures(Fbank(FbankConfig(num_mel_bins=80)))
|
test = K2SpeechRecognitionDataset(
|
||||||
if self.args.on_the_fly_feats
|
input_strategy=OnTheFlyFeatures(Fbank(FbankConfig(num_mel_bins=80))),
|
||||||
else eval(self.args.input_strategy)(),
|
return_cuts=self.args.return_cuts,
|
||||||
return_cuts=self.args.return_cuts,
|
)
|
||||||
)
|
else:
|
||||||
sampler = DynamicBucketingSampler(
|
test = K2SpeechRecognitionDataset(
|
||||||
cuts,
|
return_cuts=self.args.return_cuts,
|
||||||
|
)
|
||||||
|
|
||||||
|
test_sampler = DynamicBucketingSampler(
|
||||||
|
cuts_test,
|
||||||
max_duration=self.args.max_duration,
|
max_duration=self.args.max_duration,
|
||||||
shuffle=False,
|
shuffle=False,
|
||||||
)
|
)
|
||||||
|
|
||||||
logging.debug("About to create test dataloader")
|
logging.debug("About to create test dataloader")
|
||||||
test_dl = DataLoader(
|
test_dl = DataLoader(
|
||||||
test,
|
test,
|
||||||
batch_size=None,
|
batch_size=None,
|
||||||
sampler=sampler,
|
sampler=test_sampler,
|
||||||
num_workers=self.args.num_workers,
|
num_workers=self.args.num_workers,
|
||||||
|
persistent_workers=False,
|
||||||
)
|
)
|
||||||
return test_dl
|
return test_dl
|
||||||
|
|
||||||
@lru_cache()
|
|
||||||
def train_clean_10_cuts(self, option=None) -> CutSet:
|
|
||||||
logging.info("About to get train-clean-10 cuts")
|
|
||||||
if option is None:
|
|
||||||
return load_manifest_lazy(
|
|
||||||
self.args.manifest_dir / f"librispeech_cuts_train-clean-100.jsonl"
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
return load_manifest_lazy(
|
|
||||||
self.args.manifest_dir / f"librispeech_cuts_train-clean-10_{option}.jsonl"
|
|
||||||
)
|
|
||||||
|
|
||||||
@lru_cache()
|
@lru_cache()
|
||||||
def train_clean_100_cuts(self, option=None) -> CutSet:
|
def train_cuts(self) -> CutSet:
|
||||||
logging.info("About to get train-clean-100 cuts")
|
logging.info("About to get train cuts")
|
||||||
if option is None:
|
|
||||||
return load_manifest_lazy(
|
|
||||||
self.args.manifest_dir / f"librispeech_cuts_train-clean-100.jsonl"
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
return load_manifest_lazy(
|
|
||||||
self.args.manifest_dir / f"librispeech_cuts_train-clean-100_{option}.jsonl"
|
|
||||||
)
|
|
||||||
|
|
||||||
@lru_cache()
|
|
||||||
def train_clean_360_cuts(self, option=None) -> CutSet:
|
|
||||||
logging.info("About to get train-clean-360 cuts")
|
|
||||||
if option is None:
|
|
||||||
return load_manifest_lazy(
|
|
||||||
self.args.manifest_dir / f"librispeech_cuts_train-clean-360.jsonl"
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
return load_manifest_lazy(
|
|
||||||
self.args.manifest_dir / f"librispeech_cuts_train-clean-360_{option}.jsonl"
|
|
||||||
)
|
|
||||||
|
|
||||||
@lru_cache()
|
|
||||||
def train_other_500_cuts(self, option=None) -> CutSet:
|
|
||||||
logging.info("About to get train-other-500 cuts")
|
|
||||||
if option is None:
|
|
||||||
return load_manifest_lazy(
|
|
||||||
self.args.manifest_dir / f"librispeech_cuts_train-other-500.jsonl"
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
return load_manifest_lazy(
|
|
||||||
self.args.manifest_dir / f"librispeech_cuts_train-other-500_{option}.jsonl"
|
|
||||||
)
|
|
||||||
|
|
||||||
@lru_cache()
|
|
||||||
def train_all_shuf_cuts(self, option=None) -> CutSet:
|
|
||||||
logging.info(
|
|
||||||
"About to get the shuffled train-clean-100, \
|
|
||||||
train-clean-360 and train-other-500 cuts"
|
|
||||||
)
|
|
||||||
if option is None:
|
|
||||||
return load_manifest_lazy(
|
|
||||||
self.args.manifest_dir / f"librispeech_cuts_train-all-shuf.jsonl"
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
return load_manifest_lazy(
|
|
||||||
self.args.manifest_dir / f"librispeech_cuts_train-all-shuf_{option}.jsonl"
|
|
||||||
)
|
|
||||||
|
|
||||||
@lru_cache()
|
|
||||||
def dev_clean_cuts(self, option=None) -> CutSet:
|
|
||||||
logging.info("About to get dev-clean cuts")
|
|
||||||
if option is None:
|
|
||||||
return load_manifest_lazy(
|
|
||||||
self.args.manifest_dir / f"librispeech_cuts_dev-clean.jsonl"
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
return load_manifest_lazy(
|
|
||||||
self.args.manifest_dir / f"librispeech_cuts_dev-clean_{option}.jsonl"
|
|
||||||
)
|
|
||||||
|
|
||||||
@lru_cache()
|
|
||||||
def dev_other_cuts(self, option=None) -> CutSet:
|
|
||||||
logging.info("About to get dev-other cuts")
|
|
||||||
if option is None:
|
|
||||||
return load_manifest_lazy(
|
|
||||||
self.args.manifest_dir / f"librispeech_cuts_dev-other.jsonl"
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
return load_manifest_lazy(
|
|
||||||
self.args.manifest_dir / f"librispeech_cuts_dev-other_{option}.jsonl"
|
|
||||||
)
|
|
||||||
|
|
||||||
@lru_cache()
|
|
||||||
def test_clean_cuts(self, option=None) -> CutSet:
|
|
||||||
logging.info("About to get test-clean cuts")
|
|
||||||
if option is None:
|
|
||||||
return load_manifest_lazy(
|
|
||||||
self.args.manifest_dir / f"librispeech_cuts_test-clean.jsonl"
|
|
||||||
)
|
|
||||||
elif option == 'user':
|
|
||||||
json_list = sorted(glob(str(self.args.manifest_dir) + "/userlibri/test-clean/*"))
|
|
||||||
spk_list = [json.split('/')[-1][:-6] for json in json_list]
|
|
||||||
|
|
||||||
return [load_manifest_lazy(json) for json in json_list], spk_list
|
|
||||||
else:
|
|
||||||
return load_manifest_lazy(
|
|
||||||
self.args.manifest_dir / f"librispeech_cuts_test-clean_{option}.jsonl"
|
|
||||||
)
|
|
||||||
|
|
||||||
@lru_cache()
|
|
||||||
def test_other_cuts(self, option=None) -> CutSet:
|
|
||||||
logging.info("About to get test-other cuts")
|
|
||||||
if option is None:
|
|
||||||
return load_manifest_lazy(
|
|
||||||
self.args.manifest_dir / f"librispeech_cuts_test-other_{option}.jsonl"
|
|
||||||
)
|
|
||||||
elif option == 'user':
|
|
||||||
json_list = sorted(glob(str(self.args.manifest_dir) + "/userlibri/test-other/*"))
|
|
||||||
spk_list = [json.split('/')[-1][:-6] for json in json_list]
|
|
||||||
|
|
||||||
return [load_manifest_lazy(json) for json in json_list], spk_list
|
|
||||||
else:
|
|
||||||
return load_manifest_lazy(
|
|
||||||
self.args.manifest_dir / f"librispeech_cuts_test-other_{option}.jsonl"
|
|
||||||
)
|
|
||||||
|
|
||||||
@lru_cache()
|
|
||||||
def test_clean_user(self, option=None) -> CutSet:
|
|
||||||
logging.info("About to get test-clean user cuts")
|
|
||||||
return load_manifest_lazy(
|
return load_manifest_lazy(
|
||||||
self.args.manifest_dir / f"userlibri/test-clean_sampling/{option}.jsonl"
|
self.args.manifest_dir / "tedlium_cuts_train.jsonl.gz"
|
||||||
)
|
|
||||||
|
|
||||||
@lru_cache()
|
|
||||||
def test_other_user(self, option=None) -> CutSet:
|
|
||||||
logging.info("About to get test-other user cuts")
|
|
||||||
return load_manifest_lazy(
|
|
||||||
self.args.manifest_dir / f"userlibri/test-other_sampling/{option}.jsonl"
|
|
||||||
)
|
|
||||||
|
|
||||||
@lru_cache()
|
|
||||||
def vox_cuts(self, option=None) -> CutSet:
|
|
||||||
logging.info("About to get test-other user cuts")
|
|
||||||
return load_manifest_lazy(
|
|
||||||
self.args.manifest_dir / f"{self.args.prefix}_cuts_{option}.jsonl.gz"
|
|
||||||
)
|
|
||||||
|
|
||||||
@lru_cache()
|
|
||||||
def userlibri_cuts(self, option=None) -> CutSet:
|
|
||||||
logging.info("About to get userlibri cuts")
|
|
||||||
return load_manifest_lazy(
|
|
||||||
self.args.manifest_dir / f"{option}.jsonl"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@lru_cache()
|
||||||
|
def dev_cuts(self) -> CutSet:
|
||||||
|
logging.info("About to get dev cuts")
|
||||||
|
return load_manifest_lazy(self.args.manifest_dir / "tedlium_cuts_dev.jsonl.gz")
|
||||||
|
|
||||||
|
@lru_cache()
|
||||||
|
def test_cuts(self) -> CutSet:
|
||||||
|
logging.info("About to get test cuts")
|
||||||
|
return load_manifest_lazy(self.args.manifest_dir / "tedlium_cuts_test.jsonl.gz")
|
||||||
|
|||||||
@ -0,0 +1,559 @@
|
|||||||
|
# Copyright 2021 Piotr Żelasko
|
||||||
|
# Copyright 2022 Xiaomi Corporation (Author: Mingshuang Luo)
|
||||||
|
#
|
||||||
|
# See ../../../../LICENSE for clarification regarding multiple authors
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import inspect
|
||||||
|
import logging
|
||||||
|
from glob import glob
|
||||||
|
from functools import lru_cache
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, Dict, Optional
|
||||||
|
|
||||||
|
import torch
|
||||||
|
from lhotse import CutSet, Fbank, FbankConfig, load_manifest, load_manifest_lazy
|
||||||
|
from lhotse.dataset import ( # noqa F401 for PrecomputedFeatures
|
||||||
|
CutConcatenate,
|
||||||
|
CutMix,
|
||||||
|
DynamicBucketingSampler,
|
||||||
|
K2SpeechRecognitionDataset,
|
||||||
|
PrecomputedFeatures,
|
||||||
|
SingleCutSampler,
|
||||||
|
SpecAugment,
|
||||||
|
)
|
||||||
|
from lhotse.dataset.input_strategies import ( # noqa F401 For AudioSamples
|
||||||
|
AudioSamples,
|
||||||
|
OnTheFlyFeatures,
|
||||||
|
)
|
||||||
|
from lhotse.utils import fix_random_seed
|
||||||
|
from torch.utils.data import DataLoader
|
||||||
|
|
||||||
|
from icefall.utils import str2bool
|
||||||
|
|
||||||
|
|
||||||
|
class _SeedWorkers:
|
||||||
|
def __init__(self, seed: int):
|
||||||
|
self.seed = seed
|
||||||
|
|
||||||
|
def __call__(self, worker_id: int):
|
||||||
|
fix_random_seed(self.seed + worker_id)
|
||||||
|
|
||||||
|
|
||||||
|
class LibriSpeechAsrDataModule:
|
||||||
|
"""
|
||||||
|
DataModule for k2 ASR experiments.
|
||||||
|
It assumes there is always one train and valid dataloader,
|
||||||
|
but there can be multiple test dataloaders (e.g. LibriSpeech test-clean
|
||||||
|
and test-other).
|
||||||
|
|
||||||
|
It contains all the common data pipeline modules used in ASR
|
||||||
|
experiments, e.g.:
|
||||||
|
- dynamic batch size,
|
||||||
|
- bucketing samplers,
|
||||||
|
- cut concatenation,
|
||||||
|
- augmentation,
|
||||||
|
- on-the-fly feature extraction
|
||||||
|
|
||||||
|
This class should be derived for specific corpora used in ASR tasks.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, args: argparse.Namespace):
|
||||||
|
self.args = args
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def add_arguments(cls, parser: argparse.ArgumentParser):
|
||||||
|
group = parser.add_argument_group(
|
||||||
|
title="ASR data related options",
|
||||||
|
description="These options are used for the preparation of "
|
||||||
|
"PyTorch DataLoaders from Lhotse CutSet's -- they control the "
|
||||||
|
"effective batch sizes, sampling strategies, applied data "
|
||||||
|
"augmentations, etc.",
|
||||||
|
)
|
||||||
|
group.add_argument(
|
||||||
|
"--full-libri",
|
||||||
|
type=str2bool,
|
||||||
|
default=False,
|
||||||
|
help="When enabled, use 960h LibriSpeech. Otherwise, use 100h subset.",
|
||||||
|
)
|
||||||
|
group.add_argument(
|
||||||
|
"--manifest-dir",
|
||||||
|
type=Path,
|
||||||
|
default=Path("data/fbank"),
|
||||||
|
help="Path to directory with train/valid/test cuts.",
|
||||||
|
)
|
||||||
|
group.add_argument(
|
||||||
|
"--max-duration",
|
||||||
|
type=int,
|
||||||
|
default=250.0,
|
||||||
|
help="Maximum pooled recordings duration (seconds) in a "
|
||||||
|
"single batch. You can reduce it if it causes CUDA OOM.",
|
||||||
|
)
|
||||||
|
group.add_argument(
|
||||||
|
"--bucketing-sampler",
|
||||||
|
type=str2bool,
|
||||||
|
default=True,
|
||||||
|
help="When enabled, the batches will come from buckets of "
|
||||||
|
"similar duration (saves padding frames).",
|
||||||
|
)
|
||||||
|
group.add_argument(
|
||||||
|
"--num-buckets",
|
||||||
|
type=int,
|
||||||
|
default=30,
|
||||||
|
help="The number of buckets for the DynamicBucketingSampler"
|
||||||
|
"(you might want to increase it for larger datasets).",
|
||||||
|
)
|
||||||
|
group.add_argument(
|
||||||
|
"--concatenate-cuts",
|
||||||
|
type=str2bool,
|
||||||
|
default=False,
|
||||||
|
help="When enabled, utterances (cuts) will be concatenated "
|
||||||
|
"to minimize the amount of padding.",
|
||||||
|
)
|
||||||
|
group.add_argument(
|
||||||
|
"--duration-factor",
|
||||||
|
type=float,
|
||||||
|
default=1.0,
|
||||||
|
help="Determines the maximum duration of a concatenated cut "
|
||||||
|
"relative to the duration of the longest cut in a batch.",
|
||||||
|
)
|
||||||
|
group.add_argument(
|
||||||
|
"--gap",
|
||||||
|
type=float,
|
||||||
|
default=1.0,
|
||||||
|
help="The amount of padding (in seconds) inserted between "
|
||||||
|
"concatenated cuts. This padding is filled with noise when "
|
||||||
|
"noise augmentation is used.",
|
||||||
|
)
|
||||||
|
group.add_argument(
|
||||||
|
"--on-the-fly-feats",
|
||||||
|
type=str2bool,
|
||||||
|
default=False,
|
||||||
|
help="When enabled, use on-the-fly cut mixing and feature "
|
||||||
|
"extraction. Will drop existing precomputed feature manifests "
|
||||||
|
"if available.",
|
||||||
|
)
|
||||||
|
group.add_argument(
|
||||||
|
"--shuffle",
|
||||||
|
type=str2bool,
|
||||||
|
default=True,
|
||||||
|
help="When enabled (=default), the examples will be "
|
||||||
|
"shuffled for each epoch.",
|
||||||
|
)
|
||||||
|
group.add_argument(
|
||||||
|
"--drop-last",
|
||||||
|
type=str2bool,
|
||||||
|
default=True,
|
||||||
|
help="Whether to drop last batch. Used by sampler.",
|
||||||
|
)
|
||||||
|
group.add_argument(
|
||||||
|
"--return-cuts",
|
||||||
|
type=str2bool,
|
||||||
|
default=True,
|
||||||
|
help="When enabled, each batch will have the "
|
||||||
|
"field: batch['supervisions']['cut'] with the cuts that "
|
||||||
|
"were used to construct it.",
|
||||||
|
)
|
||||||
|
|
||||||
|
group.add_argument(
|
||||||
|
"--num-workers",
|
||||||
|
type=int,
|
||||||
|
default=2,
|
||||||
|
help="The number of training dataloader workers that "
|
||||||
|
"collect the batches.",
|
||||||
|
)
|
||||||
|
|
||||||
|
group.add_argument(
|
||||||
|
"--enable-spec-aug",
|
||||||
|
type=str2bool,
|
||||||
|
default=False,
|
||||||
|
help="When enabled, use SpecAugment for training dataset.",
|
||||||
|
)
|
||||||
|
|
||||||
|
group.add_argument(
|
||||||
|
"--spec-aug-time-warp-factor",
|
||||||
|
type=int,
|
||||||
|
default=80,
|
||||||
|
help="Used only when --enable-spec-aug is True. "
|
||||||
|
"It specifies the factor for time warping in SpecAugment. "
|
||||||
|
"Larger values mean more warping. "
|
||||||
|
"A value less than 1 means to disable time warp.",
|
||||||
|
)
|
||||||
|
|
||||||
|
group.add_argument(
|
||||||
|
"--enable-musan",
|
||||||
|
type=str2bool,
|
||||||
|
default=True,
|
||||||
|
help="When enabled, select noise from MUSAN and mix it"
|
||||||
|
"with training dataset. ",
|
||||||
|
)
|
||||||
|
|
||||||
|
group.add_argument(
|
||||||
|
"--input-strategy",
|
||||||
|
type=str,
|
||||||
|
default="AudioSamples",
|
||||||
|
help="AudioSamples or PrecomputedFeatures",
|
||||||
|
)
|
||||||
|
|
||||||
|
group.add_argument(
|
||||||
|
"--spk-id",
|
||||||
|
type=int,
|
||||||
|
default=0,
|
||||||
|
)
|
||||||
|
|
||||||
|
group.add_argument(
|
||||||
|
"--prefix",
|
||||||
|
type=str,
|
||||||
|
default='vox',
|
||||||
|
)
|
||||||
|
|
||||||
|
def train_dataloaders(
|
||||||
|
self,
|
||||||
|
cuts_train: CutSet,
|
||||||
|
sampler_state_dict: Optional[Dict[str, Any]] = None,
|
||||||
|
) -> DataLoader:
|
||||||
|
"""
|
||||||
|
Args:
|
||||||
|
cuts_train:
|
||||||
|
CutSet for training.
|
||||||
|
sampler_state_dict:
|
||||||
|
The state dict for the training sampler.
|
||||||
|
"""
|
||||||
|
transforms = []
|
||||||
|
if self.args.enable_musan:
|
||||||
|
logging.info("Enable MUSAN")
|
||||||
|
logging.info("About to get Musan cuts")
|
||||||
|
cuts_musan = load_manifest(self.args.manifest_dir / "musan_cuts.jsonl.gz")
|
||||||
|
transforms.append(
|
||||||
|
CutMix(cuts=cuts_musan, prob=0.5, snr=(10, 20), preserve_id=True)
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
logging.info("Disable MUSAN")
|
||||||
|
|
||||||
|
if self.args.concatenate_cuts:
|
||||||
|
logging.info(
|
||||||
|
f"Using cut concatenation with duration factor "
|
||||||
|
f"{self.args.duration_factor} and gap {self.args.gap}."
|
||||||
|
)
|
||||||
|
# Cut concatenation should be the first transform in the list,
|
||||||
|
# so that if we e.g. mix noise in, it will fill the gaps between
|
||||||
|
# different utterances.
|
||||||
|
transforms = [
|
||||||
|
CutConcatenate(
|
||||||
|
duration_factor=self.args.duration_factor, gap=self.args.gap
|
||||||
|
)
|
||||||
|
] + transforms
|
||||||
|
|
||||||
|
input_transforms = []
|
||||||
|
if self.args.enable_spec_aug:
|
||||||
|
logging.info("Enable SpecAugment")
|
||||||
|
logging.info(f"Time warp factor: {self.args.spec_aug_time_warp_factor}")
|
||||||
|
# Set the value of num_frame_masks according to Lhotse's version.
|
||||||
|
# In different Lhotse's versions, the default of num_frame_masks is
|
||||||
|
# different.
|
||||||
|
num_frame_masks = 10
|
||||||
|
num_frame_masks_parameter = inspect.signature(
|
||||||
|
SpecAugment.__init__
|
||||||
|
).parameters["num_frame_masks"]
|
||||||
|
if num_frame_masks_parameter.default == 1:
|
||||||
|
num_frame_masks = 2
|
||||||
|
logging.info(f"Num frame mask: {num_frame_masks}")
|
||||||
|
input_transforms.append(
|
||||||
|
SpecAugment(
|
||||||
|
time_warp_factor=self.args.spec_aug_time_warp_factor,
|
||||||
|
num_frame_masks=num_frame_masks,
|
||||||
|
features_mask_size=27,
|
||||||
|
num_feature_masks=2,
|
||||||
|
frames_mask_size=100,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
logging.info("Disable SpecAugment")
|
||||||
|
|
||||||
|
logging.info("About to create train dataset")
|
||||||
|
train = K2SpeechRecognitionDataset(
|
||||||
|
input_strategy=eval(self.args.input_strategy)(),
|
||||||
|
cut_transforms=transforms,
|
||||||
|
input_transforms=input_transforms,
|
||||||
|
return_cuts=self.args.return_cuts,
|
||||||
|
)
|
||||||
|
|
||||||
|
if self.args.on_the_fly_feats:
|
||||||
|
# NOTE: the PerturbSpeed transform should be added only if we
|
||||||
|
# remove it from data prep stage.
|
||||||
|
# Add on-the-fly speed perturbation; since originally it would
|
||||||
|
# have increased epoch size by 3, we will apply prob 2/3 and use
|
||||||
|
# 3x more epochs.
|
||||||
|
# Speed perturbation probably should come first before
|
||||||
|
# concatenation, but in principle the transforms order doesn't have
|
||||||
|
# to be strict (e.g. could be randomized)
|
||||||
|
# transforms = [PerturbSpeed(factors=[0.9, 1.1], p=2/3)] + transforms # noqa
|
||||||
|
# Drop feats to be on the safe side.
|
||||||
|
train = K2SpeechRecognitionDataset(
|
||||||
|
cut_transforms=transforms,
|
||||||
|
input_strategy=OnTheFlyFeatures(Fbank(FbankConfig(num_mel_bins=80))),
|
||||||
|
input_transforms=input_transforms,
|
||||||
|
return_cuts=self.args.return_cuts,
|
||||||
|
)
|
||||||
|
|
||||||
|
if self.args.bucketing_sampler:
|
||||||
|
logging.info("Using DynamicBucketingSampler.")
|
||||||
|
train_sampler = DynamicBucketingSampler(
|
||||||
|
cuts_train,
|
||||||
|
max_duration=self.args.max_duration,
|
||||||
|
shuffle=self.args.shuffle,
|
||||||
|
num_buckets=self.args.num_buckets,
|
||||||
|
drop_last=self.args.drop_last,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
logging.info("Using SingleCutSampler.")
|
||||||
|
train_sampler = SingleCutSampler(
|
||||||
|
cuts_train,
|
||||||
|
max_duration=self.args.max_duration,
|
||||||
|
shuffle=self.args.shuffle,
|
||||||
|
)
|
||||||
|
logging.info("About to create train dataloader")
|
||||||
|
|
||||||
|
if sampler_state_dict is not None:
|
||||||
|
logging.info("Loading sampler state dict")
|
||||||
|
train_sampler.load_state_dict(sampler_state_dict)
|
||||||
|
|
||||||
|
# 'seed' is derived from the current random state, which will have
|
||||||
|
# previously been set in the main process.
|
||||||
|
seed = torch.randint(0, 100000, ()).item()
|
||||||
|
worker_init_fn = _SeedWorkers(seed)
|
||||||
|
|
||||||
|
train_dl = DataLoader(
|
||||||
|
train,
|
||||||
|
sampler=train_sampler,
|
||||||
|
batch_size=None,
|
||||||
|
num_workers=self.args.num_workers,
|
||||||
|
persistent_workers=False,
|
||||||
|
worker_init_fn=worker_init_fn,
|
||||||
|
)
|
||||||
|
|
||||||
|
return train_dl
|
||||||
|
|
||||||
|
def valid_dataloaders(self, cuts_valid: CutSet) -> DataLoader:
|
||||||
|
transforms = []
|
||||||
|
if self.args.concatenate_cuts:
|
||||||
|
transforms = [
|
||||||
|
CutConcatenate(
|
||||||
|
duration_factor=self.args.duration_factor, gap=self.args.gap
|
||||||
|
)
|
||||||
|
] + transforms
|
||||||
|
|
||||||
|
logging.info("About to create dev dataset")
|
||||||
|
if self.args.on_the_fly_feats:
|
||||||
|
validate = K2SpeechRecognitionDataset(
|
||||||
|
cut_transforms=transforms,
|
||||||
|
input_strategy=eval(self.args.input_strategy)(),
|
||||||
|
#input_strategy=OnTheFlyFeatures(Fbank(FbankConfig(num_mel_bins=80))),
|
||||||
|
return_cuts=self.args.return_cuts,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
validate = K2SpeechRecognitionDataset(
|
||||||
|
cut_transforms=transforms,
|
||||||
|
input_strategy=eval(self.args.input_strategy)(),
|
||||||
|
return_cuts=self.args.return_cuts,
|
||||||
|
)
|
||||||
|
valid_sampler = DynamicBucketingSampler(
|
||||||
|
cuts_valid,
|
||||||
|
max_duration=self.args.max_duration,
|
||||||
|
shuffle=False,
|
||||||
|
)
|
||||||
|
logging.info("About to create dev dataloader")
|
||||||
|
valid_dl = DataLoader(
|
||||||
|
validate,
|
||||||
|
sampler=valid_sampler,
|
||||||
|
batch_size=None,
|
||||||
|
num_workers=2,
|
||||||
|
persistent_workers=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
return valid_dl
|
||||||
|
|
||||||
|
def test_dataloaders(self, cuts: CutSet) -> DataLoader:
|
||||||
|
logging.debug("About to create test dataset")
|
||||||
|
test = K2SpeechRecognitionDataset(
|
||||||
|
input_strategy=OnTheFlyFeatures(Fbank(FbankConfig(num_mel_bins=80)))
|
||||||
|
if self.args.on_the_fly_feats
|
||||||
|
else eval(self.args.input_strategy)(),
|
||||||
|
return_cuts=self.args.return_cuts,
|
||||||
|
)
|
||||||
|
sampler = DynamicBucketingSampler(
|
||||||
|
cuts,
|
||||||
|
max_duration=self.args.max_duration,
|
||||||
|
shuffle=False,
|
||||||
|
)
|
||||||
|
logging.debug("About to create test dataloader")
|
||||||
|
test_dl = DataLoader(
|
||||||
|
test,
|
||||||
|
batch_size=None,
|
||||||
|
sampler=sampler,
|
||||||
|
num_workers=self.args.num_workers,
|
||||||
|
)
|
||||||
|
return test_dl
|
||||||
|
|
||||||
|
@lru_cache()
|
||||||
|
def train_clean_10_cuts(self, option=None) -> CutSet:
|
||||||
|
logging.info("About to get train-clean-10 cuts")
|
||||||
|
if option is None:
|
||||||
|
return load_manifest_lazy(
|
||||||
|
self.args.manifest_dir / f"librispeech_cuts_train-clean-100.jsonl"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
return load_manifest_lazy(
|
||||||
|
self.args.manifest_dir / f"librispeech_cuts_train-clean-10_{option}.jsonl"
|
||||||
|
)
|
||||||
|
|
||||||
|
@lru_cache()
|
||||||
|
def train_clean_100_cuts(self, option=None) -> CutSet:
|
||||||
|
logging.info("About to get train-clean-100 cuts")
|
||||||
|
if option is None:
|
||||||
|
return load_manifest_lazy(
|
||||||
|
self.args.manifest_dir / f"librispeech_cuts_train-clean-100.jsonl"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
return load_manifest_lazy(
|
||||||
|
self.args.manifest_dir / f"librispeech_cuts_train-clean-100_{option}.jsonl"
|
||||||
|
)
|
||||||
|
|
||||||
|
@lru_cache()
|
||||||
|
def train_clean_360_cuts(self, option=None) -> CutSet:
|
||||||
|
logging.info("About to get train-clean-360 cuts")
|
||||||
|
if option is None:
|
||||||
|
return load_manifest_lazy(
|
||||||
|
self.args.manifest_dir / f"librispeech_cuts_train-clean-360.jsonl"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
return load_manifest_lazy(
|
||||||
|
self.args.manifest_dir / f"librispeech_cuts_train-clean-360_{option}.jsonl"
|
||||||
|
)
|
||||||
|
|
||||||
|
@lru_cache()
|
||||||
|
def train_other_500_cuts(self, option=None) -> CutSet:
|
||||||
|
logging.info("About to get train-other-500 cuts")
|
||||||
|
if option is None:
|
||||||
|
return load_manifest_lazy(
|
||||||
|
self.args.manifest_dir / f"librispeech_cuts_train-other-500.jsonl"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
return load_manifest_lazy(
|
||||||
|
self.args.manifest_dir / f"librispeech_cuts_train-other-500_{option}.jsonl"
|
||||||
|
)
|
||||||
|
|
||||||
|
@lru_cache()
|
||||||
|
def train_all_shuf_cuts(self, option=None) -> CutSet:
|
||||||
|
logging.info(
|
||||||
|
"About to get the shuffled train-clean-100, \
|
||||||
|
train-clean-360 and train-other-500 cuts"
|
||||||
|
)
|
||||||
|
if option is None:
|
||||||
|
return load_manifest_lazy(
|
||||||
|
self.args.manifest_dir / f"librispeech_cuts_train-all-shuf.jsonl"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
return load_manifest_lazy(
|
||||||
|
self.args.manifest_dir / f"librispeech_cuts_train-all-shuf_{option}.jsonl"
|
||||||
|
)
|
||||||
|
|
||||||
|
@lru_cache()
|
||||||
|
def dev_clean_cuts(self, option=None) -> CutSet:
|
||||||
|
logging.info("About to get dev-clean cuts")
|
||||||
|
if option is None:
|
||||||
|
return load_manifest_lazy(
|
||||||
|
self.args.manifest_dir / f"librispeech_cuts_dev-clean.jsonl"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
return load_manifest_lazy(
|
||||||
|
self.args.manifest_dir / f"librispeech_cuts_dev-clean_{option}.jsonl"
|
||||||
|
)
|
||||||
|
|
||||||
|
@lru_cache()
|
||||||
|
def dev_other_cuts(self, option=None) -> CutSet:
|
||||||
|
logging.info("About to get dev-other cuts")
|
||||||
|
if option is None:
|
||||||
|
return load_manifest_lazy(
|
||||||
|
self.args.manifest_dir / f"librispeech_cuts_dev-other.jsonl"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
return load_manifest_lazy(
|
||||||
|
self.args.manifest_dir / f"librispeech_cuts_dev-other_{option}.jsonl"
|
||||||
|
)
|
||||||
|
|
||||||
|
@lru_cache()
|
||||||
|
def test_clean_cuts(self, option=None) -> CutSet:
|
||||||
|
logging.info("About to get test-clean cuts")
|
||||||
|
if option is None:
|
||||||
|
return load_manifest_lazy(
|
||||||
|
self.args.manifest_dir / f"librispeech_cuts_test-clean.jsonl"
|
||||||
|
)
|
||||||
|
elif option == 'user':
|
||||||
|
json_list = sorted(glob(str(self.args.manifest_dir) + "/userlibri/test-clean/*"))
|
||||||
|
spk_list = [json.split('/')[-1][:-6] for json in json_list]
|
||||||
|
|
||||||
|
return [load_manifest_lazy(json) for json in json_list], spk_list
|
||||||
|
else:
|
||||||
|
return load_manifest_lazy(
|
||||||
|
self.args.manifest_dir / f"librispeech_cuts_test-clean_{option}.jsonl"
|
||||||
|
)
|
||||||
|
|
||||||
|
@lru_cache()
|
||||||
|
def test_other_cuts(self, option=None) -> CutSet:
|
||||||
|
logging.info("About to get test-other cuts")
|
||||||
|
if option is None:
|
||||||
|
return load_manifest_lazy(
|
||||||
|
self.args.manifest_dir / f"librispeech_cuts_test-other_{option}.jsonl"
|
||||||
|
)
|
||||||
|
elif option == 'user':
|
||||||
|
json_list = sorted(glob(str(self.args.manifest_dir) + "/userlibri/test-other/*"))
|
||||||
|
spk_list = [json.split('/')[-1][:-6] for json in json_list]
|
||||||
|
|
||||||
|
return [load_manifest_lazy(json) for json in json_list], spk_list
|
||||||
|
else:
|
||||||
|
return load_manifest_lazy(
|
||||||
|
self.args.manifest_dir / f"librispeech_cuts_test-other_{option}.jsonl"
|
||||||
|
)
|
||||||
|
|
||||||
|
@lru_cache()
|
||||||
|
def test_clean_user(self, option=None) -> CutSet:
|
||||||
|
logging.info("About to get test-clean user cuts")
|
||||||
|
return load_manifest_lazy(
|
||||||
|
self.args.manifest_dir / f"userlibri/test-clean_sampling/{option}.jsonl"
|
||||||
|
)
|
||||||
|
|
||||||
|
@lru_cache()
|
||||||
|
def test_other_user(self, option=None) -> CutSet:
|
||||||
|
logging.info("About to get test-other user cuts")
|
||||||
|
return load_manifest_lazy(
|
||||||
|
self.args.manifest_dir / f"userlibri/test-other_sampling/{option}.jsonl"
|
||||||
|
)
|
||||||
|
|
||||||
|
@lru_cache()
|
||||||
|
def vox_cuts(self, option=None) -> CutSet:
|
||||||
|
logging.info("About to get test-other user cuts")
|
||||||
|
return load_manifest_lazy(
|
||||||
|
self.args.manifest_dir / f"{self.args.prefix}_cuts_{option}.jsonl.gz"
|
||||||
|
)
|
||||||
|
|
||||||
|
@lru_cache()
|
||||||
|
def userlibri_cuts(self, option=None) -> CutSet:
|
||||||
|
logging.info("About to get userlibri cuts")
|
||||||
|
return load_manifest_lazy(
|
||||||
|
self.args.manifest_dir / f"{option}.jsonl"
|
||||||
|
)
|
||||||
|
|
||||||
@ -1,11 +0,0 @@
|
|||||||
import torch
|
|
||||||
|
|
||||||
base_model = torch.load('./d2v-base-T.pt')
|
|
||||||
bias_model = torch.load('./bitfit_533_v2/checkpoint-100.pt')
|
|
||||||
|
|
||||||
base_model, bias_model = base_model['model'], bias_model['model']
|
|
||||||
|
|
||||||
for key in base_model.keys():
|
|
||||||
if 'bias' in key:
|
|
||||||
l1_diff = torch.abs(base_model[key]-bias_model[key]).sum() / base_model[key].size(0)
|
|
||||||
print(key, l1_diff.item())
|
|
||||||
@ -1,834 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
#
|
|
||||||
# Copyright 2021-2022 Xiaomi Corporation (Author: Fangjun Kuang,
|
|
||||||
# Zengwei Yao)
|
|
||||||
#
|
|
||||||
# See ../../../../LICENSE for clarification regarding multiple authors
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
# you may not use this file except in compliance with the License.
|
|
||||||
# You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
"""
|
|
||||||
Usage:
|
|
||||||
(0) for d2v-T decoding
|
|
||||||
for method in greedy_search modified_beam_search fast_beam_search; do
|
|
||||||
./pruned_transducer_stateless_d2v_v2/decode.py \
|
|
||||||
--input-strategy AudioSamples \
|
|
||||||
--enable-spec-aug False \
|
|
||||||
--additional-block True \
|
|
||||||
--model-name epoc.pt \
|
|
||||||
--exp-dir ./pruned_transducer_stateless_d2v_v2/960h_sweep_v3_388 \
|
|
||||||
--max-duration 400 \
|
|
||||||
--decoding-method $method \
|
|
||||||
--max-sym-per-frame 1 \
|
|
||||||
--encoder-type d2v \
|
|
||||||
--encoder-dim 768 \
|
|
||||||
--decoder-dim 768 \
|
|
||||||
--joiner-dim 768
|
|
||||||
done
|
|
||||||
"""
|
|
||||||
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import logging
|
|
||||||
import math
|
|
||||||
from collections import defaultdict
|
|
||||||
from pathlib import Path
|
|
||||||
from typing import Dict, List, Optional, Tuple
|
|
||||||
|
|
||||||
import k2
|
|
||||||
import sentencepiece as spm
|
|
||||||
import torch
|
|
||||||
import torch.nn as nn
|
|
||||||
from asr_datamodule import LibriSpeechAsrDataModule
|
|
||||||
from beam_search import (
|
|
||||||
beam_search,
|
|
||||||
fast_beam_search_nbest,
|
|
||||||
fast_beam_search_nbest_LG,
|
|
||||||
fast_beam_search_nbest_oracle,
|
|
||||||
fast_beam_search_one_best,
|
|
||||||
greedy_search,
|
|
||||||
greedy_search_batch,
|
|
||||||
modified_beam_search,
|
|
||||||
)
|
|
||||||
#from train import add_model_arguments, add_rep_arguments, get_params, get_transducer_model
|
|
||||||
from prompt_tuning import add_model_arguments, add_rep_arguments, get_params, get_transducer_model
|
|
||||||
|
|
||||||
from icefall.checkpoint import (
|
|
||||||
average_checkpoints,
|
|
||||||
average_checkpoints_with_averaged_model,
|
|
||||||
find_checkpoints,
|
|
||||||
load_checkpoint,
|
|
||||||
)
|
|
||||||
from icefall.lexicon import Lexicon
|
|
||||||
from icefall.utils import (
|
|
||||||
AttributeDict,
|
|
||||||
setup_logger,
|
|
||||||
store_transcripts,
|
|
||||||
str2bool,
|
|
||||||
write_error_stats,
|
|
||||||
)
|
|
||||||
|
|
||||||
from train_lora import LoRAHook
|
|
||||||
|
|
||||||
LOG_EPS = math.log(1e-10)
|
|
||||||
|
|
||||||
|
|
||||||
def get_parser():
|
|
||||||
parser = argparse.ArgumentParser(
|
|
||||||
formatter_class=argparse.ArgumentDefaultsHelpFormatter
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--model-name",
|
|
||||||
type=str,
|
|
||||||
default="",
|
|
||||||
help="""It specifies the model file name to use for decoding.""",
|
|
||||||
)
|
|
||||||
|
|
||||||
parser.add_argument(
|
|
||||||
"--epoch",
|
|
||||||
type=int,
|
|
||||||
default=30,
|
|
||||||
help="""It specifies the checkpoint to use for decoding.
|
|
||||||
Note: Epoch counts from 1.
|
|
||||||
You can specify --avg to use more checkpoints for model averaging.""",
|
|
||||||
)
|
|
||||||
|
|
||||||
parser.add_argument(
|
|
||||||
"--iter",
|
|
||||||
type=int,
|
|
||||||
default=0,
|
|
||||||
help="""If positive, --epoch is ignored and it
|
|
||||||
will use the checkpoint exp_dir/checkpoint-iter.pt.
|
|
||||||
You can specify --avg to use more checkpoints for model averaging.
|
|
||||||
""",
|
|
||||||
)
|
|
||||||
|
|
||||||
parser.add_argument(
|
|
||||||
"--avg",
|
|
||||||
type=int,
|
|
||||||
default=9,
|
|
||||||
help="Number of checkpoints to average. Automatically select "
|
|
||||||
"consecutive checkpoints before the checkpoint specified by "
|
|
||||||
"'--epoch' and '--iter'",
|
|
||||||
)
|
|
||||||
|
|
||||||
parser.add_argument(
|
|
||||||
"--use-averaged-model",
|
|
||||||
type=str2bool,
|
|
||||||
default=True,
|
|
||||||
help="Whether to load averaged model. Currently it only supports "
|
|
||||||
"using --epoch. If True, it would decode with the averaged model "
|
|
||||||
"over the epoch range from `epoch-avg` (excluded) to `epoch`."
|
|
||||||
"Actually only the models with epoch number of `epoch-avg` and "
|
|
||||||
"`epoch` are loaded for averaging. ",
|
|
||||||
)
|
|
||||||
|
|
||||||
parser.add_argument(
|
|
||||||
"--exp-dir",
|
|
||||||
type=str,
|
|
||||||
default="pruned_transducer_stateless7_ctc/exp",
|
|
||||||
help="The experiment dir",
|
|
||||||
)
|
|
||||||
|
|
||||||
parser.add_argument(
|
|
||||||
"--bpe-model",
|
|
||||||
type=str,
|
|
||||||
default="data/lang_bpe_500/bpe.model",
|
|
||||||
help="Path to the BPE model",
|
|
||||||
)
|
|
||||||
|
|
||||||
parser.add_argument(
|
|
||||||
"--lang-dir",
|
|
||||||
type=Path,
|
|
||||||
default="data/lang_bpe_500",
|
|
||||||
help="The lang dir containing word table and LG graph",
|
|
||||||
)
|
|
||||||
|
|
||||||
parser.add_argument(
|
|
||||||
"--decoding-method",
|
|
||||||
type=str,
|
|
||||||
default="greedy_search",
|
|
||||||
help="""Possible values are:
|
|
||||||
- greedy_search
|
|
||||||
- beam_search
|
|
||||||
- modified_beam_search
|
|
||||||
- fast_beam_search
|
|
||||||
- fast_beam_search_nbest
|
|
||||||
- fast_beam_search_nbest_oracle
|
|
||||||
- fast_beam_search_nbest_LG
|
|
||||||
If you use fast_beam_search_nbest_LG, you have to specify
|
|
||||||
`--lang-dir`, which should contain `LG.pt`.
|
|
||||||
""",
|
|
||||||
)
|
|
||||||
|
|
||||||
parser.add_argument(
|
|
||||||
"--beam-size",
|
|
||||||
type=int,
|
|
||||||
default=4,
|
|
||||||
help="""An integer indicating how many candidates we will keep for each
|
|
||||||
frame. Used only when --decoding-method is beam_search or
|
|
||||||
modified_beam_search.""",
|
|
||||||
)
|
|
||||||
|
|
||||||
parser.add_argument(
|
|
||||||
"--beam",
|
|
||||||
type=float,
|
|
||||||
default=20.0,
|
|
||||||
help="""A floating point value to calculate the cutoff score during beam
|
|
||||||
search (i.e., `cutoff = max-score - beam`), which is the same as the
|
|
||||||
`beam` in Kaldi.
|
|
||||||
Used only when --decoding-method is fast_beam_search,
|
|
||||||
fast_beam_search_nbest, fast_beam_search_nbest_LG,
|
|
||||||
and fast_beam_search_nbest_oracle
|
|
||||||
""",
|
|
||||||
)
|
|
||||||
|
|
||||||
parser.add_argument(
|
|
||||||
"--ngram-lm-scale",
|
|
||||||
type=float,
|
|
||||||
default=0.01,
|
|
||||||
help="""
|
|
||||||
Used only when --decoding_method is fast_beam_search_nbest_LG.
|
|
||||||
It specifies the scale for n-gram LM scores.
|
|
||||||
""",
|
|
||||||
)
|
|
||||||
|
|
||||||
parser.add_argument(
|
|
||||||
"--max-contexts",
|
|
||||||
type=int,
|
|
||||||
default=8,
|
|
||||||
help="""Used only when --decoding-method is
|
|
||||||
fast_beam_search, fast_beam_search_nbest, fast_beam_search_nbest_LG,
|
|
||||||
and fast_beam_search_nbest_oracle""",
|
|
||||||
)
|
|
||||||
|
|
||||||
parser.add_argument(
|
|
||||||
"--max-states",
|
|
||||||
type=int,
|
|
||||||
default=64,
|
|
||||||
help="""Used only when --decoding-method is
|
|
||||||
fast_beam_search, fast_beam_search_nbest, fast_beam_search_nbest_LG,
|
|
||||||
and fast_beam_search_nbest_oracle""",
|
|
||||||
)
|
|
||||||
|
|
||||||
parser.add_argument(
|
|
||||||
"--context-size",
|
|
||||||
type=int,
|
|
||||||
default=2,
|
|
||||||
help="The context size in the decoder. 1 means bigram; 2 means tri-gram",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--max-sym-per-frame",
|
|
||||||
type=int,
|
|
||||||
default=1,
|
|
||||||
help="""Maximum number of symbols per frame.
|
|
||||||
Used only when --decoding_method is greedy_search""",
|
|
||||||
)
|
|
||||||
|
|
||||||
parser.add_argument(
|
|
||||||
"--num-paths",
|
|
||||||
type=int,
|
|
||||||
default=200,
|
|
||||||
help="""Number of paths for nbest decoding.
|
|
||||||
Used only when the decoding method is fast_beam_search_nbest,
|
|
||||||
fast_beam_search_nbest_LG, and fast_beam_search_nbest_oracle""",
|
|
||||||
)
|
|
||||||
|
|
||||||
parser.add_argument(
|
|
||||||
"--nbest-scale",
|
|
||||||
type=float,
|
|
||||||
default=0.5,
|
|
||||||
help="""Scale applied to lattice scores when computing nbest paths.
|
|
||||||
Used only when the decoding method is fast_beam_search_nbest,
|
|
||||||
fast_beam_search_nbest_LG, and fast_beam_search_nbest_oracle""",
|
|
||||||
)
|
|
||||||
|
|
||||||
parser.add_argument(
|
|
||||||
"--simulate-streaming",
|
|
||||||
type=str2bool,
|
|
||||||
default=False,
|
|
||||||
help="""Whether to simulate streaming in decoding, this is a good way to
|
|
||||||
test a streaming model.
|
|
||||||
""",
|
|
||||||
)
|
|
||||||
|
|
||||||
parser.add_argument(
|
|
||||||
"--decode-chunk-size",
|
|
||||||
type=int,
|
|
||||||
default=16,
|
|
||||||
help="The chunk size for decoding (in frames after subsampling)",
|
|
||||||
)
|
|
||||||
|
|
||||||
parser.add_argument(
|
|
||||||
"--left-context",
|
|
||||||
type=int,
|
|
||||||
default=64,
|
|
||||||
help="left context can be seen during decoding (in frames after subsampling)",
|
|
||||||
)
|
|
||||||
|
|
||||||
parser.add_argument(
|
|
||||||
"--res-name",
|
|
||||||
type=str,
|
|
||||||
)
|
|
||||||
|
|
||||||
add_model_arguments(parser)
|
|
||||||
add_rep_arguments(parser)
|
|
||||||
|
|
||||||
return parser
|
|
||||||
|
|
||||||
|
|
||||||
def decode_one_batch(
|
|
||||||
params: AttributeDict,
|
|
||||||
model: nn.Module,
|
|
||||||
sp: spm.SentencePieceProcessor,
|
|
||||||
batch: dict,
|
|
||||||
word_table: Optional[k2.SymbolTable] = None,
|
|
||||||
decoding_graph: Optional[k2.Fsa] = None,
|
|
||||||
) -> Dict[str, List[List[str]]]:
|
|
||||||
"""Decode one batch and return the result in a dict. The dict has the
|
|
||||||
following format:
|
|
||||||
|
|
||||||
- key: It indicates the setting used for decoding. For example,
|
|
||||||
if greedy_search is used, it would be "greedy_search"
|
|
||||||
If beam search with a beam size of 7 is used, it would be
|
|
||||||
"beam_7"
|
|
||||||
- value: It contains the decoding result. `len(value)` equals to
|
|
||||||
batch size. `value[i]` is the decoding result for the i-th
|
|
||||||
utterance in the given batch.
|
|
||||||
Args:
|
|
||||||
params:
|
|
||||||
It's the return value of :func:`get_params`.
|
|
||||||
model:
|
|
||||||
The neural model.
|
|
||||||
sp:
|
|
||||||
The BPE model.
|
|
||||||
batch:
|
|
||||||
It is the return value from iterating
|
|
||||||
`lhotse.dataset.K2SpeechRecognitionDataset`. See its documentation
|
|
||||||
for the format of the `batch`.
|
|
||||||
word_table:
|
|
||||||
The word symbol table.
|
|
||||||
decoding_graph:
|
|
||||||
The decoding graph. Can be either a `k2.trivial_graph` or HLG, Used
|
|
||||||
only when --decoding_method is fast_beam_search, fast_beam_search_nbest,
|
|
||||||
fast_beam_search_nbest_oracle, and fast_beam_search_nbest_LG.
|
|
||||||
Returns:
|
|
||||||
Return the decoding result. See above description for the format of
|
|
||||||
the returned dict.
|
|
||||||
"""
|
|
||||||
device = next(model.parameters()).device
|
|
||||||
feature = batch["inputs"]
|
|
||||||
assert feature.ndim == 2 or feature.ndim == 3
|
|
||||||
|
|
||||||
feature = feature.to(device)
|
|
||||||
# at entry, feature is (N, T, C)
|
|
||||||
|
|
||||||
supervisions = batch["supervisions"]
|
|
||||||
#feature_lens = supervisions["num_frames"].to(device)
|
|
||||||
if feature.ndim == 2:
|
|
||||||
feature_lens = []
|
|
||||||
for supervision in supervisions['cut']:
|
|
||||||
try: feature_lens.append(supervision.tracks[0].cut.recording.num_samples)
|
|
||||||
except: feature_lens.append(supervision.recording.num_samples)
|
|
||||||
feature_lens = torch.tensor(feature_lens)
|
|
||||||
|
|
||||||
elif feature.ndim == 3:
|
|
||||||
feature_lens = supervisions["num_frames"].to(device)
|
|
||||||
|
|
||||||
if params.simulate_streaming:
|
|
||||||
feature_lens += params.left_context
|
|
||||||
feature = torch.nn.functional.pad(
|
|
||||||
feature,
|
|
||||||
pad=(0, 0, 0, params.left_context),
|
|
||||||
value=LOG_EPS,
|
|
||||||
)
|
|
||||||
encoder_out, encoder_out_lens, _ = model.encoder.streaming_forward(
|
|
||||||
x=feature,
|
|
||||||
x_lens=feature_lens,
|
|
||||||
chunk_size=params.decode_chunk_size,
|
|
||||||
left_context=params.left_context,
|
|
||||||
simulate_streaming=True,
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
encoder_out, encoder_out_lens = model.encoder(x=feature, x_lens=feature_lens)
|
|
||||||
|
|
||||||
hyps = []
|
|
||||||
|
|
||||||
if params.decoding_method == "fast_beam_search":
|
|
||||||
hyp_tokens = fast_beam_search_one_best(
|
|
||||||
model=model,
|
|
||||||
decoding_graph=decoding_graph,
|
|
||||||
encoder_out=encoder_out,
|
|
||||||
encoder_out_lens=encoder_out_lens,
|
|
||||||
beam=params.beam,
|
|
||||||
max_contexts=params.max_contexts,
|
|
||||||
max_states=params.max_states,
|
|
||||||
)
|
|
||||||
for hyp in sp.decode(hyp_tokens):
|
|
||||||
hyps.append(hyp.split())
|
|
||||||
elif params.decoding_method == "fast_beam_search_nbest_LG":
|
|
||||||
hyp_tokens = fast_beam_search_nbest_LG(
|
|
||||||
model=model,
|
|
||||||
decoding_graph=decoding_graph,
|
|
||||||
encoder_out=encoder_out,
|
|
||||||
encoder_out_lens=encoder_out_lens,
|
|
||||||
beam=params.beam,
|
|
||||||
max_contexts=params.max_contexts,
|
|
||||||
max_states=params.max_states,
|
|
||||||
num_paths=params.num_paths,
|
|
||||||
nbest_scale=params.nbest_scale,
|
|
||||||
)
|
|
||||||
for hyp in hyp_tokens:
|
|
||||||
hyps.append([word_table[i] for i in hyp])
|
|
||||||
elif params.decoding_method == "fast_beam_search_nbest":
|
|
||||||
hyp_tokens = fast_beam_search_nbest(
|
|
||||||
model=model,
|
|
||||||
decoding_graph=decoding_graph,
|
|
||||||
encoder_out=encoder_out,
|
|
||||||
encoder_out_lens=encoder_out_lens,
|
|
||||||
beam=params.beam,
|
|
||||||
max_contexts=params.max_contexts,
|
|
||||||
max_states=params.max_states,
|
|
||||||
num_paths=params.num_paths,
|
|
||||||
nbest_scale=params.nbest_scale,
|
|
||||||
)
|
|
||||||
for hyp in sp.decode(hyp_tokens):
|
|
||||||
hyps.append(hyp.split())
|
|
||||||
elif params.decoding_method == "fast_beam_search_nbest_oracle":
|
|
||||||
hyp_tokens = fast_beam_search_nbest_oracle(
|
|
||||||
model=model,
|
|
||||||
decoding_graph=decoding_graph,
|
|
||||||
encoder_out=encoder_out,
|
|
||||||
encoder_out_lens=encoder_out_lens,
|
|
||||||
beam=params.beam,
|
|
||||||
max_contexts=params.max_contexts,
|
|
||||||
max_states=params.max_states,
|
|
||||||
num_paths=params.num_paths,
|
|
||||||
ref_texts=sp.encode(supervisions["text"]),
|
|
||||||
nbest_scale=params.nbest_scale,
|
|
||||||
)
|
|
||||||
for hyp in sp.decode(hyp_tokens):
|
|
||||||
hyps.append(hyp.split())
|
|
||||||
elif params.decoding_method == "greedy_search" and params.max_sym_per_frame == 1:
|
|
||||||
hyp_tokens = greedy_search_batch(
|
|
||||||
model=model,
|
|
||||||
encoder_out=encoder_out,
|
|
||||||
encoder_out_lens=encoder_out_lens,
|
|
||||||
)
|
|
||||||
for hyp in sp.decode(hyp_tokens):
|
|
||||||
hyps.append(hyp.split())
|
|
||||||
elif params.decoding_method == "modified_beam_search":
|
|
||||||
hyp_tokens = modified_beam_search(
|
|
||||||
model=model,
|
|
||||||
encoder_out=encoder_out,
|
|
||||||
encoder_out_lens=encoder_out_lens,
|
|
||||||
beam=params.beam_size,
|
|
||||||
)
|
|
||||||
for hyp in sp.decode(hyp_tokens):
|
|
||||||
hyps.append(hyp.split())
|
|
||||||
else:
|
|
||||||
batch_size = encoder_out.size(0)
|
|
||||||
|
|
||||||
for i in range(batch_size):
|
|
||||||
# fmt: off
|
|
||||||
encoder_out_i = encoder_out[i:i+1, :encoder_out_lens[i]]
|
|
||||||
# fmt: on
|
|
||||||
if params.decoding_method == "greedy_search":
|
|
||||||
hyp = greedy_search(
|
|
||||||
model=model,
|
|
||||||
encoder_out=encoder_out_i,
|
|
||||||
max_sym_per_frame=params.max_sym_per_frame,
|
|
||||||
)
|
|
||||||
elif params.decoding_method == "beam_search":
|
|
||||||
hyp = beam_search(
|
|
||||||
model=model,
|
|
||||||
encoder_out=encoder_out_i,
|
|
||||||
beam=params.beam_size,
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
raise ValueError(
|
|
||||||
f"Unsupported decoding method: {params.decoding_method}"
|
|
||||||
)
|
|
||||||
hyps.append(sp.decode(hyp).split())
|
|
||||||
|
|
||||||
if params.decoding_method == "greedy_search":
|
|
||||||
return {"greedy_search": hyps}
|
|
||||||
elif "fast_beam_search" in params.decoding_method:
|
|
||||||
key = f"beam_{params.beam}_"
|
|
||||||
key += f"max_contexts_{params.max_contexts}_"
|
|
||||||
key += f"max_states_{params.max_states}"
|
|
||||||
if "nbest" in params.decoding_method:
|
|
||||||
key += f"_num_paths_{params.num_paths}_"
|
|
||||||
key += f"nbest_scale_{params.nbest_scale}"
|
|
||||||
if "LG" in params.decoding_method:
|
|
||||||
key += f"_ngram_lm_scale_{params.ngram_lm_scale}"
|
|
||||||
|
|
||||||
return {key: hyps}
|
|
||||||
else:
|
|
||||||
return {f"beam_size_{params.beam_size}": hyps}
|
|
||||||
|
|
||||||
|
|
||||||
def decode_dataset(
|
|
||||||
dl: torch.utils.data.DataLoader,
|
|
||||||
params: AttributeDict,
|
|
||||||
model: nn.Module,
|
|
||||||
sp: spm.SentencePieceProcessor,
|
|
||||||
word_table: Optional[k2.SymbolTable] = None,
|
|
||||||
decoding_graph: Optional[k2.Fsa] = None,
|
|
||||||
) -> Dict[str, List[Tuple[str, List[str], List[str]]]]:
|
|
||||||
"""Decode dataset.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
dl:
|
|
||||||
PyTorch's dataloader containing the dataset to decode.
|
|
||||||
params:
|
|
||||||
It is returned by :func:`get_params`.
|
|
||||||
model:
|
|
||||||
The neural model.
|
|
||||||
sp:
|
|
||||||
The BPE model.
|
|
||||||
word_table:
|
|
||||||
The word symbol table.
|
|
||||||
decoding_graph:
|
|
||||||
The decoding graph. Can be either a `k2.trivial_graph` or HLG, Used
|
|
||||||
only when --decoding_method is fast_beam_search, fast_beam_search_nbest,
|
|
||||||
fast_beam_search_nbest_oracle, and fast_beam_search_nbest_LG.
|
|
||||||
Returns:
|
|
||||||
Return a dict, whose key may be "greedy_search" if greedy search
|
|
||||||
is used, or it may be "beam_7" if beam size of 7 is used.
|
|
||||||
Its value is a list of tuples. Each tuple contains two elements:
|
|
||||||
The first is the reference transcript, and the second is the
|
|
||||||
predicted result.
|
|
||||||
"""
|
|
||||||
num_cuts = 0
|
|
||||||
|
|
||||||
try:
|
|
||||||
num_batches = len(dl)
|
|
||||||
except TypeError:
|
|
||||||
num_batches = "?"
|
|
||||||
|
|
||||||
if params.decoding_method == "greedy_search":
|
|
||||||
log_interval = 50
|
|
||||||
else:
|
|
||||||
log_interval = 20
|
|
||||||
|
|
||||||
results = defaultdict(list)
|
|
||||||
for batch_idx, batch in enumerate(dl):
|
|
||||||
texts = batch["supervisions"]["text"]
|
|
||||||
cut_ids = [cut.id for cut in batch["supervisions"]["cut"]]
|
|
||||||
|
|
||||||
hyps_dict = decode_one_batch(
|
|
||||||
params=params,
|
|
||||||
model=model,
|
|
||||||
sp=sp,
|
|
||||||
decoding_graph=decoding_graph,
|
|
||||||
word_table=word_table,
|
|
||||||
batch=batch,
|
|
||||||
)
|
|
||||||
|
|
||||||
for name, hyps in hyps_dict.items():
|
|
||||||
this_batch = []
|
|
||||||
assert len(hyps) == len(texts)
|
|
||||||
for cut_id, hyp_words, ref_text in zip(cut_ids, hyps, texts):
|
|
||||||
ref_words = ref_text.split()
|
|
||||||
this_batch.append((cut_id, ref_words, hyp_words))
|
|
||||||
|
|
||||||
results[name].extend(this_batch)
|
|
||||||
|
|
||||||
num_cuts += len(texts)
|
|
||||||
|
|
||||||
if batch_idx % log_interval == 0:
|
|
||||||
batch_str = f"{batch_idx}/{num_batches}"
|
|
||||||
|
|
||||||
logging.info(f"batch {batch_str}, cuts processed until now is {num_cuts}")
|
|
||||||
return results
|
|
||||||
|
|
||||||
|
|
||||||
def save_results(
|
|
||||||
params: AttributeDict,
|
|
||||||
test_set_name: str,
|
|
||||||
results_dict: Dict[str, List[Tuple[str, List[str], List[str]]]],
|
|
||||||
):
|
|
||||||
test_set_wers = dict()
|
|
||||||
for key, results in results_dict.items():
|
|
||||||
recog_path = (
|
|
||||||
params.res_dir / f"recogs-{test_set_name}-{key}-{params.suffix}.txt"
|
|
||||||
)
|
|
||||||
results = sorted(results)
|
|
||||||
store_transcripts(filename=recog_path, texts=results)
|
|
||||||
logging.info(f"The transcripts are stored in {recog_path}")
|
|
||||||
|
|
||||||
# The following prints out WERs, per-word error statistics and aligned
|
|
||||||
# ref/hyp pairs.
|
|
||||||
errs_filename = (
|
|
||||||
params.res_dir / f"errs-{test_set_name}-{key}-{params.suffix}.txt"
|
|
||||||
)
|
|
||||||
with open(errs_filename, "w") as f:
|
|
||||||
wer = write_error_stats(
|
|
||||||
f, f"{test_set_name}-{key}", results, enable_log=True
|
|
||||||
)
|
|
||||||
test_set_wers[key] = wer
|
|
||||||
|
|
||||||
logging.info("Wrote detailed error stats to {}".format(errs_filename))
|
|
||||||
|
|
||||||
test_set_wers = sorted(test_set_wers.items(), key=lambda x: x[1])
|
|
||||||
errs_info = (
|
|
||||||
params.res_dir / f"wer-summary-{test_set_name}-{key}-{params.suffix}.txt"
|
|
||||||
)
|
|
||||||
with open(errs_info, "w") as f:
|
|
||||||
print("settings\tWER", file=f)
|
|
||||||
for key, val in test_set_wers:
|
|
||||||
print("{}\t{}".format(key, val), file=f)
|
|
||||||
|
|
||||||
s = "\nFor {}, WER of different settings are:\n".format(test_set_name)
|
|
||||||
note = "\tbest for {}".format(test_set_name)
|
|
||||||
for key, val in test_set_wers:
|
|
||||||
s += "{}\t{}{}\n".format(key, val, note)
|
|
||||||
note = ""
|
|
||||||
logging.info(s)
|
|
||||||
|
|
||||||
|
|
||||||
@torch.no_grad()
|
|
||||||
def main():
|
|
||||||
parser = get_parser()
|
|
||||||
LibriSpeechAsrDataModule.add_arguments(parser)
|
|
||||||
args = parser.parse_args()
|
|
||||||
args.exp_dir = Path(args.exp_dir)
|
|
||||||
|
|
||||||
params = get_params()
|
|
||||||
params.update(vars(args))
|
|
||||||
|
|
||||||
assert params.decoding_method in (
|
|
||||||
"greedy_search",
|
|
||||||
"beam_search",
|
|
||||||
"fast_beam_search",
|
|
||||||
"fast_beam_search_nbest",
|
|
||||||
"fast_beam_search_nbest_LG",
|
|
||||||
"fast_beam_search_nbest_oracle",
|
|
||||||
"modified_beam_search",
|
|
||||||
)
|
|
||||||
params.res_dir = params.exp_dir / params.decoding_method
|
|
||||||
|
|
||||||
if params.iter > 0:
|
|
||||||
params.suffix = f"iter-{params.iter}-avg-{params.avg}"
|
|
||||||
else:
|
|
||||||
params.suffix = f"epoch-{params.epoch}-avg-{params.avg}"
|
|
||||||
|
|
||||||
if params.simulate_streaming:
|
|
||||||
params.suffix += f"-streaming-chunk-size-{params.decode_chunk_size}"
|
|
||||||
params.suffix += f"-left-context-{params.left_context}"
|
|
||||||
|
|
||||||
if "fast_beam_search" in params.decoding_method:
|
|
||||||
params.suffix += f"-beam-{params.beam}"
|
|
||||||
params.suffix += f"-max-contexts-{params.max_contexts}"
|
|
||||||
params.suffix += f"-max-states-{params.max_states}"
|
|
||||||
if "nbest" in params.decoding_method:
|
|
||||||
params.suffix += f"-nbest-scale-{params.nbest_scale}"
|
|
||||||
params.suffix += f"-num-paths-{params.num_paths}"
|
|
||||||
if "LG" in params.decoding_method:
|
|
||||||
params.suffix += f"-ngram-lm-scale-{params.ngram_lm_scale}"
|
|
||||||
elif "beam_search" in params.decoding_method:
|
|
||||||
params.suffix += f"-{params.decoding_method}-beam-size-{params.beam_size}"
|
|
||||||
else:
|
|
||||||
params.suffix += f"-context-{params.context_size}"
|
|
||||||
params.suffix += f"-max-sym-per-frame-{params.max_sym_per_frame}"
|
|
||||||
|
|
||||||
if params.use_averaged_model:
|
|
||||||
params.suffix += "-use-averaged-model"
|
|
||||||
|
|
||||||
setup_logger(f"{params.res_dir}/log-decode-{params.suffix}")
|
|
||||||
logging.info("Decoding started")
|
|
||||||
|
|
||||||
device = torch.device("cpu")
|
|
||||||
if torch.cuda.is_available():
|
|
||||||
device = torch.device("cuda", 0)
|
|
||||||
|
|
||||||
logging.info(f"Device: {device}")
|
|
||||||
|
|
||||||
sp = spm.SentencePieceProcessor()
|
|
||||||
sp.load(params.bpe_model)
|
|
||||||
|
|
||||||
# <blk> and <unk> are defined in local/train_bpe_model.py
|
|
||||||
params.blank_id = sp.piece_to_id("<blk>")
|
|
||||||
params.unk_id = sp.piece_to_id("<unk>")
|
|
||||||
params.vocab_size = sp.get_piece_size()
|
|
||||||
|
|
||||||
if params.simulate_streaming:
|
|
||||||
assert (
|
|
||||||
params.causal_convolution
|
|
||||||
), "Decoding in streaming requires causal convolution"
|
|
||||||
|
|
||||||
logging.info(params)
|
|
||||||
|
|
||||||
logging.info("About to create model")
|
|
||||||
model = get_transducer_model(params)
|
|
||||||
|
|
||||||
if '.pt' in params.model_name:
|
|
||||||
load_checkpoint(f"{params.exp_dir}/{params.model_name}", model)
|
|
||||||
elif 'lora' in params.model_name:
|
|
||||||
load_checkpoint(f"{params.exp_dir}/../d2v-base-T.pt", model)
|
|
||||||
|
|
||||||
## for lora hooking
|
|
||||||
lora_modules = []
|
|
||||||
for modules in model.modules():
|
|
||||||
if isinstance(modules, fairseq.modules.multihead_attention.MultiheadAttention):
|
|
||||||
for module in modules.modules():
|
|
||||||
if isinstance(module, torch.nn.Linear):
|
|
||||||
lora_modules.append(LoRAHook(module))
|
|
||||||
|
|
||||||
for i, lora in enumerate(lora_modules):
|
|
||||||
lora_param = torch.load(f"{params.exp_dir}/lora_{params.iter}_{i}.pt")
|
|
||||||
lora.lora.load_state_dict(lora_param)
|
|
||||||
lora.lora.to(device)
|
|
||||||
logging.info("lora params load done")
|
|
||||||
else:
|
|
||||||
if not params.use_averaged_model:
|
|
||||||
if params.iter > 0:
|
|
||||||
filenames = find_checkpoints(params.exp_dir, iteration=-params.iter)[
|
|
||||||
: params.avg
|
|
||||||
]
|
|
||||||
if len(filenames) == 0:
|
|
||||||
raise ValueError(
|
|
||||||
f"No checkpoints found for"
|
|
||||||
f" --iter {params.iter}, --avg {params.avg}"
|
|
||||||
)
|
|
||||||
elif len(filenames) < params.avg:
|
|
||||||
raise ValueError(
|
|
||||||
f"Not enough checkpoints ({len(filenames)}) found for"
|
|
||||||
f" --iter {params.iter}, --avg {params.avg}"
|
|
||||||
)
|
|
||||||
logging.info(f"averaging {filenames}")
|
|
||||||
model.to(device)
|
|
||||||
model.load_state_dict(average_checkpoints(filenames, device=device))
|
|
||||||
elif params.avg == 1:
|
|
||||||
load_checkpoint(f"{params.exp_dir}/epoch-{params.epoch}.pt", model)
|
|
||||||
else:
|
|
||||||
start = params.epoch - params.avg + 1
|
|
||||||
filenames = []
|
|
||||||
for i in range(start, params.epoch + 1):
|
|
||||||
if i >= 1:
|
|
||||||
filenames.append(f"{params.exp_dir}/epoch-{i}.pt")
|
|
||||||
logging.info(f"averaging {filenames}")
|
|
||||||
model.to(device)
|
|
||||||
model.load_state_dict(average_checkpoints(filenames, device=device))
|
|
||||||
else:
|
|
||||||
if params.iter > 0:
|
|
||||||
filenames = find_checkpoints(params.exp_dir, iteration=-params.iter)[
|
|
||||||
: params.avg + 1
|
|
||||||
]
|
|
||||||
if len(filenames) == 0:
|
|
||||||
raise ValueError(
|
|
||||||
f"No checkpoints found for"
|
|
||||||
f" --iter {params.iter}, --avg {params.avg}"
|
|
||||||
)
|
|
||||||
elif len(filenames) < params.avg + 1:
|
|
||||||
raise ValueError(
|
|
||||||
f"Not enough checkpoints ({len(filenames)}) found for"
|
|
||||||
f" --iter {params.iter}, --avg {params.avg}"
|
|
||||||
)
|
|
||||||
filename_start = filenames[-1]
|
|
||||||
filename_end = filenames[0]
|
|
||||||
logging.info(
|
|
||||||
"Calculating the averaged model over iteration checkpoints"
|
|
||||||
f" from {filename_start} (excluded) to {filename_end}"
|
|
||||||
)
|
|
||||||
model.to(device)
|
|
||||||
model.load_state_dict(
|
|
||||||
average_checkpoints_with_averaged_model(
|
|
||||||
filename_start=filename_start,
|
|
||||||
filename_end=filename_end,
|
|
||||||
device=device,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
assert params.avg > 0, params.avg
|
|
||||||
start = params.epoch - params.avg
|
|
||||||
assert start >= 1, start
|
|
||||||
filename_start = f"{params.exp_dir}/epoch-{start}.pt"
|
|
||||||
filename_end = f"{params.exp_dir}/epoch-{params.epoch}.pt"
|
|
||||||
logging.info(
|
|
||||||
f"Calculating the averaged model over epoch range from "
|
|
||||||
f"{start} (excluded) to {params.epoch}"
|
|
||||||
)
|
|
||||||
model.to(device)
|
|
||||||
model.load_state_dict(
|
|
||||||
average_checkpoints_with_averaged_model(
|
|
||||||
filename_start=filename_start,
|
|
||||||
filename_end=filename_end,
|
|
||||||
device=device,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
model.to(device)
|
|
||||||
model.eval()
|
|
||||||
|
|
||||||
if "fast_beam_search" in params.decoding_method:
|
|
||||||
if params.decoding_method == "fast_beam_search_nbest_LG":
|
|
||||||
lexicon = Lexicon(params.lang_dir)
|
|
||||||
word_table = lexicon.word_table
|
|
||||||
lg_filename = params.lang_dir / "LG.pt"
|
|
||||||
logging.info(f"Loading {lg_filename}")
|
|
||||||
decoding_graph = k2.Fsa.from_dict(
|
|
||||||
torch.load(lg_filename, map_location=device)
|
|
||||||
)
|
|
||||||
decoding_graph.scores *= params.ngram_lm_scale
|
|
||||||
else:
|
|
||||||
word_table = None
|
|
||||||
decoding_graph = k2.trivial_graph(params.vocab_size - 1, device=device)
|
|
||||||
else:
|
|
||||||
decoding_graph = None
|
|
||||||
word_table = None
|
|
||||||
|
|
||||||
num_param = sum([p.numel() for p in model.parameters()])
|
|
||||||
logging.info(f"Number of model parameters: {num_param}")
|
|
||||||
|
|
||||||
# we need cut ids to display recognition results.
|
|
||||||
args.return_cuts = True
|
|
||||||
librispeech = LibriSpeechAsrDataModule(args)
|
|
||||||
|
|
||||||
'''
|
|
||||||
test_clean_cuts = librispeech.test_clean_cuts()
|
|
||||||
test_other_cuts = librispeech.test_other_cuts()
|
|
||||||
|
|
||||||
test_clean_dl = librispeech.test_dataloaders(test_clean_cuts)
|
|
||||||
test_other_dl = librispeech.test_dataloaders(test_other_cuts)
|
|
||||||
|
|
||||||
test_sets = ["test-clean", "test-other"]
|
|
||||||
test_dl = [test_clean_dl, test_other_dl]
|
|
||||||
'''
|
|
||||||
|
|
||||||
test_clean_cuts = librispeech.userlibri_cuts(option=params.spk_id)
|
|
||||||
test_clean_dl = librispeech.test_dataloaders(test_clean_cuts)
|
|
||||||
test_sets = [f"{params.spk_id}"]
|
|
||||||
test_dl = [test_clean_dl]
|
|
||||||
|
|
||||||
for test_set, test_dl in zip(test_sets, test_dl):
|
|
||||||
results_dict = decode_dataset(
|
|
||||||
dl=test_dl,
|
|
||||||
params=params,
|
|
||||||
model=model,
|
|
||||||
sp=sp,
|
|
||||||
word_table=word_table,
|
|
||||||
decoding_graph=decoding_graph,
|
|
||||||
)
|
|
||||||
|
|
||||||
save_results(
|
|
||||||
params=params,
|
|
||||||
test_set_name=test_set,
|
|
||||||
results_dict=results_dict,
|
|
||||||
)
|
|
||||||
|
|
||||||
logging.info("Done!")
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
File diff suppressed because it is too large
Load Diff
Loading…
x
Reference in New Issue
Block a user