mirror of
https://github.com/k2-fsa/icefall.git
synced 2025-12-11 06:55:27 +00:00
updated bash scripts to incorporate with the speed-perturb arg
This commit is contained in:
parent
1e29c781fc
commit
53aa106180
@ -77,7 +77,7 @@ if [ $stage -le 4 ] && [ $stop_stage -ge 4 ]; then
|
|||||||
log "Stage 4: Compute fbank for aidatatang_200zh"
|
log "Stage 4: Compute fbank for aidatatang_200zh"
|
||||||
if [ ! -f data/fbank/.aidatatang_200zh.done ]; then
|
if [ ! -f data/fbank/.aidatatang_200zh.done ]; then
|
||||||
mkdir -p data/fbank
|
mkdir -p data/fbank
|
||||||
./local/compute_fbank_aidatatang_200zh.py
|
./local/compute_fbank_aidatatang_200zh.py --speed-perturb True
|
||||||
touch data/fbank/.aidatatang_200zh.done
|
touch data/fbank/.aidatatang_200zh.done
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|||||||
@ -42,7 +42,7 @@ torch.set_num_threads(1)
|
|||||||
torch.set_num_interop_threads(1)
|
torch.set_num_interop_threads(1)
|
||||||
|
|
||||||
|
|
||||||
def compute_fbank_aidatatang_200zh(num_mel_bins: int = 80):
|
def compute_fbank_aidatatang_200zh(num_mel_bins: int = 80, speed_perturb: bool = False):
|
||||||
src_dir = Path("data/manifests")
|
src_dir = Path("data/manifests")
|
||||||
output_dir = Path("data/fbank")
|
output_dir = Path("data/fbank")
|
||||||
num_jobs = min(15, os.cpu_count())
|
num_jobs = min(15, os.cpu_count())
|
||||||
@ -85,7 +85,7 @@ def compute_fbank_aidatatang_200zh(num_mel_bins: int = 80):
|
|||||||
recordings=m["recordings"],
|
recordings=m["recordings"],
|
||||||
supervisions=m["supervisions"],
|
supervisions=m["supervisions"],
|
||||||
)
|
)
|
||||||
if "train" in partition:
|
if "train" in partition and speed_perturb:
|
||||||
cut_set = (
|
cut_set = (
|
||||||
cut_set + cut_set.perturb_speed(0.9) + cut_set.perturb_speed(1.1)
|
cut_set + cut_set.perturb_speed(0.9) + cut_set.perturb_speed(1.1)
|
||||||
)
|
)
|
||||||
@ -109,7 +109,12 @@ def get_args():
|
|||||||
default=80,
|
default=80,
|
||||||
help="""The number of mel bins for Fbank""",
|
help="""The number of mel bins for Fbank""",
|
||||||
)
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--speed-perturb",
|
||||||
|
type=bool,
|
||||||
|
default=False,
|
||||||
|
help="Enable 0.9 and 1.1 speed perturbation for data augmentation. Default: False.",
|
||||||
|
)
|
||||||
return parser.parse_args()
|
return parser.parse_args()
|
||||||
|
|
||||||
|
|
||||||
@ -119,4 +124,6 @@ if __name__ == "__main__":
|
|||||||
logging.basicConfig(format=formatter, level=logging.INFO)
|
logging.basicConfig(format=formatter, level=logging.INFO)
|
||||||
|
|
||||||
args = get_args()
|
args = get_args()
|
||||||
compute_fbank_aidatatang_200zh(num_mel_bins=args.num_mel_bins)
|
compute_fbank_aidatatang_200zh(
|
||||||
|
num_mel_bins=args.num_mel_bins, speed_perturb=args.speed_perturb
|
||||||
|
)
|
||||||
|
|||||||
@ -114,7 +114,7 @@ if [ $stage -le 3 ] && [ $stop_stage -ge 3 ]; then
|
|||||||
log "Stage 3: Compute fbank for aishell"
|
log "Stage 3: Compute fbank for aishell"
|
||||||
if [ ! -f data/fbank/.aishell.done ]; then
|
if [ ! -f data/fbank/.aishell.done ]; then
|
||||||
mkdir -p data/fbank
|
mkdir -p data/fbank
|
||||||
./local/compute_fbank_aishell.py
|
./local/compute_fbank_aishell.py --speed-perturb True
|
||||||
touch data/fbank/.aishell.done
|
touch data/fbank/.aishell.done
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|||||||
@ -53,7 +53,7 @@ if [ $stage -le 2 ] && [ $stop_stage -ge 2 ]; then
|
|||||||
log "Stage 2: Process aidatatang_200zh"
|
log "Stage 2: Process aidatatang_200zh"
|
||||||
if [ ! -f data/fbank/.aidatatang_200zh_fbank.done ]; then
|
if [ ! -f data/fbank/.aidatatang_200zh_fbank.done ]; then
|
||||||
mkdir -p data/fbank
|
mkdir -p data/fbank
|
||||||
./local/compute_fbank_aidatatang_200zh.py
|
./local/compute_fbank_aidatatang_200zh.py --speed-perturb True
|
||||||
touch data/fbank/.aidatatang_200zh_fbank.done
|
touch data/fbank/.aidatatang_200zh_fbank.done
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|||||||
@ -101,7 +101,7 @@ if [ $stage -le 3 ] && [ $stop_stage -ge 3 ]; then
|
|||||||
log "Stage 3: Compute fbank for aishell2"
|
log "Stage 3: Compute fbank for aishell2"
|
||||||
if [ ! -f data/fbank/.aishell2.done ]; then
|
if [ ! -f data/fbank/.aishell2.done ]; then
|
||||||
mkdir -p data/fbank
|
mkdir -p data/fbank
|
||||||
./local/compute_fbank_aishell2.py
|
./local/compute_fbank_aishell2.py --speed-perturb True
|
||||||
touch data/fbank/.aishell2.done
|
touch data/fbank/.aishell2.done
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|||||||
@ -107,7 +107,7 @@ if [ $stage -le 5 ] && [ $stop_stage -ge 5 ]; then
|
|||||||
log "Stage 5: Compute fbank for aishell4"
|
log "Stage 5: Compute fbank for aishell4"
|
||||||
if [ ! -f data/fbank/.aishell4.done ]; then
|
if [ ! -f data/fbank/.aishell4.done ]; then
|
||||||
mkdir -p data/fbank
|
mkdir -p data/fbank
|
||||||
./local/compute_fbank_aishell4.py
|
./local/compute_fbank_aishell4.py --speed-perturb True
|
||||||
touch data/fbank/.aishell4.done
|
touch data/fbank/.aishell4.done
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|||||||
@ -97,7 +97,7 @@ if [ $stage -le 5 ] && [ $stop_stage -ge 5 ]; then
|
|||||||
log "Stage 5: Compute fbank for alimeeting"
|
log "Stage 5: Compute fbank for alimeeting"
|
||||||
if [ ! -f data/fbank/.alimeeting.done ]; then
|
if [ ! -f data/fbank/.alimeeting.done ]; then
|
||||||
mkdir -p data/fbank
|
mkdir -p data/fbank
|
||||||
./local/compute_fbank_alimeeting.py
|
./local/compute_fbank_alimeeting.py --speed-perturb True
|
||||||
touch data/fbank/.alimeeting.done
|
touch data/fbank/.alimeeting.done
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|||||||
@ -25,6 +25,7 @@ It looks for manifests in the directory data/manifests.
|
|||||||
|
|
||||||
The generated fbank features are saved in data/fbank.
|
The generated fbank features are saved in data/fbank.
|
||||||
"""
|
"""
|
||||||
|
import argparse
|
||||||
import logging
|
import logging
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
@ -48,7 +49,7 @@ torch.set_num_interop_threads(1)
|
|||||||
torch.multiprocessing.set_sharing_strategy("file_system")
|
torch.multiprocessing.set_sharing_strategy("file_system")
|
||||||
|
|
||||||
|
|
||||||
def compute_fbank_ami():
|
def compute_fbank_ami(speed_perturb: bool = False):
|
||||||
src_dir = Path("data/manifests")
|
src_dir = Path("data/manifests")
|
||||||
output_dir = Path("data/fbank")
|
output_dir = Path("data/fbank")
|
||||||
|
|
||||||
@ -84,7 +85,10 @@ def compute_fbank_ami():
|
|||||||
suffix="jsonl.gz",
|
suffix="jsonl.gz",
|
||||||
)
|
)
|
||||||
|
|
||||||
def _extract_feats(cuts: CutSet, storage_path: Path, manifest_path: Path) -> None:
|
def _extract_feats(
|
||||||
|
cuts: CutSet, storage_path: Path, manifest_path: Path, speed_perturb: bool
|
||||||
|
) -> None:
|
||||||
|
if speed_perturb:
|
||||||
cuts = cuts + cuts.perturb_speed(0.9) + cuts.perturb_speed(1.1)
|
cuts = cuts + cuts.perturb_speed(0.9) + cuts.perturb_speed(1.1)
|
||||||
_ = cuts.compute_and_store_features_batch(
|
_ = cuts.compute_and_store_features_batch(
|
||||||
extractor=extractor,
|
extractor=extractor,
|
||||||
@ -109,6 +113,7 @@ def compute_fbank_ami():
|
|||||||
cuts_ihm,
|
cuts_ihm,
|
||||||
output_dir / "feats_train_ihm",
|
output_dir / "feats_train_ihm",
|
||||||
src_dir / "cuts_train_ihm.jsonl.gz",
|
src_dir / "cuts_train_ihm.jsonl.gz",
|
||||||
|
speed_perturb,
|
||||||
)
|
)
|
||||||
|
|
||||||
logging.info("Processing train split IHM + reverberated IHM")
|
logging.info("Processing train split IHM + reverberated IHM")
|
||||||
@ -117,6 +122,7 @@ def compute_fbank_ami():
|
|||||||
cuts_ihm_rvb,
|
cuts_ihm_rvb,
|
||||||
output_dir / "feats_train_ihm_rvb",
|
output_dir / "feats_train_ihm_rvb",
|
||||||
src_dir / "cuts_train_ihm_rvb.jsonl.gz",
|
src_dir / "cuts_train_ihm_rvb.jsonl.gz",
|
||||||
|
speed_perturb,
|
||||||
)
|
)
|
||||||
|
|
||||||
logging.info("Processing train split SDM")
|
logging.info("Processing train split SDM")
|
||||||
@ -129,6 +135,7 @@ def compute_fbank_ami():
|
|||||||
cuts_sdm,
|
cuts_sdm,
|
||||||
output_dir / "feats_train_sdm",
|
output_dir / "feats_train_sdm",
|
||||||
src_dir / "cuts_train_sdm.jsonl.gz",
|
src_dir / "cuts_train_sdm.jsonl.gz",
|
||||||
|
speed_perturb,
|
||||||
)
|
)
|
||||||
|
|
||||||
logging.info("Processing train split GSS")
|
logging.info("Processing train split GSS")
|
||||||
@ -141,6 +148,7 @@ def compute_fbank_ami():
|
|||||||
cuts_gss,
|
cuts_gss,
|
||||||
output_dir / "feats_train_gss",
|
output_dir / "feats_train_gss",
|
||||||
src_dir / "cuts_train_gss.jsonl.gz",
|
src_dir / "cuts_train_gss.jsonl.gz",
|
||||||
|
speed_perturb,
|
||||||
)
|
)
|
||||||
|
|
||||||
logging.info("Preparing test cuts: IHM, SDM, GSS (optional)")
|
logging.info("Preparing test cuts: IHM, SDM, GSS (optional)")
|
||||||
@ -186,8 +194,21 @@ def compute_fbank_ami():
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def get_args():
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument(
|
||||||
|
"--speed-perturb",
|
||||||
|
type=bool,
|
||||||
|
default=False,
|
||||||
|
help="Enable 0.9 and 1.1 speed perturbation for data augmentation. Default: False.",
|
||||||
|
)
|
||||||
|
return parser.parse_args()
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
formatter = "%(asctime)s %(levelname)s [%(filename)s:%(lineno)d] %(message)s"
|
formatter = "%(asctime)s %(levelname)s [%(filename)s:%(lineno)d] %(message)s"
|
||||||
logging.basicConfig(format=formatter, level=logging.INFO)
|
logging.basicConfig(format=formatter, level=logging.INFO)
|
||||||
|
|
||||||
compute_fbank_ami()
|
args = get_args()
|
||||||
|
|
||||||
|
compute_fbank_ami(speed_perturb=args.speed_perturb)
|
||||||
|
|||||||
@ -85,7 +85,7 @@ fi
|
|||||||
if [ $stage -le 5 ] && [ $stop_stage -ge 5 ]; then
|
if [ $stage -le 5 ] && [ $stop_stage -ge 5 ]; then
|
||||||
log "Stage 5: Compute fbank for alimeeting"
|
log "Stage 5: Compute fbank for alimeeting"
|
||||||
mkdir -p data/fbank
|
mkdir -p data/fbank
|
||||||
python local/compute_fbank_alimeeting.py
|
python local/compute_fbank_alimeeting.py --speed-perturb True
|
||||||
log "Combine features from train splits"
|
log "Combine features from train splits"
|
||||||
lhotse combine data/manifests/cuts_train_{ihm,ihm_rvb,sdm,gss}.jsonl.gz - | shuf |\
|
lhotse combine data/manifests/cuts_train_{ihm,ihm_rvb,sdm,gss}.jsonl.gz - | shuf |\
|
||||||
gzip -c > data/manifests/cuts_train_all.jsonl.gz
|
gzip -c > data/manifests/cuts_train_all.jsonl.gz
|
||||||
|
|||||||
@ -91,7 +91,7 @@ fi
|
|||||||
if [ $stage -le 3 ] && [ $stop_stage -ge 3 ]; then
|
if [ $stage -le 3 ] && [ $stop_stage -ge 3 ]; then
|
||||||
log "Stage 3: Preprocess WenetSpeech manifest"
|
log "Stage 3: Preprocess WenetSpeech manifest"
|
||||||
if [ ! -f data/fbank/.preprocess_complete ]; then
|
if [ ! -f data/fbank/.preprocess_complete ]; then
|
||||||
python3 ./local/preprocess_wenetspeech.py
|
python3 ./local/preprocess_wenetspeech.py --speed-perturb True
|
||||||
touch data/fbank/.preprocess_complete
|
touch data/fbank/.preprocess_complete
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user