updated bash scripts to incorporate with the speed-perturb arg

This commit is contained in:
zr_jin 2023-07-27 17:35:55 +08:00
parent 1e29c781fc
commit 53aa106180
10 changed files with 44 additions and 16 deletions

View File

@ -77,7 +77,7 @@ if [ $stage -le 4 ] && [ $stop_stage -ge 4 ]; then
log "Stage 4: Compute fbank for aidatatang_200zh"
if [ ! -f data/fbank/.aidatatang_200zh.done ]; then
mkdir -p data/fbank
./local/compute_fbank_aidatatang_200zh.py
./local/compute_fbank_aidatatang_200zh.py --speed-perturb True
touch data/fbank/.aidatatang_200zh.done
fi
fi

View File

@ -42,7 +42,7 @@ torch.set_num_threads(1)
torch.set_num_interop_threads(1)
def compute_fbank_aidatatang_200zh(num_mel_bins: int = 80):
def compute_fbank_aidatatang_200zh(num_mel_bins: int = 80, speed_perturb: bool = False):
src_dir = Path("data/manifests")
output_dir = Path("data/fbank")
num_jobs = min(15, os.cpu_count())
@ -85,7 +85,7 @@ def compute_fbank_aidatatang_200zh(num_mel_bins: int = 80):
recordings=m["recordings"],
supervisions=m["supervisions"],
)
if "train" in partition:
if "train" in partition and speed_perturb:
cut_set = (
cut_set + cut_set.perturb_speed(0.9) + cut_set.perturb_speed(1.1)
)
@ -109,7 +109,12 @@ def get_args():
default=80,
help="""The number of mel bins for Fbank""",
)
parser.add_argument(
"--speed-perturb",
type=bool,
default=False,
help="Enable 0.9 and 1.1 speed perturbation for data augmentation. Default: False.",
)
return parser.parse_args()
@ -119,4 +124,6 @@ if __name__ == "__main__":
logging.basicConfig(format=formatter, level=logging.INFO)
args = get_args()
compute_fbank_aidatatang_200zh(num_mel_bins=args.num_mel_bins)
compute_fbank_aidatatang_200zh(
num_mel_bins=args.num_mel_bins, speed_perturb=args.speed_perturb
)

View File

@ -114,7 +114,7 @@ if [ $stage -le 3 ] && [ $stop_stage -ge 3 ]; then
log "Stage 3: Compute fbank for aishell"
if [ ! -f data/fbank/.aishell.done ]; then
mkdir -p data/fbank
./local/compute_fbank_aishell.py
./local/compute_fbank_aishell.py --speed-perturb True
touch data/fbank/.aishell.done
fi
fi

View File

@ -53,7 +53,7 @@ if [ $stage -le 2 ] && [ $stop_stage -ge 2 ]; then
log "Stage 2: Process aidatatang_200zh"
if [ ! -f data/fbank/.aidatatang_200zh_fbank.done ]; then
mkdir -p data/fbank
./local/compute_fbank_aidatatang_200zh.py
./local/compute_fbank_aidatatang_200zh.py --speed-perturb True
touch data/fbank/.aidatatang_200zh_fbank.done
fi
fi

View File

@ -101,7 +101,7 @@ if [ $stage -le 3 ] && [ $stop_stage -ge 3 ]; then
log "Stage 3: Compute fbank for aishell2"
if [ ! -f data/fbank/.aishell2.done ]; then
mkdir -p data/fbank
./local/compute_fbank_aishell2.py
./local/compute_fbank_aishell2.py --speed-perturb True
touch data/fbank/.aishell2.done
fi
fi

View File

@ -107,7 +107,7 @@ if [ $stage -le 5 ] && [ $stop_stage -ge 5 ]; then
log "Stage 5: Compute fbank for aishell4"
if [ ! -f data/fbank/.aishell4.done ]; then
mkdir -p data/fbank
./local/compute_fbank_aishell4.py
./local/compute_fbank_aishell4.py --speed-perturb True
touch data/fbank/.aishell4.done
fi
fi

View File

@ -97,7 +97,7 @@ if [ $stage -le 5 ] && [ $stop_stage -ge 5 ]; then
log "Stage 5: Compute fbank for alimeeting"
if [ ! -f data/fbank/.alimeeting.done ]; then
mkdir -p data/fbank
./local/compute_fbank_alimeeting.py
./local/compute_fbank_alimeeting.py --speed-perturb True
touch data/fbank/.alimeeting.done
fi
fi

View File

@ -25,6 +25,7 @@ It looks for manifests in the directory data/manifests.
The generated fbank features are saved in data/fbank.
"""
import argparse
import logging
from pathlib import Path
@ -48,7 +49,7 @@ torch.set_num_interop_threads(1)
torch.multiprocessing.set_sharing_strategy("file_system")
def compute_fbank_ami():
def compute_fbank_ami(speed_perturb: bool = False):
src_dir = Path("data/manifests")
output_dir = Path("data/fbank")
@ -84,8 +85,11 @@ def compute_fbank_ami():
suffix="jsonl.gz",
)
def _extract_feats(cuts: CutSet, storage_path: Path, manifest_path: Path) -> None:
cuts = cuts + cuts.perturb_speed(0.9) + cuts.perturb_speed(1.1)
def _extract_feats(
cuts: CutSet, storage_path: Path, manifest_path: Path, speed_perturb: bool
) -> None:
if speed_perturb:
cuts = cuts + cuts.perturb_speed(0.9) + cuts.perturb_speed(1.1)
_ = cuts.compute_and_store_features_batch(
extractor=extractor,
storage_path=storage_path,
@ -109,6 +113,7 @@ def compute_fbank_ami():
cuts_ihm,
output_dir / "feats_train_ihm",
src_dir / "cuts_train_ihm.jsonl.gz",
speed_perturb,
)
logging.info("Processing train split IHM + reverberated IHM")
@ -117,6 +122,7 @@ def compute_fbank_ami():
cuts_ihm_rvb,
output_dir / "feats_train_ihm_rvb",
src_dir / "cuts_train_ihm_rvb.jsonl.gz",
speed_perturb,
)
logging.info("Processing train split SDM")
@ -129,6 +135,7 @@ def compute_fbank_ami():
cuts_sdm,
output_dir / "feats_train_sdm",
src_dir / "cuts_train_sdm.jsonl.gz",
speed_perturb,
)
logging.info("Processing train split GSS")
@ -141,6 +148,7 @@ def compute_fbank_ami():
cuts_gss,
output_dir / "feats_train_gss",
src_dir / "cuts_train_gss.jsonl.gz",
speed_perturb,
)
logging.info("Preparing test cuts: IHM, SDM, GSS (optional)")
@ -186,8 +194,21 @@ def compute_fbank_ami():
)
def get_args():
parser = argparse.ArgumentParser()
parser.add_argument(
"--speed-perturb",
type=bool,
default=False,
help="Enable 0.9 and 1.1 speed perturbation for data augmentation. Default: False.",
)
return parser.parse_args()
if __name__ == "__main__":
formatter = "%(asctime)s %(levelname)s [%(filename)s:%(lineno)d] %(message)s"
logging.basicConfig(format=formatter, level=logging.INFO)
compute_fbank_ami()
args = get_args()
compute_fbank_ami(speed_perturb=args.speed_perturb)

View File

@ -85,7 +85,7 @@ fi
if [ $stage -le 5 ] && [ $stop_stage -ge 5 ]; then
log "Stage 5: Compute fbank for alimeeting"
mkdir -p data/fbank
python local/compute_fbank_alimeeting.py
python local/compute_fbank_alimeeting.py --speed-perturb True
log "Combine features from train splits"
lhotse combine data/manifests/cuts_train_{ihm,ihm_rvb,sdm,gss}.jsonl.gz - | shuf |\
gzip -c > data/manifests/cuts_train_all.jsonl.gz

View File

@ -91,7 +91,7 @@ fi
if [ $stage -le 3 ] && [ $stop_stage -ge 3 ]; then
log "Stage 3: Preprocess WenetSpeech manifest"
if [ ! -f data/fbank/.preprocess_complete ]; then
python3 ./local/preprocess_wenetspeech.py
python3 ./local/preprocess_wenetspeech.py --speed-perturb True
touch data/fbank/.preprocess_complete
fi
fi