mirror of
https://github.com/k2-fsa/icefall.git
synced 2025-08-27 18:54:18 +00:00
added scripts for processing validated data
This commit is contained in:
parent
e979bf5e93
commit
ed3d25b768
@ -42,6 +42,13 @@ torch.set_num_interop_threads(1)
|
||||
def get_args():
|
||||
parser = argparse.ArgumentParser()
|
||||
|
||||
parser.add_argument(
|
||||
"--subset",
|
||||
type=str,
|
||||
default="train"
|
||||
help="""Dataset parts to compute fbank. """,
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--language",
|
||||
type=str,
|
||||
@ -95,7 +102,7 @@ def get_args():
|
||||
|
||||
|
||||
def compute_fbank_commonvoice_splits(args):
|
||||
subset = "train"
|
||||
subset = args.subset
|
||||
num_splits = args.num_splits
|
||||
language = args.language
|
||||
output_dir = f"data/{language}/fbank/cv-{language}_{subset}_split_{num_splits}"
|
||||
|
@ -164,6 +164,26 @@ def preprocess_commonvoice(
|
||||
supervisions=m["supervisions"],
|
||||
).resample(16000)
|
||||
|
||||
if partition == "validated":
|
||||
logging.warning(
|
||||
"""
|
||||
The 'validated' partition contains the data of both 'train', 'dev'
|
||||
and 'test' partitions. We filter out the 'dev' and 'test' partition
|
||||
here.
|
||||
"""
|
||||
)
|
||||
dev_ids = src_dir / f"cv-{language}_dev_ids"
|
||||
test_ids = src_dir / f"cv-{language}_test_ids"
|
||||
assert (
|
||||
dev_ids.is_file()
|
||||
), f"{dev_ids} does not exist, please check stage 1 of the prepare.sh"
|
||||
assert (
|
||||
test_ids.is_file()
|
||||
), f"{test_ids} does not exist, please check stage 1 of the prepare.sh"
|
||||
dev_ids = dev_ids.read_text().strip().split("\n")
|
||||
test_ids = test_ids.read_text().strip().split("\n")
|
||||
cut_set = cut_set.filter(lambda x: x.id not in dev_ids + test_ids)
|
||||
|
||||
# Run data augmentation that needs to be done in the
|
||||
# time domain.
|
||||
logging.info(f"Saving to {raw_cuts_path}")
|
||||
|
@ -101,8 +101,35 @@ if [ $stage -le 1 ] && [ $stop_stage -ge 1 ]; then
|
||||
mkdir -p data/${lang}/manifests
|
||||
if [ ! -e data/${lang}/manifests/.cv-${lang}.done ]; then
|
||||
lhotse prepare commonvoice --language $lang -j $nj $dl_dir/$release data/${lang}/manifests
|
||||
|
||||
# In case you want to use all validated data
|
||||
# lhotse prepare commonvoice \
|
||||
# --split validated \
|
||||
# --language $lang \
|
||||
# -j $nj $dl_dir/$release data/${lang}/manifests
|
||||
# touch data/${lang}/manifests/.cv-${lang}.validated.done
|
||||
|
||||
# In case you want to take the risk and use invalidated data
|
||||
# lhotse prepare commonvoice \
|
||||
# --split invalidated \
|
||||
# --language $lang \
|
||||
# -j $nj $dl_dir/$release data/${lang}/manifests
|
||||
# touch data/${lang}/manifests/.cv-${lang}.invalidated.done
|
||||
|
||||
touch data/${lang}/manifests/.cv-${lang}.done
|
||||
fi
|
||||
|
||||
# Note: in Linux, you can install jq with the following command:
|
||||
# 1. wget -O jq https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64
|
||||
# 2. chmod +x ./jq
|
||||
# 3. cp jq /usr/bin
|
||||
if [ -f data/${lang}/manifests/.cv-${lang}.validated.done ]; then
|
||||
gunzip -c data/${lang}/manifests/cv-${lang}_supervisions_test.jsonl.gz \
|
||||
| jq '.id' | sed 's/"//g' > data/${lang}/manifests/cv-${lang}_test_ids
|
||||
|
||||
gunzip -c data/${lang}/manifests/cv-${lang}_supervisions_dev.jsonl.gz \
|
||||
| jq '.id' | sed 's/"//g' > data/${lang}/manifests/cv-${lang}_dev_ids
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ $stage -le 2 ] && [ $stop_stage -ge 2 ]; then
|
||||
@ -121,6 +148,18 @@ if [ $stage -le 3 ] && [ $stop_stage -ge 3 ]; then
|
||||
if [ ! -e data/${lang}/fbank/.preprocess_complete ]; then
|
||||
./local/preprocess_commonvoice.py --language $lang
|
||||
touch data/${lang}/fbank/.preprocess_complete
|
||||
|
||||
if [ -f data/${lang}/manifests/.cv-${lang}.validated.done ]; then
|
||||
log "Also preprocess validated data"
|
||||
./local/preprocess_commonvoice.py --language $lang --split validated
|
||||
touch data/${lang}/fbank/.validated.preprocess_complete
|
||||
fi
|
||||
|
||||
if [ -f data/${lang}/manifests/.cv-${lang}.invalidated.done ]; then
|
||||
log "Also preprocess invalidated data"
|
||||
./local/preprocess_commonvoice.py --language $lang --split invalidated
|
||||
touch data/${lang}/fbank/.invalidated.preprocess_complete
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
@ -140,6 +179,20 @@ if [ $stage -le 5 ] && [ $stop_stage -ge 5 ]; then
|
||||
lhotse split $num_splits ./data/${lang}/fbank/cv-${lang}_cuts_train_raw.jsonl.gz $split_dir
|
||||
touch $split_dir/.cv-${lang}_train_split.done
|
||||
fi
|
||||
|
||||
if [ -f data/${lang}/fbank/.validated.preprocess_complete ]; then
|
||||
log "Also split validated data"
|
||||
split_dir=data/${lang}/fbank/cv-${lang}_validated_split_${num_splits}
|
||||
lhotse split $num_splits ./data/${lang}/fbank/cv-${lang}_cuts_validated_raw.jsonl.gz $split_dir
|
||||
touch $split_dir/.cv-${lang}_validated.done
|
||||
fi
|
||||
|
||||
if [ -f data/${lang}/fbank/.invalidated.preprocess_complete ]; then
|
||||
log "Also split invalidated data"
|
||||
split_dir=data/${lang}/fbank/cv-${lang}_invalidated_split_${num_splits}
|
||||
lhotse split $num_splits ./data/${lang}/fbank/cv-${lang}_cuts_invalidated_raw.jsonl.gz $split_dir
|
||||
touch $split_dir/.cv-${lang}_invalidated.done
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ $stage -le 6 ] && [ $stop_stage -ge 6 ]; then
|
||||
@ -154,6 +207,34 @@ if [ $stage -le 6 ] && [ $stop_stage -ge 6 ]; then
|
||||
--perturb-speed $perturb_speed
|
||||
touch data/${lang}/fbank/.cv-${lang}_train.done
|
||||
fi
|
||||
|
||||
split_dir=data/${lang}/fbank/cv-${lang}_validated_split_${num_splits}
|
||||
if [ -f $split_dir/.cv-${lang}_validated.done ]; then
|
||||
log "Also compute features for validated data"
|
||||
./local/compute_fbank_commonvoice_splits.py \
|
||||
--subset validated \
|
||||
--num-workers $nj \
|
||||
--batch-duration 200 \
|
||||
--start 0 \
|
||||
--num-splits $num_splits \
|
||||
--language $lang \
|
||||
--perturb-speed $perturb_speed
|
||||
touch data/${lang}/fbank/.cv-${lang}_validated.done
|
||||
fi
|
||||
|
||||
split_dir=data/${lang}/fbank/cv-${lang}_invalidated_split_${num_splits}
|
||||
if [ -f $split_dir/.cv-${lang}_invalidated.done ]; then
|
||||
log "Also compute features for invalidated data"
|
||||
./local/compute_fbank_commonvoice_splits.py \
|
||||
--subset invalidated \
|
||||
--num-workers $nj \
|
||||
--batch-duration 200 \
|
||||
--start 0 \
|
||||
--num-splits $num_splits \
|
||||
--language $lang \
|
||||
--perturb-speed $perturb_speed
|
||||
touch data/${lang}/fbank/.cv-${lang}_invalidated.done
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ $stage -le 7 ] && [ $stop_stage -ge 7 ]; then
|
||||
@ -162,6 +243,18 @@ if [ $stage -le 7 ] && [ $stop_stage -ge 7 ]; then
|
||||
pieces=$(find data/${lang}/fbank/cv-${lang}_train_split_${num_splits} -name "cv-${lang}_cuts_train.*.jsonl.gz")
|
||||
lhotse combine $pieces data/${lang}/fbank/cv-${lang}_cuts_train.jsonl.gz
|
||||
fi
|
||||
|
||||
if [ -f data/${lang}/fbank/.cv-${lang}_validated.done ]; then
|
||||
log "Also combine features for validated data"
|
||||
pieces=$(find data/${lang}/fbank/cv-${lang}_validated_split_${num_splits} -name "cv-${lang}_cuts_validated.*.jsonl.gz")
|
||||
lhotse combine $pieces data/${lang}/fbank/cv-${lang}_cuts_validated.jsonl.gz
|
||||
fi
|
||||
|
||||
if [ -f data/${lang}/fbank/.cv-${lang}_invalidated.done ]; then
|
||||
log "Also combine features for invalidated data"
|
||||
pieces=$(find data/${lang}/fbank/cv-${lang}_inalidated_split_${num_splits} -name "cv-${lang}_cuts_invalidated.*.jsonl.gz")
|
||||
lhotse combine $pieces data/${lang}/fbank/cv-${lang}_cuts_invalidated.jsonl.gz
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ $stage -le 8 ] && [ $stop_stage -ge 8 ]; then
|
||||
|
Loading…
x
Reference in New Issue
Block a user