mirror of
https://github.com/k2-fsa/icefall.git
synced 2025-08-26 18:24:18 +00:00
scripts updated
This commit is contained in:
parent
ed3d25b768
commit
53fb384488
@ -45,7 +45,7 @@ def get_args():
|
|||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--subset",
|
"--subset",
|
||||||
type=str,
|
type=str,
|
||||||
default="train"
|
default="train",
|
||||||
help="""Dataset parts to compute fbank. """,
|
help="""Dataset parts to compute fbank. """,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -10,6 +10,12 @@ stop_stage=100
|
|||||||
# This is to avoid OOM during feature extraction.
|
# This is to avoid OOM during feature extraction.
|
||||||
num_splits=1000
|
num_splits=1000
|
||||||
|
|
||||||
|
# In case you want to use all validated data
|
||||||
|
use_validated=false
|
||||||
|
|
||||||
|
# In case you are willing to take the risk and use invalidated data
|
||||||
|
use_invalidated=false
|
||||||
|
|
||||||
# We assume dl_dir (download dir) contains the following
|
# We assume dl_dir (download dir) contains the following
|
||||||
# directories and files. If not, they will be downloaded
|
# directories and files. If not, they will be downloaded
|
||||||
# by this script automatically.
|
# by this script automatically.
|
||||||
@ -102,19 +108,23 @@ if [ $stage -le 1 ] && [ $stop_stage -ge 1 ]; then
|
|||||||
if [ ! -e data/${lang}/manifests/.cv-${lang}.done ]; then
|
if [ ! -e data/${lang}/manifests/.cv-${lang}.done ]; then
|
||||||
lhotse prepare commonvoice --language $lang -j $nj $dl_dir/$release data/${lang}/manifests
|
lhotse prepare commonvoice --language $lang -j $nj $dl_dir/$release data/${lang}/manifests
|
||||||
|
|
||||||
# In case you want to use all validated data
|
if [ $use_validated = true ] && [ ! -f data/${lang}/manifests/.cv-${lang}.validated.done ]; then
|
||||||
# lhotse prepare commonvoice \
|
log "Also prepare validated data"
|
||||||
# --split validated \
|
lhotse prepare commonvoice \
|
||||||
# --language $lang \
|
--split validated \
|
||||||
# -j $nj $dl_dir/$release data/${lang}/manifests
|
--language $lang \
|
||||||
# touch data/${lang}/manifests/.cv-${lang}.validated.done
|
-j $nj $dl_dir/$release data/${lang}/manifests
|
||||||
|
touch data/${lang}/manifests/.cv-${lang}.validated.done
|
||||||
|
fi
|
||||||
|
|
||||||
# In case you want to take the risk and use invalidated data
|
if [ $use_invalidated = true ] && [ ! -f data/${lang}/manifests/.cv-${lang}.invalidated.done ]; then
|
||||||
# lhotse prepare commonvoice \
|
log "Also prepare invalidated data"
|
||||||
# --split invalidated \
|
lhotse prepare commonvoice \
|
||||||
# --language $lang \
|
--split invalidated \
|
||||||
# -j $nj $dl_dir/$release data/${lang}/manifests
|
--language $lang \
|
||||||
# touch data/${lang}/manifests/.cv-${lang}.invalidated.done
|
-j $nj $dl_dir/$release data/${lang}/manifests
|
||||||
|
touch data/${lang}/manifests/.cv-${lang}.invalidated.done
|
||||||
|
fi
|
||||||
|
|
||||||
touch data/${lang}/manifests/.cv-${lang}.done
|
touch data/${lang}/manifests/.cv-${lang}.done
|
||||||
fi
|
fi
|
||||||
@ -123,7 +133,8 @@ if [ $stage -le 1 ] && [ $stop_stage -ge 1 ]; then
|
|||||||
# 1. wget -O jq https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64
|
# 1. wget -O jq https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64
|
||||||
# 2. chmod +x ./jq
|
# 2. chmod +x ./jq
|
||||||
# 3. cp jq /usr/bin
|
# 3. cp jq /usr/bin
|
||||||
if [ -f data/${lang}/manifests/.cv-${lang}.validated.done ]; then
|
if [ $use_validated = true ]; then
|
||||||
|
log "Getting cut ids from dev/test sets for later use"
|
||||||
gunzip -c data/${lang}/manifests/cv-${lang}_supervisions_test.jsonl.gz \
|
gunzip -c data/${lang}/manifests/cv-${lang}_supervisions_test.jsonl.gz \
|
||||||
| jq '.id' | sed 's/"//g' > data/${lang}/manifests/cv-${lang}_test_ids
|
| jq '.id' | sed 's/"//g' > data/${lang}/manifests/cv-${lang}_test_ids
|
||||||
|
|
||||||
@ -148,18 +159,18 @@ if [ $stage -le 3 ] && [ $stop_stage -ge 3 ]; then
|
|||||||
if [ ! -e data/${lang}/fbank/.preprocess_complete ]; then
|
if [ ! -e data/${lang}/fbank/.preprocess_complete ]; then
|
||||||
./local/preprocess_commonvoice.py --language $lang
|
./local/preprocess_commonvoice.py --language $lang
|
||||||
touch data/${lang}/fbank/.preprocess_complete
|
touch data/${lang}/fbank/.preprocess_complete
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ $use_validated = true ] && [ ! -f data/${lang}/fbank/.validated.preprocess_complete ]; then
|
||||||
|
log "Also preprocess validated data"
|
||||||
|
./local/preprocess_commonvoice.py --language $lang --split validated
|
||||||
|
touch data/${lang}/fbank/.validated.preprocess_complete
|
||||||
|
fi
|
||||||
|
|
||||||
if [ -f data/${lang}/manifests/.cv-${lang}.validated.done ]; then
|
if [ $use_invalidated = true ] && [ ! -f data/${lang}/fbank/.invalidated.preprocess_complete ]; then
|
||||||
log "Also preprocess validated data"
|
log "Also preprocess invalidated data"
|
||||||
./local/preprocess_commonvoice.py --language $lang --split validated
|
./local/preprocess_commonvoice.py --language $lang --split invalidated
|
||||||
touch data/${lang}/fbank/.validated.preprocess_complete
|
touch data/${lang}/fbank/.invalidated.preprocess_complete
|
||||||
fi
|
|
||||||
|
|
||||||
if [ -f data/${lang}/manifests/.cv-${lang}.invalidated.done ]; then
|
|
||||||
log "Also preprocess invalidated data"
|
|
||||||
./local/preprocess_commonvoice.py --language $lang --split invalidated
|
|
||||||
touch data/${lang}/fbank/.invalidated.preprocess_complete
|
|
||||||
fi
|
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
@ -180,16 +191,16 @@ if [ $stage -le 5 ] && [ $stop_stage -ge 5 ]; then
|
|||||||
touch $split_dir/.cv-${lang}_train_split.done
|
touch $split_dir/.cv-${lang}_train_split.done
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [ -f data/${lang}/fbank/.validated.preprocess_complete ]; then
|
split_dir=data/${lang}/fbank/cv-${lang}_validated_split_${num_splits}
|
||||||
|
if [ $use_validated = true ] && [ ! -f $split_dir/.cv-${lang}_validated.done ]; then
|
||||||
log "Also split validated data"
|
log "Also split validated data"
|
||||||
split_dir=data/${lang}/fbank/cv-${lang}_validated_split_${num_splits}
|
|
||||||
lhotse split $num_splits ./data/${lang}/fbank/cv-${lang}_cuts_validated_raw.jsonl.gz $split_dir
|
lhotse split $num_splits ./data/${lang}/fbank/cv-${lang}_cuts_validated_raw.jsonl.gz $split_dir
|
||||||
touch $split_dir/.cv-${lang}_validated.done
|
touch $split_dir/.cv-${lang}_validated.done
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [ -f data/${lang}/fbank/.invalidated.preprocess_complete ]; then
|
split_dir=data/${lang}/fbank/cv-${lang}_invalidated_split_${num_splits}
|
||||||
|
if [ $use_invalidated = true ] && [ ! -f $split_dir/.cv-${lang}_invalidated.done ]; then
|
||||||
log "Also split invalidated data"
|
log "Also split invalidated data"
|
||||||
split_dir=data/${lang}/fbank/cv-${lang}_invalidated_split_${num_splits}
|
|
||||||
lhotse split $num_splits ./data/${lang}/fbank/cv-${lang}_cuts_invalidated_raw.jsonl.gz $split_dir
|
lhotse split $num_splits ./data/${lang}/fbank/cv-${lang}_cuts_invalidated_raw.jsonl.gz $split_dir
|
||||||
touch $split_dir/.cv-${lang}_invalidated.done
|
touch $split_dir/.cv-${lang}_invalidated.done
|
||||||
fi
|
fi
|
||||||
@ -208,8 +219,7 @@ if [ $stage -le 6 ] && [ $stop_stage -ge 6 ]; then
|
|||||||
touch data/${lang}/fbank/.cv-${lang}_train.done
|
touch data/${lang}/fbank/.cv-${lang}_train.done
|
||||||
fi
|
fi
|
||||||
|
|
||||||
split_dir=data/${lang}/fbank/cv-${lang}_validated_split_${num_splits}
|
if [ $use_validated = true ] && [ ! -f data/${lang}/fbank/.cv-${lang}_validated.done ]; then
|
||||||
if [ -f $split_dir/.cv-${lang}_validated.done ]; then
|
|
||||||
log "Also compute features for validated data"
|
log "Also compute features for validated data"
|
||||||
./local/compute_fbank_commonvoice_splits.py \
|
./local/compute_fbank_commonvoice_splits.py \
|
||||||
--subset validated \
|
--subset validated \
|
||||||
@ -222,8 +232,7 @@ if [ $stage -le 6 ] && [ $stop_stage -ge 6 ]; then
|
|||||||
touch data/${lang}/fbank/.cv-${lang}_validated.done
|
touch data/${lang}/fbank/.cv-${lang}_validated.done
|
||||||
fi
|
fi
|
||||||
|
|
||||||
split_dir=data/${lang}/fbank/cv-${lang}_invalidated_split_${num_splits}
|
if [ $use_invalidated = true ] && [ ! -f data/${lang}/fbank/.cv-${lang}_invalidated.done ]; then
|
||||||
if [ -f $split_dir/.cv-${lang}_invalidated.done ]; then
|
|
||||||
log "Also compute features for invalidated data"
|
log "Also compute features for invalidated data"
|
||||||
./local/compute_fbank_commonvoice_splits.py \
|
./local/compute_fbank_commonvoice_splits.py \
|
||||||
--subset invalidated \
|
--subset invalidated \
|
||||||
@ -244,13 +253,13 @@ if [ $stage -le 7 ] && [ $stop_stage -ge 7 ]; then
|
|||||||
lhotse combine $pieces data/${lang}/fbank/cv-${lang}_cuts_train.jsonl.gz
|
lhotse combine $pieces data/${lang}/fbank/cv-${lang}_cuts_train.jsonl.gz
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [ -f data/${lang}/fbank/.cv-${lang}_validated.done ]; then
|
if [ $use_validated = true ] && [ -f data/${lang}/fbank/.cv-${lang}_validated.done ]; then
|
||||||
log "Also combine features for validated data"
|
log "Also combine features for validated data"
|
||||||
pieces=$(find data/${lang}/fbank/cv-${lang}_validated_split_${num_splits} -name "cv-${lang}_cuts_validated.*.jsonl.gz")
|
pieces=$(find data/${lang}/fbank/cv-${lang}_validated_split_${num_splits} -name "cv-${lang}_cuts_validated.*.jsonl.gz")
|
||||||
lhotse combine $pieces data/${lang}/fbank/cv-${lang}_cuts_validated.jsonl.gz
|
lhotse combine $pieces data/${lang}/fbank/cv-${lang}_cuts_validated.jsonl.gz
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [ -f data/${lang}/fbank/.cv-${lang}_invalidated.done ]; then
|
if [ $use_invalidated = true ] && [ -f data/${lang}/fbank/.cv-${lang}_invalidated.done ]; then
|
||||||
log "Also combine features for invalidated data"
|
log "Also combine features for invalidated data"
|
||||||
pieces=$(find data/${lang}/fbank/cv-${lang}_inalidated_split_${num_splits} -name "cv-${lang}_cuts_invalidated.*.jsonl.gz")
|
pieces=$(find data/${lang}/fbank/cv-${lang}_inalidated_split_${num_splits} -name "cv-${lang}_cuts_invalidated.*.jsonl.gz")
|
||||||
lhotse combine $pieces data/${lang}/fbank/cv-${lang}_cuts_invalidated.jsonl.gz
|
lhotse combine $pieces data/${lang}/fbank/cv-${lang}_cuts_invalidated.jsonl.gz
|
||||||
|
Loading…
x
Reference in New Issue
Block a user