icefall/egs/must_c/ST/prepare.sh

#!/usr/bin/env bash

# fix segmentation fault reported in https://github.com/k2-fsa/icefall/issues/674
export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python

set -eou pipefail

nj=10
stage=0
stop_stage=100

version=v1.0
tgt_lang=de
dl_dir=$PWD/download

must_c_dir=$dl_dir/must-c/$version/en-$tgt_lang/data

# We assume dl_dir (download dir) contains the following
# directories and files.
#  - $dl_dir/must-c/$version/en-$tgt_lang/data/{dev,train,tst-COMMON,tst-HE}
#
# Please go to https://ict.fbk.eu/must-c-releases/
# to download and untar the dataset if you have not already done this.

#  - $dl_dir/musan
#      This directory contains the following directories downloaded from
#       http://www.openslr.org/17/
#
#     - music
#     - noise
#     - speech

. shared/parse_options.sh || exit 1

# vocab size for sentence piece models.
# It will generate
#  data/lang_bpe_${tgt_lang}_xxx
#  data/lang_bpe_${tgt_lang}_yyy
# if the array contains xxx, yyy
vocab_sizes=(
  # 5000
  # 2000
  # 1000
  500
)

# All files generated by this script are saved in "data".
# You can safely remove "data" and rerun this script to regenerate it.
mkdir -p data

log() {
  # This function is from espnet
  local fname=${BASH_SOURCE[1]##*/}
  echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*"
}

log "dl_dir: $dl_dir"

if [ ! -d $must_c_dir ]; then
  log "$must_c_dir does not exist"
  exit 1
fi

for d in dev train tst-COMMON tst-HE; do
  if [ ! -d $must_c_dir/$d ]; then
    log "$must_c_dir/$d does not exist!"
    exit 1
  fi
done

if [ $stage -le 0 ] && [ $stop_stage -ge 0 ]; then
  log "Stage 0: Download musan"
  if [ ! -d $dl_dir/musan ]; then
    lhotse download musan $dl_dir
  fi
fi

if [ $stage -le 1 ] && [ $stop_stage -ge 1 ]; then
  log "Stage 1: Prepare musan manifest"
  # We assume that you have downloaded the musan corpus
  # to $dl_dir/musan
  mkdir -p data/manifests
  if [ ! -e data/manifests/.musan.done ]; then
    lhotse prepare musan $dl_dir/musan data/manifests
    touch data/manifests/.musan.done
  fi
fi

if [ $stage -le 2 ] && [ $stop_stage -ge 2 ]; then
  log "Stage 2: Prepare must-c $version manifest for target language $tgt_lang"
  mkdir -p data/manifests/$version
  if [ ! -e data/manifests/$version/.${tgt_lang}.manifests.done ]; then
    lhotse prepare must-c \
      -j $nj \
      --tgt-lang $tgt_lang \
      $dl_dir/must-c/$version/ \
      data/manifests/$version/

    touch data/manifests/$version/.${tgt_lang}.manifests.done
  fi
fi

if [ $stage -le 3 ] && [ $stop_stage -ge 3 ]; then
  log "Stage 3: Text normalization for $version with target language $tgt_lang"
  if [ ! -f ./data/manifests/$version/.$tgt_lang.norm.done ]; then
    ./local/preprocess_must_c.py \
      --manifest-dir ./data/manifests/$version/ \
      --tgt-lang $tgt_lang
    touch ./data/manifests/$version/.$tgt_lang.norm.done
  fi
fi

if [ $stage -le 4 ] && [ $stop_stage -ge 4 ]; then
  log "Stage 4: Compute fbank for musan"
  mkdir -p data/fbank
  if [ ! -e data/fbank/.musan.done ]; then
    ./local/compute_fbank_musan.py
    touch data/fbank/.musan.done
  fi
fi

if [ $stage -le 5 ] && [ $stop_stage -ge 5 ]; then
  log "Stage 5: Compute fbank for $version with target language $tgt_lang"
  mkdir -p data/fbank/$version/
  if [ ! -e data/fbank/$version/.$tgt_lang.done ]; then
    ./local/compute_fbank_must_c.py \
      --in-dir ./data/manifests/$version/ \
      --out-dir ./data/fbank/$version/ \
      --tgt-lang $tgt_lang \
      --num-jobs $nj

    ./local/compute_fbank_must_c.py \
      --in-dir ./data/manifests/$version/ \
      --out-dir ./data/fbank/$version/ \
      --tgt-lang $tgt_lang \
      --num-jobs $nj \
      --perturb-speed 1

    touch data/fbank/$version/.$tgt_lang.done
  fi
fi

if [ $stage -le 6 ] && [ $stop_stage -ge 6 ]; then
  log "Stage 6: Prepare BPE based lang for $version with target language $tgt_lang"

  for vocab_size in ${vocab_sizes[@]}; do
    lang_dir=data/lang_bpe_${vocab_size}/$version/$tgt_lang/
    mkdir -p $lang_dir
    if [ ! -f $lang_dir/transcript_words.txt ]; then
      ./local/get_text.py ./data/fbank/$version/must_c_feats_en-${tgt_lang}_train.jsonl.gz > $lang_dir/transcript_words.txt
    fi

    if [ ! -f $lang_dir/words.txt ]; then
      ./local/get_words.py $lang_dir/transcript_words.txt > $lang_dir/words.txt
    fi

    if [ ! -f $lang_dir/bpe.model ]; then
      ./local/train_bpe_model.py \
        --lang-dir $lang_dir \
        --vocab-size $vocab_size \
        --transcript $lang_dir/transcript_words.txt
    fi

    if [ ! -f $lang_dir/L_disambig.pt ]; then
      ./local/prepare_lang_bpe.py --lang-dir $lang_dir

      log "Validating $lang_dir/lexicon.txt"
      ./local/validate_bpe_lexicon.py \
        --lexicon $lang_dir/lexicon.txt \
        --bpe-model $lang_dir/bpe.model
    fi
  done
fi