#!/usr/bin/env bash

# fix segmentation fault reported in https://github.com/k2-fsa/icefall/issues/674
export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python

set -eou pipefail

stage=-1
stop_stage=100

dl_dir=$PWD/download

. shared/parse_options.sh || exit 1

vocab_sizes=(
  2000
)

# All files generated by this script are saved in "data".
# You can safely remove "data" and rerun this script to regenerate it.
mkdir -p data

log() {
  # This function is from espnet
  local fname=${BASH_SOURCE[1]##*/}
  echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*"
}

log "dl_dir: $dl_dir"

log "Dataset: musan"
if [ $stage -le 1 ] && [ $stop_stage -ge 1 ]; then
  log "Stage 1: Soft link fbank of musan"
  mkdir -p data/fbank
  if [ -e ../../librispeech/ASR/data/fbank/.musan.done ]; then
    cd data/fbank
    ln -svf $(realpath ../../../../librispeech/ASR/data/fbank/musan_feats) .
    ln -svf $(realpath ../../../../librispeech/ASR/data/fbank/musan_cuts.jsonl.gz) .
    cd ../..
  else
    log "Abort! Please run ../../librispeech/ASR/prepare.sh --stage 4 --stop-stage 4"
    exit 1
  fi
fi

log "Dataset: LibriSpeech"
if [ $stage -le 2 ] && [ $stop_stage -ge 2 ]; then
  log "Stage 2: Soft link fbank of LibriSpeech"
  mkdir -p data/fbank
  if [ -e ../../librispeech/ASR/data/fbank/.librispeech.done ]; then
    cd data/fbank
    ln -svf $(realpath ../../../../librispeech/ASR/data/fbank/librispeech_cuts*) .
    ln -svf $(realpath ../../../../librispeech/ASR/data/fbank/librispeech_feats*) .
    cd ../..
  else
    log "Abort! Please run ../../librispeech/ASR/prepare.sh --stage 3 --stop-stage 3"
    exit 1
  fi
fi

log "Dataset: AiShell-2"
if [ $stage -le 3 ] && [ $stop_stage -ge 3 ]; then
  log "Stage 3: Soft link fbank of AiShell-2"
  mkdir -p data/fbank
  if [ -e ../../aishell2/ASR/data/fbank/.aishell2.done ]; then
    cd data/fbank
    ln -svf $(realpath ../../../../aishell2/ASR/data/fbank/aishell2_cuts*) .
    ln -svf $(realpath ../../../../aishell2/ASR/data/fbank/aishell2_feats*) .
    cd ../..
  else
    log "Abort! Please run ../../aishell2/ASR/prepare.sh --stage 3 --stop-stage 3"
    exit 1
  fi
fi

if [ $stage -le 4 ] && [ $stop_stage -ge 4 ]; then
  log "Stage 4: preparation for training BPE model"
  mkdir -p data/fbank
  if [ ! -d ../../aishell2/ASR/data/lang_char ]; then
    log "Abort! Please run ../../aishell2/ASR/prepare.sh --stage 3 --stop-stage 3"
    exit 1
  fi

  if [ ! -d ../../librispeech/ASR/data/lang_phone ]; then
    log "Abort! Please run ../../librispeech/ASR/prepare.sh --stage 5 --stop-stage 5"
    exit 1
  fi

  if [ ! -d ../../librispeech/ASR/data/lang_bpe_500 ]; then
    log "Abort! Please run ../../librispeech/ASR/prepare.sh --stage 6 --stop-stage 6"
    exit 1
  fi

  cd data/
  ln -svf $(realpath ../../../aishell2/ASR/data/lang_char) .
  ln -svf $(realpath ../../../librispeech/ASR/data/lang_phone) .
  ln -svf $(realpath ../../../librispeech/ASR/data/lang_bpe_500) .
  cd ../

  for vocab_size in ${vocab_sizes[@]}; do
    lang_dir=data/lang_bbpe_${vocab_size}
    mkdir -p $lang_dir

    cat data/lang_char/text data/lang_bpe_500/transcript_words.txt \
      > $lang_dir/text

    ./local/prepare_for_bpe_model.py \
      --lang-dir ./$lang_dir \
      --text $lang_dir/text
  done 
fi

if [ $stage -le 5 ] && [ $stop_stage -ge 5 ]; then
  log "Stage 5: training BPE model"
  if [ ! -d data/lang_char ]; then
    log "Abort! Please run stage 4 first"
    exit 1
  fi

  if [ ! -d data/lang_phone ]; then
    log "Abort! Please run stage 4 first"
    exit 1
  fi

  for vocab_size in ${vocab_sizes[@]}; do
    lang_dir=data/lang_bbpe_${vocab_size}
    mkdir -p $lang_dir

    if [ ! -f $lang_dir/bpe.model ]; then
      log "Training BPE model with vocab size ${vocab_size}"
      python3 -m k2_cli.prepares.prep_bpe \
        --input-dir data/lang_char \
        --output-dir data/lang_char \
        --vocab-size ${vocab_size}
    fi

    if [ ! -f data/lang_phone/bpe${vocab_size}.model ]; then
      log "Training BPE model with vocab size ${vocab_size}"
      python3 -m k2_cli.prepares.prep_bpe \
        --input-dir data/lang_phone \
        --output-dir data/lang_phone \
        --vocab-size ${vocab_size}
    fi
  done
fi