Merge branch 'master' into spgispeech_zipformer

.flake8

@@ -15,7 +15,7 @@ per-file-ignores =
     egs/librispeech/ASR/zipformer_mmi/*.py: E501, E203
     egs/librispeech/ASR/zipformer/*.py: E501, E203
     egs/librispeech/ASR/RESULTS.md: E999,
-
+    egs/ljspeech/TTS/vits/*.py: E501, E203
     # invalid escape sequence (cause by tex formular), W605
     icefall/utils.py: E501, W605
 

.github/scripts/aishell/ASR/run.sh

@@ -0,0 +1,343 @@
+#!/usr/bin/env bash
+
+set -ex
+
+log() {
+  # This function is from espnet
+  local fname=${BASH_SOURCE[1]##*/}
+  echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*"
+}
+
+cd egs/aishell/ASR
+
+function download_test_dev_manifests() {
+  git lfs install
+
+  fbank_url=https://huggingface.co/csukuangfj/aishell-test-dev-manifests
+  log "Downloading pre-commputed fbank from $fbank_url"
+
+  git clone https://huggingface.co/csukuangfj/aishell-test-dev-manifests
+  ln -s $PWD/aishell-test-dev-manifests/data .
+}
+
+function test_transducer_stateless3_2022_06_20() {
+  repo_url=https://huggingface.co/csukuangfj/icefall-aishell-pruned-transducer-stateless3-2022-06-20
+  log "Downloading pre-trained model from $repo_url"
+  git clone $repo_url
+  repo=$(basename $repo_url)
+
+  log "Display test files"
+  tree $repo/
+  ls -lh $repo/test_wavs/*.wav
+
+  pushd $repo/exp
+  ln -s pretrained-epoch-29-avg-5-torch-1.10.0.pt pretrained.pt
+  popd
+
+  log "test greedy_search with pretrained.py"
+
+  for sym in 1 2 3; do
+    log "Greedy search with --max-sym-per-frame $sym"
+
+    ./pruned_transducer_stateless3/pretrained.py \
+      --method greedy_search \
+      --max-sym-per-frame $sym \
+      --checkpoint $repo/exp/pretrained.pt \
+      --lang-dir $repo/data/lang_char \
+      $repo/test_wavs/BAC009S0764W0121.wav \
+      $repo/test_wavs/BAC009S0764W0122.wav \
+      $repo/test_wavs/BAC009S0764W0123.wav
+  done
+
+  log "test beam search with pretrained.py"
+
+  for method in modified_beam_search beam_search fast_beam_search; do
+    log "$method"
+
+    ./pruned_transducer_stateless3/pretrained.py \
+      --method $method \
+      --beam-size 4 \
+      --checkpoint $repo/exp/pretrained.pt \
+      --lang-dir $repo/data/lang_char \
+      $repo/test_wavs/BAC009S0764W0121.wav \
+      $repo/test_wavs/BAC009S0764W0122.wav \
+      $repo/test_wavs/BAC009S0764W0123.wav
+  done
+
+  echo "GITHUB_EVENT_NAME: ${GITHUB_EVENT_NAME}"
+  echo "GITHUB_EVENT_LABEL_NAME: ${GITHUB_EVENT_LABEL_NAME}"
+  if [[ x"${GITHUB_EVENT_NAME}" == x"schedule" || x"${GITHUB_EVENT_LABEL_NAME}" == x"run-decode"  ]]; then
+    mkdir -p pruned_transducer_stateless3/exp
+    ln -s $PWD/$repo/exp/pretrained.pt pruned_transducer_stateless3/exp/epoch-999.pt
+    ln -s $PWD/$repo/data/lang_char data/
+
+    ls -lh data
+    ls -lh pruned_transducer_stateless3/exp
+
+    log "Decoding test and dev"
+
+    # use a small value for decoding with CPU
+    max_duration=100
+
+    for method in greedy_search fast_beam_search modified_beam_search; do
+      log "Decoding with $method"
+
+      ./pruned_transducer_stateless3/decode.py \
+        --decoding-method $method \
+        --epoch 999 \
+        --avg 1 \
+        --max-duration $max_duration \
+        --exp-dir pruned_transducer_stateless3/exp
+    done
+
+    rm pruned_transducer_stateless3/exp/*.pt
+  fi
+
+  rm -rf $repo
+}
+
+function test_zipformer_large_2023_10_24() {
+  log "CI testing large model"
+  repo_url=https://huggingface.co/zrjin/icefall-asr-aishell-zipformer-large-2023-10-24/
+  log "Downloading pre-trained model from $repo_url"
+  git clone $repo_url
+  repo=$(basename $repo_url)
+
+  log "Display test files"
+  tree $repo/
+  ls -lh $repo/test_wavs/*.wav
+
+  for method in modified_beam_search greedy_search fast_beam_search; do
+    log "$method"
+
+    ./zipformer/pretrained.py \
+      --method $method \
+      --context-size 1 \
+      --checkpoint $repo/exp/pretrained.pt \
+      --tokens $repo/data/lang_char/tokens.txt \
+      --num-encoder-layers 2,2,4,5,4,2 \
+      --feedforward-dim 512,768,1536,2048,1536,768 \
+      --encoder-dim 192,256,512,768,512,256 \
+      --encoder-unmasked-dim 192,192,256,320,256,192 \
+      $repo/test_wavs/BAC009S0764W0121.wav \
+      $repo/test_wavs/BAC009S0764W0122.wav \
+      $repo/test_wavs/BAC009S0764W0123.wav
+  done
+  rm -rf $repo
+}
+
+function test_zipformer_2023_10_24() {
+  repo_url=https://huggingface.co/zrjin/icefall-asr-aishell-zipformer-2023-10-24/
+  log "Downloading pre-trained model from $repo_url"
+  git clone $repo_url
+  repo=$(basename $repo_url)
+
+  log "Display test files"
+  tree $repo/
+  ls -lh $repo/test_wavs/*.wav
+
+
+  for method in modified_beam_search greedy_search fast_beam_search; do
+    log "$method"
+
+    ./zipformer/pretrained.py \
+      --method $method \
+      --context-size 1 \
+      --checkpoint $repo/exp/pretrained.pt \
+      --tokens $repo/data/lang_char/tokens.txt \
+      $repo/test_wavs/BAC009S0764W0121.wav \
+      $repo/test_wavs/BAC009S0764W0122.wav \
+      $repo/test_wavs/BAC009S0764W0123.wav
+  done
+  rm -rf $repo
+}
+
+function test_zipformer_small_2023_10_24() {
+  log "CI testing small model"
+  repo_url=https://huggingface.co/zrjin/icefall-asr-aishell-zipformer-small-2023-10-24/
+  log "Downloading pre-trained model from $repo_url"
+  git clone $repo_url
+  repo=$(basename $repo_url)
+
+  log "Display test files"
+  tree $repo/
+  ls -lh $repo/test_wavs/*.wav
+
+
+  for method in modified_beam_search greedy_search fast_beam_search; do
+    log "$method"
+
+    ./zipformer/pretrained.py \
+      --method $method \
+      --context-size 1 \
+      --checkpoint $repo/exp/pretrained.pt \
+      --tokens $repo/data/lang_char/tokens.txt \
+      --num-encoder-layers 2,2,2,2,2,2 \
+      --feedforward-dim 512,768,768,768,768,768 \
+      --encoder-dim 192,256,256,256,256,256 \
+      --encoder-unmasked-dim 192,192,192,192,192,192 \
+      $repo/test_wavs/BAC009S0764W0121.wav \
+      $repo/test_wavs/BAC009S0764W0122.wav \
+      $repo/test_wavs/BAC009S0764W0123.wav
+  done
+  rm -rf $repo
+}
+
+function test_transducer_stateless_modified_2022_03_01() {
+  repo_url=https://huggingface.co/csukuangfj/icefall-aishell-transducer-stateless-modified-2022-03-01
+
+  log "Downloading pre-trained model from $repo_url"
+  git lfs install
+  git clone $repo_url
+  repo=$(basename $repo_url)
+
+  log "Display test files"
+  tree $repo/
+  ls -lh $repo/test_wavs/*.wav
+
+  for sym in 1 2 3; do
+    log "Greedy search with --max-sym-per-frame $sym"
+
+    ./transducer_stateless_modified/pretrained.py \
+      --method greedy_search \
+      --max-sym-per-frame $sym \
+      --checkpoint $repo/exp/pretrained.pt \
+      --lang-dir $repo/data/lang_char \
+      $repo/test_wavs/BAC009S0764W0121.wav \
+      $repo/test_wavs/BAC009S0764W0122.wav \
+      $repo/test_wavs/BAC009S0764W0123.wav
+  done
+
+  for method in modified_beam_search beam_search; do
+    log "$method"
+
+    ./transducer_stateless_modified/pretrained.py \
+      --method $method \
+      --beam-size 4 \
+      --checkpoint $repo/exp/pretrained.pt \
+      --lang-dir $repo/data/lang_char \
+      $repo/test_wavs/BAC009S0764W0121.wav \
+      $repo/test_wavs/BAC009S0764W0122.wav \
+      $repo/test_wavs/BAC009S0764W0123.wav
+  done
+  rm -rf $repo
+}
+
+function test_transducer_stateless_modified_2_2022_03_01() {
+  repo_url=https://huggingface.co/csukuangfj/icefall-aishell-transducer-stateless-modified-2-2022-03-01
+
+  log "Downloading pre-trained model from $repo_url"
+  git lfs install
+  git clone $repo_url
+  repo=$(basename $repo_url)
+
+  log "Display test files"
+  tree $repo/
+  ls -lh $repo/test_wavs/*.wav
+
+  for sym in 1 2 3; do
+    log "Greedy search with --max-sym-per-frame $sym"
+
+    ./transducer_stateless_modified-2/pretrained.py \
+      --method greedy_search \
+      --max-sym-per-frame $sym \
+      --checkpoint $repo/exp/pretrained.pt \
+      --lang-dir $repo/data/lang_char \
+      $repo/test_wavs/BAC009S0764W0121.wav \
+      $repo/test_wavs/BAC009S0764W0122.wav \
+      $repo/test_wavs/BAC009S0764W0123.wav
+  done
+
+  for method in modified_beam_search beam_search; do
+    log "$method"
+
+    ./transducer_stateless_modified-2/pretrained.py \
+      --method $method \
+      --beam-size 4 \
+      --checkpoint $repo/exp/pretrained.pt \
+      --lang-dir $repo/data/lang_char \
+      $repo/test_wavs/BAC009S0764W0121.wav \
+      $repo/test_wavs/BAC009S0764W0122.wav \
+      $repo/test_wavs/BAC009S0764W0123.wav
+  done
+  rm -rf $repo
+}
+
+function test_conformer_ctc() {
+  repo_url=https://huggingface.co/csukuangfj/icefall_asr_aishell_conformer_ctc
+  log "Downloading pre-trained model from $repo_url"
+  GIT_LFS_SKIP_SMUDGE=1 git clone $repo_url
+  repo=$(basename $repo_url)
+  pushd $repo
+
+  git lfs pull --include "exp/pretrained.pt"
+  git lfs pull --include "data/lang_char/H.fst"
+  git lfs pull --include "data/lang_char/HL.fst"
+  git lfs pull --include "data/lang_char/HLG.fst"
+
+  popd
+
+  log "Display test files"
+  tree $repo/
+  ls -lh $repo/test_wavs/*.wav
+
+  log "CTC decoding"
+
+  log "Exporting model with torchscript"
+
+  pushd $repo/exp
+  ln -s pretrained.pt epoch-99.pt
+  popd
+
+  ./conformer_ctc/export.py \
+    --epoch 99 \
+    --avg 1 \
+    --exp-dir $repo/exp \
+    --tokens $repo/data/lang_char/tokens.txt \
+    --jit 1
+
+  ls -lh $repo/exp
+
+  ls -lh $repo/data/lang_char
+
+  log "Decoding with H on CPU with OpenFst"
+
+  ./conformer_ctc/jit_pretrained_decode_with_H.py \
+    --nn-model $repo/exp/cpu_jit.pt \
+    --H $repo/data/lang_char/H.fst \
+    --tokens $repo/data/lang_char/tokens.txt \
+    $repo/test_wavs/0.wav \
+    $repo/test_wavs/1.wav \
+    $repo/test_wavs/2.wav
+
+  log "Decoding with HL on CPU with OpenFst"
+
+  ./conformer_ctc/jit_pretrained_decode_with_HL.py \
+    --nn-model $repo/exp/cpu_jit.pt \
+    --HL $repo/data/lang_char/HL.fst \
+    --words $repo/data/lang_char/words.txt \
+    $repo/test_wavs/0.wav \
+    $repo/test_wavs/1.wav \
+    $repo/test_wavs/2.wav
+
+  log "Decoding with HLG on CPU with OpenFst"
+
+  ./conformer_ctc/jit_pretrained_decode_with_HLG.py \
+    --nn-model $repo/exp/cpu_jit.pt \
+    --HLG $repo/data/lang_char/HLG.fst \
+    --words $repo/data/lang_char/words.txt \
+    $repo/test_wavs/0.wav \
+    $repo/test_wavs/1.wav \
+    $repo/test_wavs/2.wav
+
+  rm -rf $repo
+}
+
+download_test_dev_manifests
+test_transducer_stateless3_2022_06_20
+test_zipformer_large_2023_10_24
+test_zipformer_2023_10_24
+test_zipformer_small_2023_10_24
+test_transducer_stateless_modified_2022_03_01
+test_transducer_stateless_modified_2_2022_03_01
+# test_conformer_ctc # fails for torch 1.13.x and torch 2.0.x

.github/scripts/docker/Dockerfile

@@ -0,0 +1,63 @@
+ARG PYTHON_VERSION=3.8
+FROM python:${PYTHON_VERSION}
+
+ARG TORCHAUDIO_VERSION="0.13.0"
+ARG TORCH_VERSION="1.13.0"
+ARG K2_VERSION="1.24.4.dev20231220"
+ARG KALDIFEAT_VERSION="1.25.3.dev20231221"
+
+ARG _K2_VERSION="${K2_VERSION}+cpu.torch${TORCH_VERSION}"
+ARG _KALDIFEAT_VERSION="${KALDIFEAT_VERSION}+cpu.torch${TORCH_VERSION}"
+
+RUN apt-get update -y && \
+    apt-get install -qq -y \
+    ffmpeg \
+    git \
+    git-lfs \
+    graphviz \
+    less \
+    tree \
+    vim \
+    && \
+    apt-get clean && \
+    rm -rf /var/cache/apt/archives /var/lib/apt/lists
+
+
+LABEL authors="Fangjun Kuang <csukuangfj@gmail.com>"
+LABEL k2_version=${_K2_VERSION}
+LABEL kaldifeat_version=${_KALDIFEAT_VERSION}
+LABEL github_repo="https://github.com/k2-fsa/icefall"
+
+# Install dependencies
+RUN pip install --no-cache-dir \
+      torch==${TORCH_VERSION} torchaudio==${TORCHAUDIO_VERSION} -f https://download.pytorch.org/whl/cpu/torch_stable.html \
+      k2==${_K2_VERSION} -f https://k2-fsa.github.io/k2/cpu.html \
+      \
+      git+https://github.com/lhotse-speech/lhotse \
+      kaldifeat==${_KALDIFEAT_VERSION} -f https://csukuangfj.github.io/kaldifeat/cpu.html \
+      dill \
+      graphviz \
+      kaldi-decoder \
+      kaldi_native_io \
+      kaldialign \
+      kaldifst \
+      kaldilm \
+      matplotlib \
+      multi_quantization \
+      numpy \
+      onnx \
+      onnxmltools \
+      onnxruntime \
+      pytest \
+      sentencepiece>=0.1.96 \
+      six \
+      tensorboard \
+      typeguard
+
+# RUN git clone https://github.com/k2-fsa/icefall /workspace/icefall && \
+#     cd /workspace/icefall && \
+#     pip install --no-cache-dir -r requirements.txt
+#
+# ENV PYTHONPATH /workspace/icefall:$PYTHONPATH
+#
+# WORKDIR /workspace/icefall

.github/scripts/docker/generate_build_matrix.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+# Copyright    2023  Xiaomi Corp.        (authors: Fangjun Kuang)
+
+
+import json
+
+
+def version_gt(a, b):
+    a_major, a_minor = a.split(".")[:2]
+    b_major, b_minor = b.split(".")[:2]
+    if a_major > b_major:
+        return True
+
+    if a_major == b_major and a_minor > b_minor:
+        return True
+
+    return False
+
+
+def version_ge(a, b):
+    a_major, a_minor = a.split(".")[:2]
+    b_major, b_minor = b.split(".")[:2]
+    if a_major > b_major:
+        return True
+
+    if a_major == b_major and a_minor >= b_minor:
+        return True
+
+    return False
+
+
+def get_torchaudio_version(torch_version):
+    if torch_version == "1.13.0":
+        return "0.13.0"
+    elif torch_version == "1.13.1":
+        return "0.13.1"
+    elif torch_version == "2.0.0":
+        return "2.0.1"
+    elif torch_version == "2.0.1":
+        return "2.0.2"
+    else:
+        return torch_version
+
+
+def get_matrix():
+    k2_version = "1.24.4.dev20231220"
+    kaldifeat_version = "1.25.3.dev20231221"
+    version = "1.2"
+    python_version = ["3.8", "3.9", "3.10", "3.11"]
+    torch_version = ["1.13.0", "1.13.1", "2.0.0", "2.0.1", "2.1.0", "2.1.1", "2.1.2"]
+
+    matrix = []
+    for p in python_version:
+        for t in torch_version:
+            # torchaudio <= 1.13.x supports only python <= 3.10
+
+            if version_gt(p, "3.10") and not version_gt(t, "2.0"):
+                continue
+
+            matrix.append(
+                {
+                    "k2-version": k2_version,
+                    "kaldifeat-version": kaldifeat_version,
+                    "version": version,
+                    "python-version": p,
+                    "torch-version": t,
+                    "torchaudio-version": get_torchaudio_version(t),
+                }
+            )
+    return matrix
+
+
+def main():
+    matrix = get_matrix()
+    print(json.dumps({"include": matrix}))
+
+
+if __name__ == "__main__":
+    main()

.github/scripts/multi-zh-hans.sh

@@ -0,0 +1,158 @@
+#!/usr/bin/env bash
+
+set -ex
+
+git config --global user.name "k2-fsa"
+git config --global user.email "csukuangfj@gmail.com"
+git config --global lfs.allowincompletepush true
+
+log() {
+  # This function is from espnet
+  local fname=${BASH_SOURCE[1]##*/}
+  echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*"
+}
+
+log "pwd: $PWD"
+
+cd egs/multi_zh-hans/ASR
+
+repo_url=https://huggingface.co/zrjin/icefall-asr-multi-zh-hans-zipformer-ctc-streaming-2023-11-05
+log "Downloading pre-trained model from $repo_url"
+GIT_LFS_SKIP_SMUDGE=1 git clone $repo_url
+repo=$(basename $repo_url)
+
+pushd $repo
+cd exp/
+git lfs pull --include pretrained.pt
+rm -fv epoch-20.pt
+rm -fv *.onnx
+ln -s pretrained.pt epoch-20.pt
+cd ../data/lang_bpe_2000
+ls -lh
+git lfs pull --include L.pt L_disambig.pt Linv.pt bpe.model
+git lfs pull --include "*.model"
+ls -lh
+popd
+
+log "----------------------------------------"
+log "Export streaming ONNX CTC models "
+log "----------------------------------------"
+./zipformer/export-onnx-streaming-ctc.py \
+  --exp-dir $repo/exp \
+  --tokens $repo/data/lang_bpe_2000/tokens.txt \
+  --causal 1 \
+  --avg 1 \
+  --epoch 20 \
+  --use-averaged-model 0 \
+  --chunk-size 16 \
+  --left-context-frames 128 \
+  --use-ctc 1
+
+ls -lh $repo/exp/
+
+log "------------------------------------------------------------"
+log "Test exported streaming ONNX CTC models (greedy search)     "
+log "------------------------------------------------------------"
+
+test_wavs=(
+DEV_T0000000000.wav
+DEV_T0000000001.wav
+DEV_T0000000002.wav
+TEST_MEETING_T0000000113.wav
+TEST_MEETING_T0000000219.wav
+TEST_MEETING_T0000000351.wav
+)
+
+for w in ${test_wavs[@]}; do
+  ./zipformer/onnx_pretrained-streaming-ctc.py \
+    --model-filename $repo/exp/ctc-epoch-20-avg-1-chunk-16-left-128.int8.onnx \
+    --tokens $repo/data/lang_bpe_2000/tokens.txt \
+    $repo/test_wavs/$w
+done
+
+log "Upload onnx CTC models to huggingface"
+url=https://huggingface.co/k2-fsa/sherpa-onnx-streaming-zipformer-ctc-multi-zh-hans-2023-12-13
+GIT_LFS_SKIP_SMUDGE=1 git clone $url
+dst=$(basename $url)
+cp -v $repo/exp/ctc*.onnx $dst
+cp -v $repo/data/lang_bpe_2000/tokens.txt $dst
+cp -v $repo/data/lang_bpe_2000/bpe.model $dst
+mkdir -p $dst/test_wavs
+cp -v $repo/test_wavs/*.wav $dst/test_wavs
+cd $dst
+git lfs track "*.onnx" "bpe.model"
+ls -lh
+file bpe.model
+git status
+git add .
+git commit -m "upload model" && git push https://k2-fsa:${HF_TOKEN}@huggingface.co/k2-fsa/$dst main || true
+
+log "Upload models to https://github.com/k2-fsa/sherpa-onnx"
+rm -rf .git
+rm -fv .gitattributes
+cd ..
+tar cjfv $dst.tar.bz2 $dst
+ls -lh *.tar.bz2
+mv -v $dst.tar.bz2 ../../../
+
+log "----------------------------------------"
+log "Export streaming ONNX transducer models "
+log "----------------------------------------"
+
+./zipformer/export-onnx-streaming.py \
+  --exp-dir $repo/exp \
+  --tokens $repo/data/lang_bpe_2000/tokens.txt \
+  --causal 1 \
+  --avg 1 \
+  --epoch 20 \
+  --use-averaged-model 0 \
+  --chunk-size 16 \
+  --left-context-frames 128 \
+  --use-ctc 0
+
+ls -lh $repo/exp
+
+log "------------------------------------------------------------"
+log "Test exported streaming ONNX transducer models (Python code)"
+log "------------------------------------------------------------"
+
+log "test fp32"
+./zipformer/onnx_pretrained-streaming.py \
+  --encoder-model-filename $repo/exp/encoder-epoch-20-avg-1-chunk-16-left-128.onnx \
+  --decoder-model-filename $repo/exp/decoder-epoch-20-avg-1-chunk-16-left-128.onnx \
+  --joiner-model-filename $repo/exp/joiner-epoch-20-avg-1-chunk-16-left-128.onnx \
+  --tokens $repo/data/lang_bpe_2000/tokens.txt \
+  $repo/test_wavs/DEV_T0000000000.wav
+
+log "test int8"
+./zipformer/onnx_pretrained-streaming.py \
+  --encoder-model-filename $repo/exp/encoder-epoch-20-avg-1-chunk-16-left-128.int8.onnx \
+  --decoder-model-filename $repo/exp/decoder-epoch-20-avg-1-chunk-16-left-128.onnx \
+  --joiner-model-filename $repo/exp/joiner-epoch-20-avg-1-chunk-16-left-128.int8.onnx \
+  --tokens $repo/data/lang_bpe_2000/tokens.txt \
+  $repo/test_wavs/DEV_T0000000000.wav
+
+log "Upload onnx transducer models to huggingface"
+
+url=https://huggingface.co/k2-fsa/sherpa-onnx-streaming-zipformer-multi-zh-hans-2023-12-12
+GIT_LFS_SKIP_SMUDGE=1 git clone $url
+dst=$(basename $url)
+cp -v $repo/exp/encoder*.onnx $dst
+cp -v $repo/exp/decoder*.onnx $dst
+cp -v $repo/exp/joiner*.onnx $dst
+cp -v $repo/data/lang_bpe_2000/tokens.txt $dst
+cp -v $repo/data/lang_bpe_2000/bpe.model $dst
+mkdir -p $dst/test_wavs
+cp -v $repo/test_wavs/*.wav $dst/test_wavs
+cd $dst
+git lfs track "*.onnx" bpe.model
+git add .
+git commit -m "upload model" && git push https://k2-fsa:${HF_TOKEN}@huggingface.co/k2-fsa/$dst main || true
+
+log "Upload models to https://github.com/k2-fsa/sherpa-onnx"
+rm -rf .git
+rm -fv .gitattributes
+cd ..
+tar cjfv $dst.tar.bz2 $dst
+ls -lh *.tar.bz2
+mv -v $dst.tar.bz2 ../../../

.github/scripts/run-aishell-pruned-transducer-stateless3-2022-06-20.sh

@@ -1,87 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-log() {
-  # This function is from espnet
-  local fname=${BASH_SOURCE[1]##*/}
-  echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*"
-}
-
-cd egs/aishell/ASR
-
-git lfs install
-
-fbank_url=https://huggingface.co/csukuangfj/aishell-test-dev-manifests
-log "Downloading pre-commputed fbank from $fbank_url"
-
-git clone https://huggingface.co/csukuangfj/aishell-test-dev-manifests
-ln -s $PWD/aishell-test-dev-manifests/data .
-
-log "Downloading pre-trained model from $repo_url"
-repo_url=https://huggingface.co/csukuangfj/icefall-aishell-pruned-transducer-stateless3-2022-06-20
-git clone $repo_url
-repo=$(basename $repo_url)
-
-log "Display test files"
-tree $repo/
-ls -lh $repo/test_wavs/*.wav
-
-pushd $repo/exp
-ln -s pretrained-epoch-29-avg-5-torch-1.10.0.pt pretrained.pt
-popd
-
-for sym in 1 2 3; do
-  log "Greedy search with --max-sym-per-frame $sym"
-
-  ./pruned_transducer_stateless3/pretrained.py \
-    --method greedy_search \
-    --max-sym-per-frame $sym \
-    --checkpoint $repo/exp/pretrained.pt \
-    --lang-dir $repo/data/lang_char \
-    $repo/test_wavs/BAC009S0764W0121.wav \
-    $repo/test_wavs/BAC009S0764W0122.wav \
-    $repo/test_wavs/BAC009S0764W0123.wav
-done
-
-for method in modified_beam_search beam_search fast_beam_search; do
-  log "$method"
-
-  ./pruned_transducer_stateless3/pretrained.py \
-    --method $method \
-    --beam-size 4 \
-    --checkpoint $repo/exp/pretrained.pt \
-    --lang-dir $repo/data/lang_char \
-    $repo/test_wavs/BAC009S0764W0121.wav \
-    $repo/test_wavs/BAC009S0764W0122.wav \
-    $repo/test_wavs/BAC009S0764W0123.wav
-done
-
-echo "GITHUB_EVENT_NAME: ${GITHUB_EVENT_NAME}"
-echo "GITHUB_EVENT_LABEL_NAME: ${GITHUB_EVENT_LABEL_NAME}"
-if [[ x"${GITHUB_EVENT_NAME}" == x"schedule" || x"${GITHUB_EVENT_LABEL_NAME}" == x"run-decode"  ]]; then
-  mkdir -p pruned_transducer_stateless3/exp
-  ln -s $PWD/$repo/exp/pretrained.pt pruned_transducer_stateless3/exp/epoch-999.pt
-  ln -s $PWD/$repo/data/lang_char data/
-
-  ls -lh data
-  ls -lh pruned_transducer_stateless3/exp
-
-  log "Decoding test and dev"
-
-  # use a small value for decoding with CPU
-  max_duration=100
-
-  for method in greedy_search fast_beam_search modified_beam_search; do
-    log "Decoding with $method"
-
-    ./pruned_transducer_stateless3/decode.py \
-      --decoding-method $method \
-      --epoch 999 \
-      --avg 1 \
-      --max-duration $max_duration \
-      --exp-dir pruned_transducer_stateless3/exp
-  done
-
-  rm pruned_transducer_stateless3/exp/*.pt
-fi

.github/scripts/run-gigaspeech-zipformer-2023-10-17.sh

@@ -26,16 +26,80 @@ git lfs pull --include "data/lang_bpe_500/bpe.model"
 git lfs pull --include "data/lang_bpe_500/tokens.txt"
 git lfs pull --include "exp/jit_script.pt"
 git lfs pull --include "exp/pretrained.pt"
-ln -s pretrained.pt epoch-99.pt
+rm epoch-30.pt
-ls -lh *.pt
+ln -s pretrained.pt epoch-30.pt
+rm *.onnx
+ls -lh
 popd
 
+log "----------------------------------------"
+log "Export ONNX transducer models "
+log "----------------------------------------"
+
+./zipformer/export-onnx.py \
+  --tokens $repo/data/lang_bpe_500/tokens.txt \
+  --use-averaged-model 0 \
+  --epoch 30 \
+  --avg 1 \
+  --exp-dir $repo/exp
+
+ls -lh $repo/exp
+
+log "------------------------------------------------------------"
+log "Test exported ONNX transducer models (Python code)          "
+log "------------------------------------------------------------"
+
+log "test fp32"
+./zipformer/onnx_pretrained.py \
+  --encoder-model-filename $repo/exp/encoder-epoch-30-avg-1.onnx \
+  --decoder-model-filename $repo/exp/decoder-epoch-30-avg-1.onnx \
+  --joiner-model-filename $repo/exp/joiner-epoch-30-avg-1.onnx \
+  --tokens $repo/data/lang_bpe_500/tokens.txt \
+  $repo/test_wavs/1089-134686-0001.wav \
+  $repo/test_wavs/1221-135766-0001.wav \
+  $repo/test_wavs/1221-135766-0002.wav
+
+log "test int8"
+./zipformer/onnx_pretrained.py \
+  --encoder-model-filename $repo/exp/encoder-epoch-30-avg-1.int8.onnx \
+  --decoder-model-filename $repo/exp/decoder-epoch-30-avg-1.onnx \
+  --joiner-model-filename $repo/exp/joiner-epoch-30-avg-1.int8.onnx \
+  --tokens $repo/data/lang_bpe_500/tokens.txt \
+  $repo/test_wavs/1089-134686-0001.wav \
+  $repo/test_wavs/1221-135766-0001.wav \
+  $repo/test_wavs/1221-135766-0002.wav
+
+log "Upload models to huggingface"
+git config --global user.name "k2-fsa"
+git config --global user.email "xxx@gmail.com"
+
+url=https://huggingface.co/k2-fsa/sherpa-onnx-zipformer-gigaspeech-2023-12-12
+GIT_LFS_SKIP_SMUDGE=1 git clone $url
+dst=$(basename $url)
+cp -v $repo/exp/*.onnx $dst
+cp -v $repo/data/lang_bpe_500/tokens.txt $dst
+cp -v $repo/data/lang_bpe_500/bpe.model $dst
+mkdir -p $dst/test_wavs
+cp -v $repo/test_wavs/*.wav $dst/test_wavs
+cd $dst
+git lfs track "*.onnx"
+git add .
+git commit -m "upload model" && git push https://k2-fsa:${HF_TOKEN}@huggingface.co/k2-fsa/$dst main || true
+
+log "Upload models to https://github.com/k2-fsa/sherpa-onnx"
+rm -rf .git
+rm -fv .gitattributes
+cd ..
+tar cjfv $dst.tar.bz2 $dst
+ls -lh
+mv -v $dst.tar.bz2 ../../../
+
 log "Export to torchscript model"
 ./zipformer/export.py \
   --exp-dir $repo/exp \
   --use-averaged-model false \
   --tokens $repo/data/lang_bpe_500/tokens.txt \
-  --epoch 99 \
+  --epoch 30 \
   --avg 1 \
   --jit 1
 
@@ -67,7 +131,7 @@ echo "GITHUB_EVENT_NAME: ${GITHUB_EVENT_NAME}"
 echo "GITHUB_EVENT_LABEL_NAME: ${GITHUB_EVENT_LABEL_NAME}"
 if [[ x"${GITHUB_EVENT_NAME}" == x"schedule" || x"${GITHUB_EVENT_LABEL_NAME}" == x"run-decode"  ]]; then
   mkdir -p zipformer/exp
-  ln -s $PWD/$repo/exp/pretrained.pt zipformer/exp/epoch-999.pt
+  ln -s $PWD/$repo/exp/pretrained.pt zipformer/exp/epoch-30.pt
   ln -s $PWD/$repo/data/lang_bpe_500 data/
 
   ls -lh data
@@ -83,7 +147,7 @@ if [[ x"${GITHUB_EVENT_NAME}" == x"schedule" || x"${GITHUB_EVENT_LABEL_NAME}" ==
 
     ./zipformer/decode.py \
       --decoding-method $method \
-      --epoch 999 \
+      --epoch 30 \
       --avg 1 \
       --use-averaged-model 0 \
       --max-duration $max_duration \

.github/scripts/run-librispeech-conformer-ctc3-2022-11-28.sh

@@ -1,122 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-log() {
-  # This function is from espnet
-  local fname=${BASH_SOURCE[1]##*/}
-  echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*"
-}
-
-cd egs/librispeech/ASR
-
-repo_url=https://huggingface.co/Zengwei/icefall-asr-librispeech-conformer-ctc3-2022-11-27
-
-log "Downloading pre-trained model from $repo_url"
-GIT_LFS_SKIP_SMUDGE=1 git clone $repo_url
-repo=$(basename $repo_url)
-
-log "Display test files"
-tree $repo/
-ls -lh $repo/test_wavs/*.wav
-
-pushd $repo/exp
-git lfs pull --include "data/lang_bpe_500/HLG.pt"
-git lfs pull --include "data/lang_bpe_500/L.pt"
-git lfs pull --include "data/lang_bpe_500/LG.pt"
-git lfs pull --include "data/lang_bpe_500/Linv.pt"
-git lfs pull --include "data/lang_bpe_500/bpe.model"
-git lfs pull --include "data/lm/G_4_gram.pt"
-git lfs pull --include "exp/jit_trace.pt"
-git lfs pull --include "exp/pretrained.pt"
-ln -s pretrained.pt epoch-99.pt
-ls -lh *.pt
-popd
-
-log "Decode with models exported by torch.jit.trace()"
-
-for m in ctc-decoding 1best; do
-  ./conformer_ctc3/jit_pretrained.py \
-    --model-filename $repo/exp/jit_trace.pt \
-    --words-file $repo/data/lang_bpe_500/words.txt \
-    --HLG $repo/data/lang_bpe_500/HLG.pt \
-    --bpe-model $repo/data/lang_bpe_500/bpe.model \
-    --G $repo/data/lm/G_4_gram.pt \
-    --method $m \
-    --sample-rate 16000 \
-    $repo/test_wavs/1089-134686-0001.wav \
-    $repo/test_wavs/1221-135766-0001.wav \
-    $repo/test_wavs/1221-135766-0002.wav
-done
-
-log "Export to torchscript model"
-
-./conformer_ctc3/export.py \
-  --exp-dir $repo/exp \
-  --tokens $repo/data/lang_bpe_500/tokens.txt \
-  --jit-trace 1 \
-  --epoch 99 \
-  --avg 1 \
-  --use-averaged-model 0
-
-ls -lh $repo/exp/*.pt
-
-log "Decode with models exported by torch.jit.trace()"
-
-for m in ctc-decoding 1best; do
-  ./conformer_ctc3/jit_pretrained.py \
-    --model-filename $repo/exp/jit_trace.pt \
-    --words-file $repo/data/lang_bpe_500/words.txt  \
-    --HLG $repo/data/lang_bpe_500/HLG.pt \
-    --bpe-model $repo/data/lang_bpe_500/bpe.model \
-    --G $repo/data/lm/G_4_gram.pt \
-    --method $m \
-    --sample-rate 16000 \
-    $repo/test_wavs/1089-134686-0001.wav \
-    $repo/test_wavs/1221-135766-0001.wav \
-    $repo/test_wavs/1221-135766-0002.wav
-done
-
-for m in ctc-decoding 1best; do
-  ./conformer_ctc3/pretrained.py \
-    --checkpoint $repo/exp/pretrained.pt \
-    --words-file $repo/data/lang_bpe_500/words.txt \
-    --HLG $repo/data/lang_bpe_500/HLG.pt \
-    --tokens $repo/data/lang_bpe_500/tokens.txt \
-    --G $repo/data/lm/G_4_gram.pt \
-    --method $m \
-    --sample-rate 16000 \
-    $repo/test_wavs/1089-134686-0001.wav \
-    $repo/test_wavs/1221-135766-0001.wav \
-    $repo/test_wavs/1221-135766-0002.wav
-done
-
-echo "GITHUB_EVENT_NAME: ${GITHUB_EVENT_NAME}"
-echo "GITHUB_EVENT_LABEL_NAME: ${GITHUB_EVENT_LABEL_NAME}"
-if [[ x"${GITHUB_EVENT_NAME}" == x"schedule" || x"${GITHUB_EVENT_LABEL_NAME}" == x"run-decode" ]]; then
-  mkdir -p conformer_ctc3/exp
-  ln -s $PWD/$repo/exp/pretrained.pt conformer_ctc3/exp/epoch-999.pt
-  ln -s $PWD/$repo/data/lang_bpe_500 data/
-
-  ls -lh data
-  ls -lh conformer_ctc3/exp
-
-  log "Decoding test-clean and test-other"
-
-  # use a small value for decoding with CPU
-  max_duration=100
-
-  for method in ctc-decoding 1best; do
-    log "Decoding with $method"
-    ./conformer_ctc3/decode.py \
-      --epoch 999 \
-      --avg 1 \
-      --use-averaged-model 0 \
-      --exp-dir conformer_ctc3/exp/ \
-      --max-duration $max_duration \
-      --decoding-method $method \
-      --lm-dir data/lm
-  done
-
-  rm conformer_ctc3/exp/*.pt
-fi

.github/scripts/run-librispeech-pruned-transducer-stateless-2022-03-12.sh

@@ -1,77 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-log() {
-  # This function is from espnet
-  local fname=${BASH_SOURCE[1]##*/}
-  echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*"
-}
-
-cd egs/librispeech/ASR
-
-repo_url=https://huggingface.co/csukuangfj/icefall-asr-librispeech-pruned-transducer-stateless-2022-03-12
-
-log "Downloading pre-trained model from $repo_url"
-git lfs install
-git clone $repo_url
-repo=$(basename $repo_url)
-
-log "Display test files"
-tree $repo/
-ls -lh $repo/test_wavs/*.wav
-
-for sym in 1 2 3; do
-  log "Greedy search with --max-sym-per-frame $sym"
-
-  ./pruned_transducer_stateless/pretrained.py \
-    --method greedy_search \
-    --max-sym-per-frame $sym \
-    --checkpoint $repo/exp/pretrained.pt \
-    --tokens $repo/data/lang_bpe_500/tokens.txt \
-    $repo/test_wavs/1089-134686-0001.wav \
-    $repo/test_wavs/1221-135766-0001.wav \
-    $repo/test_wavs/1221-135766-0002.wav
-done
-
-for method in fast_beam_search modified_beam_search beam_search; do
-  log "$method"
-
-  ./pruned_transducer_stateless/pretrained.py \
-    --method $method \
-    --beam-size 4 \
-    --checkpoint $repo/exp/pretrained.pt \
-    --tokens $repo/data/lang_bpe_500/tokens.txt \
-    $repo/test_wavs/1089-134686-0001.wav \
-    $repo/test_wavs/1221-135766-0001.wav \
-    $repo/test_wavs/1221-135766-0002.wav
-done
-
-echo "GITHUB_EVENT_NAME: ${GITHUB_EVENT_NAME}"
-echo "GITHUB_EVENT_LABEL_NAME: ${GITHUB_EVENT_LABEL_NAME}"
-if [[ x"${GITHUB_EVENT_NAME}" == x"schedule" || x"${GITHUB_EVENT_LABEL_NAME}" == x"run-decode"  ]]; then
-  mkdir -p pruned_transducer_stateless/exp
-  ln -s $PWD/$repo/exp/pretrained.pt pruned_transducer_stateless/exp/epoch-999.pt
-  ln -s $PWD/$repo/data/lang_bpe_500 data/
-
-  ls -lh data
-  ls -lh pruned_transducer_stateless/exp
-
-  log "Decoding test-clean and test-other"
-
-  # use a small value for decoding with CPU
-  max_duration=100
-
-  for method in greedy_search fast_beam_search modified_beam_search; do
-    log "Decoding with $method"
-
-    ./pruned_transducer_stateless/decode.py \
-      --decoding-method $method \
-      --epoch 999 \
-      --avg 1 \
-      --max-duration $max_duration \
-      --exp-dir pruned_transducer_stateless/exp
-  done
-
-  rm pruned_transducer_stateless/exp/*.pt
-fi

.github/scripts/run-librispeech-pruned-transducer-stateless2-2022-04-29.sh

@@ -1,86 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-log() {
-  # This function is from espnet
-  local fname=${BASH_SOURCE[1]##*/}
-  echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*"
-}
-
-cd egs/librispeech/ASR
-
-repo_url=https://huggingface.co/csukuangfj/icefall-asr-librispeech-pruned-transducer-stateless2-2022-04-29
-
-log "Downloading pre-trained model from $repo_url"
-GIT_LFS_SKIP_SMUDGE=1 git clone $repo_url
-repo=$(basename $repo_url)
-
-pushd $repo
-git lfs pull --include "data/lang_bpe_500/bpe.model"
-git lfs pull --include "exp/pretrained-epoch-38-avg-10.pt"
-popd
-
-log "Display test files"
-tree $repo/
-ls -lh $repo/test_wavs/*.wav
-
-pushd $repo/exp
-ln -s pretrained-epoch-38-avg-10.pt pretrained.pt
-popd
-
-for sym in 1 2 3; do
-  log "Greedy search with --max-sym-per-frame $sym"
-
-  ./pruned_transducer_stateless2/pretrained.py \
-    --method greedy_search \
-    --max-sym-per-frame $sym \
-    --checkpoint $repo/exp/pretrained.pt \
-    --tokens $repo/data/lang_bpe_500/tokens.txt \
-    $repo/test_wavs/1089-134686-0001.wav \
-    $repo/test_wavs/1221-135766-0001.wav \
-    $repo/test_wavs/1221-135766-0002.wav
-done
-
-for method in modified_beam_search beam_search fast_beam_search; do
-  log "$method"
-
-  ./pruned_transducer_stateless2/pretrained.py \
-    --method $method \
-    --beam-size 4 \
-    --checkpoint $repo/exp/pretrained.pt \
-    --tokens $repo/data/lang_bpe_500/tokens.txt \
-    $repo/test_wavs/1089-134686-0001.wav \
-    $repo/test_wavs/1221-135766-0001.wav \
-    $repo/test_wavs/1221-135766-0002.wav
-done
-
-echo "GITHUB_EVENT_NAME: ${GITHUB_EVENT_NAME}"
-echo "GITHUB_EVENT_LABEL_NAME: ${GITHUB_EVENT_LABEL_NAME}"
-if [[ x"${GITHUB_EVENT_NAME}" == x"schedule" || x"${GITHUB_EVENT_LABEL_NAME}" == x"run-decode"  ]]; then
-  mkdir -p pruned_transducer_stateless2/exp
-  ln -s $PWD/$repo/exp/pretrained.pt pruned_transducer_stateless2/exp/epoch-999.pt
-  ln -s $PWD/$repo/data/lang_bpe_500 data/
-
-  ls -lh data
-  ls -lh pruned_transducer_stateless2/exp
-
-  log "Decoding test-clean and test-other"
-
-  # use a small value for decoding with CPU
-  max_duration=100
-
-  for method in greedy_search fast_beam_search modified_beam_search; do
-    log "Decoding with $method"
-
-    ./pruned_transducer_stateless2/decode.py \
-      --decoding-method $method \
-      --epoch 999 \
-      --avg 1 \
-      --max-duration $max_duration \
-      --exp-dir pruned_transducer_stateless2/exp
-  done
-
-  rm pruned_transducer_stateless2/exp/*.pt
-  rm -r data/lang_bpe_500
-fi

.github/scripts/run-librispeech-pruned-transducer-stateless3-2022-04-29.sh

@@ -1,85 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-log() {
-  # This function is from espnet
-  local fname=${BASH_SOURCE[1]##*/}
-  echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*"
-}
-
-cd egs/librispeech/ASR
-
-repo_url=https://huggingface.co/csukuangfj/icefall-asr-librispeech-pruned-transducer-stateless3-2022-04-29
-
-log "Downloading pre-trained model from $repo_url"
-GIT_LFS_SKIP_SMUDGE=1 git clone $repo_url
-repo=$(basename $repo_url)
-pushd $repo
-git lfs pull --include "data/lang_bpe_500/bpe.model"
-git lfs pull --include "exp/pretrained-epoch-25-avg-6.pt"
-popd
-
-log "Display test files"
-tree $repo/
-ls -lh $repo/test_wavs/*.wav
-
-pushd $repo/exp
-ln -s pretrained-epoch-25-avg-6.pt pretrained.pt
-popd
-
-for sym in 1 2 3; do
-  log "Greedy search with --max-sym-per-frame $sym"
-
-  ./pruned_transducer_stateless3/pretrained.py \
-    --method greedy_search \
-    --max-sym-per-frame $sym \
-    --checkpoint $repo/exp/pretrained.pt \
-    --tokens $repo/data/lang_bpe_500/tokens.txt \
-    $repo/test_wavs/1089-134686-0001.wav \
-    $repo/test_wavs/1221-135766-0001.wav \
-    $repo/test_wavs/1221-135766-0002.wav
-done
-
-for method in modified_beam_search beam_search fast_beam_search; do
-  log "$method"
-
-  ./pruned_transducer_stateless3/pretrained.py \
-    --method $method \
-    --beam-size 4 \
-    --checkpoint $repo/exp/pretrained.pt \
-    --tokens $repo/data/lang_bpe_500/tokens.txt \
-    $repo/test_wavs/1089-134686-0001.wav \
-    $repo/test_wavs/1221-135766-0001.wav \
-    $repo/test_wavs/1221-135766-0002.wav
-done
-
-echo "GITHUB_EVENT_NAME: ${GITHUB_EVENT_NAME}"
-echo "GITHUB_EVENT_LABEL_NAME: ${GITHUB_EVENT_LABEL_NAME}"
-if [[ x"${GITHUB_EVENT_NAME}" == x"schedule" || x"${GITHUB_EVENT_LABEL_NAME}" == x"run-decode"  ]]; then
-  mkdir -p pruned_transducer_stateless3/exp
-  ln -s $PWD/$repo/exp/pretrained.pt pruned_transducer_stateless3/exp/epoch-999.pt
-  ln -s $PWD/$repo/data/lang_bpe_500 data/
-
-  ls -lh data
-  ls -lh pruned_transducer_stateless3/exp
-
-  log "Decoding test-clean and test-other"
-
-  # use a small value for decoding with CPU
-  max_duration=100
-
-  for method in greedy_search fast_beam_search modified_beam_search; do
-    log "Decoding with $method"
-
-    ./pruned_transducer_stateless3/decode.py \
-      --decoding-method $method \
-      --epoch 999 \
-      --avg 1 \
-      --max-duration $max_duration \
-      --exp-dir pruned_transducer_stateless3/exp
-  done
-
-  rm pruned_transducer_stateless3/exp/*.pt
-  rm -r data/lang_bpe_500
-fi

.github/scripts/run-librispeech-pruned-transducer-stateless3-2022-05-13.sh

@@ -1,123 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-log() {
-  # This function is from espnet
-  local fname=${BASH_SOURCE[1]##*/}
-  echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*"
-}
-
-cd egs/librispeech/ASR
-
-repo_url=https://huggingface.co/csukuangfj/icefall-asr-librispeech-pruned-transducer-stateless3-2022-05-13
-
-log "Downloading pre-trained model from $repo_url"
-git lfs install
-git clone $repo_url
-repo=$(basename $repo_url)
-
-log "Display test files"
-tree $repo/
-ls -lh $repo/test_wavs/*.wav
-
-pushd $repo/exp
-ln -s pretrained-iter-1224000-avg-14.pt pretrained.pt
-ln -s pretrained-iter-1224000-avg-14.pt epoch-99.pt
-popd
-
-
-log "Export to torchscript model"
-./pruned_transducer_stateless3/export.py \
-  --exp-dir $repo/exp \
-  --tokens $repo/data/lang_bpe_500/tokens.txt \
-  --epoch 99 \
-  --avg 1 \
-  --jit 1
-
-./pruned_transducer_stateless3/export.py \
-  --exp-dir $repo/exp \
-  --tokens $repo/data/lang_bpe_500/tokens.txt \
-  --epoch 99 \
-  --avg 1 \
-  --jit-trace 1
-
-ls -lh $repo/exp/*.pt
-
-log "Decode with models exported by torch.jit.trace()"
-
-./pruned_transducer_stateless3/jit_pretrained.py \
-  --bpe-model $repo/data/lang_bpe_500/bpe.model \
-  --encoder-model-filename $repo/exp/encoder_jit_trace.pt \
-  --decoder-model-filename $repo/exp/decoder_jit_trace.pt \
-  --joiner-model-filename $repo/exp/joiner_jit_trace.pt \
-  $repo/test_wavs/1089-134686-0001.wav \
-  $repo/test_wavs/1221-135766-0001.wav \
-  $repo/test_wavs/1221-135766-0002.wav
-
-log "Decode with models exported by torch.jit.script()"
-
-./pruned_transducer_stateless3/jit_pretrained.py \
-  --bpe-model $repo/data/lang_bpe_500/bpe.model \
-  --encoder-model-filename $repo/exp/encoder_jit_script.pt \
-  --decoder-model-filename $repo/exp/decoder_jit_script.pt \
-  --joiner-model-filename $repo/exp/joiner_jit_script.pt \
-  $repo/test_wavs/1089-134686-0001.wav \
-  $repo/test_wavs/1221-135766-0001.wav \
-  $repo/test_wavs/1221-135766-0002.wav
-
-
-for sym in 1 2 3; do
-  log "Greedy search with --max-sym-per-frame $sym"
-
-  ./pruned_transducer_stateless3/pretrained.py \
-    --method greedy_search \
-    --max-sym-per-frame $sym \
-    --checkpoint $repo/exp/pretrained.pt \
-    --tokens $repo/data/lang_bpe_500/tokens.txt \
-    $repo/test_wavs/1089-134686-0001.wav \
-    $repo/test_wavs/1221-135766-0001.wav \
-    $repo/test_wavs/1221-135766-0002.wav
-done
-
-for method in modified_beam_search beam_search fast_beam_search; do
-  log "$method"
-
-  ./pruned_transducer_stateless3/pretrained.py \
-    --method $method \
-    --beam-size 4 \
-    --checkpoint $repo/exp/pretrained.pt \
-    --tokens $repo/data/lang_bpe_500/tokens.txt \
-    $repo/test_wavs/1089-134686-0001.wav \
-    $repo/test_wavs/1221-135766-0001.wav \
-    $repo/test_wavs/1221-135766-0002.wav
-done
-
-echo "GITHUB_EVENT_NAME: ${GITHUB_EVENT_NAME}"
-echo "GITHUB_EVENT_LABEL_NAME: ${GITHUB_EVENT_LABEL_NAME}"
-if [[ x"${GITHUB_EVENT_NAME}" == x"schedule" || x"${GITHUB_EVENT_LABEL_NAME}" == x"run-decode"  ]]; then
-  mkdir -p pruned_transducer_stateless3/exp
-  ln -s $PWD/$repo/exp/pretrained.pt pruned_transducer_stateless3/exp/epoch-999.pt
-  ln -s $PWD/$repo/data/lang_bpe_500 data/
-
-  ls -lh data
-  ls -lh pruned_transducer_stateless3/exp
-
-  log "Decoding test-clean and test-other"
-
-  # use a small value for decoding with CPU
-  max_duration=100
-
-  for method in greedy_search fast_beam_search modified_beam_search; do
-    log "Decoding with $method"
-
-    ./pruned_transducer_stateless3/decode.py \
-      --decoding-method $method \
-      --epoch 999 \
-      --avg 1 \
-      --max-duration $max_duration \
-      --exp-dir pruned_transducer_stateless3/exp
-  done
-
-  rm pruned_transducer_stateless3/exp/*.pt
-fi

.github/scripts/run-librispeech-pruned-transducer-stateless5-2022-05-13.sh

@@ -1,100 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-log() {
-  # This function is from espnet
-  local fname=${BASH_SOURCE[1]##*/}
-  echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*"
-}
-
-cd egs/librispeech/ASR
-
-repo_url=https://huggingface.co/csukuangfj/icefall-asr-librispeech-pruned-transducer-stateless5-2022-05-13
-
-log "Downloading pre-trained model from $repo_url"
-git lfs install
-git clone $repo_url
-repo=$(basename $repo_url)
-
-log "Display test files"
-tree $repo/
-ls -lh $repo/test_wavs/*.wav
-
-pushd $repo/exp
-ln -s pretrained-epoch-39-avg-7.pt pretrained.pt
-popd
-
-for sym in 1 2 3; do
-  log "Greedy search with --max-sym-per-frame $sym"
-
-  ./pruned_transducer_stateless5/pretrained.py \
-    --method greedy_search \
-    --max-sym-per-frame $sym \
-    --checkpoint $repo/exp/pretrained.pt \
-    --tokens $repo/data/lang_bpe_500/tokens.txt \
-    --num-encoder-layers 18 \
-    --dim-feedforward 2048 \
-    --nhead 8 \
-    --encoder-dim 512 \
-    --decoder-dim 512 \
-    --joiner-dim 512 \
-    $repo/test_wavs/1089-134686-0001.wav \
-    $repo/test_wavs/1221-135766-0001.wav \
-    $repo/test_wavs/1221-135766-0002.wav
-done
-
-for method in modified_beam_search beam_search fast_beam_search; do
-  log "$method"
-
-  ./pruned_transducer_stateless5/pretrained.py \
-    --method $method \
-    --beam-size 4 \
-    --checkpoint $repo/exp/pretrained.pt \
-    --tokens $repo/data/lang_bpe_500/tokens.txt \
-    $repo/test_wavs/1089-134686-0001.wav \
-    $repo/test_wavs/1221-135766-0001.wav \
-    $repo/test_wavs/1221-135766-0002.wav \
-    --num-encoder-layers 18 \
-    --dim-feedforward 2048 \
-    --nhead 8 \
-    --encoder-dim 512 \
-    --decoder-dim 512 \
-    --joiner-dim 512
-done
-
-echo "GITHUB_EVENT_NAME: ${GITHUB_EVENT_NAME}"
-echo "GITHUB_EVENT_LABEL_NAME: ${GITHUB_EVENT_LABEL_NAME}"
-if [[ x"${GITHUB_EVENT_NAME}" == x"schedule" || x"${GITHUB_EVENT_LABEL_NAME}" == x"run-decode"  ]]; then
-  mkdir -p pruned_transducer_stateless5/exp
-  ln -s $PWD/$repo/exp/pretrained-epoch-39-avg-7.pt pruned_transducer_stateless5/exp/epoch-999.pt
-  ln -s $PWD/$repo/data/lang_bpe_500 data/
-
-  ls -lh data
-  ls -lh pruned_transducer_stateless5/exp
-
-  log "Decoding test-clean and test-other"
-
-  # use a small value for decoding with CPU
-  max_duration=100
-
-  for method in greedy_search fast_beam_search modified_beam_search; do
-    log "Decoding with $method"
-
-    ./pruned_transducer_stateless5/decode.py \
-      --decoding-method $method \
-      --use-averaged-model 0 \
-      --epoch 999 \
-      --avg 1 \
-      --max-duration $max_duration \
-      --exp-dir pruned_transducer_stateless5/exp \
-      --num-encoder-layers 18 \
-      --dim-feedforward 2048 \
-      --nhead 8 \
-      --encoder-dim 512 \
-      --decoder-dim 512 \
-      --joiner-dim 512
-  done
-
-  rm pruned_transducer_stateless5/exp/*.pt
-fi

.github/scripts/run-librispeech-pruned-transducer-stateless7-2022-11-11.sh

@@ -1,106 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-log() {
-  # This function is from espnet
-  local fname=${BASH_SOURCE[1]##*/}
-  echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*"
-}
-
-cd egs/librispeech/ASR
-
-repo_url=https://huggingface.co/csukuangfj/icefall-asr-librispeech-pruned-transducer-stateless7-2022-11-11
-
-log "Downloading pre-trained model from $repo_url"
-git lfs install
-GIT_LFS_SKIP_SMUDGE=1 git clone $repo_url
-repo=$(basename $repo_url)
-
-log "Display test files"
-tree $repo/
-ls -lh $repo/test_wavs/*.wav
-
-pushd $repo/exp
-git lfs pull --include "data/lang_bpe_500/bpe.model"
-git lfs pull --include "exp/cpu_jit.pt"
-git lfs pull --include "exp/pretrained.pt"
-ln -s pretrained.pt epoch-99.pt
-ls -lh *.pt
-popd
-
-log "Export to torchscript model"
-./pruned_transducer_stateless7/export.py \
-  --exp-dir $repo/exp \
-  --use-averaged-model false \
-  --tokens $repo/data/lang_bpe_500/tokens.txt \
-  --epoch 99 \
-  --avg 1 \
-  --jit 1
-
-ls -lh $repo/exp/*.pt
-
-log "Decode with models exported by torch.jit.script()"
-
-./pruned_transducer_stateless7/jit_pretrained.py \
-  --bpe-model $repo/data/lang_bpe_500/bpe.model \
-  --nn-model-filename $repo/exp/cpu_jit.pt \
-  $repo/test_wavs/1089-134686-0001.wav \
-  $repo/test_wavs/1221-135766-0001.wav \
-  $repo/test_wavs/1221-135766-0002.wav
-
-for sym in 1 2 3; do
-  log "Greedy search with --max-sym-per-frame $sym"
-
-  ./pruned_transducer_stateless7/pretrained.py \
-    --method greedy_search \
-    --max-sym-per-frame $sym \
-    --checkpoint $repo/exp/pretrained.pt \
-    --tokens $repo/data/lang_bpe_500/tokens.txt \
-    $repo/test_wavs/1089-134686-0001.wav \
-    $repo/test_wavs/1221-135766-0001.wav \
-    $repo/test_wavs/1221-135766-0002.wav
-done
-
-for method in modified_beam_search beam_search fast_beam_search; do
-  log "$method"
-
-  ./pruned_transducer_stateless7/pretrained.py \
-    --method $method \
-    --beam-size 4 \
-    --checkpoint $repo/exp/pretrained.pt \
-    --tokens $repo/data/lang_bpe_500/tokens.txt \
-    $repo/test_wavs/1089-134686-0001.wav \
-    $repo/test_wavs/1221-135766-0001.wav \
-    $repo/test_wavs/1221-135766-0002.wav
-done
-
-echo "GITHUB_EVENT_NAME: ${GITHUB_EVENT_NAME}"
-echo "GITHUB_EVENT_LABEL_NAME: ${GITHUB_EVENT_LABEL_NAME}"
-if [[ x"${GITHUB_EVENT_NAME}" == x"schedule" || x"${GITHUB_EVENT_LABEL_NAME}" == x"run-decode"  ]]; then
-  mkdir -p pruned_transducer_stateless7/exp
-  ln -s $PWD/$repo/exp/pretrained.pt pruned_transducer_stateless7/exp/epoch-999.pt
-  ln -s $PWD/$repo/data/lang_bpe_500 data/
-
-  ls -lh data
-  ls -lh pruned_transducer_stateless7/exp
-
-  log "Decoding test-clean and test-other"
-
-  # use a small value for decoding with CPU
-  max_duration=100
-
-  for method in greedy_search fast_beam_search modified_beam_search; do
-    log "Decoding with $method"
-
-    ./pruned_transducer_stateless7/decode.py \
-      --decoding-method $method \
-      --epoch 999 \
-      --avg 1 \
-      --use-averaged-model 0 \
-      --max-duration $max_duration \
-      --exp-dir pruned_transducer_stateless7/exp
-  done
-
-  rm pruned_transducer_stateless7/exp/*.pt
-fi

.github/scripts/run-librispeech-pruned-transducer-stateless7-ctc-2022-12-01.sh

@@ -1,150 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-log() {
-  # This function is from espnet
-  local fname=${BASH_SOURCE[1]##*/}
-  echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*"
-}
-
-cd egs/librispeech/ASR
-
-repo_url=https://huggingface.co/Zengwei/icefall-asr-librispeech-pruned-transducer-stateless7-ctc-2022-12-01
-
-log "Downloading pre-trained model from $repo_url"
-GIT_LFS_SKIP_SMUDGE=1 git clone $repo_url
-repo=$(basename $repo_url)
-
-log "Display test files"
-tree $repo/
-ls -lh $repo/test_wavs/*.wav
-
-pushd $repo/exp
-git lfs pull --include "data/lang_bpe_500/HLG.pt"
-git lfs pull --include "data/lang_bpe_500/L.pt"
-git lfs pull --include "data/lang_bpe_500/LG.pt"
-git lfs pull --include "data/lang_bpe_500/Linv.pt"
-git lfs pull --include "data/lang_bpe_500/bpe.model"
-git lfs pull --include "data/lm/G_4_gram.pt"
-git lfs pull --include "exp/cpu_jit.pt"
-git lfs pull --include "exp/pretrained.pt"
-ln -s pretrained.pt epoch-99.pt
-ls -lh *.pt
-popd
-
-log "Export to torchscript model"
-./pruned_transducer_stateless7_ctc/export.py \
-  --exp-dir $repo/exp \
-  --use-averaged-model false \
-  --tokens $repo/data/lang_bpe_500/tokens.txt \
-  --epoch 99 \
-  --avg 1 \
-  --jit 1
-
-ls -lh $repo/exp/*.pt
-
-log "Decode with models exported by torch.jit.script()"
-
-./pruned_transducer_stateless7_ctc/jit_pretrained.py \
-  --bpe-model $repo/data/lang_bpe_500/bpe.model \
-  --nn-model-filename $repo/exp/cpu_jit.pt \
-  $repo/test_wavs/1089-134686-0001.wav \
-  $repo/test_wavs/1221-135766-0001.wav \
-  $repo/test_wavs/1221-135766-0002.wav
-
-for m in ctc-decoding 1best; do
-  ./pruned_transducer_stateless7_ctc/jit_pretrained_ctc.py \
-    --model-filename $repo/exp/cpu_jit.pt \
-    --words-file $repo/data/lang_bpe_500/words.txt  \
-    --HLG $repo/data/lang_bpe_500/HLG.pt \
-    --bpe-model $repo/data/lang_bpe_500/bpe.model \
-    --G $repo/data/lm/G_4_gram.pt \
-    --method $m \
-    --sample-rate 16000 \
-    $repo/test_wavs/1089-134686-0001.wav \
-    $repo/test_wavs/1221-135766-0001.wav \
-    $repo/test_wavs/1221-135766-0002.wav
-done
-
-for sym in 1 2 3; do
-  log "Greedy search with --max-sym-per-frame $sym"
-
-  ./pruned_transducer_stateless7_ctc/pretrained.py \
-    --method greedy_search \
-    --max-sym-per-frame $sym \
-    --checkpoint $repo/exp/pretrained.pt \
-    --tokens $repo/data/lang_bpe_500/tokens.txt \
-    $repo/test_wavs/1089-134686-0001.wav \
-    $repo/test_wavs/1221-135766-0001.wav \
-    $repo/test_wavs/1221-135766-0002.wav
-done
-
-for method in modified_beam_search beam_search fast_beam_search; do
-  log "$method"
-
-  ./pruned_transducer_stateless7_ctc/pretrained.py \
-    --method $method \
-    --beam-size 4 \
-    --checkpoint $repo/exp/pretrained.pt \
-    --tokens $repo/data/lang_bpe_500/tokens.txt \
-    $repo/test_wavs/1089-134686-0001.wav \
-    $repo/test_wavs/1221-135766-0001.wav \
-    $repo/test_wavs/1221-135766-0002.wav
-done
-
-for m in ctc-decoding 1best; do
-  ./pruned_transducer_stateless7_ctc/pretrained_ctc.py \
-    --checkpoint $repo/exp/pretrained.pt \
-    --words-file $repo/data/lang_bpe_500/words.txt  \
-    --HLG $repo/data/lang_bpe_500/HLG.pt \
-    --bpe-model $repo/data/lang_bpe_500/bpe.model \
-    --G $repo/data/lm/G_4_gram.pt \
-    --method $m \
-    --sample-rate 16000 \
-    $repo/test_wavs/1089-134686-0001.wav \
-    $repo/test_wavs/1221-135766-0001.wav \
-    $repo/test_wavs/1221-135766-0002.wav
-done
-
-echo "GITHUB_EVENT_NAME: ${GITHUB_EVENT_NAME}"
-echo "GITHUB_EVENT_LABEL_NAME: ${GITHUB_EVENT_LABEL_NAME}"
-if [[ x"${GITHUB_EVENT_NAME}" == x"schedule" || x"${GITHUB_EVENT_LABEL_NAME}" == x"run-decode"  ]]; then
-  mkdir -p pruned_transducer_stateless7_ctc/exp
-  ln -s $PWD/$repo/exp/pretrained.pt pruned_transducer_stateless7_ctc/exp/epoch-999.pt
-  ln -s $PWD/$repo/data/lang_bpe_500 data/
-
-  ls -lh data
-  ls -lh pruned_transducer_stateless7_ctc/exp
-
-  log "Decoding test-clean and test-other"
-
-  # use a small value for decoding with CPU
-  max_duration=100
-
-  for method in greedy_search fast_beam_search modified_beam_search; do
-    log "Decoding with $method"
-
-    ./pruned_transducer_stateless7_ctc/decode.py \
-      --decoding-method $method \
-      --epoch 999 \
-      --avg 1 \
-      --use-averaged-model 0 \
-      --max-duration $max_duration \
-      --exp-dir pruned_transducer_stateless7_ctc/exp
-  done
-
-  for m in ctc-decoding 1best; do
-    ./pruned_transducer_stateless7_ctc/ctc_decode.py \
-        --epoch 999 \
-        --avg 1 \
-        --exp-dir ./pruned_transducer_stateless7_ctc/exp \
-        --max-duration $max_duration \
-        --use-averaged-model 0 \
-        --decoding-method $m \
-        --hlg-scale 0.6 \
-        --lm-dir data/lm
-  done
-
-  rm pruned_transducer_stateless7_ctc/exp/*.pt
-fi

.github/scripts/run-librispeech-pruned-transducer-stateless7-ctc-bs-2023-01-29.sh

@@ -1,147 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-log() {
-  # This function is from espnet
-  local fname=${BASH_SOURCE[1]##*/}
-  echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*"
-}
-
-cd egs/librispeech/ASR
-
-repo_url=https://huggingface.co/yfyeung/icefall-asr-librispeech-pruned_transducer_stateless7_ctc_bs-2023-01-29
-
-log "Downloading pre-trained model from $repo_url"
-GIT_LFS_SKIP_SMUDGE=1 git clone $repo_url
-repo=$(basename $repo_url)
-
-log "Display test files"
-tree $repo/
-ls -lh $repo/test_wavs/*.wav
-
-pushd $repo/exp
-git lfs pull --include "data/lang_bpe_500/L.pt"
-git lfs pull --include "data/lang_bpe_500/LG.pt"
-git lfs pull --include "data/lang_bpe_500/HLG.pt"
-git lfs pull --include "data/lang_bpe_500/Linv.pt"
-git lfs pull --include "data/lang_bpe_500/bpe.model"
-git lfs pull --include "exp/cpu_jit.pt"
-git lfs pull --include "exp/pretrained.pt"
-ln -s pretrained.pt epoch-99.pt
-ls -lh *.pt
-popd
-
-log "Export to torchscript model"
-./pruned_transducer_stateless7_ctc_bs/export.py \
-  --exp-dir $repo/exp \
-  --use-averaged-model false \
-  --tokens $repo/data/lang_bpe_500/tokens.txt \
-  --epoch 99 \
-  --avg 1 \
-  --jit 1
-
-ls -lh $repo/exp/*.pt
-
-log "Decode with models exported by torch.jit.script()"
-
-./pruned_transducer_stateless7_ctc_bs/jit_pretrained.py \
-  --bpe-model $repo/data/lang_bpe_500/bpe.model \
-  --nn-model-filename $repo/exp/cpu_jit.pt \
-  $repo/test_wavs/1089-134686-0001.wav \
-  $repo/test_wavs/1221-135766-0001.wav \
-  $repo/test_wavs/1221-135766-0002.wav
-
-for m in ctc-decoding 1best; do
-  ./pruned_transducer_stateless7_ctc_bs/jit_pretrained_ctc.py \
-    --model-filename $repo/exp/cpu_jit.pt \
-    --words-file $repo/data/lang_bpe_500/words.txt  \
-    --HLG $repo/data/lang_bpe_500/HLG.pt \
-    --bpe-model $repo/data/lang_bpe_500/bpe.model \
-    --method $m \
-    --sample-rate 16000 \
-    $repo/test_wavs/1089-134686-0001.wav \
-    $repo/test_wavs/1221-135766-0001.wav \
-    $repo/test_wavs/1221-135766-0002.wav
-done
-
-for sym in 1 2 3; do
-  log "Greedy search with --max-sym-per-frame $sym"
-
-  ./pruned_transducer_stateless7_ctc_bs/pretrained.py \
-    --method greedy_search \
-    --max-sym-per-frame $sym \
-    --checkpoint $repo/exp/pretrained.pt \
-    --tokens $repo/data/lang_bpe_500/tokens.txt \
-    $repo/test_wavs/1089-134686-0001.wav \
-    $repo/test_wavs/1221-135766-0001.wav \
-    $repo/test_wavs/1221-135766-0002.wav
-done
-
-for method in modified_beam_search beam_search fast_beam_search; do
-  log "$method"
-
-  ./pruned_transducer_stateless7_ctc_bs/pretrained.py \
-    --method $method \
-    --beam-size 4 \
-    --checkpoint $repo/exp/pretrained.pt \
-    --tokens $repo/data/lang_bpe_500/tokens.txt \
-    $repo/test_wavs/1089-134686-0001.wav \
-    $repo/test_wavs/1221-135766-0001.wav \
-    $repo/test_wavs/1221-135766-0002.wav
-done
-
-for m in ctc-decoding 1best; do
-  ./pruned_transducer_stateless7_ctc_bs/pretrained_ctc.py \
-    --checkpoint $repo/exp/pretrained.pt \
-    --words-file $repo/data/lang_bpe_500/words.txt  \
-    --HLG $repo/data/lang_bpe_500/HLG.pt \
-    --bpe-model $repo/data/lang_bpe_500/bpe.model \
-    --method $m \
-    --sample-rate 16000 \
-    $repo/test_wavs/1089-134686-0001.wav \
-    $repo/test_wavs/1221-135766-0001.wav \
-    $repo/test_wavs/1221-135766-0002.wav
-done
-
-echo "GITHUB_EVENT_NAME: ${GITHUB_EVENT_NAME}"
-echo "GITHUB_EVENT_LABEL_NAME: ${GITHUB_EVENT_LABEL_NAME}"
-
-if [[ x"${GITHUB_EVENT_NAME}" == x"schedule" || x"${GITHUB_EVENT_LABEL_NAME}" == x"run-decode"  ]]; then
-  mkdir -p pruned_transducer_stateless7_ctc_bs/exp
-  ln -s $PWD/$repo/exp/pretrained.pt pruned_transducer_stateless7_ctc_bs/exp/epoch-999.pt
-  ln -s $PWD/$repo/data/lang_bpe_500 data/
-
-  ls -lh data
-  ls -lh pruned_transducer_stateless7_ctc_bs/exp
-
-  log "Decoding test-clean and test-other"
-
-  # use a small value for decoding with CPU
-  max_duration=100
-
-  for method in greedy_search fast_beam_search modified_beam_search; do
-    log "Decoding with $method"
-
-    ./pruned_transducer_stateless7_ctc_bs/decode.py \
-      --decoding-method $method \
-      --epoch 999 \
-      --avg 1 \
-      --use-averaged-model 0 \
-      --max-duration $max_duration \
-      --exp-dir pruned_transducer_stateless7_ctc_bs/exp
-  done
-
-  for m in ctc-decoding 1best; do
-    ./pruned_transducer_stateless7_ctc_bs/ctc_decode.py \
-        --epoch 999 \
-        --avg 1 \
-        --exp-dir ./pruned_transducer_stateless7_ctc_bs/exp \
-        --max-duration $max_duration \
-        --use-averaged-model 0 \
-        --decoding-method $m \
-        --hlg-scale 0.6
-  done
-  
-  rm pruned_transducer_stateless7_ctc_bs/exp/*.pt
-fi

.github/scripts/run-librispeech-pruned-transducer-stateless7-streaming-2022-12-29.sh

@@ -1,148 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-log() {
-  # This function is from espnet
-  local fname=${BASH_SOURCE[1]##*/}
-  echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*"
-}
-
-cd egs/librispeech/ASR
-
-repo_url=https://huggingface.co/Zengwei/icefall-asr-librispeech-pruned-transducer-stateless7-streaming-2022-12-29
-
-log "Downloading pre-trained model from $repo_url"
-git lfs install
-GIT_LFS_SKIP_SMUDGE=1 git clone $repo_url
-repo=$(basename $repo_url)
-
-log "Display test files"
-tree $repo/
-ls -lh $repo/test_wavs/*.wav
-
-pushd $repo
-git lfs pull --include "data/lang_bpe_500/bpe.model"
-git lfs pull --include "exp/cpu_jit.pt"
-git lfs pull --include "exp/pretrained.pt"
-git lfs pull --include "exp/encoder_jit_trace.pt"
-git lfs pull --include "exp/decoder_jit_trace.pt"
-git lfs pull --include "exp/joiner_jit_trace.pt"
-cd exp
-ln -s pretrained.pt epoch-99.pt
-ls -lh *.pt
-popd
-
-log "Export to torchscript model"
-./pruned_transducer_stateless7_streaming/export.py \
-  --exp-dir $repo/exp \
-  --use-averaged-model false \
-  --tokens $repo/data/lang_bpe_500/tokens.txt \
-  --decode-chunk-len 32 \
-  --epoch 99 \
-  --avg 1 \
-  --jit 1
-
-ls -lh $repo/exp/*.pt
-
-log "Decode with models exported by torch.jit.script()"
-
-./pruned_transducer_stateless7_streaming/jit_pretrained.py \
-  --bpe-model $repo/data/lang_bpe_500/bpe.model \
-  --nn-model-filename $repo/exp/cpu_jit.pt \
-  --decode-chunk-len 32 \
-  $repo/test_wavs/1089-134686-0001.wav \
-  $repo/test_wavs/1221-135766-0001.wav \
-  $repo/test_wavs/1221-135766-0002.wav
-
-log "Export to torchscript model by torch.jit.trace()"
-./pruned_transducer_stateless7_streaming/jit_trace_export.py \
-  --exp-dir $repo/exp \
-  --use-averaged-model false \
-  --bpe-model $repo/data/lang_bpe_500/bpe.model \
-  --decode-chunk-len 32 \
-  --epoch 99 \
-  --avg 1
-
-log "Decode with models exported by torch.jit.trace()"
-
-./pruned_transducer_stateless7_streaming/jit_trace_pretrained.py \
-  --bpe-model $repo/data/lang_bpe_500/bpe.model \
-  --encoder-model-filename $repo/exp/encoder_jit_trace.pt \
-  --decoder-model-filename $repo/exp/decoder_jit_trace.pt \
-  --joiner-model-filename $repo/exp/joiner_jit_trace.pt \
-  --decode-chunk-len 32 \
-  $repo/test_wavs/1089-134686-0001.wav
-
-for sym in 1 2 3; do
-  log "Greedy search with --max-sym-per-frame $sym"
-
-  ./pruned_transducer_stateless7_streaming/pretrained.py \
-    --method greedy_search \
-    --max-sym-per-frame $sym \
-    --checkpoint $repo/exp/pretrained.pt \
-    --tokens $repo/data/lang_bpe_500/tokens.txt \
-    --decode-chunk-len 32 \
-    $repo/test_wavs/1089-134686-0001.wav \
-    $repo/test_wavs/1221-135766-0001.wav \
-    $repo/test_wavs/1221-135766-0002.wav
-done
-
-for method in modified_beam_search beam_search fast_beam_search; do
-  log "$method"
-
-  ./pruned_transducer_stateless7_streaming/pretrained.py \
-    --method $method \
-    --beam-size 4 \
-    --checkpoint $repo/exp/pretrained.pt \
-    --tokens $repo/data/lang_bpe_500/tokens.txt \
-    --decode-chunk-len 32 \
-    $repo/test_wavs/1089-134686-0001.wav \
-    $repo/test_wavs/1221-135766-0001.wav \
-    $repo/test_wavs/1221-135766-0002.wav
-done
-
-echo "GITHUB_EVENT_NAME: ${GITHUB_EVENT_NAME}"
-echo "GITHUB_EVENT_LABEL_NAME: ${GITHUB_EVENT_LABEL_NAME}"
-if [[ x"${GITHUB_EVENT_NAME}" == x"schedule" || x"${GITHUB_EVENT_LABEL_NAME}" == x"run-decode"  ]]; then
-  mkdir -p pruned_transducer_stateless7_streaming/exp
-  ln -s $PWD/$repo/exp/pretrained.pt pruned_transducer_stateless7_streaming/exp/epoch-999.pt
-  ln -s $PWD/$repo/data/lang_bpe_500 data/
-
-  ls -lh data
-  ls -lh pruned_transducer_stateless7_streaming/exp
-
-  log "Decoding test-clean and test-other"
-
-  # use a small value for decoding with CPU
-  max_duration=100
-  num_decode_stream=200
-
-  for method in greedy_search fast_beam_search modified_beam_search; do
-    log "decoding with $method"
-
-    ./pruned_transducer_stateless7_streaming/decode.py \
-      --decoding-method $method \
-      --epoch 999 \
-      --avg 1 \
-      --use-averaged-model 0 \
-      --max-duration $max_duration \
-      --decode-chunk-len 32 \
-      --exp-dir pruned_transducer_stateless7_streaming/exp
-  done
-
-  for method in greedy_search fast_beam_search modified_beam_search; do
-    log "Decoding with $method"
-
-    ./pruned_transducer_stateless7_streaming/streaming_decode.py \
-      --decoding-method $method \
-      --epoch 999 \
-      --avg 1 \
-      --use-averaged-model 0 \
-      --decode-chunk-len 32 \
-      --num-decode-streams $num_decode_stream
-      --exp-dir pruned_transducer_stateless7_streaming/exp
-  done
-
-  rm pruned_transducer_stateless7_streaming/exp/*.pt
-fi

.github/scripts/run-librispeech-pruned-transducer-stateless8-2022-11-14.sh

@@ -1,115 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-log() {
-  # This function is from espnet
-  local fname=${BASH_SOURCE[1]##*/}
-  echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*"
-}
-
-cd egs/librispeech/ASR
-
-repo_url=https://huggingface.co/csukuangfj/icefall-asr-librispeech-pruned-transducer-stateless8-2022-11-14
-
-log "Downloading pre-trained model from $repo_url"
-git lfs install
-GIT_LFS_SKIP_SMUDGE=1 git clone $repo_url
-repo=$(basename $repo_url)
-
-log "Display test files"
-tree $repo/
-ls -lh $repo/test_wavs/*.wav
-
-pushd $repo/exp
-git lfs pull --include "data/lang_bpe_500/bpe.model"
-git lfs pull --include "exp/cpu_jit.pt"
-git lfs pull --include "exp/pretrained.pt"
-ln -s pretrained.pt epoch-99.pt
-ls -lh *.pt
-popd
-
-log "Decode with models exported by torch.jit.script()"
-
-./pruned_transducer_stateless8/jit_pretrained.py \
-  --bpe-model $repo/data/lang_bpe_500/bpe.model \
-  --nn-model-filename $repo/exp/cpu_jit.pt \
-  $repo/test_wavs/1089-134686-0001.wav \
-  $repo/test_wavs/1221-135766-0001.wav \
-  $repo/test_wavs/1221-135766-0002.wav
-
-log "Export to torchscript model"
-./pruned_transducer_stateless8/export.py \
-  --exp-dir $repo/exp \
-  --tokens $repo/data/lang_bpe_500/tokens.txt \
-  --use-averaged-model false \
-  --epoch 99 \
-  --avg 1 \
-  --jit 1
-
-ls -lh $repo/exp/*.pt
-
-log "Decode with models exported by torch.jit.script()"
-
-./pruned_transducer_stateless8/jit_pretrained.py \
-  --bpe-model $repo/data/lang_bpe_500/bpe.model \
-  --nn-model-filename $repo/exp/cpu_jit.pt \
-  $repo/test_wavs/1089-134686-0001.wav \
-  $repo/test_wavs/1221-135766-0001.wav \
-  $repo/test_wavs/1221-135766-0002.wav
-
-for sym in 1 2 3; do
-  log "Greedy search with --max-sym-per-frame $sym"
-
-  ./pruned_transducer_stateless8/pretrained.py \
-    --method greedy_search \
-    --max-sym-per-frame $sym \
-    --checkpoint $repo/exp/pretrained.pt \
-    --tokens $repo/data/lang_bpe_500/tokens.txt \
-    $repo/test_wavs/1089-134686-0001.wav \
-    $repo/test_wavs/1221-135766-0001.wav \
-    $repo/test_wavs/1221-135766-0002.wav
-done
-
-for method in modified_beam_search beam_search fast_beam_search; do
-  log "$method"
-
-  ./pruned_transducer_stateless8/pretrained.py \
-    --method $method \
-    --beam-size 4 \
-    --checkpoint $repo/exp/pretrained.pt \
-    --tokens $repo/data/lang_bpe_500/tokens.txt \
-    $repo/test_wavs/1089-134686-0001.wav \
-    $repo/test_wavs/1221-135766-0001.wav \
-    $repo/test_wavs/1221-135766-0002.wav
-done
-
-echo "GITHUB_EVENT_NAME: ${GITHUB_EVENT_NAME}"
-echo "GITHUB_EVENT_LABEL_NAME: ${GITHUB_EVENT_LABEL_NAME}"
-if [[ x"${GITHUB_EVENT_NAME}" == x"schedule" || x"${GITHUB_EVENT_LABEL_NAME}" == x"run-decode"  ]]; then
-  mkdir -p pruned_transducer_stateless8/exp
-  ln -s $PWD/$repo/exp/pretrained.pt pruned_transducer_stateless8/exp/epoch-999.pt
-  ln -s $PWD/$repo/data/lang_bpe_500 data/
-
-  ls -lh data
-  ls -lh pruned_transducer_stateless8/exp
-
-  log "Decoding test-clean and test-other"
-
-  # use a small value for decoding with CPU
-  max_duration=100
-
-  for method in greedy_search fast_beam_search modified_beam_search; do
-    log "Decoding with $method"
-
-    ./pruned_transducer_stateless8/decode.py \
-      --decoding-method $method \
-      --epoch 999 \
-      --avg 1 \
-      --use-averaged-model 0 \
-      --max-duration $max_duration \
-      --exp-dir pruned_transducer_stateless8/exp
-  done
-
-  rm pruned_transducer_stateless8/exp/*.pt
-fi

.github/scripts/run-librispeech-streaming-pruned-transducer-stateless2-2022-06-26.sh

@@ -1,101 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-log() {
-  # This function is from espnet
-  local fname=${BASH_SOURCE[1]##*/}
-  echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*"
-}
-
-cd egs/librispeech/ASR
-
-repo_url=https://huggingface.co/pkufool/icefall_librispeech_streaming_pruned_transducer_stateless2_20220625
-
-log "Downloading pre-trained model from $repo_url"
-git lfs install
-git clone $repo_url
-repo=$(basename $repo_url)
-
-log "Display test files"
-tree $repo/
-ls -lh $repo/test_wavs/*.wav
-
-pushd $repo/exp
-ln -s pretrained-epoch-24-avg-10.pt pretrained.pt
-popd
-
-for sym in 1 2 3; do
-  log "Greedy search with --max-sym-per-frame $sym"
-
-  ./pruned_transducer_stateless2/pretrained.py \
-    --method greedy_search \
-    --max-sym-per-frame $sym \
-    --checkpoint $repo/exp/pretrained.pt \
-    --tokens $repo/data/lang_bpe_500/tokens.txt \
-    --simulate-streaming 1 \
-    --causal-convolution 1 \
-    $repo/test_wavs/1089-134686-0001.wav \
-    $repo/test_wavs/1221-135766-0001.wav \
-    $repo/test_wavs/1221-135766-0002.wav
-done
-
-for method in modified_beam_search beam_search fast_beam_search; do
-  log "$method"
-
-  ./pruned_transducer_stateless2/pretrained.py \
-    --method $method \
-    --beam-size 4 \
-    --checkpoint $repo/exp/pretrained.pt \
-    --tokens $repo/data/lang_bpe_500/tokens.txt \
-    --simulate-streaming 1 \
-    --causal-convolution 1 \
-    $repo/test_wavs/1089-134686-0001.wav \
-    $repo/test_wavs/1221-135766-0001.wav \
-    $repo/test_wavs/1221-135766-0002.wav
-done
-
-echo "GITHUB_EVENT_NAME: ${GITHUB_EVENT_NAME}"
-echo "GITHUB_EVENT_LABEL_NAME: ${GITHUB_EVENT_LABEL_NAME}"
-if [[ x"${GITHUB_EVENT_NAME}" == x"schedule" || x"${GITHUB_EVENT_LABEL_NAME}" == x"run-decode"  ]]; then
-  mkdir -p pruned_transducer_stateless2/exp
-  ln -s $PWD/$repo/exp/pretrained-epoch-24-avg-10.pt pruned_transducer_stateless2/exp/epoch-999.pt
-  ln -s $PWD/$repo/data/lang_bpe_500 data/
-
-  ls -lh data
-  ls -lh pruned_transducer_stateless2/exp
-
-  log "Decoding test-clean and test-other"
-
-  # use a small value for decoding with CPU
-  max_duration=100
-
-  for method in greedy_search fast_beam_search modified_beam_search; do
-    log "Simulate streaming decoding with $method"
-
-    ./pruned_transducer_stateless2/decode.py \
-      --decoding-method $method \
-      --epoch 999 \
-      --avg 1 \
-      --max-duration $max_duration \
-      --exp-dir pruned_transducer_stateless2/exp \
-      --simulate-streaming 1 \
-      --causal-convolution 1
-  done
-
-  for method in greedy_search fast_beam_search modified_beam_search; do
-    log "Real streaming decoding with $method"
-
-    ./pruned_transducer_stateless2/streaming_decode.py \
-      --decoding-method $method \
-      --epoch 999 \
-      --avg 1 \
-      --num-decode-streams 100 \
-      --exp-dir pruned_transducer_stateless2/exp \
-      --left-context 32 \
-      --decode-chunk-size 8 \
-      --right-context 0
-  done
-
-  rm pruned_transducer_stateless2/exp/*.pt
-fi

.github/scripts/run-librispeech-streaming-zipformer-2023-05-18.sh

@@ -1,116 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-log() {
-  # This function is from espnet
-  local fname=${BASH_SOURCE[1]##*/}
-  echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*"
-}
-
-cd egs/librispeech/ASR
-
-repo_url=https://huggingface.co/Zengwei/icefall-asr-librispeech-streaming-zipformer-2023-05-17
-
-log "Downloading pre-trained model from $repo_url"
-git lfs install
-GIT_LFS_SKIP_SMUDGE=1 git clone $repo_url
-repo=$(basename $repo_url)
-
-log "Display test files"
-tree $repo/
-ls -lh $repo/test_wavs/*.wav
-
-pushd $repo/exp
-git lfs pull --include "data/lang_bpe_500/bpe.model"
-git lfs pull --include "data/lang_bpe_500/tokens.txt"
-git lfs pull --include "exp/jit_script_chunk_16_left_128.pt"
-git lfs pull --include "exp/pretrained.pt"
-ln -s pretrained.pt epoch-99.pt
-ls -lh *.pt
-popd
-
-log "Export to torchscript model"
-./zipformer/export.py \
-  --exp-dir $repo/exp \
-  --use-averaged-model false \
-  --tokens $repo/data/lang_bpe_500/tokens.txt \
-  --causal 1 \
-  --chunk-size 16 \
-  --left-context-frames 128 \
-  --epoch 99 \
-  --avg 1 \
-  --jit 1
-
-ls -lh $repo/exp/*.pt
-
-log "Decode with models exported by torch.jit.script()"
-
-./zipformer/jit_pretrained_streaming.py \
-  --tokens $repo/data/lang_bpe_500/tokens.txt \
-  --nn-model-filename $repo/exp/jit_script_chunk_16_left_128.pt \
-  $repo/test_wavs/1089-134686-0001.wav
-
-for method in greedy_search modified_beam_search fast_beam_search; do
-  log "$method"
-
-  ./zipformer/pretrained.py \
-    --causal 1 \
-    --chunk-size 16 \
-    --left-context-frames 128 \
-    --method $method \
-    --beam-size 4 \
-    --checkpoint $repo/exp/pretrained.pt \
-    --tokens $repo/data/lang_bpe_500/tokens.txt \
-    $repo/test_wavs/1089-134686-0001.wav \
-    $repo/test_wavs/1221-135766-0001.wav \
-    $repo/test_wavs/1221-135766-0002.wav
-done
-
-echo "GITHUB_EVENT_NAME: ${GITHUB_EVENT_NAME}"
-echo "GITHUB_EVENT_LABEL_NAME: ${GITHUB_EVENT_LABEL_NAME}"
-if [[ x"${GITHUB_EVENT_NAME}" == x"schedule" || x"${GITHUB_EVENT_LABEL_NAME}" == x"run-decode"  ]]; then
-  mkdir -p zipformer/exp
-  ln -s $PWD/$repo/exp/pretrained.pt zipformer/exp/epoch-999.pt
-  ln -s $PWD/$repo/data/lang_bpe_500 data/
-
-  ls -lh data
-  ls -lh zipformer/exp
-
-  log "Decoding test-clean and test-other"
-
-  # use a small value for decoding with CPU
-  max_duration=100
-
-  for method in greedy_search fast_beam_search modified_beam_search; do
-    log "Simulated streaming decoding with $method"
-
-    ./zipformer/decode.py \
-      --causal 1 \
-      --chunk-size 16 \
-      --left-context-frames 128 \
-      --decoding-method $method \
-      --epoch 999 \
-      --avg 1 \
-      --use-averaged-model 0 \
-      --max-duration $max_duration \
-      --exp-dir zipformer/exp
-  done
-
-  for method in greedy_search fast_beam_search modified_beam_search; do
-    log "Chunk-wise streaming decoding with $method"
-
-    ./zipformer/streaming_decode.py \
-      --causal 1 \
-      --chunk-size 16 \
-      --left-context-frames 128 \
-      --decoding-method $method \
-      --epoch 999 \
-      --avg 1 \
-      --use-averaged-model 0 \
-      --max-duration $max_duration \
-      --exp-dir zipformer/exp
-  done
-
-  rm zipformer/exp/*.pt
-fi

.github/scripts/run-librispeech-transducer-stateless2-2022-04-19.sh

@@ -1,77 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-log() {
-  # This function is from espnet
-  local fname=${BASH_SOURCE[1]##*/}
-  echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*"
-}
-
-cd egs/librispeech/ASR
-
-repo_url=https://huggingface.co/csukuangfj/icefall-asr-librispeech-transducer-stateless2-torchaudio-2022-04-19
-
-log "Downloading pre-trained model from $repo_url"
-git lfs install
-git clone $repo_url
-repo=$(basename $repo_url)
-
-log "Display test files"
-tree $repo/
-ls -lh $repo/test_wavs/*.wav
-
-for sym in 1 2 3; do
-  log "Greedy search with --max-sym-per-frame $sym"
-
-  ./transducer_stateless2/pretrained.py \
-    --method greedy_search \
-    --max-sym-per-frame $sym \
-    --checkpoint $repo/exp/pretrained.pt \
-    --tokens $repo/data/lang_bpe_500/tokens.txt \
-    $repo/test_wavs/1089-134686-0001.wav \
-    $repo/test_wavs/1221-135766-0001.wav \
-    $repo/test_wavs/1221-135766-0002.wav
-done
-
-for method in fast_beam_search modified_beam_search beam_search; do
-  log "$method"
-
-  ./transducer_stateless2/pretrained.py \
-    --method $method \
-    --beam-size 4 \
-    --checkpoint $repo/exp/pretrained.pt \
-    --tokens $repo/data/lang_bpe_500/tokens.txt \
-    $repo/test_wavs/1089-134686-0001.wav \
-    $repo/test_wavs/1221-135766-0001.wav \
-    $repo/test_wavs/1221-135766-0002.wav
-done
-
-echo "GITHUB_EVENT_NAME: ${GITHUB_EVENT_NAME}"
-echo "GITHUB_EVENT_LABEL_NAME: ${GITHUB_EVENT_LABEL_NAME}"
-if [[ x"${GITHUB_EVENT_NAME}" == x"schedule" || x"${GITHUB_EVENT_LABEL_NAME}" == x"run-decode"  ]]; then
-  mkdir -p transducer_stateless2/exp
-  ln -s $PWD/$repo/exp/pretrained.pt transducer_stateless2/exp/epoch-999.pt
-  ln -s $PWD/$repo/data/lang_bpe_500 data/
-
-  ls -lh data
-  ls -lh transducer_stateless2/exp
-
-  log "Decoding test-clean and test-other"
-
-  # use a small value for decoding with CPU
-  max_duration=100
-
-  for method in greedy_search fast_beam_search modified_beam_search; do
-    log "Decoding with $method"
-
-    ./transducer_stateless2/decode.py \
-      --decoding-method $method \
-      --epoch 999 \
-      --avg 1 \
-      --max-duration $max_duration \
-      --exp-dir transducer_stateless2/exp
-  done
-
-  rm transducer_stateless2/exp/*.pt
-fi

.github/scripts/run-librispeech-zipformer-2023-05-18.sh

@@ -1,94 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-log() {
-  # This function is from espnet
-  local fname=${BASH_SOURCE[1]##*/}
-  echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*"
-}
-
-cd egs/librispeech/ASR
-
-repo_url=https://huggingface.co/Zengwei/icefall-asr-librispeech-zipformer-2023-05-15
-
-log "Downloading pre-trained model from $repo_url"
-git lfs install
-GIT_LFS_SKIP_SMUDGE=1 git clone $repo_url
-repo=$(basename $repo_url)
-
-log "Display test files"
-tree $repo/
-ls -lh $repo/test_wavs/*.wav
-
-pushd $repo/exp
-git lfs pull --include "data/lang_bpe_500/bpe.model"
-git lfs pull --include "data/lang_bpe_500/tokens.txt"
-git lfs pull --include "exp/jit_script.pt"
-git lfs pull --include "exp/pretrained.pt"
-ln -s pretrained.pt epoch-99.pt
-ls -lh *.pt
-popd
-
-log "Export to torchscript model"
-./zipformer/export.py \
-  --exp-dir $repo/exp \
-  --use-averaged-model false \
-  --tokens $repo/data/lang_bpe_500/tokens.txt \
-  --epoch 99 \
-  --avg 1 \
-  --jit 1
-
-ls -lh $repo/exp/*.pt
-
-log "Decode with models exported by torch.jit.script()"
-
-./zipformer/jit_pretrained.py \
-  --tokens $repo/data/lang_bpe_500/tokens.txt \
-  --nn-model-filename $repo/exp/jit_script.pt \
-  $repo/test_wavs/1089-134686-0001.wav \
-  $repo/test_wavs/1221-135766-0001.wav \
-  $repo/test_wavs/1221-135766-0002.wav
-
-for method in greedy_search modified_beam_search fast_beam_search; do
-  log "$method"
-
-  ./zipformer/pretrained.py \
-    --method $method \
-    --beam-size 4 \
-    --checkpoint $repo/exp/pretrained.pt \
-    --tokens $repo/data/lang_bpe_500/tokens.txt \
-    $repo/test_wavs/1089-134686-0001.wav \
-    $repo/test_wavs/1221-135766-0001.wav \
-    $repo/test_wavs/1221-135766-0002.wav
-done
-
-echo "GITHUB_EVENT_NAME: ${GITHUB_EVENT_NAME}"
-echo "GITHUB_EVENT_LABEL_NAME: ${GITHUB_EVENT_LABEL_NAME}"
-if [[ x"${GITHUB_EVENT_NAME}" == x"schedule" || x"${GITHUB_EVENT_LABEL_NAME}" == x"run-decode"  ]]; then
-  mkdir -p zipformer/exp
-  ln -s $PWD/$repo/exp/pretrained.pt zipformer/exp/epoch-999.pt
-  ln -s $PWD/$repo/data/lang_bpe_500 data/
-
-  ls -lh data
-  ls -lh zipformer/exp
-
-  log "Decoding test-clean and test-other"
-
-  # use a small value for decoding with CPU
-  max_duration=100
-
-  for method in greedy_search fast_beam_search modified_beam_search; do
-    log "Decoding with $method"
-
-    ./zipformer/decode.py \
-      --decoding-method $method \
-      --epoch 999 \
-      --avg 1 \
-      --use-averaged-model 0 \
-      --max-duration $max_duration \
-      --exp-dir zipformer/exp
-  done
-
-  rm zipformer/exp/*.pt
-fi

.github/scripts/run-librispeech-zipformer-ctc-2023-06-14.sh

@@ -1,117 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-log() {
-  # This function is from espnet
-  local fname=${BASH_SOURCE[1]##*/}
-  echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*"
-}
-
-cd egs/librispeech/ASR
-
-repo_url=https://huggingface.co/Zengwei/icefall-asr-librispeech-zipformer-transducer-ctc-2023-06-13
-
-log "Downloading pre-trained model from $repo_url"
-git lfs install
-GIT_LFS_SKIP_SMUDGE=1 git clone $repo_url
-repo=$(basename $repo_url)
-
-log "Display test files"
-tree $repo/
-ls -lh $repo/test_wavs/*.wav
-
-pushd $repo/exp
-git lfs pull --include "data/lang_bpe_500/bpe.model"
-git lfs pull --include "data/lang_bpe_500/tokens.txt"
-git lfs pull --include "data/lang_bpe_500/HLG.pt"
-git lfs pull --include "data/lang_bpe_500/L.pt"
-git lfs pull --include "data/lang_bpe_500/LG.pt"
-git lfs pull --include "data/lang_bpe_500/Linv.pt"
-git lfs pull --include "data/lm/G_4_gram.pt"
-git lfs pull --include "exp/jit_script.pt"
-git lfs pull --include "exp/pretrained.pt"
-ln -s pretrained.pt epoch-99.pt
-ls -lh *.pt
-popd
-
-log "Export to torchscript model"
-./zipformer/export.py \
-  --exp-dir $repo/exp \
-  --use-transducer 1 \
-  --use-ctc 1 \
-  --use-averaged-model false \
-  --tokens $repo/data/lang_bpe_500/tokens.txt \
-  --epoch 99 \
-  --avg 1 \
-  --jit 1
-
-ls -lh $repo/exp/*.pt
-
-log "Decode with models exported by torch.jit.script()"
-
-for method in ctc-decoding 1best; do
-  ./zipformer/jit_pretrained_ctc.py \
-    --tokens $repo/data/lang_bpe_500/tokens.txt \
-    --model-filename $repo/exp/jit_script.pt \
-    --HLG $repo/data/lang_bpe_500/HLG.pt \
-    --words-file $repo/data/lang_bpe_500/words.txt  \
-    --G $repo/data/lm/G_4_gram.pt \
-    --method $method \
-    --sample-rate 16000 \
-    $repo/test_wavs/1089-134686-0001.wav \
-    $repo/test_wavs/1221-135766-0001.wav \
-    $repo/test_wavs/1221-135766-0002.wav
-done
-
-for method in ctc-decoding 1best; do
-  log "$method"
-
-  ./zipformer/pretrained_ctc.py \
-    --use-transducer 1 \
-    --use-ctc 1 \
-    --method $method \
-    --checkpoint $repo/exp/pretrained.pt \
-    --tokens $repo/data/lang_bpe_500/tokens.txt \
-    --HLG $repo/data/lang_bpe_500/HLG.pt \
-    --G $repo/data/lm/G_4_gram.pt \
-    --words-file $repo/data/lang_bpe_500/words.txt  \
-    --sample-rate 16000 \
-    $repo/test_wavs/1089-134686-0001.wav \
-    $repo/test_wavs/1221-135766-0001.wav \
-    $repo/test_wavs/1221-135766-0002.wav
-done
-
-echo "GITHUB_EVENT_NAME: ${GITHUB_EVENT_NAME}"
-echo "GITHUB_EVENT_LABEL_NAME: ${GITHUB_EVENT_LABEL_NAME}"
-if [[ x"${GITHUB_EVENT_NAME}" == x"schedule" || x"${GITHUB_EVENT_LABEL_NAME}" == x"run-decode"  ]]; then
-  mkdir -p zipformer/exp
-  ln -s $PWD/$repo/exp/pretrained.pt zipformer/exp/epoch-999.pt
-  ln -s $PWD/$repo/data/lang_bpe_500 data/
-
-  ls -lh data
-  ls -lh zipformer/exp
-
-  log "Decoding test-clean and test-other"
-
-  # use a small value for decoding with CPU
-  max_duration=100
-
-  for method in ctc-decoding 1best; do
-    log "Decoding with $method"
-
-    ./zipformer/ctc_decode.py \
-      --use-transducer 1 \
-      --use-ctc 1 \
-      --decoding-method $method \
-      --nbest-scale 1.0 \
-      --hlg-scale 0.6 \
-      --epoch 999 \
-      --avg 1 \
-      --use-averaged-model 0 \
-      --max-duration $max_duration \
-      --exp-dir zipformer/exp
-  done
-
-  rm zipformer/exp/*.pt
-fi

.github/scripts/run-librispeech-zipformer-mmi-2022-12-08.sh

@@ -1,102 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-log() {
-  # This function is from espnet
-  local fname=${BASH_SOURCE[1]##*/}
-  echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*"
-}
-
-cd egs/librispeech/ASR
-
-repo_url=https://huggingface.co/Zengwei/icefall-asr-librispeech-zipformer-mmi-2022-12-08
-
-log "Downloading pre-trained model from $repo_url"
-GIT_LFS_SKIP_SMUDGE=1 git clone $repo_url
-repo=$(basename $repo_url)
-
-log "Display test files"
-tree $repo/
-ls -lh $repo/test_wavs/*.wav
-
-pushd $repo/exp
-git lfs pull --include "data/lang_bpe_500/3gram.pt"
-git lfs pull --include "data/lang_bpe_500/4gram.pt"
-git lfs pull --include "data/lang_bpe_500/L.pt"
-git lfs pull --include "data/lang_bpe_500/LG.pt"
-git lfs pull --include "data/lang_bpe_500/Linv.pt"
-git lfs pull --include "data/lang_bpe_500/bpe.model"
-git lfs pull --include "exp/cpu_jit.pt"
-git lfs pull --include "exp/pretrained.pt"
-ln -s pretrained.pt epoch-99.pt
-ls -lh *.pt
-popd
-
-log "Export to torchscript model"
-./zipformer_mmi/export.py \
-  --exp-dir $repo/exp \
-  --use-averaged-model false \
-  --tokens $repo/data/lang_bpe_500/tokens.txt \
-  --epoch 99 \
-  --avg 1 \
-  --jit 1
-
-ls -lh $repo/exp/*.pt
-
-log "Decode with models exported by torch.jit.script()"
-
-./zipformer_mmi/jit_pretrained.py \
-  --bpe-model $repo/data/lang_bpe_500/bpe.model \
-  --nn-model-filename $repo/exp/cpu_jit.pt \
-  --lang-dir $repo/data/lang_bpe_500 \
-  $repo/test_wavs/1089-134686-0001.wav \
-  $repo/test_wavs/1221-135766-0001.wav \
-  $repo/test_wavs/1221-135766-0002.wav
-
-for method in 1best nbest nbest-rescoring-LG nbest-rescoring-3-gram nbest-rescoring-4-gram; do
-  log "$method"
-
-  ./zipformer_mmi/pretrained.py \
-    --method $method \
-    --checkpoint $repo/exp/pretrained.pt \
-    --lang-dir $repo/data/lang_bpe_500 \
-    --tokens $repo/data/lang_bpe_500/tokens.txt \
-    $repo/test_wavs/1089-134686-0001.wav \
-    $repo/test_wavs/1221-135766-0001.wav \
-    $repo/test_wavs/1221-135766-0002.wav
-done
-
-
-echo "GITHUB_EVENT_NAME: ${GITHUB_EVENT_NAME}"
-echo "GITHUB_EVENT_LABEL_NAME: ${GITHUB_EVENT_LABEL_NAME}"
-if [[ x"${GITHUB_EVENT_NAME}" == x"schedule" || x"${GITHUB_EVENT_LABEL_NAME}" == x"run-decode"  ]]; then
-  mkdir -p zipformer_mmi/exp
-  ln -s $PWD/$repo/exp/pretrained.pt zipformer_mmi/exp/epoch-999.pt
-  ln -s $PWD/$repo/data/lang_bpe_500 data/
-
-  ls -lh data
-  ls -lh zipformer_mmi/exp
-
-  log "Decoding test-clean and test-other"
-
-  # use a small value for decoding with CPU
-  max_duration=100
-
-  for method in 1best nbest nbest-rescoring-LG nbest-rescoring-3-gram nbest-rescoring-4-gram; do
-    log "Decoding with $method"
-
-    ./zipformer_mmi/decode.py \
-      --decoding-method $method \
-      --epoch 999 \
-      --avg 1 \
-      --use-averaged-model 0 \
-      --nbest-scale 1.2 \
-      --hp-scale 1.0 \
-      --max-duration $max_duration \
-      --lang-dir $repo/data/lang_bpe_500 \
-      --exp-dir zipformer_mmi/exp
-  done
-
-  rm zipformer_mmi/exp/*.pt
-fi

.github/scripts/run-multi-corpora-zipformer.sh

@@ -0,0 +1,135 @@
+#!/usr/bin/env bash
+
+set -e
+
+log() {
+  # This function is from espnet
+  local fname=${BASH_SOURCE[1]##*/}
+  echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*"
+}
+
+cd egs/multi_zh-hans/ASR
+
+log "==== Test icefall-asr-multi-zh-hans-zipformer-2023-9-2 ===="
+repo_url=https://huggingface.co/zrjin/icefall-asr-multi-zh-hans-zipformer-2023-9-2/
+
+log "Downloading pre-trained model from $repo_url"
+git lfs install
+git clone $repo_url
+repo=$(basename $repo_url)
+
+
+log "Display test files"
+tree $repo/
+ls -lh $repo/test_wavs/*.wav
+
+pushd $repo/exp
+ln -s epoch-20.pt epoch-99.pt
+popd
+
+ls -lh $repo/exp/*.pt
+
+
+./zipformer/pretrained.py \
+  --checkpoint $repo/exp/epoch-99.pt \
+  --tokens $repo/data/lang_bpe_2000/tokens.txt \
+  --method greedy_search \
+$repo/test_wavs/DEV_T0000000000.wav \
+$repo/test_wavs/DEV_T0000000001.wav \
+$repo/test_wavs/DEV_T0000000002.wav
+
+for method in modified_beam_search fast_beam_search; do
+  log "$method"
+
+  ./zipformer/pretrained.py \
+    --method $method \
+    --beam-size 4 \
+    --checkpoint $repo/exp/epoch-99.pt \
+    --tokens $repo/data/lang_bpe_2000/tokens.txt \
+  $repo/test_wavs/DEV_T0000000000.wav \
+  $repo/test_wavs/DEV_T0000000001.wav \
+  $repo/test_wavs/DEV_T0000000002.wav
+done
+
+rm -rf $repo
+
+log "==== Test icefall-asr-multi-zh-hans-zipformer-ctc-2023-10-24 ===="
+repo_url=https://huggingface.co/zrjin/icefall-asr-multi-zh-hans-zipformer-ctc-2023-10-24/
+
+log "Downloading pre-trained model from $repo_url"
+git lfs install
+git clone $repo_url
+repo=$(basename $repo_url)
+
+
+log "Display test files"
+tree $repo/
+ls -lh $repo/test_wavs/*.wav
+
+pushd $repo/exp
+ln -s epoch-20.pt epoch-99.pt
+popd
+
+ls -lh $repo/exp/*.pt
+
+
+./zipformer/pretrained.py \
+  --checkpoint $repo/exp/epoch-99.pt \
+  --tokens $repo/data/lang_bpe_2000/tokens.txt \
+  --use-ctc 1 \
+  --method greedy_search \
+$repo/test_wavs/DEV_T0000000000.wav \
+$repo/test_wavs/DEV_T0000000001.wav \
+$repo/test_wavs/DEV_T0000000002.wav
+
+for method in modified_beam_search fast_beam_search; do
+  log "$method"
+
+  ./zipformer/pretrained.py \
+    --method $method \
+    --beam-size 4 \
+    --use-ctc 1 \
+    --checkpoint $repo/exp/epoch-99.pt \
+    --tokens $repo/data/lang_bpe_2000/tokens.txt \
+  $repo/test_wavs/DEV_T0000000000.wav \
+  $repo/test_wavs/DEV_T0000000001.wav \
+  $repo/test_wavs/DEV_T0000000002.wav
+done
+
+rm -rf $repo
+
+cd ../../../egs/multi_zh_en/ASR
+log "==== Test icefall-asr-zipformer-multi-zh-en-2023-11-22 ===="
+repo_url=https://huggingface.co/zrjin/icefall-asr-zipformer-multi-zh-en-2023-11-22/
+
+log "Downloading pre-trained model from $repo_url"
+git lfs install
+git clone $repo_url
+repo=$(basename $repo_url)
+
+log "Display test files"
+tree $repo/
+ls -lh $repo/test_wavs/*.wav
+
+./zipformer/pretrained.py \
+  --checkpoint $repo/exp/pretrained.pt \
+  --bpe-model $repo/data/lang_bbpe_2000/bbpe.model \
+  --method greedy_search \
+$repo/test_wavs/_1634_210_2577_1_1525157964032_3712259_29.wav \
+$repo/test_wavs/_1634_210_2577_1_1525157964032_3712259_55.wav \
+$repo/test_wavs/_1634_210_2577_1_1525157964032_3712259_75.wav
+
+for method in modified_beam_search fast_beam_search; do
+  log "$method"
+
+  ./zipformer/pretrained.py \
+    --method $method \
+    --beam-size 4 \
+    --checkpoint $repo/exp/pretrained.pt \
+    --bpe-model $repo/data/lang_bbpe_2000/bbpe.model \
+  $repo/test_wavs/_1634_210_2577_1_1525157964032_3712259_29.wav \
+  $repo/test_wavs/_1634_210_2577_1_1525157964032_3712259_55.wav \
+  $repo/test_wavs/_1634_210_2577_1_1525157964032_3712259_75.wav
+done
+
+rm -rf $repo

.github/scripts/run-multi-zh_hans-zipformer.sh

@@ -1,51 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-log() {
-  # This function is from espnet
-  local fname=${BASH_SOURCE[1]##*/}
-  echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*"
-}
-
-cd egs/multi_zh-hans/ASR
-
-repo_url=https://huggingface.co/zrjin/icefall-asr-multi-zh-hans-zipformer-2023-9-2/
-
-log "Downloading pre-trained model from $repo_url"
-git lfs install
-git clone $repo_url
-repo=$(basename $repo_url)
-
-
-log "Display test files"
-tree $repo/
-ls -lh $repo/test_wavs/*.wav
-
-pushd $repo/exp
-ln -s epoch-20.pt epoch-99.pt
-popd
-
-ls -lh $repo/exp/*.pt
-
-
-./zipformer/pretrained.py \
-  --checkpoint $repo/exp/epoch-99.pt \
-  --tokens $repo/data/lang_bpe_2000/tokens.txt \
-  --method greedy_search \
-$repo/test_wavs/DEV_T0000000000.wav \
-$repo/test_wavs/DEV_T0000000001.wav \
-$repo/test_wavs/DEV_T0000000002.wav
-
-for method in modified_beam_search fast_beam_search; do
-  log "$method"
-
-  ./zipformer/pretrained.py \
-    --method $method \
-    --beam-size 4 \
-    --checkpoint $repo/exp/epoch-99.pt \
-    --tokens $repo/data/lang_bpe_2000/tokens.txt \
-  $repo/test_wavs/DEV_T0000000000.wav \
-  $repo/test_wavs/DEV_T0000000001.wav \
-  $repo/test_wavs/DEV_T0000000002.wav
-done

.github/scripts/run-pre-trained-ctc.sh

@@ -1,240 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-log() {
-  # This function is from espnet
-  local fname=${BASH_SOURCE[1]##*/}
-  echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*"
-}
-
-pushd egs/librispeech/ASR
-
-repo_url=https://huggingface.co/csukuangfj/sherpa-onnx-zipformer-ctc-en-2023-10-02
-log "Downloading pre-trained model from $repo_url"
-git lfs install
-git clone $repo_url
-repo=$(basename $repo_url)
-
-log "Display test files"
-tree $repo/
-ls -lh $repo/test_wavs/*.wav
-
-log "CTC greedy search"
-
-./zipformer/onnx_pretrained_ctc.py \
-  --nn-model $repo/model.onnx \
-  --tokens $repo/tokens.txt \
-  $repo/test_wavs/0.wav \
-  $repo/test_wavs/1.wav \
-  $repo/test_wavs/2.wav
-
-log "CTC H decoding"
-
-./zipformer/onnx_pretrained_ctc_H.py \
-  --nn-model $repo/model.onnx \
-  --tokens $repo/tokens.txt \
-  --H $repo/H.fst \
-  $repo/test_wavs/0.wav \
-  $repo/test_wavs/1.wav \
-  $repo/test_wavs/2.wav
-
-log "CTC HL decoding"
-
-./zipformer/onnx_pretrained_ctc_HL.py \
-  --nn-model $repo/model.onnx \
-  --words $repo/words.txt \
-  --HL $repo/HL.fst \
-  $repo/test_wavs/0.wav \
-  $repo/test_wavs/1.wav \
-  $repo/test_wavs/2.wav
-
-log "CTC HLG decoding"
-
-./zipformer/onnx_pretrained_ctc_HLG.py \
-  --nn-model $repo/model.onnx \
-  --words $repo/words.txt \
-  --HLG $repo/HLG.fst \
-  $repo/test_wavs/0.wav \
-  $repo/test_wavs/1.wav \
-  $repo/test_wavs/2.wav
-
-rm -rf $repo
-
-repo_url=https://huggingface.co/csukuangfj/icefall-asr-librispeech-conformer-ctc-jit-bpe-500-2021-11-09
-log "Downloading pre-trained model from $repo_url"
-GIT_LFS_SKIP_SMUDGE=1 git clone $repo_url
-repo=$(basename $repo_url)
-pushd $repo
-
-git lfs pull --include "exp/pretrained.pt"
-git lfs pull --include "data/lang_bpe_500/HLG.pt"
-git lfs pull --include "data/lang_bpe_500/L.pt"
-git lfs pull --include "data/lang_bpe_500/L_disambig.pt"
-git lfs pull --include "data/lang_bpe_500/Linv.pt"
-git lfs pull --include "data/lang_bpe_500/bpe.model"
-git lfs pull --include "data/lang_bpe_500/lexicon.txt"
-git lfs pull --include "data/lang_bpe_500/lexicon_disambig.txt"
-git lfs pull --include "data/lang_bpe_500/tokens.txt"
-git lfs pull --include "data/lang_bpe_500/words.txt"
-git lfs pull --include "data/lm/G_3_gram.fst.txt"
-
-popd
-
-log "Display test files"
-tree $repo/
-ls -lh $repo/test_wavs/*.wav
-
-log "CTC decoding"
-
-./conformer_ctc/pretrained.py \
-  --method ctc-decoding \
-  --num-classes 500 \
-  --checkpoint $repo/exp/pretrained.pt \
-  --tokens $repo/data/lang_bpe_500/tokens.txt \
-  $repo/test_wavs/1089-134686-0001.wav \
-  $repo/test_wavs/1221-135766-0001.wav \
-  $repo/test_wavs/1221-135766-0002.wav
-
-log "HLG decoding"
-
-./conformer_ctc/pretrained.py \
-  --method 1best \
-  --num-classes 500 \
-  --checkpoint $repo/exp/pretrained.pt \
-  --tokens $repo/data/lang_bpe_500/tokens.txt \
-  --words-file $repo/data/lang_bpe_500/words.txt \
-  --HLG $repo/data/lang_bpe_500/HLG.pt \
-  $repo/test_wavs/1089-134686-0001.wav \
-  $repo/test_wavs/1221-135766-0001.wav \
-  $repo/test_wavs/1221-135766-0002.wav
-
-log "CTC decoding on CPU with kaldi decoders using OpenFst"
-
-log "Exporting model with torchscript"
-
-pushd $repo/exp
-ln -s pretrained.pt epoch-99.pt
-popd
-
-./conformer_ctc/export.py \
-  --epoch 99 \
-  --avg 1 \
-  --exp-dir $repo/exp \
-  --tokens $repo/data/lang_bpe_500/tokens.txt \
-  --jit 1
-
-ls -lh $repo/exp
-
-
-log "Generating H.fst, HL.fst"
-
-./local/prepare_lang_fst.py  --lang-dir $repo/data/lang_bpe_500 --ngram-G $repo/data/lm/G_3_gram.fst.txt
-
-ls -lh $repo/data/lang_bpe_500
-
-log "Decoding with H on CPU with OpenFst"
-
-./conformer_ctc/jit_pretrained_decode_with_H.py \
-  --nn-model $repo/exp/cpu_jit.pt \
-  --H $repo/data/lang_bpe_500/H.fst \
-  --tokens $repo/data/lang_bpe_500/tokens.txt \
-  $repo/test_wavs/1089-134686-0001.wav \
-  $repo/test_wavs/1221-135766-0001.wav \
-  $repo/test_wavs/1221-135766-0002.wav
-
-log "Decoding with HL on CPU with OpenFst"
-
-./conformer_ctc/jit_pretrained_decode_with_HL.py \
-  --nn-model $repo/exp/cpu_jit.pt \
-  --HL $repo/data/lang_bpe_500/HL.fst \
-  --words $repo/data/lang_bpe_500/words.txt \
-  $repo/test_wavs/1089-134686-0001.wav \
-  $repo/test_wavs/1221-135766-0001.wav \
-  $repo/test_wavs/1221-135766-0002.wav
-
-log "Decoding with HLG on CPU with OpenFst"
-
-./conformer_ctc/jit_pretrained_decode_with_HLG.py \
-  --nn-model $repo/exp/cpu_jit.pt \
-  --HLG $repo/data/lang_bpe_500/HLG.fst \
-  --words $repo/data/lang_bpe_500/words.txt \
-  $repo/test_wavs/1089-134686-0001.wav \
-  $repo/test_wavs/1221-135766-0001.wav \
-  $repo/test_wavs/1221-135766-0002.wav
-
-rm -rf $repo
-
-popd
-
-log "Test aishell"
-
-pushd egs/aishell/ASR
-
-repo_url=https://huggingface.co/csukuangfj/icefall_asr_aishell_conformer_ctc
-log "Downloading pre-trained model from $repo_url"
-GIT_LFS_SKIP_SMUDGE=1 git clone $repo_url
-repo=$(basename $repo_url)
-pushd $repo
-
-git lfs pull --include "exp/pretrained.pt"
-git lfs pull --include "data/lang_char/H.fst"
-git lfs pull --include "data/lang_char/HL.fst"
-git lfs pull --include "data/lang_char/HLG.fst"
-
-popd
-
-log "Display test files"
-tree $repo/
-ls -lh $repo/test_wavs/*.wav
-
-log "CTC decoding"
-
-log "Exporting model with torchscript"
-
-pushd $repo/exp
-ln -s pretrained.pt epoch-99.pt
-popd
-
-./conformer_ctc/export.py \
-  --epoch 99 \
-  --avg 1 \
-  --exp-dir $repo/exp \
-  --tokens $repo/data/lang_char/tokens.txt \
-  --jit 1
-
-ls -lh $repo/exp
-
-ls -lh $repo/data/lang_char
-
-log "Decoding with H on CPU with OpenFst"
-
-./conformer_ctc/jit_pretrained_decode_with_H.py \
-  --nn-model $repo/exp/cpu_jit.pt \
-  --H $repo/data/lang_char/H.fst \
-  --tokens $repo/data/lang_char/tokens.txt \
-  $repo/test_wavs/0.wav \
-  $repo/test_wavs/1.wav \
-  $repo/test_wavs/2.wav
-
-log "Decoding with HL on CPU with OpenFst"
-
-./conformer_ctc/jit_pretrained_decode_with_HL.py \
-  --nn-model $repo/exp/cpu_jit.pt \
-  --HL $repo/data/lang_char/HL.fst \
-  --words $repo/data/lang_char/words.txt \
-  $repo/test_wavs/0.wav \
-  $repo/test_wavs/1.wav \
-  $repo/test_wavs/2.wav
-
-log "Decoding with HLG on CPU with OpenFst"
-
-./conformer_ctc/jit_pretrained_decode_with_HLG.py \
-  --nn-model $repo/exp/cpu_jit.pt \
-  --HLG $repo/data/lang_char/HLG.fst \
-  --words $repo/data/lang_char/words.txt \
-  $repo/test_wavs/0.wav \
-  $repo/test_wavs/1.wav \
-  $repo/test_wavs/2.wav
-
-rm -rf $repo

.github/scripts/run-pre-trained-transducer-stateless-librispeech-100h.sh

@@ -1,77 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-log() {
-  # This function is from espnet
-  local fname=${BASH_SOURCE[1]##*/}
-  echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*"
-}
-
-cd egs/librispeech/ASR
-
-repo_url=https://huggingface.co/csukuangfj/icefall-asr-librispeech-100h-transducer-stateless-multi-datasets-bpe-500-2022-02-21
-
-log "Downloading pre-trained model from $repo_url"
-git lfs install
-git clone $repo_url
-repo=$(basename $repo_url)
-
-log "Display test files"
-tree $repo/
-ls -lh $repo/test_wavs/*.wav
-
-for sym in 1 2 3; do
-  log "Greedy search with --max-sym-per-frame $sym"
-
-  ./transducer_stateless_multi_datasets/pretrained.py \
-    --method greedy_search \
-    --max-sym-per-frame $sym \
-    --checkpoint $repo/exp/pretrained.pt \
-    --tokens $repo/data/lang_bpe_500/tokens.txt \
-    $repo/test_wavs/1089-134686-0001.wav \
-    $repo/test_wavs/1221-135766-0001.wav \
-    $repo/test_wavs/1221-135766-0002.wav
-done
-
-for method in modified_beam_search beam_search fast_beam_search; do
-  log "$method"
-
-  ./transducer_stateless_multi_datasets/pretrained.py \
-    --method $method \
-    --beam-size 4 \
-    --checkpoint $repo/exp/pretrained.pt \
-    --tokens $repo/data/lang_bpe_500/tokens.txt \
-    $repo/test_wavs/1089-134686-0001.wav \
-    $repo/test_wavs/1221-135766-0001.wav \
-    $repo/test_wavs/1221-135766-0002.wav
-done
-
-echo "GITHUB_EVENT_NAME: ${GITHUB_EVENT_NAME}"
-echo "GITHUB_EVENT_LABEL_NAME: ${GITHUB_EVENT_LABEL_NAME}"
-if [[ x"${GITHUB_EVENT_NAME}" == x"schedule" || x"${GITHUB_EVENT_LABEL_NAME}" == x"run-decode"  ]]; then
-  mkdir -p transducer_stateless_multi_datasets/exp
-  ln -s $PWD/$repo/exp/pretrained.pt transducer_stateless_multi_datasets/exp/epoch-999.pt
-  ln -s $PWD/$repo/data/lang_bpe_500 data/
-
-  ls -lh data
-  ls -lh transducer_stateless_multi_datasets/exp
-
-  log "Decoding test-clean and test-other"
-
-  # use a small value for decoding with CPU
-  max_duration=100
-
-  for method in greedy_search fast_beam_search modified_beam_search; do
-    log "Decoding with $method"
-
-    ./transducer_stateless_multi_datasets/decode.py \
-      --decoding-method $method \
-      --epoch 999 \
-      --avg 1 \
-      --max-duration $max_duration \
-      --exp-dir transducer_stateless_multi_datasets/exp
-  done
-
-  rm transducer_stateless_multi_datasets/exp/*.pt
-fi

.github/scripts/run-pre-trained-transducer-stateless-librispeech-960h.sh

@@ -1,77 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-log() {
-  # This function is from espnet
-  local fname=${BASH_SOURCE[1]##*/}
-  echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*"
-}
-
-cd egs/librispeech/ASR
-
-repo_url=https://huggingface.co/csukuangfj/icefall-asr-librispeech-transducer-stateless-multi-datasets-bpe-500-2022-03-01
-
-log "Downloading pre-trained model from $repo_url"
-git lfs install
-git clone $repo_url
-repo=$(basename $repo_url)
-
-log "Display test files"
-tree $repo/
-ls -lh $repo/test_wavs/*.wav
-
-for sym in 1 2 3; do
-  log "Greedy search with --max-sym-per-frame $sym"
-
-  ./transducer_stateless_multi_datasets/pretrained.py \
-    --method greedy_search \
-    --max-sym-per-frame $sym \
-    --checkpoint $repo/exp/pretrained.pt \
-    --tokens $repo/data/lang_bpe_500/tokens.txt \
-    $repo/test_wavs/1089-134686-0001.wav \
-    $repo/test_wavs/1221-135766-0001.wav \
-    $repo/test_wavs/1221-135766-0002.wav
-done
-
-for method in modified_beam_search beam_search fast_beam_search; do
-  log "$method"
-
-  ./transducer_stateless_multi_datasets/pretrained.py \
-    --method $method \
-    --beam-size 4 \
-    --checkpoint $repo/exp/pretrained.pt \
-    --tokens $repo/data/lang_bpe_500/tokens.txt \
-    $repo/test_wavs/1089-134686-0001.wav \
-    $repo/test_wavs/1221-135766-0001.wav \
-    $repo/test_wavs/1221-135766-0002.wav
-done
-
-echo "GITHUB_EVENT_NAME: ${GITHUB_EVENT_NAME}"
-echo "GITHUB_EVENT_LABEL_NAME: ${GITHUB_EVENT_LABEL_NAME}"
-if [[ x"${GITHUB_EVENT_NAME}" == x"schedule" || x"${GITHUB_EVENT_LABEL_NAME}" == x"run-decode"  ]]; then
-  mkdir -p transducer_stateless_multi_datasets/exp
-  ln -s $PWD/$repo/exp/pretrained.pt transducer_stateless_multi_datasets/exp/epoch-999.pt
-  ln -s $PWD/$repo/data/lang_bpe_500 data/
-
-  ls -lh data
-  ls -lh transducer_stateless_multi_datasets/exp
-
-  log "Decoding test-clean and test-other"
-
-  # use a small value for decoding with CPU
-  max_duration=100
-
-  for method in greedy_search fast_beam_search modified_beam_search; do
-    log "Decoding with $method"
-
-    ./transducer_stateless_multi_datasets/decode.py \
-      --decoding-method $method \
-      --epoch 999 \
-      --avg 1 \
-      --max-duration $max_duration \
-      --exp-dir transducer_stateless_multi_datasets/exp
-  done
-
-  rm transducer_stateless_multi_datasets/exp/*.pt
-fi

.github/scripts/run-pre-trained-transducer-stateless-modified-2-aishell.sh

@@ -1,48 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-log() {
-  # This function is from espnet
-  local fname=${BASH_SOURCE[1]##*/}
-  echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*"
-}
-
-cd egs/aishell/ASR
-
-repo_url=https://huggingface.co/csukuangfj/icefall-aishell-transducer-stateless-modified-2-2022-03-01
-
-log "Downloading pre-trained model from $repo_url"
-git lfs install
-git clone $repo_url
-repo=$(basename $repo_url)
-
-log "Display test files"
-tree $repo/
-ls -lh $repo/test_wavs/*.wav
-
-for sym in 1 2 3; do
-  log "Greedy search with --max-sym-per-frame $sym"
-
-  ./transducer_stateless_modified-2/pretrained.py \
-    --method greedy_search \
-    --max-sym-per-frame $sym \
-    --checkpoint $repo/exp/pretrained.pt \
-    --lang-dir $repo/data/lang_char \
-    $repo/test_wavs/BAC009S0764W0121.wav \
-    $repo/test_wavs/BAC009S0764W0122.wav \
-    $repo/test_wavs/BAC009S0764W0123.wav
-done
-
-for method in modified_beam_search beam_search; do
-  log "$method"
-
-  ./transducer_stateless_modified-2/pretrained.py \
-    --method $method \
-    --beam-size 4 \
-    --checkpoint $repo/exp/pretrained.pt \
-    --lang-dir $repo/data/lang_char \
-    $repo/test_wavs/BAC009S0764W0121.wav \
-    $repo/test_wavs/BAC009S0764W0122.wav \
-    $repo/test_wavs/BAC009S0764W0123.wav
-done

.github/scripts/run-pre-trained-transducer-stateless-modified-aishell.sh

@@ -1,48 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-log() {
-  # This function is from espnet
-  local fname=${BASH_SOURCE[1]##*/}
-  echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*"
-}
-
-cd egs/aishell/ASR
-
-repo_url=https://huggingface.co/csukuangfj/icefall-aishell-transducer-stateless-modified-2022-03-01
-
-log "Downloading pre-trained model from $repo_url"
-git lfs install
-git clone $repo_url
-repo=$(basename $repo_url)
-
-log "Display test files"
-tree $repo/
-ls -lh $repo/test_wavs/*.wav
-
-for sym in 1 2 3; do
-  log "Greedy search with --max-sym-per-frame $sym"
-
-  ./transducer_stateless_modified/pretrained.py \
-    --method greedy_search \
-    --max-sym-per-frame $sym \
-    --checkpoint $repo/exp/pretrained.pt \
-    --lang-dir $repo/data/lang_char \
-    $repo/test_wavs/BAC009S0764W0121.wav \
-    $repo/test_wavs/BAC009S0764W0122.wav \
-    $repo/test_wavs/BAC009S0764W0123.wav
-done
-
-for method in modified_beam_search beam_search; do
-  log "$method"
-
-  ./transducer_stateless_modified/pretrained.py \
-    --method $method \
-    --beam-size 4 \
-    --checkpoint $repo/exp/pretrained.pt \
-    --lang-dir $repo/data/lang_char \
-    $repo/test_wavs/BAC009S0764W0121.wav \
-    $repo/test_wavs/BAC009S0764W0122.wav \
-    $repo/test_wavs/BAC009S0764W0123.wav
-done

.github/scripts/run-pre-trained-transducer-stateless.sh

@@ -1,77 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-log() {
-  # This function is from espnet
-  local fname=${BASH_SOURCE[1]##*/}
-  echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*"
-}
-
-cd egs/librispeech/ASR
-
-repo_url=https://huggingface.co/csukuangfj/icefall-asr-librispeech-transducer-stateless-bpe-500-2022-02-07
-
-log "Downloading pre-trained model from $repo_url"
-git lfs install
-git clone $repo_url
-repo=$(basename $repo_url)
-
-log "Display test files"
-tree $repo/
-ls -lh $repo/test_wavs/*.wav
-
-for sym in 1 2 3; do
-  log "Greedy search with --max-sym-per-frame $sym"
-
-  ./transducer_stateless/pretrained.py \
-    --method greedy_search \
-    --max-sym-per-frame $sym \
-    --checkpoint $repo/exp/pretrained.pt \
-    --tokens $repo/data/lang_bpe_500/tokens.txt \
-    $repo/test_wavs/1089-134686-0001.wav \
-    $repo/test_wavs/1221-135766-0001.wav \
-    $repo/test_wavs/1221-135766-0002.wav
-done
-
-for method in fast_beam_search modified_beam_search beam_search; do
-  log "$method"
-
-  ./transducer_stateless/pretrained.py \
-    --method $method \
-    --beam-size 4 \
-    --checkpoint $repo/exp/pretrained.pt \
-    --tokens $repo/data/lang_bpe_500/tokens.txt \
-    $repo/test_wavs/1089-134686-0001.wav \
-    $repo/test_wavs/1221-135766-0001.wav \
-    $repo/test_wavs/1221-135766-0002.wav
-done
-
-echo "GITHUB_EVENT_NAME: ${GITHUB_EVENT_NAME}"
-echo "GITHUB_EVENT_LABEL_NAME: ${GITHUB_EVENT_LABEL_NAME}"
-if [[ x"${GITHUB_EVENT_NAME}" == x"schedule" || x"${GITHUB_EVENT_LABEL_NAME}" == x"run-decode"  ]]; then
-  mkdir -p transducer_stateless/exp
-  ln -s $PWD/$repo/exp/pretrained.pt transducer_stateless/exp/epoch-999.pt
-  ln -s $PWD/$repo/data/lang_bpe_500 data/
-
-  ls -lh data
-  ls -lh transducer_stateless/exp
-
-  log "Decoding test-clean and test-other"
-
-  # use a small value for decoding with CPU
-  max_duration=100
-
-  for method in greedy_search fast_beam_search modified_beam_search; do
-    log "Decoding with $method"
-
-    ./transducer_stateless/decode.py \
-      --decoding-method $method \
-      --epoch 999 \
-      --avg 1 \
-      --max-duration $max_duration \
-      --exp-dir transducer_stateless/exp
-  done
-
-  rm transducer_stateless/exp/*.pt
-fi

.github/scripts/run-pre-trained-transducer.sh

@@ -1,33 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-log() {
-  # This function is from espnet
-  local fname=${BASH_SOURCE[1]##*/}
-  echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*"
-}
-
-cd egs/librispeech/ASR
-
-repo_url=https://huggingface.co/csukuangfj/icefall-asr-librispeech-transducer-bpe-500-2021-12-23
-
-log "Downloading pre-trained model from $repo_url"
-git lfs install
-git clone $repo_url
-repo=$(basename $repo_url)
-
-log "Display test files"
-tree $repo/
-ls -lh $repo/test_wavs/*.wav
-
-log "Beam search decoding"
-
-./transducer/pretrained.py \
-  --method beam_search \
-  --beam-size 4 \
-  --checkpoint $repo/exp/pretrained.pt \
-  --tokens $repo/data/lang_bpe_500/tokens.txt \
-  $repo/test_wavs/1089-134686-0001.wav \
-  $repo/test_wavs/1221-135766-0001.wav \
-  $repo/test_wavs/1221-135766-0002.wav

.github/scripts/yesno/ASR/run.sh

@@ -0,0 +1,86 @@
+#!/usr/bin/env bash
+
+set -ex
+
+log() {
+  # This function is from espnet
+  local fname=${BASH_SOURCE[1]##*/}
+  echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*"
+}
+
+cd egs/yesno/ASR
+
+log "data preparation"
+./prepare.sh
+
+log "training"
+python3 ./tdnn/train.py
+
+log "decoding"
+python3 ./tdnn/decode.py
+
+log "export to pretrained.pt"
+
+python3 ./tdnn/export.py --epoch 14 --avg 2
+
+python3 ./tdnn/pretrained.py \
+  --checkpoint ./tdnn/exp/pretrained.pt \
+  --HLG ./data/lang_phone/HLG.pt \
+  --words-file ./data/lang_phone/words.txt \
+  download/waves_yesno/0_0_0_1_0_0_0_1.wav \
+  download/waves_yesno/0_0_1_0_0_0_1_0.wav
+
+log "Test exporting to torchscript"
+python3 ./tdnn/export.py --epoch 14 --avg 2 --jit 1
+
+python3 ./tdnn/jit_pretrained.py \
+  --nn-model ./tdnn/exp/cpu_jit.pt \
+  --HLG ./data/lang_phone/HLG.pt \
+  --words-file ./data/lang_phone/words.txt \
+  download/waves_yesno/0_0_0_1_0_0_0_1.wav \
+  download/waves_yesno/0_0_1_0_0_0_1_0.wav
+
+log "Test exporting to onnx"
+python3 ./tdnn/export_onnx.py --epoch 14 --avg 2
+
+log "Test float32 model"
+python3 ./tdnn/onnx_pretrained.py \
+  --nn-model ./tdnn/exp/model-epoch-14-avg-2.onnx \
+  --HLG ./data/lang_phone/HLG.pt \
+  --words-file ./data/lang_phone/words.txt \
+  download/waves_yesno/0_0_0_1_0_0_0_1.wav \
+  download/waves_yesno/0_0_1_0_0_0_1_0.wav
+
+log "Test int8 model"
+python3 ./tdnn/onnx_pretrained.py \
+  --nn-model ./tdnn/exp/model-epoch-14-avg-2.int8.onnx \
+  --HLG ./data/lang_phone/HLG.pt \
+  --words-file ./data/lang_phone/words.txt \
+  download/waves_yesno/0_0_0_1_0_0_0_1.wav \
+  download/waves_yesno/0_0_1_0_0_0_1_0.wav
+
+log "Test decoding with H"
+python3 ./tdnn/export.py --epoch 14 --avg 2 --jit 1
+
+python3 ./tdnn/jit_pretrained_decode_with_H.py \
+    --nn-model ./tdnn/exp/cpu_jit.pt \
+    --H ./data/lang_phone/H.fst \
+    --tokens ./data/lang_phone/tokens.txt \
+    ./download/waves_yesno/0_0_0_1_0_0_0_1.wav \
+    ./download/waves_yesno/0_0_1_0_0_0_1_0.wav \
+    ./download/waves_yesno/0_0_1_0_0_1_1_1.wav
+
+log "Test decoding with HL"
+python3 ./tdnn/export.py --epoch 14 --avg 2 --jit 1
+
+python3 ./tdnn/jit_pretrained_decode_with_HL.py \
+    --nn-model ./tdnn/exp/cpu_jit.pt \
+    --HL ./data/lang_phone/HL.fst \
+    --words ./data/lang_phone/words.txt \
+    ./download/waves_yesno/0_0_0_1_0_0_0_1.wav \
+    ./download/waves_yesno/0_0_1_0_0_0_1_0.wav \
+    ./download/waves_yesno/0_0_1_0_0_1_1_1.wav
+
+log "Show generated files"
+ls -lh tdnn/exp
+ls -lh data/lang_phone

.github/workflows/aishell.yml

@@ -0,0 +1,72 @@
+name: aishell
+
+on:
+  push:
+    branches:
+      - master
+
+  pull_request:
+    branches:
+      - master
+
+  workflow_dispatch:
+
+concurrency:
+  group: aishell-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  generate_build_matrix:
+    if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && (github.event.label.name == 'ready' || github.event_name == 'push' || github.event_name == 'aishell')
+
+    # see https://github.com/pytorch/pytorch/pull/50633
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Generating build matrix
+        id: set-matrix
+        run: |
+          # outputting for debugging purposes
+          python ./.github/scripts/docker/generate_build_matrix.py
+          MATRIX=$(python ./.github/scripts/docker/generate_build_matrix.py)
+          echo "::set-output name=matrix::${MATRIX}"
+  aishell:
+    needs: generate_build_matrix
+    name: py${{ matrix.python-version }} torch${{ matrix.torch-version }} v${{ matrix.version }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        ${{ fromJson(needs.generate_build_matrix.outputs.matrix) }}
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Free space
+        shell: bash
+        run: |
+          df -h
+          rm -rf /opt/hostedtoolcache
+          df -h
+          echo "pwd: $PWD"
+          echo "github.workspace ${{ github.workspace }}"
+
+      - name: Run aishell tests
+        uses: addnab/docker-run-action@v3
+        with:
+            image: ghcr.io/${{ github.repository_owner }}/icefall:cpu-py${{ matrix.python-version }}-torch${{ matrix.torch-version }}-v${{ matrix.version }}
+            options: |
+              --volume ${{ github.workspace }}/:/icefall
+            shell: bash
+            run: |
+              export PYTHONPATH=/icefall:$PYTHONPATH
+              cd /icefall
+              git config --global --add safe.directory /icefall
+
+              .github/scripts/aishell/ASR/run.sh

.github/workflows/build-cpu-docker.yml

@@ -0,0 +1,81 @@
+name: build-cpu-docker
+on:
+  workflow_dispatch:
+
+concurrency:
+  group: build-cpu-docker-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  generate_build_matrix:
+    if: github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa'
+    # see https://github.com/pytorch/pytorch/pull/50633
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Generating build matrix
+        id: set-matrix
+        run: |
+          # outputting for debugging purposes
+          python ./.github/scripts/docker/generate_build_matrix.py
+          MATRIX=$(python ./.github/scripts/docker/generate_build_matrix.py)
+          echo "::set-output name=matrix::${MATRIX}"
+  build-cpu-docker:
+    needs: generate_build_matrix
+    name: py${{ matrix.python-version }} torch${{ matrix.torch-version }} v${{ matrix.version }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        ${{ fromJson(needs.generate_build_matrix.outputs.matrix) }}
+
+    steps:
+      # refer to https://github.com/actions/checkout
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Free space
+        shell: bash
+        run: |
+          df -h
+          rm -rf /opt/hostedtoolcache
+          df -h
+
+      - name: 'Login to GitHub Container Registry'
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build docker Image
+        shell: bash
+        run: |
+          cd .github/scripts/docker
+          torch_version=${{ matrix.torch-version }}
+          torchaudio_version=${{ matrix.torchaudio-version }}
+
+          echo "torch_version: $torch_version"
+          echo "torchaudio_version: $torchaudio_version"
+
+          version=${{ matrix.version }}
+
+          tag=ghcr.io/${{ github.repository_owner }}/icefall:cpu-py${{ matrix.python-version }}-torch${{ matrix.torch-version }}-v$version
+          echo "tag: $tag"
+
+          docker build \
+            -t $tag \
+            --build-arg PYTHON_VERSION=${{ matrix.python-version }} \
+            --build-arg TORCH_VERSION=$torch_version \
+            --build-arg TORCHAUDIO_VERSION=$torchaudio_version \
+            --build-arg K2_VERSION=${{ matrix.k2-version }} \
+            --build-arg KALDIFEAT_VERSION=${{ matrix.kaldifeat-version }} \
+            .
+
+          docker image ls
+          docker push $tag

.github/workflows/librispeech.yml

@@ -0,0 +1,71 @@
+name: librispeech
+on:
+  push:
+    branches:
+      - master
+
+  pull_request:
+    branches:
+      - master
+
+  workflow_dispatch:
+
+concurrency:
+  group: librispeech-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  generate_build_matrix:
+    if: github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa'
+    # see https://github.com/pytorch/pytorch/pull/50633
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Generating build matrix
+        id: set-matrix
+        run: |
+          # outputting for debugging purposes
+          python ./.github/scripts/docker/generate_build_matrix.py
+          MATRIX=$(python ./.github/scripts/docker/generate_build_matrix.py)
+          echo "::set-output name=matrix::${MATRIX}"
+  librispeech:
+    needs: generate_build_matrix
+    name: py${{ matrix.python-version }} torch${{ matrix.torch-version }} v${{ matrix.version }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        ${{ fromJson(needs.generate_build_matrix.outputs.matrix) }}
+
+    steps:
+      # refer to https://github.com/actions/checkout
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Free space
+        shell: bash
+        run: |
+          df -h
+          rm -rf /opt/hostedtoolcache
+          df -h
+          echo "pwd: $PWD"
+          echo "github.workspace ${{ github.workspace }}"
+
+      - name: Test zipformer/train.py with LibriSpeech
+        uses: addnab/docker-run-action@v3
+        with:
+            image: ghcr.io/${{ github.repository_owner }}/icefall:cpu-py${{ matrix.python-version }}-torch${{ matrix.torch-version }}-v${{ matrix.version }}
+            options: |
+              --volume ${{ github.workspace }}/:/icefall
+            shell: bash
+            run: |
+              export PYTHONPATH=/icefall:$PYTHONPATH
+              cd /icefall
+              git config --global --add safe.directory /icefall
+
+              .github/scripts/librispeech/ASR/run.sh

.github/workflows/multi-zh-hans.yml

@@ -1,45 +1,30 @@
-# Copyright      2021  Fangjun Kuang (csukuangfj@gmail.com)
+name: run-multi-zh-hans
-
-# See ../../LICENSE for clarification regarding multiple authors
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: run-pre-trained-transducer
 
 on:
   push:
     branches:
       - master
-  pull_request:
+
-    types: [labeled]
+  workflow_dispatch:
 
 concurrency:
-  group: run_pre_trained_transducer-${{ github.ref }}
+  group: run-multi-zh-hans-${{ github.ref }}
   cancel-in-progress: true
 
+permissions:
+  contents: write
+
 jobs:
-  run_pre_trained_transducer:
+  multi-zh-hans:
-    if: github.event.label.name == 'ready' || github.event_name == 'push'
     runs-on: ${{ matrix.os }}
     strategy:
+      fail-fast: false
       matrix:
         os: [ubuntu-latest]
         python-version: [3.8]
 
-      fail-fast: false
-
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
@@ -68,13 +53,27 @@ jobs:
         if: steps.my-cache.outputs.cache-hit != 'true'
         shell: bash
         run: |
-          make -j2 _kaldifeat
+          .github/scripts/install-kaldifeat.sh
 
-      - name: Inference with pre-trained model
+      - name: export-model
         shell: bash
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
         run: |
           sudo apt-get -qq install git-lfs tree
           export PYTHONPATH=$PWD:$PYTHONPATH
           export PYTHONPATH=~/tmp/kaldifeat/kaldifeat/python:$PYTHONPATH
           export PYTHONPATH=~/tmp/kaldifeat/build/lib:$PYTHONPATH
-          .github/scripts/run-pre-trained-transducer.sh
+
+          .github/scripts/multi-zh-hans.sh
+          ls -lh
+
+      - name: upload model to https://github.com/k2-fsa/sherpa-onnx
+        uses: svenstaro/upload-release-action@v2
+        with:
+          file_glob: true
+          file: ./*.tar.bz2
+          overwrite: true
+          repo_name: k2-fsa/sherpa-onnx
+          repo_token: ${{ secrets.UPLOAD_GH_SHERPA_ONNX_TOKEN }}
+          tag: asr-models

.github/workflows/run-aishell-2022-06-20.yml

@@ -1,123 +0,0 @@
-# Copyright      2022  Fangjun Kuang (csukuangfj@gmail.com)
-
-# See ../../LICENSE for clarification regarding multiple authors
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: run-aishell-2022-06-20
-# pruned RNN-T + reworked model with random combiner
-# https://huggingface.co/csukuangfj/icefall-aishell-pruned-transducer-stateless3-2022-06-20
-
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-    types: [labeled]
-
-  schedule:
-    # minute (0-59)
-    # hour (0-23)
-    # day of the month (1-31)
-    # month (1-12)
-    # day of the week (0-6)
-    # nightly build at 15:50 UTC time every day
-    - cron: "50 15 * * *"
-
-concurrency:
-  group: run_aishell_2022_06_20-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  run_aishell_2022_06_20:
-    if: github.event.label.name == 'ready' || github.event.label.name == 'run-decode' || github.event_name == 'push' || github.event_name == 'schedule'
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-latest]
-        python-version: [3.8]
-
-      fail-fast: false
-
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-
-      - name: Setup Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: 'pip'
-          cache-dependency-path: '**/requirements-ci.txt'
-
-      - name: Install Python dependencies
-        run: |
-          grep -v '^#' ./requirements-ci.txt  | xargs -n 1 -L 1 pip install
-          pip uninstall -y protobuf
-          pip install --no-binary protobuf protobuf==3.20.*
-
-      - name: Cache kaldifeat
-        id: my-cache
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/tmp/kaldifeat
-          key: cache-tmp-${{ matrix.python-version }}-2023-05-22
-
-      - name: Install kaldifeat
-        if: steps.my-cache.outputs.cache-hit != 'true'
-        shell: bash
-        run: |
-          .github/scripts/install-kaldifeat.sh
-
-      - name: Inference with pre-trained model
-        shell: bash
-        env:
-          GITHUB_EVENT_NAME: ${{ github.event_name }}
-          GITHUB_EVENT_LABEL_NAME: ${{ github.event.label.name }}
-        run: |
-          sudo apt-get -qq install git-lfs tree
-          export PYTHONPATH=$PWD:$PYTHONPATH
-          export PYTHONPATH=~/tmp/kaldifeat/kaldifeat/python:$PYTHONPATH
-          export PYTHONPATH=~/tmp/kaldifeat/build/lib:$PYTHONPATH
-
-          .github/scripts/run-aishell-pruned-transducer-stateless3-2022-06-20.sh
-
-      - name: Display decoding results for aishell pruned_transducer_stateless3
-        if: github.event_name == 'schedule' || github.event.label.name == 'run-decode'
-        shell: bash
-        run: |
-          cd egs/aishell/ASR/
-          tree ./pruned_transducer_stateless3/exp
-
-          cd pruned_transducer_stateless3
-          echo "results for pruned_transducer_stateless3"
-          echo "===greedy search==="
-          find exp/greedy_search -name "log-*" -exec grep -n --color "best for test" {} + | sort -n -k2
-          find exp/greedy_search -name "log-*" -exec grep -n --color "best for dev" {} + | sort -n -k2
-
-          echo "===fast_beam_search==="
-          find exp/fast_beam_search -name "log-*" -exec grep -n --color "best for test" {} + | sort -n -k2
-          find exp/fast_beam_search -name "log-*" -exec grep -n --color "best for dev" {} + | sort -n -k2
-
-          echo "===modified beam search==="
-          find exp/modified_beam_search -name "log-*" -exec grep -n --color "best for test" {} + | sort -n -k2
-          find exp/modified_beam_search -name "log-*" -exec grep -n --color "best for dev" {} + | sort -n -k2
-
-      - name: Upload decoding results for aishell pruned_transducer_stateless3
-        uses: actions/upload-artifact@v2
-        if: github.event_name == 'schedule' || github.event.label.name == 'run-decode'
-        with:
-          name: aishell-torch-${{ matrix.torch }}-python-${{ matrix.python-version }}-ubuntu-latest-cpu-pruned_transducer_stateless3-2022-06-20
-          path: egs/aishell/ASR/pruned_transducer_stateless3/exp/

.github/workflows/run-gigaspeech-zipformer-2023-10-17.yml

@@ -21,6 +21,7 @@ on:
   push:
     branches:
       - master
+
   pull_request:
     types: [labeled]
 
@@ -33,6 +34,8 @@ on:
     # nightly build at 15:50 UTC time every day
     - cron: "50 15 * * *"
 
+  workflow_dispatch:
+
 concurrency:
   group: run_gigaspeech_2023_10_17_zipformer-${{ github.ref }}
   cancel-in-progress: true
@@ -85,6 +88,7 @@ jobs:
         env:
           GITHUB_EVENT_NAME: ${{ github.event_name }}
           GITHUB_EVENT_LABEL_NAME: ${{ github.event.label.name }}
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
         run: |
           mkdir -p egs/gigaspeech/ASR/data
           ln -sfv ~/tmp/fbank-libri egs/gigaspeech/ASR/data/fbank
@@ -97,6 +101,16 @@ jobs:
 
           .github/scripts/run-gigaspeech-zipformer-2023-10-17.sh
 
+      - name: upload model to https://github.com/k2-fsa/sherpa-onnx
+        uses: svenstaro/upload-release-action@v2
+        with:
+          file_glob: true
+          file: ./*.tar.bz2
+          overwrite: true
+          repo_name: k2-fsa/sherpa-onnx
+          repo_token: ${{ secrets.UPLOAD_GH_SHERPA_ONNX_TOKEN }}
+          tag: asr-models
+
       - name: Display decoding results for gigaspeech zipformer
         if: github.event_name == 'schedule' || github.event.label.name == 'run-decode'
         shell: bash

.github/workflows/run-librispeech-2022-03-12.yml

@@ -1,159 +0,0 @@
-# Copyright      2021  Fangjun Kuang (csukuangfj@gmail.com)
-
-# See ../../LICENSE for clarification regarding multiple authors
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: run-librispeech-2022-03-12
-# stateless transducer + k2 pruned rnnt-loss
-
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-    types: [labeled]
-
-  schedule:
-    # minute (0-59)
-    # hour (0-23)
-    # day of the month (1-31)
-    # month (1-12)
-    # day of the week (0-6)
-    # nightly build at 15:50 UTC time every day
-    - cron: "50 15 * * *"
-
-concurrency:
-  group: run_librispeech_2022_03_12-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  run_librispeech_2022_03_12:
-    if: github.event.label.name == 'ready' || github.event.label.name == 'run-decode' || github.event_name == 'push' || github.event_name == 'schedule'
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-latest]
-        python-version: [3.8]
-
-      fail-fast: false
-
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-
-      - name: Setup Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: 'pip'
-          cache-dependency-path: '**/requirements-ci.txt'
-
-      - name: Install Python dependencies
-        run: |
-          grep -v '^#' ./requirements-ci.txt  | xargs -n 1 -L 1 pip install
-          pip uninstall -y protobuf
-          pip install --no-binary protobuf protobuf==3.20.*
-
-      - name: Cache kaldifeat
-        id: my-cache
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/tmp/kaldifeat
-          key: cache-tmp-${{ matrix.python-version }}-2023-05-22
-
-      - name: Install kaldifeat
-        if: steps.my-cache.outputs.cache-hit != 'true'
-        shell: bash
-        run: |
-          .github/scripts/install-kaldifeat.sh
-
-      - name: Cache LibriSpeech test-clean and test-other datasets
-        id: libri-test-clean-and-test-other-data
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/tmp/download
-          key: cache-libri-test-clean-and-test-other
-
-      - name: Download LibriSpeech test-clean and test-other
-        if: steps.libri-test-clean-and-test-other-data.outputs.cache-hit != 'true'
-        shell: bash
-        run: |
-          .github/scripts/download-librispeech-test-clean-and-test-other-dataset.sh
-
-      - name: Prepare manifests for LibriSpeech test-clean and test-other
-        shell: bash
-        run: |
-          .github/scripts/prepare-librispeech-test-clean-and-test-other-manifests.sh
-
-      - name: Cache LibriSpeech test-clean and test-other fbank features
-        id: libri-test-clean-and-test-other-fbank
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/tmp/fbank-libri
-          key: cache-libri-fbank-test-clean-and-test-other-v2
-
-      - name: Compute fbank for LibriSpeech test-clean and test-other
-        if: steps.libri-test-clean-and-test-other-fbank.outputs.cache-hit != 'true'
-        shell: bash
-        run: |
-          .github/scripts/compute-fbank-librispeech-test-clean-and-test-other.sh
-
-      - name: Inference with pre-trained model
-        shell: bash
-        env:
-          GITHUB_EVENT_NAME: ${{ github.event_name }}
-          GITHUB_EVENT_LABEL_NAME: ${{ github.event.label.name }}
-        run: |
-          mkdir -p egs/librispeech/ASR/data
-          ln -sfv ~/tmp/fbank-libri egs/librispeech/ASR/data/fbank
-          ls -lh egs/librispeech/ASR/data/*
-
-          sudo apt-get -qq install git-lfs tree
-          export PYTHONPATH=$PWD:$PYTHONPATH
-          export PYTHONPATH=~/tmp/kaldifeat/kaldifeat/python:$PYTHONPATH
-          export PYTHONPATH=~/tmp/kaldifeat/build/lib:$PYTHONPATH
-
-          .github/scripts/run-librispeech-pruned-transducer-stateless-2022-03-12.sh
-
-      - name: Display decoding results for pruned_transducer_stateless
-        if: github.event_name == 'schedule' || github.event.label.name == 'run-decode'
-        shell: bash
-        run: |
-          cd egs/librispeech/ASR/
-          tree ./pruned_transducer_stateless/exp
-
-          cd pruned_transducer_stateless
-          echo "results for pruned_transducer_stateless"
-          echo "===greedy search==="
-          find exp/greedy_search -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find exp/greedy_search -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-          echo "===fast_beam_search==="
-          find exp/fast_beam_search -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find exp/fast_beam_search -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-          echo "===modified beam search==="
-          find exp/modified_beam_search -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find exp/modified_beam_search -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-      - name: Upload decoding results for pruned_transducer_stateless
-        uses: actions/upload-artifact@v2
-        if: github.event_name == 'schedule' || github.event.label.name == 'run-decode'
-        with:
-          name: torch-${{ matrix.torch }}-python-${{ matrix.python-version }}-ubuntu-latest-cpu-pruned_transducer_stateless-2022-03-12
-          path: egs/librispeech/ASR/pruned_transducer_stateless/exp/

.github/workflows/run-librispeech-2022-04-29.yml

@@ -1,185 +0,0 @@
-# Copyright      2021  Fangjun Kuang (csukuangfj@gmail.com)
-
-# See ../../LICENSE for clarification regarding multiple authors
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: run-librispeech-2022-04-29
-# stateless pruned transducer (reworked model) + giga speech
-
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-    types: [labeled]
-
-  schedule:
-    # minute (0-59)
-    # hour (0-23)
-    # day of the month (1-31)
-    # month (1-12)
-    # day of the week (0-6)
-    # nightly build at 15:50 UTC time every day
-    - cron: "50 15 * * *"
-
-concurrency:
-  group: run_librispeech_2022_04_29-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  run_librispeech_2022_04_29:
-    if: github.event.label.name == 'ready' || github.event.label.name == 'run-decode' || github.event_name == 'push' || github.event_name == 'schedule'
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-latest]
-        python-version: [3.8]
-
-      fail-fast: false
-
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-
-      - name: Setup Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: 'pip'
-          cache-dependency-path: '**/requirements-ci.txt'
-
-      - name: Install Python dependencies
-        run: |
-          grep -v '^#' ./requirements-ci.txt  | xargs -n 1 -L 1 pip install
-          pip uninstall -y protobuf
-          pip install --no-binary protobuf protobuf==3.20.*
-
-      - name: Cache kaldifeat
-        id: my-cache
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/tmp/kaldifeat
-          key: cache-tmp-${{ matrix.python-version }}-2023-05-22
-
-      - name: Install kaldifeat
-        if: steps.my-cache.outputs.cache-hit != 'true'
-        shell: bash
-        run: |
-          .github/scripts/install-kaldifeat.sh
-
-      - name: Cache LibriSpeech test-clean and test-other datasets
-        id: libri-test-clean-and-test-other-data
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/tmp/download
-          key: cache-libri-test-clean-and-test-other
-
-      - name: Download LibriSpeech test-clean and test-other
-        if: steps.libri-test-clean-and-test-other-data.outputs.cache-hit != 'true'
-        shell: bash
-        run: |
-          .github/scripts/download-librispeech-test-clean-and-test-other-dataset.sh
-
-      - name: Prepare manifests for LibriSpeech test-clean and test-other
-        shell: bash
-        run: |
-          .github/scripts/prepare-librispeech-test-clean-and-test-other-manifests.sh
-
-      - name: Cache LibriSpeech test-clean and test-other fbank features
-        id: libri-test-clean-and-test-other-fbank
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/tmp/fbank-libri
-          key: cache-libri-fbank-test-clean-and-test-other-v2
-
-      - name: Compute fbank for LibriSpeech test-clean and test-other
-        if: steps.libri-test-clean-and-test-other-fbank.outputs.cache-hit != 'true'
-        shell: bash
-        run: |
-          .github/scripts/compute-fbank-librispeech-test-clean-and-test-other.sh
-
-      - name: Inference with pre-trained model
-        shell: bash
-        env:
-          GITHUB_EVENT_NAME: ${{ github.event_name }}
-          GITHUB_EVENT_LABEL_NAME: ${{ github.event.label.name }}
-        run: |
-          mkdir -p egs/librispeech/ASR/data
-          ln -sfv ~/tmp/fbank-libri egs/librispeech/ASR/data/fbank
-          ls -lh egs/librispeech/ASR/data/*
-
-          sudo apt-get -qq install git-lfs tree
-          export PYTHONPATH=$PWD:$PYTHONPATH
-          export PYTHONPATH=~/tmp/kaldifeat/kaldifeat/python:$PYTHONPATH
-          export PYTHONPATH=~/tmp/kaldifeat/build/lib:$PYTHONPATH
-
-          .github/scripts/run-librispeech-pruned-transducer-stateless2-2022-04-29.sh
-
-          .github/scripts/run-librispeech-pruned-transducer-stateless3-2022-04-29.sh
-
-      - name: Display decoding results for pruned_transducer_stateless2
-        if: github.event_name == 'schedule' || github.event.label.name == 'run-decode'
-        shell: bash
-        run: |
-          cd egs/librispeech/ASR
-          tree pruned_transducer_stateless2/exp
-          cd pruned_transducer_stateless2/exp
-          echo "===greedy search==="
-          find greedy_search -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find greedy_search -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-          echo "===fast_beam_search==="
-          find fast_beam_search -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find fast_beam_search -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-          echo "===modified beam search==="
-          find modified_beam_search -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find modified_beam_search -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-      - name: Display decoding results for pruned_transducer_stateless3
-        if: github.event_name == 'schedule' || github.event.label.name == 'run-decode'
-        shell: bash
-        run: |
-          cd egs/librispeech/ASR
-          tree pruned_transducer_stateless3/exp
-          cd pruned_transducer_stateless3/exp
-          echo "===greedy search==="
-          find greedy_search -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find greedy_search -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-          echo "===fast_beam_search==="
-          find fast_beam_search -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find fast_beam_search -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-          echo "===modified beam search==="
-          find modified_beam_search -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find modified_beam_search -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-      - name: Upload decoding results for pruned_transducer_stateless2
-        uses: actions/upload-artifact@v2
-        if: github.event_name == 'schedule' || github.event.label.name == 'run-decode'
-        with:
-          name: torch-${{ matrix.torch }}-python-${{ matrix.python-version }}-ubuntu-latest-cpu-pruned_transducer_stateless2-2022-04-29
-          path: egs/librispeech/ASR/pruned_transducer_stateless2/exp/
-
-      - name: Upload decoding results for pruned_transducer_stateless3
-        uses: actions/upload-artifact@v2
-        if: github.event_name == 'schedule' || github.event.label.name == 'run-decode'
-        with:
-          name: torch-${{ matrix.torch }}-python-${{ matrix.python-version }}-ubuntu-latest-cpu-pruned_transducer_stateless3-2022-04-29
-          path: egs/librispeech/ASR/pruned_transducer_stateless3/exp/

.github/workflows/run-librispeech-2022-05-13.yml

@@ -1,159 +0,0 @@
-# Copyright      2022  Fangjun Kuang (csukuangfj@gmail.com)
-
-# See ../../LICENSE for clarification regarding multiple authors
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: run-librispeech-2022-05-13
-# stateless transducer + k2 pruned rnnt-loss + deeper model
-
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-    types: [labeled]
-
-  schedule:
-    # minute (0-59)
-    # hour (0-23)
-    # day of the month (1-31)
-    # month (1-12)
-    # day of the week (0-6)
-    # nightly build at 15:50 UTC time every day
-    - cron: "50 15 * * *"
-
-concurrency:
-  group: run_librispeech_2022_05_13-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  run_librispeech_2022_05_13:
-    if: github.event.label.name == 'ready' || github.event.label.name == 'run-decode' || github.event_name == 'push' || github.event_name == 'schedule'
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-latest]
-        python-version: [3.8]
-
-      fail-fast: false
-
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-
-      - name: Setup Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: 'pip'
-          cache-dependency-path: '**/requirements-ci.txt'
-
-      - name: Install Python dependencies
-        run: |
-          grep -v '^#' ./requirements-ci.txt  | xargs -n 1 -L 1 pip install
-          pip uninstall -y protobuf
-          pip install --no-binary protobuf protobuf==3.20.*
-
-      - name: Cache kaldifeat
-        id: my-cache
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/tmp/kaldifeat
-          key: cache-tmp-${{ matrix.python-version }}-2023-05-22
-
-      - name: Install kaldifeat
-        if: steps.my-cache.outputs.cache-hit != 'true'
-        shell: bash
-        run: |
-          .github/scripts/install-kaldifeat.sh
-
-      - name: Cache LibriSpeech test-clean and test-other datasets
-        id: libri-test-clean-and-test-other-data
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/tmp/download
-          key: cache-libri-test-clean-and-test-other
-
-      - name: Download LibriSpeech test-clean and test-other
-        if: steps.libri-test-clean-and-test-other-data.outputs.cache-hit != 'true'
-        shell: bash
-        run: |
-          .github/scripts/download-librispeech-test-clean-and-test-other-dataset.sh
-
-      - name: Prepare manifests for LibriSpeech test-clean and test-other
-        shell: bash
-        run: |
-          .github/scripts/prepare-librispeech-test-clean-and-test-other-manifests.sh
-
-      - name: Cache LibriSpeech test-clean and test-other fbank features
-        id: libri-test-clean-and-test-other-fbank
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/tmp/fbank-libri
-          key: cache-libri-fbank-test-clean-and-test-other-v2
-
-      - name: Compute fbank for LibriSpeech test-clean and test-other
-        if: steps.libri-test-clean-and-test-other-fbank.outputs.cache-hit != 'true'
-        shell: bash
-        run: |
-          .github/scripts/compute-fbank-librispeech-test-clean-and-test-other.sh
-
-      - name: Inference with pre-trained model
-        shell: bash
-        env:
-          GITHUB_EVENT_NAME: ${{ github.event_name }}
-          GITHUB_EVENT_LABEL_NAME: ${{ github.event.label.name }}
-        run: |
-          mkdir -p egs/librispeech/ASR/data
-          ln -sfv ~/tmp/fbank-libri egs/librispeech/ASR/data/fbank
-          ls -lh egs/librispeech/ASR/data/*
-
-          sudo apt-get -qq install git-lfs tree
-          export PYTHONPATH=$PWD:$PYTHONPATH
-          export PYTHONPATH=~/tmp/kaldifeat/kaldifeat/python:$PYTHONPATH
-          export PYTHONPATH=~/tmp/kaldifeat/build/lib:$PYTHONPATH
-
-          .github/scripts/run-librispeech-pruned-transducer-stateless5-2022-05-13.sh
-
-      - name: Display decoding results for librispeech pruned_transducer_stateless5
-        if: github.event_name == 'schedule' || github.event.label.name == 'run-decode'
-        shell: bash
-        run: |
-          cd egs/librispeech/ASR/
-          tree ./pruned_transducer_stateless5/exp
-
-          cd pruned_transducer_stateless5
-          echo "results for pruned_transducer_stateless5"
-          echo "===greedy search==="
-          find exp/greedy_search -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find exp/greedy_search -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-          echo "===fast_beam_search==="
-          find exp/fast_beam_search -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find exp/fast_beam_search -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-          echo "===modified beam search==="
-          find exp/modified_beam_search -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find exp/modified_beam_search -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-      - name: Upload decoding results for librispeech pruned_transducer_stateless5
-        uses: actions/upload-artifact@v2
-        if: github.event_name == 'schedule' || github.event.label.name == 'run-decode'
-        with:
-          name: torch-${{ matrix.torch }}-python-${{ matrix.python-version }}-ubuntu-latest-cpu-pruned_transducer_stateless5-2022-05-13
-          path: egs/librispeech/ASR/pruned_transducer_stateless5/exp/

.github/workflows/run-librispeech-2022-11-11-stateless7.yml

@@ -1,159 +0,0 @@
-# Copyright      2022  Fangjun Kuang (csukuangfj@gmail.com)
-
-# See ../../LICENSE for clarification regarding multiple authors
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: run-librispeech-2022-11-11-stateless7
-# zipformer
-
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-    types: [labeled]
-
-  schedule:
-    # minute (0-59)
-    # hour (0-23)
-    # day of the month (1-31)
-    # month (1-12)
-    # day of the week (0-6)
-    # nightly build at 15:50 UTC time every day
-    - cron: "50 15 * * *"
-
-concurrency:
-  group: run_librispeech_2022_11_11_zipformer-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  run_librispeech_2022_11_11_zipformer:
-    if: github.event.label.name == 'ready' || github.event.label.name == 'run-decode' || github.event_name == 'push' || github.event_name == 'schedule'
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-latest]
-        python-version: [3.8]
-
-      fail-fast: false
-
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-
-      - name: Setup Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: 'pip'
-          cache-dependency-path: '**/requirements-ci.txt'
-
-      - name: Install Python dependencies
-        run: |
-          grep -v '^#' ./requirements-ci.txt  | xargs -n 1 -L 1 pip install
-          pip uninstall -y protobuf
-          pip install --no-binary protobuf protobuf==3.20.*
-
-      - name: Cache kaldifeat
-        id: my-cache
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/tmp/kaldifeat
-          key: cache-tmp-${{ matrix.python-version }}-2023-05-22
-
-      - name: Install kaldifeat
-        if: steps.my-cache.outputs.cache-hit != 'true'
-        shell: bash
-        run: |
-          .github/scripts/install-kaldifeat.sh
-
-      - name: Cache LibriSpeech test-clean and test-other datasets
-        id: libri-test-clean-and-test-other-data
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/tmp/download
-          key: cache-libri-test-clean-and-test-other
-
-      - name: Download LibriSpeech test-clean and test-other
-        if: steps.libri-test-clean-and-test-other-data.outputs.cache-hit != 'true'
-        shell: bash
-        run: |
-          .github/scripts/download-librispeech-test-clean-and-test-other-dataset.sh
-
-      - name: Prepare manifests for LibriSpeech test-clean and test-other
-        shell: bash
-        run: |
-          .github/scripts/prepare-librispeech-test-clean-and-test-other-manifests.sh
-
-      - name: Cache LibriSpeech test-clean and test-other fbank features
-        id: libri-test-clean-and-test-other-fbank
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/tmp/fbank-libri
-          key: cache-libri-fbank-test-clean-and-test-other-v2
-
-      - name: Compute fbank for LibriSpeech test-clean and test-other
-        if: steps.libri-test-clean-and-test-other-fbank.outputs.cache-hit != 'true'
-        shell: bash
-        run: |
-          .github/scripts/compute-fbank-librispeech-test-clean-and-test-other.sh
-
-      - name: Inference with pre-trained model
-        shell: bash
-        env:
-          GITHUB_EVENT_NAME: ${{ github.event_name }}
-          GITHUB_EVENT_LABEL_NAME: ${{ github.event.label.name }}
-        run: |
-          mkdir -p egs/librispeech/ASR/data
-          ln -sfv ~/tmp/fbank-libri egs/librispeech/ASR/data/fbank
-          ls -lh egs/librispeech/ASR/data/*
-
-          sudo apt-get -qq install git-lfs tree
-          export PYTHONPATH=$PWD:$PYTHONPATH
-          export PYTHONPATH=~/tmp/kaldifeat/kaldifeat/python:$PYTHONPATH
-          export PYTHONPATH=~/tmp/kaldifeat/build/lib:$PYTHONPATH
-
-          .github/scripts/run-librispeech-pruned-transducer-stateless7-2022-11-11.sh
-
-      - name: Display decoding results for librispeech pruned_transducer_stateless7
-        if: github.event_name == 'schedule' || github.event.label.name == 'run-decode'
-        shell: bash
-        run: |
-          cd egs/librispeech/ASR/
-          tree ./pruned_transducer_stateless7/exp
-
-          cd pruned_transducer_stateless7
-          echo "results for pruned_transducer_stateless7"
-          echo "===greedy search==="
-          find exp/greedy_search -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find exp/greedy_search -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-          echo "===fast_beam_search==="
-          find exp/fast_beam_search -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find exp/fast_beam_search -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-          echo "===modified beam search==="
-          find exp/modified_beam_search -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find exp/modified_beam_search -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-      - name: Upload decoding results for librispeech pruned_transducer_stateless7
-        uses: actions/upload-artifact@v2
-        if: github.event_name == 'schedule' || github.event.label.name == 'run-decode'
-        with:
-          name: torch-${{ matrix.torch }}-python-${{ matrix.python-version }}-ubuntu-latest-cpu-pruned_transducer_stateless7-2022-11-11
-          path: egs/librispeech/ASR/pruned_transducer_stateless7/exp/

.github/workflows/run-librispeech-2022-11-14-stateless8.yml

@@ -1,159 +0,0 @@
-# Copyright      2022  Fangjun Kuang (csukuangfj@gmail.com)
-
-# See ../../LICENSE for clarification regarding multiple authors
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: run-librispeech-2022-11-14-stateless8
-# zipformer
-
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-    types: [labeled]
-
-  schedule:
-    # minute (0-59)
-    # hour (0-23)
-    # day of the month (1-31)
-    # month (1-12)
-    # day of the week (0-6)
-    # nightly build at 15:50 UTC time every day
-    - cron: "50 15 * * *"
-
-concurrency:
-  group: run_librispeech_2022_11_14_zipformer_stateless8-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  run_librispeech_2022_11_14_zipformer_stateless8:
-    if: github.event.label.name == 'ready' || github.event.label.name == 'run-decode' || github.event_name == 'push' || github.event_name == 'schedule'
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-latest]
-        python-version: [3.8]
-
-      fail-fast: false
-
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-
-      - name: Setup Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: 'pip'
-          cache-dependency-path: '**/requirements-ci.txt'
-
-      - name: Install Python dependencies
-        run: |
-          grep -v '^#' ./requirements-ci.txt  | xargs -n 1 -L 1 pip install
-          pip uninstall -y protobuf
-          pip install --no-binary protobuf protobuf==3.20.*
-
-      - name: Cache kaldifeat
-        id: my-cache
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/tmp/kaldifeat
-          key: cache-tmp-${{ matrix.python-version }}-2023-05-22
-
-      - name: Install kaldifeat
-        if: steps.my-cache.outputs.cache-hit != 'true'
-        shell: bash
-        run: |
-          .github/scripts/install-kaldifeat.sh
-
-      - name: Cache LibriSpeech test-clean and test-other datasets
-        id: libri-test-clean-and-test-other-data
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/tmp/download
-          key: cache-libri-test-clean-and-test-other
-
-      - name: Download LibriSpeech test-clean and test-other
-        if: steps.libri-test-clean-and-test-other-data.outputs.cache-hit != 'true'
-        shell: bash
-        run: |
-          .github/scripts/download-librispeech-test-clean-and-test-other-dataset.sh
-
-      - name: Prepare manifests for LibriSpeech test-clean and test-other
-        shell: bash
-        run: |
-          .github/scripts/prepare-librispeech-test-clean-and-test-other-manifests.sh
-
-      - name: Cache LibriSpeech test-clean and test-other fbank features
-        id: libri-test-clean-and-test-other-fbank
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/tmp/fbank-libri
-          key: cache-libri-fbank-test-clean-and-test-other-v2
-
-      - name: Compute fbank for LibriSpeech test-clean and test-other
-        if: steps.libri-test-clean-and-test-other-fbank.outputs.cache-hit != 'true'
-        shell: bash
-        run: |
-          .github/scripts/compute-fbank-librispeech-test-clean-and-test-other.sh
-
-      - name: Inference with pre-trained model
-        shell: bash
-        env:
-          GITHUB_EVENT_NAME: ${{ github.event_name }}
-          GITHUB_EVENT_LABEL_NAME: ${{ github.event.label.name }}
-        run: |
-          mkdir -p egs/librispeech/ASR/data
-          ln -sfv ~/tmp/fbank-libri egs/librispeech/ASR/data/fbank
-          ls -lh egs/librispeech/ASR/data/*
-
-          sudo apt-get -qq install git-lfs tree
-          export PYTHONPATH=$PWD:$PYTHONPATH
-          export PYTHONPATH=~/tmp/kaldifeat/kaldifeat/python:$PYTHONPATH
-          export PYTHONPATH=~/tmp/kaldifeat/build/lib:$PYTHONPATH
-
-          .github/scripts/run-librispeech-pruned-transducer-stateless8-2022-11-14.sh
-
-      - name: Display decoding results for librispeech pruned_transducer_stateless8
-        if: github.event_name == 'schedule' || github.event.label.name == 'run-decode'
-        shell: bash
-        run: |
-          cd egs/librispeech/ASR/
-          tree ./pruned_transducer_stateless8/exp
-
-          cd pruned_transducer_stateless8
-          echo "results for pruned_transducer_stateless8"
-          echo "===greedy search==="
-          find exp/greedy_search -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find exp/greedy_search -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-          echo "===fast_beam_search==="
-          find exp/fast_beam_search -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find exp/fast_beam_search -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-          echo "===modified beam search==="
-          find exp/modified_beam_search -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find exp/modified_beam_search -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-      - name: Upload decoding results for librispeech pruned_transducer_stateless8
-        uses: actions/upload-artifact@v2
-        if: github.event_name == 'schedule' || github.event.label.name == 'run-decode'
-        with:
-          name: torch-${{ matrix.torch }}-python-${{ matrix.python-version }}-ubuntu-latest-cpu-pruned_transducer_stateless8-2022-11-14
-          path: egs/librispeech/ASR/pruned_transducer_stateless8/exp/

.github/workflows/run-librispeech-2022-12-01-stateless7-ctc.yml

@@ -1,163 +0,0 @@
-# Copyright      2022  Fangjun Kuang (csukuangfj@gmail.com)
-
-# See ../../LICENSE for clarification regarding multiple authors
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: run-librispeech-2022-12-01-stateless7-ctc
-# zipformer
-
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-    types: [labeled]
-
-  schedule:
-    # minute (0-59)
-    # hour (0-23)
-    # day of the month (1-31)
-    # month (1-12)
-    # day of the week (0-6)
-    # nightly build at 15:50 UTC time every day
-    - cron: "50 15 * * *"
-
-jobs:
-  run_librispeech_2022_11_11_zipformer:
-    if: github.event.label.name == 'ready' || github.event.label.name == 'run-decode' || github.event_name == 'push' || github.event_name == 'schedule'
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-latest]
-        python-version: [3.8]
-
-      fail-fast: false
-
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-
-      - name: Setup Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: 'pip'
-          cache-dependency-path: '**/requirements-ci.txt'
-
-      - name: Install Python dependencies
-        run: |
-          grep -v '^#' ./requirements-ci.txt  | xargs -n 1 -L 1 pip install
-          pip uninstall -y protobuf
-          pip install --no-binary protobuf protobuf==3.20.*
-
-      - name: Cache kaldifeat
-        id: my-cache
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/tmp/kaldifeat
-          key: cache-tmp-${{ matrix.python-version }}-2023-05-22
-
-      - name: Install kaldifeat
-        if: steps.my-cache.outputs.cache-hit != 'true'
-        shell: bash
-        run: |
-          .github/scripts/install-kaldifeat.sh
-
-      - name: Cache LibriSpeech test-clean and test-other datasets
-        id: libri-test-clean-and-test-other-data
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/tmp/download
-          key: cache-libri-test-clean-and-test-other
-
-      - name: Download LibriSpeech test-clean and test-other
-        if: steps.libri-test-clean-and-test-other-data.outputs.cache-hit != 'true'
-        shell: bash
-        run: |
-          .github/scripts/download-librispeech-test-clean-and-test-other-dataset.sh
-
-      - name: Prepare manifests for LibriSpeech test-clean and test-other
-        shell: bash
-        run: |
-          .github/scripts/prepare-librispeech-test-clean-and-test-other-manifests.sh
-
-      - name: Cache LibriSpeech test-clean and test-other fbank features
-        id: libri-test-clean-and-test-other-fbank
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/tmp/fbank-libri
-          key: cache-libri-fbank-test-clean-and-test-other-v2
-
-      - name: Compute fbank for LibriSpeech test-clean and test-other
-        if: steps.libri-test-clean-and-test-other-fbank.outputs.cache-hit != 'true'
-        shell: bash
-        run: |
-          .github/scripts/compute-fbank-librispeech-test-clean-and-test-other.sh
-
-      - name: Inference with pre-trained model
-        shell: bash
-        env:
-          GITHUB_EVENT_NAME: ${{ github.event_name }}
-          GITHUB_EVENT_LABEL_NAME: ${{ github.event.label.name }}
-        run: |
-          mkdir -p egs/librispeech/ASR/data
-          ln -sfv ~/tmp/fbank-libri egs/librispeech/ASR/data/fbank
-          ls -lh egs/librispeech/ASR/data/*
-
-          sudo apt-get -qq install git-lfs tree
-          export PYTHONPATH=$PWD:$PYTHONPATH
-          export PYTHONPATH=~/tmp/kaldifeat/kaldifeat/python:$PYTHONPATH
-          export PYTHONPATH=~/tmp/kaldifeat/build/lib:$PYTHONPATH
-
-          .github/scripts/run-librispeech-pruned-transducer-stateless7-ctc-2022-12-01.sh
-
-      - name: Display decoding results for librispeech pruned_transducer_stateless7_ctc
-        if: github.event_name == 'schedule' || github.event.label.name == 'run-decode'
-        shell: bash
-        run: |
-          cd egs/librispeech/ASR/
-          tree ./pruned_transducer_stateless7_ctc/exp
-
-          cd pruned_transducer_stateless7_ctc
-          echo "results for pruned_transducer_stateless7_ctc"
-          echo "===greedy search==="
-          find exp/greedy_search -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find exp/greedy_search -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-          echo "===fast_beam_search==="
-          find exp/fast_beam_search -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find exp/fast_beam_search -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-          echo "===modified beam search==="
-          find exp/modified_beam_search -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find exp/modified_beam_search -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-          echo "===ctc decoding==="
-          find exp/ctc-decoding -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find exp/ctc-decoding -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-          echo "===1best==="
-          find exp/1best -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find exp/1best -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-      - name: Upload decoding results for librispeech pruned_transducer_stateless7_ctc
-        uses: actions/upload-artifact@v2
-        if: github.event_name == 'schedule' || github.event.label.name == 'run-decode'
-        with:
-          name: torch-${{ matrix.torch }}-python-${{ matrix.python-version }}-ubuntu-latest-cpu-pruned_transducer_stateless7-ctc-2022-12-01
-          path: egs/librispeech/ASR/pruned_transducer_stateless7_ctc/exp/

.github/workflows/run-librispeech-2022-12-08-zipformer-mmi.yml

@@ -1,167 +0,0 @@
-# Copyright      2022  Zengwei Yao
-
-# See ../../LICENSE for clarification regarding multiple authors
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: run-librispeech-2022-12-08-zipformer-mmi
-# zipformer
-
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-    types: [labeled]
-
-  schedule:
-    # minute (0-59)
-    # hour (0-23)
-    # day of the month (1-31)
-    # month (1-12)
-    # day of the week (0-6)
-    # nightly build at 15:50 UTC time every day
-    - cron: "50 15 * * *"
-
-concurrency:
-  group: run_librispeech_2022_12_08_zipformer-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  run_librispeech_2022_12_08_zipformer:
-    if: github.event.label.name == 'ready' || github.event.label.name == 'run-decode' || github.event_name == 'push' || github.event_name == 'schedule'
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-latest]
-        python-version: [3.8]
-
-      fail-fast: false
-
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-
-      - name: Setup Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: 'pip'
-          cache-dependency-path: '**/requirements-ci.txt'
-
-      - name: Install Python dependencies
-        run: |
-          grep -v '^#' ./requirements-ci.txt  | xargs -n 1 -L 1 pip install
-          pip uninstall -y protobuf
-          pip install --no-binary protobuf protobuf==3.20.*
-
-      - name: Cache kaldifeat
-        id: my-cache
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/tmp/kaldifeat
-          key: cache-tmp-${{ matrix.python-version }}-2023-05-22
-
-      - name: Install kaldifeat
-        if: steps.my-cache.outputs.cache-hit != 'true'
-        shell: bash
-        run: |
-          .github/scripts/install-kaldifeat.sh
-
-      - name: Cache LibriSpeech test-clean and test-other datasets
-        id: libri-test-clean-and-test-other-data
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/tmp/download
-          key: cache-libri-test-clean-and-test-other
-
-      - name: Download LibriSpeech test-clean and test-other
-        if: steps.libri-test-clean-and-test-other-data.outputs.cache-hit != 'true'
-        shell: bash
-        run: |
-          .github/scripts/download-librispeech-test-clean-and-test-other-dataset.sh
-
-      - name: Prepare manifests for LibriSpeech test-clean and test-other
-        shell: bash
-        run: |
-          .github/scripts/prepare-librispeech-test-clean-and-test-other-manifests.sh
-
-      - name: Cache LibriSpeech test-clean and test-other fbank features
-        id: libri-test-clean-and-test-other-fbank
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/tmp/fbank-libri
-          key: cache-libri-fbank-test-clean-and-test-other-v2
-
-      - name: Compute fbank for LibriSpeech test-clean and test-other
-        if: steps.libri-test-clean-and-test-other-fbank.outputs.cache-hit != 'true'
-        shell: bash
-        run: |
-          .github/scripts/compute-fbank-librispeech-test-clean-and-test-other.sh
-
-      - name: Inference with pre-trained model
-        shell: bash
-        env:
-          GITHUB_EVENT_NAME: ${{ github.event_name }}
-          GITHUB_EVENT_LABEL_NAME: ${{ github.event.label.name }}
-        run: |
-          mkdir -p egs/librispeech/ASR/data
-          ln -sfv ~/tmp/fbank-libri egs/librispeech/ASR/data/fbank
-          ls -lh egs/librispeech/ASR/data/*
-
-          sudo apt-get -qq install git-lfs tree
-          export PYTHONPATH=$PWD:$PYTHONPATH
-          export PYTHONPATH=~/tmp/kaldifeat/kaldifeat/python:$PYTHONPATH
-          export PYTHONPATH=~/tmp/kaldifeat/build/lib:$PYTHONPATH
-
-          .github/scripts/run-librispeech-zipformer-mmi-2022-12-08.sh
-
-      - name: Display decoding results for librispeech zipformer-mmi
-        if: github.event_name == 'schedule' || github.event.label.name == 'run-decode'
-        shell: bash
-        run: |
-          cd egs/librispeech/ASR/
-          tree ./zipformer-mmi/exp
-
-          cd zipformer-mmi
-          echo "results for zipformer-mmi"
-          echo "===1best==="
-          find exp/1best -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find exp/1best -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-          echo "===nbest==="
-          find exp/nbest -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find exp/nbest -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-          echo "===nbest-rescoring-LG==="
-          find exp/nbest-rescoring-LG -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find exp/nbest-rescoring-LG -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-          echo "===nbest-rescoring-3-gram==="
-          find exp/nbest-rescoring-3-gram -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find exp/nbest-rescoring-3-gram -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-          echo "===nbest-rescoring-4-gram==="
-          find exp/nbest-rescoring-4-gram -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find exp/nbest-rescoring-4-gram -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-      - name: Upload decoding results for librispeech zipformer-mmi
-        uses: actions/upload-artifact@v2
-        if: github.event_name == 'schedule' || github.event.label.name == 'run-decode'
-        with:
-          name: torch-${{ matrix.torch }}-python-${{ matrix.python-version }}-ubuntu-latest-cpu-zipformer_mmi-2022-12-08
-          path: egs/librispeech/ASR/zipformer_mmi/exp/

.github/workflows/run-librispeech-2022-12-29-stateless7-streaming.yml

@@ -1,172 +0,0 @@
-# Copyright      2022  Fangjun Kuang (csukuangfj@gmail.com)
-
-# See ../../LICENSE for clarification regarding multiple authors
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: run-librispeech-2022-12-29-stateless7-streaming
-# zipformer
-
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-    types: [labeled]
-
-  schedule:
-    # minute (0-59)
-    # hour (0-23)
-    # day of the month (1-31)
-    # month (1-12)
-    # day of the week (0-6)
-    # nightly build at 15:50 UTC time every day
-    - cron: "50 15 * * *"
-
-concurrency:
-  group: run_librispeech_2022_12_29_zipformer_streaming-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  run_librispeech_2022_12_29_zipformer_streaming:
-    if: github.event.label.name == 'ready' || github.event.label.name == 'run-decode' || github.event.label.name == 'streaming-zipformer' || github.event_name == 'push' || github.event_name == 'schedule'
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-latest]
-        python-version: [3.8]
-
-      fail-fast: false
-
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-
-      - name: Setup Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: 'pip'
-          cache-dependency-path: '**/requirements-ci.txt'
-
-      - name: Install Python dependencies
-        run: |
-          grep -v '^#' ./requirements-ci.txt  | xargs -n 1 -L 1 pip install
-          pip uninstall -y protobuf
-          pip install --no-binary protobuf protobuf==3.20.*
-
-      - name: Cache kaldifeat
-        id: my-cache
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/tmp/kaldifeat
-          key: cache-tmp-${{ matrix.python-version }}-2023-05-22
-
-      - name: Install kaldifeat
-        if: steps.my-cache.outputs.cache-hit != 'true'
-        shell: bash
-        run: |
-          .github/scripts/install-kaldifeat.sh
-
-      - name: Cache LibriSpeech test-clean and test-other datasets
-        id: libri-test-clean-and-test-other-data
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/tmp/download
-          key: cache-libri-test-clean-and-test-other
-
-      - name: Download LibriSpeech test-clean and test-other
-        if: steps.libri-test-clean-and-test-other-data.outputs.cache-hit != 'true'
-        shell: bash
-        run: |
-          .github/scripts/download-librispeech-test-clean-and-test-other-dataset.sh
-
-      - name: Prepare manifests for LibriSpeech test-clean and test-other
-        shell: bash
-        run: |
-          .github/scripts/prepare-librispeech-test-clean-and-test-other-manifests.sh
-
-      - name: Cache LibriSpeech test-clean and test-other fbank features
-        id: libri-test-clean-and-test-other-fbank
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/tmp/fbank-libri
-          key: cache-libri-fbank-test-clean-and-test-other-v2
-
-      - name: Compute fbank for LibriSpeech test-clean and test-other
-        if: steps.libri-test-clean-and-test-other-fbank.outputs.cache-hit != 'true'
-        shell: bash
-        run: |
-          .github/scripts/compute-fbank-librispeech-test-clean-and-test-other.sh
-
-      - name: Inference with pre-trained model
-        shell: bash
-        env:
-          GITHUB_EVENT_NAME: ${{ github.event_name }}
-          GITHUB_EVENT_LABEL_NAME: ${{ github.event.label.name }}
-        run: |
-          mkdir -p egs/librispeech/ASR/data
-          ln -sfv ~/tmp/fbank-libri egs/librispeech/ASR/data/fbank
-          ls -lh egs/librispeech/ASR/data/*
-
-          sudo apt-get -qq install git-lfs tree
-          export PYTHONPATH=$PWD:$PYTHONPATH
-          export PYTHONPATH=~/tmp/kaldifeat/kaldifeat/python:$PYTHONPATH
-          export PYTHONPATH=~/tmp/kaldifeat/build/lib:$PYTHONPATH
-
-          .github/scripts/run-librispeech-pruned-transducer-stateless7-streaming-2022-12-29.sh
-
-      - name: Display decoding results for librispeech pruned_transducer_stateless7_streaming
-        if: github.event_name == 'schedule' || github.event.label.name == 'run-decode'
-        shell: bash
-        run: |
-          cd egs/librispeech/ASR/
-          tree ./pruned_transducer_stateless7_streaming/exp
-
-          cd pruned_transducer_stateless7_streaming
-          echo "results for pruned_transducer_stateless7_streaming"
-          echo "===greedy search==="
-          find exp/greedy_search -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find exp/greedy_search -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-          echo "===fast_beam_search==="
-          find exp/fast_beam_search -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find exp/fast_beam_search -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-          echo "===modified beam search==="
-          find exp/modified_beam_search -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find exp/modified_beam_search -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-          echo "===streaming greedy search==="
-          find exp/streaming/greedy_search -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find exp/streaming/greedy_search -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-          echo "===streaming fast_beam_search==="
-          find exp/streaming/fast_beam_search -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find exp/streaming/fast_beam_search -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-          echo "===streaming modified beam search==="
-          find exp/streaming/modified_beam_search -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find exp/streaming/modified_beam_search -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-
-      - name: Upload decoding results for librispeech pruned_transducer_stateless7_streaming
-        uses: actions/upload-artifact@v2
-        if: github.event_name == 'schedule' || github.event.label.name == 'run-decode'
-        with:
-          name: torch-${{ matrix.torch }}-python-${{ matrix.python-version }}-ubuntu-latest-cpu-pruned_transducer_stateless7-streaming-2022-12-29
-          path: egs/librispeech/ASR/pruned_transducer_stateless7_streaming/exp/

.github/workflows/run-librispeech-2023-01-29-stateless7-ctc-bs.yml

@@ -1,163 +0,0 @@
-# Copyright      2022  Fangjun Kuang (csukuangfj@gmail.com)
-
-# See ../../LICENSE for clarification regarding multiple authors
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: run-librispeech-2023-01-29-stateless7-ctc-bs
-# zipformer
-
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-    types: [labeled]
-
-  schedule:
-    # minute (0-59)
-    # hour (0-23)
-    # day of the month (1-31)
-    # month (1-12)
-    # day of the week (0-6)
-    # nightly build at 15:50 UTC time every day
-    - cron: "50 15 * * *"
-
-jobs:
-  run_librispeech_2023_01_29_zipformer_ctc_bs:
-    if: github.event.label.name == 'run-decode' || github.event.label.name == 'blank-skip' || github.event_name == 'push' || github.event_name == 'schedule'
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-latest]
-        python-version: [3.8]
-
-      fail-fast: false
-
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-
-      - name: Setup Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: 'pip'
-          cache-dependency-path: '**/requirements-ci.txt'
-
-      - name: Install Python dependencies
-        run: |
-          grep -v '^#' ./requirements-ci.txt  | xargs -n 1 -L 1 pip install
-          pip uninstall -y protobuf
-          pip install --no-binary protobuf protobuf==3.20.*
-
-      - name: Cache kaldifeat
-        id: my-cache
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/tmp/kaldifeat
-          key: cache-tmp-${{ matrix.python-version }}-2023-05-22
-
-      - name: Install kaldifeat
-        if: steps.my-cache.outputs.cache-hit != 'true'
-        shell: bash
-        run: |
-          .github/scripts/install-kaldifeat.sh
-
-      - name: Cache LibriSpeech test-clean and test-other datasets
-        id: libri-test-clean-and-test-other-data
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/tmp/download
-          key: cache-libri-test-clean-and-test-other
-
-      - name: Download LibriSpeech test-clean and test-other
-        if: steps.libri-test-clean-and-test-other-data.outputs.cache-hit != 'true'
-        shell: bash
-        run: |
-          .github/scripts/download-librispeech-test-clean-and-test-other-dataset.sh
-
-      - name: Prepare manifests for LibriSpeech test-clean and test-other
-        shell: bash
-        run: |
-          .github/scripts/prepare-librispeech-test-clean-and-test-other-manifests.sh
-
-      - name: Cache LibriSpeech test-clean and test-other fbank features
-        id: libri-test-clean-and-test-other-fbank
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/tmp/fbank-libri
-          key: cache-libri-fbank-test-clean-and-test-other-v2
-
-      - name: Compute fbank for LibriSpeech test-clean and test-other
-        if: steps.libri-test-clean-and-test-other-fbank.outputs.cache-hit != 'true'
-        shell: bash
-        run: |
-          .github/scripts/compute-fbank-librispeech-test-clean-and-test-other.sh
-
-      - name: Inference with pre-trained model
-        shell: bash
-        env:
-          GITHUB_EVENT_NAME: ${{ github.event_name }}
-          GITHUB_EVENT_LABEL_NAME: ${{ github.event.label.name }}
-        run: |
-          mkdir -p egs/librispeech/ASR/data
-          ln -sfv ~/tmp/fbank-libri egs/librispeech/ASR/data/fbank
-          ls -lh egs/librispeech/ASR/data/*
-
-          sudo apt-get -qq install git-lfs tree
-          export PYTHONPATH=$PWD:$PYTHONPATH
-          export PYTHONPATH=~/tmp/kaldifeat/kaldifeat/python:$PYTHONPATH
-          export PYTHONPATH=~/tmp/kaldifeat/build/lib:$PYTHONPATH
-
-          .github/scripts/run-librispeech-pruned-transducer-stateless7-ctc-bs-2023-01-29.sh
-
-      - name: Display decoding results for librispeech pruned_transducer_stateless7_ctc_bs
-        if: github.event_name == 'schedule' || github.event.label.name == 'run-decode'
-        shell: bash
-        run: |
-          cd egs/librispeech/ASR/
-          tree ./pruned_transducer_stateless7_ctc_bs/exp
-
-          cd pruned_transducer_stateless7_ctc_bs
-          echo "results for pruned_transducer_stateless7_ctc_bs"
-          echo "===greedy search==="
-          find exp/greedy_search -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find exp/greedy_search -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-          echo "===fast_beam_search==="
-          find exp/fast_beam_search -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find exp/fast_beam_search -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-          echo "===modified beam search==="
-          find exp/modified_beam_search -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find exp/modified_beam_search -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-          echo "===ctc decoding==="
-          find exp/ctc-decoding -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find exp/ctc-decoding -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-          echo "===1best==="
-          find exp/1best -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find exp/1best -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-      - name: Upload decoding results for librispeech pruned_transducer_stateless7_ctc_bs
-        uses: actions/upload-artifact@v2
-        if: github.event_name == 'schedule' || github.event.label.name == 'run-decode'
-        with:
-          name: torch-${{ matrix.torch }}-python-${{ matrix.python-version }}-ubuntu-latest-cpu-pruned_transducer_stateless7-ctc-bs-2023-01-29
-          path: egs/librispeech/ASR/pruned_transducer_stateless7_ctc_bs/exp/

.github/workflows/run-librispeech-conformer-ctc3-2022-11-28.yml

@@ -1,155 +0,0 @@
-# Copyright      2022  Fangjun Kuang (csukuangfj@gmail.com)
-
-# See ../../LICENSE for clarification regarding multiple authors
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: run-librispeech-conformer-ctc3-2022-11-28
-# zipformer
-
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-    types: [labeled]
-
-  schedule:
-    # minute (0-59)
-    # hour (0-23)
-    # day of the month (1-31)
-    # month (1-12)
-    # day of the week (0-6)
-    # nightly build at 15:50 UTC time every day
-    - cron: "50 15 * * *"
-
-concurrency:
-  group: run_librispeech_2022_11_28_conformer_ctc3-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  run_librispeech_2022_11_28_conformer_ctc3:
-    if: github.event.label.name == 'ready' || github.event.label.name == 'run-decode' || github.event_name == 'push' || github.event_name == 'schedule'
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-latest]
-        python-version: [3.8]
-
-      fail-fast: false
-
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-
-      - name: Setup Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: 'pip'
-          cache-dependency-path: '**/requirements-ci.txt'
-
-      - name: Install Python dependencies
-        run: |
-          grep -v '^#' ./requirements-ci.txt  | xargs -n 1 -L 1 pip install
-          pip uninstall -y protobuf
-          pip install --no-binary protobuf protobuf==3.20.*
-
-      - name: Cache kaldifeat
-        id: my-cache
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/tmp/kaldifeat
-          key: cache-tmp-${{ matrix.python-version }}-2023-05-22
-
-      - name: Install kaldifeat
-        if: steps.my-cache.outputs.cache-hit != 'true'
-        shell: bash
-        run: |
-          .github/scripts/install-kaldifeat.sh
-
-      - name: Cache LibriSpeech test-clean and test-other datasets
-        id: libri-test-clean-and-test-other-data
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/tmp/download
-          key: cache-libri-test-clean-and-test-other
-
-      - name: Download LibriSpeech test-clean and test-other
-        if: steps.libri-test-clean-and-test-other-data.outputs.cache-hit != 'true'
-        shell: bash
-        run: |
-          .github/scripts/download-librispeech-test-clean-and-test-other-dataset.sh
-
-      - name: Prepare manifests for LibriSpeech test-clean and test-other
-        shell: bash
-        run: |
-          .github/scripts/prepare-librispeech-test-clean-and-test-other-manifests.sh
-
-      - name: Cache LibriSpeech test-clean and test-other fbank features
-        id: libri-test-clean-and-test-other-fbank
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/tmp/fbank-libri
-          key: cache-libri-fbank-test-clean-and-test-other-v2
-
-      - name: Compute fbank for LibriSpeech test-clean and test-other
-        if: steps.libri-test-clean-and-test-other-fbank.outputs.cache-hit != 'true'
-        shell: bash
-        run: |
-          .github/scripts/compute-fbank-librispeech-test-clean-and-test-other.sh
-
-      - name: Inference with pre-trained model
-        shell: bash
-        env:
-          GITHUB_EVENT_NAME: ${{ github.event_name }}
-          GITHUB_EVENT_LABEL_NAME: ${{ github.event.label.name }}
-        run: |
-          mkdir -p egs/librispeech/ASR/data
-          ln -sfv ~/tmp/fbank-libri egs/librispeech/ASR/data/fbank
-          ls -lh egs/librispeech/ASR/data/*
-
-          sudo apt-get -qq install git-lfs tree
-          export PYTHONPATH=$PWD:$PYTHONPATH
-          export PYTHONPATH=~/tmp/kaldifeat/kaldifeat/python:$PYTHONPATH
-          export PYTHONPATH=~/tmp/kaldifeat/build/lib:$PYTHONPATH
-
-          .github/scripts/run-librispeech-conformer-ctc3-2022-11-28.sh
-
-      - name: Display decoding results for librispeech conformer_ctc3
-        if: github.event_name == 'schedule' || github.event.label.name == 'run-decode'
-        shell: bash
-        run: |
-          cd egs/librispeech/ASR/
-          tree ./conformer_ctc3/exp
-
-          cd conformer_ctc3
-          echo "results for conformer_ctc3"
-          echo "===ctc-decoding==="
-          find exp/ctc-decoding -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find exp/ctc-decoding -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-          echo "===1best==="
-          find exp/1best -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find exp/1best -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-      - name: Upload decoding results for librispeech conformer_ctc3
-        uses: actions/upload-artifact@v2
-        if: github.event_name == 'schedule' || github.event.label.name == 'run-decode'
-        with:
-          name: torch-${{ matrix.torch }}-python-${{ matrix.python-version }}-ubuntu-latest-cpu-conformer_ctc3-2022-11-28
-          path: egs/librispeech/ASR/conformer_ctc3/exp/

.github/workflows/run-librispeech-pruned-transducer-stateless3-2022-05-13.yml

@@ -1,157 +0,0 @@
-# Copyright      2021  Fangjun Kuang (csukuangfj@gmail.com)
-
-# See ../../LICENSE for clarification regarding multiple authors
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: run-librispeech-pruned-transducer-stateless3-2022-05-13
-# stateless pruned transducer (reworked model) + giga speech
-
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-    types: [labeled]
-
-  schedule:
-    # minute (0-59)
-    # hour (0-23)
-    # day of the month (1-31)
-    # month (1-12)
-    # day of the week (0-6)
-    # nightly build at 15:50 UTC time every day
-    - cron: "50 15 * * *"
-
-concurrency:
-  group: run_librispeech_pruned_transducer_stateless3_2022_05_13-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  run_librispeech_pruned_transducer_stateless3_2022_05_13:
-    if: github.event.label.name == 'ready' || github.event.label.name == 'run-decode' || github.event_name == 'push' || github.event_name == 'schedule'
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-latest]
-        python-version: [3.8]
-
-      fail-fast: false
-
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-
-      - name: Setup Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: 'pip'
-          cache-dependency-path: '**/requirements-ci.txt'
-
-      - name: Install Python dependencies
-        run: |
-          grep -v '^#' ./requirements-ci.txt  | xargs -n 1 -L 1 pip install
-          pip uninstall -y protobuf
-          pip install --no-binary protobuf protobuf==3.20.*
-
-      - name: Cache kaldifeat
-        id: my-cache
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/tmp/kaldifeat
-          key: cache-tmp-${{ matrix.python-version }}-2023-05-22
-
-      - name: Install kaldifeat
-        if: steps.my-cache.outputs.cache-hit != 'true'
-        shell: bash
-        run: |
-          .github/scripts/install-kaldifeat.sh
-
-      - name: Cache LibriSpeech test-clean and test-other datasets
-        id: libri-test-clean-and-test-other-data
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/tmp/download
-          key: cache-libri-test-clean-and-test-other
-
-      - name: Download LibriSpeech test-clean and test-other
-        if: steps.libri-test-clean-and-test-other-data.outputs.cache-hit != 'true'
-        shell: bash
-        run: |
-          .github/scripts/download-librispeech-test-clean-and-test-other-dataset.sh
-
-      - name: Prepare manifests for LibriSpeech test-clean and test-other
-        shell: bash
-        run: |
-          .github/scripts/prepare-librispeech-test-clean-and-test-other-manifests.sh
-
-      - name: Cache LibriSpeech test-clean and test-other fbank features
-        id: libri-test-clean-and-test-other-fbank
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/tmp/fbank-libri
-          key: cache-libri-fbank-test-clean-and-test-other-v2
-
-      - name: Compute fbank for LibriSpeech test-clean and test-other
-        if: steps.libri-test-clean-and-test-other-fbank.outputs.cache-hit != 'true'
-        shell: bash
-        run: |
-          .github/scripts/compute-fbank-librispeech-test-clean-and-test-other.sh
-
-      - name: Inference with pre-trained model
-        shell: bash
-        env:
-          GITHUB_EVENT_NAME: ${{ github.event_name }}
-          GITHUB_EVENT_LABEL_NAME: ${{ github.event.label.name }}
-        run: |
-          mkdir -p egs/librispeech/ASR/data
-          ln -sfv ~/tmp/fbank-libri egs/librispeech/ASR/data/fbank
-          ls -lh egs/librispeech/ASR/data/*
-
-          sudo apt-get -qq install git-lfs tree
-          export PYTHONPATH=$PWD:$PYTHONPATH
-          export PYTHONPATH=~/tmp/kaldifeat/kaldifeat/python:$PYTHONPATH
-          export PYTHONPATH=~/tmp/kaldifeat/build/lib:$PYTHONPATH
-
-          .github/scripts/run-librispeech-pruned-transducer-stateless3-2022-05-13.sh
-
-      - name: Display decoding results for pruned_transducer_stateless3
-        if: github.event_name == 'schedule' || github.event.label.name == 'run-decode'
-        shell: bash
-        run: |
-          cd egs/librispeech/ASR
-          tree pruned_transducer_stateless3/exp
-          cd pruned_transducer_stateless3/exp
-          echo "===greedy search==="
-          find greedy_search -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find greedy_search -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-          echo "===fast_beam_search==="
-          find fast_beam_search -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find fast_beam_search -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-          echo "===modified beam search==="
-          find modified_beam_search -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find modified_beam_search -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-      - name: Upload decoding results for pruned_transducer_stateless3
-        uses: actions/upload-artifact@v2
-        if: github.event_name == 'schedule' || github.event.label.name == 'run-decode'
-        with:
-          name: torch-${{ matrix.torch }}-python-${{ matrix.python-version }}-ubuntu-latest-cpu-pruned_transducer_stateless3-2022-04-29
-          path: egs/librispeech/ASR/pruned_transducer_stateless3/exp/

.github/workflows/run-librispeech-streaming-transducer-stateless2-2022-06-26.yml

@@ -1,159 +0,0 @@
-# Copyright      2021  Fangjun Kuang (csukuangfj@gmail.com)
-
-# See ../../LICENSE for clarification regarding multiple authors
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: run-librispeech-streaming-2022-06-26
-# streaming conformer stateless transducer2
-
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-    types: [labeled]
-
-  schedule:
-    # minute (0-59)
-    # hour (0-23)
-    # day of the month (1-31)
-    # month (1-12)
-    # day of the week (0-6)
-    # nightly build at 15:50 UTC time every day
-    - cron: "50 15 * * *"
-
-concurrency:
-  group: run_librispeech_streaming_2022_06_26-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  run_librispeech_streaming_2022_06_26:
-    if: github.event.label.name == 'ready' || github.event.label.name == 'run-decode' || github.event_name == 'push' || github.event_name == 'schedule'
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-latest]
-        python-version: [3.8]
-
-      fail-fast: false
-
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-
-      - name: Setup Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: 'pip'
-          cache-dependency-path: '**/requirements-ci.txt'
-
-      - name: Install Python dependencies
-        run: |
-          grep -v '^#' ./requirements-ci.txt  | xargs -n 1 -L 1 pip install
-          pip uninstall -y protobuf
-          pip install --no-binary protobuf protobuf==3.20.*
-
-      - name: Cache kaldifeat
-        id: my-cache
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/tmp/kaldifeat
-          key: cache-tmp-${{ matrix.python-version }}-2023-05-22
-
-      - name: Install kaldifeat
-        if: steps.my-cache.outputs.cache-hit != 'true'
-        shell: bash
-        run: |
-          .github/scripts/install-kaldifeat.sh
-
-      - name: Cache LibriSpeech test-clean and test-other datasets
-        id: libri-test-clean-and-test-other-data
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/tmp/download
-          key: cache-libri-test-clean-and-test-other
-
-      - name: Download LibriSpeech test-clean and test-other
-        if: steps.libri-test-clean-and-test-other-data.outputs.cache-hit != 'true'
-        shell: bash
-        run: |
-          .github/scripts/download-librispeech-test-clean-and-test-other-dataset.sh
-
-      - name: Prepare manifests for LibriSpeech test-clean and test-other
-        shell: bash
-        run: |
-          .github/scripts/prepare-librispeech-test-clean-and-test-other-manifests.sh
-
-      - name: Cache LibriSpeech test-clean and test-other fbank features
-        id: libri-test-clean-and-test-other-fbank
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/tmp/fbank-libri
-          key: cache-libri-fbank-test-clean-and-test-other-v2
-
-      - name: Compute fbank for LibriSpeech test-clean and test-other
-        if: steps.libri-test-clean-and-test-other-fbank.outputs.cache-hit != 'true'
-        shell: bash
-        run: |
-          .github/scripts/compute-fbank-librispeech-test-clean-and-test-other.sh
-
-      - name: Inference with pre-trained model
-        shell: bash
-        env:
-          GITHUB_EVENT_NAME: ${{ github.event_name }}
-          GITHUB_EVENT_LABEL_NAME: ${{ github.event.label.name }}
-        run: |
-          mkdir -p egs/librispeech/ASR/data
-          ln -sfv ~/tmp/fbank-libri egs/librispeech/ASR/data/fbank
-          ls -lh egs/librispeech/ASR/data/*
-
-          sudo apt-get -qq install git-lfs tree
-          export PYTHONPATH=$PWD:$PYTHONPATH
-          export PYTHONPATH=~/tmp/kaldifeat/kaldifeat/python:$PYTHONPATH
-          export PYTHONPATH=~/tmp/kaldifeat/build/lib:$PYTHONPATH
-
-          .github/scripts/run-librispeech-streaming-pruned-transducer-stateless2-2022-06-26.sh
-
-      - name: Display decoding results
-        if: github.event_name == 'schedule' || github.event.label.name == 'run-decode'
-        shell: bash
-        run: |
-          cd egs/librispeech/ASR/
-          tree ./pruned_transducer_stateless2/exp
-
-          cd pruned_transducer_stateless2
-          echo "results for pruned_transducer_stateless2"
-          echo "===greedy search==="
-          find exp/greedy_search -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find exp/greedy_search -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-          echo "===fast_beam_search==="
-          find exp/fast_beam_search -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find exp/fast_beam_search -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-          echo "===modified_beam_search==="
-          find exp/modified_beam_search -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find exp/modified_beam_search -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-      - name: Upload decoding results for pruned_transducer_stateless2
-        uses: actions/upload-artifact@v2
-        if: github.event_name == 'schedule' || github.event.label.name == 'run-decode'
-        with:
-          name: torch-${{ matrix.torch }}-python-${{ matrix.python-version }}-ubuntu-latest-cpu-pruned_transducer_stateless2-2022-06-26
-          path: egs/librispeech/ASR/pruned_transducer_stateless2/exp/

.github/workflows/run-librispeech-streaming-zipformer-2023-05-18.yml

@@ -1,174 +0,0 @@
-# Copyright      2022  Fangjun Kuang (csukuangfj@gmail.com)
-
-# See ../../LICENSE for clarification regarding multiple authors
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: run-librispeech-streaming-zipformer-2023-05-18
-# zipformer
-
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-    types: [labeled]
-
-  schedule:
-    # minute (0-59)
-    # hour (0-23)
-    # day of the month (1-31)
-    # month (1-12)
-    # day of the week (0-6)
-    # nightly build at 15:50 UTC time every day
-    - cron: "50 15 * * *"
-
-concurrency:
-  group: run_librispeech_2023_05_18_streaming_zipformer-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  run_librispeech_2023_05_18_streaming_zipformer:
-    if: github.event.label.name == 'zipformer' ||github.event.label.name == 'ready' || github.event.label.name == 'run-decode' || github.event_name == 'push' || github.event_name == 'schedule'
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-latest]
-        python-version: [3.8]
-
-      fail-fast: false
-
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-
-      - name: Setup Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: 'pip'
-          cache-dependency-path: '**/requirements-ci.txt'
-
-      - name: Install Python dependencies
-        run: |
-          grep -v '^#' ./requirements-ci.txt  | xargs -n 1 -L 1 pip install
-          pip uninstall -y protobuf
-          pip install --no-binary protobuf protobuf==3.20.*
-
-      - name: Cache kaldifeat
-        id: my-cache
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/tmp/kaldifeat
-          key: cache-tmp-${{ matrix.python-version }}-2023-05-22
-
-      - name: Install kaldifeat
-        if: steps.my-cache.outputs.cache-hit != 'true'
-        shell: bash
-        run: |
-          .github/scripts/install-kaldifeat.sh
-
-      - name: Cache LibriSpeech test-clean and test-other datasets
-        id: libri-test-clean-and-test-other-data
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/tmp/download
-          key: cache-libri-test-clean-and-test-other
-
-      - name: Download LibriSpeech test-clean and test-other
-        if: steps.libri-test-clean-and-test-other-data.outputs.cache-hit != 'true'
-        shell: bash
-        run: |
-          .github/scripts/download-librispeech-test-clean-and-test-other-dataset.sh
-
-      - name: Prepare manifests for LibriSpeech test-clean and test-other
-        shell: bash
-        run: |
-          .github/scripts/prepare-librispeech-test-clean-and-test-other-manifests.sh
-
-      - name: Cache LibriSpeech test-clean and test-other fbank features
-        id: libri-test-clean-and-test-other-fbank
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/tmp/fbank-libri
-          key: cache-libri-fbank-test-clean-and-test-other-v2
-
-      - name: Compute fbank for LibriSpeech test-clean and test-other
-        if: steps.libri-test-clean-and-test-other-fbank.outputs.cache-hit != 'true'
-        shell: bash
-        run: |
-          .github/scripts/compute-fbank-librispeech-test-clean-and-test-other.sh
-
-      - name: Inference with pre-trained model
-        shell: bash
-        env:
-          GITHUB_EVENT_NAME: ${{ github.event_name }}
-          GITHUB_EVENT_LABEL_NAME: ${{ github.event.label.name }}
-        run: |
-          mkdir -p egs/librispeech/ASR/data
-          ln -sfv ~/tmp/fbank-libri egs/librispeech/ASR/data/fbank
-          ls -lh egs/librispeech/ASR/data/*
-
-          sudo apt-get -qq install git-lfs tree
-          export PYTHONPATH=$PWD:$PYTHONPATH
-          export PYTHONPATH=~/tmp/kaldifeat/kaldifeat/python:$PYTHONPATH
-          export PYTHONPATH=~/tmp/kaldifeat/build/lib:$PYTHONPATH
-
-          .github/scripts/run-librispeech-streaming-zipformer-2023-05-18.sh
-
-      - name: Display decoding results for librispeech zipformer
-        if: github.event_name == 'schedule' || github.event.label.name == 'run-decode'
-        shell: bash
-        run: |
-          cd egs/librispeech/ASR/
-          tree ./zipformer/exp
-
-          cd zipformer
-
-          echo "results for zipformer, simulated streaming decoding"
-          echo "===greedy search==="
-          find exp/greedy_search -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find exp/greedy_search -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-          echo "===fast_beam_search==="
-          find exp/fast_beam_search -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find exp/fast_beam_search -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-          echo "===modified beam search==="
-          find exp/modified_beam_search -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find exp/modified_beam_search -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-          echo "results for zipformer, chunk-wise streaming decoding"
-          echo "===greedy search==="
-          find exp/streaming/greedy_search -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find exp/streaming/greedy_search -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-          echo "===fast_beam_search==="
-          find exp/streaming/fast_beam_search -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find exp/streaming/fast_beam_search -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-          echo "===modified beam search==="
-          find exp/streaming/modified_beam_search -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find exp/streaming/modified_beam_search -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-
-      - name: Upload decoding results for librispeech zipformer
-        uses: actions/upload-artifact@v2
-        if: github.event_name == 'schedule' || github.event.label.name == 'run-decode'
-        with:
-          name: torch-${{ matrix.torch }}-python-${{ matrix.python-version }}-ubuntu-latest-cpu-zipformer-2022-11-11
-          path: egs/librispeech/ASR/zipformer/exp/

.github/workflows/run-librispeech-transducer-stateless2-2022-04-19.yml

@@ -1,159 +0,0 @@
-# Copyright      2021  Fangjun Kuang (csukuangfj@gmail.com)
-
-# See ../../LICENSE for clarification regarding multiple authors
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: run-librispeech-2022-04-19
-# stateless transducer + torchaudio rnn-t loss
-
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-    types: [labeled]
-
-  schedule:
-    # minute (0-59)
-    # hour (0-23)
-    # day of the month (1-31)
-    # month (1-12)
-    # day of the week (0-6)
-    # nightly build at 15:50 UTC time every day
-    - cron: "50 15 * * *"
-
-concurrency:
-  group: run_librispeech_2022_04_19-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  run_librispeech_2022_04_19:
-    if: github.event.label.name == 'ready' || github.event.label.name == 'run-decode' || github.event_name == 'push' || github.event_name == 'schedule'
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-latest]
-        python-version: [3.8]
-
-      fail-fast: false
-
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-
-      - name: Setup Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: 'pip'
-          cache-dependency-path: '**/requirements-ci.txt'
-
-      - name: Install Python dependencies
-        run: |
-          grep -v '^#' ./requirements-ci.txt  | xargs -n 1 -L 1 pip install
-          pip uninstall -y protobuf
-          pip install --no-binary protobuf protobuf==3.20.*
-
-      - name: Cache kaldifeat
-        id: my-cache
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/tmp/kaldifeat
-          key: cache-tmp-${{ matrix.python-version }}-2023-05-22
-
-      - name: Install kaldifeat
-        if: steps.my-cache.outputs.cache-hit != 'true'
-        shell: bash
-        run: |
-          .github/scripts/install-kaldifeat.sh
-
-      - name: Cache LibriSpeech test-clean and test-other datasets
-        id: libri-test-clean-and-test-other-data
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/tmp/download
-          key: cache-libri-test-clean-and-test-other
-
-      - name: Download LibriSpeech test-clean and test-other
-        if: steps.libri-test-clean-and-test-other-data.outputs.cache-hit != 'true'
-        shell: bash
-        run: |
-          .github/scripts/download-librispeech-test-clean-and-test-other-dataset.sh
-
-      - name: Prepare manifests for LibriSpeech test-clean and test-other
-        shell: bash
-        run: |
-          .github/scripts/prepare-librispeech-test-clean-and-test-other-manifests.sh
-
-      - name: Cache LibriSpeech test-clean and test-other fbank features
-        id: libri-test-clean-and-test-other-fbank
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/tmp/fbank-libri
-          key: cache-libri-fbank-test-clean-and-test-other-v2
-
-      - name: Compute fbank for LibriSpeech test-clean and test-other
-        if: steps.libri-test-clean-and-test-other-fbank.outputs.cache-hit != 'true'
-        shell: bash
-        run: |
-          .github/scripts/compute-fbank-librispeech-test-clean-and-test-other.sh
-
-      - name: Inference with pre-trained model
-        shell: bash
-        env:
-          GITHUB_EVENT_NAME: ${{ github.event_name }}
-          GITHUB_EVENT_LABEL_NAME: ${{ github.event.label.name }}
-        run: |
-          mkdir -p egs/librispeech/ASR/data
-          ln -sfv ~/tmp/fbank-libri egs/librispeech/ASR/data/fbank
-          ls -lh egs/librispeech/ASR/data/*
-
-          sudo apt-get -qq install git-lfs tree
-          export PYTHONPATH=$PWD:$PYTHONPATH
-          export PYTHONPATH=~/tmp/kaldifeat/kaldifeat/python:$PYTHONPATH
-          export PYTHONPATH=~/tmp/kaldifeat/build/lib:$PYTHONPATH
-
-          .github/scripts/run-librispeech-transducer-stateless2-2022-04-19.sh
-
-      - name: Display decoding results
-        if: github.event_name == 'schedule' || github.event.label.name == 'run-decode'
-        shell: bash
-        run: |
-          cd egs/librispeech/ASR/
-          tree ./transducer_stateless2/exp
-
-          cd transducer_stateless2
-          echo "results for transducer_stateless2"
-          echo "===greedy search==="
-          find exp/greedy_search -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find exp/greedy_search -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-          echo "===fast_beam_search==="
-          find exp/fast_beam_search -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find exp/fast_beam_search -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-          echo "===modified_beam_search==="
-          find exp/modified_beam_search -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find exp/modified_beam_search -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-      - name: Upload decoding results for transducer_stateless2
-        uses: actions/upload-artifact@v2
-        if: github.event_name == 'schedule' || github.event.label.name == 'run-decode'
-        with:
-          name: torch-${{ matrix.torch }}-python-${{ matrix.python-version }}-ubuntu-latest-cpu-transducer_stateless2-2022-04-19
-          path: egs/librispeech/ASR/transducer_stateless2/exp/

.github/workflows/run-librispeech-zipformer-2023-05-18.yml

@@ -1,159 +0,0 @@
-# Copyright      2022  Fangjun Kuang (csukuangfj@gmail.com)
-
-# See ../../LICENSE for clarification regarding multiple authors
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: run-librispeech-zipformer-2023-05-18
-# zipformer
-
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-    types: [labeled]
-
-  schedule:
-    # minute (0-59)
-    # hour (0-23)
-    # day of the month (1-31)
-    # month (1-12)
-    # day of the week (0-6)
-    # nightly build at 15:50 UTC time every day
-    - cron: "50 15 * * *"
-
-concurrency:
-  group: run_librispeech_2023_05_18_zipformer-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  run_librispeech_2023_05_18_zipformer:
-    if: github.event.label.name == 'zipformer' ||github.event.label.name == 'ready' || github.event.label.name == 'run-decode' || github.event_name == 'push' || github.event_name == 'schedule'
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-latest]
-        python-version: [3.8]
-
-      fail-fast: false
-
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-
-      - name: Setup Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: 'pip'
-          cache-dependency-path: '**/requirements-ci.txt'
-
-      - name: Install Python dependencies
-        run: |
-          grep -v '^#' ./requirements-ci.txt  | xargs -n 1 -L 1 pip install
-          pip uninstall -y protobuf
-          pip install --no-binary protobuf protobuf==3.20.*
-
-      - name: Cache kaldifeat
-        id: my-cache
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/tmp/kaldifeat
-          key: cache-tmp-${{ matrix.python-version }}-2023-05-22
-
-      - name: Install kaldifeat
-        if: steps.my-cache.outputs.cache-hit != 'true'
-        shell: bash
-        run: |
-          .github/scripts/install-kaldifeat.sh
-
-      - name: Cache LibriSpeech test-clean and test-other datasets
-        id: libri-test-clean-and-test-other-data
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/tmp/download
-          key: cache-libri-test-clean-and-test-other
-
-      - name: Download LibriSpeech test-clean and test-other
-        if: steps.libri-test-clean-and-test-other-data.outputs.cache-hit != 'true'
-        shell: bash
-        run: |
-          .github/scripts/download-librispeech-test-clean-and-test-other-dataset.sh
-
-      - name: Prepare manifests for LibriSpeech test-clean and test-other
-        shell: bash
-        run: |
-          .github/scripts/prepare-librispeech-test-clean-and-test-other-manifests.sh
-
-      - name: Cache LibriSpeech test-clean and test-other fbank features
-        id: libri-test-clean-and-test-other-fbank
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/tmp/fbank-libri
-          key: cache-libri-fbank-test-clean-and-test-other-v2
-
-      - name: Compute fbank for LibriSpeech test-clean and test-other
-        if: steps.libri-test-clean-and-test-other-fbank.outputs.cache-hit != 'true'
-        shell: bash
-        run: |
-          .github/scripts/compute-fbank-librispeech-test-clean-and-test-other.sh
-
-      - name: Inference with pre-trained model
-        shell: bash
-        env:
-          GITHUB_EVENT_NAME: ${{ github.event_name }}
-          GITHUB_EVENT_LABEL_NAME: ${{ github.event.label.name }}
-        run: |
-          mkdir -p egs/librispeech/ASR/data
-          ln -sfv ~/tmp/fbank-libri egs/librispeech/ASR/data/fbank
-          ls -lh egs/librispeech/ASR/data/*
-
-          sudo apt-get -qq install git-lfs tree
-          export PYTHONPATH=$PWD:$PYTHONPATH
-          export PYTHONPATH=~/tmp/kaldifeat/kaldifeat/python:$PYTHONPATH
-          export PYTHONPATH=~/tmp/kaldifeat/build/lib:$PYTHONPATH
-
-          .github/scripts/run-librispeech-zipformer-2023-05-18.sh
-
-      - name: Display decoding results for librispeech zipformer
-        if: github.event_name == 'schedule' || github.event.label.name == 'run-decode'
-        shell: bash
-        run: |
-          cd egs/librispeech/ASR/
-          tree ./zipformer/exp
-
-          cd zipformer
-          echo "results for zipformer"
-          echo "===greedy search==="
-          find exp/greedy_search -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find exp/greedy_search -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-          echo "===fast_beam_search==="
-          find exp/fast_beam_search -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find exp/fast_beam_search -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-          echo "===modified beam search==="
-          find exp/modified_beam_search -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find exp/modified_beam_search -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-      - name: Upload decoding results for librispeech zipformer
-        uses: actions/upload-artifact@v2
-        if: github.event_name == 'schedule' || github.event.label.name == 'run-decode'
-        with:
-          name: torch-${{ matrix.torch }}-python-${{ matrix.python-version }}-ubuntu-latest-cpu-zipformer-2022-11-11
-          path: egs/librispeech/ASR/zipformer/exp/

.github/workflows/run-librispeech-zipformer-ctc-2023-06-14.yml

@@ -1,155 +0,0 @@
-# Copyright      2022  Fangjun Kuang (csukuangfj@gmail.com)
-
-# See ../../LICENSE for clarification regarding multiple authors
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: run-librispeech-zipformer-ctc-2023-06-14
-# zipformer
-
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-    types: [labeled]
-
-  schedule:
-    # minute (0-59)
-    # hour (0-23)
-    # day of the month (1-31)
-    # month (1-12)
-    # day of the week (0-6)
-    # nightly build at 15:50 UTC time every day
-    - cron: "50 15 * * *"
-
-concurrency:
-  group: run_librispeech_2023_06_14_zipformer-ctc-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  run_librispeech_2023_06_14_zipformer_ctc:
-    if: github.event.label.name == 'zipformer' ||github.event.label.name == 'ready' || github.event.label.name == 'run-decode' || github.event_name == 'push' || github.event_name == 'schedule'
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-latest]
-        python-version: [3.8]
-
-      fail-fast: false
-
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-
-      - name: Setup Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: 'pip'
-          cache-dependency-path: '**/requirements-ci.txt'
-
-      - name: Install Python dependencies
-        run: |
-          grep -v '^#' ./requirements-ci.txt  | xargs -n 1 -L 1 pip install
-          pip uninstall -y protobuf
-          pip install --no-binary protobuf protobuf==3.20.*
-
-      - name: Cache kaldifeat
-        id: my-cache
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/tmp/kaldifeat
-          key: cache-tmp-${{ matrix.python-version }}-2023-05-22
-
-      - name: Install kaldifeat
-        if: steps.my-cache.outputs.cache-hit != 'true'
-        shell: bash
-        run: |
-          .github/scripts/install-kaldifeat.sh
-
-      - name: Cache LibriSpeech test-clean and test-other datasets
-        id: libri-test-clean-and-test-other-data
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/tmp/download
-          key: cache-libri-test-clean-and-test-other
-
-      - name: Download LibriSpeech test-clean and test-other
-        if: steps.libri-test-clean-and-test-other-data.outputs.cache-hit != 'true'
-        shell: bash
-        run: |
-          .github/scripts/download-librispeech-test-clean-and-test-other-dataset.sh
-
-      - name: Prepare manifests for LibriSpeech test-clean and test-other
-        shell: bash
-        run: |
-          .github/scripts/prepare-librispeech-test-clean-and-test-other-manifests.sh
-
-      - name: Cache LibriSpeech test-clean and test-other fbank features
-        id: libri-test-clean-and-test-other-fbank
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/tmp/fbank-libri
-          key: cache-libri-fbank-test-clean-and-test-other-v2
-
-      - name: Compute fbank for LibriSpeech test-clean and test-other
-        if: steps.libri-test-clean-and-test-other-fbank.outputs.cache-hit != 'true'
-        shell: bash
-        run: |
-          .github/scripts/compute-fbank-librispeech-test-clean-and-test-other.sh
-
-      - name: Inference with pre-trained model
-        shell: bash
-        env:
-          GITHUB_EVENT_NAME: ${{ github.event_name }}
-          GITHUB_EVENT_LABEL_NAME: ${{ github.event.label.name }}
-        run: |
-          mkdir -p egs/librispeech/ASR/data
-          ln -sfv ~/tmp/fbank-libri egs/librispeech/ASR/data/fbank
-          ls -lh egs/librispeech/ASR/data/*
-
-          sudo apt-get -qq install git-lfs tree
-          export PYTHONPATH=$PWD:$PYTHONPATH
-          export PYTHONPATH=~/tmp/kaldifeat/kaldifeat/python:$PYTHONPATH
-          export PYTHONPATH=~/tmp/kaldifeat/build/lib:$PYTHONPATH
-
-          .github/scripts/run-librispeech-zipformer-ctc-2023-06-14.sh
-
-      - name: Display decoding results for librispeech zipformer
-        if: github.event_name == 'schedule' || github.event.label.name == 'run-decode'
-        shell: bash
-        run: |
-          cd egs/librispeech/ASR/
-          tree ./zipformer/exp
-
-          cd zipformer
-          echo "results for zipformer"
-          echo "===ctc-decoding==="
-          find exp/ctc-decoding -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find exp/ctc-decoding -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-          echo "===1best==="
-          find exp/1best -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find exp/1best -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-      - name: Upload decoding results for librispeech zipformer
-        uses: actions/upload-artifact@v2
-        if: github.event_name == 'schedule' || github.event.label.name == 'run-decode'
-        with:
-          name: torch-${{ matrix.torch }}-python-${{ matrix.python-version }}-ubuntu-latest-cpu-zipformer-2022-11-11
-          path: egs/librispeech/ASR/zipformer/exp/

.github/workflows/run-multi-corpora-zipformer.yml

@@ -14,7 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-name: run-multi-zh_hans-zipformer
+name: run-multi-corpora-zipformer
 
 on:
   push:
@@ -24,12 +24,12 @@ on:
     types: [labeled]
 
 concurrency:
-  group: run_multi-zh_hans_zipformer-${{ github.ref }}
+  group: run_multi-corpora_zipformer-${{ github.ref }}
   cancel-in-progress: true
 
 jobs:
-  run_multi-zh_hans_zipformer:
+  run_multi-corpora_zipformer:
-    if: github.event.label.name == 'onnx' || github.event.label.name == 'ready' || github.event_name == 'push' || github.event.label.name == 'multi-zh_hans'
+    if: github.event.label.name == 'onnx' || github.event.label.name == 'ready' || github.event_name == 'push' || github.event.label.name == 'multi-zh_hans' || github.event.label.name == 'zipformer' || github.event.label.name == 'multi-corpora'
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
@@ -81,4 +81,4 @@ jobs:
           export PYTHONPATH=~/tmp/kaldifeat/kaldifeat/python:$PYTHONPATH
           export PYTHONPATH=~/tmp/kaldifeat/build/lib:$PYTHONPATH
 
-          .github/scripts/run-multi-zh_hans-zipformer.sh
+          .github/scripts/run-multi-corpora-zipformer.sh

.github/workflows/run-pretrained-ctc.yml

@@ -1,87 +0,0 @@
-# Copyright      2021  Fangjun Kuang (csukuangfj@gmail.com)
-
-# See ../../LICENSE for clarification regarding multiple authors
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: run-pre-trained-ctc
-
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-    types: [labeled]
-
-  workflow_dispatch:
-    inputs:
-      test-run:
-        description: 'Test (y/n)?'
-        required: true
-        default: 'y'
-
-concurrency:
-  group: run_pre_trained_ctc-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  run_pre_trained_ctc:
-    if: github.event.label.name == 'ready' || github.event_name == 'push' || github.event.inputs.test-run == 'y' || github.event.label.name == 'ctc'
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-latest]
-        python-version: [3.8]
-
-      fail-fast: false
-
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-
-      - name: Setup Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: 'pip'
-          cache-dependency-path: '**/requirements-ci.txt'
-
-      - name: Install Python dependencies
-        run: |
-          grep -v '^#' ./requirements-ci.txt  | xargs -n 1 -L 1 pip install
-          pip uninstall -y protobuf
-          pip install --no-binary protobuf protobuf==3.20.*
-
-      - name: Cache kaldifeat
-        id: my-cache
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/tmp/kaldifeat
-          key: cache-tmp-${{ matrix.python-version }}-2023-05-22
-
-      - name: Install kaldifeat
-        if: steps.my-cache.outputs.cache-hit != 'true'
-        shell: bash
-        run: |
-          .github/scripts/install-kaldifeat.sh
-
-      - name: Inference with pre-trained model
-        shell: bash
-        run: |
-          sudo apt-get -qq install git-lfs tree
-          export PYTHONPATH=$PWD:$PYTHONPATH
-          export PYTHONPATH=~/tmp/kaldifeat/kaldifeat/python:$PYTHONPATH
-          export PYTHONPATH=~/tmp/kaldifeat/build/lib:$PYTHONPATH
-          .github/scripts/run-pre-trained-ctc.sh

.github/workflows/run-pretrained-transducer-stateless-librispeech-100h.yml

@@ -1,158 +0,0 @@
-# Copyright      2021  Fangjun Kuang (csukuangfj@gmail.com)
-
-# See ../../LICENSE for clarification regarding multiple authors
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: run-pre-trained-trandsucer-stateless-multi-datasets-librispeech-100h
-
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-    types: [labeled]
-
-  schedule:
-    # minute (0-59)
-    # hour (0-23)
-    # day of the month (1-31)
-    # month (1-12)
-    # day of the week (0-6)
-    # nightly build at 15:50 UTC time every day
-    - cron: "50 15 * * *"
-
-concurrency:
-  group: run_pre_trained_transducer_stateless_multi_datasets_librispeech_100h-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  run_pre_trained_transducer_stateless_multi_datasets_librispeech_100h:
-    if: github.event.label.name == 'ready' || github.event.label.name == 'run-decode' || github.event_name == 'push' || github.event_name == 'schedule'
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-latest]
-        python-version: [3.8]
-
-      fail-fast: false
-
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-
-      - name: Setup Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: 'pip'
-          cache-dependency-path: '**/requirements-ci.txt'
-
-      - name: Install Python dependencies
-        run: |
-          grep -v '^#' ./requirements-ci.txt  | xargs -n 1 -L 1 pip install
-          pip uninstall -y protobuf
-          pip install --no-binary protobuf protobuf==3.20.*
-
-      - name: Cache kaldifeat
-        id: my-cache
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/tmp/kaldifeat
-          key: cache-tmp-${{ matrix.python-version }}-2023-05-22
-
-      - name: Install kaldifeat
-        if: steps.my-cache.outputs.cache-hit != 'true'
-        shell: bash
-        run: |
-          .github/scripts/install-kaldifeat.sh
-
-      - name: Cache LibriSpeech test-clean and test-other datasets
-        id: libri-test-clean-and-test-other-data
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/tmp/download
-          key: cache-libri-test-clean-and-test-other
-
-      - name: Download LibriSpeech test-clean and test-other
-        if: steps.libri-test-clean-and-test-other-data.outputs.cache-hit != 'true'
-        shell: bash
-        run: |
-          .github/scripts/download-librispeech-test-clean-and-test-other-dataset.sh
-
-      - name: Prepare manifests for LibriSpeech test-clean and test-other
-        shell: bash
-        run: |
-          .github/scripts/prepare-librispeech-test-clean-and-test-other-manifests.sh
-
-      - name: Cache LibriSpeech test-clean and test-other fbank features
-        id: libri-test-clean-and-test-other-fbank
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/tmp/fbank-libri
-          key: cache-libri-fbank-test-clean-and-test-other-v2
-
-      - name: Compute fbank for LibriSpeech test-clean and test-other
-        if: steps.libri-test-clean-and-test-other-fbank.outputs.cache-hit != 'true'
-        shell: bash
-        run: |
-          .github/scripts/compute-fbank-librispeech-test-clean-and-test-other.sh
-
-      - name: Inference with pre-trained model
-        shell: bash
-        env:
-          GITHUB_EVENT_NAME: ${{ github.event_name }}
-          GITHUB_EVENT_LABEL_NAME: ${{ github.event.label.name }}
-        run: |
-          mkdir -p egs/librispeech/ASR/data
-          ln -sfv ~/tmp/fbank-libri egs/librispeech/ASR/data/fbank
-          ls -lh egs/librispeech/ASR/data/*
-
-          sudo apt-get -qq install git-lfs tree
-          export PYTHONPATH=$PWD:$PYTHONPATH
-          export PYTHONPATH=~/tmp/kaldifeat/kaldifeat/python:$PYTHONPATH
-          export PYTHONPATH=~/tmp/kaldifeat/build/lib:$PYTHONPATH
-
-          .github/scripts/run-pre-trained-transducer-stateless-librispeech-100h.sh
-
-      - name: Display decoding results for transducer_stateless_multi_datasets
-        if: github.event_name == 'schedule' || github.event.label.name == 'run-decode'
-        shell: bash
-        run: |
-          cd egs/librispeech/ASR/
-          tree ./transducer_stateless_multi_datasets/exp
-
-          cd transducer_stateless_multi_datasets
-          echo "results for transducer_stateless_multi_datasets"
-          echo "===greedy search==="
-          find exp/greedy_search -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find exp/greedy_search -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-          echo "===fast_beam_search==="
-          find exp/fast_beam_search -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find exp/fast_beam_search -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-          echo "===modified beam search==="
-          find exp/modified_beam_search -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find exp/modified_beam_search -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-      - name: Upload decoding results for transducer_stateless_multi_datasets
-        uses: actions/upload-artifact@v2
-        if: github.event_name == 'schedule' || github.event.label.name == 'run-decode'
-        with:
-          name: torch-${{ matrix.torch }}-python-${{ matrix.python-version }}-ubuntu-latest-cpu-transducer_stateless_multi_datasets-100h-2022-02-21
-          path: egs/librispeech/ASR/transducer_stateless_multi_datasets/exp/

.github/workflows/run-pretrained-transducer-stateless-librispeech-multi-datasets.yml

@@ -1,158 +0,0 @@
-# Copyright      2021  Fangjun Kuang (csukuangfj@gmail.com)
-
-# See ../../LICENSE for clarification regarding multiple authors
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: run-pre-trained-trandsucer-stateless-multi-datasets-librispeech-960h
-
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-    types: [labeled]
-
-  schedule:
-    # minute (0-59)
-    # hour (0-23)
-    # day of the month (1-31)
-    # month (1-12)
-    # day of the week (0-6)
-    # nightly build at 15:50 UTC time every day
-    - cron: "50 15 * * *"
-
-concurrency:
-  group: run_pre_trained_transducer_stateless_multi_datasets_librispeech_960h-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  run_pre_trained_transducer_stateless_multi_datasets_librispeech_960h:
-    if: github.event.label.name == 'ready' || github.event.label.name == 'run-decode' || github.event_name == 'push' || github.event_name == 'schedule'
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-latest]
-        python-version: [3.8]
-
-      fail-fast: false
-
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-
-      - name: Setup Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: 'pip'
-          cache-dependency-path: '**/requirements-ci.txt'
-
-      - name: Install Python dependencies
-        run: |
-          grep -v '^#' ./requirements-ci.txt  | xargs -n 1 -L 1 pip install
-          pip uninstall -y protobuf
-          pip install --no-binary protobuf protobuf==3.20.*
-
-      - name: Cache kaldifeat
-        id: my-cache
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/tmp/kaldifeat
-          key: cache-tmp-${{ matrix.python-version }}-2023-05-22
-
-      - name: Install kaldifeat
-        if: steps.my-cache.outputs.cache-hit != 'true'
-        shell: bash
-        run: |
-          .github/scripts/install-kaldifeat.sh
-
-      - name: Cache LibriSpeech test-clean and test-other datasets
-        id: libri-test-clean-and-test-other-data
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/tmp/download
-          key: cache-libri-test-clean-and-test-other
-
-      - name: Download LibriSpeech test-clean and test-other
-        if: steps.libri-test-clean-and-test-other-data.outputs.cache-hit != 'true'
-        shell: bash
-        run: |
-          .github/scripts/download-librispeech-test-clean-and-test-other-dataset.sh
-
-      - name: Prepare manifests for LibriSpeech test-clean and test-other
-        shell: bash
-        run: |
-          .github/scripts/prepare-librispeech-test-clean-and-test-other-manifests.sh
-
-      - name: Cache LibriSpeech test-clean and test-other fbank features
-        id: libri-test-clean-and-test-other-fbank
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/tmp/fbank-libri
-          key: cache-libri-fbank-test-clean-and-test-other-v2
-
-      - name: Compute fbank for LibriSpeech test-clean and test-other
-        if: steps.libri-test-clean-and-test-other-fbank.outputs.cache-hit != 'true'
-        shell: bash
-        run: |
-          .github/scripts/compute-fbank-librispeech-test-clean-and-test-other.sh
-
-      - name: Inference with pre-trained model
-        shell: bash
-        env:
-          GITHUB_EVENT_NAME: ${{ github.event_name }}
-          GITHUB_EVENT_LABEL_NAME: ${{ github.event.label.name }}
-        run: |
-          mkdir -p egs/librispeech/ASR/data
-          ln -sfv ~/tmp/fbank-libri egs/librispeech/ASR/data/fbank
-          ls -lh egs/librispeech/ASR/data/*
-
-          sudo apt-get -qq install git-lfs tree
-          export PYTHONPATH=$PWD:$PYTHONPATH
-          export PYTHONPATH=~/tmp/kaldifeat/kaldifeat/python:$PYTHONPATH
-          export PYTHONPATH=~/tmp/kaldifeat/build/lib:$PYTHONPATH
-
-          .github/scripts/run-pre-trained-transducer-stateless-librispeech-960h.sh
-
-      - name: Display decoding results for transducer_stateless_multi_datasets
-        if: github.event_name == 'schedule' || github.event.label.name == 'run-decode'
-        shell: bash
-        run: |
-          cd egs/librispeech/ASR/
-          tree ./transducer_stateless_multi_datasets/exp
-
-          cd transducer_stateless_multi_datasets
-          echo "results for transducer_stateless_multi_datasets"
-          echo "===greedy search==="
-          find exp/greedy_search -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find exp/greedy_search -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-          echo "===fast_beam_search==="
-          find exp/fast_beam_search -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find exp/fast_beam_search -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-          echo "===modified beam search==="
-          find exp/modified_beam_search -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find exp/modified_beam_search -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-      - name: Upload decoding results for transducer_stateless_multi_datasets
-        uses: actions/upload-artifact@v2
-        if: github.event_name == 'schedule' || github.event.label.name == 'run-decode'
-        with:
-          name: torch-${{ matrix.torch }}-python-${{ matrix.python-version }}-ubuntu-latest-cpu-transducer_stateless_multi_datasets-100h-2022-03-01
-          path: egs/librispeech/ASR/transducer_stateless_multi_datasets/exp/

.github/workflows/run-pretrained-transducer-stateless-modified-2-aishell.yml

@@ -1,80 +0,0 @@
-# Copyright      2021  Fangjun Kuang (csukuangfj@gmail.com)
-
-# See ../../LICENSE for clarification regarding multiple authors
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: run-pre-trained-trandsucer-stateless-modified-2-aishell
-
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-    types: [labeled]
-
-concurrency:
-  group: run_pre_trained_transducer_stateless_modified_2_aishell-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  run_pre_trained_transducer_stateless_modified_2_aishell:
-    if: github.event.label.name == 'ready' || github.event_name == 'push'
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-latest]
-        python-version: [3.8]
-
-      fail-fast: false
-
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-
-      - name: Setup Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: 'pip'
-          cache-dependency-path: '**/requirements-ci.txt'
-
-      - name: Install Python dependencies
-        run: |
-          grep -v '^#' ./requirements-ci.txt  | xargs -n 1 -L 1 pip install
-          pip uninstall -y protobuf
-          pip install --no-binary protobuf protobuf==3.20.*
-
-      - name: Cache kaldifeat
-        id: my-cache
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/tmp/kaldifeat
-          key: cache-tmp-${{ matrix.python-version }}-2023-05-22
-
-      - name: Install kaldifeat
-        if: steps.my-cache.outputs.cache-hit != 'true'
-        shell: bash
-        run: |
-          .github/scripts/install-kaldifeat.sh
-
-      - name: Inference with pre-trained model
-        shell: bash
-        run: |
-          sudo apt-get -qq install git-lfs tree
-          export PYTHONPATH=$PWD:$PYTHONPATH
-          export PYTHONPATH=~/tmp/kaldifeat/kaldifeat/python:$PYTHONPATH
-          export PYTHONPATH=~/tmp/kaldifeat/build/lib:$PYTHONPATH
-          .github/scripts/run-pre-trained-transducer-stateless-modified-2-aishell.sh

.github/workflows/run-pretrained-transducer-stateless-modified-aishell.yml

@@ -1,80 +0,0 @@
-# Copyright      2021  Fangjun Kuang (csukuangfj@gmail.com)
-
-# See ../../LICENSE for clarification regarding multiple authors
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: run-pre-trained-trandsucer-stateless-modified-aishell
-
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-    types: [labeled]
-
-concurrency:
-  group: run_pre_trained_transducer_stateless_modified_aishell-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  run_pre_trained_transducer_stateless_modified_aishell:
-    if: github.event.label.name == 'ready' || github.event_name == 'push'
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-latest]
-        python-version: [3.8]
-
-      fail-fast: false
-
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-
-      - name: Setup Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: 'pip'
-          cache-dependency-path: '**/requirements-ci.txt'
-
-      - name: Install Python dependencies
-        run: |
-          grep -v '^#' ./requirements-ci.txt  | xargs -n 1 -L 1 pip install
-          pip uninstall -y protobuf
-          pip install --no-binary protobuf protobuf==3.20.*
-
-      - name: Cache kaldifeat
-        id: my-cache
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/tmp/kaldifeat
-          key: cache-tmp-${{ matrix.python-version }}-2023-05-22
-
-      - name: Install kaldifeat
-        if: steps.my-cache.outputs.cache-hit != 'true'
-        shell: bash
-        run: |
-          .github/scripts/install-kaldifeat.sh
-
-      - name: Inference with pre-trained model
-        shell: bash
-        run: |
-          sudo apt-get -qq install git-lfs tree
-          export PYTHONPATH=$PWD:$PYTHONPATH
-          export PYTHONPATH=~/tmp/kaldifeat/kaldifeat/python:$PYTHONPATH
-          export PYTHONPATH=~/tmp/kaldifeat/build/lib:$PYTHONPATH
-          .github/scripts/run-pre-trained-transducer-stateless-modified-aishell.sh

.github/workflows/run-pretrained-transducer-stateless.yml

@@ -1,158 +0,0 @@
-# Copyright      2021  Fangjun Kuang (csukuangfj@gmail.com)
-
-# See ../../LICENSE for clarification regarding multiple authors
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: run-pre-trained-transducer-stateless
-
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-    types: [labeled]
-
-  schedule:
-    # minute (0-59)
-    # hour (0-23)
-    # day of the month (1-31)
-    # month (1-12)
-    # day of the week (0-6)
-    # nightly build at 15:50 UTC time every day
-    - cron: "50 15 * * *"
-
-concurrency:
-  group: run_pre_trained_transducer_stateless-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  run_pre_trained_transducer_stateless:
-    if: github.event.label.name == 'ready' || github.event.label.name == 'run-decode' || github.event_name == 'push' || github.event_name == 'schedule'
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-latest]
-        python-version: [3.8]
-
-      fail-fast: false
-
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-
-      - name: Setup Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: 'pip'
-          cache-dependency-path: '**/requirements-ci.txt'
-
-      - name: Install Python dependencies
-        run: |
-          grep -v '^#' ./requirements-ci.txt  | xargs -n 1 -L 1 pip install
-          pip uninstall -y protobuf
-          pip install --no-binary protobuf protobuf==3.20.*
-
-      - name: Cache kaldifeat
-        id: my-cache
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/tmp/kaldifeat
-          key: cache-tmp-${{ matrix.python-version }}-2023-05-22
-
-      - name: Install kaldifeat
-        if: steps.my-cache.outputs.cache-hit != 'true'
-        shell: bash
-        run: |
-          .github/scripts/install-kaldifeat.sh
-
-      - name: Cache LibriSpeech test-clean and test-other datasets
-        id: libri-test-clean-and-test-other-data
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/tmp/download
-          key: cache-libri-test-clean-and-test-other
-
-      - name: Download LibriSpeech test-clean and test-other
-        if: steps.libri-test-clean-and-test-other-data.outputs.cache-hit != 'true'
-        shell: bash
-        run: |
-          .github/scripts/download-librispeech-test-clean-and-test-other-dataset.sh
-
-      - name: Prepare manifests for LibriSpeech test-clean and test-other
-        shell: bash
-        run: |
-          .github/scripts/prepare-librispeech-test-clean-and-test-other-manifests.sh
-
-      - name: Cache LibriSpeech test-clean and test-other fbank features
-        id: libri-test-clean-and-test-other-fbank
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/tmp/fbank-libri
-          key: cache-libri-fbank-test-clean-and-test-other-v2
-
-      - name: Compute fbank for LibriSpeech test-clean and test-other
-        if: steps.libri-test-clean-and-test-other-fbank.outputs.cache-hit != 'true'
-        shell: bash
-        run: |
-          .github/scripts/compute-fbank-librispeech-test-clean-and-test-other.sh
-
-      - name: Inference with pre-trained model
-        shell: bash
-        env:
-          GITHUB_EVENT_NAME: ${{ github.event_name }}
-          GITHUB_EVENT_LABEL_NAME: ${{ github.event.label.name }}
-        run: |
-          mkdir -p egs/librispeech/ASR/data
-          ln -sfv ~/tmp/fbank-libri egs/librispeech/ASR/data/fbank
-          ls -lh egs/librispeech/ASR/data/*
-
-          sudo apt-get -qq install git-lfs tree
-          export PYTHONPATH=$PWD:$PYTHONPATH
-          export PYTHONPATH=~/tmp/kaldifeat/kaldifeat/python:$PYTHONPATH
-          export PYTHONPATH=~/tmp/kaldifeat/build/lib:$PYTHONPATH
-
-          .github/scripts/run-pre-trained-transducer-stateless.sh
-
-      - name: Display decoding results for transducer_stateless
-        if: github.event_name == 'schedule' || github.event.label.name == 'run-decode'
-        shell: bash
-        run: |
-          cd egs/librispeech/ASR/
-          tree ./transducer_stateless/exp
-
-          cd transducer_stateless
-          echo "results for transducer_stateless"
-          echo "===greedy search==="
-          find exp/greedy_search -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find exp/greedy_search -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-          echo "===fast_beam_search==="
-          find exp/fast_beam_search -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find exp/fast_beam_search -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-          echo "===modified beam search==="
-          find exp/modified_beam_search -name "log-*" -exec grep -n --color "best for test-clean" {} + | sort -n -k2
-          find exp/modified_beam_search -name "log-*" -exec grep -n --color "best for test-other" {} + | sort -n -k2
-
-      - name: Upload decoding results for transducer_stateless
-        uses: actions/upload-artifact@v2
-        if: github.event_name == 'schedule' || github.event.label.name == 'run-decode'
-        with:
-          name: torch-${{ matrix.torch }}-python-${{ matrix.python-version }}-ubuntu-latest-cpu-transducer_stateless-2022-02-07
-          path: egs/librispeech/ASR/transducer_stateless/exp/

.github/workflows/run-yesno-recipe.yml

@@ -1,185 +0,0 @@
-# Copyright      2021  Fangjun Kuang (csukuangfj@gmail.com)
-
-# See ../../LICENSE for clarification regarding multiple authors
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: run-yesno-recipe
-
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-    branches:
-      - master
-
-concurrency:
-  group: run-yesno-recipe-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  run-yesno-recipe:
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        # os: [ubuntu-latest, macos-10.15]
-        # TODO: enable macOS for CPU testing
-        os: [ubuntu-latest]
-        python-version: [3.8]
-      fail-fast: false
-
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-
-      - name: Setup Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: 'pip'
-          cache-dependency-path: '**/requirements-ci.txt'
-
-      - name: Install libnsdfile and libsox
-        if: startsWith(matrix.os, 'ubuntu')
-        run: |
-          sudo apt update
-          sudo apt install -q -y libsndfile1-dev libsndfile1 ffmpeg
-          sudo apt install -q -y --fix-missing sox libsox-dev libsox-fmt-all
-
-      - name: Install Python dependencies
-        run: |
-          grep -v '^#' ./requirements-ci.txt  | xargs -n 1 -L 1 pip install
-          pip uninstall -y protobuf
-          pip install --no-binary protobuf protobuf==3.20.*
-
-          pip install --no-deps --force-reinstall k2==1.24.4.dev20231021+cpu.torch1.13.1 -f https://k2-fsa.github.io/k2/cpu.html
-          pip install kaldifeat==1.25.1.dev20231022+cpu.torch1.13.1 -f https://csukuangfj.github.io/kaldifeat/cpu.html
-
-      - name: Run yesno recipe
-        shell: bash
-        working-directory: ${{github.workspace}}
-        run: |
-          export PYTHONPATH=$PWD:$PYTHONPATH
-          echo $PYTHONPATH
-
-          cd egs/yesno/ASR
-          ./prepare.sh
-          python3 ./tdnn/train.py
-          python3 ./tdnn/decode.py
-
-      - name: Test exporting to pretrained.pt
-        shell: bash
-        working-directory: ${{github.workspace}}
-        run: |
-          export PYTHONPATH=$PWD:$PYTHONPATH
-          echo $PYTHONPATH
-
-          cd egs/yesno/ASR
-          python3 ./tdnn/export.py --epoch 14 --avg 2
-
-          python3 ./tdnn/pretrained.py \
-            --checkpoint ./tdnn/exp/pretrained.pt \
-            --HLG ./data/lang_phone/HLG.pt \
-            --words-file ./data/lang_phone/words.txt \
-            download/waves_yesno/0_0_0_1_0_0_0_1.wav \
-            download/waves_yesno/0_0_1_0_0_0_1_0.wav
-
-      - name: Test exporting to torchscript
-        shell: bash
-        working-directory: ${{github.workspace}}
-        run: |
-          export PYTHONPATH=$PWD:$PYTHONPATH
-          echo $PYTHONPATH
-
-          cd egs/yesno/ASR
-          python3 ./tdnn/export.py --epoch 14 --avg 2 --jit 1
-
-          python3 ./tdnn/jit_pretrained.py \
-            --nn-model ./tdnn/exp/cpu_jit.pt \
-            --HLG ./data/lang_phone/HLG.pt \
-            --words-file ./data/lang_phone/words.txt \
-            download/waves_yesno/0_0_0_1_0_0_0_1.wav \
-            download/waves_yesno/0_0_1_0_0_0_1_0.wav
-
-      - name: Test exporting to onnx
-        shell: bash
-        working-directory: ${{github.workspace}}
-        run: |
-          export PYTHONPATH=$PWD:$PYTHONPATH
-          echo $PYTHONPATH
-
-          cd egs/yesno/ASR
-          python3 ./tdnn/export_onnx.py --epoch 14 --avg 2
-
-          echo "Test float32 model"
-          python3 ./tdnn/onnx_pretrained.py \
-            --nn-model ./tdnn/exp/model-epoch-14-avg-2.onnx \
-            --HLG ./data/lang_phone/HLG.pt \
-            --words-file ./data/lang_phone/words.txt \
-            download/waves_yesno/0_0_0_1_0_0_0_1.wav \
-            download/waves_yesno/0_0_1_0_0_0_1_0.wav
-
-
-          echo "Test int8 model"
-          python3 ./tdnn/onnx_pretrained.py \
-            --nn-model ./tdnn/exp/model-epoch-14-avg-2.int8.onnx \
-            --HLG ./data/lang_phone/HLG.pt \
-            --words-file ./data/lang_phone/words.txt \
-            download/waves_yesno/0_0_0_1_0_0_0_1.wav \
-            download/waves_yesno/0_0_1_0_0_0_1_0.wav
-
-      - name: Test decoding with H
-        shell: bash
-        working-directory: ${{github.workspace}}
-        run: |
-          export PYTHONPATH=$PWD:$PYTHONPATH
-          echo $PYTHONPATH
-
-          cd egs/yesno/ASR
-          python3 ./tdnn/export.py --epoch 14 --avg 2 --jit 1
-
-          python3 ./tdnn/jit_pretrained_decode_with_H.py \
-              --nn-model ./tdnn/exp/cpu_jit.pt \
-              --H ./data/lang_phone/H.fst \
-              --tokens ./data/lang_phone/tokens.txt \
-              ./download/waves_yesno/0_0_0_1_0_0_0_1.wav \
-              ./download/waves_yesno/0_0_1_0_0_0_1_0.wav \
-              ./download/waves_yesno/0_0_1_0_0_1_1_1.wav
-
-      - name: Test decoding with HL
-        shell: bash
-        working-directory: ${{github.workspace}}
-        run: |
-          export PYTHONPATH=$PWD:$PYTHONPATH
-          echo $PYTHONPATH
-
-          cd egs/yesno/ASR
-          python3 ./tdnn/export.py --epoch 14 --avg 2 --jit 1
-
-          python3 ./tdnn/jit_pretrained_decode_with_HL.py \
-              --nn-model ./tdnn/exp/cpu_jit.pt \
-              --HL ./data/lang_phone/HL.fst \
-              --words ./data/lang_phone/words.txt \
-              ./download/waves_yesno/0_0_0_1_0_0_0_1.wav \
-              ./download/waves_yesno/0_0_1_0_0_0_1_0.wav \
-              ./download/waves_yesno/0_0_1_0_0_1_1_1.wav
-
-      - name: Show generated files
-        shell: bash
-        working-directory: ${{github.workspace}}
-        run: |
-          cd egs/yesno/ASR
-          ls -lh tdnn/exp
-          ls -lh data/lang_phone

.github/workflows/test.yml

@@ -1,129 +1,109 @@
-# Copyright      2021  Fangjun Kuang (csukuangfj@gmail.com)
-
-# See ../../LICENSE for clarification regarding multiple authors
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
 name: test
 
 on:
   push:
     branches:
       - master
+
   pull_request:
     branches:
       - master
 
+  workflow_dispatch:
+
 concurrency:
   group: test-${{ github.ref }}
   cancel-in-progress: true
 
 jobs:
+  generate_build_matrix:
+    if: github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa'
+    # see https://github.com/pytorch/pytorch/pull/50633
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Generating build matrix
+        id: set-matrix
+        run: |
+          # outputting for debugging purposes
+          python ./.github/scripts/docker/generate_build_matrix.py
+          MATRIX=$(python ./.github/scripts/docker/generate_build_matrix.py)
+          echo "::set-output name=matrix::${MATRIX}"
   test:
-    runs-on: ${{ matrix.os }}
+    needs: generate_build_matrix
+    name: py${{ matrix.python-version }} torch${{ matrix.torch-version }} v${{ matrix.version }}
+    runs-on: ubuntu-latest
     strategy:
-      matrix:
-        os: [ubuntu-latest]
-        python-version: ["3.8"]
-        torch: ["1.13.0"]
-        torchaudio: ["0.13.0"]
-        k2-version: ["1.24.3.dev20230719"]
-
       fail-fast: false
+      matrix:
+        ${{ fromJson(needs.generate_build_matrix.outputs.matrix) }}
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
-      - name: Setup Python ${{ matrix.python-version }}
+      - name: Free space
-        uses: actions/setup-python@v1
-        with:
-          python-version: ${{ matrix.python-version }}
-
-      - name: Install libnsdfile and libsox
-        if: startsWith(matrix.os, 'ubuntu')
-        run: |
-          sudo apt update
-          sudo apt install -q -y libsndfile1-dev libsndfile1 ffmpeg
-          sudo apt install -q -y --fix-missing libsox-dev libsox-fmt-all
-
-      - name: Install Python dependencies
-        run: |
-          python3 -m pip install --upgrade pip pytest
-          # numpy 1.20.x does not support python 3.6
-          pip install numpy==1.19
-          pip install torch==${{ matrix.torch }}+cpu -f https://download.pytorch.org/whl/cpu/torch_stable.html
-          pip install torchaudio==${{ matrix.torchaudio }}+cpu -f https://download.pytorch.org/whl/cpu/torch_stable.html
-
-          pip install k2==${{ matrix.k2-version }}+cpu.torch${{ matrix.torch }} -f https://k2-fsa.github.io/k2/cpu.html
-          pip install git+https://github.com/lhotse-speech/lhotse
-          # icefall requirements
-          pip uninstall -y protobuf
-          pip install --no-binary protobuf protobuf==3.20.*
-
-          pip install kaldifst
-          pip install onnxruntime matplotlib
-          pip install -r requirements.txt
-
-      - name: Install graphviz
-        if: startsWith(matrix.os, 'ubuntu')
         shell: bash
         run: |
-          python3 -m pip install -qq graphviz
+          df -h
-          sudo apt-get -qq install graphviz
+          rm -rf /opt/hostedtoolcache
+          df -h
+          echo "pwd: $PWD"
+          echo "github.workspace ${{ github.workspace }}"
 
       - name: Run tests
-        if: startsWith(matrix.os, 'ubuntu')
+        uses: addnab/docker-run-action@v3
-        run: |
+        with:
-          ls -lh
+            image: ghcr.io/${{ github.repository_owner }}/icefall:cpu-py${{ matrix.python-version }}-torch${{ matrix.torch-version }}-v${{ matrix.version }}
-          export PYTHONPATH=$PWD:$PWD/lhotse:$PYTHONPATH
+            options: |
-          echo $PYTHONPATH
+              --volume ${{ github.workspace }}/:/icefall
-          pytest -v -s ./test
+            shell: bash
-          # runt tests for conformer ctc
+            run: |
-          cd egs/librispeech/ASR/conformer_ctc
+              export PYTHONPATH=/icefall:$PYTHONPATH
-          pytest -v -s
+              cd /icefall
+              git config --global --add safe.directory /icefall
 
-          cd ../pruned_transducer_stateless
+              pytest -v -s ./test
-          pytest -v -s
 
-          cd ../pruned_transducer_stateless2
+              # runt tests for conformer ctc
-          pytest -v -s
+              cd egs/librispeech/ASR/conformer_ctc
+              pytest -v -s
 
-          cd ../pruned_transducer_stateless3
+              cd ../pruned_transducer_stateless
-          pytest -v -s
+              pytest -v -s
 
-          cd ../pruned_transducer_stateless4
+              cd ../pruned_transducer_stateless2
-          pytest -v -s
+              pytest -v -s
 
-          echo $PYTHONPATH
+              cd ../pruned_transducer_stateless3
-          cd ../pruned_transducer_stateless7
+              pytest -v -s
-          pytest -v -s
 
-          cd ../transducer_stateless
+              cd ../pruned_transducer_stateless4
-          pytest -v -s
+              pytest -v -s
 
-          # cd ../transducer
+              echo $PYTHONPATH
-          # pytest -v -s
+              cd ../pruned_transducer_stateless7
+              pytest -v -s
 
-          cd ../transducer_stateless2
+              cd ../transducer_stateless
-          pytest -v -s
+              pytest -v -s
 
-          cd ../transducer_lstm
+              # cd ../transducer
-          pytest -v -s
+              # pytest -v -s
 
-          cd ../zipformer
+              cd ../transducer_stateless2
-          pytest -v -s
+              pytest -v -s
+
+              cd ../transducer_lstm
+              pytest -v -s
+
+              cd ../zipformer
+              pytest -v -s
 
       - uses: actions/upload-artifact@v2
         with:

.github/workflows/yesno.yml

@@ -0,0 +1,62 @@
+name: yesno
+
+on:
+  push:
+    branches:
+      - master
+
+  pull_request:
+    branches:
+      - master
+
+  workflow_dispatch:
+
+concurrency:
+  group: yesno-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  generate_build_matrix:
+    if: github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa'
+    # see https://github.com/pytorch/pytorch/pull/50633
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Generating build matrix
+        id: set-matrix
+        run: |
+          # outputting for debugging purposes
+          python ./.github/scripts/docker/generate_build_matrix.py
+          MATRIX=$(python ./.github/scripts/docker/generate_build_matrix.py)
+          echo "::set-output name=matrix::${MATRIX}"
+  yesno:
+    needs: generate_build_matrix
+    name: py${{ matrix.python-version }} torch${{ matrix.torch-version }} v${{ matrix.version }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        ${{ fromJson(needs.generate_build_matrix.outputs.matrix) }}
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Run the yesno recipe
+        uses: addnab/docker-run-action@v3
+        with:
+            image: ghcr.io/${{ github.repository_owner }}/icefall:cpu-py${{ matrix.python-version }}-torch${{ matrix.torch-version }}-v${{ matrix.version }}
+            options: |
+              --volume ${{ github.workspace }}/:/icefall
+            shell: bash
+            run: |
+              export PYTHONPATH=/icefall:$PYTHONPATH
+              cd /icefall
+              git config --global --add safe.directory /icefall
+
+              .github/scripts/yesno/ASR/run.sh

README.md

@@ -118,11 +118,12 @@ We provide a Colab notebook to run a pre-trained transducer conformer + stateles
 
 #### k2 pruned RNN-T
 
-| Encoder         | Params | test-clean | test-other |
+| Encoder         | Params | test-clean | test-other | epochs  | devices    |
-|-----------------|--------|------------|------------|
+|-----------------|--------|------------|------------|---------|------------|
-| zipformer       | 65.5M  | 2.21       | 4.79       |
+| zipformer       | 65.5M  | 2.21       | 4.79       | 50      | 4 32G-V100 |
-| zipformer-small | 23.2M  | 2.42       | 5.73       |
+| zipformer-small | 23.2M  | 2.42       | 5.73       | 50      | 2 32G-V100 |
-| zipformer-large | 148.4M | 2.06       | 4.63       |
+| zipformer-large | 148.4M | 2.06       | 4.63       | 50      | 4 32G-V100 |
+| zipformer-large | 148.4M | 2.00       | 4.38       | 174     | 8 80G-A100 |
 
 Note: No auxiliary losses are used in the training and no LMs are used
 in the decoding.
@@ -366,7 +367,7 @@ Once you have trained a model in icefall, you may want to deploy it with C++,
 without Python dependencies.
 
 Please refer to the documentation
-<https://icefall.readthedocs.io/en/latest/recipes/librispeech/conformer_ctc.html#deployment-with-c>
+<https://icefall.readthedocs.io/en/latest/recipes/Non-streaming-ASR/librispeech/conformer_ctc.html#deployment-with-c>
 for how to do this.
 
 We also provide a Colab notebook, showing you how to run a torch scripted model in [k2][k2] with C++.

contributing.md

@@ -1,39 +1,37 @@
+# Contributing to Our Project
 
-## Pre-commit hooks
+Thank you for your interest in contributing to our project! We use Git pre-commit hooks to ensure code quality and consistency. Before contributing, please follow these guidelines to enable and use the pre-commit hooks.
 
-We use [git][git] [pre-commit][pre-commit] [hooks][hooks] to check that files
+## Pre-Commit Hooks
-going to be committed:
 
-  - contain no trailing spaces
+We have set up pre-commit hooks to check that the files you're committing meet our coding and formatting standards. These checks include:
-  - are formatted with [black][black]
-  - are compatible to [PEP8][PEP8] (checked by [flake8][flake8])
-  - end in a newline and only a newline
-  - contain sorted `imports` (checked by [isort][isort])
 
-These hooks are disabled by default. Please use the following commands to enable them:
+- Ensuring there are no trailing spaces.
+- Formatting code with [black](https://github.com/psf/black).
+- Checking compliance with PEP8 using [flake8](https://flake8.pycqa.org/).
+- Verifying that files end with a newline character (and only a newline).
+- Sorting imports using [isort](https://pycqa.github.io/isort/).
 
-```bash
+Please note that these hooks are disabled by default. To enable them, follow these steps:
-pip install pre-commit  # run it only once
-pre-commit install      # run it only once, it will install all hooks
 
-# modify some files
+### Installation (Run only once)
-git add <some files>
-git commit              # It runs all hooks automatically.
 
-# If all hooks run successfully, you can write the commit message now. Done!
+1. Install the `pre-commit` package using pip:
-#
+   ```bash
-# If any hook failed, your commit was not successful.
+   pip install pre-commit
-# Please read the error messages and make changes accordingly.
+   ```
-# And rerun
+1. Install the Git hooks using:
+   ```bash
+   pre-commit install
+   ```
+### Making a Commit
+Once you have enabled the pre-commit hooks, follow these steps when making a commit:
+1. Make your changes to the codebase.
+2. Stage your changes by using git add for the files you modified.
+3. Commit your changes using git commit. The pre-commit hooks will run automatically at this point.
+4. If all hooks run successfully, you can write your commit message, and your changes will be successfully committed.
+5. If any hook fails, your commit will not be successful. Please read and follow the error messages provided, make the necessary changes, and then re-run git add and git commit.
 
-git add <some files>
+### Your Contribution
-git commit
+Your contributions are valuable to us, and by following these guidelines, you help maintain code consistency and quality in our project. We appreciate your dedication to ensuring high-quality code. If you have questions or need assistance, feel free to reach out to us. Thank you for being part of our open-source community!
-```
 
-[git]: https://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks
-[flake8]: https://github.com/PyCQA/flake8
-[PEP8]: https://www.python.org/dev/peps/pep-0008/
-[black]: https://github.com/psf/black
-[hooks]: https://github.com/pre-commit/pre-commit-hooks
-[pre-commit]: https://github.com/pre-commit/pre-commit
-[isort]: https://github.com/PyCQA/isort

docker/torch1.12.1-cuda11.3.dockerfile

@@ -18,7 +18,7 @@ RUN apt-get update && \
     apt-get install -y --no-install-recommends \
         curl \
         vim \
-    		libssl-dev \
+    	libssl-dev \
         autoconf \
         automake \
         bzip2 \
@@ -44,7 +44,6 @@ RUN pip install --no-cache-dir \
       k2==${K2_VERSION} -f https://k2-fsa.github.io/k2/cuda.html \
       git+https://github.com/lhotse-speech/lhotse \
       kaldifeat==${KALDIFEAT_VERSION} -f https://csukuangfj.github.io/kaldifeat/cuda.html \
-      \
       kaldi_native_io \
       kaldialign \
       kaldifst \

docker/torch1.13.0-cuda11.6.dockerfile

@@ -18,7 +18,7 @@ RUN apt-get update && \
     apt-get install -y --no-install-recommends \
         curl \
         vim \
-    		libssl-dev \
+    	libssl-dev \
         autoconf \
         automake \
         bzip2 \
@@ -44,7 +44,6 @@ RUN pip install --no-cache-dir \
       k2==${K2_VERSION} -f https://k2-fsa.github.io/k2/cuda.html \
       git+https://github.com/lhotse-speech/lhotse \
       kaldifeat==${KALDIFEAT_VERSION} -f https://csukuangfj.github.io/kaldifeat/cuda.html \
-      \
       kaldi_native_io \
       kaldialign \
       kaldifst \

docker/torch1.9.0-cuda10.2.dockerfile

@@ -25,7 +25,7 @@ RUN apt-get update && \
     apt-get install -y --no-install-recommends \
         curl \
         vim \
-    		libssl-dev \
+    	libssl-dev \
         autoconf \
         automake \
         bzip2 \
@@ -58,7 +58,6 @@ RUN pip uninstall -y tqdm && \
       k2==${K2_VERSION} -f https://k2-fsa.github.io/k2/cuda.html \
       kaldifeat==${KALDIFEAT_VERSION} -f https://csukuangfj.github.io/kaldifeat/cuda.html \
       git+https://github.com/lhotse-speech/lhotse \
-      \
       kaldi_native_io \
       kaldialign \
       kaldifst \

docker/torch2.0.0-cuda11.7.dockerfile

@@ -18,7 +18,7 @@ RUN apt-get update && \
     apt-get install -y --no-install-recommends \
         curl \
         vim \
-    		libssl-dev \
+    	libssl-dev \
         autoconf \
         automake \
         bzip2 \
@@ -44,7 +44,6 @@ RUN pip install --no-cache-dir \
       k2==${K2_VERSION} -f https://k2-fsa.github.io/k2/cuda.html \
       git+https://github.com/lhotse-speech/lhotse \
       kaldifeat==${KALDIFEAT_VERSION} -f https://csukuangfj.github.io/kaldifeat/cuda.html \
-      \
       kaldi_native_io \
       kaldialign \
       kaldifst \

docker/torch2.1.0-cuda11.8.dockerfile

@@ -18,7 +18,7 @@ RUN apt-get update && \
     apt-get install -y --no-install-recommends \
         curl \
         vim \
-    		libssl-dev \
+    	libssl-dev \
         autoconf \
         automake \
         bzip2 \
@@ -44,7 +44,6 @@ RUN pip install --no-cache-dir \
       k2==${K2_VERSION} -f https://k2-fsa.github.io/k2/cuda.html \
       git+https://github.com/lhotse-speech/lhotse \
       kaldifeat==${KALDIFEAT_VERSION} -f https://csukuangfj.github.io/kaldifeat/cuda.html \
-      \
       kaldi_native_io \
       kaldialign \
       kaldifst \

docker/torch2.1.0-cuda12.1.dockerfile

@@ -18,7 +18,7 @@ RUN apt-get update && \
     apt-get install -y --no-install-recommends \
         curl \
         vim \
-    		libssl-dev \
+    	libssl-dev \
         autoconf \
         automake \
         bzip2 \
@@ -44,7 +44,6 @@ RUN pip install --no-cache-dir \
       k2==${K2_VERSION} -f https://k2-fsa.github.io/k2/cuda.html \
       git+https://github.com/lhotse-speech/lhotse \
       kaldifeat==${KALDIFEAT_VERSION} -f https://csukuangfj.github.io/kaldifeat/cuda.html \
-      \
       kaldi_native_io \
       kaldialign \
       kaldifst \

docs/source/contributing/how-to-create-a-recipe.rst

@@ -3,7 +3,7 @@ How to create a recipe
 
 .. HINT::
 
-  Please read :ref:`follow the code style` to adjust your code sytle.
+  Please read :ref:`follow the code style` to adjust your code style.
 
 .. CAUTION::
 

docs/source/docker/intro.rst

@@ -20,7 +20,11 @@ We describe the following items in this section:
 View available tags
 ===================
 
-You can use the following command to view available tags:
+CUDA-enabled docker images
+--------------------------
+
+You can use the following command to view available tags for CUDA-enabled
+docker images:
 
 .. code-block:: bash
 
@@ -43,8 +47,25 @@ which will give you something like below:
 
 Please select an appropriate combination of `torch`_ and  CUDA.
 
-Download a docker image
+CPU-only docker images
-=======================
+----------------------
+
+To view CPU-only docker images, please visit `<https://github.com/k2-fsa/icefall/pkgs/container/icefall>`_
+for available tags.
+
+You can select different combinations of ``Python`` and ``torch``. For instance,
+to select ``Python 3.8`` and ``torch 2.1.2``, you can use the following tag
+
+.. code-block:: bash
+
+    cpu-py3.8-torch2.1.2-v1.1
+
+where ``v1.1`` is the current version of the docker image. You may see
+``ghcr.io/k2-fsa/icefall:cpu-py3.8-torch2.1.2-v1.2`` or some other versions.
+We recommend that you always use the latest version.
+
+Download a docker image (CUDA)
+==============================
 
 Suppose that you select the tag ``torch1.13.0-cuda11.6``, you can use
 the following command to download it:
@@ -53,6 +74,16 @@ the following command to download it:
 
    sudo docker image pull k2fsa/icefall:torch1.13.0-cuda11.6
 
+Download a docker image (CPU)
+==============================
+
+Suppose that you select the tag ``cpu-py3.8-torch2.1.2-v1.1``, you can use
+the following command to download it:
+
+.. code-block:: bash
+
+  sudo docker pull ghcr.io/k2-fsa/icefall:cpu-py3.8-torch2.1.2-v1.1
+
 Run a docker image with GPU
 ===========================
 
@@ -65,7 +96,7 @@ Run a docker image with CPU
 
 .. code-block:: bash
 
-  sudo docker run --rm -it k2fsa/icefall:torch1.13.0-cuda11.6 /bin/bash
+  sudo docker run --rm -it ghcr.io/k2-fsa/icefall:cpu-py3.8-torch2.1.2-v1.1 /bin/bash
 
 Run yesno within a docker container
 ===================================
@@ -74,8 +105,13 @@ After starting the container, the following interface is presented:
 
 .. code-block:: bash
 
+  # GPU-enabled docker
   root@60c947eac59c:/workspace/icefall#
 
+  # CPU-only docker
+  root@60c947eac59c:# mkdir /workspace; git clone https://github.com/k2-fsa/icefall
+  root@60c947eac59c:# export PYTHONPATH=/workspace/icefall:$PYTHONPATH
+
 It shows the current user is ``root`` and the current working directory
 is ``/workspace/icefall``.
 
@@ -107,7 +143,7 @@ to switch to the ``yesno`` recipe and run
 
 .. hint::
 
-   If you are running without GPU, it may report the following error:
+   If you are running without GPU with a GPU-enabled docker, it may report the following error:
 
     .. code-block:: bash
 

docs/source/for-dummies/environment-setup.rst

@@ -66,13 +66,13 @@ to install dependencies of `icefall`_:
 
    pip install torch==2.0.0+cpu torchaudio==2.0.0+cpu -f https://download.pytorch.org/whl/torch_stable.html
 
-   # If you are using macOS or Windows, please use the following command to install torch and torchaudio
+   # If you are using macOS, please use the following command to install torch and torchaudio
    # pip install torch==2.0.0 torchaudio==2.0.0 -f https://download.pytorch.org/whl/torch_stable.html
 
    # Now install k2
    # Please refer to https://k2-fsa.github.io/k2/installation/from_wheels.html#linux-cpu-example
 
-   pip install k2==1.24.3.dev20230726+cpu.torch2.0.0 -f https://k2-fsa.github.io/k2/cpu.html
+   pip install k2==1.24.4.dev20231220+cpu.torch2.0.0 -f https://k2-fsa.github.io/k2/cpu.html
 
    # Install the latest version of lhotse
 

docs/source/for-dummies/model-export.rst

@@ -85,7 +85,7 @@ We can also use it to decode files with the following command:
   # Please refer to https://csukuangfj.github.io/kaldifeat/installation/from_wheels.html
   # for how to install kaldifeat
 
-  pip install kaldifeat==1.25.0.dev20230726+cpu.torch2.0.0 -f https://csukuangfj.github.io/kaldifeat/cpu.html
+  pip install kaldifeat==1.25.3.dev20231221+cpu.torch2.0.0 -f https://csukuangfj.github.io/kaldifeat/cpu.html
 
   ./tdnn/pretrained.py \
     --checkpoint ./tdnn/exp/pretrained.pt \
@@ -162,7 +162,7 @@ To use ``tdnn/exp/cpu_jit.pt`` with `icefall`_ to decode files, we can use:
   # Please refer to https://csukuangfj.github.io/kaldifeat/installation/from_wheels.html
   # for how to install kaldifeat
 
-  pip install kaldifeat==1.25.0.dev20230726+cpu.torch2.0.0 -f https://csukuangfj.github.io/kaldifeat/cpu.html
+  pip install kaldifeat==1.25.3.dev20231221+cpu.torch2.0.0 -f https://csukuangfj.github.io/kaldifeat/cpu.html
 
 
   ./tdnn/jit_pretrained.py \
@@ -249,7 +249,7 @@ To use the generated ONNX model files for decoding with `onnxruntime`_, we can u
   # Please refer to https://csukuangfj.github.io/kaldifeat/installation/from_wheels.html
   # for how to install kaldifeat
 
-  pip install kaldifeat==1.25.0.dev20230726+cpu.torch2.0.0 -f https://csukuangfj.github.io/kaldifeat/cpu.html
+  pip install kaldifeat==1.25.3.dev20231221+cpu.torch2.0.0 -f https://csukuangfj.github.io/kaldifeat/cpu.html
 
   ./tdnn/onnx_pretrained.py \
     --nn-model ./tdnn/exp/model-epoch-14-avg-2.onnx \

docs/source/recipes/Non-streaming-ASR/aishell/conformer_ctc.rst

@@ -67,7 +67,7 @@ To run stage 2 to stage 5, use:
 .. HINT::
 
   A 3-gram language model will be downloaded from huggingface, we assume you have
-  intalled and initialized ``git-lfs``. If not, you could install ``git-lfs`` by
+  installed and initialized ``git-lfs``. If not, you could install ``git-lfs`` by
 
   .. code-block:: bash
 

docs/source/recipes/Non-streaming-ASR/aishell/tdnn_lstm_ctc.rst

@@ -67,7 +67,7 @@ To run stage 2 to stage 5, use:
 .. HINT::
 
   A 3-gram language model will be downloaded from huggingface, we assume you have
-  intalled and initialized ``git-lfs``. If not, you could install ``git-lfs`` by
+  installed and initialized ``git-lfs``. If not, you could install ``git-lfs`` by
 
   .. code-block:: bash
 

docs/source/recipes/Non-streaming-ASR/librispeech/pruned_transducer_stateless.rst

@@ -418,7 +418,7 @@ The following shows two examples (for two types of checkpoints):
 
     - ``beam_search`` :  It implements Algorithm 1 in https://arxiv.org/pdf/1211.3711.pdf and
       `espnet/nets/beam_search_transducer.py <https://github.com/espnet/espnet/blob/master/espnet/nets/beam_search_transducer.py#L247>`_
-      is used as a reference. Basicly, it keeps topk states for each frame, and expands the kept states with their own contexts to
+      is used as a reference. Basically, it keeps topk states for each frame, and expands the kept states with their own contexts to
       next frame.
 
     - ``modified_beam_search`` : It implements the same algorithm as ``beam_search`` above, but it

docs/source/recipes/RNN-LM/librispeech/lm-training.rst

@@ -1,10 +1,10 @@
 .. _train_nnlm:
 
-Train an RNN langugage model
+Train an RNN language model
 ======================================
 
 If you have enough text data, you can train a neural network language model (NNLM) to improve
-the WER of your E2E ASR system. This tutorial shows you how to train an RNNLM from 
+the WER of your E2E ASR system. This tutorial shows you how to train an RNNLM from
 scratch.
 
 .. HINT::
@@ -15,23 +15,23 @@ scratch.
 .. note::
 
     This tutorial is based on the LibriSpeech recipe. Please check it out for the necessary
-    python scripts for this tutorial. We use the LibriSpeech LM-corpus as the LM training set 
+    python scripts for this tutorial. We use the LibriSpeech LM-corpus as the LM training set
     for illustration purpose. You can also collect your own data. The data format is quite simple:
     each line should contain a complete sentence, and words should be separated by space.
 
-First, let's download the training data for the RNNLM. This can be done via the 
+First, let's download the training data for the RNNLM. This can be done via the
 following command:
 
 .. code-block:: bash
 
-    $ wget https://www.openslr.org/resources/11/librispeech-lm-norm.txt.gz 
+    $ wget https://www.openslr.org/resources/11/librispeech-lm-norm.txt.gz
     $ gzip -d librispeech-lm-norm.txt.gz
 
 As we are training a BPE-level RNNLM, we need to tokenize the training text, which requires a
 BPE tokenizer. This can be achieved by executing the following command:
 
 .. code-block:: bash
-    
+
     $ # if you don't have the BPE
     $ GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/Zengwei/icefall-asr-librispeech-zipformer-2023-05-15
     $ cd icefall-asr-librispeech-zipformer-2023-05-15/data/lang_bpe_500
@@ -56,11 +56,11 @@ sentence length.
         --out-statistics data/lang_bpe_500/lm_data_stats.txt
 
 
-The aforementioned steps can be repeated to create a a validation set for you RNNLM. Let's say 
+The aforementioned steps can be repeated to create a a validation set for you RNNLM. Let's say
-you have a validation set in ``valid.txt``, you can just set ``--lm-data valid.txt`` 
+you have a validation set in ``valid.txt``, you can just set ``--lm-data valid.txt``
 and ``--lm-archive data/lang_bpe_500/lm-data-valid.pt`` when calling ``./local/prepare_lm_training_data.py``.
 
-After completing the previous steps, the training and testing sets for training RNNLM are ready. 
+After completing the previous steps, the training and testing sets for training RNNLM are ready.
 The next step is to train the RNNLM model. The training command is as follows:
 
 .. code-block:: bash
@@ -77,7 +77,7 @@ The next step is to train the RNNLM model. The training command is as follows:
         --use-fp16 0 \
         --tie-weights 1 \
         --embedding-dim 2048 \
-        --hidden_dim 2048 \
+        --hidden-dim 2048 \
         --num-layers 3 \
         --batch-size 300 \
         --lm-data rnn_lm/data/lang_bpe_500/sorted_lm_data.pt \
@@ -93,12 +93,3 @@ The next step is to train the RNNLM model. The training command is as follows:
 .. note::
 
     The training of RNNLM can take a long time (usually a couple of days).
-
-
-
-
-
-
-
-
-

docs/source/recipes/Streaming-ASR/introduction.rst

@@ -32,7 +32,7 @@ In icefall, we implement the streaming conformer the way just like what `WeNet <
 .. HINT::
    If you want to modify a non-streaming conformer recipe to support both streaming and non-streaming, please refer
    to `this pull request <https://github.com/k2-fsa/icefall/pull/454>`_.  After adding the code needed by streaming training,
-   you have to re-train it with the extra arguments metioned in the docs above to get a streaming model.
+   you have to re-train it with the extra arguments mentioned in the docs above to get a streaming model.
 
 
 Streaming Emformer

docs/source/recipes/Streaming-ASR/librispeech/pruned_transducer_stateless.rst

@@ -584,7 +584,7 @@ The following shows two examples (for the two types of checkpoints):
 
     - ``beam_search`` :  It implements Algorithm 1 in https://arxiv.org/pdf/1211.3711.pdf and
       `espnet/nets/beam_search_transducer.py <https://github.com/espnet/espnet/blob/master/espnet/nets/beam_search_transducer.py#L247>`_
-      is used as a reference. Basicly, it keeps topk states for each frame, and expands the kept states with their own contexts to
+      is used as a reference. Basically, it keeps topk states for each frame, and expands the kept states with their own contexts to
       next frame.
 
     - ``modified_beam_search`` : It implements the same algorithm as ``beam_search`` above, but it
@@ -648,7 +648,7 @@ command to extract ``model.state_dict()``.
 .. caution::
 
    ``--streaming-model`` and ``--causal-convolution`` require to be True to export
-   a streaming mdoel.
+   a streaming model.
 
 It will generate a file ``./pruned_transducer_stateless4/exp/pretrained.pt``.
 
@@ -697,7 +697,7 @@ Export model using ``torch.jit.script()``
 .. caution::
 
    ``--streaming-model`` and ``--causal-convolution`` require to be True to export
-   a streaming mdoel.
+   a streaming model.
 
 It will generate a file ``cpu_jit.pt`` in the given ``exp_dir``. You can later
 load it by ``torch.jit.load("cpu_jit.pt")``.

docs/source/recipes/TTS/index.rst

@@ -0,0 +1,8 @@
+TTS
+======
+
+.. toctree::
+   :maxdepth: 2
+
+   ljspeech/vits
+   vctk/vits

docs/source/recipes/TTS/ljspeech/vits.rst

@@ -0,0 +1,123 @@
+VITS
+===============
+
+This tutorial shows you how to train an VITS model
+with the `LJSpeech <https://keithito.com/LJ-Speech-Dataset/>`_ dataset.
+
+.. note::
+  
+   TTS related recipes require packages in ``requirements-tts.txt``.
+
+.. note::
+
+   The VITS paper: `Conditional Variational Autoencoder with Adversarial Learning for End-to-End Text-to-Speech <https://arxiv.org/pdf/2106.06103.pdf>`_
+
+
+Data preparation
+----------------
+
+.. code-block:: bash
+
+  $ cd egs/ljspeech/TTS
+  $ ./prepare.sh
+
+To run stage 1 to stage 5, use
+
+.. code-block:: bash
+
+  $ ./prepare.sh --stage 1 --stop_stage 5
+
+
+Build Monotonic Alignment Search
+--------------------------------
+
+.. code-block:: bash
+
+  $ ./prepare.sh --stage -1 --stop_stage -1
+
+or
+
+.. code-block:: bash
+
+  $ cd vits/monotonic_align
+  $ python setup.py build_ext --inplace
+  $ cd ../../
+
+
+Training
+--------
+
+.. code-block:: bash
+
+  $ export CUDA_VISIBLE_DEVICES="0,1,2,3"
+  $ ./vits/train.py \
+      --world-size 4 \
+      --num-epochs 1000 \
+      --start-epoch 1 \
+      --use-fp16 1 \
+      --exp-dir vits/exp \
+      --tokens data/tokens.txt
+      --max-duration 500
+
+.. note::
+
+    You can adjust the hyper-parameters to control the size of the VITS model and
+    the training configurations. For more details, please run ``./vits/train.py --help``.
+
+.. note::
+
+    The training can take a long time (usually a couple of days).
+
+Training logs, checkpoints and tensorboard logs are saved in ``vits/exp``.
+
+
+Inference
+---------
+
+The inference part uses checkpoints saved by the training part, so you have to run the
+training part first. It will save the ground-truth and generated wavs to the directory
+``vits/exp/infer/epoch-*/wav``, e.g., ``vits/exp/infer/epoch-1000/wav``.
+
+.. code-block:: bash
+
+  $ export CUDA_VISIBLE_DEVICES="0"
+  $ ./vits/infer.py \
+      --epoch 1000 \
+      --exp-dir vits/exp \
+      --tokens data/tokens.txt \
+      --max-duration 500
+
+.. note::
+
+    For more details, please run ``./vits/infer.py --help``.
+
+
+Export models
+-------------
+
+Currently we only support ONNX model exporting. It will generate two files in the given ``exp-dir``:
+``vits-epoch-*.onnx`` and ``vits-epoch-*.int8.onnx``.
+
+.. code-block:: bash
+
+  $ ./vits/export-onnx.py \
+      --epoch 1000 \
+      --exp-dir vits/exp \
+      --tokens data/tokens.txt
+
+You can test the exported ONNX model with:
+
+.. code-block:: bash
+
+  $ ./vits/test_onnx.py \
+      --model-filename vits/exp/vits-epoch-1000.onnx \
+      --tokens data/tokens.txt
+
+
+Download pretrained models
+--------------------------
+
+If you don't want to train from scratch, you can download the pretrained models
+by visiting the following link:
+
+  - `<https://huggingface.co/Zengwei/icefall-tts-ljspeech-vits-2023-11-29>`_

docs/source/recipes/TTS/vctk/vits.rst

@@ -0,0 +1,125 @@
+VITS
+===============
+
+This tutorial shows you how to train an VITS model
+with the `VCTK <https://datashare.ed.ac.uk/handle/10283/3443>`_ dataset.
+
+.. note::
+  
+   TTS related recipes require packages in ``requirements-tts.txt``.
+
+.. note::
+
+   The VITS paper: `Conditional Variational Autoencoder with Adversarial Learning for End-to-End Text-to-Speech <https://arxiv.org/pdf/2106.06103.pdf>`_
+
+
+Data preparation
+----------------
+
+.. code-block:: bash
+
+  $ cd egs/vctk/TTS
+  $ ./prepare.sh
+
+To run stage 1 to stage 6, use
+
+.. code-block:: bash
+
+  $ ./prepare.sh --stage 1 --stop_stage 6
+
+
+Build Monotonic Alignment Search
+--------------------------------
+
+To build the monotonic alignment search, use the following commands:
+
+.. code-block:: bash
+
+  $ ./prepare.sh --stage -1 --stop_stage -1
+
+or
+
+.. code-block:: bash
+
+  $ cd vits/monotonic_align
+  $ python setup.py build_ext --inplace
+  $ cd ../../
+
+
+Training
+--------
+
+.. code-block:: bash
+
+  $ export CUDA_VISIBLE_DEVICES="0,1,2,3"
+  $ ./vits/train.py \
+      --world-size 4 \
+      --num-epochs 1000 \
+      --start-epoch 1 \
+      --use-fp16 1 \
+      --exp-dir vits/exp \
+      --tokens data/tokens.txt
+      --max-duration 350
+
+.. note::
+
+    You can adjust the hyper-parameters to control the size of the VITS model and
+    the training configurations. For more details, please run ``./vits/train.py --help``.
+
+.. note::
+
+    The training can take a long time (usually a couple of days).
+
+Training logs, checkpoints and tensorboard logs are saved in ``vits/exp``.
+
+
+Inference
+---------
+
+The inference part uses checkpoints saved by the training part, so you have to run the
+training part first. It will save the ground-truth and generated wavs to the directory
+``vits/exp/infer/epoch-*/wav``, e.g., ``vits/exp/infer/epoch-1000/wav``.
+
+.. code-block:: bash
+
+  $ export CUDA_VISIBLE_DEVICES="0"
+  $ ./vits/infer.py \
+      --epoch 1000 \
+      --exp-dir vits/exp \
+      --tokens data/tokens.txt \
+      --max-duration 500
+
+.. note::
+
+    For more details, please run ``./vits/infer.py --help``.
+
+
+Export models
+-------------
+
+Currently we only support ONNX model exporting. It will generate two files in the given ``exp-dir``:
+``vits-epoch-*.onnx`` and ``vits-epoch-*.int8.onnx``.
+
+.. code-block:: bash
+
+  $ ./vits/export-onnx.py \
+      --epoch 1000 \
+      --exp-dir vits/exp \
+      --tokens data/tokens.txt
+
+You can test the exported ONNX model with:
+
+.. code-block:: bash
+
+  $ ./vits/test_onnx.py \
+      --model-filename vits/exp/vits-epoch-1000.onnx \
+      --tokens data/tokens.txt
+
+
+Download pretrained models
+--------------------------
+
+If you don't want to train from scratch, you can download the pretrained models
+by visiting the following link:
+
+  - `<https://huggingface.co/zrjin/icefall-tts-vctk-vits-2023-12-05>`_

docs/source/recipes/index.rst

@@ -2,7 +2,7 @@ Recipes
 =======
 
 This page contains various recipes in ``icefall``.
-Currently, only speech recognition recipes are provided.
+Currently, we provide recipes for speech recognition, language model, and speech synthesis.
 
 We may add recipes for other tasks as well in the future.
 
@@ -16,3 +16,4 @@ We may add recipes for other tasks as well in the future.
    Non-streaming-ASR/index
    Streaming-ASR/index
    RNN-LM/index
+   TTS/index

egs/aidatatang_200zh/ASR/prepare.sh

@@ -7,6 +7,8 @@ set -eou pipefail
 
 stage=-1
 stop_stage=100
+perturb_speed=true
+
 
 # We assume dl_dir (download dir) contains the following
 # directories and files. If not, they will be downloaded
@@ -77,7 +79,7 @@ if [ $stage -le 4 ] && [ $stop_stage -ge 4 ]; then
   log "Stage 4: Compute fbank for aidatatang_200zh"
   if [ ! -f data/fbank/.aidatatang_200zh.done ]; then
     mkdir -p data/fbank
-    ./local/compute_fbank_aidatatang_200zh.py --perturb-speed True
+    ./local/compute_fbank_aidatatang_200zh.py --perturb-speed ${perturb_speed}
     touch data/fbank/.aidatatang_200zh.done
   fi
 fi

egs/aidatatang_200zh/ASR/pruned_transducer_stateless2/asr_datamodule.py

@@ -102,7 +102,7 @@ class Aidatatang_200zhAsrDataModule:
         group.add_argument(
             "--bucketing-sampler",
             type=str2bool,
-            default=True,
+            default=False,
             help="When enabled, the batches will come from buckets of "
             "similar duration (saves padding frames).",
         )
@@ -288,6 +288,8 @@ class Aidatatang_200zhAsrDataModule:
                 max_duration=self.args.max_duration,
                 shuffle=self.args.shuffle,
                 num_buckets=self.args.num_buckets,
+                buffer_size=self.args.num_buckets * 2000,
+                shuffle_buffer_size=self.args.num_buckets * 5000,
                 drop_last=True,
             )
         else:

egs/aidatatang_200zh/ASR/pruned_transducer_stateless2/pretrained.py

@@ -242,6 +242,7 @@ def main():
     opts.frame_opts.snip_edges = False
     opts.frame_opts.samp_freq = params.sample_rate
     opts.mel_opts.num_bins = params.feature_dim
+    opts.mel_opts.high_freq = -400
 
     fbank = kaldifeat.Fbank(opts)
 

egs/aishell/ASR/README.md

@@ -1,10 +1,12 @@
 
 # Introduction
 
-Please refer to <https://icefall.readthedocs.io/en/latest/recipes/Non-streaming-ASR/aishell/index.html>
+Please refer to <https://k2-fsa.github.io/icefall/recipes/Non-streaming-ASR/aishell/index.html> for how to run models in this recipe.
-for how to run models in this recipe.
 
+Aishell is an open-source Chinese Mandarin speech corpus published by Beijing Shell Shell Technology Co., Ltd.
+400 people from different accent areas in China are invited to participate in the recording, which is conducted in a quiet indoor environment using high fidelity microphone and downsampled to 16kHz. The manual transcription accuracy is above 95%, through professional speech annotation and strict quality inspection. The data is free for academic use. We hope to provide moderate amount of data for new researchers in the field of speech recognition.
 
+(From [Open Speech and Language Resources](https://www.openslr.org/33/))
 
 # Transducers
 

egs/aishell/ASR/RESULTS.md

@@ -1,6 +1,214 @@
 ## Results
 
-### Aishell training result(Stateless Transducer)
+### Aishell training result (Stateless Transducer)
+
+#### Zipformer (Byte-level BPE)
+
+[./zipformer](./zipformer/)
+
+It's reworked Zipformer with Pruned RNNT loss, trained with Byte-level BPE, `vocab_size` set to 500.
+
+##### normal-scaled model, number of model parameters: 65549011, i.e., 65.55 M
+
+|                        | test | dev  | comment                                 |
+|------------------------|------|------|-----------------------------------------|
+| greedy search          | 4.54 | 4.31 | --epoch 40 --avg 10                     |
+| modified beam search   | 4.37 | 4.11 | --epoch 40 --avg 10                     |
+| fast beam search       | 4.43 | 4.17 | --epoch 40 --avg 10                     |
+
+```bash
+./prepare.sh 
+
+export CUDA_VISIBLE_DEVICES="0,1"
+
+./zipformer/train_bbpe.py \
+  --world-size 2 \
+  --num-epochs 40 \
+  --start-epoch 1 \
+  --use-fp16 1 \
+  --context-size 2 \
+  --enable-musan 0 \
+  --exp-dir zipformer/exp_bbpe \
+  --max-duration 1000 \
+  --enable-musan 0 \
+  --base-lr 0.045 \
+  --lr-batches 7500 \
+  --lr-epochs 10 \
+  --spec-aug-time-warp-factor 20
+```
+
+Command for decoding is:
+```bash
+for m in greedy_search modified_beam_search fast_beam_search ; do
+  ./zipformer/decode_bbpe.py \
+    --epoch 40 \
+    --avg 10 \
+    --exp-dir ./zipformer_bbpe/exp \
+    --bpe-model data/lang_bbpe_500/bbpe.model \
+    --context-size 2 \
+    --decoding-method $m
+done
+```
+Pretrained models, training logs, decoding logs, tensorboard and decoding results
+are available at
+<https://huggingface.co/zrjin/icefall-asr-aishell-zipformer-bbpe-2024-01-16>
+
+
+#### Zipformer (Non-streaming)
+
+[./zipformer](./zipformer/)
+
+It's reworked Zipformer with Pruned RNNT loss.
+**Caution**: It uses `--context-size=1`.
+
+##### normal-scaled model, number of model parameters: 73412551, i.e., 73.41 M
+
+|                        | test | dev  | comment                                 |
+|------------------------|------|------|-----------------------------------------|
+| greedy search          | 4.67 | 4.37 | --epoch 55 --avg 17                     |
+| modified beam search   | 4.40 | 4.13 | --epoch 55 --avg 17                     |
+| fast beam search       | 4.60 | 4.31 | --epoch 55 --avg 17                     |
+
+Command for training is:
+```bash
+./prepare.sh 
+
+export CUDA_VISIBLE_DEVICES="0,1"
+
+./zipformer/train.py \
+  --world-size 2 \
+  --num-epochs 60 \
+  --start-epoch 1 \
+  --use-fp16 1 \
+  --context-size 1 \
+  --enable-musan 0 \
+  --exp-dir zipformer/exp \
+  --max-duration 1000 \
+  --enable-musan 0 \
+  --base-lr 0.045 \
+  --lr-batches 7500 \
+  --lr-epochs 18 \
+  --spec-aug-time-warp-factor 20
+```
+
+Command for decoding is:
+```bash
+for m in greedy_search modified_beam_search fast_beam_search ; do
+  ./zipformer/decode.py \
+    --epoch 55 \
+    --avg 17 \
+    --exp-dir ./zipformer/exp \
+    --lang-dir data/lang_char \
+    --context-size 1 \
+    --decoding-method $m
+done
+```
+Pretrained models, training logs, decoding logs, tensorboard and decoding results
+are available at
+<https://huggingface.co/zrjin/icefall-asr-aishell-zipformer-2023-10-24>
+
+
+##### small-scaled model, number of model parameters: 30167139, i.e., 30.17 M
+
+|                        | test | dev  | comment                                 |
+|------------------------|------|------|-----------------------------------------|
+| greedy search          | 4.97 | 4.67 | --epoch 55 --avg 21                     |
+| modified beam search   | 4.67 | 4.40 | --epoch 55 --avg 21                     |
+| fast beam search       | 4.85 | 4.61 | --epoch 55 --avg 21                     |
+
+Command for training is:
+```bash
+export CUDA_VISIBLE_DEVICES="0,1"
+
+./zipformer/train.py \
+  --world-size 2 \
+  --num-epochs 60 \
+  --start-epoch 1 \
+  --use-fp16 1 \
+  --context-size 1 \
+  --exp-dir zipformer/exp-small \
+  --enable-musan 0 \
+  --base-lr 0.045 \
+  --lr-batches 7500 \
+  --lr-epochs 18 \
+  --spec-aug-time-warp-factor 20 \
+  --num-encoder-layers 2,2,2,2,2,2 \
+  --feedforward-dim 512,768,768,768,768,768 \
+  --encoder-dim 192,256,256,256,256,256 \
+  --encoder-unmasked-dim 192,192,192,192,192,192 \
+  --max-duration 1200 
+```
+
+Command for decoding is:
+```bash
+for m in greedy_search modified_beam_search fast_beam_search ; do
+  ./zipformer/decode.py \
+    --epoch 55 \
+    --avg 21 \
+    --exp-dir ./zipformer/exp-small \
+    --lang-dir data/lang_char \
+    --context-size 1 \
+    --decoding-method $m \
+    --num-encoder-layers 2,2,2,2,2,2 \
+    --feedforward-dim 512,768,768,768,768,768 \
+    --encoder-dim 192,256,256,256,256,256 \
+    --encoder-unmasked-dim 192,192,192,192,192,192
+done
+```
+
+Pretrained models, training logs, decoding logs, tensorboard and decoding results
+are available at
+<https://huggingface.co/zrjin/icefall-asr-aishell-zipformer-small-2023-10-24/>
+
+##### large-scaled model, number of model parameters: 157285130, i.e., 157.29 M
+
+|                        | test | dev  | comment                                 |
+|------------------------|------|------|-----------------------------------------|
+| greedy search          | 4.49 | 4.22 | --epoch 56 --avg 23                     |
+| modified beam search   | 4.28 | 4.03 | --epoch 56 --avg 23                     |
+| fast beam search       | 4.44 | 4.18 | --epoch 56 --avg 23                     |
+
+Command for training is:
+```bash
+export CUDA_VISIBLE_DEVICES="0,1"
+
+./zipformer/train.py \
+  --world-size 2 \
+  --num-epochs 60 \
+  --use-fp16 1 \
+  --context-size 1 \
+  --exp-dir ./zipformer/exp-large \
+  --enable-musan 0 \
+  --lr-batches 7500 \
+  --lr-epochs 18 \
+  --spec-aug-time-warp-factor 20 \
+  --num-encoder-layers 2,2,4,5,4,2 \
+  --feedforward-dim 512,768,1536,2048,1536,768 \
+  --encoder-dim 192,256,512,768,512,256 \
+  --encoder-unmasked-dim 192,192,256,320,256,192 \
+  --max-duration 800 
+```
+
+Command for decoding is:
+```bash
+for m in greedy_search modified_beam_search fast_beam_search ; do
+  ./zipformer/decode.py \
+    --epoch 56 \
+    --avg 23 \
+    --exp-dir ./zipformer/exp-large \
+    --lang-dir data/lang_char \
+    --context-size 1 \
+    --decoding-method $m \
+    --num-encoder-layers 2,2,4,5,4,2 \
+    --feedforward-dim 512,768,1536,2048,1536,768 \
+    --encoder-dim 192,256,512,768,512,256 \
+    --encoder-unmasked-dim 192,192,256,320,256,192 
+done
+```
+
+Pretrained models, training logs, decoding logs, tensorboard and decoding results
+are available at
+<https://huggingface.co/zrjin/icefall-asr-aishell-zipformer-large-2023-10-24/>
 
 #### Pruned transducer stateless 7 streaming
 [./pruned_transducer_stateless7_streaming](./pruned_transducer_stateless7_streaming)
@@ -104,7 +312,7 @@ done
 Pretrained models, training logs, decoding logs, and decoding results
 are available at
 <https://huggingface.co/marcoyang/icefall-asr-aishell-zipformer-pruned-transducer-stateless7-2023-03-21>
-#### Pruned transducer stateless 7 (zipformer)
+#### Pruned transducer stateless 7 (Byte-level BPE)
 
 See <https://github.com/k2-fsa/icefall/pull/986>
 

egs/aishell/ASR/conformer_ctc/pretrained.py

@@ -261,6 +261,7 @@ def main():
     opts.frame_opts.snip_edges = False
     opts.frame_opts.samp_freq = params.sample_rate
     opts.mel_opts.num_bins = params.feature_dim
+    opts.mel_opts.high_freq = -400
 
     fbank = kaldifeat.Fbank(opts)
 

egs/aishell/ASR/prepare.sh

@@ -8,6 +8,7 @@ set -eou pipefail
 nj=15
 stage=-1
 stop_stage=11
+perturb_speed=true
 
 # We assume dl_dir (download dir) contains the following
 # directories and files. If not, they will be downloaded
@@ -114,7 +115,7 @@ if [ $stage -le 3 ] && [ $stop_stage -ge 3 ]; then
   log "Stage 3: Compute fbank for aishell"
   if [ ! -f data/fbank/.aishell.done ]; then
     mkdir -p data/fbank
-    ./local/compute_fbank_aishell.py --perturb-speed True
+    ./local/compute_fbank_aishell.py --perturb-speed ${perturb_speed}
     touch data/fbank/.aishell.done
   fi
 fi
@@ -242,7 +243,7 @@ if [ $stage -le 8 ] && [ $stop_stage -ge 8 ]; then
       -lm data/lm/3-gram.unpruned.arpa
   fi
 
-  # We assume you have install kaldilm, if not, please install
+  # We assume you have installed kaldilm, if not, please install
   # it using: pip install kaldilm
   if [ ! -f data/lm/G_3_gram_char.fst.txt ]; then
     # It is used in building HLG
@@ -260,10 +261,9 @@ if [ $stage -le 8 ] && [ $stop_stage -ge 8 ]; then
   fi
 
   if [ ! -f $lang_char_dir/HLG.fst ]; then
-    lang_phone_dir=data/lang_phone
     ./local/prepare_lang_fst.py  \
-      --lang-dir $lang_phone_dir \
+      --lang-dir $lang_char_dir \
-      --ngram-G ./data/lm/G_3_gram.fst.txt
+      --ngram-G ./data/lm/G_3_gram_char.fst.txt
   fi
 fi
 

egs/aishell/ASR/pruned_transducer_stateless2/pretrained.py

@@ -240,6 +240,7 @@ def main():
     opts.frame_opts.snip_edges = False
     opts.frame_opts.samp_freq = params.sample_rate
     opts.mel_opts.num_bins = params.feature_dim
+    opts.mel_opts.high_freq = -400
 
     fbank = kaldifeat.Fbank(opts)
 

egs/aishell/ASR/pruned_transducer_stateless3/pretrained.py

@@ -241,6 +241,7 @@ def main():
     opts.frame_opts.snip_edges = False
     opts.frame_opts.samp_freq = params.sample_rate
     opts.mel_opts.num_bins = params.feature_dim
+    opts.mel_opts.high_freq = -400
 
     fbank = kaldifeat.Fbank(opts)
 

egs/aishell/ASR/pruned_transducer_stateless7/decode.py

@@ -641,7 +641,7 @@ def main():
                 contexts_text.append(line.strip())
             contexts = graph_compiler.texts_to_ids(contexts_text)
             context_graph = ContextGraph(params.context_score)
-            context_graph.build(contexts)
+            context_graph.build([(c, 0.0) for c in contexts])
         else:
             context_graph = None
     else:

egs/aishell/ASR/pruned_transducer_stateless7/do_not_use_it_directly.py

@@ -1234,6 +1234,7 @@ def scan_pessimistic_batches_for_oom(
 
 
 def main():
+    raise RuntimeError("Please don't use this file directly!")
     parser = get_parser()
     AsrDataModule.add_arguments(parser)
     args = parser.parse_args()