From a7643301ecc7ca704b1fdc44b30b435fceda663f Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Tue, 15 Mar 2022 15:34:21 +0800 Subject: [PATCH] Cache pip packages for GitHub actions (#253) * Cache pip packages in GitHub actions. --- .../workflows/run-librispeech-2022-03-12.yml | 99 ++++++++++++------- .../run-pretrained-conformer-ctc.yml | 60 ++++++----- ...-transducer-stateless-librispeech-100h.yml | 70 ++++++++----- ...r-stateless-librispeech-multi-datasets.yml | 70 ++++++++----- ...ransducer-stateless-modified-2-aishell.yml | 70 ++++++++----- ...-transducer-stateless-modified-aishell.yml | 70 ++++++++----- .../run-pretrained-transducer-stateless.yml | 70 ++++++++----- .../workflows/run-pretrained-transducer.yml | 60 ++++++----- .github/workflows/run-yesno-recipe.yml | 20 ++-- requirements-ci.txt | 21 ++++ 10 files changed, 390 insertions(+), 220 deletions(-) create mode 100644 requirements-ci.txt diff --git a/.github/workflows/run-librispeech-2022-03-12.yml b/.github/workflows/run-librispeech-2022-03-12.yml index 74052312e..221104f8f 100644 --- a/.github/workflows/run-librispeech-2022-03-12.yml +++ b/.github/workflows/run-librispeech-2022-03-12.yml @@ -32,9 +32,6 @@ jobs: matrix: os: [ubuntu-18.04] python-version: [3.7, 3.8, 3.9] - torch: ["1.10.0"] - torchaudio: ["0.10.0"] - k2-version: ["1.9.dev20211101"] fail-fast: false @@ -43,49 +40,67 @@ jobs: with: fetch-depth: 0 - - name: Setup Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - - name: Install Python dependencies - run: | - python3 -m pip install --upgrade pip pytest - # numpy 1.20.x does not support python 3.6 - pip install numpy==1.19 - pip install torch==${{ matrix.torch }}+cpu torchaudio==${{ matrix.torchaudio }}+cpu -f https://download.pytorch.org/whl/cpu/torch_stable.html - pip install k2==${{ matrix.k2-version }}+cpu.torch${{ matrix.torch }} -f https://k2-fsa.org/nightly/ - - python3 -m pip install git+https://github.com/lhotse-speech/lhotse - python3 -m pip install kaldifeat - # We are in ./icefall and there is a file: requirements.txt in it - pip install -r requirements.txt - - name: Install graphviz shell: bash run: | - python3 -m pip install -qq graphviz sudo apt-get -qq install graphviz + - name: Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + cache-dependency-path: '**/requirements-ci.txt' + + - name: Install Python dependencies + run: | + grep -v '^#' ./requirements-ci.txt | xargs -n 1 -L 1 pip install + + - name: Cache kaldifeat + id: my-cache + uses: actions/cache@v2 + with: + path: | + ~/tmp/kaldifeat + key: cache-tmp-${{ matrix.python-version }} + + - name: Install kaldifeat + if: steps.my-cache.outputs.cache-hit != 'true' + shell: bash + run: | + mkdir -p ~/tmp + cd ~/tmp + git clone https://github.com/csukuangfj/kaldifeat + cd kaldifeat + mkdir build + cd build + cmake -DCMAKE_BUILD_TYPE=Release .. + make -j2 _kaldifeat + - name: Download pre-trained model shell: bash run: | - sudo apt-get -qq install git-lfs tree sox - cd egs/librispeech/ASR - mkdir tmp - cd tmp + sudo apt-get -qq install git-lfs + mkdir -p ~/tmp + cd ~/tmp git lfs install git clone https://huggingface.co/csukuangfj/icefall-asr-librispeech-pruned-transducer-stateless-2022-03-12 - cd .. - tree tmp - soxi tmp/icefall-asr-librispeech-pruned-transducer-stateless-2022-03-12/test_wavs/*.wav - ls -lh tmp/icefall-asr-librispeech-pruned-transducer-stateless-2022-03-12/test_wavs/*.wav + + - name: Display test files + shell: bash + run: | + sudo apt-get -qq install tree sox + tree ~/tmp/icefall-asr-librispeech-pruned-transducer-stateless-2022-03-12 + soxi ~/tmp/icefall-asr-librispeech-pruned-transducer-stateless-2022-03-12/test_wavs/*.wav + ls -lh ~/tmp/icefall-asr-librispeech-pruned-transducer-stateless-2022-03-12/test_wavs/*.wav - name: Run greedy search decoding (max-sym-per-frame 1) shell: bash run: | - export PYTHONPATH=$PWD:PYTHONPATH - dir=./tmp/icefall-asr-librispeech-pruned-transducer-stateless-2022-03-12 + export PYTHONPATH=$PWD:$PYTHONPATH + export PYTHONPATH=~/tmp/kaldifeat/kaldifeat/python:$PYTHONPATH + export PYTHONPATH=~/tmp/kaldifeat/build/lib:$PYTHONPATH + dir=~/tmp/icefall-asr-librispeech-pruned-transducer-stateless-2022-03-12 cd egs/librispeech/ASR ./pruned_transducer_stateless/pretrained.py \ --method greedy_search \ @@ -99,8 +114,10 @@ jobs: - name: Run greedy search decoding (max-sym-per-frame 2) shell: bash run: | - export PYTHONPATH=$PWD:PYTHONPATH - dir=./tmp/icefall-asr-librispeech-pruned-transducer-stateless-2022-03-12 + export PYTHONPATH=$PWD:$PYTHONPATH + export PYTHONPATH=~/tmp/kaldifeat/kaldifeat/python:$PYTHONPATH + export PYTHONPATH=~/tmp/kaldifeat/build/lib:$PYTHONPATH + dir=~/tmp/icefall-asr-librispeech-pruned-transducer-stateless-2022-03-12 cd egs/librispeech/ASR ./pruned_transducer_stateless/pretrained.py \ --method greedy_search \ @@ -114,8 +131,10 @@ jobs: - name: Run greedy search decoding (max-sym-per-frame 3) shell: bash run: | - export PYTHONPATH=$PWD:PYTHONPATH - dir=./tmp/icefall-asr-librispeech-pruned-transducer-stateless-2022-03-12 + export PYTHONPATH=$PWD:$PYTHONPATH + export PYTHONPATH=~/tmp/kaldifeat/kaldifeat/python:$PYTHONPATH + export PYTHONPATH=~/tmp/kaldifeat/build/lib:$PYTHONPATH + dir=~/tmp/icefall-asr-librispeech-pruned-transducer-stateless-2022-03-12 cd egs/librispeech/ASR ./pruned_transducer_stateless/pretrained.py \ --method greedy_search \ @@ -130,7 +149,9 @@ jobs: shell: bash run: | export PYTHONPATH=$PWD:$PYTHONPATH - dir=./tmp/icefall-asr-librispeech-pruned-transducer-stateless-2022-03-12 + export PYTHONPATH=~/tmp/kaldifeat/kaldifeat/python:$PYTHONPATH + export PYTHONPATH=~/tmp/kaldifeat/build/lib:$PYTHONPATH + dir=~/tmp/icefall-asr-librispeech-pruned-transducer-stateless-2022-03-12 cd egs/librispeech/ASR ./pruned_transducer_stateless/pretrained.py \ --method beam_search \ @@ -145,7 +166,9 @@ jobs: shell: bash run: | export PYTHONPATH=$PWD:$PYTHONPATH - dir=./tmp/icefall-asr-librispeech-pruned-transducer-stateless-2022-03-12 + export PYTHONPATH=~/tmp/kaldifeat/kaldifeat/python:$PYTHONPATH + export PYTHONPATH=~/tmp/kaldifeat/build/lib:$PYTHONPATH + dir=~/tmp/icefall-asr-librispeech-pruned-transducer-stateless-2022-03-12 cd egs/librispeech/ASR ./pruned_transducer_stateless/pretrained.py \ --method modified_beam_search \ diff --git a/.github/workflows/run-pretrained-conformer-ctc.yml b/.github/workflows/run-pretrained-conformer-ctc.yml index 1758a3521..cd24c9c44 100644 --- a/.github/workflows/run-pretrained-conformer-ctc.yml +++ b/.github/workflows/run-pretrained-conformer-ctc.yml @@ -31,9 +31,6 @@ jobs: matrix: os: [ubuntu-18.04] python-version: [3.7, 3.8, 3.9] - torch: ["1.10.0"] - torchaudio: ["0.10.0"] - k2-version: ["1.9.dev20211101"] fail-fast: false @@ -42,30 +39,43 @@ jobs: with: fetch-depth: 0 - - name: Setup Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - - name: Install Python dependencies - run: | - python3 -m pip install --upgrade pip pytest - # numpy 1.20.x does not support python 3.6 - pip install numpy==1.19 - pip install torch==${{ matrix.torch }}+cpu torchaudio==${{ matrix.torchaudio }}+cpu -f https://download.pytorch.org/whl/cpu/torch_stable.html - pip install k2==${{ matrix.k2-version }}+cpu.torch${{ matrix.torch }} -f https://k2-fsa.org/nightly/ - - python3 -m pip install git+https://github.com/lhotse-speech/lhotse - python3 -m pip install kaldifeat - # We are in ./icefall and there is a file: requirements.txt in it - pip install -r requirements.txt - - name: Install graphviz shell: bash run: | - python3 -m pip install -qq graphviz sudo apt-get -qq install graphviz + - name: Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + cache-dependency-path: '**/requirements-ci.txt' + + - name: Install Python dependencies + run: | + grep -v '^#' ./requirements-ci.txt | xargs -n 1 -L 1 pip install + + - name: Cache kaldifeat + id: my-cache + uses: actions/cache@v2 + with: + path: | + ~/tmp/kaldifeat + key: cache-tmp-${{ matrix.python-version }} + + - name: Install kaldifeat + if: steps.my-cache.outputs.cache-hit != 'true' + shell: bash + run: | + mkdir -p ~/tmp + cd ~/tmp + git clone https://github.com/csukuangfj/kaldifeat + cd kaldifeat + mkdir build + cd build + cmake -DCMAKE_BUILD_TYPE=Release .. + make -j2 _kaldifeat + - name: Download pre-trained model shell: bash run: | @@ -83,7 +93,9 @@ jobs: - name: Run CTC decoding shell: bash run: | - export PYTHONPATH=$PWD:PYTHONPATH + export PYTHONPATH=$PWD:$PYTHONPATH + export PYTHONPATH=~/tmp/kaldifeat/kaldifeat/python:$PYTHONPATH + export PYTHONPATH=~/tmp/kaldifeat/build/lib:$PYTHONPATH cd egs/librispeech/ASR ./conformer_ctc/pretrained.py \ --num-classes 500 \ @@ -98,6 +110,8 @@ jobs: shell: bash run: | export PYTHONPATH=$PWD:$PYTHONPATH + export PYTHONPATH=~/tmp/kaldifeat/kaldifeat/python:$PYTHONPATH + export PYTHONPATH=~/tmp/kaldifeat/build/lib:$PYTHONPATH cd egs/librispeech/ASR ./conformer_ctc/pretrained.py \ --num-classes 500 \ diff --git a/.github/workflows/run-pretrained-transducer-stateless-librispeech-100h.yml b/.github/workflows/run-pretrained-transducer-stateless-librispeech-100h.yml index efea5366b..b827ec82e 100644 --- a/.github/workflows/run-pretrained-transducer-stateless-librispeech-100h.yml +++ b/.github/workflows/run-pretrained-transducer-stateless-librispeech-100h.yml @@ -31,9 +31,6 @@ jobs: matrix: os: [ubuntu-18.04] python-version: [3.7, 3.8, 3.9] - torch: ["1.10.0"] - torchaudio: ["0.10.0"] - k2-version: ["1.9.dev20211101"] fail-fast: false @@ -42,30 +39,43 @@ jobs: with: fetch-depth: 0 - - name: Setup Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - - name: Install Python dependencies - run: | - python3 -m pip install --upgrade pip pytest - # numpy 1.20.x does not support python 3.6 - pip install numpy==1.19 - pip install torch==${{ matrix.torch }}+cpu torchaudio==${{ matrix.torchaudio }}+cpu -f https://download.pytorch.org/whl/cpu/torch_stable.html - pip install k2==${{ matrix.k2-version }}+cpu.torch${{ matrix.torch }} -f https://k2-fsa.org/nightly/ - - python3 -m pip install git+https://github.com/lhotse-speech/lhotse - python3 -m pip install kaldifeat - # We are in ./icefall and there is a file: requirements.txt in it - pip install -r requirements.txt - - name: Install graphviz shell: bash run: | - python3 -m pip install -qq graphviz sudo apt-get -qq install graphviz + - name: Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + cache-dependency-path: '**/requirements-ci.txt' + + - name: Install Python dependencies + run: | + grep -v '^#' ./requirements-ci.txt | xargs -n 1 -L 1 pip install + + - name: Cache kaldifeat + id: my-cache + uses: actions/cache@v2 + with: + path: | + ~/tmp/kaldifeat + key: cache-tmp-${{ matrix.python-version }} + + - name: Install kaldifeat + if: steps.my-cache.outputs.cache-hit != 'true' + shell: bash + run: | + mkdir -p ~/tmp + cd ~/tmp + git clone https://github.com/csukuangfj/kaldifeat + cd kaldifeat + mkdir build + cd build + cmake -DCMAKE_BUILD_TYPE=Release .. + make -j2 _kaldifeat + - name: Download pre-trained model shell: bash run: | @@ -84,7 +94,9 @@ jobs: - name: Run greedy search decoding (max-sym-per-frame 1) shell: bash run: | - export PYTHONPATH=$PWD:PYTHONPATH + export PYTHONPATH=$PWD:$PYTHONPATH + export PYTHONPATH=~/tmp/kaldifeat/kaldifeat/python:$PYTHONPATH + export PYTHONPATH=~/tmp/kaldifeat/build/lib:$PYTHONPATH cd egs/librispeech/ASR ./transducer_stateless_multi_datasets/pretrained.py \ --method greedy_search \ @@ -98,7 +110,9 @@ jobs: - name: Run greedy search decoding (max-sym-per-frame 2) shell: bash run: | - export PYTHONPATH=$PWD:PYTHONPATH + export PYTHONPATH=$PWD:$PYTHONPATH + export PYTHONPATH=~/tmp/kaldifeat/kaldifeat/python:$PYTHONPATH + export PYTHONPATH=~/tmp/kaldifeat/build/lib:$PYTHONPATH cd egs/librispeech/ASR ./transducer_stateless_multi_datasets/pretrained.py \ --method greedy_search \ @@ -112,7 +126,9 @@ jobs: - name: Run greedy search decoding (max-sym-per-frame 3) shell: bash run: | - export PYTHONPATH=$PWD:PYTHONPATH + export PYTHONPATH=$PWD:$PYTHONPATH + export PYTHONPATH=~/tmp/kaldifeat/kaldifeat/python:$PYTHONPATH + export PYTHONPATH=~/tmp/kaldifeat/build/lib:$PYTHONPATH cd egs/librispeech/ASR ./transducer_stateless_multi_datasets/pretrained.py \ --method greedy_search \ @@ -127,6 +143,8 @@ jobs: shell: bash run: | export PYTHONPATH=$PWD:$PYTHONPATH + export PYTHONPATH=~/tmp/kaldifeat/kaldifeat/python:$PYTHONPATH + export PYTHONPATH=~/tmp/kaldifeat/build/lib:$PYTHONPATH cd egs/librispeech/ASR ./transducer_stateless_multi_datasets/pretrained.py \ --method beam_search \ @@ -141,6 +159,8 @@ jobs: shell: bash run: | export PYTHONPATH=$PWD:$PYTHONPATH + export PYTHONPATH=~/tmp/kaldifeat/kaldifeat/python:$PYTHONPATH + export PYTHONPATH=~/tmp/kaldifeat/build/lib:$PYTHONPATH cd egs/librispeech/ASR ./transducer_stateless_multi_datasets/pretrained.py \ --method modified_beam_search \ diff --git a/.github/workflows/run-pretrained-transducer-stateless-librispeech-multi-datasets.yml b/.github/workflows/run-pretrained-transducer-stateless-librispeech-multi-datasets.yml index ccf9028cb..ffd9bdaec 100644 --- a/.github/workflows/run-pretrained-transducer-stateless-librispeech-multi-datasets.yml +++ b/.github/workflows/run-pretrained-transducer-stateless-librispeech-multi-datasets.yml @@ -31,9 +31,6 @@ jobs: matrix: os: [ubuntu-18.04] python-version: [3.7, 3.8, 3.9] - torch: ["1.10.0"] - torchaudio: ["0.10.0"] - k2-version: ["1.9.dev20211101"] fail-fast: false @@ -42,30 +39,43 @@ jobs: with: fetch-depth: 0 - - name: Setup Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - - name: Install Python dependencies - run: | - python3 -m pip install --upgrade pip pytest - # numpy 1.20.x does not support python 3.6 - pip install numpy==1.19 - pip install torch==${{ matrix.torch }}+cpu torchaudio==${{ matrix.torchaudio }}+cpu -f https://download.pytorch.org/whl/cpu/torch_stable.html - pip install k2==${{ matrix.k2-version }}+cpu.torch${{ matrix.torch }} -f https://k2-fsa.org/nightly/ - - python3 -m pip install git+https://github.com/lhotse-speech/lhotse - python3 -m pip install kaldifeat - # We are in ./icefall and there is a file: requirements.txt in it - pip install -r requirements.txt - - name: Install graphviz shell: bash run: | - python3 -m pip install -qq graphviz sudo apt-get -qq install graphviz + - name: Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + cache-dependency-path: '**/requirements-ci.txt' + + - name: Install Python dependencies + run: | + grep -v '^#' ./requirements-ci.txt | xargs -n 1 -L 1 pip install + + - name: Cache kaldifeat + id: my-cache + uses: actions/cache@v2 + with: + path: | + ~/tmp/kaldifeat + key: cache-tmp-${{ matrix.python-version }} + + - name: Install kaldifeat + if: steps.my-cache.outputs.cache-hit != 'true' + shell: bash + run: | + mkdir -p ~/tmp + cd ~/tmp + git clone https://github.com/csukuangfj/kaldifeat + cd kaldifeat + mkdir build + cd build + cmake -DCMAKE_BUILD_TYPE=Release .. + make -j2 _kaldifeat + - name: Download pre-trained model shell: bash run: | @@ -85,7 +95,9 @@ jobs: - name: Run greedy search decoding (max-sym-per-frame 1) shell: bash run: | - export PYTHONPATH=$PWD:PYTHONPATH + export PYTHONPATH=$PWD:$PYTHONPATH + export PYTHONPATH=~/tmp/kaldifeat/kaldifeat/python:$PYTHONPATH + export PYTHONPATH=~/tmp/kaldifeat/build/lib:$PYTHONPATH cd egs/librispeech/ASR ./transducer_stateless_multi_datasets/pretrained.py \ --method greedy_search \ @@ -99,7 +111,9 @@ jobs: - name: Run greedy search decoding (max-sym-per-frame 2) shell: bash run: | - export PYTHONPATH=$PWD:PYTHONPATH + export PYTHONPATH=$PWD:$PYTHONPATH + export PYTHONPATH=~/tmp/kaldifeat/kaldifeat/python:$PYTHONPATH + export PYTHONPATH=~/tmp/kaldifeat/build/lib:$PYTHONPATH cd egs/librispeech/ASR ./transducer_stateless_multi_datasets/pretrained.py \ --method greedy_search \ @@ -113,7 +127,9 @@ jobs: - name: Run greedy search decoding (max-sym-per-frame 3) shell: bash run: | - export PYTHONPATH=$PWD:PYTHONPATH + export PYTHONPATH=$PWD:$PYTHONPATH + export PYTHONPATH=~/tmp/kaldifeat/kaldifeat/python:$PYTHONPATH + export PYTHONPATH=~/tmp/kaldifeat/build/lib:$PYTHONPATH cd egs/librispeech/ASR ./transducer_stateless_multi_datasets/pretrained.py \ --method greedy_search \ @@ -128,6 +144,8 @@ jobs: shell: bash run: | export PYTHONPATH=$PWD:$PYTHONPATH + export PYTHONPATH=~/tmp/kaldifeat/kaldifeat/python:$PYTHONPATH + export PYTHONPATH=~/tmp/kaldifeat/build/lib:$PYTHONPATH cd egs/librispeech/ASR ./transducer_stateless_multi_datasets/pretrained.py \ --method beam_search \ @@ -143,6 +161,8 @@ jobs: shell: bash run: | export PYTHONPATH=$PWD:$PYTHONPATH + export PYTHONPATH=~/tmp/kaldifeat/kaldifeat/python:$PYTHONPATH + export PYTHONPATH=~/tmp/kaldifeat/build/lib:$PYTHONPATH cd egs/librispeech/ASR ./transducer_stateless_multi_datasets/pretrained.py \ --method modified_beam_search \ diff --git a/.github/workflows/run-pretrained-transducer-stateless-modified-2-aishell.yml b/.github/workflows/run-pretrained-transducer-stateless-modified-2-aishell.yml index c27ffc374..12652a22d 100644 --- a/.github/workflows/run-pretrained-transducer-stateless-modified-2-aishell.yml +++ b/.github/workflows/run-pretrained-transducer-stateless-modified-2-aishell.yml @@ -31,9 +31,6 @@ jobs: matrix: os: [ubuntu-18.04] python-version: [3.7, 3.8, 3.9] - torch: ["1.10.0"] - torchaudio: ["0.10.0"] - k2-version: ["1.9.dev20211101"] fail-fast: false @@ -42,30 +39,43 @@ jobs: with: fetch-depth: 0 - - name: Setup Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - - name: Install Python dependencies - run: | - python3 -m pip install --upgrade pip pytest - # numpy 1.20.x does not support python 3.6 - pip install numpy==1.19 - pip install torch==${{ matrix.torch }}+cpu torchaudio==${{ matrix.torchaudio }}+cpu -f https://download.pytorch.org/whl/cpu/torch_stable.html - pip install k2==${{ matrix.k2-version }}+cpu.torch${{ matrix.torch }} -f https://k2-fsa.org/nightly/ - - python3 -m pip install git+https://github.com/lhotse-speech/lhotse - python3 -m pip install kaldifeat - # We are in ./icefall and there is a file: requirements.txt in it - pip install -r requirements.txt - - name: Install graphviz shell: bash run: | - python3 -m pip install -qq graphviz sudo apt-get -qq install graphviz + - name: Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + cache-dependency-path: '**/requirements-ci.txt' + + - name: Install Python dependencies + run: | + grep -v '^#' ./requirements-ci.txt | xargs -n 1 -L 1 pip install + + - name: Cache kaldifeat + id: my-cache + uses: actions/cache@v2 + with: + path: | + ~/tmp/kaldifeat + key: cache-tmp-${{ matrix.python-version }} + + - name: Install kaldifeat + if: steps.my-cache.outputs.cache-hit != 'true' + shell: bash + run: | + mkdir -p ~/tmp + cd ~/tmp + git clone https://github.com/csukuangfj/kaldifeat + cd kaldifeat + mkdir build + cd build + cmake -DCMAKE_BUILD_TYPE=Release .. + make -j2 _kaldifeat + - name: Download pre-trained model shell: bash run: | @@ -84,7 +94,9 @@ jobs: - name: Run greedy search decoding (max-sym-per-frame 1) shell: bash run: | - export PYTHONPATH=$PWD:PYTHONPATH + export PYTHONPATH=$PWD:$PYTHONPATH + export PYTHONPATH=~/tmp/kaldifeat/kaldifeat/python:$PYTHONPATH + export PYTHONPATH=~/tmp/kaldifeat/build/lib:$PYTHONPATH cd egs/aishell/ASR ./transducer_stateless_modified-2/pretrained.py \ --method greedy_search \ @@ -98,7 +110,9 @@ jobs: - name: Run greedy search decoding (max-sym-per-frame 2) shell: bash run: | - export PYTHONPATH=$PWD:PYTHONPATH + export PYTHONPATH=$PWD:$PYTHONPATH + export PYTHONPATH=~/tmp/kaldifeat/kaldifeat/python:$PYTHONPATH + export PYTHONPATH=~/tmp/kaldifeat/build/lib:$PYTHONPATH cd egs/aishell/ASR ./transducer_stateless_modified-2/pretrained.py \ --method greedy_search \ @@ -112,7 +126,9 @@ jobs: - name: Run greedy search decoding (max-sym-per-frame 3) shell: bash run: | - export PYTHONPATH=$PWD:PYTHONPATH + export PYTHONPATH=$PWD:$PYTHONPATH + export PYTHONPATH=~/tmp/kaldifeat/kaldifeat/python:$PYTHONPATH + export PYTHONPATH=~/tmp/kaldifeat/build/lib:$PYTHONPATH cd egs/aishell/ASR ./transducer_stateless_modified-2/pretrained.py \ --method greedy_search \ @@ -127,6 +143,8 @@ jobs: shell: bash run: | export PYTHONPATH=$PWD:$PYTHONPATH + export PYTHONPATH=~/tmp/kaldifeat/kaldifeat/python:$PYTHONPATH + export PYTHONPATH=~/tmp/kaldifeat/build/lib:$PYTHONPATH cd egs/aishell/ASR ./transducer_stateless_modified-2/pretrained.py \ --method beam_search \ @@ -142,6 +160,8 @@ jobs: shell: bash run: | export PYTHONPATH=$PWD:$PYTHONPATH + export PYTHONPATH=~/tmp/kaldifeat/kaldifeat/python:$PYTHONPATH + export PYTHONPATH=~/tmp/kaldifeat/build/lib:$PYTHONPATH cd egs/aishell/ASR ./transducer_stateless_modified-2/pretrained.py \ --method modified_beam_search \ diff --git a/.github/workflows/run-pretrained-transducer-stateless-modified-aishell.yml b/.github/workflows/run-pretrained-transducer-stateless-modified-aishell.yml index 2e38abb5a..aa69d1500 100644 --- a/.github/workflows/run-pretrained-transducer-stateless-modified-aishell.yml +++ b/.github/workflows/run-pretrained-transducer-stateless-modified-aishell.yml @@ -31,9 +31,6 @@ jobs: matrix: os: [ubuntu-18.04] python-version: [3.7, 3.8, 3.9] - torch: ["1.10.0"] - torchaudio: ["0.10.0"] - k2-version: ["1.9.dev20211101"] fail-fast: false @@ -42,30 +39,43 @@ jobs: with: fetch-depth: 0 - - name: Setup Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - - name: Install Python dependencies - run: | - python3 -m pip install --upgrade pip pytest - # numpy 1.20.x does not support python 3.6 - pip install numpy==1.19 - pip install torch==${{ matrix.torch }}+cpu torchaudio==${{ matrix.torchaudio }}+cpu -f https://download.pytorch.org/whl/cpu/torch_stable.html - pip install k2==${{ matrix.k2-version }}+cpu.torch${{ matrix.torch }} -f https://k2-fsa.org/nightly/ - - python3 -m pip install git+https://github.com/lhotse-speech/lhotse - python3 -m pip install kaldifeat - # We are in ./icefall and there is a file: requirements.txt in it - pip install -r requirements.txt - - name: Install graphviz shell: bash run: | - python3 -m pip install -qq graphviz sudo apt-get -qq install graphviz + - name: Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + cache-dependency-path: '**/requirements-ci.txt' + + - name: Install Python dependencies + run: | + grep -v '^#' ./requirements-ci.txt | xargs -n 1 -L 1 pip install + + - name: Cache kaldifeat + id: my-cache + uses: actions/cache@v2 + with: + path: | + ~/tmp/kaldifeat + key: cache-tmp-${{ matrix.python-version }} + + - name: Install kaldifeat + if: steps.my-cache.outputs.cache-hit != 'true' + shell: bash + run: | + mkdir -p ~/tmp + cd ~/tmp + git clone https://github.com/csukuangfj/kaldifeat + cd kaldifeat + mkdir build + cd build + cmake -DCMAKE_BUILD_TYPE=Release .. + make -j2 _kaldifeat + - name: Download pre-trained model shell: bash run: | @@ -84,7 +94,9 @@ jobs: - name: Run greedy search decoding (max-sym-per-frame 1) shell: bash run: | - export PYTHONPATH=$PWD:PYTHONPATH + export PYTHONPATH=$PWD:$PYTHONPATH + export PYTHONPATH=~/tmp/kaldifeat/kaldifeat/python:$PYTHONPATH + export PYTHONPATH=~/tmp/kaldifeat/build/lib:$PYTHONPATH cd egs/aishell/ASR ./transducer_stateless_modified/pretrained.py \ --method greedy_search \ @@ -98,7 +110,9 @@ jobs: - name: Run greedy search decoding (max-sym-per-frame 2) shell: bash run: | - export PYTHONPATH=$PWD:PYTHONPATH + export PYTHONPATH=$PWD:$PYTHONPATH + export PYTHONPATH=~/tmp/kaldifeat/kaldifeat/python:$PYTHONPATH + export PYTHONPATH=~/tmp/kaldifeat/build/lib:$PYTHONPATH cd egs/aishell/ASR ./transducer_stateless_modified/pretrained.py \ --method greedy_search \ @@ -112,7 +126,9 @@ jobs: - name: Run greedy search decoding (max-sym-per-frame 3) shell: bash run: | - export PYTHONPATH=$PWD:PYTHONPATH + export PYTHONPATH=$PWD:$PYTHONPATH + export PYTHONPATH=~/tmp/kaldifeat/kaldifeat/python:$PYTHONPATH + export PYTHONPATH=~/tmp/kaldifeat/build/lib:$PYTHONPATH cd egs/aishell/ASR ./transducer_stateless_modified/pretrained.py \ --method greedy_search \ @@ -127,6 +143,8 @@ jobs: shell: bash run: | export PYTHONPATH=$PWD:$PYTHONPATH + export PYTHONPATH=~/tmp/kaldifeat/kaldifeat/python:$PYTHONPATH + export PYTHONPATH=~/tmp/kaldifeat/build/lib:$PYTHONPATH cd egs/aishell/ASR ./transducer_stateless_modified/pretrained.py \ --method beam_search \ @@ -142,6 +160,8 @@ jobs: shell: bash run: | export PYTHONPATH=$PWD:$PYTHONPATH + export PYTHONPATH=~/tmp/kaldifeat/kaldifeat/python:$PYTHONPATH + export PYTHONPATH=~/tmp/kaldifeat/build/lib:$PYTHONPATH cd egs/aishell/ASR ./transducer_stateless_modified/pretrained.py \ --method modified_beam_search \ diff --git a/.github/workflows/run-pretrained-transducer-stateless.yml b/.github/workflows/run-pretrained-transducer-stateless.yml index de66b90c5..535e46261 100644 --- a/.github/workflows/run-pretrained-transducer-stateless.yml +++ b/.github/workflows/run-pretrained-transducer-stateless.yml @@ -31,9 +31,6 @@ jobs: matrix: os: [ubuntu-18.04] python-version: [3.7, 3.8, 3.9] - torch: ["1.10.0"] - torchaudio: ["0.10.0"] - k2-version: ["1.9.dev20211101"] fail-fast: false @@ -42,30 +39,43 @@ jobs: with: fetch-depth: 0 - - name: Setup Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - - name: Install Python dependencies - run: | - python3 -m pip install --upgrade pip pytest - # numpy 1.20.x does not support python 3.6 - pip install numpy==1.19 - pip install torch==${{ matrix.torch }}+cpu torchaudio==${{ matrix.torchaudio }}+cpu -f https://download.pytorch.org/whl/cpu/torch_stable.html - pip install k2==${{ matrix.k2-version }}+cpu.torch${{ matrix.torch }} -f https://k2-fsa.org/nightly/ - - python3 -m pip install git+https://github.com/lhotse-speech/lhotse - python3 -m pip install kaldifeat - # We are in ./icefall and there is a file: requirements.txt in it - pip install -r requirements.txt - - name: Install graphviz shell: bash run: | - python3 -m pip install -qq graphviz sudo apt-get -qq install graphviz + - name: Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + cache-dependency-path: '**/requirements-ci.txt' + + - name: Install Python dependencies + run: | + grep -v '^#' ./requirements-ci.txt | xargs -n 1 -L 1 pip install + + - name: Cache kaldifeat + id: my-cache + uses: actions/cache@v2 + with: + path: | + ~/tmp/kaldifeat + key: cache-tmp-${{ matrix.python-version }} + + - name: Install kaldifeat + if: steps.my-cache.outputs.cache-hit != 'true' + shell: bash + run: | + mkdir -p ~/tmp + cd ~/tmp + git clone https://github.com/csukuangfj/kaldifeat + cd kaldifeat + mkdir build + cd build + cmake -DCMAKE_BUILD_TYPE=Release .. + make -j2 _kaldifeat + - name: Download pre-trained model shell: bash run: | @@ -83,7 +93,9 @@ jobs: - name: Run greedy search decoding (max-sym-per-frame 1) shell: bash run: | - export PYTHONPATH=$PWD:PYTHONPATH + export PYTHONPATH=$PWD:$PYTHONPATH + export PYTHONPATH=~/tmp/kaldifeat/kaldifeat/python:$PYTHONPATH + export PYTHONPATH=~/tmp/kaldifeat/build/lib:$PYTHONPATH cd egs/librispeech/ASR ./transducer_stateless/pretrained.py \ --method greedy_search \ @@ -97,7 +109,9 @@ jobs: - name: Run greedy search decoding (max-sym-per-frame 2) shell: bash run: | - export PYTHONPATH=$PWD:PYTHONPATH + export PYTHONPATH=$PWD:$PYTHONPATH + export PYTHONPATH=~/tmp/kaldifeat/kaldifeat/python:$PYTHONPATH + export PYTHONPATH=~/tmp/kaldifeat/build/lib:$PYTHONPATH cd egs/librispeech/ASR ./transducer_stateless/pretrained.py \ --method greedy_search \ @@ -111,7 +125,9 @@ jobs: - name: Run greedy search decoding (max-sym-per-frame 3) shell: bash run: | - export PYTHONPATH=$PWD:PYTHONPATH + export PYTHONPATH=$PWD:$PYTHONPATH + export PYTHONPATH=~/tmp/kaldifeat/kaldifeat/python:$PYTHONPATH + export PYTHONPATH=~/tmp/kaldifeat/build/lib:$PYTHONPATH cd egs/librispeech/ASR ./transducer_stateless/pretrained.py \ --method greedy_search \ @@ -126,6 +142,8 @@ jobs: shell: bash run: | export PYTHONPATH=$PWD:$PYTHONPATH + export PYTHONPATH=~/tmp/kaldifeat/kaldifeat/python:$PYTHONPATH + export PYTHONPATH=~/tmp/kaldifeat/build/lib:$PYTHONPATH cd egs/librispeech/ASR ./transducer_stateless/pretrained.py \ --method beam_search \ @@ -140,6 +158,8 @@ jobs: shell: bash run: | export PYTHONPATH=$PWD:$PYTHONPATH + export PYTHONPATH=~/tmp/kaldifeat/kaldifeat/python:$PYTHONPATH + export PYTHONPATH=~/tmp/kaldifeat/build/lib:$PYTHONPATH cd egs/librispeech/ASR ./transducer_stateless/pretrained.py \ --method modified_beam_search \ diff --git a/.github/workflows/run-pretrained-transducer.yml b/.github/workflows/run-pretrained-transducer.yml index f0ebddba3..41e4cfe0d 100644 --- a/.github/workflows/run-pretrained-transducer.yml +++ b/.github/workflows/run-pretrained-transducer.yml @@ -31,9 +31,6 @@ jobs: matrix: os: [ubuntu-18.04] python-version: [3.7, 3.8, 3.9] - torch: ["1.10.0"] - torchaudio: ["0.10.0"] - k2-version: ["1.9.dev20211101"] fail-fast: false @@ -42,30 +39,43 @@ jobs: with: fetch-depth: 0 - - name: Setup Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - - name: Install Python dependencies - run: | - python3 -m pip install --upgrade pip pytest - # numpy 1.20.x does not support python 3.6 - pip install numpy==1.19 - pip install torch==${{ matrix.torch }}+cpu torchaudio==${{ matrix.torchaudio }}+cpu -f https://download.pytorch.org/whl/cpu/torch_stable.html - pip install k2==${{ matrix.k2-version }}+cpu.torch${{ matrix.torch }} -f https://k2-fsa.org/nightly/ - - python3 -m pip install git+https://github.com/lhotse-speech/lhotse - python3 -m pip install kaldifeat - # We are in ./icefall and there is a file: requirements.txt in it - pip install -r requirements.txt - - name: Install graphviz shell: bash run: | - python3 -m pip install -qq graphviz sudo apt-get -qq install graphviz + - name: Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + cache-dependency-path: '**/requirements-ci.txt' + + - name: Install Python dependencies + run: | + grep -v '^#' ./requirements-ci.txt | xargs -n 1 -L 1 pip install + + - name: Cache kaldifeat + id: my-cache + uses: actions/cache@v2 + with: + path: | + ~/tmp/kaldifeat + key: cache-tmp-${{ matrix.python-version }} + + - name: Install kaldifeat + if: steps.my-cache.outputs.cache-hit != 'true' + shell: bash + run: | + mkdir -p ~/tmp + cd ~/tmp + git clone https://github.com/csukuangfj/kaldifeat + cd kaldifeat + mkdir build + cd build + cmake -DCMAKE_BUILD_TYPE=Release .. + make -j2 _kaldifeat + - name: Download pre-trained model shell: bash run: | @@ -84,7 +94,9 @@ jobs: - name: Run greedy search decoding shell: bash run: | - export PYTHONPATH=$PWD:PYTHONPATH + export PYTHONPATH=$PWD:$PYTHONPATH + export PYTHONPATH=~/tmp/kaldifeat/kaldifeat/python:$PYTHONPATH + export PYTHONPATH=~/tmp/kaldifeat/build/lib:$PYTHONPATH cd egs/librispeech/ASR ./transducer/pretrained.py \ --method greedy_search \ @@ -98,6 +110,8 @@ jobs: shell: bash run: | export PYTHONPATH=$PWD:$PYTHONPATH + export PYTHONPATH=~/tmp/kaldifeat/kaldifeat/python:$PYTHONPATH + export PYTHONPATH=~/tmp/kaldifeat/build/lib:$PYTHONPATH cd egs/librispeech/ASR ./transducer/pretrained.py \ --method beam_search \ diff --git a/.github/workflows/run-yesno-recipe.yml b/.github/workflows/run-yesno-recipe.yml index 98b2e4ebd..38c36a7c6 100644 --- a/.github/workflows/run-yesno-recipe.yml +++ b/.github/workflows/run-yesno-recipe.yml @@ -33,9 +33,6 @@ jobs: # TODO: enable macOS for CPU testing os: [ubuntu-18.04] python-version: [3.8] - torch: ["1.10.0"] - torchaudio: ["0.10.0"] - k2-version: ["1.9.dev20211101"] fail-fast: false steps: @@ -43,10 +40,17 @@ jobs: with: fetch-depth: 0 + - name: Install graphviz + shell: bash + run: | + sudo apt-get -qq install graphviz + - name: Setup Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 + uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} + cache: 'pip' + cache-dependency-path: '**/requirements-ci.txt' - name: Install libnsdfile and libsox if: startsWith(matrix.os, 'ubuntu') @@ -57,13 +61,7 @@ jobs: - name: Install Python dependencies run: | - python3 -m pip install -U pip - pip install torch==${{ matrix.torch }}+cpu torchaudio==${{ matrix.torchaudio }}+cpu -f https://download.pytorch.org/whl/cpu/torch_stable.html - pip install k2==${{ matrix.k2-version }}+cpu.torch${{ matrix.torch }} -f https://k2-fsa.org/nightly/ - python3 -m pip install git+https://github.com/lhotse-speech/lhotse - - # We are in ./icefall and there is a file: requirements.txt in it - python3 -m pip install -r requirements.txt + grep -v '^#' ./requirements-ci.txt | xargs -n 1 -L 1 pip install - name: Run yesno recipe shell: bash diff --git a/requirements-ci.txt b/requirements-ci.txt new file mode 100644 index 000000000..b5ee6b51c --- /dev/null +++ b/requirements-ci.txt @@ -0,0 +1,21 @@ +# Usage: grep -v '^#' requirements-ci.txt | xargs -n 1 -L 1 pip install +# dependencies for GitHub actions +# +# See https://github.com/actions/setup-python#caching-packages-dependencies + +# numpy 1.20.x does not support python 3.6 +numpy==1.19 +pytest==7.1.0 +graphviz==0.19.1 + +-f https://download.pytorch.org/whl/cpu/torch_stable.html torch==1.10.0+cpu +-f https://download.pytorch.org/whl/cpu/torch_stable.html torchaudio==0.10.0+cpu + +-f https://k2-fsa.org/nightly/ k2==1.9.dev20211101+cpu.torch1.10.0 + +git+https://github.com/lhotse-speech/lhotse +kaldilm==1.11 +kaldialign==0.2 +sentencepiece==0.1.96 +tensorboard==2.8.0 +typeguard==2.13.3