diff --git a/.github/workflows/build_conda_ubuntu_cpu.yml b/.github/workflows/build_conda_ubuntu_cpu.yml new file mode 100644 index 0000000..3f0692c --- /dev/null +++ b/.github/workflows/build_conda_ubuntu_cpu.yml @@ -0,0 +1,108 @@ +# Copyright 2021 Xiaomi Corp. (author: Fangjun Kuang) + +# See ../../LICENSE for clarification regarding multiple authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# refer to https://github.com/actions/starter-workflows/pull/47/files + +name: build_conda_ubuntu_cpu + +on: + push: + tags: + - '*' + +env: + KALDIFEAT_BUILD_TYPE: Release + +jobs: + generate_build_matrix: + # see https://github.com/pytorch/pytorch/pull/50633 + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Generating build matrix + id: set-matrix + run: | + # outputting for debugging purposes + python scripts/github_actions/generate_build_matrix.py + MATRIX=$(python scripts/github_actions/generate_build_matrix.py) + echo "::set-output name=matrix::${MATRIX}" + + build_conda_ubuntu_cpu: + needs: generate_build_matrix + runs-on: ubuntu-18.04 + strategy: + fail-fast: false + matrix: + ${{ fromJson(needs.generate_build_matrix.outputs.matrix) }} + + steps: + # refer to https://github.com/actions/checkout + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - uses: conda-incubator/setup-miniconda@v2 + with: + auto-update-conda: true + python-version: ${{ matrix.python-version }} + activate-environment: kaldifeat + + - name: Display Python version + shell: bash -l {0} + run: | + python3 -c "import sys; print(sys.version)" + which python3 + + - name: Install conda dependencies + shell: bash -l {0} + run: | + conda install -y -q anaconda-client + conda install -y -q conda-build + conda install -y -q -c pytorch pytorch=${{ matrix.torch }} cpuonly + + - name: Display conda info + shell: bash -l {0} + run: | + which conda + conda env list + conda info + nproc + + - name: Build kaldifeat + shell: bash -l {0} + env: + KALDIFEAT_PYTHON_VERSION: ${{ matrix.python-version}} + KALDIFEAT_TORCH_VERSION: ${{ matrix.torch }} + KALDIFEAT_CONDA_TOKEN: ${{ secrets.KALDIFEAT_CONDA_TOKEN}} + KALDIFEAT_IS_GITHUB_ACTIONS: 1 + KALDIFEAT_IS_FOR_CONDA: 1 + run: | + export KALDIFEAT_BUILD_TYPE=$KALDIFEAT_BUILD_TYPE + ./scripts/build_conda_cpu.sh + + - name: Display generated files + run: | + ls -lh /usr/share/miniconda/envs/kaldifeat/conda-bld/linux-64 + + - name: Upload generated files + uses: actions/upload-artifact@v2 + with: + name: cpu-torch-${{ matrix.torch }}-python-${{ matrix.python-version }}-${{ matrix.os }} + path: /usr/share/miniconda/envs/kaldifeat/conda-bld/linux-64/*.tar.bz2 diff --git a/.github/workflows/build_conda.yml b/.github/workflows/build_conda_ubuntu_cuda.yml similarity index 88% rename from .github/workflows/build_conda.yml rename to .github/workflows/build_conda_ubuntu_cuda.yml index 882ad72..b604a62 100644 --- a/.github/workflows/build_conda.yml +++ b/.github/workflows/build_conda_ubuntu_cuda.yml @@ -16,7 +16,7 @@ # refer to https://github.com/actions/starter-workflows/pull/47/files -name: build_conda_cuda +name: build_conda_ubuntu_cuda on: push: @@ -40,11 +40,11 @@ jobs: id: set-matrix run: | # outputting for debugging purposes - python scripts/github_actions/generate_build_matrix.py + python scripts/github_actions/generate_build_matrix.py --enable-cuda MATRIX=$(python scripts/github_actions/generate_build_matrix.py --enable-cuda) echo "::set-output name=matrix::${MATRIX}" - build_conda_cuda: + build_conda_ubuntu_cuda: needs: generate_build_matrix runs-on: ubuntu-18.04 strategy: @@ -91,7 +91,6 @@ jobs: run: | conda install -y -q anaconda-client conda install -y -q conda-build - conda install -y -q bs4 requests tqdm conda install -y -q -c pytorch -c conda-forge pytorch=${{ matrix.torch }} cudatoolkit=${{ matrix.cuda }} - name: Display conda info @@ -121,3 +120,13 @@ jobs: run: | export KALDIFEAT_BUILD_TYPE=$KALDIFEAT_BUILD_TYPE ./scripts/build_conda.sh + + - name: Display generated files + run: | + ls -lh /usr/share/miniconda/envs/kaldifeat/conda-bld/linux-64 + + - name: Upload generated files + uses: actions/upload-artifact@v2 + with: + name: cuda-${{ matrix.cuda }}-torch-${{ matrix.torch }}-python-${{ matrix.python-version }}-${{ matrix.os }} + path: /usr/share/miniconda/envs/kaldifeat/conda-bld/linux-64/*.tar.bz2 diff --git a/.github/workflows/nightly-tests-macos-cpu.yml b/.github/workflows/nightly-tests-macos-cpu.yml new file mode 100644 index 0000000..6eedadb --- /dev/null +++ b/.github/workflows/nightly-tests-macos-cpu.yml @@ -0,0 +1,99 @@ +# Copyright 2021 Xiaomi Corp. (author: Fangjun Kuang) + +# See ../../LICENSE for clarification regarding multiple authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +name: Nightly tests macos cpu + +on: + schedule: + # minute (0-59) + # hour (0-23) + # day of the month (1-31) + # month (1-12) + # day of the week (0-6) + # nightly build at 23:50 UTC time every day + - cron: "50 23 * * *" + +jobs: + generate_build_matrix: + # see https://github.com/pytorch/pytorch/pull/50633 + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Generating build matrix + id: set-matrix + run: | + # outputting for debugging purposes + python scripts/github_actions/generate_build_matrix.py + MATRIX=$(python scripts/github_actions/generate_build_matrix.py) + echo "::set-output name=matrix::${MATRIX}" + + run_nightly_tests_macos_cpu: + needs: generate_build_matrix + runs-on: macos-10.15 + strategy: + fail-fast: false + matrix: + ${{ fromJson(needs.generate_build_matrix.outputs.matrix) }} + + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Display Python version + run: python -c "import sys; print(sys.version)" + + - name: Display clang version + run: | + clang --version + + - name: Install PyTorch ${{ matrix.torch }} + shell: bash + run: | + python3 -m pip install -qq --upgrade pip + python3 -m pip install -qq wheel twine numpy typing_extensions + python3 -m pip install -qq torch==${{ matrix.torch }} numpy + python3 -m pip install -qq dataclasses soundfile + + python3 -c "import torch; print('torch version:', torch.__version__)" + + - name: Install from PyPI + shell: bash + run: | + export KALDIFEAT_MAKE_ARGS="-j 3" + pip install --verbose kaldifeat + + - name: Run tests + shell: bash + run: | + lib_path=$(python -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())") + echo "lib_path: $lib_path" + ls -lh $lib_path + # export DYLD_LIBRARY_PATH=$lib_path:$DYLD_LIBRARY_PATH + otool -L $lib_path/_kaldifeat.*.so + otool -l $lib_path/_kaldifeat.*.so + + cd kaldifeat/python/tests + make test diff --git a/.github/workflows/nightly-tests-ubuntu-conda-cpu.yml b/.github/workflows/nightly-tests-ubuntu-conda-cpu.yml new file mode 100644 index 0000000..d5e7689 --- /dev/null +++ b/.github/workflows/nightly-tests-ubuntu-conda-cpu.yml @@ -0,0 +1,83 @@ +# Copyright 2021 Xiaomi Corp. (author: Fangjun Kuang) + +# See ../../LICENSE for clarification regarding multiple authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +name: Nightly tests ubuntu conda cpu + +on: + schedule: + # minute (0-59) + # hour (0-23) + # day of the month (1-31) + # month (1-12) + # day of the week (0-6) + # nightly build at 23:50 UTC time every day + - cron: "50 23 * * *" + +jobs: + generate_build_matrix: + # see https://github.com/pytorch/pytorch/pull/50633 + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Generating build matrix + id: set-matrix + run: | + # outputting for debugging purposes + python scripts/github_actions/generate_build_matrix.py --test-only-latest-torch + MATRIX=$(python scripts/github_actions/generate_build_matrix.py --test-only-latest-torch) + echo "::set-output name=matrix::${MATRIX}" + + run_nightly_tests_ubuntu_conda_cpu: + needs: generate_build_matrix + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + ${{ fromJson(needs.generate_build_matrix.outputs.matrix) }} + + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - uses: conda-incubator/setup-miniconda@v2 + with: + auto-update-conda: true + python-version: ${{ matrix.python-version }} + activate-environment: kaldifeat + + - name: Display Python version + shell: bash -l {0} + run: | + python3 -c "import sys; print(sys.version)" + which python3 + + - name: Install kaldifeat and run tests + shell: bash -l {0} + run: | + sudo apt-get update + sudo apt-get install -y libsndfile1-dev libsndfile1 ffmpeg + + conda install -y -c kaldifeat -c pytorch cpuonly pytorch=${{ matrix.torch }} kaldifeat python=${{ matrix.python-version }} + conda install -y -c conda-forge pysoundfile + + + cd kaldifeat/python/tests + make test diff --git a/.github/workflows/nightly-tests-ubuntu-conda-cuda.yml b/.github/workflows/nightly-tests-ubuntu-conda-cuda.yml new file mode 100644 index 0000000..eb5fda0 --- /dev/null +++ b/.github/workflows/nightly-tests-ubuntu-conda-cuda.yml @@ -0,0 +1,79 @@ +# Copyright 2021 Xiaomi Corp. (author: Fangjun Kuang) + +# See ../../LICENSE for clarification regarding multiple authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +name: Nightly tests ubuntu conda cuda + +on: + schedule: + # minute (0-59) + # hour (0-23) + # day of the month (1-31) + # month (1-12) + # day of the week (0-6) + # nightly build at 23:50 UTC time every day + - cron: "50 23 * * *" + +jobs: + generate_build_matrix: + # see https://github.com/pytorch/pytorch/pull/50633 + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Generating build matrix + id: set-matrix + run: | + # outputting for debugging purposes + python scripts/github_actions/generate_build_matrix.py --enable-cuda --test-only-latest-torch + MATRIX=$(python scripts/github_actions/generate_build_matrix.py --enable-cuda --test-only-latest-torch) + echo "::set-output name=matrix::${MATRIX}" + + run_nightly_tests_ubuntu_conda_cuda: + needs: generate_build_matrix + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + ${{ fromJson(needs.generate_build_matrix.outputs.matrix) }} + + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - uses: conda-incubator/setup-miniconda@v2 + with: + auto-update-conda: true + python-version: ${{ matrix.python-version }} + activate-environment: kaldifeat + + - name: Display Python version + shell: bash -l {0} + run: | + python3 -c "import sys; print(sys.version)" + which python3 + + - name: Install kaldifeat and run tests + shell: bash -l {0} + run: | + conda install -y -c kaldifeat -c pytorch -c conda-forge pytorch=${{ matrix.torch }} cudatoolkit=${{ matrix.cuda }} kaldifeat python=${{ matrix.python-version }} + conda install -y -c conda-forge pysoundfile + + cd kaldifeat/python/tests + make test diff --git a/.github/workflows/nightly-tests-ubuntu-pip-cpu.yml b/.github/workflows/nightly-tests-ubuntu-pip-cpu.yml new file mode 100644 index 0000000..5433e2e --- /dev/null +++ b/.github/workflows/nightly-tests-ubuntu-pip-cpu.yml @@ -0,0 +1,101 @@ +# Copyright 2021 Xiaomi Corp. (author: Fangjun Kuang) + +# See ../../LICENSE for clarification regarding multiple authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +name: Nightly tests ubuntu pip cpu + +on: + schedule: + # minute (0-59) + # hour (0-23) + # day of the month (1-31) + # month (1-12) + # day of the week (0-6) + # nightly build at 23:50 UTC time every day + - cron: "50 23 * * *" + +jobs: + generate_build_matrix: + # see https://github.com/pytorch/pytorch/pull/50633 + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Generating build matrix + id: set-matrix + run: | + # outputting for debugging purposes + python scripts/github_actions/generate_build_matrix.py + MATRIX=$(python scripts/github_actions/generate_build_matrix.py) + echo "::set-output name=matrix::${MATRIX}" + + run_nightly_tests_ubuntu_pip_cpu: + needs: generate_build_matrix + runs-on: ubuntu-18.04 + strategy: + fail-fast: false + matrix: + ${{ fromJson(needs.generate_build_matrix.outputs.matrix) }} + + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Display Python version + run: python -c "import sys; print(sys.version)" + + - name: Install GCC 7 + run: | + sudo apt-get install -y gcc-7 g++-7 + echo "CC=/usr/bin/gcc-7" >> $GITHUB_ENV + echo "CXX=/usr/bin/g++-7" >> $GITHUB_ENV + + - name: Display GCC version + run: | + gcc --version + + - name: Install PyTorch ${{ matrix.torch }} + shell: bash + run: | + sudo apt-get update + sudo apt-get install -y libsndfile1-dev libsndfile1 ffmpeg + + python3 -m pip install -qq --upgrade pip + python3 -m pip install -qq wheel twine typing_extensions numpy soundfile + python3 -m pip install -qq torch==${{ matrix.torch }}+cpu -f https://download.pytorch.org/whl/torch_stable.html + python3 -m pip install -qq dataclasses + + python3 -c "import torch; print('torch version:', torch.__version__)" + + - name: Install from PyPI + shell: bash + run: | + export KALDIFEAT_MAKE_ARGS="-j 3" + pip install --verbose kaldifeat + + - name: Run tests + shell: bash + run: | + cd kaldifeat/python/tests + make test diff --git a/.github/workflows/publish_to_pypi.yml-bak b/.github/workflows/nightly-tests-ubuntu-pip-cuda.yml similarity index 58% rename from .github/workflows/publish_to_pypi.yml-bak rename to .github/workflows/nightly-tests-ubuntu-pip-cuda.yml index e0d7b1f..b39b9d6 100644 --- a/.github/workflows/publish_to_pypi.yml-bak +++ b/.github/workflows/nightly-tests-ubuntu-pip-cuda.yml @@ -14,23 +14,44 @@ # See the License for the specific language governing permissions and # limitations under the License. # -name: Publish to PyPI +name: Nightly tests ubuntu pip cuda on: - push: - tags: - - '*' + schedule: + # minute (0-59) + # hour (0-23) + # day of the month (1-31) + # month (1-12) + # day of the week (0-6) + # nightly build at 23:50 UTC time every day + - cron: "50 23 * * *" jobs: - pypi: - runs-on: ${{ matrix.os }} + generate_build_matrix: + # see https://github.com/pytorch/pytorch/pull/50633 + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Generating build matrix + id: set-matrix + run: | + # outputting for debugging purposes + python scripts/github_actions/generate_build_matrix.py --enable-cuda + MATRIX=$(python scripts/github_actions/generate_build_matrix.py --enable-cuda) + echo "::set-output name=matrix::${MATRIX}" + + run_nightly_tests_ubuntu_pip_cuda: + needs: generate_build_matrix + runs-on: ubuntu-18.04 strategy: fail-fast: false matrix: - os: [ubuntu-18.04, macos-10.15] - cuda: ["10.1"] - torch: ["1.7.1"] - python-version: [3.6, 3.7, 3.8] + ${{ fromJson(needs.generate_build_matrix.outputs.matrix) }} + steps: - uses: actions/checkout@v2 with: @@ -42,15 +63,12 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install GCC 7 - if: startsWith(matrix.os, 'ubuntu') run: | sudo apt-get install -y gcc-7 g++-7 echo "CC=/usr/bin/gcc-7" >> $GITHUB_ENV echo "CXX=/usr/bin/g++-7" >> $GITHUB_ENV - - name: Install CUDA Toolkit ${{ matrix.cuda }} - if: startsWith(matrix.os, 'ubuntu') shell: bash env: cuda: ${{ matrix.cuda }} @@ -61,61 +79,39 @@ jobs: echo "LD_LIBRARY_PATH=${CUDA_HOME}/lib:${CUDA_HOME}/lib64:${LD_LIBRARY_PATH}" >> $GITHUB_ENV - name: Display NVCC version - if: startsWith(matrix.os, 'ubuntu') run: | which nvcc nvcc --version - name: Install PyTorch ${{ matrix.torch }} - if: startsWith(matrix.os, 'ubuntu') env: cuda: ${{ matrix.cuda }} torch: ${{ matrix.torch }} shell: bash run: | + sudo apt-get update + sudo apt-get install -y libsndfile1-dev libsndfile1 ffmpeg python3 -m pip install --upgrade pip - python3 -m pip install wheel twine typing_extensions - python3 -m pip install bs4 requests tqdm + python3 -m pip install wheel twine typing_extensions soundfile + python3 -m pip install bs4 requests tqdm numpy ./scripts/github_actions/install_torch.sh python3 -c "import torch; print('torch version:', torch.__version__)" - - name: Install PyTorch ${{ matrix.torch }} - if: startsWith(matrix.os, 'macos') - shell: bash - run: | - python3 -m pip install -qq --upgrade pip - python3 -m pip install -qq wheel twine typing_extensions - python3 -m pip install -qq torch==${{ matrix.torch }} - - python3 -c "import torch; print('torch version:', torch.__version__)" - - name: Download cudnn 8.0 - if: startsWith(matrix.os, 'ubuntu') env: cuda: ${{ matrix.cuda }} run: | ./scripts/github_actions/install_cudnn.sh - - name: Build pip packages + - name: Install from PyPI shell: bash - env: - KALDIFEAT_IS_FOR_PYPI: 1 run: | - tag=$(python3 -c "import sys; print(''.join(sys.version[:3].split('.')))") - export KALDIFEAT_MAKE_ARGS="-j2" - python3 setup.py bdist_wheel --python-tag=py${tag} - ls -lh dist/ + export KALDIFEAT_MAKE_ARGS="-j 3" + pip install --verbose kaldifeat - - name: Publish wheels to PyPI - env: - TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + - name: Run tests + shell: bash run: | - twine upload dist/kaldifeat-*.whl - - - name: Upload Wheel - uses: actions/upload-artifact@v2 - with: - name: torch-${{ matrix.torch }}-python-${{ matrix.python-version }}-${{ matrix.os }} - path: dist/*.whl + cd kaldifeat/python/tests + make test diff --git a/.github/workflows/nightly-tests-windows-pip-cpu.yml b/.github/workflows/nightly-tests-windows-pip-cpu.yml new file mode 100644 index 0000000..815d62d --- /dev/null +++ b/.github/workflows/nightly-tests-windows-pip-cpu.yml @@ -0,0 +1,102 @@ +# Copyright 2021 Xiaomi Corp. (author: Fangjun Kuang) + +# See ../../LICENSE for clarification regarding multiple authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +name: Nightly tests windows pip cpu + +on: + schedule: + # minute (0-59) + # hour (0-23) + # day of the month (1-31) + # month (1-12) + # day of the week (0-6) + # nightly build at 23:50 UTC time every day + - cron: "50 23 * * *" + +jobs: + generate_build_matrix: + # see https://github.com/pytorch/pytorch/pull/50633 + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Generating build matrix + id: set-matrix + run: | + # outputting for debugging purposes + python scripts/github_actions/generate_build_matrix.py + MATRIX=$(python scripts/github_actions/generate_build_matrix.py) + echo "::set-output name=matrix::${MATRIX}" + + run_nightly_tests_windows_pip_cpu: + needs: generate_build_matrix + runs-on: windows-2019 + strategy: + fail-fast: false + matrix: + ${{ fromJson(needs.generate_build_matrix.outputs.matrix) }} + + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + + # see https://github.com/microsoft/setup-msbuild + - name: Add msbuild to PATH + uses: microsoft/setup-msbuild@v1.0.2 + + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Display Python version + run: python -c "import sys; print(sys.version)" + + - name: Install PyTorch ${{ matrix.torch }} + shell: bash + run: | + python3 -m pip install -qq --upgrade pip + python3 -m pip install -qq wheel twine numpy typing_extensions + python3 -m pip install -qq dataclasses soundfile numpy + python3 -m pip install -qq torch==${{ matrix.torch }}+cpu -f https://download.pytorch.org/whl/torch_stable.html + + python3 -c "import torch; print('torch version:', torch.__version__)" + + - name: Install from PyPI + shell: bash + run: | + export KALDIFEAT_MAKE_ARGS="-j 3" + pip install --verbose kaldifeat + + - name: Run tests + shell: bash + run: | + cd kaldifeat/python/tests + + python3 ./test_fbank.py + python3 ./test_fbank_options.py + python3 ./test_frame_extraction_options.py + python3 ./test_mel_bank_options.py + python3 ./test_mfcc.py + python3 ./test_mfcc_options.py + python3 ./test_plp.py + python3 ./test_plp_options.py + python3 ./test_spectrogram.py + python3 ./test_spectrogram_options.py diff --git a/.github/workflows/nightly-tests-windows-pip-cuda.yml b/.github/workflows/nightly-tests-windows-pip-cuda.yml new file mode 100644 index 0000000..10ca353 --- /dev/null +++ b/.github/workflows/nightly-tests-windows-pip-cuda.yml @@ -0,0 +1,143 @@ +# Copyright 2021 Xiaomi Corp. (author: Fangjun Kuang) + +# See ../../LICENSE for clarification regarding multiple authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +name: Nightly tests windows pip cuda + +on: + schedule: + # minute (0-59) + # hour (0-23) + # day of the month (1-31) + # month (1-12) + # day of the week (0-6) + # nightly build at 23:50 UTC time every day + - cron: "50 23 * * *" + +jobs: + generate_build_matrix: + # see https://github.com/pytorch/pytorch/pull/50633 + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Generating build matrix + id: set-matrix + run: | + # outputting for debugging purposes + python scripts/github_actions/generate_build_matrix.py --enable-cuda --for-windows + MATRIX=$(python scripts/github_actions/generate_build_matrix.py --enable-cuda --for-windows) + echo "::set-output name=matrix::${MATRIX}" + + run_nightly_tests_windows_pip_cuda: + needs: generate_build_matrix + runs-on: windows-2019 + strategy: + fail-fast: false + matrix: + ${{ fromJson(needs.generate_build_matrix.outputs.matrix) }} + + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + + # see https://github.com/microsoft/setup-msbuild + - name: Add msbuild to PATH + uses: microsoft/setup-msbuild@v1.0.2 + + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Display Python version + run: python -c "import sys; print(sys.version)" + + # See https://github.com/Jimver/cuda-toolkit/blob/master/src/links/windows-links.ts + # for available CUDA versions + - uses: Jimver/cuda-toolkit@v0.2.7 + id: cuda-toolkit + with: + cuda: ${{ matrix.cuda }} + + - name: Display CUDA version + run: | + echo "Installed cuda version is: ${{ steps.cuda-toolkit.outputs.cuda }}" + echo "Cuda install location: ${{steps.cuda-toolkit.outputs.CUDA_PATH}}" + nvcc --version + + - name: Remove CUDA installation package + shell: bash + run: | + rm "C:/hostedtoolcache/windows/cuda_installer-windows/${{ matrix.cuda }}/x64/cuda_installer_${{ matrix.cuda }}.exe" + + - name: Download cuDNN + shell: bash + run: | + GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/csukuangfj/cudnn-for-windows + cd cudnn-for-windows + git lfs pull --include="cudnn-windows-x86_64-8.4.1.50_cuda11.6-archive.zip" + unzip cudnn-windows-x86_64-8.4.1.50_cuda11.6-archive.zip + rm cudnn-windows-x86_64-8.4.1.50_cuda11.6-archive.zip + ls -lh * + ls -lh */* + + echo "PWD: $PWD" + + - name: Install PyTorch ${{ matrix.torch }} + shell: bash + run: | + version=${{ matrix.cuda }} + major=${version:0:2} + minor=${version:3:1} + v=${major}${minor} + if [ ${v} -eq 102 ]; then v=""; else v="+cu${v}"; fi + + python3 -m pip install -qq --upgrade pip + python3 -m pip install -qq wheel twine numpy typing_extensions + python3 -m pip install -qq dataclasses soundfile numpy + python3 -m pip install -qq torch==${{ matrix.torch }}${v} -f https://download.pytorch.org/whl/torch_stable.html numpy + + python3 -c "import torch; print('torch version:', torch.__version__)" + + python3 -m torch.utils.collect_env + + - name: Install from PyPI + shell: bash + run: | + echo "PWD: $PWD" + export KALDIFEAT_CMAKE_ARGS="-DCMAKE_BUILD_TYPE=$BUILD_TYPE -DCUDNN_INCLUDE_PATH=d:/a/kaldifeat/kaldifeat/cudnn-for-windows/cudnn-windows-x86_64-8.4.1.50_cuda11.6-archive/include -DCUDNN_LIBRARY_PATH=d:/a/kaldifeat/kaldifeat/cudnn-for-windows/cudnn-windows-x86_64-8.4.1.50_cuda11.6-archive/lib/cudnn.lib" + export KALDIFEAT_MAKE_ARGS="-j 3" + pip install --verbose kaldifeat + + - name: Run tests + shell: bash + run: | + cd kaldifeat/python/tests + + python3 ./test_fbank.py + python3 ./test_fbank_options.py + python3 ./test_frame_extraction_options.py + python3 ./test_mel_bank_options.py + python3 ./test_mfcc.py + python3 ./test_mfcc_options.py + python3 ./test_plp.py + python3 ./test_plp_options.py + python3 ./test_spectrogram.py + python3 ./test_spectrogram_options.py diff --git a/.github/workflows/run-tests-macos-cpu.yml b/.github/workflows/run-tests-macos-cpu.yml new file mode 100644 index 0000000..99a81e5 --- /dev/null +++ b/.github/workflows/run-tests-macos-cpu.yml @@ -0,0 +1,85 @@ +# Copyright 2021 Xiaomi Corp. (author: Fangjun Kuang) + +# See ../../LICENSE for clarification regarding multiple authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +name: Run tests macos cpu + +on: + push: + branches: + - master + + pull_request: + branches: + - master + +jobs: + generate_build_matrix: + # see https://github.com/pytorch/pytorch/pull/50633 + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Generating build matrix + id: set-matrix + run: | + # outputting for debugging purposes + python scripts/github_actions/generate_build_matrix.py + MATRIX=$(python scripts/github_actions/generate_build_matrix.py) + echo "::set-output name=matrix::${MATRIX}" + + run_tests_macos_cpu: + needs: generate_build_matrix + runs-on: macos-10.15 + strategy: + fail-fast: false + matrix: + ${{ fromJson(needs.generate_build_matrix.outputs.matrix) }} + + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Install PyTorch ${{ matrix.torch }} + shell: bash + run: | + python3 -m pip install -qq --upgrade pip + python3 -m pip install -qq wheel twine typing_extensions soundfile numpy + python3 -m pip install -qq torch==${{ matrix.torch }} -f https://download.pytorch.org/whl/torch_stable.html + + python3 -c "import torch; print('torch version:', torch.__version__)" + + - name: Build + shell: bash + run: | + mkdir build_release + cd build_release + cmake .. + make VERBOSE=1 -j3 + + - name: Run tests + shell: bash + run: | + cd build_release + ctest --output-on-failure diff --git a/.github/workflows/run-tests-ubuntu-cpu.yml b/.github/workflows/run-tests-ubuntu-cpu.yml new file mode 100644 index 0000000..7c2e45f --- /dev/null +++ b/.github/workflows/run-tests-ubuntu-cpu.yml @@ -0,0 +1,94 @@ +# Copyright 2021 Xiaomi Corp. (author: Fangjun Kuang) + +# See ../../LICENSE for clarification regarding multiple authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +name: Run tests ubuntu cpu + +on: + push: + branches: + - master + + pull_request: + branches: + - master + +jobs: + generate_build_matrix: + # see https://github.com/pytorch/pytorch/pull/50633 + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Generating build matrix + id: set-matrix + run: | + # outputting for debugging purposes + python scripts/github_actions/generate_build_matrix.py + MATRIX=$(python scripts/github_actions/generate_build_matrix.py) + echo "::set-output name=matrix::${MATRIX}" + + run_tests_ubuntu_cpu: + needs: generate_build_matrix + runs-on: ubuntu-18.04 + strategy: + fail-fast: false + matrix: + ${{ fromJson(needs.generate_build_matrix.outputs.matrix) }} + + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Install GCC 7 + run: | + sudo apt-get install -y gcc-7 g++-7 + echo "CC=/usr/bin/gcc-7" >> $GITHUB_ENV + echo "CXX=/usr/bin/g++-7" >> $GITHUB_ENV + + - name: Install PyTorch ${{ matrix.torch }} + shell: bash + run: | + sudo apt-get update + sudo apt-get install -y libsndfile1-dev libsndfile1 ffmpeg + python3 -m pip install --upgrade pip + python3 -m pip install wheel twine typing_extensions soundfile + python3 -m pip install bs4 requests tqdm numpy + python3 -m pip install -qq torch==${{ matrix.torch }}+cpu -f https://download.pytorch.org/whl/torch_stable.html + + python3 -c "import torch; print('torch version:', torch.__version__)" + + - name: Build + shell: bash + run: | + mkdir build_release + cd build_release + cmake .. + make VERBOSE=1 -j3 + + - name: Run tests + shell: bash + run: | + cd build_release + ctest --output-on-failure diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests-ubuntu-cuda.yml similarity index 71% rename from .github/workflows/run-tests.yml rename to .github/workflows/run-tests-ubuntu-cuda.yml index 656a792..51251b5 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests-ubuntu-cuda.yml @@ -14,26 +14,43 @@ # See the License for the specific language governing permissions and # limitations under the License. # -name: Run tests +name: Run tests ubuntu cuda on: push: branches: - master + pull_request: branches: - master jobs: - run_tests: - runs-on: ${{ matrix.os }} + generate_build_matrix: + # see https://github.com/pytorch/pytorch/pull/50633 + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Generating build matrix + id: set-matrix + run: | + # outputting for debugging purposes + python scripts/github_actions/generate_build_matrix.py --enable-cuda --test-only-latest-torch + MATRIX=$(python scripts/github_actions/generate_build_matrix.py --enable-cuda --test-only-latest-torch) + echo "::set-output name=matrix::${MATRIX}" + + run_tests_ubuntu_cuda: + needs: generate_build_matrix + runs-on: ubuntu-18.04 strategy: fail-fast: false matrix: - os: [ubuntu-18.04, macos-10.15] - cuda: ["10.1"] - torch: ["1.7.1"] - python-version: [3.6, 3.7, 3.8] + ${{ fromJson(needs.generate_build_matrix.outputs.matrix) }} + steps: - uses: actions/checkout@v2 with: @@ -53,7 +70,6 @@ jobs: - name: Install CUDA Toolkit ${{ matrix.cuda }} - if: startsWith(matrix.os, 'ubuntu') shell: bash env: cuda: ${{ matrix.cuda }} @@ -64,39 +80,26 @@ jobs: echo "LD_LIBRARY_PATH=${CUDA_HOME}/lib:${CUDA_HOME}/lib64:${LD_LIBRARY_PATH}" >> $GITHUB_ENV - name: Display NVCC version - if: startsWith(matrix.os, 'ubuntu') run: | which nvcc nvcc --version - name: Install PyTorch ${{ matrix.torch }} - if: startsWith(matrix.os, 'ubuntu') env: cuda: ${{ matrix.cuda }} torch: ${{ matrix.torch }} shell: bash run: | - sudo apt update - sudo apt install libsndfile1-dev libsndfile1 ffmpeg + sudo apt-get update + sudo apt-get install -y libsndfile1-dev libsndfile1 ffmpeg python3 -m pip install --upgrade pip python3 -m pip install wheel twine typing_extensions soundfile - python3 -m pip install bs4 requests tqdm + python3 -m pip install bs4 requests tqdm numpy ./scripts/github_actions/install_torch.sh python3 -c "import torch; print('torch version:', torch.__version__)" - - name: Install PyTorch ${{ matrix.torch }} - if: startsWith(matrix.os, 'macos') - shell: bash - run: | - python3 -m pip install -qq --upgrade pip - python3 -m pip install -qq wheel twine typing_extensions soundfile - python3 -m pip install -qq torch==${{ matrix.torch }} - - python3 -c "import torch; print('torch version:', torch.__version__)" - - name: Download cudnn 8.0 - if: startsWith(matrix.os, 'ubuntu') env: cuda: ${{ matrix.cuda }} run: | @@ -108,10 +111,10 @@ jobs: mkdir build_release cd build_release cmake .. - make VERBOSE=1 + make VERBOSE=1 -j3 - name: Run tests shell: bash run: | cd build_release - ctest -R py --output-on-failure + ctest --output-on-failure diff --git a/.github/workflows/build_windows.yml b/.github/workflows/run-tests-windows-cpu.yml similarity index 60% rename from .github/workflows/build_windows.yml rename to .github/workflows/run-tests-windows-cpu.yml index 6a7402c..cf4f547 100644 --- a/.github/workflows/build_windows.yml +++ b/.github/workflows/run-tests-windows-cpu.yml @@ -1,25 +1,56 @@ -name: build-windows-cpu +# Copyright 2021 Xiaomi Corp. (author: Fangjun Kuang) + +# See ../../LICENSE for clarification regarding multiple authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +name: Run tests windows cpu on: push: branches: - master + pull_request: branches: - master jobs: - build-windows: + generate_build_matrix: + # see https://github.com/pytorch/pytorch/pull/50633 + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Generating build matrix + id: set-matrix + run: | + # outputting for debugging purposes + python scripts/github_actions/generate_build_matrix.py + MATRIX=$(python scripts/github_actions/generate_build_matrix.py) + echo "::set-output name=matrix::${MATRIX}" + + run_tests_windows_cpu: # see https://github.com/actions/virtual-environments/blob/win19/20210525.0/images/win/Windows2019-Readme.md - runs-on: ${{ matrix.os }} + needs: generate_build_matrix + runs-on: windows-2019 strategy: fail-fast: false matrix: - os: [windows-2019] - # torch: ["1.6.0", "1.7.0", "1.7.1", "1.8.0", "1.8.1", "1.9.0", "1.10.0", "1.11.0"] - # python-version: [3.6, 3.7, 3.8, 3.9] - torch: ["1.10.0", "1.11.0"] - python-version: [3.8] + ${{ fromJson(needs.generate_build_matrix.outputs.matrix) }} steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/run-tests-windows-cuda.yml b/.github/workflows/run-tests-windows-cuda.yml new file mode 100644 index 0000000..6d04dd5 --- /dev/null +++ b/.github/workflows/run-tests-windows-cuda.yml @@ -0,0 +1,173 @@ +# Copyright 2021 Xiaomi Corp. (author: Fangjun Kuang) + +# See ../../LICENSE for clarification regarding multiple authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +name: Run tests windows cuda + +on: + push: + branches: + - master + + pull_request: + branches: + - master + +jobs: + generate_build_matrix: + # see https://github.com/pytorch/pytorch/pull/50633 + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Generating build matrix + id: set-matrix + run: | + # outputting for debugging purposes + python scripts/github_actions/generate_build_matrix.py --enable-cuda --for-windows --test-only-latest-torch + MATRIX=$(python scripts/github_actions/generate_build_matrix.py --enable-cuda --for-windows --test-only-latest-torch) + echo "::set-output name=matrix::${MATRIX}" + + run_tests_windows_cuda: + needs: generate_build_matrix + runs-on: windows-2019 + strategy: + fail-fast: false + matrix: + ${{ fromJson(needs.generate_build_matrix.outputs.matrix) }} + + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + + # see https://github.com/microsoft/setup-msbuild + - name: Add msbuild to PATH + uses: microsoft/setup-msbuild@v1.0.2 + + - name: Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Display Python version + run: python -c "import sys; print(sys.version)" + + # See https://github.com/Jimver/cuda-toolkit/blob/master/src/links/windows-links.ts + # for available CUDA versions + - uses: Jimver/cuda-toolkit@v0.2.7 + id: cuda-toolkit + with: + cuda: ${{ matrix.cuda }} + + - name: Display CUDA version + shell: bash + run: | + echo "Installed cuda version is: ${{ steps.cuda-toolkit.outputs.cuda }}" + echo "Cuda install location: ${{steps.cuda-toolkit.outputs.CUDA_PATH}}" + nvcc --version + + - name: Remove CUDA installation package + shell: bash + run: | + rm "C:/hostedtoolcache/windows/cuda_installer-windows/${{ matrix.cuda }}/x64/cuda_installer_${{ matrix.cuda }}.exe" + + - name: Download cuDNN + shell: bash + run: | + GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/csukuangfj/cudnn-for-windows + cd cudnn-for-windows + git lfs pull --include="cudnn-windows-x86_64-8.4.1.50_cuda11.6-archive.zip" + unzip cudnn-windows-x86_64-8.4.1.50_cuda11.6-archive.zip + rm cudnn-windows-x86_64-8.4.1.50_cuda11.6-archive.zip + ls -lh * + ls -lh */* + + echo "PWD: $PWD" + + - name: Install PyTorch ${{ matrix.torch }} + shell: bash + run: | + version=${{ matrix.cuda }} + major=${version:0:2} + minor=${version:3:1} + v=${major}${minor} + if [ ${v} -eq 102 ]; then v=""; else v="+cu${v}"; fi + + python3 -m pip install -qq --upgrade pip + python3 -m pip install -qq wheel twine numpy typing_extensions + python3 -m pip install -qq dataclasses soundfile numpy + python3 -m pip install -qq torch==${{ matrix.torch }}${v} -f https://download.pytorch.org/whl/torch_stable.html numpy + + python3 -c "import torch; print('torch version:', torch.__version__)" + + python3 -m torch.utils.collect_env + + - name: Display CMake version + run: | + cmake --version + cmake --help + + - name: Configure CMake + shell: bash + run: | + echo "PWD: $PWD" + ls -lh + + mkdir build_release + cd build_release + cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DCUDNN_INCLUDE_PATH=d:/a/kaldifeat/kaldifeat/cudnn-for-windows/cudnn-windows-x86_64-8.4.1.50_cuda11.6-archive/include -DCUDNN_LIBRARY_PATH=d:/a/kaldifeat/kaldifeat/cudnn-for-windows/cudnn-windows-x86_64-8.4.1.50_cuda11.6-archive/lib/cudnn.lib .. + ls -lh + + - name: Build kaldifeat + shell: bash + run: | + cd build_release + cmake --build . --target _kaldifeat --config Release + + - name: Display generated files + shell: bash + run: | + cd build_release + ls -lh lib/*/* + + - name: Build wheel + shell: bash + run: | + echo $PWD + ls -lh ./* + export KALDIFEAT_CMAKE_ARGS="-DCMAKE_BUILD_TYPE=$BUILD_TYPE -DCUDNN_INCLUDE_PATH=d:/a/kaldifeat/kaldifeat/cudnn-for-windows/cudnn-windows-x86_64-8.4.1.50_cuda11.6-archive/include -DCUDNN_LIBRARY_PATH=d:/a/kaldifeat/kaldifeat/cudnn-for-windows/cudnn-windows-x86_64-8.4.1.50_cuda11.6-archive/lib/cudnn.lib" + python3 setup.py bdist_wheel + ls -lh dist/ + pip install ./dist/*.whl + python3 -c "import kaldifeat; print(kaldifeat.__version__)" + + - name: Upload Wheel + uses: actions/upload-artifact@v2 + with: + name: python-${{ matrix.python-version }}-${{ matrix.os }}-cuda-${{ matrix.cuda }} + path: dist/*.whl + + - name: Build tests + shell: bash + run: | + cd build_release + cmake --build . --target ALL_BUILD --config Release + ls -lh bin/*/* + ctest -C Release --verbose --output-on-failure diff --git a/README.md b/README.md index 170c0e7..9513b14 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # kaldifeat
- + @@ -268,15 +268,22 @@ See . See . +## sherpa + +[sherpa](https://github.com/k2-fsa/sherpa) uses kaldifeat for streaming feature +extraction. + +See + # Installation -## From conda (Only for Linux + CUDA) +## From conda (Only for Linux) Supported versions of Python, PyTorch, and CUDA toolkit are listed below: -[![Supported Python versions](/doc/source/images/python-3.6_3.7_3.8-blue.svg)](/doc/source/images/python-3.6_3.7_3.8-blue.svg) -[![Supported PyTorch versions](/doc/source/images/pytorch-1.6.0_1.7.0_1.7.1_1.8.0_1.8.1_1.9.0-green.svg)](/doc/source/images/pytorch-1.6.0_1.7.0_1.7.1_1.8.0_1.8.1_1.9.0-green.svg) -[![Supported CUDA versions](/doc/source/images/cuda-10.1_10.2_11.0_11.1-orange.svg)](/doc/source/images/cuda-10.1_10.2_11.0_11.1-orange.svg) +[![Supported Python versions](/doc/source/images/python_ge_3.6-blue.svg)](/doc/source/images/python_ge_3.6-blue.svg) +[![Supported PyTorch versions](/doc/source/images/pytorch_ge_1.5.0-green.svg)](/doc/source/images/pytorch_ge_1.5.0-green.svg) +[![Supported CUDA versions](/doc/source/images/cuda_ge_10.1-orange.svg)](/doc/source/images/cuda_ge_10.1-orange.svg) ```bash conda install -c kaldifeat -c pytorch -c conda-forge kaldifeat python=3.8 cudatoolkit=11.1 pytorch=1.8.1 @@ -284,11 +291,17 @@ conda install -c kaldifeat -c pytorch -c conda-forge kaldifeat python=3.8 cudato You can select the supported Python version, CUDA toolkit version and PyTorch version as you wish. -**Note:** If you want a CPU only version or want to install `kaldifeat` on other operating systems, -e.g., macOS, please use `pip install` or compile `kaldifeat` from source. +To install a CPU version, use + +```bash +conda install -c kaldifeat -c pytorch cpuonly kaldifeat python=3.8 pytorch=1.8.1 +``` -## From PyPi with pip +**Note:** If you want to install `kaldifeat` on macOS or Windows, please +use either `pip install -v kaldifeat` or compile `kaldifeat` from source. + +## From PyPi with pip (Support Linux/macOS/Windows) You need to install PyTorch and CMake first. CMake 3.11 is known to work. Other CMake versions may also work. @@ -298,7 +311,7 @@ PyTorch 1.6.0 and above are known to work. Other PyTorch versions may also work. pip install -v kaldifeat ``` -## From source +## From source (Support Linux/macOS/Windows) The following are the commands to compile `kaldifeat` from source. We assume that you have installed `CMake` and PyTorch. @@ -321,14 +334,41 @@ python3 -c "import kaldifeat; print(kaldifeat.__version__)" ### How to install a CUDA version -There are two approaches: +You need to first install a CUDA version of PyTorch and then install `kaldifeat`. - - (1) Install using `conda`. It always installs a CUDA version of kaldifeat. - - (2) Install a CUDA version of PyTorch and then install kaldifeat from source - or use `pip install kaldifeat`. +**Note**: You can use a CUDA version of kaldifeat on machines with no GPUs. ### How to install a CPU-only version -You have to first install a CPU-only version of PyTorch and then install kaldifeat -from source or use `pip install kaldifeat`. +You need to first install a CPU version of PyTorch and then install `kaldifeat`. + +### How to fix `Caffe2: Cannot find cuDNN library` + +``` + Your installed Caffe2 version uses cuDNN but I cannot find the cuDNN + libraries. Please set the proper cuDNN prefixes and / or install cuDNN. +``` + +This error happens when you want to install a CUDA version of kaldifeat +by `pip install kaldifeat` or from source. + +You need to first install cuDNN. Assume you have installed cuDNN to the +path `/path/to/cudnn`. You can fix the error by the following commands. + +(1) Fix for installation using `pip install` + +```bash +export KALDIFEAT_CMAKE_ARGS="-DCMAKE_BUILD_TYPE=Release -DCUDNN_LIBRARY_PATH=/path/to/cudnn/lib/libcudnn.so -DCUDNN_INCLUDE_PATH=/path/to/cudnn/include" +pip install --verbose kaldifeat +``` + +(2) Fix for installation from source + +```bash +mkdir /some/path +git clone https://github.com/csukuangfj/kaldifeat.git +cd kaldifeat +export KALDIFEAT_CMAKE_ARGS="-DCMAKE_BUILD_TYPE=Release -DCUDNN_LIBRARY_PATH=/path/to/cudnn/lib/libcudnn.so -DCUDNN_INCLUDE_PATH=/path/to/cudnn/include" +python setup.py install +``` diff --git a/doc/source/images/README.md b/doc/source/images/README.md new file mode 100644 index 0000000..542998b --- /dev/null +++ b/doc/source/images/README.md @@ -0,0 +1,8 @@ +## File description + + is used to create the following files: + +- ./os.svg +- ./python_ge_3.6-blue.svg +- ./cuda_ge_10.1-orange.svg +- ./pytorch_ge_1.5.0-green.svg diff --git a/doc/source/images/os-green.svg b/doc/source/images/os-green.svg new file mode 100644 index 0000000..b78017a --- /dev/null +++ b/doc/source/images/os-green.svg @@ -0,0 +1 @@ +os: Linux | macOS | WindowsosLinux | macOS | Windows diff --git a/doc/source/images/os.svg b/doc/source/images/os.svg deleted file mode 100644 index 314bf44..0000000 --- a/doc/source/images/os.svg +++ /dev/null @@ -1 +0,0 @@ -OS: Linux | macOSOSLinux | macOS diff --git a/kaldifeat/python/tests/Makefile b/kaldifeat/python/tests/Makefile new file mode 100644 index 0000000..20f98ff --- /dev/null +++ b/kaldifeat/python/tests/Makefile @@ -0,0 +1,13 @@ + +.PHONY: test +test: + python3 ./test_fbank.py + python3 ./test_fbank_options.py + python3 ./test_frame_extraction_options.py + python3 ./test_mel_bank_options.py + python3 ./test_mfcc.py + python3 ./test_mfcc_options.py + python3 ./test_plp.py + python3 ./test_plp_options.py + python3 ./test_spectrogram.py + python3 ./test_spectrogram_options.py diff --git a/scripts/build_conda_cpu.sh b/scripts/build_conda_cpu.sh new file mode 100755 index 0000000..3d2c47a --- /dev/null +++ b/scripts/build_conda_cpu.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# +# Copyright 2021 Xiaomi Corp. (author: Fangjun Kuang) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The following environment variables are supposed to be set by users +# +# - KALDIFEAT_TORCH_VERSION +# The PyTorch version. Example: +# +# export KALDIFEAT_TORCH_VERSION=1.7.1 +# +# Defaults to 1.7.1 if not set. +# +# - KALDIFEAT_CONDA_TOKEN +# If not set, auto upload to anaconda.org is disabled. +# +# Its value is from https://anaconda.org/kaldifeat/settings/access +# (You need to login as user kaldifeat to see its value) +# +# - KALDIFEAT_BUILD_TYPE +# If not set, defaults to Release. + +set -e +export CONDA_BUILD=1 + +cur_dir=$(cd $(dirname $BASH_SOURCE) && pwd) +kaldifeat_dir=$(cd $cur_dir/.. && pwd) + +cd $kaldifeat_dir + +export KALDIFEAT_ROOT_DIR=$kaldifeat_dir +echo "KALDIFEAT_ROOT_DIR: $KALDIFEAT_ROOT_DIR" + +KALDIFEAT_PYTHON_VERSION=$(python3 -c "import sys; print(sys.version[:3])") + +if [ -z $KALDIFEAT_TORCH_VERSION ]; then + echo "env var KALDIFEAT_TORCH_VERSION is not set, defaults to 1.7.1" + KALDIFEAT_TORCH_VERSION=1.7.1 +fi + +if [ -z $KALDIFEAT_BUILD_TYPE ]; then + echo "env var KALDIFEAT_BUILD_TYPE is not set, defaults to Release" + KALDIFEAT_BUILD_TYPE=Release +fi + +export KALDIFEAT_IS_FOR_CONDA=1 + +# Example value: 3.8 +export KALDIFEAT_PYTHON_VERSION + +# Example value: 1.7.1 +export KALDIFEAT_TORCH_VERSION + +export KALDIFEAT_BUILD_TYPE + +if [ ! -z $KALDIFEAT_IS_GITHUB_ACTIONS ]; then + export KALDIFEAT_IS_GITHUB_ACTIONS + conda remove -q pytorch + conda clean -q -a +else + export KALDIFEAT_IS_GITHUB_ACTIONS=0 +fi + +if [ -z $KALDIFEAT_CONDA_TOKEN ]; then + echo "Auto upload to anaconda.org is disabled since KALDIFEAT_CONDA_TOKEN is not set" + conda build --no-test --no-anaconda-upload -c pytorch -c conda-forge ./scripts/conda-cpu/kaldifeat +else + conda build --no-test -c pytorch -c conda-forge --token $KALDIFEAT_CONDA_TOKEN ./scripts/conda-cpu/kaldifeat +fi diff --git a/scripts/conda-cpu/cpuonly/meta.yaml b/scripts/conda-cpu/cpuonly/meta.yaml new file mode 100644 index 0000000..33ec762 --- /dev/null +++ b/scripts/conda-cpu/cpuonly/meta.yaml @@ -0,0 +1,10 @@ +# this file is copied from +# https://github.com/pytorch/builder/tree/master/conda/cpuonly +package: + name: cpuonly + version: 1.0 + +build: + track_features: + - cpuonly + noarch: generic diff --git a/scripts/conda-cpu/kaldifeat/build.sh b/scripts/conda-cpu/kaldifeat/build.sh new file mode 100644 index 0000000..6e24b9d --- /dev/null +++ b/scripts/conda-cpu/kaldifeat/build.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# +# Copyright 2021 Xiaomi Corp. (author: Fangjun Kuang) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +CONDA_ENV_DIR=$CONDA_PREFIX + +echo "KALDIFEAT_PYTHON_VERSION: $KALDIFEAT_PYTHON_VERSION" +echo "KALDIFEAT_TORCH_VERSION: $KALDIFEAT_TORCH_VERSION" +echo "KALDIFEAT_BUILD_TYPE: $KALDIFEAT_BUILD_TYPE" +echo "KALDIFEAT_BUILD_VERSION: $KALDIFEAT_BUILD_VERSION" + +export KALDIFEAT_CMAKE_ARGS="-DCMAKE_BUILD_TYPE=${KALDIFEAT_BUILD_TYPE}" +export KALDIFEAT_MAKE_ARGS="-j1 VERBOSE=1" + +export LIBRARY_PATH="/usr/local/miniconda/envs/kaldifeat/lib":$LIBRARY_PATH +export LD_LIBRARY_PATH="/usr/local/miniconda/envs/kaldifeat/lib":$LD_LIBRARY_PATH +export DYLD_LIBRARY_PATH="/usr/local/miniconda/envs/kaldifeat/lib":$DYLD_LIBRARY_PATH + +python3 setup.py install --single-version-externally-managed --record=record.txt diff --git a/scripts/conda-cpu/kaldifeat/meta.yaml b/scripts/conda-cpu/kaldifeat/meta.yaml new file mode 100644 index 0000000..9805868 --- /dev/null +++ b/scripts/conda-cpu/kaldifeat/meta.yaml @@ -0,0 +1,44 @@ +package: + name: kaldifeat + version: "1.15.4" + +source: + path: "{{ environ.get('KALDIFEAT_ROOT_DIR') }}" + +build: + number: 0 + string: cpu_py{{ environ.get('KALDIFEAT_PYTHON_VERSION') }}_torch{{ environ.get('KALDIFEAT_TORCH_VERSION') }} + script_env: + - KALDIFEAT_IS_GITHUB_ACTIONS + - KALDIFEAT_TORCH_VERSION + - KALDIFEAT_PYTHON_VERSION + - KALDIFEAT_BUILD_TYPE + - KALDIFEAT_BUILD_VERSION + - KALDIFEAT_IS_FOR_CONDA + +requirements: + build: + - {{ compiler('c') }} # [win] + - {{ compiler('cxx') }} # [win] + + host: + - cmake=3.18 + - python + - pytorch={{ environ.get('KALDIFEAT_TORCH_VERSION') }} + - gcc_linux-64=7 # [linux] + - cpuonly + - numpy + + run: + - python + - pytorch={{ environ.get('KALDIFEAT_TORCH_VERSION') }} + - numpy + +about: + home: https://github.com/csukuangfj/kaldifeat + license: Apache V2 + license_file: LICENSE + summary: Kaldi-compatible feature extraction with PyTorch + description: | + Kaldi-compatible feature extraction with PyTorch, + supporting CUDA, batch processing, chunk processing, and autograd diff --git a/scripts/conda/kaldifeat/build.sh b/scripts/conda/kaldifeat/build.sh index 3897511..4539872 100644 --- a/scripts/conda/kaldifeat/build.sh +++ b/scripts/conda/kaldifeat/build.sh @@ -32,6 +32,6 @@ echo "gcc version: $($CC --version)" echo "nvcc version: $(nvcc --version)" export KALDIFEAT_CMAKE_ARGS="-DCMAKE_BUILD_TYPE=${KALDIFEAT_BUILD_TYPE}" -export KALDIFEAT_MAKE_ARGS="-j2" +export KALDIFEAT_MAKE_ARGS="-j3" python3 setup.py install --single-version-externally-managed --record=record.txt diff --git a/scripts/github_actions/generate_build_matrix.py b/scripts/github_actions/generate_build_matrix.py index 67d10a0..6b85131 100755 --- a/scripts/github_actions/generate_build_matrix.py +++ b/scripts/github_actions/generate_build_matrix.py @@ -14,6 +14,13 @@ def get_args(): help="True to enable CUDA", ) + parser.add_argument( + "--for-windows", + action="store_true", + default=False, + help="True for windows", + ) + parser.add_argument( "--test-only-latest-torch", action="store_true", @@ -24,7 +31,7 @@ def get_args(): return parser.parse_args() -def generate_build_matrix(enable_cuda, test_only_latest_torch): +def generate_build_matrix(enable_cuda, for_windows, test_only_latest_torch): matrix = { # 1.5.x is removed because there are compilation errors. # See @@ -41,47 +48,69 @@ def generate_build_matrix(enable_cuda, test_only_latest_torch): # }, "1.6.0": { "python-version": ["3.6", "3.7", "3.8"], - "cuda": ["10.1", "10.2"], + "cuda": ["10.1", "10.2"] + if not for_windows + else ["10.1.243", "10.2.89"], }, "1.7.0": { "python-version": ["3.6", "3.7", "3.8"], - "cuda": ["10.1", "10.2", "11.0"], + "cuda": ["10.1", "10.2", "11.0"] + if not for_windows + else ["10.1.243", "10.2.89", "11.0.3"], }, "1.7.1": { "python-version": ["3.6", "3.7", "3.8", "3.9"], - "cuda": ["10.1", "10.2", "11.0"], + "cuda": ["10.1", "10.2", "11.0"] + if not for_windows + else ["10.1.243", "10.2.89", "11.0.3"], }, "1.8.0": { "python-version": ["3.6", "3.7", "3.8", "3.9"], - "cuda": ["10.1", "10.2", "11.1"], + "cuda": ["10.1", "10.2", "11.1"] + if not for_windows + else ["10.1.243", "10.2.89", "11.1.1"], }, "1.8.1": { "python-version": ["3.6", "3.7", "3.8", "3.9"], - "cuda": ["10.1", "10.2", "11.1"], + "cuda": ["10.1", "10.2", "11.1"] + if not for_windows + else ["10.1.243", "10.2.89", "11.1.1"], }, "1.9.0": { "python-version": ["3.6", "3.7", "3.8", "3.9"], - "cuda": ["10.2", "11.1"], + "cuda": ["10.2", "11.1"] + if not for_windows + else ["10.2.89", "11.1.1"], }, "1.9.1": { "python-version": ["3.6", "3.7", "3.8", "3.9"], - "cuda": ["10.2", "11.1"], + "cuda": ["10.2", "11.1"] + if not for_windows + else ["10.2.89", "11.1.1"], }, "1.10.0": { "python-version": ["3.6", "3.7", "3.8", "3.9"], - "cuda": ["10.2", "11.1", "11.3"], + "cuda": ["10.2", "11.1", "11.3"] + if not for_windows + else ["10.2.89", "11.1.1", "11.3.1"], }, "1.10.1": { "python-version": ["3.6", "3.7", "3.8", "3.9"], - "cuda": ["10.2", "11.1", "11.3"], + "cuda": ["10.2", "11.1", "11.3"] + if not for_windows + else ["10.2.89", "11.1.1", "11.3.1"], }, "1.10.2": { "python-version": ["3.6", "3.7", "3.8", "3.9"], - "cuda": ["10.2", "11.1", "11.3"], + "cuda": ["10.2", "11.1", "11.3"] + if not for_windows + else ["10.2.89", "11.1.1", "11.3.1"], }, "1.11.0": { "python-version": ["3.7", "3.8", "3.9", "3.10"], - "cuda": ["10.2", "11.3", "11.5"], + "cuda": ["10.2", "11.3", "11.5"] + if not for_windows + else ["11.3.1", "11.5.2"], }, } if test_only_latest_torch: @@ -107,6 +136,7 @@ def main(): args = get_args() generate_build_matrix( enable_cuda=args.enable_cuda, + for_windows=args.for_windows, test_only_latest_torch=args.test_only_latest_torch, )