From 4f47e0dd0c2ece1eab75eadefef2afebc60d0745 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Mon, 19 Jul 2021 11:08:40 +0800 Subject: [PATCH] Minor fixes. --- .github/workflows/publish_to_pypi.yml | 74 +++++++++++----- .github/workflows/run-tests.yml | 59 ++++++++++--- scripts/github_actions/install_cuda.sh | 59 +++++++++++++ scripts/github_actions/install_cudnn.sh | 58 +++++++++++++ scripts/github_actions/install_torch.sh | 108 ++++++++++++++++++++++++ 5 files changed, 327 insertions(+), 31 deletions(-) create mode 100755 scripts/github_actions/install_cuda.sh create mode 100755 scripts/github_actions/install_cudnn.sh create mode 100755 scripts/github_actions/install_torch.sh diff --git a/.github/workflows/publish_to_pypi.yml b/.github/workflows/publish_to_pypi.yml index 8f3826c..23bb692 100644 --- a/.github/workflows/publish_to_pypi.yml +++ b/.github/workflows/publish_to_pypi.yml @@ -17,15 +17,9 @@ name: Publish to PyPI on: - # push: - # tags: - # - '*' push: - branches: - - master - pull_request: - branches: - - master + tags: + - '*' jobs: pypi: @@ -34,7 +28,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-18.04, macos-10.15] - gcc: ["5"] + cuda: ["10.1"] torch: ["1.8.1"] python-version: [3.6, 3.7, 3.8] steps: @@ -47,22 +41,62 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Install GCC ${{ matrix.gcc }} + - name: Install GCC 7 + if: startsWith(matrix.os, 'ubuntu') run: | - sudo apt-get install -y gcc-${{ matrix.gcc }} g++-${{ matrix.gcc }} - echo "CC=/usr/bin/gcc-${{ matrix.gcc }}" >> $GITHUB_ENV - echo "CXX=/usr/bin/g++-${{ matrix.gcc }}" >> $GITHUB_ENV + sudo apt-get install -y gcc-7 g++-7 + echo "CC=/usr/bin/gcc-7" >> $GITHUB_ENV + echo "CXX=/usr/bin/g++-7" >> $GITHUB_ENV + + + - name: Install CUDA Toolkit ${{ matrix.cuda }} + if: startsWith(matrix.os, 'ubuntu') + shell: bash + env: + cuda: ${{ matrix.cuda }} + run: | + source ./scripts/github_actions/install_cuda.sh + echo "CUDA_HOME=${CUDA_HOME}" >> $GITHUB_ENV + echo "${CUDA_HOME}/bin" >> $GITHUB_PATH + echo "LD_LIBRARY_PATH=${CUDA_HOME}/lib:${CUDA_HOME}/lib64:${LD_LIBRARY_PATH}" >> $GITHUB_ENV + + - name: Display NVCC version + if: startsWith(matrix.os, 'ubuntu') + run: | + which nvcc + nvcc --version - name: Install PyTorch ${{ matrix.torch }} + if: startsWith(matrix.os, 'ubuntu') + env: + cuda: ${{ matrix.cuda }} + torch: ${{ matrix.torch }} shell: bash run: | python3 -m pip install --upgrade pip python3 -m pip install wheel twine typing_extensions python3 -m pip install bs4 requests tqdm - python3 -m pip install torch==${{ matrix.torch }} + + ./scripts/github_actions/install_torch.sh + python3 -c "import torch; print('torch version:', torch.__version__)" + + - name: Install PyTorch ${{ matrix.torch }} + if: startsWith(matrix.os, 'macos') + shell: bash + run: | + python3 -m pip install -qq --upgrade pip + python3 -m pip install -qq wheel twine typing_extensions + python3 -m pip install -qq torch==${{ matrix.torch }} python3 -c "import torch; print('torch version:', torch.__version__)" + - name: Download cudnn 8.0 + if: startsWith(matrix.os, 'ubuntu') + env: + cuda: ${{ matrix.cuda }} + run: | + ./scripts/github_actions/install_cudnn.sh + - name: Build pip packages shell: bash env: @@ -73,12 +107,12 @@ jobs: python3 setup.py bdist_wheel --python-tag=py${tag} ls -lh dist/ - - name: Publish wheels to PyPI - env: - TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - run: | - twine upload dist/kaldifeat-*.whl + # - name: Publish wheels to PyPI + # env: + # TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} + # TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + # run: | + # twine upload dist/kaldifeat-*.whl - name: Upload Wheel uses: actions/upload-artifact@v2 diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 1eb2ba5..d943c96 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -18,11 +18,8 @@ name: Run tests on: push: - branches: - - master - pull_request: - branches: - - master + tags: + - '*' jobs: run_tests: @@ -31,7 +28,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-18.04, macos-10.15] - gcc: ["5"] + cuda: ["10.1"] torch: ["1.8.1"] python-version: [3.6, 3.7, 3.8] steps: @@ -44,22 +41,62 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Install GCC ${{ matrix.gcc }} + - name: Install GCC 7 + if: startsWith(matrix.os, 'ubuntu') run: | - sudo apt-get install -y gcc-${{ matrix.gcc }} g++-${{ matrix.gcc }} - echo "CC=/usr/bin/gcc-${{ matrix.gcc }}" >> $GITHUB_ENV - echo "CXX=/usr/bin/g++-${{ matrix.gcc }}" >> $GITHUB_ENV + sudo apt-get install -y gcc-7 g++-7 + echo "CC=/usr/bin/gcc-7" >> $GITHUB_ENV + echo "CXX=/usr/bin/g++-7" >> $GITHUB_ENV + + + - name: Install CUDA Toolkit ${{ matrix.cuda }} + if: startsWith(matrix.os, 'ubuntu') + shell: bash + env: + cuda: ${{ matrix.cuda }} + run: | + source ./scripts/github_actions/install_cuda.sh + echo "CUDA_HOME=${CUDA_HOME}" >> $GITHUB_ENV + echo "${CUDA_HOME}/bin" >> $GITHUB_PATH + echo "LD_LIBRARY_PATH=${CUDA_HOME}/lib:${CUDA_HOME}/lib64:${LD_LIBRARY_PATH}" >> $GITHUB_ENV + + - name: Display NVCC version + if: startsWith(matrix.os, 'ubuntu') + run: | + which nvcc + nvcc --version - name: Install PyTorch ${{ matrix.torch }} + if: startsWith(matrix.os, 'ubuntu') + env: + cuda: ${{ matrix.cuda }} + torch: ${{ matrix.torch }} shell: bash run: | python3 -m pip install --upgrade pip python3 -m pip install wheel twine typing_extensions python3 -m pip install bs4 requests tqdm - python3 -m pip install torch==${{ matrix.torch }} + + ./scripts/github_actions/install_torch.sh + python3 -c "import torch; print('torch version:', torch.__version__)" + + - name: Install PyTorch ${{ matrix.torch }} + if: startsWith(matrix.os, 'macos') + shell: bash + run: | + python3 -m pip install -qq --upgrade pip + python3 -m pip install -qq wheel twine typing_extensions + python3 -m pip install -qq torch==${{ matrix.torch }} python3 -c "import torch; print('torch version:', torch.__version__)" + - name: Download cudnn 8.0 + if: startsWith(matrix.os, 'ubuntu') + env: + cuda: ${{ matrix.cuda }} + run: | + ./scripts/github_actions/install_cudnn.sh + - name: Build shell: bash run: | diff --git a/scripts/github_actions/install_cuda.sh b/scripts/github_actions/install_cuda.sh new file mode 100755 index 0000000..7d023b9 --- /dev/null +++ b/scripts/github_actions/install_cuda.sh @@ -0,0 +1,59 @@ +#!/bin/bash +# +# Copyright 2020 Mobvoi Inc. (authors: Fangjun Kuang) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +echo "cuda version: $cuda" + +case "$cuda" in + 10.0) + url=https://developer.nvidia.com/compute/cuda/10.0/Prod/local_installers/cuda_10.0.130_410.48_linux + ;; + 10.1) + # WARNING: there are bugs in + # https://developer.nvidia.com/compute/cuda/10.1/Prod/local_installers/cuda_10.1.105_418.39_linux.run + # with GCC 7. Please use the following version + url=http://developer.download.nvidia.com/compute/cuda/10.1/Prod/local_installers/cuda_10.1.243_418.87.00_linux.run + ;; + 10.2) + url=http://developer.download.nvidia.com/compute/cuda/10.2/Prod/local_installers/cuda_10.2.89_440.33.01_linux.run + ;; + 11.0) + url=http://developer.download.nvidia.com/compute/cuda/11.0.2/local_installers/cuda_11.0.2_450.51.05_linux.run + ;; + 11.1) + # url=https://developer.download.nvidia.com/compute/cuda/11.1.0/local_installers/cuda_11.1.0_455.23.05_linux.run + url=https://developer.download.nvidia.com/compute/cuda/11.1.1/local_installers/cuda_11.1.1_455.32.00_linux.run + ;; + *) + echo "Unknown cuda version: $cuda" + exit 1 + ;; +esac + +function retry() { + $* || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*) +} + +retry curl -LSs -O $url +filename=$(basename $url) +echo "filename: $filename" +chmod +x ./$filename +sudo ./$filename --toolkit --silent +rm -fv ./$filename + +export CUDA_HOME=/usr/local/cuda +export PATH=$CUDA_HOME/bin:$PATH +export LD_LIBRARY_PATH=$CUDA_HOME/lib:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=$CUDA_HOME/lib64:$LD_LIBRARY_PATH diff --git a/scripts/github_actions/install_cudnn.sh b/scripts/github_actions/install_cudnn.sh new file mode 100755 index 0000000..853eba5 --- /dev/null +++ b/scripts/github_actions/install_cudnn.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# +# Copyright 2020 Mobvoi Inc. (authors: Fangjun Kuang) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +case $cuda in + 10.0) + filename=cudnn-10.0-linux-x64-v7.6.5.32.tgz + url=http://www.mediafire.com/file/1037lb1vmj9qdtq/cudnn-10.0-linux-x64-v7.6.5.32.tgz/file + ;; + 10.1) + filename=cudnn-10.1-linux-x64-v8.0.2.39.tgz + url=http://www.mediafire.com/file/fnl2wg0h757qhd7/cudnn-10.1-linux-x64-v8.0.2.39.tgz/file + ;; + 10.2) + filename=cudnn-10.2-linux-x64-v8.0.2.39.tgz + url=http://www.mediafire.com/file/sc2nvbtyg0f7ien/cudnn-10.2-linux-x64-v8.0.2.39.tgz/file + ;; + 11.0) + filename=cudnn-11.0-linux-x64-v8.0.5.39.tgz + url=https://www.mediafire.com/file/abyhnls106ko9kp/cudnn-11.0-linux-x64-v8.0.5.39.tgz/file + ;; + 11.1) + filename=cudnn-11.1-linux-x64-v8.0.5.39.tgz + url=https://www.mediafire.com/file/qx55zd65773xonv/cudnn-11.1-linux-x64-v8.0.5.39.tgz/file + ;; + *) + echo "Unsupported cuda version: $cuda" + exit 1 + ;; +esac + +function retry() { + $* || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*) +} + +# It is forked from https://github.com/Juvenal-Yescas/mediafire-dl +# https://github.com/Juvenal-Yescas/mediafire-dl/pull/2 changes the filename and breaks the CI. +# We use a separate fork to keep the link fixed. +retry wget https://raw.githubusercontent.com/csukuangfj/mediafire-dl/master/mediafire_dl.py + +sed -i 's/quiet=False/quiet=True/' mediafire_dl.py +retry python3 mediafire_dl.py "$url" +sudo tar xf ./$filename -C /usr/local +rm -v ./$filename + +sudo sed -i '59i#define CUDNN_MAJOR 8' /usr/local/cuda/include/cudnn.h diff --git a/scripts/github_actions/install_torch.sh b/scripts/github_actions/install_torch.sh new file mode 100755 index 0000000..3ad1717 --- /dev/null +++ b/scripts/github_actions/install_torch.sh @@ -0,0 +1,108 @@ +#!/bin/bash +# +# Copyright 2020 Mobvoi Inc. (authors: Fangjun Kuang) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +case ${torch} in + 1.5.*) + case ${cuda} in + 10.1) + package="torch==${torch}+cu101" + url=https://download.pytorch.org/whl/torch_stable.html + ;; + 10.2) + package="torch==${torch}" + # Leave url empty to use PyPI. + # torch_stable provides cu92 but we want cu102 + url= + ;; + esac + ;; + 1.6.0) + case ${cuda} in + 10.1) + package="torch==1.6.0+cu101" + url=https://download.pytorch.org/whl/torch_stable.html + ;; + 10.2) + package="torch==1.6.0" + # Leave it empty to use PyPI. + # torch_stable provides cu92 but we want cu102 + url= + ;; + esac + ;; + 1.7.*) + case ${cuda} in + 10.1) + package="torch==${torch}+cu101" + url=https://download.pytorch.org/whl/torch_stable.html + ;; + 10.2) + package="torch==${torch}" + # Leave it empty to use PyPI. + # torch_stable provides cu92 but we want cu102 + url= + ;; + 11.0) + package="torch==${torch}+cu110" + url=https://download.pytorch.org/whl/torch_stable.html + ;; + esac + ;; + 1.8.*) + case ${cuda} in + 10.1) + package="torch==${torch}+cu101" + url=https://download.pytorch.org/whl/torch_stable.html + ;; + 10.2) + package="torch==${torch}" + # Leave it empty to use PyPI. + url= + ;; + 11.1) + package="torch==${torch}+cu111" + url=https://download.pytorch.org/whl/torch_stable.html + ;; + esac + ;; + 1.9.0) + case ${cuda} in + 10.2) + package="torch==${torch}" + # Leave it empty to use PyPI. + url= + ;; + 11.1) + package="torch==${torch}+cu111" + url=https://download.pytorch.org/whl/torch_stable.html + ;; + esac + ;; + *) + echo "Unsupported PyTorch version: ${torch}" + exit 1 + ;; +esac + +function retry() { + $* || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*) +} + +if [ x"${url}" == "x" ]; then + retry python3 -m pip install -q $package +else + retry python3 -m pip install -q $package -f $url +fi