diff --git a/.github/workflows/publish_to_pypi.yml b/.github/workflows/publish_to_pypi.yml index 74c255a..38ddc21 100644 --- a/.github/workflows/publish_to_pypi.yml +++ b/.github/workflows/publish_to_pypi.yml @@ -1,36 +1,91 @@ name: Publish to PyPI on: + # push: + # tags: + # - '*' push: - tags: - - '*' + branches: + - master + pull_request: + branches: + - master jobs: pypi: - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-18.04] + cuda: ["10.1"] + gcc: ["5"] + torch: ["1.8.1"] + python-version: [3.6, 3.7, 3.8] steps: - uses: actions/checkout@v2 + with: + fetch-depth: 0 - name: Setup Python uses: actions/setup-python@v2 with: - python-version: 3.6 + python-version: ${{ matrix.python-version }} - - name: Install Python dependencies + - name: Install CUDA Toolkit ${{ matrix.cuda }} + shell: bash + env: + cuda: ${{ matrix.cuda }} + run: | + source ./scripts/github_actions/install_cuda.sh + echo "CUDA_HOME=${CUDA_HOME}" >> $GITHUB_ENV + echo "${CUDA_HOME}/bin" >> $GITHUB_PATH + echo "LD_LIBRARY_PATH=${CUDA_HOME}/lib:${CUDA_HOME}/lib64:${LD_LIBRARY_PATH}" >> $GITHUB_ENV + + - name: Display NVCC version + run: | + which nvcc + nvcc --version + + - name: Install GCC ${{ matrix.gcc }} + run: | + sudo apt-get install -y gcc-${{ matrix.gcc }} g++-${{ matrix.gcc }} + echo "CC=/usr/bin/gcc-${{ matrix.gcc }}" >> $GITHUB_ENV + echo "CXX=/usr/bin/g++-${{ matrix.gcc }}" >> $GITHUB_ENV + echo "CUDAHOSTCXX=/usr/bin/g++-${{ matrix.gcc }}" >> $GITHUB_ENV + + - name: Install PyTorch ${{ matrix.torch }} + env: + cuda: ${{ matrix.cuda }} + torch: ${{ matrix.torch }} shell: bash run: | python3 -m pip install --upgrade pip - python3 -m pip install wheel twine setuptools + python3 -m pip install wheel twine typing_extensions + python3 -m pip install bs4 requests tqdm - - name: Build - shell: bash - run: | - python3 setup.py sdist - ls -l dist/* + ./scripts/github_actions/install_torch.sh + python3 -c "import torch; print('torch version:', torch.__version__)" - - name: Publish wheels to PyPI + - name: Download cudnn 8.0 env: - TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + cuda: ${{ matrix.cuda }} run: | - twine upload dist/kaldifeat-*.tar.gz + ./scripts/github_actions/install_cudnn.sh + + - name: Build pip packages + shell: bash + env: + KALDIFEAT_IS_FOR_PYPI: 1 + run: | + tag=$(python3 -c "import sys; print(''.join(sys.version[:3].split('.')))") + export KALDIFEAT_MAKE_ARGS="-j2" + python3 setup.py bdist_wheel --python-tag=py${tag} + ls -lh dist/ + + # - name: Publish wheels to PyPI + # env: + # TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} + # TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + # run: | + # twine upload dist/kaldifeat-*.whl diff --git a/cmake/cmake_extension.py b/cmake/cmake_extension.py index 9f2d879..413a87d 100644 --- a/cmake/cmake_extension.py +++ b/cmake/cmake_extension.py @@ -9,6 +9,12 @@ from pathlib import Path import setuptools from setuptools.command.build_ext import build_ext + +def is_for_pypi(): + ans = os.environ.get("KALDIFEAT_IS_FOR_PYPI", None) + return ans is not None + + try: from wheel.bdist_wheel import bdist_wheel as _bdist_wheel @@ -17,11 +23,12 @@ try: _bdist_wheel.finalize_options(self) # In this case, the generated wheel has a name in the form # k2-xxx-pyxx-none-any.whl - # self.root_is_pure = True - - # The generated wheel has a name ending with - # -linux_x86_64.whl - self.root_is_pure = False + if is_for_pypi(): + self.root_is_pure = True + else: + # The generated wheel has a name ending with + # -linux_x86_64.whl + self.root_is_pure = False except ImportError: diff --git a/scripts/github_actions/install_cuda.sh b/scripts/github_actions/install_cuda.sh new file mode 100755 index 0000000..7d023b9 --- /dev/null +++ b/scripts/github_actions/install_cuda.sh @@ -0,0 +1,59 @@ +#!/bin/bash +# +# Copyright 2020 Mobvoi Inc. (authors: Fangjun Kuang) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +echo "cuda version: $cuda" + +case "$cuda" in + 10.0) + url=https://developer.nvidia.com/compute/cuda/10.0/Prod/local_installers/cuda_10.0.130_410.48_linux + ;; + 10.1) + # WARNING: there are bugs in + # https://developer.nvidia.com/compute/cuda/10.1/Prod/local_installers/cuda_10.1.105_418.39_linux.run + # with GCC 7. Please use the following version + url=http://developer.download.nvidia.com/compute/cuda/10.1/Prod/local_installers/cuda_10.1.243_418.87.00_linux.run + ;; + 10.2) + url=http://developer.download.nvidia.com/compute/cuda/10.2/Prod/local_installers/cuda_10.2.89_440.33.01_linux.run + ;; + 11.0) + url=http://developer.download.nvidia.com/compute/cuda/11.0.2/local_installers/cuda_11.0.2_450.51.05_linux.run + ;; + 11.1) + # url=https://developer.download.nvidia.com/compute/cuda/11.1.0/local_installers/cuda_11.1.0_455.23.05_linux.run + url=https://developer.download.nvidia.com/compute/cuda/11.1.1/local_installers/cuda_11.1.1_455.32.00_linux.run + ;; + *) + echo "Unknown cuda version: $cuda" + exit 1 + ;; +esac + +function retry() { + $* || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*) +} + +retry curl -LSs -O $url +filename=$(basename $url) +echo "filename: $filename" +chmod +x ./$filename +sudo ./$filename --toolkit --silent +rm -fv ./$filename + +export CUDA_HOME=/usr/local/cuda +export PATH=$CUDA_HOME/bin:$PATH +export LD_LIBRARY_PATH=$CUDA_HOME/lib:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=$CUDA_HOME/lib64:$LD_LIBRARY_PATH diff --git a/scripts/github_actions/install_cudnn.sh b/scripts/github_actions/install_cudnn.sh new file mode 100755 index 0000000..853eba5 --- /dev/null +++ b/scripts/github_actions/install_cudnn.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# +# Copyright 2020 Mobvoi Inc. (authors: Fangjun Kuang) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +case $cuda in + 10.0) + filename=cudnn-10.0-linux-x64-v7.6.5.32.tgz + url=http://www.mediafire.com/file/1037lb1vmj9qdtq/cudnn-10.0-linux-x64-v7.6.5.32.tgz/file + ;; + 10.1) + filename=cudnn-10.1-linux-x64-v8.0.2.39.tgz + url=http://www.mediafire.com/file/fnl2wg0h757qhd7/cudnn-10.1-linux-x64-v8.0.2.39.tgz/file + ;; + 10.2) + filename=cudnn-10.2-linux-x64-v8.0.2.39.tgz + url=http://www.mediafire.com/file/sc2nvbtyg0f7ien/cudnn-10.2-linux-x64-v8.0.2.39.tgz/file + ;; + 11.0) + filename=cudnn-11.0-linux-x64-v8.0.5.39.tgz + url=https://www.mediafire.com/file/abyhnls106ko9kp/cudnn-11.0-linux-x64-v8.0.5.39.tgz/file + ;; + 11.1) + filename=cudnn-11.1-linux-x64-v8.0.5.39.tgz + url=https://www.mediafire.com/file/qx55zd65773xonv/cudnn-11.1-linux-x64-v8.0.5.39.tgz/file + ;; + *) + echo "Unsupported cuda version: $cuda" + exit 1 + ;; +esac + +function retry() { + $* || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*) +} + +# It is forked from https://github.com/Juvenal-Yescas/mediafire-dl +# https://github.com/Juvenal-Yescas/mediafire-dl/pull/2 changes the filename and breaks the CI. +# We use a separate fork to keep the link fixed. +retry wget https://raw.githubusercontent.com/csukuangfj/mediafire-dl/master/mediafire_dl.py + +sed -i 's/quiet=False/quiet=True/' mediafire_dl.py +retry python3 mediafire_dl.py "$url" +sudo tar xf ./$filename -C /usr/local +rm -v ./$filename + +sudo sed -i '59i#define CUDNN_MAJOR 8' /usr/local/cuda/include/cudnn.h diff --git a/scripts/github_actions/install_torch.sh b/scripts/github_actions/install_torch.sh new file mode 100755 index 0000000..3ad1717 --- /dev/null +++ b/scripts/github_actions/install_torch.sh @@ -0,0 +1,108 @@ +#!/bin/bash +# +# Copyright 2020 Mobvoi Inc. (authors: Fangjun Kuang) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +case ${torch} in + 1.5.*) + case ${cuda} in + 10.1) + package="torch==${torch}+cu101" + url=https://download.pytorch.org/whl/torch_stable.html + ;; + 10.2) + package="torch==${torch}" + # Leave url empty to use PyPI. + # torch_stable provides cu92 but we want cu102 + url= + ;; + esac + ;; + 1.6.0) + case ${cuda} in + 10.1) + package="torch==1.6.0+cu101" + url=https://download.pytorch.org/whl/torch_stable.html + ;; + 10.2) + package="torch==1.6.0" + # Leave it empty to use PyPI. + # torch_stable provides cu92 but we want cu102 + url= + ;; + esac + ;; + 1.7.*) + case ${cuda} in + 10.1) + package="torch==${torch}+cu101" + url=https://download.pytorch.org/whl/torch_stable.html + ;; + 10.2) + package="torch==${torch}" + # Leave it empty to use PyPI. + # torch_stable provides cu92 but we want cu102 + url= + ;; + 11.0) + package="torch==${torch}+cu110" + url=https://download.pytorch.org/whl/torch_stable.html + ;; + esac + ;; + 1.8.*) + case ${cuda} in + 10.1) + package="torch==${torch}+cu101" + url=https://download.pytorch.org/whl/torch_stable.html + ;; + 10.2) + package="torch==${torch}" + # Leave it empty to use PyPI. + url= + ;; + 11.1) + package="torch==${torch}+cu111" + url=https://download.pytorch.org/whl/torch_stable.html + ;; + esac + ;; + 1.9.0) + case ${cuda} in + 10.2) + package="torch==${torch}" + # Leave it empty to use PyPI. + url= + ;; + 11.1) + package="torch==${torch}+cu111" + url=https://download.pytorch.org/whl/torch_stable.html + ;; + esac + ;; + *) + echo "Unsupported PyTorch version: ${torch}" + exit 1 + ;; +esac + +function retry() { + $* || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*) +} + +if [ x"${url}" == "x" ]; then + retry python3 -m pip install -q $package +else + retry python3 -m pip install -q $package -f $url +fi