Publish to PyPI.

.github/workflows/publish_to_pypi.yml

@@ -1,36 +1,91 @@
 name: Publish to PyPI
 
 on:
+  # push:
+  #   tags:
+  #     - '*'
   push:
-    tags:
+    branches:
-      - '*'
+      - master
+  pull_request:
+    branches:
+      - master
 
 jobs:
   pypi:
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-18.04]
+        cuda: ["10.1"]
+        gcc: ["5"]
+        torch: ["1.8.1"]
+        python-version: [3.6, 3.7, 3.8]
     steps:
       - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
 
       - name: Setup Python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.6
+          python-version: ${{ matrix.python-version }}
 
-      - name: Install Python dependencies
+      - name: Install CUDA Toolkit ${{ matrix.cuda }}
+        shell: bash
+        env:
+          cuda: ${{ matrix.cuda }}
+        run: |
+          source ./scripts/github_actions/install_cuda.sh
+          echo "CUDA_HOME=${CUDA_HOME}" >> $GITHUB_ENV
+          echo "${CUDA_HOME}/bin" >> $GITHUB_PATH
+          echo "LD_LIBRARY_PATH=${CUDA_HOME}/lib:${CUDA_HOME}/lib64:${LD_LIBRARY_PATH}" >> $GITHUB_ENV
+
+      - name: Display NVCC version
+        run: |
+          which nvcc
+          nvcc --version
+
+      - name: Install GCC ${{ matrix.gcc }}
+        run: |
+          sudo apt-get install -y gcc-${{ matrix.gcc }} g++-${{ matrix.gcc }}
+          echo "CC=/usr/bin/gcc-${{ matrix.gcc }}" >> $GITHUB_ENV
+          echo "CXX=/usr/bin/g++-${{ matrix.gcc }}" >> $GITHUB_ENV
+          echo "CUDAHOSTCXX=/usr/bin/g++-${{ matrix.gcc }}" >> $GITHUB_ENV
+
+      - name: Install PyTorch ${{ matrix.torch }}
+        env:
+          cuda: ${{ matrix.cuda }}
+          torch: ${{ matrix.torch }}
         shell: bash
         run: |
           python3 -m pip install --upgrade pip
-          python3 -m pip install wheel twine setuptools
+          python3 -m pip install wheel twine typing_extensions
+          python3 -m pip install bs4 requests tqdm
 
-      - name: Build
+          ./scripts/github_actions/install_torch.sh
-        shell: bash
+          python3 -c "import torch; print('torch version:', torch.__version__)"
-        run: |
-          python3 setup.py sdist
-          ls -l dist/*
 
-      - name: Publish wheels to PyPI
+      - name: Download cudnn 8.0
         env:
-          TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
+          cuda: ${{ matrix.cuda }}
-          TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
         run: |
-          twine upload dist/kaldifeat-*.tar.gz
+          ./scripts/github_actions/install_cudnn.sh
+
+      - name: Build pip packages
+        shell: bash
+        env:
+          KALDIFEAT_IS_FOR_PYPI: 1
+        run: |
+          tag=$(python3 -c "import sys; print(''.join(sys.version[:3].split('.')))")
+          export KALDIFEAT_MAKE_ARGS="-j2"
+          python3 setup.py bdist_wheel --python-tag=py${tag}
+          ls -lh dist/
+
+      # - name: Publish wheels to PyPI
+      #   env:
+      #     TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
+      #     TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+      #   run: |
+      #     twine upload dist/kaldifeat-*.whl

cmake/cmake_extension.py

@@ -9,6 +9,12 @@ from pathlib import Path
 import setuptools
 from setuptools.command.build_ext import build_ext
 
+
+def is_for_pypi():
+    ans = os.environ.get("KALDIFEAT_IS_FOR_PYPI", None)
+    return ans is not None
+
+
 try:
     from wheel.bdist_wheel import bdist_wheel as _bdist_wheel
 
@@ -17,8 +23,9 @@ try:
             _bdist_wheel.finalize_options(self)
             # In this case, the generated wheel has a name in the form
             # k2-xxx-pyxx-none-any.whl
-            #  self.root_is_pure = True
+            if is_for_pypi():
-
+                self.root_is_pure = True
+            else:
                 # The generated wheel has a name ending with
                 # -linux_x86_64.whl
                 self.root_is_pure = False

scripts/github_actions/install_cuda.sh

@@ -0,0 +1,59 @@
+#!/bin/bash
+#
+# Copyright      2020  Mobvoi Inc. (authors: Fangjun Kuang)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+echo "cuda version: $cuda"
+
+case "$cuda" in
+  10.0)
+    url=https://developer.nvidia.com/compute/cuda/10.0/Prod/local_installers/cuda_10.0.130_410.48_linux
+    ;;
+  10.1)
+    # WARNING: there are bugs in
+    # https://developer.nvidia.com/compute/cuda/10.1/Prod/local_installers/cuda_10.1.105_418.39_linux.run
+    # with GCC 7. Please use the following version
+    url=http://developer.download.nvidia.com/compute/cuda/10.1/Prod/local_installers/cuda_10.1.243_418.87.00_linux.run
+    ;;
+  10.2)
+    url=http://developer.download.nvidia.com/compute/cuda/10.2/Prod/local_installers/cuda_10.2.89_440.33.01_linux.run
+    ;;
+  11.0)
+    url=http://developer.download.nvidia.com/compute/cuda/11.0.2/local_installers/cuda_11.0.2_450.51.05_linux.run
+    ;;
+  11.1)
+    # url=https://developer.download.nvidia.com/compute/cuda/11.1.0/local_installers/cuda_11.1.0_455.23.05_linux.run
+    url=https://developer.download.nvidia.com/compute/cuda/11.1.1/local_installers/cuda_11.1.1_455.32.00_linux.run
+    ;;
+  *)
+    echo "Unknown cuda version: $cuda"
+    exit 1
+    ;;
+esac
+
+function retry() {
+  $* || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*)
+}
+
+retry curl -LSs -O $url
+filename=$(basename $url)
+echo "filename: $filename"
+chmod +x ./$filename
+sudo ./$filename --toolkit --silent
+rm -fv ./$filename
+
+export CUDA_HOME=/usr/local/cuda
+export PATH=$CUDA_HOME/bin:$PATH
+export LD_LIBRARY_PATH=$CUDA_HOME/lib:$LD_LIBRARY_PATH
+export LD_LIBRARY_PATH=$CUDA_HOME/lib64:$LD_LIBRARY_PATH

scripts/github_actions/install_cudnn.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+#
+# Copyright      2020  Mobvoi Inc. (authors: Fangjun Kuang)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+case $cuda in
+  10.0)
+    filename=cudnn-10.0-linux-x64-v7.6.5.32.tgz
+    url=http://www.mediafire.com/file/1037lb1vmj9qdtq/cudnn-10.0-linux-x64-v7.6.5.32.tgz/file
+    ;;
+  10.1)
+    filename=cudnn-10.1-linux-x64-v8.0.2.39.tgz
+    url=http://www.mediafire.com/file/fnl2wg0h757qhd7/cudnn-10.1-linux-x64-v8.0.2.39.tgz/file
+    ;;
+  10.2)
+    filename=cudnn-10.2-linux-x64-v8.0.2.39.tgz
+    url=http://www.mediafire.com/file/sc2nvbtyg0f7ien/cudnn-10.2-linux-x64-v8.0.2.39.tgz/file
+    ;;
+  11.0)
+    filename=cudnn-11.0-linux-x64-v8.0.5.39.tgz
+    url=https://www.mediafire.com/file/abyhnls106ko9kp/cudnn-11.0-linux-x64-v8.0.5.39.tgz/file
+    ;;
+  11.1)
+    filename=cudnn-11.1-linux-x64-v8.0.5.39.tgz
+    url=https://www.mediafire.com/file/qx55zd65773xonv/cudnn-11.1-linux-x64-v8.0.5.39.tgz/file
+    ;;
+  *)
+    echo "Unsupported cuda version: $cuda"
+    exit 1
+    ;;
+esac
+
+function retry() {
+  $* || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*)
+}
+
+# It is forked from https://github.com/Juvenal-Yescas/mediafire-dl
+# https://github.com/Juvenal-Yescas/mediafire-dl/pull/2 changes the filename and breaks the CI.
+# We use a separate fork to keep the link fixed.
+retry wget https://raw.githubusercontent.com/csukuangfj/mediafire-dl/master/mediafire_dl.py
+
+sed -i 's/quiet=False/quiet=True/' mediafire_dl.py
+retry python3 mediafire_dl.py "$url"
+sudo tar xf ./$filename -C /usr/local
+rm -v ./$filename
+
+sudo sed -i '59i#define CUDNN_MAJOR 8' /usr/local/cuda/include/cudnn.h

scripts/github_actions/install_torch.sh

@@ -0,0 +1,108 @@
+#!/bin/bash
+#
+# Copyright      2020  Mobvoi Inc. (authors: Fangjun Kuang)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+case ${torch} in
+  1.5.*)
+    case ${cuda} in
+      10.1)
+        package="torch==${torch}+cu101"
+        url=https://download.pytorch.org/whl/torch_stable.html
+        ;;
+      10.2)
+        package="torch==${torch}"
+        # Leave url empty to use PyPI.
+        # torch_stable provides cu92 but we want cu102
+        url=
+        ;;
+    esac
+    ;;
+  1.6.0)
+    case ${cuda} in
+      10.1)
+        package="torch==1.6.0+cu101"
+        url=https://download.pytorch.org/whl/torch_stable.html
+        ;;
+      10.2)
+        package="torch==1.6.0"
+        # Leave it empty to use PyPI.
+        # torch_stable provides cu92 but we want cu102
+        url=
+        ;;
+    esac
+    ;;
+  1.7.*)
+    case ${cuda} in
+      10.1)
+        package="torch==${torch}+cu101"
+        url=https://download.pytorch.org/whl/torch_stable.html
+        ;;
+      10.2)
+        package="torch==${torch}"
+        # Leave it empty to use PyPI.
+        # torch_stable provides cu92 but we want cu102
+        url=
+        ;;
+      11.0)
+        package="torch==${torch}+cu110"
+        url=https://download.pytorch.org/whl/torch_stable.html
+        ;;
+    esac
+    ;;
+  1.8.*)
+    case ${cuda} in
+      10.1)
+        package="torch==${torch}+cu101"
+        url=https://download.pytorch.org/whl/torch_stable.html
+        ;;
+      10.2)
+        package="torch==${torch}"
+        # Leave it empty to use PyPI.
+        url=
+        ;;
+      11.1)
+        package="torch==${torch}+cu111"
+        url=https://download.pytorch.org/whl/torch_stable.html
+        ;;
+    esac
+    ;;
+  1.9.0)
+    case ${cuda} in
+      10.2)
+        package="torch==${torch}"
+        # Leave it empty to use PyPI.
+        url=
+        ;;
+      11.1)
+        package="torch==${torch}+cu111"
+        url=https://download.pytorch.org/whl/torch_stable.html
+        ;;
+    esac
+    ;;
+  *)
+    echo "Unsupported PyTorch version: ${torch}"
+    exit 1
+    ;;
+esac
+
+function retry() {
+  $* || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*)
+}
+
+if [ x"${url}" == "x" ]; then
+  retry python3 -m pip install -q $package
+else
+  retry python3 -m pip install -q $package -f $url
+fi