Build conda packages.

.github/workflows/build_conda.yml

@@ -0,0 +1,163 @@
+# Copyright      2021  Xiaomi Corp.       (author: Fangjun Kuang)
+
+# See ../../LICENSE for clarification regarding multiple authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# refer to https://github.com/actions/starter-workflows/pull/47/files
+
+name: build_conda_cuda
+
+on:
+  push:
+    branches:
+      - conda-cuda
+
+env:
+  KALDIFEAT_BUILD_TYPE: Release
+
+jobs:
+  build_conda_cuda:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-16.04]
+        # anaconda does not support 3.9 as of 2021.05.08
+        python-version: [3.6, 3.7, 3.8, 3.9]
+        # python-version: [3.6, 3.7, 3.8]
+        cuda: ["10.1", "10.2", "11.0", "11.1"]
+        # from https://download.pytorch.org/whl/torch_stable.html
+        #
+        # PyTorch 1.9.0 supports: 10.2 (default), 11.1
+        # PyTorch 1.8.1 supports: cuda 10.1, 10.2 (default), 11.1
+        # PyTorch 1.8.0 supports: cuda 10.1, 10.2 (default), 11.1
+        # PyTorch 1.7.x supports: cuda 10.1, 10.2 (default), 11.0, 9.2 (not included in this setup)
+        # PyTorch 1.6.0 supports: cuda 10.1, 10.2 (default), 9.2 (not included in this setup)
+        # PyTorch 1.5.x supports: cuda 10.1, 10.2 (default), 9.2 (not included in this setup)
+        #
+        # PyTorch 1.8.x and 1.7.1 support 3.6, 3.7, 3.8, 3.9
+        # PyTorch 1.7.0, 1.6.0, and 1.5.x support 3.6, 3.7, 3.8
+        #
+        # Other PyTorch versions are not tested
+        #
+        # torch: ["1.5.0", "1.5.1", "1.6.0", "1.7.0", "1.7.1", "1.8.0", "1.8.1"]
+        # 1.5.x is removed because there are compilation errors.
+        #  See
+        #  https://github.com/csukuangfj/k2/runs/2533830771?check_suite_focus=true
+        #  and
+        #  https://github.com/NVIDIA/apex/issues/805
+        torch: ["1.6.0", "1.7.0", "1.7.1", "1.8.0", "1.8.1", "1.9.0"]
+        exclude:
+          # - cuda: "11.0" # exclude 11.0 for [1.5.0, 1.5.1, 1.6.0, 1.8.0, 1.8.1, 1.9.0]
+          #   torch: "1.5.0"
+          # - cuda: "11.0"
+          #   torch: "1.5.1"
+          - cuda: "11.0"
+            torch: "1.6.0"
+          - cuda: "11.0"
+            torch: "1.8.0"
+          - cuda: "11.0"
+            torch: "1.8.1"
+          - cuda: "11.0"
+            torch: "1.9.0"
+          # - cuda: "11.1" # exclude 11.1 for [1.5.0, 1.5.1, 1.6.0, 1.7.0, 1.7.1]
+          #   torch: "1.5.0"
+          # - cuda: "11.1"
+          #   torch: "1.5.1"
+          - cuda: "11.1"
+            torch: "1.6.0"
+          - cuda: "11.1"
+            torch: "1.7.0"
+          - cuda: "11.1"
+            torch: "1.7.1"
+          - cuda: "10.1" # exclude 10.1 for [1.9.0]
+            torch: "1.9.0"
+          - python-version: 3.9 # exclude Python 3.9 for [1.5.0, 1.5.1, 1.6.0, 1.7.0]
+            torch: "1.5.0"
+          - python-version: 3.9
+            torch: "1.5.1"
+          - python-version: 3.9
+            torch: "1.6.0"
+          - python-version: 3.9
+            torch: "1.7.0"
+
+    steps:
+      # refer to https://github.com/actions/checkout
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: Install CUDA Toolkit ${{ matrix.cuda }}
+        shell: bash -l {0}
+        env:
+          cuda: ${{ matrix.cuda }}
+        run: |
+          source ./scripts/github_actions/install_cuda.sh
+          echo "CUDA_HOME=${CUDA_HOME}" >> $GITHUB_ENV
+          echo "${CUDA_HOME}/bin" >> $GITHUB_PATH
+          echo "LD_LIBRARY_PATH=${CUDA_HOME}/lib:${CUDA_HOME}/lib64:${LD_LIBRARY_PATH}" >> $GITHUB_ENV
+
+      - name: Display NVCC version
+        shell: bash -l {0}
+        run: |
+          which nvcc
+          nvcc --version
+
+      - uses: conda-incubator/setup-miniconda@v2
+        with:
+          auto-update-conda: true
+          python-version: ${{ matrix.python-version }}
+          activate-environment: kaldifeat
+
+      - name: Display Python version
+        shell: bash -l {0}
+        run: |
+          python3 -c "import sys; print(sys.version)"
+          which python3
+
+      - name: Install conda dependencies
+        shell: bash -l {0}
+        run: |
+          conda install -y -q anaconda-client
+          conda install -y -q conda-build
+          conda install -y -q bs4 requests tqdm
+          conda install -y -q -c pytorch -c conda-forge pytorch=${{ matrix.torch }} cudatoolkit=${{ matrix.cuda }}
+
+      - name: Display conda info
+        shell: bash -l {0}
+        run: |
+          which conda
+          conda env list
+          conda info
+          nproc
+
+      - name: Download cudnn 8.0
+        shell: bash -l {0}
+        env:
+          cuda: ${{ matrix.cuda }}
+        run: |
+          ./scripts/github_actions/install_cudnn.sh
+
+      - name: Build kaldifeat
+        shell: bash -l {0}
+        env:
+          KALDIFEAT_CUDA_VERSION: ${{ matrix.cuda }}
+          KALDIFEAT_PYTHON_VERSION: ${{ matrix.python-version}}
+          KALDIFEAT_TORCH_VERSION: ${{ matrix.torch }}
+          KALDIFEAT_CONDA_TOKEN: ${{ secrets.KALDIFEAT_CONDA_TOKEN}}
+          KALDIFEAT_IS_GITHUB_ACTIONS: 1
+          KALDIFEAT_IS_FOR_CONDA: 1
+        run: |
+          export KALDIFEAT_BUILD_TYPE=$KALDIFEAT_BUILD_TYPE
+          ./scripts/build_conda.sh

CMakeLists.txt

@@ -4,6 +4,8 @@ cmake_minimum_required(VERSION 3.8 FATAL_ERROR)
 
 project(kaldifeat)
 
+# remember to change the version in
+# scripts/conda/kaldifeat/meta.yaml
 set(kaldifeat_VERSION "1.5.4")
 
 set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib")

cmake/cmake_extension.py

@@ -27,7 +27,7 @@ try:
         def finalize_options(self):
             _bdist_wheel.finalize_options(self)
             # In this case, the generated wheel has a name in the form
-            # k2-xxx-pyxx-none-any.whl
+            # kaldifeat-xxx-pyxx-none-any.whl
             if is_for_pypi() and not is_macos():
                 self.root_is_pure = True
             else:

cmake/googletest.cmake

@@ -18,7 +18,7 @@ function(download_googltest)
     # FetchContent is available since 3.11,
     # we've copied it to ${CMAKE_SOURCE_DIR}/cmake/Modules
     # so that it can be used in lower CMake versions.
-    message(STATUS "Use FetchContent provided by k2")
+    message(STATUS "Use FetchContent provided by kaldifeat")
     list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake/Modules)
   endif()
 

scripts/build_conda.sh

@@ -0,0 +1,101 @@
+#!/usr/bin/env bash
+#
+# Copyright      2021  Xiaomi Corp.       (author: Fangjun Kuang)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# To use this script, we assume that you have installed cudatoolkit locally.
+# That is, `which nvcc` should give the path to nvcc
+#
+# We also assume that cudnn is installed locally.
+#
+# The following environment variables are supposed to be set by users
+#
+# - KALDIFEAT_CUDA_VERSION
+#     It represents the cuda version. Example:
+#
+#       export KALDIFEAT_CUDA_VERSION=10.1
+#
+#     Defaults to 10.1 if not set.
+#
+# - KALDIFEAT_TORCH_VERSION
+#     The PyTorch version. Example:
+#
+#       export KALDIFEAT_TORCH_VERSION=1.7.1
+#
+#     Defaults to 1.7.1 if not set.
+#
+# - KALDIFEAT_CONDA_TOKEN
+#     If not set, auto upload to anaconda.org is disabled.
+#
+#     Its value is from https://anaconda.org/kaldifeat/settings/access
+#      (You need to login as user kaldifeat to see its value)
+#
+# - KALDIFEAT_BUILD_TYPE
+#     If not set, defaults to Release.
+
+set -e
+export CONDA_BUILD=1
+
+cur_dir=$(cd $(dirname $BASH_SOURCE) && pwd)
+kaldifeat_dir=$(cd $cur_dir/.. && pwd)
+
+cd $kaldifeat_dir
+
+export KALDIFEAT_ROOT_DIR=$kaldifeat_dir
+echo "KALDIFEAT_ROOT_DIR: $KALDIFEAT_ROOT_DIR"
+
+KALDIFEAT_PYTHON_VERSION=$(python3 -c "import sys; print(sys.version[:3])")
+
+if [ -z $KALDIFEAT_CUDA_VERSION ]; then
+  echo "env var KALDIFEAT_CUDA_VERSION is not set, defaults to 10.1"
+  KALDIFEAT_CUDA_VERSION=10.1
+fi
+
+if [ -z $KALDIFEAT_TORCH_VERSION ]; then
+  echo "env var KALDIFEAT_TORCH_VERSION is not set, defaults to 1.7.1"
+  KALDIFEAT_TORCH_VERSION=1.7.1
+fi
+
+if [ -z $KALDIFEAT_BUILD_TYPE ]; then
+  echo "env var KALDIFEAT_BUILD_TYPE is not set, defaults to Release"
+  KALDIFEAT_BUILD_TYPE=Release
+fi
+
+export KALDIFEAT_IS_FOR_CONDA=1
+
+# Example value: 3.8
+export KALDIFEAT_PYTHON_VERSION
+
+# Example value: 10.1
+export KALDIFEAT_CUDA_VERSION
+
+# Example value: 1.7.1
+export KALDIFEAT_TORCH_VERSION
+
+export KALDIFEAT_BUILD_TYPE
+
+if [ ! -z $KALDIFEAT_IS_GITHUB_ACTIONS ]; then
+  export KALDIFEAT_IS_GITHUB_ACTIONS
+  conda remove -q pytorch cudatoolkit
+  conda clean -q -a
+else
+  export KALDIFEAT_IS_GITHUB_ACTIONS=0
+fi
+
+if [ -z $KALDIFEAT_CONDA_TOKEN ]; then
+  echo "Auto upload to anaconda.org is disabled since KALDIFEAT_CONDA_TOKEN is not set"
+  conda build --no-test --no-anaconda-upload -c pytorch -c conda-forge ./scripts/conda/kaldifeat
+else
+  conda build --no-test -c pytorch -c conda-forge --token $KADIFEAT_CONDA_TOKEN ./scripts/conda/kaldifeat
+fi

scripts/conda/kaldifeat/build.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+#
+# Copyright      2021  Xiaomi Corp.       (author: Fangjun Kuang)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+CONDA_ENV_DIR=$CONDA_PREFIX
+
+echo "KALDIFEAT_PYTHON_VERSION: $KALDIFEAT_PYTHON_VERSION"
+echo "KALDIFEAT_TORCH_VERSION: $KALDIFEAT_TORCH_VERSION"
+echo "KALDIFEAT_CUDA_VERSION: $KALDIFEAT_CUDA_VERSION"
+echo "KALDIFEAT_BUILD_TYPE: $KALDIFEAT_BUILD_TYPE"
+echo "KALDIFEAT_BUILD_VERSION: $KALDIFEAT_BUILD_VERSION"
+python3 --version
+
+echo "CC is: $CC"
+echo "GCC is: $GCC"
+echo "which nvcc: $(which nvcc)"
+echo "gcc version: $($CC --version)"
+echo "nvcc version: $(nvcc --version)"
+
+export KALDIFEAT_CMAKE_ARGS="-DCMAKE_BUILD_TYPE=${KALDIFEAT_BUILD_TYPE}"
+export KALDIFEAT_MAKE_ARGS="-j2"
+
+python3 setup.py install --single-version-externally-managed --record=record.txt

scripts/conda/kaldifeat/meta.yaml

@@ -0,0 +1,39 @@
+package:
+  name: kaldifeat
+  version: "1.5.4"
+
+source:
+  path: "{{ environ.get('KALDIFEAT_ROOT_DIR') }}"
+
+build:
+  number: 0
+  string: cuda{{ environ.get('KALDIFEAT_CUDA_VERSION') }}_py{{ environ.get('KALDIFEAT_PYTHON_VERSION') }}_torch{{ environ.get('KALDIFEAT_TORCH_VERSION') }}
+  script_env:
+    - KALDIFEAT_IS_GITHUB_ACTIONS
+    - KALDIFEAT_CUDA_VERSION
+    - KALDIFEAT_TORCH_VERSION
+    - KALDIFEAT_PYTHON_VERSION
+    - KALDIFEAT_BUILD_TYPE
+    - KALDIFEAT_BUILD_VERSION
+    - KALDIFEAT_IS_FOR_CONDA
+
+requirements:
+  host:
+    - cmake=3.18
+    - python
+    - pytorch={{ environ.get('KALDIFEAT_TORCH_VERSION') }}
+    - cudatoolkit={{ environ.get('KALDIFEAT_CUDA_VERSION') }}
+    - gcc_linux-64=7
+  run:
+    - python
+    - pytorch={{ environ.get('KALDIFEAT_TORCH_VERSION') }}
+    - cudatoolkit={{ environ.get('KALDIFEAT_CUDA_VERSION') }}
+
+about:
+  home: https://github.com/csukuangfj/kaldifeat
+  license: Apache V2
+  license_file: LICENSE
+  summary: Kaldi-compatible feature extraction with PyTorch
+  description: |
+    Kaldi-compatible feature extraction with PyTorch,
+    supporting CUDA, batch processing, chunk processing, and autograd