diff --git a/.github/workflows/build_conda.yml b/.github/workflows/build_conda.yml new file mode 100644 index 0000000..a7e43fe --- /dev/null +++ b/.github/workflows/build_conda.yml @@ -0,0 +1,163 @@ +# Copyright 2021 Xiaomi Corp. (author: Fangjun Kuang) + +# See ../../LICENSE for clarification regarding multiple authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# refer to https://github.com/actions/starter-workflows/pull/47/files + +name: build_conda_cuda + +on: + push: + branches: + - conda-cuda + +env: + KALDIFEAT_BUILD_TYPE: Release + +jobs: + build_conda_cuda: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-16.04] + # anaconda does not support 3.9 as of 2021.05.08 + python-version: [3.6, 3.7, 3.8, 3.9] + # python-version: [3.6, 3.7, 3.8] + cuda: ["10.1", "10.2", "11.0", "11.1"] + # from https://download.pytorch.org/whl/torch_stable.html + # + # PyTorch 1.9.0 supports: 10.2 (default), 11.1 + # PyTorch 1.8.1 supports: cuda 10.1, 10.2 (default), 11.1 + # PyTorch 1.8.0 supports: cuda 10.1, 10.2 (default), 11.1 + # PyTorch 1.7.x supports: cuda 10.1, 10.2 (default), 11.0, 9.2 (not included in this setup) + # PyTorch 1.6.0 supports: cuda 10.1, 10.2 (default), 9.2 (not included in this setup) + # PyTorch 1.5.x supports: cuda 10.1, 10.2 (default), 9.2 (not included in this setup) + # + # PyTorch 1.8.x and 1.7.1 support 3.6, 3.7, 3.8, 3.9 + # PyTorch 1.7.0, 1.6.0, and 1.5.x support 3.6, 3.7, 3.8 + # + # Other PyTorch versions are not tested + # + # torch: ["1.5.0", "1.5.1", "1.6.0", "1.7.0", "1.7.1", "1.8.0", "1.8.1"] + # 1.5.x is removed because there are compilation errors. + # See + # https://github.com/csukuangfj/k2/runs/2533830771?check_suite_focus=true + # and + # https://github.com/NVIDIA/apex/issues/805 + torch: ["1.6.0", "1.7.0", "1.7.1", "1.8.0", "1.8.1", "1.9.0"] + exclude: + # - cuda: "11.0" # exclude 11.0 for [1.5.0, 1.5.1, 1.6.0, 1.8.0, 1.8.1, 1.9.0] + # torch: "1.5.0" + # - cuda: "11.0" + # torch: "1.5.1" + - cuda: "11.0" + torch: "1.6.0" + - cuda: "11.0" + torch: "1.8.0" + - cuda: "11.0" + torch: "1.8.1" + - cuda: "11.0" + torch: "1.9.0" + # - cuda: "11.1" # exclude 11.1 for [1.5.0, 1.5.1, 1.6.0, 1.7.0, 1.7.1] + # torch: "1.5.0" + # - cuda: "11.1" + # torch: "1.5.1" + - cuda: "11.1" + torch: "1.6.0" + - cuda: "11.1" + torch: "1.7.0" + - cuda: "11.1" + torch: "1.7.1" + - cuda: "10.1" # exclude 10.1 for [1.9.0] + torch: "1.9.0" + - python-version: 3.9 # exclude Python 3.9 for [1.5.0, 1.5.1, 1.6.0, 1.7.0] + torch: "1.5.0" + - python-version: 3.9 + torch: "1.5.1" + - python-version: 3.9 + torch: "1.6.0" + - python-version: 3.9 + torch: "1.7.0" + + steps: + # refer to https://github.com/actions/checkout + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Install CUDA Toolkit ${{ matrix.cuda }} + shell: bash -l {0} + env: + cuda: ${{ matrix.cuda }} + run: | + source ./scripts/github_actions/install_cuda.sh + echo "CUDA_HOME=${CUDA_HOME}" >> $GITHUB_ENV + echo "${CUDA_HOME}/bin" >> $GITHUB_PATH + echo "LD_LIBRARY_PATH=${CUDA_HOME}/lib:${CUDA_HOME}/lib64:${LD_LIBRARY_PATH}" >> $GITHUB_ENV + + - name: Display NVCC version + shell: bash -l {0} + run: | + which nvcc + nvcc --version + + - uses: conda-incubator/setup-miniconda@v2 + with: + auto-update-conda: true + python-version: ${{ matrix.python-version }} + activate-environment: kaldifeat + + - name: Display Python version + shell: bash -l {0} + run: | + python3 -c "import sys; print(sys.version)" + which python3 + + - name: Install conda dependencies + shell: bash -l {0} + run: | + conda install -y -q anaconda-client + conda install -y -q conda-build + conda install -y -q bs4 requests tqdm + conda install -y -q -c pytorch -c conda-forge pytorch=${{ matrix.torch }} cudatoolkit=${{ matrix.cuda }} + + - name: Display conda info + shell: bash -l {0} + run: | + which conda + conda env list + conda info + nproc + + - name: Download cudnn 8.0 + shell: bash -l {0} + env: + cuda: ${{ matrix.cuda }} + run: | + ./scripts/github_actions/install_cudnn.sh + + - name: Build kaldifeat + shell: bash -l {0} + env: + KALDIFEAT_CUDA_VERSION: ${{ matrix.cuda }} + KALDIFEAT_PYTHON_VERSION: ${{ matrix.python-version}} + KALDIFEAT_TORCH_VERSION: ${{ matrix.torch }} + KALDIFEAT_CONDA_TOKEN: ${{ secrets.KALDIFEAT_CONDA_TOKEN}} + KALDIFEAT_IS_GITHUB_ACTIONS: 1 + KALDIFEAT_IS_FOR_CONDA: 1 + run: | + export KALDIFEAT_BUILD_TYPE=$KALDIFEAT_BUILD_TYPE + ./scripts/build_conda.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index f1e7c50..0f3f901 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,6 +4,8 @@ cmake_minimum_required(VERSION 3.8 FATAL_ERROR) project(kaldifeat) +# remember to change the version in +# scripts/conda/kaldifeat/meta.yaml set(kaldifeat_VERSION "1.5.4") set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib") diff --git a/cmake/cmake_extension.py b/cmake/cmake_extension.py index ceb0d61..ae27abf 100644 --- a/cmake/cmake_extension.py +++ b/cmake/cmake_extension.py @@ -27,7 +27,7 @@ try: def finalize_options(self): _bdist_wheel.finalize_options(self) # In this case, the generated wheel has a name in the form - # k2-xxx-pyxx-none-any.whl + # kaldifeat-xxx-pyxx-none-any.whl if is_for_pypi() and not is_macos(): self.root_is_pure = True else: diff --git a/cmake/googletest.cmake b/cmake/googletest.cmake index 0252268..0795a30 100644 --- a/cmake/googletest.cmake +++ b/cmake/googletest.cmake @@ -18,7 +18,7 @@ function(download_googltest) # FetchContent is available since 3.11, # we've copied it to ${CMAKE_SOURCE_DIR}/cmake/Modules # so that it can be used in lower CMake versions. - message(STATUS "Use FetchContent provided by k2") + message(STATUS "Use FetchContent provided by kaldifeat") list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake/Modules) endif() diff --git a/scripts/build_conda.sh b/scripts/build_conda.sh new file mode 100755 index 0000000..49ede93 --- /dev/null +++ b/scripts/build_conda.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +# +# Copyright 2021 Xiaomi Corp. (author: Fangjun Kuang) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# To use this script, we assume that you have installed cudatoolkit locally. +# That is, `which nvcc` should give the path to nvcc +# +# We also assume that cudnn is installed locally. +# +# The following environment variables are supposed to be set by users +# +# - KALDIFEAT_CUDA_VERSION +# It represents the cuda version. Example: +# +# export KALDIFEAT_CUDA_VERSION=10.1 +# +# Defaults to 10.1 if not set. +# +# - KALDIFEAT_TORCH_VERSION +# The PyTorch version. Example: +# +# export KALDIFEAT_TORCH_VERSION=1.7.1 +# +# Defaults to 1.7.1 if not set. +# +# - KALDIFEAT_CONDA_TOKEN +# If not set, auto upload to anaconda.org is disabled. +# +# Its value is from https://anaconda.org/kaldifeat/settings/access +# (You need to login as user kaldifeat to see its value) +# +# - KALDIFEAT_BUILD_TYPE +# If not set, defaults to Release. + +set -e +export CONDA_BUILD=1 + +cur_dir=$(cd $(dirname $BASH_SOURCE) && pwd) +kaldifeat_dir=$(cd $cur_dir/.. && pwd) + +cd $kaldifeat_dir + +export KALDIFEAT_ROOT_DIR=$kaldifeat_dir +echo "KALDIFEAT_ROOT_DIR: $KALDIFEAT_ROOT_DIR" + +KALDIFEAT_PYTHON_VERSION=$(python3 -c "import sys; print(sys.version[:3])") + +if [ -z $KALDIFEAT_CUDA_VERSION ]; then + echo "env var KALDIFEAT_CUDA_VERSION is not set, defaults to 10.1" + KALDIFEAT_CUDA_VERSION=10.1 +fi + +if [ -z $KALDIFEAT_TORCH_VERSION ]; then + echo "env var KALDIFEAT_TORCH_VERSION is not set, defaults to 1.7.1" + KALDIFEAT_TORCH_VERSION=1.7.1 +fi + +if [ -z $KALDIFEAT_BUILD_TYPE ]; then + echo "env var KALDIFEAT_BUILD_TYPE is not set, defaults to Release" + KALDIFEAT_BUILD_TYPE=Release +fi + +export KALDIFEAT_IS_FOR_CONDA=1 + +# Example value: 3.8 +export KALDIFEAT_PYTHON_VERSION + +# Example value: 10.1 +export KALDIFEAT_CUDA_VERSION + +# Example value: 1.7.1 +export KALDIFEAT_TORCH_VERSION + +export KALDIFEAT_BUILD_TYPE + +if [ ! -z $KALDIFEAT_IS_GITHUB_ACTIONS ]; then + export KALDIFEAT_IS_GITHUB_ACTIONS + conda remove -q pytorch cudatoolkit + conda clean -q -a +else + export KALDIFEAT_IS_GITHUB_ACTIONS=0 +fi + +if [ -z $KALDIFEAT_CONDA_TOKEN ]; then + echo "Auto upload to anaconda.org is disabled since KALDIFEAT_CONDA_TOKEN is not set" + conda build --no-test --no-anaconda-upload -c pytorch -c conda-forge ./scripts/conda/kaldifeat +else + conda build --no-test -c pytorch -c conda-forge --token $KADIFEAT_CONDA_TOKEN ./scripts/conda/kaldifeat +fi diff --git a/scripts/conda/kaldifeat/build.sh b/scripts/conda/kaldifeat/build.sh new file mode 100644 index 0000000..3897511 --- /dev/null +++ b/scripts/conda/kaldifeat/build.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# +# Copyright 2021 Xiaomi Corp. (author: Fangjun Kuang) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +CONDA_ENV_DIR=$CONDA_PREFIX + +echo "KALDIFEAT_PYTHON_VERSION: $KALDIFEAT_PYTHON_VERSION" +echo "KALDIFEAT_TORCH_VERSION: $KALDIFEAT_TORCH_VERSION" +echo "KALDIFEAT_CUDA_VERSION: $KALDIFEAT_CUDA_VERSION" +echo "KALDIFEAT_BUILD_TYPE: $KALDIFEAT_BUILD_TYPE" +echo "KALDIFEAT_BUILD_VERSION: $KALDIFEAT_BUILD_VERSION" +python3 --version + +echo "CC is: $CC" +echo "GCC is: $GCC" +echo "which nvcc: $(which nvcc)" +echo "gcc version: $($CC --version)" +echo "nvcc version: $(nvcc --version)" + +export KALDIFEAT_CMAKE_ARGS="-DCMAKE_BUILD_TYPE=${KALDIFEAT_BUILD_TYPE}" +export KALDIFEAT_MAKE_ARGS="-j2" + +python3 setup.py install --single-version-externally-managed --record=record.txt diff --git a/scripts/conda/kaldifeat/meta.yaml b/scripts/conda/kaldifeat/meta.yaml new file mode 100644 index 0000000..3b1c39a --- /dev/null +++ b/scripts/conda/kaldifeat/meta.yaml @@ -0,0 +1,39 @@ +package: + name: kaldifeat + version: "1.5.4" + +source: + path: "{{ environ.get('KALDIFEAT_ROOT_DIR') }}" + +build: + number: 0 + string: cuda{{ environ.get('KALDIFEAT_CUDA_VERSION') }}_py{{ environ.get('KALDIFEAT_PYTHON_VERSION') }}_torch{{ environ.get('KALDIFEAT_TORCH_VERSION') }} + script_env: + - KALDIFEAT_IS_GITHUB_ACTIONS + - KALDIFEAT_CUDA_VERSION + - KALDIFEAT_TORCH_VERSION + - KALDIFEAT_PYTHON_VERSION + - KALDIFEAT_BUILD_TYPE + - KALDIFEAT_BUILD_VERSION + - KALDIFEAT_IS_FOR_CONDA + +requirements: + host: + - cmake=3.18 + - python + - pytorch={{ environ.get('KALDIFEAT_TORCH_VERSION') }} + - cudatoolkit={{ environ.get('KALDIFEAT_CUDA_VERSION') }} + - gcc_linux-64=7 + run: + - python + - pytorch={{ environ.get('KALDIFEAT_TORCH_VERSION') }} + - cudatoolkit={{ environ.get('KALDIFEAT_CUDA_VERSION') }} + +about: + home: https://github.com/csukuangfj/kaldifeat + license: Apache V2 + license_file: LICENSE + summary: Kaldi-compatible feature extraction with PyTorch + description: | + Kaldi-compatible feature extraction with PyTorch, + supporting CUDA, batch processing, chunk processing, and autograd