From 9488509091f3432da9423d543bf98dd946eefd46 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Fri, 16 Jul 2021 19:49:31 +0800 Subject: [PATCH] Support conda. --- .github/workflows/conda.yml | 159 +++++++++++++++++++++++++ .github/workflows/publish_to_pypi.yml | 16 +++ CMakeLists.txt | 1 + kaldifeat/python/kaldifeat/__init__.py | 1 + scripts/build_conda.sh | 104 ++++++++++++++++ scripts/conda/kaldifeat/build.sh | 37 ++++++ scripts/conda/kaldifeat/meta.yaml | 39 ++++++ 7 files changed, 357 insertions(+) create mode 100644 .github/workflows/conda.yml create mode 100755 scripts/build_conda.sh create mode 100644 scripts/conda/kaldifeat/build.sh create mode 100644 scripts/conda/kaldifeat/meta.yaml diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml new file mode 100644 index 0000000..0e2f86b --- /dev/null +++ b/.github/workflows/conda.yml @@ -0,0 +1,159 @@ +# Copyright 2021 Xiaomi Corp. (author: Fangjun Kuang) + +# See ../../LICENSE for clarification regarding multiple authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +name: build_conda + +on: + push: + branches: + - conda + pull_request: + branches: + - master + +jobs: + build_conda: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-16.04] + python-version: [3.6, 3.7, 3.8, 3.9] + cuda: ["10.1", "10.2", "11.0", "11.1"] + # from https://download.pytorch.org/whl/torch_stable.html + # + # PyTorch 1.9.0 supports: 10.2 (default), 11.1 + # PyTorch 1.8.1 supports: cuda 10.1, 10.2 (default), 11.1 + # PyTorch 1.8.0 supports: cuda 10.1, 10.2 (default), 11.1 + # PyTorch 1.7.x supports: cuda 10.1, 10.2 (default), 11.0, 9.2 (not included in this setup) + # PyTorch 1.6.0 supports: cuda 10.1, 10.2 (default), 9.2 (not included in this setup) + # PyTorch 1.5.x supports: cuda 10.1, 10.2 (default), 9.2 (not included in this setup) + # + # PyTorch 1.8.x and 1.7.1 support 3.6, 3.7, 3.8, 3.9 + # PyTorch 1.7.0, 1.6.0, and 1.5.x support 3.6, 3.7, 3.8 + # + # Other PyTorch versions are not tested + # + # torch: ["1.5.0", "1.5.1", "1.6.0", "1.7.0", "1.7.1", "1.8.0", "1.8.1"] + # 1.5.x is removed because there are compilation errors. + # See + # https://github.com/csukuangfj/k2/runs/2533830771?check_suite_focus=true + # and + # https://github.com/NVIDIA/apex/issues/805 + torch: ["1.6.0", "1.7.0", "1.7.1", "1.8.0", "1.8.1", "1.9.0"] + exclude: + # - cuda: "11.0" # exclude 11.0 for [1.5.0, 1.5.1, 1.6.0, 1.8.0, 1.8.1, 1.9.0] + # torch: "1.5.0" + # - cuda: "11.0" + # torch: "1.5.1" + - cuda: "11.0" + torch: "1.6.0" + - cuda: "11.0" + torch: "1.8.0" + - cuda: "11.0" + torch: "1.8.1" + - cuda: "11.0" + torch: "1.9.0" + # - cuda: "11.1" # exclude 11.1 for [1.5.0, 1.5.1, 1.6.0, 1.7.0, 1.7.1] + # torch: "1.5.0" + # - cuda: "11.1" + # torch: "1.5.1" + - cuda: "11.1" + torch: "1.6.0" + - cuda: "11.1" + torch: "1.7.0" + - cuda: "11.1" + torch: "1.7.1" + - cuda: "10.1" # exclude 10.1 for [1.9.0] + torch: "1.9.0" + - python-version: 3.9 # exclude Python 3.9 for [1.5.0, 1.5.1, 1.6.0, 1.7.0] + torch: "1.5.0" + - python-version: 3.9 + torch: "1.5.1" + - python-version: 3.9 + torch: "1.6.0" + - python-version: 3.9 + torch: "1.7.0" + + steps: + # refer to https://github.com/actions/checkout + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Install CUDA Toolkit ${{ matrix.cuda }} + shell: bash -l {0} + env: + cuda: ${{ matrix.cuda }} + run: | + source ./scripts/github_actions/install_cuda.sh + echo "CUDA_HOME=${CUDA_HOME}" >> $GITHUB_ENV + echo "${CUDA_HOME}/bin" >> $GITHUB_PATH + echo "LD_LIBRARY_PATH=${CUDA_HOME}/lib:${CUDA_HOME}/lib64:${LD_LIBRARY_PATH}" >> $GITHUB_ENV + + - name: Display NVCC version + shell: bash -l {0} + run: | + which nvcc + nvcc --version + + - uses: conda-incubator/setup-miniconda@v2 + with: + auto-update-conda: true + python-version: ${{ matrix.python-version }} + activate-environment: kaldifeat + + - name: Display Python version + shell: bash -l {0} + run: | + python3 -c "import sys; print(sys.version)" + which python3 + + - name: Install conda dependencies + shell: bash -l {0} + run: | + conda install -y -q anaconda-client + conda install -y -q conda-build + conda install -y -q bs4 requests tqdm + conda install -y -q -c pytorch -c conda-forge pytorch=${{ matrix.torch }} cudatoolkit=${{ matrix.cuda }} + + - name: Display conda info + shell: bash -l {0} + run: | + which conda + conda env list + conda info + nproc + + - name: Download cudnn 8.0 + shell: bash -l {0} + env: + cuda: ${{ matrix.cuda }} + run: | + ./scripts/github_actions/install_cudnn.sh + + - name: Build kaldifeat + shell: bash -l {0} + env: + KALDIFEAT_CUDA_VERSION: ${{ matrix.cuda }} + KALDIFEAT_PYTHON_VERSION: ${{ matrix.python-version}} + KALDIFEAT_TORCH_VERSION: ${{ matrix.torch }} + KALDIFEAT_CONDA_TOKEN: ${{ secrets.KALDIFEAT_CONDA_TOKEN}} + KALDIFEAT_IS_GITHUB_ACTIONS: 1 + KALDIFEAT_IS_FOR_CONDA: 1 + run: | + ./scripts/build_conda.sh diff --git a/.github/workflows/publish_to_pypi.yml b/.github/workflows/publish_to_pypi.yml index 8527eb5..8e4e535 100644 --- a/.github/workflows/publish_to_pypi.yml +++ b/.github/workflows/publish_to_pypi.yml @@ -1,3 +1,19 @@ +# Copyright 2021 Xiaomi Corp. (author: Fangjun Kuang) + +# See ../../LICENSE for clarification regarding multiple authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# name: Publish to PyPI on: diff --git a/CMakeLists.txt b/CMakeLists.txt index 7ff965f..75d822f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,6 +4,7 @@ cmake_minimum_required(VERSION 3.8 FATAL_ERROR) project(kaldifeat) +# Also change KALDIFEAT_BUILD_VERSION in scripts/build_conda.sh set(kaldifeat_VERSION "0.2") set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib") diff --git a/kaldifeat/python/kaldifeat/__init__.py b/kaldifeat/python/kaldifeat/__init__.py index e177288..f86941f 100644 --- a/kaldifeat/python/kaldifeat/__init__.py +++ b/kaldifeat/python/kaldifeat/__init__.py @@ -1,3 +1,4 @@ +import torch from _kaldifeat import FbankOptions, FrameExtractionOptions, MelBanksOptions from .fbank import Fbank diff --git a/scripts/build_conda.sh b/scripts/build_conda.sh new file mode 100755 index 0000000..f6dcaa7 --- /dev/null +++ b/scripts/build_conda.sh @@ -0,0 +1,104 @@ +#!/usr/bin/env bash +# +# Copyright 2021 Xiaomi Corp. (author: Fangjun Kuang) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# To use this script, we assume that you have installed cudatoolkit locally. +# That is, `which nvcc` should give the path to nvcc +# +# We also assume that cudnn is installed locally. +# +# The following environment variables are supposed to be set by users +# +# - KALDIFEAT_CUDA_VERSION +# It represents the cuda version. Example: +# +# export KALDIFEAT_CUDA_VERSION=10.1 +# +# Defaults to 10.1 if not set. +# +# - KALDIFEAT_TORCH_VERSION +# The PyTorch version. Example: +# +# export KALDIFEAT_TORCH_VERSION=1.7.1 +# +# Defaults to 1.7.1 if not set. +# +# - KALDIFEAT_CONDA_TOKEN +# If not set, auto upload to anaconda.org is disabled. +# +# Its value is from https://anaconda.org/k2-fsa/settings/access +# (You need to login as user k2-fsa to see its value) +# +# - KALDIFEAT_BUILD_TYPE +# If not set, defaults to Release. + +set -e +export CONDA_BUILD=1 + +cur_dir=$(cd $(dirname $BASH_SOURCE) && pwd) +kaldifeat_dir=$(cd $cur_dir/.. && pwd) + +cd $kaldifeat_dir + +export KALDIFEAT_ROOT_DIR=$kaldifeat_dir +echo "KALDIFEAT_ROOT_DIR: $KALDIFEAT_ROOT_DIR" + +KALDIFEAT_PYTHON_VERSION=$(python3 -c "import sys; print(sys.version[:3])") + +if [ -z $KALDIFEAT_CUDA_VERSION ]; then + echo "env var KALDIFEAT_CUDA_VERSION is not set, defaults to 10.1" + KALDIFEAT_CUDA_VERSION=10.1 +fi + +if [ -z $KALDIFEAT_TORCH_VERSION ]; then + echo "env var KALDIFEAT_TORCH_VERSION is not set, defaults to 1.7.1" + KALDIFEAT_TORCH_VERSION=1.7.1 +fi + +if [ -z $KALDIFEAT_BUILD_TYPE ]; then + echo "env var KALDIFEAT_BUILD_TYPE is not set, defaults to Release" + KALDIFEAT_BUILD_TYPE=Release +fi + +export KALDIFEAT_IS_FOR_CONDA=1 +KALDIFEAT_BUILD_VERSION="0.2" + +# Example value: 3.8 +export KALDIFEAT_PYTHON_VERSION + +# Example value: 10.1 +export KALDIFEAT_CUDA_VERSION + +# Example value: 1.7.1 +export KALDIFEAT_TORCH_VERSION + +export KALDIFEAT_BUILD_VERSION + +export KALDIFEAT_BUILD_TYPE + +if [ ! -z $KALDIFEAT_IS_GITHUB_ACTIONS ]; then + export KALDIFEAT_IS_GITHUB_ACTIONS + conda remove -q pytorch cudatoolkit + conda clean -q -a +else + export KALDIFEAT_IS_GITHUB_ACTIONS=0 +fi + +if [ -z $KALDIFEAT_CONDA_TOKEN ]; then + echo "Auto upload to anaconda.org is disabled since KALDIFEAT_CONDA_TOKEN is not set" + conda build --no-test --no-anaconda-upload -c pytorch -c conda-forge ./scripts/conda/kaldifeat +else + conda build --no-test -c pytorch -c conda-forge --token $KALDIFEAT_CONDA_TOKEN ./scripts/conda/kaldifeat +fi diff --git a/scripts/conda/kaldifeat/build.sh b/scripts/conda/kaldifeat/build.sh new file mode 100644 index 0000000..3897511 --- /dev/null +++ b/scripts/conda/kaldifeat/build.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# +# Copyright 2021 Xiaomi Corp. (author: Fangjun Kuang) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +CONDA_ENV_DIR=$CONDA_PREFIX + +echo "KALDIFEAT_PYTHON_VERSION: $KALDIFEAT_PYTHON_VERSION" +echo "KALDIFEAT_TORCH_VERSION: $KALDIFEAT_TORCH_VERSION" +echo "KALDIFEAT_CUDA_VERSION: $KALDIFEAT_CUDA_VERSION" +echo "KALDIFEAT_BUILD_TYPE: $KALDIFEAT_BUILD_TYPE" +echo "KALDIFEAT_BUILD_VERSION: $KALDIFEAT_BUILD_VERSION" +python3 --version + +echo "CC is: $CC" +echo "GCC is: $GCC" +echo "which nvcc: $(which nvcc)" +echo "gcc version: $($CC --version)" +echo "nvcc version: $(nvcc --version)" + +export KALDIFEAT_CMAKE_ARGS="-DCMAKE_BUILD_TYPE=${KALDIFEAT_BUILD_TYPE}" +export KALDIFEAT_MAKE_ARGS="-j2" + +python3 setup.py install --single-version-externally-managed --record=record.txt diff --git a/scripts/conda/kaldifeat/meta.yaml b/scripts/conda/kaldifeat/meta.yaml new file mode 100644 index 0000000..8ef207a --- /dev/null +++ b/scripts/conda/kaldifeat/meta.yaml @@ -0,0 +1,39 @@ +package: + name: kaldifeat + version: "{{ environ.get('KALDIFEAT_BUILD_VERSION') }}" + +source: + path: "{{ environ.get('KALDIFEAT_ROOT_DIR') }}" + +build: + number: 0 + string: cuda{{ environ.get('KALDIFEAT_CUDA_VERSION') }}_py{{ environ.get('KALDIFEAT_PYTHON_VERSION') }}_torch{{ environ.get('KALDIFEAT_TORCH_VERSION') }} + script_env: + - KALDIFEAT_IS_GITHUB_ACTIONS + - KALDIFEAT_CUDA_VERSION + - KALDIFEAT_TORCH_VERSION + - KALDIFEAT_PYTHON_VERSION + - KALDIFEAT_BUILD_TYPE + - KALDIFEAT_BUILD_VERSION + - KALDIFEAT_IS_FOR_CONDA + +requirements: + host: + - cmake=3.18 + - python + - pytorch={{ environ.get('KALDIFEAT_TORCH_VERSION') }} + - cudatoolkit={{ environ.get('KALDIFEAT_CUDA_VERSION') }} + - gcc_linux-64=7 + run: + - python + - pytorch={{ environ.get('KALDIFEAT_TORCH_VERSION') }} + - cudatoolkit={{ environ.get('KALDIFEAT_CUDA_VERSION') }} + +about: + home: https://github.com/csukuangfj/kaldifeat + doc_url: https://github.com/csukuangfj/kaldifeat + license: Apache V2 + license_file: LICENSE + summary: Reimplementation of kaldi's feature computation in PyTorch. + description: | + Reimplementation of kaldi's feature computation in PyTorch.