diff --git a/.github/workflows/build_conda_macos_cpu.yml b/.github/workflows/build_conda_macos_cpu.yml new file mode 100644 index 0000000..810c413 --- /dev/null +++ b/.github/workflows/build_conda_macos_cpu.yml @@ -0,0 +1,111 @@ +# Copyright 2021 Xiaomi Corp. (author: Fangjun Kuang) + +# See ../../LICENSE for clarification regarding multiple authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# refer to https://github.com/actions/starter-workflows/pull/47/files + +name: build_conda_macos_cpu + +on: + push: + tags: + - '*' + +env: + KALDIFEAT_BUILD_TYPE: Release + +jobs: + generate_build_matrix: + # see https://github.com/pytorch/pytorch/pull/50633 + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Generating build matrix + id: set-matrix + run: | + # outputting for debugging purposes + python scripts/github_actions/generate_build_matrix.py + MATRIX=$(python scripts/github_actions/generate_build_matrix.py) + echo "::set-output name=matrix::${MATRIX}" + + build_conda_macos_cpu: + needs: generate_build_matrix + runs-on: macos-10.15 + strategy: + fail-fast: false + matrix: + ${{ fromJson(needs.generate_build_matrix.outputs.matrix) }} + + steps: + # refer to https://github.com/actions/checkout + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Display clang version + run: | + clang --version + + - uses: conda-incubator/setup-miniconda@v2 + with: + auto-update-conda: true + python-version: ${{ matrix.python-version }} + activate-environment: kaldifeat + + - name: Display Python version + shell: bash -l {0} + run: | + python3 -c "import sys; print(sys.version)" + which python3 + + - name: Install conda dependencies + shell: bash -l {0} + run: | + conda install -y -q anaconda-client + conda install -y -q conda-build + conda install -y -q -c pytorch pytorch=${{ matrix.torch }} cpuonly + + - name: Display conda info + shell: bash -l {0} + run: | + which conda + conda env list + conda info + + - name: Build kaldifeat + shell: bash -l {0} + env: + KALDIFEAT_PYTHON_VERSION: ${{ matrix.python-version}} + KALDIFEAT_TORCH_VERSION: ${{ matrix.torch }} + KALDIFEAT_CONDA_TOKEN: ${{ secrets.KALDIFEAT_CONDA_TOKEN}} + KALDIFEAT_IS_GITHUB_ACTIONS: 1 + KALDIFEAT_IS_FOR_CONDA: 1 + run: | + export KALDIFEAT_BUILD_TYPE=$KALDIFEAT_BUILD_TYPE + ./scripts/build_conda_cpu.sh + + - name: Display generated files + run: | + ls -lh /usr/local/miniconda/envs/kaldifeat/conda-bld/osx-64 + + - name: Upload generated files + uses: actions/upload-artifact@v2 + with: + name: cpu-torch-${{ matrix.torch }}-python-${{ matrix.python-version }}-macos-10.15 + path: /usr/local/miniconda/envs/kaldifeat/conda-bld/osx-64/*.tar.bz2 diff --git a/.github/workflows/build_conda_ubuntu_cpu.yml b/.github/workflows/build_conda_ubuntu_cpu.yml index 3f0692c..f14ce10 100644 --- a/.github/workflows/build_conda_ubuntu_cpu.yml +++ b/.github/workflows/build_conda_ubuntu_cpu.yml @@ -104,5 +104,5 @@ jobs: - name: Upload generated files uses: actions/upload-artifact@v2 with: - name: cpu-torch-${{ matrix.torch }}-python-${{ matrix.python-version }}-${{ matrix.os }} + name: cpu-torch-${{ matrix.torch }}-python-${{ matrix.python-version }}-ubuntu-18.04 path: /usr/share/miniconda/envs/kaldifeat/conda-bld/linux-64/*.tar.bz2 diff --git a/.github/workflows/build_conda_ubuntu_cuda.yml b/.github/workflows/build_conda_ubuntu_cuda.yml index b604a62..5c587f6 100644 --- a/.github/workflows/build_conda_ubuntu_cuda.yml +++ b/.github/workflows/build_conda_ubuntu_cuda.yml @@ -128,5 +128,5 @@ jobs: - name: Upload generated files uses: actions/upload-artifact@v2 with: - name: cuda-${{ matrix.cuda }}-torch-${{ matrix.torch }}-python-${{ matrix.python-version }}-${{ matrix.os }} + name: cuda-${{ matrix.cuda }}-torch-${{ matrix.torch }}-python-${{ matrix.python-version }}-ubuntu-18.04 path: /usr/share/miniconda/envs/kaldifeat/conda-bld/linux-64/*.tar.bz2 diff --git a/.github/workflows/build_conda_windows_cpu.yml b/.github/workflows/build_conda_windows_cpu.yml new file mode 100644 index 0000000..b95c98c --- /dev/null +++ b/.github/workflows/build_conda_windows_cpu.yml @@ -0,0 +1,113 @@ +# Copyright 2021 Xiaomi Corp. (author: Fangjun Kuang) + +# See ../../LICENSE for clarification regarding multiple authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# refer to https://github.com/actions/starter-workflows/pull/47/files + +name: build_conda_windows_cpu + +on: + push: + tags: + - '*' + +env: + KALDIFEAT_BUILD_TYPE: Release + +jobs: + generate_build_matrix: + # see https://github.com/pytorch/pytorch/pull/50633 + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Generating build matrix + id: set-matrix + run: | + # outputting for debugging purposes + python scripts/github_actions/generate_build_matrix.py + MATRIX=$(python scripts/github_actions/generate_build_matrix.py) + echo "::set-output name=matrix::${MATRIX}" + + build_conda_windows_cpu: + needs: generate_build_matrix + runs-on: windows-2019 + strategy: + fail-fast: false + matrix: + ${{ fromJson(needs.generate_build_matrix.outputs.matrix) }} + + steps: + # refer to https://github.com/actions/checkout + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - uses: conda-incubator/setup-miniconda@v2 + with: + auto-update-conda: true + python-version: ${{ matrix.python-version }} + activate-environment: kaldifeat + + - name: Install conda dependencies + shell: bash -l {0} + run: | + conda install -y -q anaconda-client + conda install -y -q conda-build + conda install -y -q -c pytorch pytorch=${{ matrix.torch }} cpuonly + + - name: Display Python version + shell: bash -l {0} + run: | + python -c "import sys; print(sys.version)" + which python + + - name: Display conda info + shell: bash -l {0} + run: | + conda env list + conda info + which conda + python --version + which python + python -m torch.utils.collect_env + + - name: Build kaldifeat + shell: bash -l {0} + env: + KALDIFEAT_PYTHON_VERSION: ${{ matrix.python-version}} + KALDIFEAT_TORCH_VERSION: ${{ matrix.torch }} + KALDIFEAT_CONDA_TOKEN: ${{ secrets.KALDIFEAT_CONDA_TOKEN}} + KALDIFEAT_IS_GITHUB_ACTIONS: 1 + KALDIFEAT_IS_FOR_CONDA: 1 + run: | + export KALDIFEAT_BUILD_TYPE=$KALDIFEAT_BUILD_TYPE + ./scripts/build_conda_cpu.sh + + - name: Display generated files + shell: bash -l {0} + run: | + ls -lh /c/Miniconda/envs/kaldifeat/conda-bld + ls -lh /c/Miniconda/envs/kaldifeat/conda-bld/*/* + ls -lh /c/Miniconda/envs/kaldifeat/conda-bld/win-64/* + + - name: Upload generated files + uses: actions/upload-artifact@v2 + with: + name: cpu-torch-${{ matrix.torch }}-python-${{ matrix.python-version }}-windows-2019 + path: c:/Miniconda/envs/kaldifeat/conda-bld/win-64/*.tar.bz2 diff --git a/CMakeLists.txt b/CMakeLists.txt index 1709a39..7515920 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -76,8 +76,6 @@ include_directories(${CMAKE_SOURCE_DIR}) message(STATUS "CMAKE_CXX_FLAGS: ${CMAKE_CXX_FLAGS}") -include(GNUInstallDirs) - message(STATUS "CMAKE_INSTALL_PREFIX: ${CMAKE_INSTALL_PREFIX}") add_subdirectory(kaldifeat) diff --git a/kaldifeat/csrc/CMakeLists.txt b/kaldifeat/csrc/CMakeLists.txt index 7a52a7d..2e140a6 100644 --- a/kaldifeat/csrc/CMakeLists.txt +++ b/kaldifeat/csrc/CMakeLists.txt @@ -18,6 +18,16 @@ target_link_libraries(kaldifeat_core PUBLIC ${TORCH_LIBRARIES}) target_compile_definitions(kaldifeat_core PUBLIC KALDIFEAT_TORCH_VERSION_MAJOR=${KALDIFEAT_TORCH_VERSION_MAJOR}) target_compile_definitions(kaldifeat_core PUBLIC KALDIFEAT_TORCH_VERSION_MINOR=${KALDIFEAT_TORCH_VERSION_MINOR}) +if(APPLE) + execute_process( + COMMAND "${PYTHON_EXECUTABLE}" -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())" + OUTPUT_STRIP_TRAILING_WHITESPACE + OUTPUT_VARIABLE PYTHON_SITE_PACKAGE_DIR + ) + message(STATUS "PYTHON_SITE_PACKAGE_DIR: ${PYTHON_SITE_PACKAGE_DIR}") + target_link_libraries(kaldifeat_core PUBLIC "-L ${PYTHON_SITE_PACKAGE_DIR}/../..") +endif() + add_executable(test_kaldifeat test_kaldifeat.cc) target_link_libraries(test_kaldifeat PRIVATE kaldifeat_core) diff --git a/scripts/build_conda.sh b/scripts/build_conda.sh index f52ebf2..73fc45f 100755 --- a/scripts/build_conda.sh +++ b/scripts/build_conda.sh @@ -55,7 +55,7 @@ cd $kaldifeat_dir export KALDIFEAT_ROOT_DIR=$kaldifeat_dir echo "KALDIFEAT_ROOT_DIR: $KALDIFEAT_ROOT_DIR" -KALDIFEAT_PYTHON_VERSION=$(python3 -c "import sys; print(sys.version[:3])") +KALDIFEAT_PYTHON_VERSION=$(python -c "import sys; print('.'.join(sys.version.split('.')[:2]))") if [ -z $KALDIFEAT_CUDA_VERSION ]; then echo "env var KALDIFEAT_CUDA_VERSION is not set, defaults to 10.1" diff --git a/scripts/build_conda_cpu.sh b/scripts/build_conda_cpu.sh index 3d2c47a..7aa5983 100755 --- a/scripts/build_conda_cpu.sh +++ b/scripts/build_conda_cpu.sh @@ -43,7 +43,7 @@ cd $kaldifeat_dir export KALDIFEAT_ROOT_DIR=$kaldifeat_dir echo "KALDIFEAT_ROOT_DIR: $KALDIFEAT_ROOT_DIR" -KALDIFEAT_PYTHON_VERSION=$(python3 -c "import sys; print(sys.version[:3])") +KALDIFEAT_PYTHON_VERSION=$(python -c "import sys; print('.'.join(sys.version.split('.')[:2]))") if [ -z $KALDIFEAT_TORCH_VERSION ]; then echo "env var KALDIFEAT_TORCH_VERSION is not set, defaults to 1.7.1" @@ -75,7 +75,7 @@ fi if [ -z $KALDIFEAT_CONDA_TOKEN ]; then echo "Auto upload to anaconda.org is disabled since KALDIFEAT_CONDA_TOKEN is not set" - conda build --no-test --no-anaconda-upload -c pytorch -c conda-forge ./scripts/conda-cpu/kaldifeat + conda build --no-test --no-anaconda-upload -c pytorch ./scripts/conda-cpu/kaldifeat else - conda build --no-test -c pytorch -c conda-forge --token $KALDIFEAT_CONDA_TOKEN ./scripts/conda-cpu/kaldifeat + conda build --no-test -c pytorch --token $KALDIFEAT_CONDA_TOKEN ./scripts/conda-cpu/kaldifeat fi diff --git a/scripts/conda-cpu/kaldifeat/build.sh b/scripts/conda-cpu/kaldifeat/build.sh deleted file mode 100644 index 6e24b9d..0000000 --- a/scripts/conda-cpu/kaldifeat/build.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright 2021 Xiaomi Corp. (author: Fangjun Kuang) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -ex - -CONDA_ENV_DIR=$CONDA_PREFIX - -echo "KALDIFEAT_PYTHON_VERSION: $KALDIFEAT_PYTHON_VERSION" -echo "KALDIFEAT_TORCH_VERSION: $KALDIFEAT_TORCH_VERSION" -echo "KALDIFEAT_BUILD_TYPE: $KALDIFEAT_BUILD_TYPE" -echo "KALDIFEAT_BUILD_VERSION: $KALDIFEAT_BUILD_VERSION" - -export KALDIFEAT_CMAKE_ARGS="-DCMAKE_BUILD_TYPE=${KALDIFEAT_BUILD_TYPE}" -export KALDIFEAT_MAKE_ARGS="-j1 VERBOSE=1" - -export LIBRARY_PATH="/usr/local/miniconda/envs/kaldifeat/lib":$LIBRARY_PATH -export LD_LIBRARY_PATH="/usr/local/miniconda/envs/kaldifeat/lib":$LD_LIBRARY_PATH -export DYLD_LIBRARY_PATH="/usr/local/miniconda/envs/kaldifeat/lib":$DYLD_LIBRARY_PATH - -python3 setup.py install --single-version-externally-managed --record=record.txt diff --git a/scripts/conda-cpu/kaldifeat/meta.yaml b/scripts/conda-cpu/kaldifeat/meta.yaml index e922b5b..019c667 100644 --- a/scripts/conda-cpu/kaldifeat/meta.yaml +++ b/scripts/conda-cpu/kaldifeat/meta.yaml @@ -8,13 +8,9 @@ source: build: number: 0 string: cpu_py{{ environ.get('KALDIFEAT_PYTHON_VERSION') }}_torch{{ environ.get('KALDIFEAT_TORCH_VERSION') }} - script_env: - - KALDIFEAT_IS_GITHUB_ACTIONS - - KALDIFEAT_TORCH_VERSION - - KALDIFEAT_PYTHON_VERSION - - KALDIFEAT_BUILD_TYPE - - KALDIFEAT_BUILD_VERSION - - KALDIFEAT_IS_FOR_CONDA + script: conda install -y -q -c pytorch pytorch={{ environ.get('KALDIFEAT_TORCH_VERSION') }} cpuonly & {{ PYTHON }} setup.py install --single-version-externally-managed --record=record.txt + features: + - cpuonly # [not osx] requirements: build: @@ -22,17 +18,15 @@ requirements: - {{ compiler('cxx') }} # [win] host: - - cmake=3.18 + - anaconda-client + - cmake - python - pytorch={{ environ.get('KALDIFEAT_TORCH_VERSION') }} - - gcc_linux-64=7 # [linux] - cpuonly - - numpy run: - python - pytorch={{ environ.get('KALDIFEAT_TORCH_VERSION') }} - - numpy about: home: https://github.com/csukuangfj/kaldifeat diff --git a/scripts/conda/kaldifeat/build.sh b/scripts/conda/kaldifeat/build.sh index 4539872..488019b 100644 --- a/scripts/conda/kaldifeat/build.sh +++ b/scripts/conda/kaldifeat/build.sh @@ -32,6 +32,6 @@ echo "gcc version: $($CC --version)" echo "nvcc version: $(nvcc --version)" export KALDIFEAT_CMAKE_ARGS="-DCMAKE_BUILD_TYPE=${KALDIFEAT_BUILD_TYPE}" -export KALDIFEAT_MAKE_ARGS="-j3" +export KALDIFEAT_MAKE_ARGS="-j" python3 setup.py install --single-version-externally-managed --record=record.txt diff --git a/scripts/github_actions/generate_build_matrix.py b/scripts/github_actions/generate_build_matrix.py index 90e18cb..88c88bd 100755 --- a/scripts/github_actions/generate_build_matrix.py +++ b/scripts/github_actions/generate_build_matrix.py @@ -118,6 +118,12 @@ def generate_build_matrix(enable_cuda, for_windows, test_only_latest_torch): if not for_windows else ["11.3.1", "11.6.2"], }, + "1.12.1": { + "python-version": ["3.7", "3.8", "3.9", "3.10"], + "cuda": ["10.2", "11.3", "11.6"] + if not for_windows + else ["11.3.1", "11.6.2"], + }, } if test_only_latest_torch: latest = "1.12.0"