Support building conda packages. (#54)

* Build CPU version for linux

* Build conda package for macos

* Support windows

.github/workflows/build_conda_macos_cpu.yml

@@ -0,0 +1,111 @@
+# Copyright      2021  Xiaomi Corp.       (author: Fangjun Kuang)
+
+# See ../../LICENSE for clarification regarding multiple authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# refer to https://github.com/actions/starter-workflows/pull/47/files
+
+name: build_conda_macos_cpu
+
+on:
+  push:
+    tags:
+      - '*'
+
+env:
+  KALDIFEAT_BUILD_TYPE: Release
+
+jobs:
+  generate_build_matrix:
+    # see https://github.com/pytorch/pytorch/pull/50633
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Generating build matrix
+        id: set-matrix
+        run: |
+          # outputting for debugging purposes
+          python scripts/github_actions/generate_build_matrix.py
+          MATRIX=$(python scripts/github_actions/generate_build_matrix.py)
+          echo "::set-output name=matrix::${MATRIX}"
+
+  build_conda_macos_cpu:
+    needs: generate_build_matrix
+    runs-on: macos-10.15
+    strategy:
+      fail-fast: false
+      matrix:
+        ${{ fromJson(needs.generate_build_matrix.outputs.matrix) }}
+
+    steps:
+      # refer to https://github.com/actions/checkout
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: Display clang version
+        run: |
+          clang --version
+
+      - uses: conda-incubator/setup-miniconda@v2
+        with:
+          auto-update-conda: true
+          python-version: ${{ matrix.python-version }}
+          activate-environment: kaldifeat
+
+      - name: Display Python version
+        shell: bash -l {0}
+        run: |
+          python3 -c "import sys; print(sys.version)"
+          which python3
+
+      - name: Install conda dependencies
+        shell: bash -l {0}
+        run: |
+          conda install -y -q anaconda-client
+          conda install -y -q conda-build
+          conda install -y -q -c pytorch pytorch=${{ matrix.torch }} cpuonly
+
+      - name: Display conda info
+        shell: bash -l {0}
+        run: |
+          which conda
+          conda env list
+          conda info
+
+      - name: Build kaldifeat
+        shell: bash -l {0}
+        env:
+          KALDIFEAT_PYTHON_VERSION: ${{ matrix.python-version}}
+          KALDIFEAT_TORCH_VERSION: ${{ matrix.torch }}
+          KALDIFEAT_CONDA_TOKEN: ${{ secrets.KALDIFEAT_CONDA_TOKEN}}
+          KALDIFEAT_IS_GITHUB_ACTIONS: 1
+          KALDIFEAT_IS_FOR_CONDA: 1
+        run: |
+          export KALDIFEAT_BUILD_TYPE=$KALDIFEAT_BUILD_TYPE
+          ./scripts/build_conda_cpu.sh
+
+      - name: Display generated files
+        run: |
+          ls -lh /usr/local/miniconda/envs/kaldifeat/conda-bld/osx-64
+
+      - name: Upload generated files
+        uses: actions/upload-artifact@v2
+        with:
+          name: cpu-torch-${{ matrix.torch }}-python-${{ matrix.python-version }}-macos-10.15
+          path: /usr/local/miniconda/envs/kaldifeat/conda-bld/osx-64/*.tar.bz2

.github/workflows/build_conda_ubuntu_cpu.yml

@@ -104,5 +104,5 @@ jobs:
       - name: Upload generated files
         uses: actions/upload-artifact@v2
         with:
-          name: cpu-torch-${{ matrix.torch }}-python-${{ matrix.python-version }}-${{ matrix.os }}
+          name: cpu-torch-${{ matrix.torch }}-python-${{ matrix.python-version }}-ubuntu-18.04
           path: /usr/share/miniconda/envs/kaldifeat/conda-bld/linux-64/*.tar.bz2

.github/workflows/build_conda_ubuntu_cuda.yml

@@ -128,5 +128,5 @@ jobs:
       - name: Upload generated files
         uses: actions/upload-artifact@v2
         with:
-          name: cuda-${{ matrix.cuda }}-torch-${{ matrix.torch }}-python-${{ matrix.python-version }}-${{ matrix.os }}
+          name: cuda-${{ matrix.cuda }}-torch-${{ matrix.torch }}-python-${{ matrix.python-version }}-ubuntu-18.04
           path: /usr/share/miniconda/envs/kaldifeat/conda-bld/linux-64/*.tar.bz2

.github/workflows/build_conda_windows_cpu.yml

@@ -0,0 +1,113 @@
+# Copyright      2021  Xiaomi Corp.       (author: Fangjun Kuang)
+
+# See ../../LICENSE for clarification regarding multiple authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# refer to https://github.com/actions/starter-workflows/pull/47/files
+
+name: build_conda_windows_cpu
+
+on:
+  push:
+    tags:
+      - '*'
+
+env:
+  KALDIFEAT_BUILD_TYPE: Release
+
+jobs:
+  generate_build_matrix:
+    # see https://github.com/pytorch/pytorch/pull/50633
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Generating build matrix
+        id: set-matrix
+        run: |
+          # outputting for debugging purposes
+          python scripts/github_actions/generate_build_matrix.py
+          MATRIX=$(python scripts/github_actions/generate_build_matrix.py)
+          echo "::set-output name=matrix::${MATRIX}"
+
+  build_conda_windows_cpu:
+    needs: generate_build_matrix
+    runs-on: windows-2019
+    strategy:
+      fail-fast: false
+      matrix:
+        ${{ fromJson(needs.generate_build_matrix.outputs.matrix) }}
+
+    steps:
+      # refer to https://github.com/actions/checkout
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - uses: conda-incubator/setup-miniconda@v2
+        with:
+          auto-update-conda: true
+          python-version: ${{ matrix.python-version }}
+          activate-environment: kaldifeat
+
+      - name: Install conda dependencies
+        shell: bash -l {0}
+        run: |
+          conda install -y -q anaconda-client
+          conda install -y -q conda-build
+          conda install -y -q -c pytorch pytorch=${{ matrix.torch }} cpuonly
+
+      - name: Display Python version
+        shell: bash -l {0}
+        run: |
+          python -c "import sys; print(sys.version)"
+          which python
+
+      - name: Display conda info
+        shell: bash -l {0}
+        run: |
+          conda env list
+          conda info
+          which conda
+          python --version
+          which python
+          python -m torch.utils.collect_env
+
+      - name: Build kaldifeat
+        shell: bash -l {0}
+        env:
+          KALDIFEAT_PYTHON_VERSION: ${{ matrix.python-version}}
+          KALDIFEAT_TORCH_VERSION: ${{ matrix.torch }}
+          KALDIFEAT_CONDA_TOKEN: ${{ secrets.KALDIFEAT_CONDA_TOKEN}}
+          KALDIFEAT_IS_GITHUB_ACTIONS: 1
+          KALDIFEAT_IS_FOR_CONDA: 1
+        run: |
+          export KALDIFEAT_BUILD_TYPE=$KALDIFEAT_BUILD_TYPE
+          ./scripts/build_conda_cpu.sh
+
+      - name: Display generated files
+        shell: bash -l {0}
+        run: |
+          ls -lh /c/Miniconda/envs/kaldifeat/conda-bld
+          ls -lh /c/Miniconda/envs/kaldifeat/conda-bld/*/*
+          ls -lh /c/Miniconda/envs/kaldifeat/conda-bld/win-64/*
+
+      - name: Upload generated files
+        uses: actions/upload-artifact@v2
+        with:
+          name: cpu-torch-${{ matrix.torch }}-python-${{ matrix.python-version }}-windows-2019
+          path: c:/Miniconda/envs/kaldifeat/conda-bld/win-64/*.tar.bz2

CMakeLists.txt

@@ -76,8 +76,6 @@ include_directories(${CMAKE_SOURCE_DIR})
 
 message(STATUS "CMAKE_CXX_FLAGS: ${CMAKE_CXX_FLAGS}")
 
-include(GNUInstallDirs)
-
 message(STATUS "CMAKE_INSTALL_PREFIX: ${CMAKE_INSTALL_PREFIX}")
 
 add_subdirectory(kaldifeat)

kaldifeat/csrc/CMakeLists.txt

@@ -18,6 +18,16 @@ target_link_libraries(kaldifeat_core PUBLIC ${TORCH_LIBRARIES})
 target_compile_definitions(kaldifeat_core PUBLIC KALDIFEAT_TORCH_VERSION_MAJOR=${KALDIFEAT_TORCH_VERSION_MAJOR})
 target_compile_definitions(kaldifeat_core PUBLIC KALDIFEAT_TORCH_VERSION_MINOR=${KALDIFEAT_TORCH_VERSION_MINOR})
 
+if(APPLE)
+  execute_process(
+    COMMAND "${PYTHON_EXECUTABLE}" -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())"
+    OUTPUT_STRIP_TRAILING_WHITESPACE
+    OUTPUT_VARIABLE PYTHON_SITE_PACKAGE_DIR
+  )
+  message(STATUS "PYTHON_SITE_PACKAGE_DIR: ${PYTHON_SITE_PACKAGE_DIR}")
+  target_link_libraries(kaldifeat_core PUBLIC "-L ${PYTHON_SITE_PACKAGE_DIR}/../..")
+endif()
+
 add_executable(test_kaldifeat test_kaldifeat.cc)
 target_link_libraries(test_kaldifeat PRIVATE kaldifeat_core)
 

scripts/build_conda.sh

@@ -55,7 +55,7 @@ cd $kaldifeat_dir
 export KALDIFEAT_ROOT_DIR=$kaldifeat_dir
 echo "KALDIFEAT_ROOT_DIR: $KALDIFEAT_ROOT_DIR"
 
-KALDIFEAT_PYTHON_VERSION=$(python3 -c "import sys; print(sys.version[:3])")
+KALDIFEAT_PYTHON_VERSION=$(python -c "import sys; print('.'.join(sys.version.split('.')[:2]))")
 
 if [ -z $KALDIFEAT_CUDA_VERSION ]; then
   echo "env var KALDIFEAT_CUDA_VERSION is not set, defaults to 10.1"

scripts/build_conda_cpu.sh

@@ -43,7 +43,7 @@ cd $kaldifeat_dir
 export KALDIFEAT_ROOT_DIR=$kaldifeat_dir
 echo "KALDIFEAT_ROOT_DIR: $KALDIFEAT_ROOT_DIR"
 
-KALDIFEAT_PYTHON_VERSION=$(python3 -c "import sys; print(sys.version[:3])")
+KALDIFEAT_PYTHON_VERSION=$(python -c "import sys; print('.'.join(sys.version.split('.')[:2]))")
 
 if [ -z $KALDIFEAT_TORCH_VERSION ]; then
   echo "env var KALDIFEAT_TORCH_VERSION is not set, defaults to 1.7.1"
@@ -75,7 +75,7 @@ fi
 
 if [ -z $KALDIFEAT_CONDA_TOKEN ]; then
   echo "Auto upload to anaconda.org is disabled since KALDIFEAT_CONDA_TOKEN is not set"
-  conda build --no-test --no-anaconda-upload -c pytorch -c conda-forge ./scripts/conda-cpu/kaldifeat
+  conda build --no-test --no-anaconda-upload -c pytorch ./scripts/conda-cpu/kaldifeat
 else
-  conda build --no-test -c pytorch -c conda-forge --token $KALDIFEAT_CONDA_TOKEN ./scripts/conda-cpu/kaldifeat
+  conda build --no-test -c pytorch --token $KALDIFEAT_CONDA_TOKEN ./scripts/conda-cpu/kaldifeat
 fi

scripts/conda-cpu/kaldifeat/build.sh

@@ -1,33 +0,0 @@
-#!/usr/bin/env bash
-#
-# Copyright      2021  Xiaomi Corp.       (author: Fangjun Kuang)
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -ex
-
-CONDA_ENV_DIR=$CONDA_PREFIX
-
-echo "KALDIFEAT_PYTHON_VERSION: $KALDIFEAT_PYTHON_VERSION"
-echo "KALDIFEAT_TORCH_VERSION: $KALDIFEAT_TORCH_VERSION"
-echo "KALDIFEAT_BUILD_TYPE: $KALDIFEAT_BUILD_TYPE"
-echo "KALDIFEAT_BUILD_VERSION: $KALDIFEAT_BUILD_VERSION"
-
-export KALDIFEAT_CMAKE_ARGS="-DCMAKE_BUILD_TYPE=${KALDIFEAT_BUILD_TYPE}"
-export KALDIFEAT_MAKE_ARGS="-j1 VERBOSE=1"
-
-export LIBRARY_PATH="/usr/local/miniconda/envs/kaldifeat/lib":$LIBRARY_PATH
-export LD_LIBRARY_PATH="/usr/local/miniconda/envs/kaldifeat/lib":$LD_LIBRARY_PATH
-export DYLD_LIBRARY_PATH="/usr/local/miniconda/envs/kaldifeat/lib":$DYLD_LIBRARY_PATH
-
-python3 setup.py install --single-version-externally-managed --record=record.txt

scripts/conda-cpu/kaldifeat/meta.yaml

@@ -8,13 +8,9 @@ source:
 build:
   number: 0
   string: cpu_py{{ environ.get('KALDIFEAT_PYTHON_VERSION') }}_torch{{ environ.get('KALDIFEAT_TORCH_VERSION') }}
-  script_env:
+  script: conda install -y -q -c pytorch pytorch={{ environ.get('KALDIFEAT_TORCH_VERSION') }} cpuonly & {{ PYTHON }} setup.py install --single-version-externally-managed --record=record.txt
-    - KALDIFEAT_IS_GITHUB_ACTIONS
+  features:
-    - KALDIFEAT_TORCH_VERSION
+    - cpuonly # [not osx]
-    - KALDIFEAT_PYTHON_VERSION
-    - KALDIFEAT_BUILD_TYPE
-    - KALDIFEAT_BUILD_VERSION
-    - KALDIFEAT_IS_FOR_CONDA
 
 requirements:
   build:
@@ -22,17 +18,15 @@ requirements:
     - {{ compiler('cxx') }} # [win]
 
   host:
-    - cmake=3.18
+    - anaconda-client
+    - cmake
     - python
     - pytorch={{ environ.get('KALDIFEAT_TORCH_VERSION') }}
-    - gcc_linux-64=7 # [linux]
     - cpuonly
-    - numpy
 
   run:
     - python
     - pytorch={{ environ.get('KALDIFEAT_TORCH_VERSION') }}
-    - numpy
 
 about:
   home: https://github.com/csukuangfj/kaldifeat

scripts/conda/kaldifeat/build.sh

@@ -32,6 +32,6 @@ echo "gcc version: $($CC --version)"
 echo "nvcc version: $(nvcc --version)"
 
 export KALDIFEAT_CMAKE_ARGS="-DCMAKE_BUILD_TYPE=${KALDIFEAT_BUILD_TYPE}"
-export KALDIFEAT_MAKE_ARGS="-j3"
+export KALDIFEAT_MAKE_ARGS="-j"
 
 python3 setup.py install --single-version-externally-managed --record=record.txt

scripts/github_actions/generate_build_matrix.py

@@ -118,6 +118,12 @@ def generate_build_matrix(enable_cuda, for_windows, test_only_latest_torch):
             if not for_windows
             else ["11.3.1", "11.6.2"],
         },
+        "1.12.1": {
+            "python-version": ["3.7", "3.8", "3.9", "3.10"],
+            "cuda": ["10.2", "11.3", "11.6"]
+            if not for_windows
+            else ["11.3.1", "11.6.2"],
+        },
     }
     if test_only_latest_torch:
         latest = "1.12.0"