Fix building for python

.github/workflows/ubuntu-arm64-cpu-wheels.yml

@@ -3,6 +3,7 @@ name: build-wheels-cpu-arm64-ubuntu
 on:
   push:
     branches:
+      # - wheel
       - fix-cuda-12.6
     tags:
       - '*'
@@ -26,11 +27,11 @@ jobs:
         id: set-matrix
         run: |
           # outputting for debugging purposes
-          # python ./scripts/github_actions/generate_build_matrix.py --for-arm64
+          python ./scripts/github_actions/generate_build_matrix.py --for-arm64
-          # MATRIX=$(python ./scripts/github_actions/generate_build_matrix.py --for-arm64)
+          MATRIX=$(python ./scripts/github_actions/generate_build_matrix.py --for-arm64)
 
-          python ./scripts/github_actions/generate_build_matrix.py --test-only-latest-torch --for-arm64
+          # python ./scripts/github_actions/generate_build_matrix.py --test-only-latest-torch --for-arm64
-          MATRIX=$(python ./scripts/github_actions/generate_build_matrix.py --test-only-latest-torch --for-arm64)
+          # MATRIX=$(python ./scripts/github_actions/generate_build_matrix.py --test-only-latest-torch --for-arm64)
           echo "::set-output name=matrix::${MATRIX}"
 
   build-manylinux-wheels:
@@ -125,9 +126,6 @@ jobs:
               export PYTHON_INSTALL_DIR=$python_dir
               export PATH=$PYTHON_INSTALL_DIR/bin:$PATH
 
-              # There are no libpython.so inside $PYTHON_INSTALL_DIR
-              # since they are statically linked.
-
               python3 --version
               which python3
 
@@ -160,7 +158,7 @@ jobs:
             cd huggingface
             git pull
 
-            d=cpu/1.25.5.dev20250306/linux-arm64
+            d=cpu/1.25.5.dev20250307/linux-arm64
             mkdir -p $d
             cp -v ../wheelhouse/*.whl ./$d
             git status

.github/workflows/ubuntu-cpu-wheels.yml

@@ -3,7 +3,8 @@ name: build-wheels-cpu-ubuntu
 on:
   push:
     branches:
-      - fix-wheel-2
+      # - fix-cuda-12.6
+      - wheel
     tags:
       - '*'
   workflow_dispatch:
@@ -56,7 +57,7 @@ jobs:
         uses: addnab/docker-run-action@v3
         with:
             image: ${{ matrix.image }}
-            options: -v ${{ github.workspace }}:/var/www -e PYTHON_VERSION=${{ matrix.python-version }} -e TORCH_VERSION=${{ matrix.torch }}
+            options: -v ${{ github.workspace }}:/var/www -e IS_2_28=${{ matrix.is_2_28 }} -e PYTHON_VERSION=${{ matrix.python-version }} -e TORCH_VERSION=${{ matrix.torch }}
             run: |
               echo "pwd: $PWD"
               uname -a
@@ -65,6 +66,69 @@ jobs:
               gcc --version
               python3 --version
               which python3
+
+              ls -lh /opt/python/
+
+              echo "---"
+
+              ls -lh /opt/python/cp*
+
+              ls -lh /opt/python/*/bin
+
+              echo "---"
+              find /opt/python/cp* -name "libpython*"
+              echo "-----"
+              find /opt/_internal/cp* -name "libpython*"
+
+              echo "-----"
+              find / -name "libpython*"
+              echo "----"
+              ls -lh /usr/lib64/libpython3.so || true
+
+              # cp36-cp36m
+              # cp37-cp37m
+              # cp38-cp38
+              # cp39-cp39
+              # cp310-cp310
+              # cp311-cp311
+              # cp312-cp312
+              # cp313-cp313
+              # cp313-cp313t  (no gil)
+              if [[ $PYTHON_VERSION == "3.6" ]]; then
+                python_dir=/opt/python/cp36-cp36m
+                export PYTHONPATH=/opt/python/cp36-cp36m/lib/python3.6/site-packages:$PYTHONPATH
+              elif [[ $PYTHON_VERSION == "3.7" ]]; then
+                python_dir=/opt/python/cp37-cp37m
+                export PYTHONPATH=/opt/python/cp37-cp37m/lib/python3.7/site-packages:$PYTHONPATH
+              elif [[ $PYTHON_VERSION == "3.8" ]]; then
+                python_dir=/opt/python/cp38-cp38
+                export PYTHONPATH=/opt/python/cp38-cp38/lib/python3.8/site-packages:$PYTHONPATH
+              elif [[ $PYTHON_VERSION == "3.9" ]]; then
+                python_dir=/opt/python/cp39-cp39
+                export PYTHONPATH=/opt/python/cp39-cp39/lib/python3.9/site-packages:$PYTHONPATH
+              elif [[ $PYTHON_VERSION == "3.10" ]]; then
+                python_dir=/opt/python/cp310-cp310
+                export PYTHONPATH=/opt/python/cp310-cp310/lib/python3.10/site-packages:$PYTHONPATH
+              elif [[ $PYTHON_VERSION == "3.11" ]]; then
+                python_dir=/opt/python/cp311-cp311
+                export PYTHONPATH=/opt/python/cp311-cp311/lib/python3.11/site-packages:$PYTHONPATH
+              elif [[ $PYTHON_VERSION == "3.12" ]]; then
+                python_dir=/opt/python/cp312-cp312
+                export PYTHONPATH=/opt/python/cp312-cp312/lib/python3.12/site-packages:$PYTHONPATH
+              elif [[ $PYTHON_VERSION == "3.13" ]]; then
+                python_dir=/opt/python/cp313-cp313
+                export PYTHONPATH=/opt/python/cp313-cp313/lib/python3.13/site-packages:$PYTHONPATH
+              else
+                echo "Unsupported Python version $PYTHON_VERSION"
+                exit 1
+              fi
+
+              export PYTHON_INSTALL_DIR=$python_dir
+              export PATH=$PYTHON_INSTALL_DIR/bin:$PATH
+
+              python3 --version
+              which python3
+
               /var/www/scripts/github_actions/build-ubuntu-cpu.sh
 
       - name: Display wheels
@@ -72,12 +136,6 @@ jobs:
         run: |
           ls -lh ./wheelhouse/
 
-      - name: Upload Wheel
-        uses: actions/upload-artifact@v4
-        with:
-          name: torch-${{ matrix.torch }}-python-${{ matrix.python-version }}-ubuntu-latest-cpu
-          path: wheelhouse/*.whl
-
       # https://huggingface.co/docs/hub/spaces-github-actions
       - name: Publish to huggingface
         if: github.repository_owner == 'csukuangfj'
@@ -100,7 +158,7 @@ jobs:
             cd huggingface
             git pull
 
-            d=cpu/1.25.5.dev20241029/linux
+            d=cpu/1.25.5.dev20250307/linux-x64
             mkdir -p $d
             cp -v ../wheelhouse/*.whl ./$d
             git status

CMakeLists.txt

@@ -43,7 +43,7 @@ list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules)
 list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake)
 
 option(BUILD_SHARED_LIBS "Whether to build shared libraries" ON)
-option(kaldifeat_BUILD_TESTS "Whether to build tests or not" ON)
+option(kaldifeat_BUILD_TESTS "Whether to build tests or not" OFF)
 option(kaldifeat_BUILD_PYMODULE "Whether to build python module or not" ON)
 
 message(STATUS "BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS}")

kaldifeat/python/csrc/CMakeLists.txt

@@ -27,13 +27,13 @@ if(NOT WIN32)
 endif()
 
 target_link_libraries(_kaldifeat PRIVATE kaldifeat_core)
-# if(UNIX AND NOT APPLE)
+if(UNIX AND NOT APPLE)
-#   target_link_libraries(_kaldifeat PUBLIC ${TORCH_DIR}/lib/libtorch_python.so)
+  target_link_libraries(_kaldifeat PUBLIC ${TORCH_DIR}/lib/libtorch_python.so)
-#   target_link_libraries(_kaldifeat PUBLIC ${PYTHON_LIBRARY})
+  # target_link_libraries(_kaldifeat PUBLIC ${PYTHON_LIBRARY})
-# elseif(WIN32)
+elseif(WIN32)
-#   target_link_libraries(_kaldifeat PUBLIC ${TORCH_DIR}/lib/torch_python.lib)
+  target_link_libraries(_kaldifeat PUBLIC ${TORCH_DIR}/lib/torch_python.lib)
-#   target_link_libraries(_kaldifeat PUBLIC ${PYTHON_LIBRARIES})
+  # target_link_libraries(_kaldifeat PUBLIC ${PYTHON_LIBRARIES})
-# endif()
+endif()
 
 install(TARGETS _kaldifeat
   DESTINATION ../

scripts/github_actions/build-ubuntu-cpu-arm64.sh

@@ -28,6 +28,7 @@ python3 -m pip install wheel twine typing_extensions
 python3 -m pip install bs4 requests tqdm auditwheel
 
 echo "Installing torch"
+python3 -m pip install -qq torch==$TORCH_VERSION || \
 python3 -m pip install -qq torch==$TORCH_VERSION -f https://download.pytorch.org/whl/torch_stable.html || \
 python3 -m pip install -qq torch==$TORCH_VERSION -f https://download.pytorch.org/whl/torch/
 
@@ -40,10 +41,38 @@ export CMAKE_CUDA_COMPILER_LAUNCHER=
 export KALDIFEAT_CMAKE_ARGS=" -DPYTHON_EXECUTABLE=$PYTHON_INSTALL_DIR/bin/python3 "
 export KALDIFEAT_MAKE_ARGS=" -j "
 
+nvcc --version || true
+rm -rf /usr/local/cuda*
+nvcc --version || true
+
+if [[ x"$IS_2_28" == x"1" ]]; then
+  plat=manylinux_2_28_aarch64
+else
+  plat=manylinux_2_17_aarch64
+fi
 
 python3 setup.py bdist_wheel
 
-mkdir /var/www/wheelhouse
+auditwheel --verbose repair \
-cp -v dist/*.whl /var/www/wheelhouse
+  --exclude libc10.so \
+  --exclude libc10_cuda.so \
+  --exclude libcuda.so.1 \
+  --exclude libcudart.so.${CUDA_VERSION} \
+  --exclude libnvToolsExt.so.1 \
+  --exclude libnvrtc.so.${CUDA_VERSION} \
+  --exclude libtorch.so \
+  --exclude libtorch_cpu.so \
+  --exclude libtorch_cuda.so \
+  --exclude libtorch_python.so \
+  \
+  --exclude libcudnn.so.8 \
+  --exclude libcublas.so.11 \
+  --exclude libcublasLt.so.11 \
+  --exclude libcudart.so.11.0 \
+  --exclude libnvrtc.so.11.2 \
+  --exclude libtorch_cuda_cu.so \
+  --exclude libtorch_cuda_cpp.so \
+  --plat $plat \
+  dist/*.whl
 
 ls -lh  /var/www

scripts/github_actions/build-ubuntu-cpu.sh

@@ -15,46 +15,13 @@ if [ -z $TORCH_VERSION ]; then
   exit 1
 fi
 
-yum -y install openssl-devel bzip2-devel libffi-devel xz-devel wget redhat-lsb-core
-
-INSTALLED_PYTHON_VERSION=${PYTHON_VERSION}.2
-if [[ $PYTHON_VERSION == "3.13" ]]; then
-  INSTALLED_PYTHON_VERSION=${PYTHON_VERSION}.0
-fi
-echo "Installing $INSTALLED_PYTHON_VERSION"
-
-curl -O https://www.python.org/ftp/python/$INSTALLED_PYTHON_VERSION/Python-$INSTALLED_PYTHON_VERSION.tgz
-tar xf Python-$INSTALLED_PYTHON_VERSION.tgz
-pushd Python-$INSTALLED_PYTHON_VERSION
-
-PYTHON_INSTALL_DIR=$PWD/py-${PYTHON_VERSION}
-
-if [[ $PYTHON_VERSION =~ 3.1. ]]; then
-  yum install -y openssl11-devel
-  sed -i 's/PKG_CONFIG openssl /PKG_CONFIG openssl11 /g' configure
-fi
-
-./configure --enable-shared --prefix=$PYTHON_INSTALL_DIR >/dev/null 2>&1
-make install >/dev/null 2>&1
-
-popd
-
 export PATH=$PYTHON_INSTALL_DIR/bin:$PATH
 export LD_LIBRARY_PATH=$PYTHON_INSTALL_DIR/lib:$LD_LIBRARY_PATH
 ls -lh $PYTHON_INSTALL_DIR/lib/
 
-nvcc --version || true
-rm -rf /usr/local/cuda*
-nvcc --version || true
-
 python3 --version
 which python3
 
-if [[ $PYTHON_VERSION != 3.6 ]]; then
-  curl -O https://bootstrap.pypa.io/get-pip.py
-  python3 get-pip.py
-fi
-
 python3 -m pip install scikit-build
 python3 -m pip install -U pip cmake
 python3 -m pip install wheel twine typing_extensions
@@ -73,8 +40,16 @@ export CMAKE_CUDA_COMPILER_LAUNCHER=
 export KALDIFEAT_CMAKE_ARGS=" -DPYTHON_EXECUTABLE=$PYTHON_INSTALL_DIR/bin/python3 "
 export KALDIFEAT_MAKE_ARGS=" -j "
 
+nvcc --version || true
+rm -rf /usr/local/cuda*
+nvcc --version || true
 
 python3 setup.py bdist_wheel
+if [[ x"$IS_2_28" == x"1" ]]; then
+  plat=manylinux_2_28_x86_64
+else
+  plat=manylinux_2_17_x86_64
+fi
 
 auditwheel --verbose repair \
   --exclude libc10.so \
@@ -95,8 +70,7 @@ auditwheel --verbose repair \
   --exclude libnvrtc.so.11.2 \
   --exclude libtorch_cuda_cu.so \
   --exclude libtorch_cuda_cpp.so \
-  --plat manylinux_2_17_x86_64 \
+  --plat $plat \
-  -w /var/www/wheelhouse \
   dist/*.whl
 
 ls -lh  /var/www

scripts/github_actions/build-ubuntu-cuda.sh

@@ -51,7 +51,7 @@ python3 -c "import torch; print(torch.__file__)"
 sed -i.bak /9.0a/d /Python-*/py-3.*/lib/python3.*/site-packages/torch/share/cmake/Caffe2/Modules_CUDA_fix/upstream/FindCUDA/select_compute_arch.cmake || true
 
 if [[ x"$IS_2_28" != x"1" ]]; then
-yum -y install openssl-devel
+  yum -y install openssl-devel
 fi
 
 yum -y install zlib-devel bzip2-devel libffi-devel xz-devel wget redhat-lsb-core

scripts/github_actions/generate_build_matrix.py

@@ -7,6 +7,7 @@ https://github.com/pytorch/test-infra/blob/main/.github/workflows/test_build_whe
 https://github.com/pytorch/test-infra/blob/main/.github/workflows/test_build_wheels_linux_without_cuda.yml
 
 https://github.com/pytorch/test-infra/actions/workflows/test_build_wheels_linux_with_cuda.yml
+https://github.com/pytorch/test-infra/blob/main/tools/scripts/generate_binary_build_matrix.py
 """
 
 import argparse
@@ -305,7 +306,7 @@ def generate_build_matrix(
         # https://github.com/Jimver/cuda-toolkit/blob/master/src/links/windows-links.ts
     }
     if test_only_latest_torch:
-        latest = "2.4.0"
+        latest = "2.6.0"
         matrix = {latest: matrix[latest]}
 
     if for_windows or for_macos:
@@ -322,9 +323,18 @@ def generate_build_matrix(
             matrix["1.13.1"]["python-version"].remove("3.11")
 
     excluded_python_versions = ["3.6"]
+    enabled_torch_versions = ["1.10.0"]
+
+    enabled_torch_versions += ["1.13.0", "1.13.1"]
+
+    min_torch_version = "2.0.0"
 
     ans = []
     for torch, python_cuda in matrix.items():
+        if enabled_torch_versions and torch not in enabled_torch_versions:
+            if not version_ge(torch, min_torch_version):
+                continue
+
         python_versions = python_cuda["python-version"]
         cuda_versions = python_cuda["cuda"]
         if enable_cuda:
@@ -371,6 +381,17 @@ def generate_build_matrix(
                     ans.append({"torch": torch, "python-version": p})
                 elif for_macos:
                     ans.append({"torch": torch, "python-version": p})
+                elif version_ge(torch, "2.6.0"):
+                    ans.append(
+                        {
+                            "torch": torch,
+                            "python-version": p,
+                            "image": "pytorch/manylinux2_28-builder:cpu"
+                            if not for_arm64
+                            else "pytorch/manylinux2_28_aarch64-builder:cpu-aarch64",
+                            "is_2_28": "1",
+                        }
+                    )
                 elif version_ge(torch, "2.4.0"):
                     ans.append(
                         {