mirrors/kaldifeat
1
0
Fork 0
mirror of https://github.com/csukuangfj/kaldifeat.git synced 2025-08-09 10:02:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix building for python

Browse Source
This commit is contained in:
Fangjun Kuang 2025-03-07 10:18:29 +08:00
parent adccd33678
commit f93e9fb6e0
8 changed files with 144 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
.github/workflows/ubuntu-arm64-cpu-wheels.yml vendored
View File

@ -3,6 +3,7 @@ name: build-wheels-cpu-arm64-ubuntu
on:
push:
branches:
# - wheel
- fix-cuda-12.6
tags:
- '*'
@ -26,11 +27,11 @@ jobs:
id: set-matrix
run: |
# outputting for debugging purposes
# python ./scripts/github_actions/generate_build_matrix.py --for-arm64
# MATRIX=$(python ./scripts/github_actions/generate_build_matrix.py --for-arm64)
python ./scripts/github_actions/generate_build_matrix.py --for-arm64
MATRIX=$(python ./scripts/github_actions/generate_build_matrix.py --for-arm64)
python ./scripts/github_actions/generate_build_matrix.py --test-only-latest-torch --for-arm64
MATRIX=$(python ./scripts/github_actions/generate_build_matrix.py --test-only-latest-torch --for-arm64)
# python ./scripts/github_actions/generate_build_matrix.py --test-only-latest-torch --for-arm64
# MATRIX=$(python ./scripts/github_actions/generate_build_matrix.py --test-only-latest-torch --for-arm64)
echo "::set-output name=matrix::${MATRIX}"
build-manylinux-wheels:
@ -125,9 +126,6 @@ jobs:
export PYTHON_INSTALL_DIR=$python_dir
export PATH=$PYTHON_INSTALL_DIR/bin:$PATH
# There are no libpython.so inside $PYTHON_INSTALL_DIR
# since they are statically linked.
python3 --version
which python3
@ -160,7 +158,7 @@ jobs:
cd huggingface
git pull
d=cpu/1.25.5.dev20250306/linux-arm64
d=cpu/1.25.5.dev20250307/linux-arm64
mkdir -p $d
cp -v ../wheelhouse/*.whl ./$d
git status

76
.github/workflows/ubuntu-cpu-wheels.yml vendored
View File

@ -3,7 +3,8 @@ name: build-wheels-cpu-ubuntu
on:
push:
branches:
- fix-wheel-2
# - fix-cuda-12.6
- wheel
tags:
- '*'
workflow_dispatch:
@ -56,7 +57,7 @@ jobs:
uses: addnab/docker-run-action@v3
with:
image: ${{ matrix.image }}
options: -v ${{ github.workspace }}:/var/www -e PYTHON_VERSION=${{ matrix.python-version }} -e TORCH_VERSION=${{ matrix.torch }}
options: -v ${{ github.workspace }}:/var/www -e IS_2_28=${{ matrix.is_2_28 }} -e PYTHON_VERSION=${{ matrix.python-version }} -e TORCH_VERSION=${{ matrix.torch }}
run: |
echo "pwd: $PWD"
uname -a
@ -65,6 +66,69 @@ jobs:
gcc --version
python3 --version
which python3
ls -lh /opt/python/
echo "---"
ls -lh /opt/python/cp*
ls -lh /opt/python/*/bin
echo "---"
find /opt/python/cp* -name "libpython*"
echo "-----"
find /opt/_internal/cp* -name "libpython*"
echo "-----"
find / -name "libpython*"
echo "----"
ls -lh /usr/lib64/libpython3.so || true
# cp36-cp36m
# cp37-cp37m
# cp38-cp38
# cp39-cp39
# cp310-cp310
# cp311-cp311
# cp312-cp312
# cp313-cp313
# cp313-cp313t (no gil)
if [[ $PYTHON_VERSION == "3.6" ]]; then
python_dir=/opt/python/cp36-cp36m
export PYTHONPATH=/opt/python/cp36-cp36m/lib/python3.6/site-packages:$PYTHONPATH
elif [[ $PYTHON_VERSION == "3.7" ]]; then
python_dir=/opt/python/cp37-cp37m
export PYTHONPATH=/opt/python/cp37-cp37m/lib/python3.7/site-packages:$PYTHONPATH
elif [[ $PYTHON_VERSION == "3.8" ]]; then
python_dir=/opt/python/cp38-cp38
export PYTHONPATH=/opt/python/cp38-cp38/lib/python3.8/site-packages:$PYTHONPATH
elif [[ $PYTHON_VERSION == "3.9" ]]; then
python_dir=/opt/python/cp39-cp39
export PYTHONPATH=/opt/python/cp39-cp39/lib/python3.9/site-packages:$PYTHONPATH
elif [[ $PYTHON_VERSION == "3.10" ]]; then
python_dir=/opt/python/cp310-cp310
export PYTHONPATH=/opt/python/cp310-cp310/lib/python3.10/site-packages:$PYTHONPATH
elif [[ $PYTHON_VERSION == "3.11" ]]; then
python_dir=/opt/python/cp311-cp311
export PYTHONPATH=/opt/python/cp311-cp311/lib/python3.11/site-packages:$PYTHONPATH
elif [[ $PYTHON_VERSION == "3.12" ]]; then
python_dir=/opt/python/cp312-cp312
export PYTHONPATH=/opt/python/cp312-cp312/lib/python3.12/site-packages:$PYTHONPATH
elif [[ $PYTHON_VERSION == "3.13" ]]; then
python_dir=/opt/python/cp313-cp313
export PYTHONPATH=/opt/python/cp313-cp313/lib/python3.13/site-packages:$PYTHONPATH
else
echo "Unsupported Python version $PYTHON_VERSION"
exit 1
fi
export PYTHON_INSTALL_DIR=$python_dir
export PATH=$PYTHON_INSTALL_DIR/bin:$PATH
python3 --version
which python3
/var/www/scripts/github_actions/build-ubuntu-cpu.sh
- name: Display wheels
@ -72,12 +136,6 @@ jobs:
run: |
ls -lh ./wheelhouse/
- name: Upload Wheel
uses: actions/upload-artifact@v4
with:
name: torch-${{ matrix.torch }}-python-${{ matrix.python-version }}-ubuntu-latest-cpu
path: wheelhouse/*.whl
# https://huggingface.co/docs/hub/spaces-github-actions
- name: Publish to huggingface
if: github.repository_owner == 'csukuangfj'
@ -100,7 +158,7 @@ jobs:
cd huggingface
git pull
d=cpu/1.25.5.dev20241029/linux
d=cpu/1.25.5.dev20250307/linux-x64
mkdir -p $d
cp -v ../wheelhouse/*.whl ./$d
git status

2
CMakeLists.txt
View File

@ -43,7 +43,7 @@ list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules)
list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake)
option(BUILD_SHARED_LIBS "Whether to build shared libraries" ON)
option(kaldifeat_BUILD_TESTS "Whether to build tests or not" ON)
option(kaldifeat_BUILD_TESTS "Whether to build tests or not" OFF)
option(kaldifeat_BUILD_PYMODULE "Whether to build python module or not" ON)
message(STATUS "BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS}")

10
kaldifeat/python/csrc/CMakeLists.txt
View File

@ -27,13 +27,13 @@ if(NOT WIN32)
endif()
target_link_libraries(_kaldifeat PRIVATE kaldifeat_core)
# if(UNIX AND NOT APPLE)
# target_link_libraries(_kaldifeat PUBLIC ${TORCH_DIR}/lib/libtorch_python.so)
if(UNIX AND NOT APPLE)
target_link_libraries(_kaldifeat PUBLIC ${TORCH_DIR}/lib/libtorch_python.so)
# target_link_libraries(_kaldifeat PUBLIC ${PYTHON_LIBRARY})
# elseif(WIN32)
# target_link_libraries(_kaldifeat PUBLIC ${TORCH_DIR}/lib/torch_python.lib)
elseif(WIN32)
target_link_libraries(_kaldifeat PUBLIC ${TORCH_DIR}/lib/torch_python.lib)
# target_link_libraries(_kaldifeat PUBLIC ${PYTHON_LIBRARIES})
# endif()
endif()
install(TARGETS _kaldifeat
DESTINATION ../

33
scripts/github_actions/build-ubuntu-cpu-arm64.sh
View File

@ -28,6 +28,7 @@ python3 -m pip install wheel twine typing_extensions
python3 -m pip install bs4 requests tqdm auditwheel
echo "Installing torch"
python3 -m pip install -qq torch==$TORCH_VERSION || \
python3 -m pip install -qq torch==$TORCH_VERSION -f https://download.pytorch.org/whl/torch_stable.html || \
python3 -m pip install -qq torch==$TORCH_VERSION -f https://download.pytorch.org/whl/torch/
@ -40,10 +41,38 @@ export CMAKE_CUDA_COMPILER_LAUNCHER=
export KALDIFEAT_CMAKE_ARGS=" -DPYTHON_EXECUTABLE=$PYTHON_INSTALL_DIR/bin/python3 "
export KALDIFEAT_MAKE_ARGS=" -j "
nvcc --version || true
rm -rf /usr/local/cuda*
nvcc --version || true
if [[ x"$IS_2_28" == x"1" ]]; then
plat=manylinux_2_28_aarch64
else
plat=manylinux_2_17_aarch64
fi
python3 setup.py bdist_wheel
mkdir /var/www/wheelhouse
cp -v dist/*.whl /var/www/wheelhouse
auditwheel --verbose repair \
--exclude libc10.so \
--exclude libc10_cuda.so \
--exclude libcuda.so.1 \
--exclude libcudart.so.${CUDA_VERSION} \
--exclude libnvToolsExt.so.1 \
--exclude libnvrtc.so.${CUDA_VERSION} \
--exclude libtorch.so \
--exclude libtorch_cpu.so \
--exclude libtorch_cuda.so \
--exclude libtorch_python.so \
\
--exclude libcudnn.so.8 \
--exclude libcublas.so.11 \
--exclude libcublasLt.so.11 \
--exclude libcudart.so.11.0 \
--exclude libnvrtc.so.11.2 \
--exclude libtorch_cuda_cu.so \
--exclude libtorch_cuda_cpp.so \
--plat $plat \
dist/*.whl
ls -lh /var/www

44
scripts/github_actions/build-ubuntu-cpu.sh
View File

@ -15,46 +15,13 @@ if [ -z $TORCH_VERSION ]; then
exit 1
fi
yum -y install openssl-devel bzip2-devel libffi-devel xz-devel wget redhat-lsb-core
INSTALLED_PYTHON_VERSION=${PYTHON_VERSION}.2
if [[ $PYTHON_VERSION == "3.13" ]]; then
INSTALLED_PYTHON_VERSION=${PYTHON_VERSION}.0
fi
echo "Installing $INSTALLED_PYTHON_VERSION"
curl -O https://www.python.org/ftp/python/$INSTALLED_PYTHON_VERSION/Python-$INSTALLED_PYTHON_VERSION.tgz
tar xf Python-$INSTALLED_PYTHON_VERSION.tgz
pushd Python-$INSTALLED_PYTHON_VERSION
PYTHON_INSTALL_DIR=$PWD/py-${PYTHON_VERSION}
if [[ $PYTHON_VERSION =~ 3.1. ]]; then
yum install -y openssl11-devel
sed -i 's/PKG_CONFIG openssl /PKG_CONFIG openssl11 /g' configure
fi
./configure --enable-shared --prefix=$PYTHON_INSTALL_DIR >/dev/null 2>&1
make install >/dev/null 2>&1
popd
export PATH=$PYTHON_INSTALL_DIR/bin:$PATH
export LD_LIBRARY_PATH=$PYTHON_INSTALL_DIR/lib:$LD_LIBRARY_PATH
ls -lh $PYTHON_INSTALL_DIR/lib/
nvcc --version || true
rm -rf /usr/local/cuda*
nvcc --version || true
python3 --version
which python3
if [[ $PYTHON_VERSION != 3.6 ]]; then
curl -O https://bootstrap.pypa.io/get-pip.py
python3 get-pip.py
fi
python3 -m pip install scikit-build
python3 -m pip install -U pip cmake
python3 -m pip install wheel twine typing_extensions
@ -73,8 +40,16 @@ export CMAKE_CUDA_COMPILER_LAUNCHER=
export KALDIFEAT_CMAKE_ARGS=" -DPYTHON_EXECUTABLE=$PYTHON_INSTALL_DIR/bin/python3 "
export KALDIFEAT_MAKE_ARGS=" -j "
nvcc --version || true
rm -rf /usr/local/cuda*
nvcc --version || true
python3 setup.py bdist_wheel
if [[ x"$IS_2_28" == x"1" ]]; then
plat=manylinux_2_28_x86_64
else
plat=manylinux_2_17_x86_64
fi
auditwheel --verbose repair \
--exclude libc10.so \
@ -95,8 +70,7 @@ auditwheel --verbose repair \
--exclude libnvrtc.so.11.2 \
--exclude libtorch_cuda_cu.so \
--exclude libtorch_cuda_cpp.so \
--plat manylinux_2_17_x86_64 \
-w /var/www/wheelhouse \
--plat $plat \
dist/*.whl
ls -lh /var/www

23
scripts/github_actions/generate_build_matrix.py
View File

@ -7,6 +7,7 @@ https://github.com/pytorch/test-infra/blob/main/.github/workflows/test_build_whe
https://github.com/pytorch/test-infra/blob/main/.github/workflows/test_build_wheels_linux_without_cuda.yml
https://github.com/pytorch/test-infra/actions/workflows/test_build_wheels_linux_with_cuda.yml
https://github.com/pytorch/test-infra/blob/main/tools/scripts/generate_binary_build_matrix.py
"""
import argparse
@ -305,7 +306,7 @@ def generate_build_matrix(
# https://github.com/Jimver/cuda-toolkit/blob/master/src/links/windows-links.ts
}
if test_only_latest_torch:
latest = "2.4.0"
latest = "2.6.0"
matrix = {latest: matrix[latest]}
if for_windows or for_macos:
@ -322,9 +323,18 @@ def generate_build_matrix(
matrix["1.13.1"]["python-version"].remove("3.11")
excluded_python_versions = ["3.6"]
enabled_torch_versions = ["1.10.0"]
enabled_torch_versions += ["1.13.0", "1.13.1"]
min_torch_version = "2.0.0"
ans = []
for torch, python_cuda in matrix.items():
if enabled_torch_versions and torch not in enabled_torch_versions:
if not version_ge(torch, min_torch_version):
continue
python_versions = python_cuda["python-version"]
cuda_versions = python_cuda["cuda"]
if enable_cuda:
@ -371,6 +381,17 @@ def generate_build_matrix(
ans.append({"torch": torch, "python-version": p})
elif for_macos:
ans.append({"torch": torch, "python-version": p})
elif version_ge(torch, "2.6.0"):
ans.append(
{
"torch": torch,
"python-version": p,
"image": "pytorch/manylinux2_28-builder:cpu"
if not for_arm64
else "pytorch/manylinux2_28_aarch64-builder:cpu-aarch64",
"is_2_28": "1",
}
)
elif version_ge(torch, "2.4.0"):
ans.append(
{