From 6f33cb5702037a5e54e4ae8c06ca130e3d84797f Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Sat, 22 Jul 2023 15:33:35 +0800 Subject: [PATCH] support macos --- .github/workflows/macos-cpu-wheels.yml | 119 ++++++++++++++++++ .github/workflows/windows-x64-cpu-wheels.yml | 11 +- .../github_actions/generate_build_matrix.py | 33 +++-- 3 files changed, 145 insertions(+), 18 deletions(-) create mode 100644 .github/workflows/macos-cpu-wheels.yml diff --git a/.github/workflows/macos-cpu-wheels.yml b/.github/workflows/macos-cpu-wheels.yml new file mode 100644 index 0000000..98ac862 --- /dev/null +++ b/.github/workflows/macos-cpu-wheels.yml @@ -0,0 +1,119 @@ +name: build-wheels-cpu-macos + +on: + push: + branches: + - wheels + tags: + - '*' + +concurrency: + group: build-wheels-cpu-macos-${{ github.ref }} + cancel-in-progress: true + +jobs: + generate_build_matrix: + # see https://github.com/pytorch/pytorch/pull/50633 + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Generating build matrix + id: set-matrix + run: | + # outputting for debugging purposes + python ./scripts/github_actions/generate_build_matrix.py --for-macos + MATRIX=$(python ./scripts/github_actions/generate_build_matrix.py --for-macos) + echo "::set-output name=matrix::${MATRIX}" + + build_wheels_macos_cpu: + needs: generate_build_matrix + name: ${{ matrix.torch }} ${{ matrix.python-version }} + runs-on: macos-latest + strategy: + fail-fast: false + matrix: + ${{ fromJson(needs.generate_build_matrix.outputs.matrix) }} + + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: display + shell: bash + run: | + function retry() { + $* || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*) + } + + retry ls -lh + + # see https://cibuildwheel.readthedocs.io/en/stable/changelog/ + # for a list of versions + - name: Build wheels + uses: pypa/cibuildwheel@v2.11.4 + env: + CIBW_BEFORE_BUILD: pip install torch==${{ matrix.torch}} cmake numpy + CIBW_BUILD: ${{ matrix.python-version }}-* + CIBW_REPAIR_WHEEL_COMMAND_MACOS: "" + CIBW_BUILD_VERBOSITY: 3 + + - name: Display wheels + shell: bash + run: | + + function retry() { + $* || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*) + } + + ls -lh ./wheelhouse/ + + ls -lh ./wheelhouse/*.whl + + - name: Upload Wheel + uses: actions/upload-artifact@v2 + with: + name: torch-${{ matrix.torch }}-python-${{ matrix.python-version }}-macos-latest-cpu + path: wheelhouse/*.whl + + # https://huggingface.co/docs/hub/spaces-github-actions + - name: Publish to huggingface + if: github.repository_owner == 'csukuangfj' + shell: bash + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + run: | + git config --global user.email "csukuangfj@gmail.com" + git config --global user.name "Fangjun Kuang" + + export GIT_LFS_SKIP_SMUDGE=1 + git clone https://huggingface.co/csukuangfj/kaldifeat huggingface + cur_path=$PWD + + function push_to_huggingface() { + cd $cur_path + rm -rf huggingface + git clone https://huggingface.co/csukuangfj/kaldifeat huggingface + cd huggingface + git pull + + mkdir -p macos + cp -v ../wheelhouse/*.whl ./macos + git status + git lfs track "*.whl" + git add . + git commit -m "upload macos-cpu wheel for torch ${{ matrix.torch }} python ${{ matrix.python-version }}" + git push https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/kaldifeat main + + cd .. + rm -rf huggingface + } + function retry() { + $* || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*) + } + + retry push_to_huggingface diff --git a/.github/workflows/windows-x64-cpu-wheels.yml b/.github/workflows/windows-x64-cpu-wheels.yml index 1e1c1c6..5cc93a0 100644 --- a/.github/workflows/windows-x64-cpu-wheels.yml +++ b/.github/workflows/windows-x64-cpu-wheels.yml @@ -57,7 +57,7 @@ jobs: - name: Build wheels uses: pypa/cibuildwheel@v2.11.4 env: - CIBW_BEFORE_BUILD: pip install torch==${{ matrix.torch}}+cpu -f https://download.pytorch.org/whl/torch_stable.html cmake==3.26 numpy && cmake --version + CIBW_BEFORE_BUILD: pip install torch==${{ matrix.torch}}+cpu -f https://download.pytorch.org/whl/torch_stable.html cmake numpy CIBW_BUILD: ${{ matrix.python-version }}-win_amd64 CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "" CIBW_BUILD_VERBOSITY: 3 @@ -92,15 +92,10 @@ jobs: export GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/csukuangfj/kaldifeat huggingface - - cd huggingface - git pull - mkdir -p windows-cpu - cp -v ../wheelhouse/*.whl ./windows-cpu - git status - git lfs track "*.whl" + cur_path=$PWD function push_to_huggingface() { + cd $cur_path rm -rf huggingface git clone https://huggingface.co/csukuangfj/kaldifeat huggingface cd huggingface diff --git a/scripts/github_actions/generate_build_matrix.py b/scripts/github_actions/generate_build_matrix.py index 88acc08..e69d2e5 100755 --- a/scripts/github_actions/generate_build_matrix.py +++ b/scripts/github_actions/generate_build_matrix.py @@ -21,6 +21,13 @@ def get_args(): help="True for windows", ) + parser.add_argument( + "--for-macos", + action="store_true", + default=False, + help="True for macOS", + ) + parser.add_argument( "--test-only-latest-torch", action="store_true", @@ -31,7 +38,7 @@ def get_args(): return parser.parse_args() -def generate_build_matrix(enable_cuda, for_windows, test_only_latest_torch): +def generate_build_matrix(enable_cuda, for_windows, for_macos, test_only_latest_torch): matrix = { # 1.5.x is removed because there are compilation errors. # See @@ -48,9 +55,7 @@ def generate_build_matrix(enable_cuda, for_windows, test_only_latest_torch): # }, "1.6.0": { "python-version": ["3.6", "3.7", "3.8"], - "cuda": ["10.1", "10.2"] - if not for_windows - else ["10.1.243", "10.2.89"], + "cuda": ["10.1", "10.2"] if not for_windows else ["10.1.243", "10.2.89"], }, "1.7.0": { "python-version": ["3.6", "3.7", "3.8"], @@ -78,15 +83,11 @@ def generate_build_matrix(enable_cuda, for_windows, test_only_latest_torch): }, "1.9.0": { "python-version": ["3.6", "3.7", "3.8", "3.9"], - "cuda": ["10.2", "11.1"] - if not for_windows - else ["10.2.89", "11.1.1"], + "cuda": ["10.2", "11.1"] if not for_windows else ["10.2.89", "11.1.1"], }, "1.9.1": { "python-version": ["3.6", "3.7", "3.8", "3.9"], - "cuda": ["10.2", "11.1"] - if not for_windows - else ["10.2.89", "11.1.1"], + "cuda": ["10.2", "11.1"] if not for_windows else ["10.2.89", "11.1.1"], }, "1.10.0": { "python-version": ["3.6", "3.7", "3.8", "3.9"], @@ -128,6 +129,10 @@ def generate_build_matrix(enable_cuda, for_windows, test_only_latest_torch): "python-version": ["3.7", "3.8", "3.9", "3.10", "3.11"], "cuda": ["11.6", "11.7"], # default 11.7 }, + "1.13.1": { + "python-version": ["3.7", "3.8", "3.9", "3.10", "3.11"], + "cuda": ["11.6", "11.7"], # default 11.7 + }, "2.0.0": { "python-version": ["3.8", "3.9", "3.10", "3.11"], "cuda": ["11.7", "11.8"], # default 11.7 @@ -141,6 +146,13 @@ def generate_build_matrix(enable_cuda, for_windows, test_only_latest_torch): latest = "2.0.1" matrix = {latest: matrix[latest]} + if for_windows or for_macos: + if "1.13.0" in matrix: + matrix["1.13.0"]["python-version"].remove("3.11") + + if "1.13.1" in matrix: + matrix["1.13.1"]["python-version"].remove("3.11") + ans = [] for torch, python_cuda in matrix.items(): python_versions = python_cuda["python-version"] @@ -163,6 +175,7 @@ def main(): generate_build_matrix( enable_cuda=args.enable_cuda, for_windows=args.for_windows, + for_macos=args.for_macos, test_only_latest_torch=args.test_only_latest_torch, )