From 25b70c9cd56a102d6412acfee7114b5dd30adc7c Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Wed, 1 Jun 2022 19:04:32 +0800 Subject: [PATCH 1/2] Fix conda build. --- .github/workflows/build_conda.yml | 82 ++++--------- .../github_actions/generate_build_matrix.py | 115 ++++++++++++++++++ 2 files changed, 137 insertions(+), 60 deletions(-) create mode 100755 scripts/github_actions/generate_build_matrix.py diff --git a/.github/workflows/build_conda.yml b/.github/workflows/build_conda.yml index 124e2db..0d2a778 100644 --- a/.github/workflows/build_conda.yml +++ b/.github/workflows/build_conda.yml @@ -20,6 +20,8 @@ name: build_conda_cuda on: push: + branches: + - fix-conda-build tags: - '*' @@ -27,70 +29,30 @@ env: KALDIFEAT_BUILD_TYPE: Release jobs: + generate_build_matrix: + # see https://github.com/pytorch/pytorch/pull/50633 + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Generating build matrix + id: set-matrix + run: | + # outputting for debugging purposes + python scripts/github_actions/generate_build_matrix.py + MATRIX=$(python scripts/github_actions/generate_build_matrix.py --enable-cuda) + echo "::set-output name=matrix::${MATRIX}" + build_conda_cuda: - runs-on: ${{ matrix.os }} + needs: generate_build_matrix + runs-on: ubuntu-18.04 strategy: fail-fast: false matrix: - os: [ubuntu-18.04] - # anaconda does not support 3.9 as of 2021.05.08 - python-version: [3.6, 3.7, 3.8, 3.9] - # python-version: [3.6, 3.7, 3.8] - cuda: ["10.1", "10.2", "11.0", "11.1"] - # from https://download.pytorch.org/whl/torch_stable.html - # - # PyTorch 1.9.0 supports: 10.2 (default), 11.1 - # PyTorch 1.8.1 supports: cuda 10.1, 10.2 (default), 11.1 - # PyTorch 1.8.0 supports: cuda 10.1, 10.2 (default), 11.1 - # PyTorch 1.7.x supports: cuda 10.1, 10.2 (default), 11.0, 9.2 (not included in this setup) - # PyTorch 1.6.0 supports: cuda 10.1, 10.2 (default), 9.2 (not included in this setup) - # PyTorch 1.5.x supports: cuda 10.1, 10.2 (default), 9.2 (not included in this setup) - # - # PyTorch 1.8.x and 1.7.1 support 3.6, 3.7, 3.8, 3.9 - # PyTorch 1.7.0, 1.6.0, and 1.5.x support 3.6, 3.7, 3.8 - # - # Other PyTorch versions are not tested - # - # torch: ["1.5.0", "1.5.1", "1.6.0", "1.7.0", "1.7.1", "1.8.0", "1.8.1"] - # 1.5.x is removed because there are compilation errors. - # See - # https://github.com/csukuangfj/k2/runs/2533830771?check_suite_focus=true - # and - # https://github.com/NVIDIA/apex/issues/805 - torch: ["1.6.0", "1.7.0", "1.7.1", "1.8.0", "1.8.1", "1.9.0"] - exclude: - # - cuda: "11.0" # exclude 11.0 for [1.5.0, 1.5.1, 1.6.0, 1.8.0, 1.8.1, 1.9.0] - # torch: "1.5.0" - # - cuda: "11.0" - # torch: "1.5.1" - - cuda: "11.0" - torch: "1.6.0" - - cuda: "11.0" - torch: "1.8.0" - - cuda: "11.0" - torch: "1.8.1" - - cuda: "11.0" - torch: "1.9.0" - # - cuda: "11.1" # exclude 11.1 for [1.5.0, 1.5.1, 1.6.0, 1.7.0, 1.7.1] - # torch: "1.5.0" - # - cuda: "11.1" - # torch: "1.5.1" - - cuda: "11.1" - torch: "1.6.0" - - cuda: "11.1" - torch: "1.7.0" - - cuda: "11.1" - torch: "1.7.1" - - cuda: "10.1" # exclude 10.1 for [1.9.0] - torch: "1.9.0" - - python-version: 3.9 # exclude Python 3.9 for [1.5.0, 1.5.1, 1.6.0, 1.7.0] - torch: "1.5.0" - - python-version: 3.9 - torch: "1.5.1" - - python-version: 3.9 - torch: "1.6.0" - - python-version: 3.9 - torch: "1.7.0" + ${{ fromJson(needs.generate_build_matrix.outputs.matrix) }} steps: # refer to https://github.com/actions/checkout diff --git a/scripts/github_actions/generate_build_matrix.py b/scripts/github_actions/generate_build_matrix.py new file mode 100755 index 0000000..67d10a0 --- /dev/null +++ b/scripts/github_actions/generate_build_matrix.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +# Copyright 2022 Xiaomi Corp. (authors: Fangjun Kuang) + +import argparse +import json + + +def get_args(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--enable-cuda", + action="store_true", + default=False, + help="True to enable CUDA", + ) + + parser.add_argument( + "--test-only-latest-torch", + action="store_true", + default=False, + help="""If True, we test only the latest PyTroch + to reduce CI running time.""", + ) + return parser.parse_args() + + +def generate_build_matrix(enable_cuda, test_only_latest_torch): + matrix = { + # 1.5.x is removed because there are compilation errors. + # See + # https://github.com/csukuangfj/k2/runs/2533830771?check_suite_focus=true + # and + # https://github.com/NVIDIA/apex/issues/805 + # "1.5.0": { + # "python-version": ["3.6", "3.7", "3.8"], + # "cuda": ["10.1", "10.2"], + # }, + # "1.5.1": { + # "python-version": ["3.6", "3.7", "3.8"], + # "cuda": ["10.1", "10.2"], + # }, + "1.6.0": { + "python-version": ["3.6", "3.7", "3.8"], + "cuda": ["10.1", "10.2"], + }, + "1.7.0": { + "python-version": ["3.6", "3.7", "3.8"], + "cuda": ["10.1", "10.2", "11.0"], + }, + "1.7.1": { + "python-version": ["3.6", "3.7", "3.8", "3.9"], + "cuda": ["10.1", "10.2", "11.0"], + }, + "1.8.0": { + "python-version": ["3.6", "3.7", "3.8", "3.9"], + "cuda": ["10.1", "10.2", "11.1"], + }, + "1.8.1": { + "python-version": ["3.6", "3.7", "3.8", "3.9"], + "cuda": ["10.1", "10.2", "11.1"], + }, + "1.9.0": { + "python-version": ["3.6", "3.7", "3.8", "3.9"], + "cuda": ["10.2", "11.1"], + }, + "1.9.1": { + "python-version": ["3.6", "3.7", "3.8", "3.9"], + "cuda": ["10.2", "11.1"], + }, + "1.10.0": { + "python-version": ["3.6", "3.7", "3.8", "3.9"], + "cuda": ["10.2", "11.1", "11.3"], + }, + "1.10.1": { + "python-version": ["3.6", "3.7", "3.8", "3.9"], + "cuda": ["10.2", "11.1", "11.3"], + }, + "1.10.2": { + "python-version": ["3.6", "3.7", "3.8", "3.9"], + "cuda": ["10.2", "11.1", "11.3"], + }, + "1.11.0": { + "python-version": ["3.7", "3.8", "3.9", "3.10"], + "cuda": ["10.2", "11.3", "11.5"], + }, + } + if test_only_latest_torch: + latest = "1.11.0" + matrix = {latest: matrix[latest]} + + ans = [] + for torch, python_cuda in matrix.items(): + python_versions = python_cuda["python-version"] + cuda_versions = python_cuda["cuda"] + if enable_cuda: + for p in python_versions: + for c in cuda_versions: + ans.append({"torch": torch, "python-version": p, "cuda": c}) + else: + for p in python_versions: + ans.append({"torch": torch, "python-version": p}) + + print(json.dumps({"include": ans})) + + +def main(): + args = get_args() + generate_build_matrix( + enable_cuda=args.enable_cuda, + test_only_latest_torch=args.test_only_latest_torch, + ) + + +if __name__ == "__main__": + main() From 32f4ba6c8ee0d6c59a622bac4850c237d0d628f9 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Wed, 1 Jun 2022 19:18:01 +0800 Subject: [PATCH 2/2] Minor fixes. --- .github/workflows/build_conda.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/build_conda.yml b/.github/workflows/build_conda.yml index 0d2a778..882ad72 100644 --- a/.github/workflows/build_conda.yml +++ b/.github/workflows/build_conda.yml @@ -20,8 +20,6 @@ name: build_conda_cuda on: push: - branches: - - fix-conda-build tags: - '*'