Merge pull request #35 from csukuangfj/fix-conda-build

Fix conda build

.github/workflows/build_conda.yml

@@ -27,70 +27,30 @@ env:
   KALDIFEAT_BUILD_TYPE: Release
 
 jobs:
+  generate_build_matrix:
+    # see https://github.com/pytorch/pytorch/pull/50633
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Generating build matrix
+        id: set-matrix
+        run: |
+          # outputting for debugging purposes
+          python scripts/github_actions/generate_build_matrix.py
+          MATRIX=$(python scripts/github_actions/generate_build_matrix.py --enable-cuda)
+          echo "::set-output name=matrix::${MATRIX}"
+
   build_conda_cuda:
-    runs-on: ${{ matrix.os }}
+    needs: generate_build_matrix
+    runs-on: ubuntu-18.04
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-18.04]
+        ${{ fromJson(needs.generate_build_matrix.outputs.matrix) }}
-        # anaconda does not support 3.9 as of 2021.05.08
-        python-version: [3.6, 3.7, 3.8, 3.9]
-        # python-version: [3.6, 3.7, 3.8]
-        cuda: ["10.1", "10.2", "11.0", "11.1"]
-        # from https://download.pytorch.org/whl/torch_stable.html
-        #
-        # PyTorch 1.9.0 supports: 10.2 (default), 11.1
-        # PyTorch 1.8.1 supports: cuda 10.1, 10.2 (default), 11.1
-        # PyTorch 1.8.0 supports: cuda 10.1, 10.2 (default), 11.1
-        # PyTorch 1.7.x supports: cuda 10.1, 10.2 (default), 11.0, 9.2 (not included in this setup)
-        # PyTorch 1.6.0 supports: cuda 10.1, 10.2 (default), 9.2 (not included in this setup)
-        # PyTorch 1.5.x supports: cuda 10.1, 10.2 (default), 9.2 (not included in this setup)
-        #
-        # PyTorch 1.8.x and 1.7.1 support 3.6, 3.7, 3.8, 3.9
-        # PyTorch 1.7.0, 1.6.0, and 1.5.x support 3.6, 3.7, 3.8
-        #
-        # Other PyTorch versions are not tested
-        #
-        # torch: ["1.5.0", "1.5.1", "1.6.0", "1.7.0", "1.7.1", "1.8.0", "1.8.1"]
-        # 1.5.x is removed because there are compilation errors.
-        #  See
-        #  https://github.com/csukuangfj/k2/runs/2533830771?check_suite_focus=true
-        #  and
-        #  https://github.com/NVIDIA/apex/issues/805
-        torch: ["1.6.0", "1.7.0", "1.7.1", "1.8.0", "1.8.1", "1.9.0"]
-        exclude:
-          # - cuda: "11.0" # exclude 11.0 for [1.5.0, 1.5.1, 1.6.0, 1.8.0, 1.8.1, 1.9.0]
-          #   torch: "1.5.0"
-          # - cuda: "11.0"
-          #   torch: "1.5.1"
-          - cuda: "11.0"
-            torch: "1.6.0"
-          - cuda: "11.0"
-            torch: "1.8.0"
-          - cuda: "11.0"
-            torch: "1.8.1"
-          - cuda: "11.0"
-            torch: "1.9.0"
-          # - cuda: "11.1" # exclude 11.1 for [1.5.0, 1.5.1, 1.6.0, 1.7.0, 1.7.1]
-          #   torch: "1.5.0"
-          # - cuda: "11.1"
-          #   torch: "1.5.1"
-          - cuda: "11.1"
-            torch: "1.6.0"
-          - cuda: "11.1"
-            torch: "1.7.0"
-          - cuda: "11.1"
-            torch: "1.7.1"
-          - cuda: "10.1" # exclude 10.1 for [1.9.0]
-            torch: "1.9.0"
-          - python-version: 3.9 # exclude Python 3.9 for [1.5.0, 1.5.1, 1.6.0, 1.7.0]
-            torch: "1.5.0"
-          - python-version: 3.9
-            torch: "1.5.1"
-          - python-version: 3.9
-            torch: "1.6.0"
-          - python-version: 3.9
-            torch: "1.7.0"
 
     steps:
       # refer to https://github.com/actions/checkout

scripts/github_actions/generate_build_matrix.py

@@ -0,0 +1,115 @@
+#!/usr/bin/env python3
+# Copyright    2022  Xiaomi Corp.        (authors: Fangjun Kuang)
+
+import argparse
+import json
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--enable-cuda",
+        action="store_true",
+        default=False,
+        help="True to enable CUDA",
+    )
+
+    parser.add_argument(
+        "--test-only-latest-torch",
+        action="store_true",
+        default=False,
+        help="""If True, we test only the latest PyTroch
+        to reduce CI running time.""",
+    )
+    return parser.parse_args()
+
+
+def generate_build_matrix(enable_cuda, test_only_latest_torch):
+    matrix = {
+        # 1.5.x is removed because there are compilation errors.
+        #  See
+        #  https://github.com/csukuangfj/k2/runs/2533830771?check_suite_focus=true
+        #  and
+        #  https://github.com/NVIDIA/apex/issues/805
+        #  "1.5.0": {
+        #      "python-version": ["3.6", "3.7", "3.8"],
+        #      "cuda": ["10.1", "10.2"],
+        #  },
+        #  "1.5.1": {
+        #      "python-version": ["3.6", "3.7", "3.8"],
+        #      "cuda": ["10.1", "10.2"],
+        #  },
+        "1.6.0": {
+            "python-version": ["3.6", "3.7", "3.8"],
+            "cuda": ["10.1", "10.2"],
+        },
+        "1.7.0": {
+            "python-version": ["3.6", "3.7", "3.8"],
+            "cuda": ["10.1", "10.2", "11.0"],
+        },
+        "1.7.1": {
+            "python-version": ["3.6", "3.7", "3.8", "3.9"],
+            "cuda": ["10.1", "10.2", "11.0"],
+        },
+        "1.8.0": {
+            "python-version": ["3.6", "3.7", "3.8", "3.9"],
+            "cuda": ["10.1", "10.2", "11.1"],
+        },
+        "1.8.1": {
+            "python-version": ["3.6", "3.7", "3.8", "3.9"],
+            "cuda": ["10.1", "10.2", "11.1"],
+        },
+        "1.9.0": {
+            "python-version": ["3.6", "3.7", "3.8", "3.9"],
+            "cuda": ["10.2", "11.1"],
+        },
+        "1.9.1": {
+            "python-version": ["3.6", "3.7", "3.8", "3.9"],
+            "cuda": ["10.2", "11.1"],
+        },
+        "1.10.0": {
+            "python-version": ["3.6", "3.7", "3.8", "3.9"],
+            "cuda": ["10.2", "11.1", "11.3"],
+        },
+        "1.10.1": {
+            "python-version": ["3.6", "3.7", "3.8", "3.9"],
+            "cuda": ["10.2", "11.1", "11.3"],
+        },
+        "1.10.2": {
+            "python-version": ["3.6", "3.7", "3.8", "3.9"],
+            "cuda": ["10.2", "11.1", "11.3"],
+        },
+        "1.11.0": {
+            "python-version": ["3.7", "3.8", "3.9", "3.10"],
+            "cuda": ["10.2", "11.3", "11.5"],
+        },
+    }
+    if test_only_latest_torch:
+        latest = "1.11.0"
+        matrix = {latest: matrix[latest]}
+
+    ans = []
+    for torch, python_cuda in matrix.items():
+        python_versions = python_cuda["python-version"]
+        cuda_versions = python_cuda["cuda"]
+        if enable_cuda:
+            for p in python_versions:
+                for c in cuda_versions:
+                    ans.append({"torch": torch, "python-version": p, "cuda": c})
+        else:
+            for p in python_versions:
+                ans.append({"torch": torch, "python-version": p})
+
+    print(json.dumps({"include": ans}))
+
+
+def main():
+    args = get_args()
+    generate_build_matrix(
+        enable_cuda=args.enable_cuda,
+        test_only_latest_torch=args.test_only_latest_torch,
+    )
+
+
+if __name__ == "__main__":
+    main()