diff --git a/.github/scripts/docker/generate_build_matrix.py b/.github/scripts/docker/generate_build_matrix.py
new file mode 100755
index 000000000..4e494d810
--- /dev/null
+++ b/.github/scripts/docker/generate_build_matrix.py
@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+# Copyright    2023  Xiaomi Corp.        (authors: Fangjun Kuang)
+
+
+import json
+
+
+def version_gt(a, b):
+    a_major, a_minor = a.split(".")[:2]
+    b_major, b_minor = b.split(".")[:2]
+    if a_major > b_major:
+        return True
+
+    if a_major == b_major and a_minor > b_minor:
+        return True
+
+    return False
+
+
+def version_ge(a, b):
+    a_major, a_minor = a.split(".")[:2]
+    b_major, b_minor = b.split(".")[:2]
+    if a_major > b_major:
+        return True
+
+    if a_major == b_major and a_minor >= b_minor:
+        return True
+
+    return False
+
+
+def get_torchaudio_version(torch_version):
+    if torch_version == "1.13.0":
+        return "0.13.0"
+    elif torch_version == "1.13.1":
+        return "0.13.1"
+    elif torch_version == "2.0.0":
+        return "2.0.1"
+    elif torch_version == "2.0.1":
+        return "2.0.2"
+    else:
+        return torch_version
+
+
+def get_matrix():
+    k2_version = "1.24.4.dev20231220"
+    kaldifeat_version = "1.25.3.dev20231221"
+    version = "1.1"
+    python_version = ["3.8", "3.9", "3.10", "3.11"]
+    torch_version = ["1.13.0", "1.13.1", "2.0.0", "2.0.1", "2.1.0", "2.1.1", "2.1.2"]
+
+    matrix = []
+    for p in python_version:
+        for t in torch_version:
+            # torchaudio <= 1.13.x supports only python <= 3.10
+
+            if version_gt(p, "3.10") and not version_gt(t, "2.0"):
+                continue
+
+            matrix.append(
+                {
+                    "k2-version": k2_version,
+                    "kaldifeat-version": kaldifeat_version,
+                    "version": version,
+                    "python-version": p,
+                    "torch-version": t,
+                    "torchaudio-version": get_torchaudio_version(t),
+                }
+            )
+    return matrix
+
+
+def main():
+    matrix = get_matrix()
+    print(json.dumps({"include": matrix}))
+
+
+if __name__ == "__main__":
+    main()
diff --git a/.github/workflows/build-cpu-docker.yml b/.github/workflows/build-cpu-docker.yml
index b26cd2095..c5d5aaeb6 100644
--- a/.github/workflows/build-cpu-docker.yml
+++ b/.github/workflows/build-cpu-docker.yml
@@ -7,18 +7,31 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  generate_build_matrix:
+    if: github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa'
+    # see https://github.com/pytorch/pytorch/pull/50633
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Generating build matrix
+        id: set-matrix
+        run: |
+          # outputting for debugging purposes
+          python ./.github/scripts/docker/generate_build_matrix.py
+          MATRIX=$(python ./.github/scripts/docker/generate_build_matrix.py)
+          echo "::set-output name=matrix::${MATRIX}"
   build-cpu-docker:
+    needs: generate_build_matrix
     name: py${{ matrix.python-version }} torch${{ matrix.torch-version }} v${{ matrix.version }}
-    runs-on: ${{ matrix.os }}
+    runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest]
-        python-version: ["3.8", "3.9", "3.10"]
-        torch-version: ["1.13.0", "1.13.1", "2.0.0", "2.0.1", "2.1.0", "2.1.1", "2.1.2"]
-        k2-version: ["1.24.4.dev20231220"]
-        kaldifeat-version: ["1.25.3.dev20231221"]
-        version: ["1.1"]
+        ${{ fromJson(needs.generate_build_matrix.outputs.matrix) }}
 
     steps:
       # refer to https://github.com/actions/checkout
@@ -45,25 +58,14 @@ jobs:
         run: |
           cd .github/scripts/docker
           torch_version=${{ matrix.torch-version }}
+          torchaudio_version=${{ matrix.torchaudio-version }}
 
-          # see https://pytorch.org/audio/stable/installation.html#compatibility-matrix
-          if [[ $torch_version == 1.13.0 ]]; then
-            torchaudio_version=0.13.0
-          elif [[ $torch_version == 1.13.1 ]]; then
-            torchaudio_version=0.13.1
-          elif [[ $torch_version == 2.0.0 ]]; then
-            torchaudio_version=2.0.1
-          elif [[ $torch_version == 2.0.1 ]]; then
-            torchaudio_version=2.0.2
-          else
-            torchaudio_version=$torch_version
-          fi
           echo "torch_version: $torch_version"
           echo "torchaudio_version: $torchaudio_version"
 
           version=${{ matrix.version }}
 
-          tag=ghcr.io/k2-fsa/icefall:cpu-py${{ matrix.python-version }}-torch${{ matrix.torch-version }}-v$version
+          tag=ghcr.io/${{ github.repository_owner }}/icefall:cpu-py${{ matrix.python-version }}-torch${{ matrix.torch-version }}-v$version
           echo "tag: $tag"
 
           docker build \
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index b3fd6f133..659681b37 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -16,16 +16,31 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  generate_build_matrix:
+    if: github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa'
+    # see https://github.com/pytorch/pytorch/pull/50633
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Generating build matrix
+        id: set-matrix
+        run: |
+          # outputting for debugging purposes
+          python ./.github/scripts/docker/generate_build_matrix.py
+          MATRIX=$(python ./.github/scripts/docker/generate_build_matrix.py)
+          echo "::set-output name=matrix::${MATRIX}"
   test:
+    needs: generate_build_matrix
     name: py${{ matrix.python-version }} torch${{ matrix.torch-version }} v${{ matrix.version }}
-    runs-on: ${{ matrix.os }}
+    runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest]
-        python-version: ["3.8", "3.9", "3.10"]
-        torch-version: ["1.13.0", "1.13.1", "2.0.0", "2.0.1", "2.1.0", "2.1.1", "2.1.2"]
-        version: ["1.1"]
+        ${{ fromJson(needs.generate_build_matrix.outputs.matrix) }}
 
     steps:
       - uses: actions/checkout@v4
@@ -44,7 +59,7 @@ jobs:
       - name: Run tests
         uses: addnab/docker-run-action@v3
         with:
-            image: ghcr.io/k2-fsa/icefall:cpu-py${{ matrix.python-version }}-torch${{ matrix.torch-version }}-v${{ matrix.version }}
+            image: ghcr.io/${{ github.repository_owner }}/icefall:cpu-py${{ matrix.python-version }}-torch${{ matrix.torch-version }}-v${{ matrix.version }}
             options: |
               --volume ${{ github.workspace }}/:/icefall
             shell: bash
diff --git a/.github/workflows/train-librispeech.yml b/.github/workflows/train-librispeech.yml
index 53a2d5843..619b5ccc3 100644
--- a/.github/workflows/train-librispeech.yml
+++ b/.github/workflows/train-librispeech.yml
@@ -15,16 +15,31 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  generate_build_matrix:
+    if: github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa'
+    # see https://github.com/pytorch/pytorch/pull/50633
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Generating build matrix
+        id: set-matrix
+        run: |
+          # outputting for debugging purposes
+          python ./.github/scripts/docker/generate_build_matrix.py
+          MATRIX=$(python ./.github/scripts/docker/generate_build_matrix.py)
+          echo "::set-output name=matrix::${MATRIX}"
   train-librispeech:
+    needs: generate_build_matrix
     name: py${{ matrix.python-version }} torch${{ matrix.torch-version }} v${{ matrix.version }}
-    runs-on: ${{ matrix.os }}
+    runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest]
-        python-version: ["3.8", "3.9", "3.10"]
-        torch-version: ["1.13.0", "1.13.1", "2.0.0", "2.0.1", "2.1.0", "2.1.1", "2.1.2"]
-        version: ["1.1"]
+        ${{ fromJson(needs.generate_build_matrix.outputs.matrix) }}
 
     steps:
       # refer to https://github.com/actions/checkout
@@ -44,7 +59,7 @@ jobs:
       - name: Test zipformer/train.py with LibriSpeech
         uses: addnab/docker-run-action@v3
         with:
-            image: ghcr.io/k2-fsa/icefall:cpu-py${{ matrix.python-version }}-torch${{ matrix.torch-version }}-v${{ matrix.version }}
+            image: ghcr.io/${{ github.repository_owner }}/icefall:cpu-py${{ matrix.python-version }}-torch${{ matrix.torch-version }}-v${{ matrix.version }}
             options: |
               --volume ${{ github.workspace }}/:/icefall
             shell: bash