mirrors/icefall
1
0
Fork 0
mirror of https://github.com/k2-fsa/icefall.git synced 2025-08-09 01:52:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

Refactor CI

Browse Source
This commit is contained in:
k2-fsa 2025-06-30 19:04:02 +08:00
parent 71377d21cd
commit fe36fcc25c
10 changed files with 46 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.github/workflows/aishell.yml vendored
View File

@ -31,8 +31,8 @@ jobs:
id: set-matrix id: set-matrix
run: | run: |
# outputting for debugging purposes # outputting for debugging purposes
python ./.github/scripts/docker/generate_build_matrix.py python ./.github/scripts/docker/generate_build_matrix.py --python-version "3.10"
MATRIX=$(python ./.github/scripts/docker/generate_build_matrix.py) MATRIX=$(python ./.github/scripts/docker/generate_build_matrix.py --python-version "3.10")
echo "::set-output name=matrix::${MATRIX}" echo "::set-output name=matrix::${MATRIX}"
aishell: aishell:
needs: generate_build_matrix needs: generate_build_matrix

10
.github/workflows/audioset.yml vendored
View File

@ -30,8 +30,8 @@ jobs:
id: set-matrix id: set-matrix
run: | run: |
# outputting for debugging purposes # outputting for debugging purposes
python ./.github/scripts/docker/generate_build_matrix.py python ./.github/scripts/docker/generate_build_matrix.py --python-version "3.10"
MATRIX=$(python ./.github/scripts/docker/generate_build_matrix.py) MATRIX=$(python ./.github/scripts/docker/generate_build_matrix.py --python-version "3.10")
echo "::set-output name=matrix::${MATRIX}" echo "::set-output name=matrix::${MATRIX}"
audioset: audioset:
@ -83,7 +83,7 @@ jobs:
ls -lh ./model-onnx/* ls -lh ./model-onnx/*
- name: Upload model to huggingface - name: Upload model to huggingface
if: matrix.python-version == '3.9' && matrix.torch-version == '2.3.0' && github.event_name == 'push' if: matrix.python-version == '3.10' && matrix.torch-version == '2.3.0' && github.event_name == 'push'
env: env:
HF_TOKEN: ${{ secrets.HF_TOKEN }} HF_TOKEN: ${{ secrets.HF_TOKEN }}
uses: nick-fields/retry@v3 uses: nick-fields/retry@v3
@ -116,7 +116,7 @@ jobs:
rm -rf huggingface rm -rf huggingface
- name: Prepare for release - name: Prepare for release
if: matrix.python-version == '3.9' && matrix.torch-version == '2.3.0' && github.event_name == 'push' if: matrix.python-version == '3.10' && matrix.torch-version == '2.3.0' && github.event_name == 'push'
shell: bash shell: bash
run: | run: |
d=sherpa-onnx-zipformer-audio-tagging-2024-04-09 d=sherpa-onnx-zipformer-audio-tagging-2024-04-09
@ -125,7 +125,7 @@ jobs:
ls -lh ls -lh
- name: Release exported onnx models - name: Release exported onnx models
if: matrix.python-version == '3.9' && matrix.torch-version == '2.3.0' && github.event_name == 'push' if: matrix.python-version == '3.10' && matrix.torch-version == '2.3.0' && github.event_name == 'push'
uses: svenstaro/upload-release-action@v2 uses: svenstaro/upload-release-action@v2
with: with:
file_glob: true file_glob: true

20
.github/workflows/baker_zh.yml vendored
View File

@ -31,8 +31,8 @@ jobs:
id: set-matrix id: set-matrix
run: | run: |
# outputting for debugging purposes # outputting for debugging purposes
python ./.github/scripts/docker/generate_build_matrix.py --min-torch-version "2.3" python ./.github/scripts/docker/generate_build_matrix.py --python-version "3.10"
MATRIX=$(python ./.github/scripts/docker/generate_build_matrix.py --min-torch-version "2.3") MATRIX=$(python ./.github/scripts/docker/generate_build_matrix.py --python-version "3.10")
echo "::set-output name=matrix::${MATRIX}" echo "::set-output name=matrix::${MATRIX}"
baker_zh: baker_zh:
@ -84,43 +84,43 @@ jobs:
ls -lh ls -lh
- uses: actions/upload-artifact@v4 - uses: actions/upload-artifact@v4
if: matrix.python-version == '3.9' && matrix.torch-version == '2.3.0' if: matrix.python-version == '3.10' && matrix.torch-version == '2.3.0'
with: with:
name: generated-test-files-${{ matrix.python-version }}-${{ matrix.torch-version }} name: generated-test-files-${{ matrix.python-version }}-${{ matrix.torch-version }}
path: ./*.wav path: ./*.wav
- uses: actions/upload-artifact@v4 - uses: actions/upload-artifact@v4
if: matrix.python-version == '3.9' && matrix.torch-version == '2.3.0' if: matrix.python-version == '3.10' && matrix.torch-version == '2.3.0'
with: with:
name: step-2 name: step-2
path: ./model-steps-2.onnx path: ./model-steps-2.onnx
- uses: actions/upload-artifact@v4 - uses: actions/upload-artifact@v4
if: matrix.python-version == '3.9' && matrix.torch-version == '2.3.0' if: matrix.python-version == '3.10' && matrix.torch-version == '2.3.0'
with: with:
name: step-3 name: step-3
path: ./model-steps-3.onnx path: ./model-steps-3.onnx
- uses: actions/upload-artifact@v4 - uses: actions/upload-artifact@v4
if: matrix.python-version == '3.9' && matrix.torch-version == '2.3.0' if: matrix.python-version == '3.10' && matrix.torch-version == '2.3.0'
with: with:
name: step-4 name: step-4
path: ./model-steps-4.onnx path: ./model-steps-4.onnx
- uses: actions/upload-artifact@v4 - uses: actions/upload-artifact@v4
if: matrix.python-version == '3.9' && matrix.torch-version == '2.3.0' if: matrix.python-version == '3.10' && matrix.torch-version == '2.3.0'
with: with:
name: step-5 name: step-5
path: ./model-steps-5.onnx path: ./model-steps-5.onnx
- uses: actions/upload-artifact@v4 - uses: actions/upload-artifact@v4
if: matrix.python-version == '3.9' && matrix.torch-version == '2.3.0' if: matrix.python-version == '3.10' && matrix.torch-version == '2.3.0'
with: with:
name: step-6 name: step-6
path: ./model-steps-6.onnx path: ./model-steps-6.onnx
- name: Upload models to huggingface - name: Upload models to huggingface
if: matrix.python-version == '3.9' && matrix.torch-version == '2.3.0' && github.event_name == 'push' if: matrix.python-version == '3.10' && matrix.torch-version == '2.3.0' && github.event_name == 'push'
shell: bash shell: bash
env: env:
HF_TOKEN: ${{ secrets.HF_TOKEN }} HF_TOKEN: ${{ secrets.HF_TOKEN }}
@ -141,7 +141,7 @@ jobs:
popd popd
- name: Release exported onnx models - name: Release exported onnx models
if: matrix.python-version == '3.9' && matrix.torch-version == '2.3.0' && github.event_name == 'push' if: matrix.python-version == '3.10' && matrix.torch-version == '2.3.0' && github.event_name == 'push'
uses: svenstaro/upload-release-action@v2 uses: svenstaro/upload-release-action@v2
with: with:
file_glob: true file_glob: true

4
.github/workflows/librispeech.yml vendored
View File

@ -29,8 +29,8 @@ jobs:
id: set-matrix id: set-matrix
run: | run: |
# outputting for debugging purposes # outputting for debugging purposes
python ./.github/scripts/docker/generate_build_matrix.py python ./.github/scripts/docker/generate_build_matrix.py --python-version "3.10"
MATRIX=$(python ./.github/scripts/docker/generate_build_matrix.py) MATRIX=$(python ./.github/scripts/docker/generate_build_matrix.py --python-version "3.10")
echo "::set-output name=matrix::${MATRIX}" echo "::set-output name=matrix::${MATRIX}"
librispeech: librispeech:
needs: generate_build_matrix needs: generate_build_matrix

22
.github/workflows/ljspeech.yml vendored
View File

@ -30,8 +30,8 @@ jobs:
id: set-matrix id: set-matrix
run: | run: |
# outputting for debugging purposes # outputting for debugging purposes
python ./.github/scripts/docker/generate_build_matrix.py --min-torch-version "2.3" python ./.github/scripts/docker/generate_build_matrix.py --python-version "3.10"
MATRIX=$(python ./.github/scripts/docker/generate_build_matrix.py --min-torch-version "2.3") MATRIX=$(python ./.github/scripts/docker/generate_build_matrix.py --python-version "3.10")
echo "::set-output name=matrix::${MATRIX}" echo "::set-output name=matrix::${MATRIX}"
ljspeech: ljspeech:
@ -83,13 +83,13 @@ jobs:
ls -lh ls -lh
- uses: actions/upload-artifact@v4 - uses: actions/upload-artifact@v4
if: matrix.python-version == '3.9' && matrix.torch-version == '2.3.0' if: matrix.python-version == '3.10' && matrix.torch-version == '2.3.0'
with: with:
name: generated-test-files-${{ matrix.python-version }}-${{ matrix.torch-version }} name: generated-test-files-${{ matrix.python-version }}-${{ matrix.torch-version }}
path: ./*.wav path: ./*.wav
- name: Release exported onnx models - name: Release exported onnx models
if: matrix.python-version == '3.9' && matrix.torch-version == '2.3.0' && github.event_name == 'push' if: matrix.python-version == '3.10' && matrix.torch-version == '2.3.0' && github.event_name == 'push'
uses: svenstaro/upload-release-action@v2 uses: svenstaro/upload-release-action@v2
with: with:
file_glob: true file_glob: true
@ -100,37 +100,37 @@ jobs:
tag: tts-models tag: tts-models
- uses: actions/upload-artifact@v4 - uses: actions/upload-artifact@v4
if: matrix.python-version == '3.9' && matrix.torch-version == '2.3.0' if: matrix.python-version == '3.10' && matrix.torch-version == '2.3.0'
with: with:
name: step-2 name: step-2
path: ./model-steps-2.onnx path: ./model-steps-2.onnx
- uses: actions/upload-artifact@v4 - uses: actions/upload-artifact@v4
if: matrix.python-version == '3.9' && matrix.torch-version == '2.3.0' if: matrix.python-version == '3.10' && matrix.torch-version == '2.3.0'
with: with:
name: step-3 name: step-3
path: ./model-steps-3.onnx path: ./model-steps-3.onnx
- uses: actions/upload-artifact@v4 - uses: actions/upload-artifact@v4
if: matrix.python-version == '3.9' && matrix.torch-version == '2.3.0' if: matrix.python-version == '3.10' && matrix.torch-version == '2.3.0'
with: with:
name: step-4 name: step-4
path: ./model-steps-4.onnx path: ./model-steps-4.onnx
- uses: actions/upload-artifact@v4 - uses: actions/upload-artifact@v4
if: matrix.python-version == '3.9' && matrix.torch-version == '2.3.0' if: matrix.python-version == '3.10' && matrix.torch-version == '2.3.0'
with: with:
name: step-5 name: step-5
path: ./model-steps-5.onnx path: ./model-steps-5.onnx
- uses: actions/upload-artifact@v4 - uses: actions/upload-artifact@v4
if: matrix.python-version == '3.9' && matrix.torch-version == '2.3.0' if: matrix.python-version == '3.10' && matrix.torch-version == '2.3.0'
with: with:
name: step-6 name: step-6
path: ./model-steps-6.onnx path: ./model-steps-6.onnx
- name: Upload models to huggingface - name: Upload models to huggingface
if: matrix.python-version == '3.9' && matrix.torch-version == '2.3.0' if: matrix.python-version == '3.10' && matrix.torch-version == '2.3.0'
shell: bash shell: bash
env: env:
HF_TOKEN: ${{ secrets.HF_TOKEN }} HF_TOKEN: ${{ secrets.HF_TOKEN }}
@ -155,7 +155,7 @@ jobs:
popd popd
- name: Release exported onnx models - name: Release exported onnx models
if: matrix.python-version == '3.9' && matrix.torch-version == '2.3.0' if: matrix.python-version == '3.10' && matrix.torch-version == '2.3.0'
uses: svenstaro/upload-release-action@v2 uses: svenstaro/upload-release-action@v2
with: with:
file_glob: true file_glob: true

3
.github/workflows/test.yml vendored
View File

@ -30,8 +30,7 @@ jobs:
id: set-matrix id: set-matrix
run: | run: |
# outputting for debugging purposes # outputting for debugging purposes
python ./.github/scripts/docker/generate_build_matrix.py MATRIX=$(python ./.github/scripts/docker/generate_build_matrix.py --min-torch-version "2.3")
MATRIX=$(python ./.github/scripts/docker/generate_build_matrix.py)
echo "::set-output name=matrix::${MATRIX}" echo "::set-output name=matrix::${MATRIX}"
test: test:
needs: generate_build_matrix needs: generate_build_matrix

4
.github/workflows/yesno.yml vendored
View File

@ -30,8 +30,8 @@ jobs:
id: set-matrix id: set-matrix
run: | run: |
# outputting for debugging purposes # outputting for debugging purposes
python ./.github/scripts/docker/generate_build_matrix.py python ./.github/scripts/docker/generate_build_matrix.py --python-version "3.10"
MATRIX=$(python ./.github/scripts/docker/generate_build_matrix.py) MATRIX=$(python ./.github/scripts/docker/generate_build_matrix.py --python-version "3.10")
echo "::set-output name=matrix::${MATRIX}" echo "::set-output name=matrix::${MATRIX}"
yesno: yesno:
needs: generate_build_matrix needs: generate_build_matrix

2
egs/baker_zh/TTS/local/compute_fbank_baker_zh.py
View File

@ -73,6 +73,8 @@ def compute_fbank_baker_zh(num_jobs: int):
f_min=0, f_min=0,
f_max=8000, f_max=8000,
) )
if not torch.cuda.is_available():
config.device = "cpu"
prefix = "baker_zh" prefix = "baker_zh"
suffix = "jsonl.gz" suffix = "jsonl.gz"

2
egs/ljspeech/TTS/local/compute_fbank_ljspeech.py
View File

@ -73,6 +73,8 @@ def compute_fbank_ljspeech(num_jobs: int):
f_min=0, f_min=0,
f_max=8000, f_max=8000,
) )
if not torch.cuda.is_available():
config.device = "cpu"
prefix = "ljspeech" prefix = "ljspeech"
suffix = "jsonl.gz" suffix = "jsonl.gz"

14
icefall/checkpoint.py
View File

@ -110,7 +110,7 @@ def load_checkpoint(
TODO: document it TODO: document it
""" """
logging.info(f"Loading checkpoint from {filename}") logging.info(f"Loading checkpoint from {filename}")
checkpoint = torch.load(filename, map_location="cpu") checkpoint = torch.load(filename, map_location="cpu", weights_only=False)
if next(iter(checkpoint["model"])).startswith("module."): if next(iter(checkpoint["model"])).startswith("module."):
logging.info("Loading checkpoint saved by DDP") logging.info("Loading checkpoint saved by DDP")
@ -163,7 +163,7 @@ def average_checkpoints(
""" """
n = len(filenames) n = len(filenames)
avg = torch.load(filenames[0], map_location=device)["model"] avg = torch.load(filenames[0], map_location=device, weights_only=False)["model"]
# Identify shared parameters. Two parameters are said to be shared # Identify shared parameters. Two parameters are said to be shared
# if they have the same data_ptr # if they have the same data_ptr
@ -178,7 +178,9 @@ def average_checkpoints(
uniqued_names = list(uniqued.values()) uniqued_names = list(uniqued.values())
for i in range(1, n): for i in range(1, n):
state_dict = torch.load(filenames[i], map_location=device)["model"] state_dict = torch.load(filenames[i], map_location=device, weights_only=False)[
"model"
]
for k in uniqued_names: for k in uniqued_names:
avg[k] += state_dict[k] avg[k] += state_dict[k]
@ -421,8 +423,10 @@ def average_checkpoints_with_averaged_model(
device: device:
Move checkpoints to this device before averaging. Move checkpoints to this device before averaging.
""" """
state_dict_start = torch.load(filename_start, map_location=device) state_dict_start = torch.load(
state_dict_end = torch.load(filename_end, map_location=device) filename_start, map_location=device, weights_only=False
)
state_dict_end = torch.load(filename_end, map_location=device, weights_only=False)
average_period = state_dict_start["average_period"] average_period = state_dict_start["average_period"]