diff --git a/egs/multi_zh-hans/ASR/RESULTS.md b/egs/multi_zh-hans/ASR/RESULTS.md index e411e80a3..6f5c93ea9 100644 --- a/egs/multi_zh-hans/ASR/RESULTS.md +++ b/egs/multi_zh-hans/ASR/RESULTS.md @@ -43,6 +43,61 @@ Fine-tuned models, training logs, decoding logs, tensorboard and decoding result are available at +### Multi Chinese datasets char-based training results (streaming) on zipformer large model + +#### Streaming (with CTC head) + +The training command for large model (num of params : ~160M): + +Please use the [script](https://github.com/k2-fsa/icefall/blob/master/egs/speech_llm/ASR_LLM/prepare.sh) to prepare fbank features. + +``` +./zipformer/train.py \ + --world-size 8 \ + --num-epochs 20 \ + --use-fp16 1 \ + --max-duration 1200 \ + --num-workers 8 \ + --use-ctc 1 \ + --exp-dir zipformer/exp-large \ + --causal 1 \ + --num-encoder-layers 2,2,4,5,4,2 \ + --feedforward-dim 768,1024,1536,2048,1536,768 \ + --encoder-dim 256,384,512,768,512,256 \ + --encoder-unmasked-dim 192,192,256,320,256,192 + +``` + +The decoding command for transducer greedy search: + +``` +./zipformer/decode.py \ + --epoch 999 \ + --avg 1 \ + --causal 1 \ + --use-averaged-model False \ + --chunk_size -1 + --left-context-frames -1 \ + --use-ctc 1 \ + --exp-dir zipformer/exp-large \ + --max-duration 1200 \ + --num-encoder-layers 2,2,4,5,4,2 \ + --feedforward-dim 768,1024,1536,2048,1536,768 \ + --encoder-dim 256,384,512,768,512,256 \ + --encoder-unmasked-dim 192,192,256,320,256,192 +``` + +Character Error Rates (CERs) listed below are produced by the checkpoint of the 18th epoch using BPE model ( # tokens is 2000, byte fallback enabled). + +| Datasets | alimeeting | alimeeting | aishell-1 | aishell-1 | aishell-2 | aishell-2 | aishell-4 | magicdata | magicdata | kespeech-asr | kespeech-asr | kespeech-asr | WenetSpeech | WenetSpeech | WenetSpeech | +|--------------------------------|-------------------|--------------|----------------|-------------|------------------|-------------|------------------|------------------|-------------|-----------------------|-----------------------|-------------|--------------------|-------------------------|---------------------| +| Zipformer CER (%) | eval | test | dev | test | dev | test | test | dev | test | dev phase1 | dev phase2 | test | dev | test meeting | test net | +| CTC Greedy Streaming | 26.50 | 28.10| 1.71 | 1.97| 3.89| 4.06 | 17.23 | 3.69 | 2.87 | 8.14 | 3.61 |9.51 | 6.11 | 8.13 | 10.62 | +| CTC Greedy Offline | 23.47 | 25.02 | 1.39 | 1.50 | 3.15 | 3.41 | 15.14 | 3.07 | 2.37 | 6.06 | 2.90 | 7.13 | 5.40 | 6.52 | 9.64 | +| Transducer Greedy Offline | 23.16 | 24.78 | 1.33 | 1.38 | 3.06 | 3.23 | 15.36 | 2.54 | 2.09 | 5.24 | 2.28 | 6.26 | 4.87 | 6.26 | 7.07 | +| Transducer Greedy Streaming | 26.83|28.74 | 1.75 | 1.91 | 3.84 | 4.12 | 17.83 | 3.23 | 2.71 | 7.31 | 3.16 | 8.69 | 5.71 | 7.91 | 8.54 | + +Pre-trained model can be found here : https://huggingface.co/yuekai/icefall-asr-multi-zh-hans-zipformer-large ### Multi Chinese datasets char-based training results (Non-streaming) on zipformer model diff --git a/egs/multi_zh-hans/ASR/whisper/multi_dataset.py b/egs/multi_zh-hans/ASR/whisper/multi_dataset.py deleted file mode 100644 index d0054c4f7..000000000 --- a/egs/multi_zh-hans/ASR/whisper/multi_dataset.py +++ /dev/null @@ -1,247 +0,0 @@ -# Copyright 2023 Xiaomi Corp. (authors: Zengrui Jin) -# -# See ../../../../LICENSE for clarification regarding multiple authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import glob -import logging -import re -from pathlib import Path -from typing import Dict, List - -import lhotse -from lhotse import CutSet, load_manifest_lazy - - -class MultiDataset: - def __init__(self, fbank_dir: str): - """ - Args: - manifest_dir: - It is expected to contain the following files: - - aishell_cuts_train.jsonl.gz - - aishell2_cuts_train.jsonl.gz - - aishell4_cuts_train_L.jsonl.gz - - aishell4_cuts_train_M.jsonl.gz - - aishell4_cuts_train_S.jsonl.gz - - alimeeting-far_cuts_train.jsonl.gz - - magicdata_cuts_train.jsonl.gz - - primewords_cuts_train.jsonl.gz - - stcmds_cuts_train.jsonl.gz - - thchs_30_cuts_train.jsonl.gz - - kespeech/kespeech-asr_cuts_train_phase1.jsonl.gz - - kespeech/kespeech-asr_cuts_train_phase2.jsonl.gz - - wenetspeech/cuts_L_fixed.jsonl.gz - """ - self.fbank_dir = Path(fbank_dir) - - def train_cuts(self) -> CutSet: - logging.info("About to get multidataset train cuts") - - # THCHS-30 - logging.info("Loading THCHS-30 in lazy mode") - thchs_30_cuts = load_manifest_lazy( - self.fbank_dir / "thchs_30_cuts_train.jsonl.gz" - ) - - # AISHELL-1 - logging.info("Loading Aishell-1 in lazy mode") - aishell_cuts = load_manifest_lazy( - self.fbank_dir / "aishell_cuts_train.jsonl.gz" - ) - - # AISHELL-2 - logging.info("Loading Aishell-2 in lazy mode") - aishell_2_cuts = load_manifest_lazy( - self.fbank_dir / "aishell2_cuts_train.jsonl.gz" - ) - - # AISHELL-4 - logging.info("Loading Aishell-4 in lazy mode") - aishell_4_L_cuts = load_manifest_lazy( - self.fbank_dir / "aishell4_cuts_train_L.jsonl.gz" - ) - aishell_4_M_cuts = load_manifest_lazy( - self.fbank_dir / "aishell4_cuts_train_M.jsonl.gz" - ) - aishell_4_S_cuts = load_manifest_lazy( - self.fbank_dir / "aishell4_cuts_train_S.jsonl.gz" - ) - - # ST-CMDS - logging.info("Loading ST-CMDS in lazy mode") - stcmds_cuts = load_manifest_lazy(self.fbank_dir / "stcmds_cuts_train.jsonl.gz") - - # Primewords - logging.info("Loading Primewords in lazy mode") - primewords_cuts = load_manifest_lazy( - self.fbank_dir / "primewords_cuts_train.jsonl.gz" - ) - - # MagicData - logging.info("Loading MagicData in lazy mode") - magicdata_cuts = load_manifest_lazy( - self.fbank_dir / "magicdata_cuts_train.jsonl.gz" - ) - - # Ali-Meeting - logging.info("Loading Ali-Meeting in lazy mode") - alimeeting_cuts = load_manifest_lazy( - self.fbank_dir / "alimeeting-far_cuts_train.jsonl.gz" - ) - - # WeNetSpeech - logging.info("Loading WeNetSpeech in lazy mode") - wenetspeech_L_cuts = load_manifest_lazy( - self.fbank_dir / "wenetspeech" / "cuts_L_fixed.jsonl.gz" - ) - - # KeSpeech - logging.info("Loading KeSpeech in lazy mode") - kespeech_1_cuts = load_manifest_lazy( - self.fbank_dir / "kespeech" / "kespeech-asr_cuts_train_phase1.jsonl.gz" - ) - kespeech_2_cuts = load_manifest_lazy( - self.fbank_dir / "kespeech" / "kespeech-asr_cuts_train_phase2.jsonl.gz" - ) - - return CutSet.mux( - thchs_30_cuts, - aishell_cuts, - aishell_2_cuts, - aishell_4_L_cuts, - aishell_4_M_cuts, - aishell_4_S_cuts, - alimeeting_cuts, - stcmds_cuts, - primewords_cuts, - magicdata_cuts, - wenetspeech_L_cuts, - kespeech_1_cuts, - kespeech_2_cuts, - weights=[ - len(thchs_30_cuts), - len(aishell_cuts), - len(aishell_2_cuts), - len(aishell_4_L_cuts), - len(aishell_4_M_cuts), - len(aishell_4_S_cuts), - len(alimeeting_cuts), - len(stcmds_cuts), - len(primewords_cuts), - len(magicdata_cuts), - len(wenetspeech_L_cuts), - len(kespeech_1_cuts), - len(kespeech_2_cuts), - ], - ) - - def dev_cuts(self) -> CutSet: - logging.info("About to get multidataset dev cuts") - - # WeNetSpeech - logging.info("Loading WeNetSpeech DEV set in lazy mode") - wenetspeech_dev_cuts = load_manifest_lazy( - self.fbank_dir / "wenetspeech" / "cuts_DEV_fixed.jsonl.gz" - ) - - return wenetspeech_dev_cuts - - def test_cuts(self) -> Dict[str, CutSet]: - logging.info("About to get multidataset test cuts") - - # AISHELL - logging.info("Loading Aishell set in lazy mode") - aishell_test_cuts = load_manifest_lazy( - self.fbank_dir / "aishell_cuts_test.jsonl.gz" - ) - aishell_dev_cuts = load_manifest_lazy( - self.fbank_dir / "aishell_cuts_dev.jsonl.gz" - ) - - # AISHELL-2 - logging.info("Loading Aishell-2 set in lazy mode") - aishell2_test_cuts = load_manifest_lazy( - self.fbank_dir / "aishell2_cuts_test.jsonl.gz" - ) - aishell2_dev_cuts = load_manifest_lazy( - self.fbank_dir / "aishell2_cuts_dev.jsonl.gz" - ) - - # AISHELL-4 - logging.info("Loading Aishell-4 TEST set in lazy mode") - aishell4_test_cuts = load_manifest_lazy( - self.fbank_dir / "aishell4_cuts_test.jsonl.gz" - ) - - # Ali-Meeting - logging.info("Loading Ali-Meeting set in lazy mode") - alimeeting_test_cuts = load_manifest_lazy( - self.fbank_dir / "alimeeting-far_cuts_test.jsonl.gz" - ) - alimeeting_eval_cuts = load_manifest_lazy( - self.fbank_dir / "alimeeting-far_cuts_eval.jsonl.gz" - ) - - # MagicData - logging.info("Loading MagicData set in lazy mode") - magicdata_test_cuts = load_manifest_lazy( - self.fbank_dir / "magicdata_cuts_test.jsonl.gz" - ) - magicdata_dev_cuts = load_manifest_lazy( - self.fbank_dir / "magicdata_cuts_dev.jsonl.gz" - ) - - # KeSpeech - logging.info("Loading KeSpeech set in lazy mode") - kespeech_test_cuts = load_manifest_lazy( - self.fbank_dir / "kespeech" / "kespeech-asr_cuts_test.jsonl.gz" - ) - kespeech_dev_phase1_cuts = load_manifest_lazy( - self.fbank_dir / "kespeech" / "kespeech-asr_cuts_dev_phase1.jsonl.gz" - ) - kespeech_dev_phase2_cuts = load_manifest_lazy( - self.fbank_dir / "kespeech" / "kespeech-asr_cuts_dev_phase2.jsonl.gz" - ) - - # WeNetSpeech - logging.info("Loading WeNetSpeech set in lazy mode") - wenetspeech_test_meeting_cuts = load_manifest_lazy( - self.fbank_dir / "wenetspeech" / "cuts_TEST_MEETING.jsonl.gz" - ) - wenetspeech_test_net_cuts = load_manifest_lazy( - self.fbank_dir / "wenetspeech" / "cuts_TEST_NET.jsonl.gz" - ) - wenetspeech_dev_cuts = load_manifest_lazy( - self.fbank_dir / "wenetspeech" / "cuts_DEV_fixed.jsonl.gz" - ) - - return { - "wenetspeech-meeting_test": wenetspeech_test_meeting_cuts, - # "aishell_test": aishell_test_cuts, - # "aishell_dev": aishell_dev_cuts, - # "ali-meeting_test": alimeeting_test_cuts, - # "ali-meeting_eval": alimeeting_eval_cuts, - # "aishell-4_test": aishell4_test_cuts, - # "aishell-2_test": aishell2_test_cuts, - # "aishell-2_dev": aishell2_dev_cuts, - # "magicdata_test": magicdata_test_cuts, - # "magicdata_dev": magicdata_dev_cuts, - # "kespeech-asr_test": kespeech_test_cuts, - # "kespeech-asr_dev_phase1": kespeech_dev_phase1_cuts, - # "kespeech-asr_dev_phase2": kespeech_dev_phase2_cuts, - # "wenetspeech-net_test": wenetspeech_test_net_cuts, - # "wenetspeech_dev": wenetspeech_dev_cuts, - } diff --git a/egs/multi_zh-hans/ASR/whisper/multi_dataset.py b/egs/multi_zh-hans/ASR/whisper/multi_dataset.py new file mode 120000 index 000000000..d2e14a1ad --- /dev/null +++ b/egs/multi_zh-hans/ASR/whisper/multi_dataset.py @@ -0,0 +1 @@ +../../../speech_llm/ASR_LLM/whisper_llm_zh/multi_dataset.py \ No newline at end of file diff --git a/egs/multi_zh-hans/ASR/zipformer/ctc_decode.py b/egs/multi_zh-hans/ASR/zipformer/ctc_decode.py index 5143f945d..8d4a81fb0 100755 --- a/egs/multi_zh-hans/ASR/zipformer/ctc_decode.py +++ b/egs/multi_zh-hans/ASR/zipformer/ctc_decode.py @@ -46,7 +46,7 @@ import torch.nn as nn from asr_datamodule import AsrDataModule from lhotse.cut import Cut from multi_dataset import MultiDataset -from train import add_model_arguments, get_model, get_params +from train import add_model_arguments, get_model, get_params, normalize_text_alimeeting from icefall.checkpoint import ( average_checkpoints, @@ -367,21 +367,18 @@ def decode_dataset( hyps_dict = decode_one_batch( params=params, model=model, - HLG=HLG, H=H, bpe_model=bpe_model, batch=batch, - word_table=word_table, - G=G, ) for name, hyps in hyps_dict.items(): this_batch = [] assert len(hyps) == len(texts) for cut_id, hyp_words, ref_text in zip(cut_ids, hyps, texts): - ref_words = list(ref_text.replace(" ", "")) - hyp_words = list("".join(hyp_words)) - this_batch.append((cut_id, ref_words, hyp_words)) + ref_text = normalize_text_alimeeting(ref_text) + hyp_text = "".join(hyp_words) + this_batch.append((cut_id, ref_text, hyp_text)) results[name].extend(this_batch) @@ -583,7 +580,7 @@ def main(): data_module = AsrDataModule(args) multi_dataset = MultiDataset(args.manifest_dir) - test_sets_cuts = multi_dataset.test_cuts() + test_sets_cuts = {**multi_dataset.test_cuts(), **multi_dataset.speechio_test_cuts()} def remove_short_utt(c: Cut): T = ((c.num_frames - 7) // 2 + 1) // 2 diff --git a/egs/multi_zh-hans/ASR/zipformer/decode.py b/egs/multi_zh-hans/ASR/zipformer/decode.py index f501c3c30..5993f243f 100755 --- a/egs/multi_zh-hans/ASR/zipformer/decode.py +++ b/egs/multi_zh-hans/ASR/zipformer/decode.py @@ -118,7 +118,7 @@ from beam_search import ( ) from lhotse.cut import Cut from multi_dataset import MultiDataset -from train import add_model_arguments, get_model, get_params +from train import add_model_arguments, get_model, get_params, normalize_text_alimeeting from icefall.checkpoint import ( average_checkpoints, @@ -532,7 +532,6 @@ def decode_dataset( results = defaultdict(list) for batch_idx, batch in enumerate(dl): texts = batch["supervisions"]["text"] - texts = [list(str(text).replace(" ", "")) for text in texts] cut_ids = [cut.id for cut in batch["supervisions"]["cut"]] hyps_dict = decode_one_batch( @@ -548,6 +547,7 @@ def decode_dataset( this_batch = [] assert len(hyps) == len(texts) for cut_id, hyp_words, ref_text in zip(cut_ids, hyps, texts): + ref_text = normalize_text_alimeeting(ref_text) hyp_text = "".join(hyp_words) this_batch.append((cut_id, ref_text, hyp_text)) @@ -795,7 +795,7 @@ def main(): ) return T > 0 - test_sets_cuts = multi_dataset.test_cuts() + test_sets_cuts = {**multi_dataset.test_cuts(), **multi_dataset.speechio_test_cuts()} test_sets = test_sets_cuts.keys() test_dl = [ diff --git a/egs/multi_zh-hans/ASR/zipformer/multi_dataset.py b/egs/multi_zh-hans/ASR/zipformer/multi_dataset.py deleted file mode 100644 index b1920e62e..000000000 --- a/egs/multi_zh-hans/ASR/zipformer/multi_dataset.py +++ /dev/null @@ -1,316 +0,0 @@ -# Copyright 2023 Xiaomi Corp. (authors: Zengrui Jin) -# -# See ../../../../LICENSE for clarification regarding multiple authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import glob -import logging -import re -from pathlib import Path -from typing import Dict, List - -import lhotse -from lhotse import CutSet, load_manifest_lazy - - -class MultiDataset: - def __init__(self, fbank_dir: str): - """ - Args: - manifest_dir: - It is expected to contain the following files: - - aidatatang_cuts_train.jsonl.gz - - aishell_cuts_train.jsonl.gz - - aishell2_cuts_train.jsonl.gz - - aishell4_cuts_train_L.jsonl.gz - - aishell4_cuts_train_M.jsonl.gz - - aishell4_cuts_train_S.jsonl.gz - - alimeeting-far_cuts_train.jsonl.gz - - magicdata_cuts_train.jsonl.gz - - primewords_cuts_train.jsonl.gz - - stcmds_cuts_train.jsonl.gz - - thchs_30_cuts_train.jsonl.gz - - kespeech/kespeech-asr_cuts_train_phase1.jsonl.gz - - kespeech/kespeech-asr_cuts_train_phase2.jsonl.gz - - wenetspeech/cuts_L.jsonl.gz - """ - self.fbank_dir = Path(fbank_dir) - - def train_cuts(self) -> CutSet: - logging.info("About to get multidataset train cuts") - - # THCHS-30 - logging.info("Loading THCHS-30 in lazy mode") - thchs_30_cuts = load_manifest_lazy( - self.fbank_dir / "thchs_30_cuts_train.jsonl.gz" - ) - - # AISHELL-1 - logging.info("Loading Aishell-1 in lazy mode") - aishell_cuts = load_manifest_lazy( - self.fbank_dir / "aishell_cuts_train.jsonl.gz" - ) - - # AISHELL-2 - logging.info("Loading Aishell-2 in lazy mode") - aishell_2_cuts = load_manifest_lazy( - self.fbank_dir / "aishell2_cuts_train.jsonl.gz" - ) - - # AISHELL-4 - logging.info("Loading Aishell-4 in lazy mode") - aishell_4_L_cuts = load_manifest_lazy( - self.fbank_dir / "aishell4_cuts_train_L.jsonl.gz" - ) - aishell_4_M_cuts = load_manifest_lazy( - self.fbank_dir / "aishell4_cuts_train_M.jsonl.gz" - ) - aishell_4_S_cuts = load_manifest_lazy( - self.fbank_dir / "aishell4_cuts_train_S.jsonl.gz" - ) - - # ST-CMDS - logging.info("Loading ST-CMDS in lazy mode") - stcmds_cuts = load_manifest_lazy(self.fbank_dir / "stcmds_cuts_train.jsonl.gz") - - # Primewords - logging.info("Loading Primewords in lazy mode") - primewords_cuts = load_manifest_lazy( - self.fbank_dir / "primewords_cuts_train.jsonl.gz" - ) - - # MagicData - logging.info("Loading MagicData in lazy mode") - magicdata_cuts = load_manifest_lazy( - self.fbank_dir / "magicdata_cuts_train.jsonl.gz" - ) - - # Aidatatang_200zh - logging.info("Loading Aidatatang_200zh in lazy mode") - aidatatang_200zh_cuts = load_manifest_lazy( - self.fbank_dir / "aidatatang_cuts_train.jsonl.gz" - ) - - # Ali-Meeting - logging.info("Loading Ali-Meeting in lazy mode") - alimeeting_cuts = load_manifest_lazy( - self.fbank_dir / "alimeeting-far_cuts_train.jsonl.gz" - ) - - # WeNetSpeech - logging.info("Loading WeNetSpeech in lazy mode") - wenetspeech_L_cuts = load_manifest_lazy( - self.fbank_dir / "wenetspeech" / "cuts_L.jsonl.gz" - ) - - # KeSpeech - logging.info("Loading KeSpeech in lazy mode") - kespeech_1_cuts = load_manifest_lazy( - self.fbank_dir / "kespeech" / "kespeech-asr_cuts_train_phase1.jsonl.gz" - ) - kespeech_2_cuts = load_manifest_lazy( - self.fbank_dir / "kespeech" / "kespeech-asr_cuts_train_phase2.jsonl.gz" - ) - - return CutSet.mux( - thchs_30_cuts, - aishell_cuts, - aishell_2_cuts, - aishell_4_L_cuts, - aishell_4_M_cuts, - aishell_4_S_cuts, - stcmds_cuts, - primewords_cuts, - magicdata_cuts, - aidatatang_200zh_cuts, - alimeeting_cuts, - wenetspeech_L_cuts, - kespeech_1_cuts, - kespeech_2_cuts, - weights=[ - len(thchs_30_cuts), - len(aishell_cuts), - len(aishell_2_cuts), - len(aishell_4_L_cuts), - len(aishell_4_M_cuts), - len(aishell_4_S_cuts), - len(stcmds_cuts), - len(primewords_cuts), - len(magicdata_cuts), - len(aidatatang_200zh_cuts), - len(alimeeting_cuts), - len(wenetspeech_L_cuts), - len(kespeech_1_cuts), - len(kespeech_2_cuts), - ], - ) - - def dev_cuts(self) -> CutSet: - logging.info("About to get multidataset dev cuts") - - # Aidatatang_200zh - logging.info("Loading Aidatatang_200zh DEV set in lazy mode") - aidatatang_dev_cuts = load_manifest_lazy( - self.fbank_dir / "aidatatang_cuts_dev.jsonl.gz" - ) - - # AISHELL - logging.info("Loading Aishell DEV set in lazy mode") - aishell_dev_cuts = load_manifest_lazy( - self.fbank_dir / "aishell_cuts_dev.jsonl.gz" - ) - - # AISHELL-2 - logging.info("Loading Aishell-2 DEV set in lazy mode") - aishell2_dev_cuts = load_manifest_lazy( - self.fbank_dir / "aishell2_cuts_dev.jsonl.gz" - ) - - # Ali-Meeting - logging.info("Loading Ali-Meeting DEV set in lazy mode") - alimeeting_dev_cuts = load_manifest_lazy( - self.fbank_dir / "alimeeting-far_cuts_eval.jsonl.gz" - ) - - # MagicData - logging.info("Loading MagicData DEV set in lazy mode") - magicdata_dev_cuts = load_manifest_lazy( - self.fbank_dir / "magicdata_cuts_dev.jsonl.gz" - ) - - # KeSpeech - logging.info("Loading KeSpeech DEV set in lazy mode") - kespeech_dev_phase1_cuts = load_manifest_lazy( - self.fbank_dir / "kespeech" / "kespeech-asr_cuts_dev_phase1.jsonl.gz" - ) - kespeech_dev_phase2_cuts = load_manifest_lazy( - self.fbank_dir / "kespeech" / "kespeech-asr_cuts_dev_phase2.jsonl.gz" - ) - - # WeNetSpeech - logging.info("Loading WeNetSpeech DEV set in lazy mode") - wenetspeech_dev_cuts = load_manifest_lazy( - self.fbank_dir / "wenetspeech" / "cuts_DEV.jsonl.gz" - ) - - return wenetspeech_dev_cuts - # return [ - # aidatatang_dev_cuts, - # aishell_dev_cuts, - # aishell2_dev_cuts, - # alimeeting_dev_cuts, - # magicdata_dev_cuts, - # kespeech_dev_phase1_cuts, - # kespeech_dev_phase2_cuts, - # wenetspeech_dev_cuts, - # ] - - def test_cuts(self) -> Dict[str, CutSet]: - logging.info("About to get multidataset test cuts") - - # Aidatatang_200zh - logging.info("Loading Aidatatang_200zh set in lazy mode") - aidatatang_test_cuts = load_manifest_lazy( - self.fbank_dir / "aidatatang_cuts_test.jsonl.gz" - ) - aidatatang_dev_cuts = load_manifest_lazy( - self.fbank_dir / "aidatatang_cuts_dev.jsonl.gz" - ) - - # AISHELL - logging.info("Loading Aishell set in lazy mode") - aishell_test_cuts = load_manifest_lazy( - self.fbank_dir / "aishell_cuts_test.jsonl.gz" - ) - aishell_dev_cuts = load_manifest_lazy( - self.fbank_dir / "aishell_cuts_dev.jsonl.gz" - ) - - # AISHELL-2 - logging.info("Loading Aishell-2 set in lazy mode") - aishell2_test_cuts = load_manifest_lazy( - self.fbank_dir / "aishell2_cuts_test.jsonl.gz" - ) - aishell2_dev_cuts = load_manifest_lazy( - self.fbank_dir / "aishell2_cuts_dev.jsonl.gz" - ) - - # AISHELL-4 - logging.info("Loading Aishell-4 TEST set in lazy mode") - aishell4_test_cuts = load_manifest_lazy( - self.fbank_dir / "aishell4_cuts_test.jsonl.gz" - ) - - # Ali-Meeting - logging.info("Loading Ali-Meeting set in lazy mode") - alimeeting_test_cuts = load_manifest_lazy( - self.fbank_dir / "alimeeting-far_cuts_test.jsonl.gz" - ) - alimeeting_eval_cuts = load_manifest_lazy( - self.fbank_dir / "alimeeting-far_cuts_eval.jsonl.gz" - ) - - # MagicData - logging.info("Loading MagicData set in lazy mode") - magicdata_test_cuts = load_manifest_lazy( - self.fbank_dir / "magicdata_cuts_test.jsonl.gz" - ) - magicdata_dev_cuts = load_manifest_lazy( - self.fbank_dir / "magicdata_cuts_dev.jsonl.gz" - ) - - # KeSpeech - logging.info("Loading KeSpeech set in lazy mode") - kespeech_test_cuts = load_manifest_lazy( - self.fbank_dir / "kespeech" / "kespeech-asr_cuts_test.jsonl.gz" - ) - kespeech_dev_phase1_cuts = load_manifest_lazy( - self.fbank_dir / "kespeech" / "kespeech-asr_cuts_dev_phase1.jsonl.gz" - ) - kespeech_dev_phase2_cuts = load_manifest_lazy( - self.fbank_dir / "kespeech" / "kespeech-asr_cuts_dev_phase2.jsonl.gz" - ) - - # WeNetSpeech - logging.info("Loading WeNetSpeech set in lazy mode") - wenetspeech_test_meeting_cuts = load_manifest_lazy( - self.fbank_dir / "wenetspeech" / "cuts_TEST_MEETING.jsonl.gz" - ) - wenetspeech_test_net_cuts = load_manifest_lazy( - self.fbank_dir / "wenetspeech" / "cuts_TEST_NET.jsonl.gz" - ) - wenetspeech_dev_cuts = load_manifest_lazy( - self.fbank_dir / "wenetspeech" / "cuts_DEV.jsonl.gz" - ) - - return { - "aidatatang_test": aidatatang_test_cuts, - "aidatatang_dev": aidatatang_dev_cuts, - "alimeeting_test": alimeeting_test_cuts, - "alimeeting_eval": alimeeting_eval_cuts, - "aishell_test": aishell_test_cuts, - "aishell_dev": aishell_dev_cuts, - "aishell-2_test": aishell2_test_cuts, - "aishell-2_dev": aishell2_dev_cuts, - "aishell-4": aishell4_test_cuts, - "magicdata_test": magicdata_test_cuts, - "magicdata_dev": magicdata_dev_cuts, - "kespeech-asr_test": kespeech_test_cuts, - "kespeech-asr_dev_phase1": kespeech_dev_phase1_cuts, - "kespeech-asr_dev_phase2": kespeech_dev_phase2_cuts, - "wenetspeech-meeting_test": wenetspeech_test_meeting_cuts, - "wenetspeech-net_test": wenetspeech_test_net_cuts, - "wenetspeech_dev": wenetspeech_dev_cuts, - } diff --git a/egs/multi_zh-hans/ASR/zipformer/multi_dataset.py b/egs/multi_zh-hans/ASR/zipformer/multi_dataset.py new file mode 120000 index 000000000..d2e14a1ad --- /dev/null +++ b/egs/multi_zh-hans/ASR/zipformer/multi_dataset.py @@ -0,0 +1 @@ +../../../speech_llm/ASR_LLM/whisper_llm_zh/multi_dataset.py \ No newline at end of file diff --git a/egs/multi_zh-hans/ASR/zipformer/train.py b/egs/multi_zh-hans/ASR/zipformer/train.py index 447ca122f..1fc4c35c1 100755 --- a/egs/multi_zh-hans/ASR/zipformer/train.py +++ b/egs/multi_zh-hans/ASR/zipformer/train.py @@ -539,6 +539,43 @@ def get_params() -> AttributeDict: return params +def normalize_text_alimeeting(text: str, normalize: str = "m2met") -> str: + """ + Text normalization similar to M2MeT challenge baseline. + See: https://github.com/yufan-aslp/AliMeeting/blob/main/asr/local/text_normalize.pl + """ + if normalize == "none": + return text + elif normalize == "m2met": + import re + + text = text.replace(" ", "") + text = text.replace("", "") + text = text.replace("<%>", "") + text = text.replace("<->", "") + text = text.replace("<$>", "") + text = text.replace("<#>", "") + text = text.replace("<_>", "") + text = text.replace("", "") + text = text.replace("`", "") + text = text.replace("&", "") + text = text.replace(",", "") + if re.search("[a-zA-Z]", text): + text = text.upper() + text = text.replace("Ａ", "A") + text = text.replace("ａ", "A") + text = text.replace("ｂ", "B") + text = text.replace("ｃ", "C") + text = text.replace("ｋ", "K") + text = text.replace("ｔ", "T") + text = text.replace("，", "") + text = text.replace("丶", "") + text = text.replace("。", "") + text = text.replace("、", "") + text = text.replace("？", "") + return text + + def _to_int_tuple(s: str): return tuple(map(int, s.split(","))) @@ -788,6 +825,9 @@ def compute_loss( warm_step = params.warm_step texts = batch["supervisions"]["text"] + # remove spaces in texts + texts = [normalize_text_alimeeting(text) for text in texts] + y = sp.encode(texts, out_type=int) y = k2.RaggedTensor(y) diff --git a/egs/speechio/ASR/local/normalize_results.py b/egs/speechio/ASR/local/normalize_results.py index 79d886617..02277e2a8 100755 --- a/egs/speechio/ASR/local/normalize_results.py +++ b/egs/speechio/ASR/local/normalize_results.py @@ -114,7 +114,8 @@ def extract_hyp_ref_wavname(filename): for line in f: if "ref" in line: ref = line.split("ref=")[1].strip() - ref = ref[2:-2] + if ref[0] == "[": + ref = ref[2:-2] list_elements = ref.split("', '") ref = "".join(list_elements) refs.append(ref)