mirrors/icefall
1
0
Fork 0
mirror of https://github.com/k2-fsa/icefall.git synced 2025-08-26 18:24:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update preprocess_commonvoice.py

Browse Source
This commit is contained in:
jinzr 2024-03-13 10:36:50 +08:00
parent a39aa8a59d
commit 09a358a23e
1 changed files with 17 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
egs/commonvoice/ASR/local/preprocess_commonvoice.py
View File

@ -56,8 +56,7 @@ def normalize_text(utt: str, language: str) -> str:
# Mozilla Common Voice uses both "yue" and "zh-HK" for Cantonese # Mozilla Common Voice uses both "yue" and "zh-HK" for Cantonese
# Not sure why they decided to do this... # Not sure why they decided to do this...
return ( return (
utt.replace(" ", "") utt.replace("，", "")
.replace("，", "")
.replace("。", " ") .replace("。", " ")
.replace("？", "") .replace("？", "")
.replace("！", "") .replace("！", "")
@ -65,6 +64,22 @@ def normalize_text(utt: str, language: str) -> str:
.replace("!", "") .replace("!", "")
.replace("‘", "") .replace("‘", "")
.replace("、", "") .replace("、", "")
.replace(",", "")
.replace(".", "")
.replace(":", "")
.replace(";", "")
.replace("「", "")
.replace("」", "")
.replace("“", "")
.replace("”", "")
.replace("\\", "")
.replace("~", "")
.replace("—", "")
.replace("ㄧ", "")
.replace("《", "")
.replace("》", "")
.replace("…", "")
.replace("⋯", "")
.upper() .upper()
) )
else: else: