mirrors/icefall
1
0
Fork 0
mirror of https://github.com/k2-fsa/icefall.git synced 2025-08-26 18:24:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update preprocess_commonvoice.py

Browse Source
This commit is contained in:
jinzr 2024-03-15 19:31:35 +08:00
parent d9a0ab59db
commit 6993183dd7
1 changed files with 10 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
egs/commonvoice/ASR/local/preprocess_commonvoice.py
View File

@ -56,48 +56,16 @@ def normalize_text(utt: str, language: str) -> str:
# Mozilla Common Voice uses both "yue" and "zh-HK" for Cantonese
# Not sure why they decided to do this...
# None en/zh-yue tokens are manually removed here
return (
utt.replace("，", "")
.replace("。", " ")
.replace("？", "")
.replace("！", "")
.replace("?", "")
.replace("!", "")
.replace("‘", "")
.replace("、", "")
.replace(",", "")
.replace(".", "")
.replace(":", "")
.replace(";", "")
.replace("「", "")
.replace("」", "")
.replace("“", "")
.replace("”", "")
.replace("\\", "")
.replace("~", "")
.replace("—", "")
.replace("ㄧ", "")
.replace("《", "")
.replace("》", "")
.replace("…", "")
.replace("⋯", "")
.replace("·", "")
.replace("﹒", "")
.replace("．", "")
.replace("：", "")
.replace("︰", "")
.replace("﹖", "")
.replace("（", "")
.replace("）", "")
.replace("－", "")
.replace("～", "")
.replace("；", "")
.replace("", "")
.replace("﹔", "")
.replace("／", "")
.replace("Ａ", "")
.replace("Ｂ", "")
.upper()
# fmt: off
tokens_to_remove = ["，", "。", "？", "！", "?", "!", "‘", "、", ",", "\.", ":", ";", "「", "」", "“", "”", "~", "—", "ㄧ", "《", "》", "…", "⋯", "·", "﹒", "．", "：", "︰", "﹖", "（", "）", "－", "～", "；", "", "⠀", "﹔", "／", "Ａ", "Ｂ", "–", "‧"]
# fmt: on
utt = utt.upper().replace("\\", "")
return re.sub(
pattern="|".join([f"[{token}]" for token in tokens_to_remove]),
repl="",
string=utt,
)
else:
raise NotImplementedError(