mirrors/icefall
1
0
Fork 0
mirror of https://github.com/k2-fsa/icefall.git synced 2025-08-26 18:24:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update preprocess_commonvoice.py

Browse Source
This commit is contained in:
jinzr 2024-03-07 01:20:35 +08:00
parent 335a9962de
commit 5ffc976afc
1 changed files with 11 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
egs/commonvoice/ASR/local/preprocess_commonvoice.py
View File

@ -48,8 +48,18 @@ def normalize_text(utt: str, language: str) -> str:
utt = re.sub("’", "'", utt) utt = re.sub("’", "'", utt)
if language == "en": if language == "en":
return re.sub(r"[^a-zA-Z\s]", "", utt).upper() return re.sub(r"[^a-zA-Z\s]", "", utt).upper()
if language == "fr": elif language == "fr":
return re.sub(r"[^A-ZÀÂÆÇÉÈÊËÎÏÔŒÙÛÜ' ]", "", utt).upper() return re.sub(r"[^A-ZÀÂÆÇÉÈÊËÎÏÔŒÙÛÜ' ]", "", utt).upper()
elif language == "pl":
return re.sub(r"[^a-ząćęłńóśźżA-ZĄĆĘŁŃÓŚŹŻ' ]", "", utt).upper()
else:
raise NotImplementedError(
f"""
Text normalization not implemented for language: {language},
please consider implementing it in the local/preprocess_commonvoice.py
or raise an issue on GitHub to request it.
"""
)
def preprocess_commonvoice( def preprocess_commonvoice(