mirrors/icefall
1
0
Fork 0
mirror of https://github.com/k2-fsa/icefall.git synced 2025-08-26 18:24:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update preprocess_mdcc.py

Browse Source
This commit is contained in:
jinzr 2024-03-08 19:20:21 +08:00
parent ba01616103
commit c8dbc01308
1 changed files with 9 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
egs/mdcc/ASR/local/preprocess_mdcc.py
View File

@ -92,16 +92,21 @@ def get_word_segments(lines: List[str]) -> List[str]:
if len(line.split(" ")) > 1: if len(line.split(" ")) > 1:
segments = [] segments = []
for segment in line.split(" "): for segment in line.split(" "):
if not is_cjk(segment[0]): # en segment try:
segments.append(segment) if not is_cjk(segment[0]): # en segment
else: # zh segment segments.append(segment)
segments.extend(pycantonese.segment(segment)) else: # zh segment
segments.extend(pycantonese.segment(segment))
except Exception as e:
logging.error(f"Failed to process segment: {segment}")
raise e
new_lines.append(" ".join(segments) + "\n") new_lines.append(" ".join(segments) + "\n")
# not code switching # not code switching
else: else:
new_lines.append(" ".join(pycantonese.segment(line)) + "\n") new_lines.append(" ".join(pycantonese.segment(line)) + "\n")
except: except:
logging.error(f"Failed to process line: {line}") logging.error(f"Failed to process line: {line}")
raise e
return new_lines return new_lines