mirrors/icefall
1
0
Fork 0
mirror of https://github.com/k2-fsa/icefall.git synced 2025-08-08 09:32:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix decoding byte bpes tokens to words. (#1966)

Browse Source
This commit is contained in:
Wei Kang 2025-06-19 12:26:01 +08:00 committed by GitHub
parent 762f965cf7
commit 3587c4b3b7
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 7 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
egs/multi_zh-hans/ASR/zipformer/pretrained.py
View File

@ -328,9 +328,14 @@ def main():
logging.info(msg) logging.info(msg)
def token_ids_to_words(token_ids: List[int]) -> str: def token_ids_to_words(token_ids: List[int]) -> str:
text = "" byte_list = []
for i in token_ids: for i in token_ids:
text += token_table[i] token = token_table[i]
if token.startswith("<0x") and token.endswith(">"):
byte_list.append(int(token[3:-1], 16))
else:
byte_list += list(token.encode("utf-8"))
text = bytes(byte_list).decode("utf-8", errors='ignore')
return text.replace("▁", " ").strip() return text.replace("▁", " ").strip()
if params.method == "fast_beam_search": if params.method == "fast_beam_search":