mirrors/icefall
1
0
Fork 0
mirror of https://github.com/k2-fsa/icefall.git synced 2025-09-06 07:34:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update prepare_lm_training_data.py

Browse Source
This commit is contained in:
jinzr 2023-11-08 10:05:29 +08:00
parent 1a11440014
commit 94f963baf8
1 changed files with 1 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
egs/multi_zh-hans/ASR/local/prepare_lm_training_data.py
View File

@ -92,7 +92,6 @@ def main():
word2bpe = [] # Will be a list-of-list-of-int, representing BPE pieces. word2bpe = [] # Will be a list-of-list-of-int, representing BPE pieces.
sentences = [] # Will be a list-of-list-of-int, representing word-ids. sentences = [] # Will be a list-of-list-of-int, representing word-ids.
num_lines_in_total = None
step = 500000 step = 500000
processed = 0 processed = 0
@ -105,10 +104,7 @@ def main():
break break
if step and processed % step == 0: if step and processed % step == 0:
logging.info( logging.info(f"Processed number of lines: {processed} ")
f"Processed number of lines: {processed} "
f"({processed/num_lines_in_total*100: .3f}%)"
)
processed += 1 processed += 1
line_words = line.split() line_words = line.split()