Update prepare_lm_training_data.py

This commit is contained in:
jinzr 2023-11-08 10:20:56 +08:00
parent 403e2e52ac
commit d29efb7345

View File

@ -102,6 +102,8 @@ def main():
if line == "":
break
line = tokenize_by_CJK_char(line)
if line == "":
continue
if step and processed % step == 0:
logging.info(f"Processed number of lines: {processed} ")