mirrors/icefall
1
0
Fork 0
mirror of https://github.com/k2-fsa/icefall.git synced 2025-08-26 18:24:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add generate_tokens function to train_bpe_model.py.

Browse Source
This commit is contained in:
Kinan Martin 2025-08-15 16:01:45 +09:00
parent aeba8b505c
commit 572eef2bd5
1 changed files with 11 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
egs/mls_english/ASR/local/train_bpe_model.py
View File

@ -70,6 +70,16 @@ def get_args():
return parser.parse_args() return parser.parse_args()
def generate_tokens(lang_dir: Path):
"""
Generate the tokens.txt from a bpe model.
"""
sp = spm.SentencePieceProcessor()
sp.load(str(lang_dir / "bpe.model"))
token2id: Dict[str, int] = {sp.id_to_piece(i): i for i in range(sp.vocab_size())}
with open(lang_dir / "tokens.txt", "w", encoding="utf-8") as f:
for sym, i in token2id.items():
f.write(f"{sym} {i}\n")
def main(): def main():
args = get_args() args = get_args()
@ -109,6 +119,7 @@ def main():
shutil.copyfile(model_file, f"{lang_dir}/bpe.model") shutil.copyfile(model_file, f"{lang_dir}/bpe.model")
generate_tokens(lang_dir)
if __name__ == "__main__": if __name__ == "__main__":
main() main()