mirror of
https://github.com/k2-fsa/icefall.git
synced 2025-09-06 23:54:17 +00:00
38 lines
650 B
Python
Executable File
38 lines
650 B
Python
Executable File
#!/usr/bin/env python3
|
|
|
|
"""
|
|
This script takes `bpe.model` as input and generates a file `tokens.txt`
|
|
from it.
|
|
|
|
Usage:
|
|
./bpe_model_to_tokens.py /path/to/input/bpe.model > tokens.txt
|
|
"""
|
|
import argparse
|
|
|
|
import sentencepiece as spm
|
|
|
|
|
|
def get_args():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument(
|
|
"bpe_model",
|
|
type=str,
|
|
help="Path to the input bpe.model",
|
|
)
|
|
|
|
return parser.parse_args()
|
|
|
|
|
|
def main():
|
|
args = get_args()
|
|
|
|
sp = spm.SentencePieceProcessor()
|
|
sp.load(args.bpe_model)
|
|
|
|
for i in range(sp.vocab_size()):
|
|
print(sp.id_to_piece(i), i)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|