fixed a formatting issue

This commit is contained in:
zr_jin 2023-07-23 12:24:25 +08:00
parent 20d4031e9b
commit a4e3186e4f
5 changed files with 5 additions and 5 deletions

View File

@ -269,7 +269,7 @@ def main():
# Load tokens.txt here # Load tokens.txt here
token_table = k2.SymbolTable.from_file(params.tokens) token_table = k2.SymbolTable.from_file(params.tokens)
# Load id of the <blk> token and the vocab size, <blk> is # Load id of the <blk> token and the vocab size, <blk> is
# defined in local/train_bpe_model.py # defined in local/train_bpe_model.py
params.blank_id = token_table["<blk>"] params.blank_id = token_table["<blk>"]
params.vocab_size = num_tokens(token_table) + 1 # +1 for <blk> params.vocab_size = num_tokens(token_table) + 1 # +1 for <blk>

View File

@ -224,7 +224,7 @@ def main():
# Load tokens.txt here # Load tokens.txt here
token_table = k2.SymbolTable.from_file(params.tokens) token_table = k2.SymbolTable.from_file(params.tokens)
# Load id of the <blk> token and the vocab size, <blk> is # Load id of the <blk> token and the vocab size, <blk> is
# defined in local/train_bpe_model.py # defined in local/train_bpe_model.py
params.blank_id = token_table["<blk>"] params.blank_id = token_table["<blk>"]
params.vocab_size = num_tokens(token_table) + 1 # +1 for <blk> params.vocab_size = num_tokens(token_table) + 1 # +1 for <blk>

View File

@ -440,7 +440,7 @@ def main():
# Load tokens.txt here # Load tokens.txt here
token_table = k2.SymbolTable.from_file(params.tokens) token_table = k2.SymbolTable.from_file(params.tokens)
# Load id of the <blk> token and the vocab size, <blk> is # Load id of the <blk> token and the vocab size, <blk> is
# defined in local/train_bpe_model.py # defined in local/train_bpe_model.py
params.blank_id = token_table["<blk>"] params.blank_id = token_table["<blk>"]
params.vocab_size = num_tokens(token_table) + 1 # +1 for <blk> params.vocab_size = num_tokens(token_table) + 1 # +1 for <blk>

View File

@ -270,7 +270,7 @@ def main():
# Load tokens.txt here # Load tokens.txt here
token_table = k2.SymbolTable.from_file(params.tokens) token_table = k2.SymbolTable.from_file(params.tokens)
# Load id of the <blk> token and the vocab size, <blk> is # Load id of the <blk> token and the vocab size, <blk> is
# defined in local/train_bpe_model.py # defined in local/train_bpe_model.py
params.blank_id = token_table["<blk>"] params.blank_id = token_table["<blk>"]
params.vocab_size = num_tokens(token_table) + 1 # +1 for <blk> params.vocab_size = num_tokens(token_table) + 1 # +1 for <blk>

View File

@ -269,7 +269,7 @@ def main():
# Load tokens.txt here # Load tokens.txt here
token_table = k2.SymbolTable.from_file(params.tokens) token_table = k2.SymbolTable.from_file(params.tokens)
# Load id of the <blk> token and the vocab size, <blk> is # Load id of the <blk> token and the vocab size, <blk> is
# defined in local/train_bpe_model.py # defined in local/train_bpe_model.py
params.blank_id = token_table["<blk>"] params.blank_id = token_table["<blk>"]
params.vocab_size = num_tokens(token_table) + 1 # +1 for <blk> params.vocab_size = num_tokens(token_table) + 1 # +1 for <blk>