mirrors/icefall
1
0
Fork 0
mirror of https://github.com/k2-fsa/icefall.git synced 2025-09-06 15:44:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Use tokens.txt to replace bpe.model

Browse Source
This commit is contained in:
Fangjun Kuang 2024-01-26 12:05:21 +08:00
parent c65b734838
commit 71ee509e7d
2 changed files with 15 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
egs/gigaspeech/ASR/pruned_transducer_stateless2/export.py
View File

@ -22,7 +22,7 @@
Usage: Usage:
./pruned_transducer_stateless2/export.py \ ./pruned_transducer_stateless2/export.py \
--exp-dir ./pruned_transducer_stateless2/exp \ --exp-dir ./pruned_transducer_stateless2/exp \
--bpe-model data/lang_bpe_500/bpe.model \ --tokens ./data/lang_bpe_500/tokens.txt \
--epoch 20 \ --epoch 20 \
--avg 10 --avg 10
@ -47,13 +47,13 @@ import argparse
import logging import logging
from pathlib import Path from pathlib import Path
import sentencepiece as spm import k2
import torch import torch
from scaling_converter import convert_scaled_to_non_scaled from scaling_converter import convert_scaled_to_non_scaled
from train import get_params, get_transducer_model from train import get_params, get_transducer_model
from icefall.checkpoint import average_checkpoints, find_checkpoints, load_checkpoint from icefall.checkpoint import average_checkpoints, find_checkpoints, load_checkpoint
from icefall.utils import str2bool from icefall.utils import num_tokens, str2bool
def get_parser(): def get_parser():
@ -99,10 +99,10 @@ def get_parser():
) )
parser.add_argument( parser.add_argument(
"--bpe-model", "--tokens",
type=str, type=str,
default="data/lang_bpe_500/bpe.model", default="data/lang_bpe_500/tokens.txt",
help="Path to the BPE model", help="Path to the tokens.txt.",
) )
parser.add_argument( parser.add_argument(
@ -136,12 +136,14 @@ def main():
logging.info(f"device: {device}") logging.info(f"device: {device}")
sp = spm.SentencePieceProcessor() # Load tokens.txt here
sp.load(params.bpe_model) token_table = k2.SymbolTable.from_file(params.tokens)
# Load id of the <blk> token and the vocab size
# <blk> is defined in local/train_bpe_model.py # <blk> is defined in local/train_bpe_model.py
params.blank_id = sp.piece_to_id("<blk>") params.blank_id = token_table["<blk>"]
params.vocab_size = sp.get_piece_size() params.unk_id = token_table["<unk>"]
params.vocab_size = num_tokens(token_table) + 1 # +1 for <blk>
logging.info(params) logging.info(params)

6
egs/wenetspeech/ASR/pruned_transducer_stateless2/export.py
View File

@ -24,7 +24,7 @@ Usage:
./pruned_transducer_stateless2/export.py \ ./pruned_transducer_stateless2/export.py \
--exp-dir ./pruned_transducer_stateless2/exp \ --exp-dir ./pruned_transducer_stateless2/exp \
--lang-dir data/lang_char \ --tokens data/lang_char/tokens.txt \
--epoch 10 \ --epoch 10 \
--avg 2 \ --avg 2 \
--jit 1 --jit 1
@ -47,7 +47,7 @@ for how to use them.
./pruned_transducer_stateless2/export.py \ ./pruned_transducer_stateless2/export.py \
--exp-dir ./pruned_transducer_stateless2/exp \ --exp-dir ./pruned_transducer_stateless2/exp \
--lang-dir data/lang_char \ --tokens data/lang_char/tokens.txt \
--epoch 10 \ --epoch 10 \
--avg 2 \ --avg 2 \
--jit-trace 1 --jit-trace 1
@ -63,7 +63,7 @@ Check ./jit_pretrained.py for usage.
./pruned_transducer_stateless2/export.py \ ./pruned_transducer_stateless2/export.py \
--exp-dir ./pruned_transducer_stateless2/exp \ --exp-dir ./pruned_transducer_stateless2/exp \
--lang-dir data/lang_char \ --tokens data/lang_char/tokens.txt \
--epoch 10 \ --epoch 10 \
--avg 2 --avg 2