mirrors/icefall
Archived
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix preparing char based lang for wenetspeech

Browse Source
This commit is contained in:
WeijiZhuang 2022-08-02 23:40:27 +08:00
parent b986ef5ddc
commit fffd3bbd9e
1 changed files with 17 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
egs/wenetspeech/ASR/prepare.sh
View File

@ -28,6 +28,7 @@ num_splits=1000
# - speech # - speech
dl_dir=$PWD/download dl_dir=$PWD/download
lang_char_dir=data/lang_char
. shared/parse_options.sh || exit 1 . shared/parse_options.sh || exit 1
@ -186,24 +187,27 @@ fi
if [ $stage -le 15 ] && [ $stop_stage -ge 15 ]; then if [ $stage -le 15 ] && [ $stop_stage -ge 15 ]; then
log "Stage 15: Prepare char based lang" log "Stage 15: Prepare char based lang"
lang_char_dir=data/lang_char
mkdir -p $lang_char_dir mkdir -p $lang_char_dir
# Prepare text. if ! which jq; then
# Note: in Linux, you can install jq with the following command: echo "This script is intended to be used with jq but you have not installed jq
# 1. wget -O jq https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 Note: in Linux, you can install jq with the following command:
# 2. chmod +x ./jq 1. wget -O jq https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64
# 3. cp jq /usr/bin 2. chmod +x ./jq
if [ ! -f $lang_char_dir/text ]; then 3. cp jq /usr/bin" && exit 1
gunzip -c data/manifests/supervisions_L.jsonl.gz \ fi
| jq 'text' | sed 's/"//g' \ if [ ! -f $lang_char_dir/text ] || [ ! -s $lang_char_dir/text ]; then
log "Prepare text."
gunzip -c data/manifests/wenetspeech_supervisions_L.jsonl.gz \
| jq '.text' | sed 's/"//g' \
| ./local/text2token.py -t "char" > $lang_char_dir/text | ./local/text2token.py -t "char" > $lang_char_dir/text
fi fi
# The implementation of chinese word segmentation for text, # The implementation of chinese word segmentation for text,
# and it will take about 15 minutes. # and it will take about 15 minutes.
if [ ! -f $lang_char_dir/text_words_segmentation ]; then if [ ! -f $lang_char_dir/text_words_segmentation ]; then
python ./local/text2segments.py \ python3 ./local/text2segments.py \
--num-process $nj \
--input-file $lang_char_dir/text \ --input-file $lang_char_dir/text \
--output-file $lang_char_dir/text_words_segmentation --output-file $lang_char_dir/text_words_segmentation
fi fi
@ -212,7 +216,7 @@ if [ $stage -le 15 ] && [ $stop_stage -ge 15 ]; then
| sort -u | sed '/^$/d' | uniq > $lang_char_dir/words_no_ids.txt | sort -u | sed '/^$/d' | uniq > $lang_char_dir/words_no_ids.txt
if [ ! -f $lang_char_dir/words.txt ]; then if [ ! -f $lang_char_dir/words.txt ]; then
python ./local/prepare_words.py \ python3 ./local/prepare_words.py \
--input-file $lang_char_dir/words_no_ids.txt \ --input-file $lang_char_dir/words_no_ids.txt \
--output-file $lang_char_dir/words.txt --output-file $lang_char_dir/words.txt
fi fi
@ -221,7 +225,7 @@ fi
if [ $stage -le 16 ] && [ $stop_stage -ge 16 ]; then if [ $stage -le 16 ] && [ $stop_stage -ge 16 ]; then
log "Stage 16: Prepare char based L_disambig.pt" log "Stage 16: Prepare char based L_disambig.pt"
if [ ! -f data/lang_char/L_disambig.pt ]; then if [ ! -f data/lang_char/L_disambig.pt ]; then
python ./local/prepare_char.py \ python3 ./local/prepare_char.py \
--lang-dir data/lang_char --lang-dir data/lang_char
fi fi
fi fi
@ -232,9 +236,8 @@ if [ $stage -le 17 ] && [ $stop_stage -ge 17 ]; then
# It will take about 20 minutes. # It will take about 20 minutes.
# We assume you have install kaldilm, if not, please install # We assume you have install kaldilm, if not, please install
# it using: pip install kaldilm # it using: pip install kaldilm
lang_char_dir=data/lang_char
if [ ! -f $lang_char_dir/3-gram.unpruned.arpa ]; then if [ ! -f $lang_char_dir/3-gram.unpruned.arpa ]; then
python ./shared/make_kn_lm.py \ python3 ./shared/make_kn_lm.py \
-ngram-order 3 \ -ngram-order 3 \
-text $lang_char_dir/text_words_segmentation \ -text $lang_char_dir/text_words_segmentation \
-lm $lang_char_dir/3-gram.unpruned.arpa -lm $lang_char_dir/3-gram.unpruned.arpa
@ -253,6 +256,5 @@ fi
if [ $stage -le 18 ] && [ $stop_stage -ge 18 ]; then if [ $stage -le 18 ] && [ $stop_stage -ge 18 ]; then
log "Stage 18: Compile LG" log "Stage 18: Compile LG"
lang_char_dir=data/lang_char
python ./local/compile_lg.py --lang-dir $lang_char_dir python ./local/compile_lg.py --lang-dir $lang_char_dir
fi fi