mirror of
https://github.com/k2-fsa/icefall.git
synced 2025-09-04 06:34:20 +00:00
minor updates
This commit is contained in:
jinzr
parent
817413f899
commit
3f89cb380a
@ -426,7 +426,7 @@ if [ $stage -le 18 ] && [ $stop_stage -ge 18 ]; then
|
||||
out_dir=data/lm_training_bpe_${vocab_size}
|
||||
python ../../../icefall/rnn_lm/train.py \
|
||||
--start-epoch 0 \
|
||||
--world-size 1 \
|
||||
--world-size 2 \
|
||||
--use-fp16 0 \
|
||||
--embedding-dim 2048 \
|
||||
--hidden-dim 2048 \
|
||||
@ -435,8 +435,7 @@ if [ $stage -le 18 ] && [ $stop_stage -ge 18 ]; then
|
||||
--exp-dir rnnlm_bpe_${vocab_size}/exp \
|
||||
--lm-data $out_dir/sorted_lm_data.pt \
|
||||
--lm-data-valid $out_dir/sorted_lm_data-dev.pt \
|
||||
--vocab-size $vocab_size \
|
||||
--master-port 12345
|
||||
--vocab-size $vocab_size
|
||||
done
|
||||
fi
|
||||
|
||||
|
||||
@ -1,93 +1,229 @@
|
||||
for subset in train dev test; do
|
||||
gunzip -c aidatatang_200zh/aidatatang_supervisions_${subset}.jsonl.gz \
|
||||
| jq '.text' \
|
||||
| sed 's/"//g' \
|
||||
| ../../local/tokenize_for_lm_training.py -t "char" \
|
||||
> aidatatang_${subset}_text
|
||||
cd data/
|
||||
|
||||
log "Preparing LM data..."
|
||||
mkdir -p lm_training_data
|
||||
mkdir -p lm_dev_data
|
||||
mkdir -p lm_test_data
|
||||
|
||||
log "aidatatang_200zh"
|
||||
gunzip -c manifests/aidatatang_200zh/aidatatang_supervisions_train.jsonl.gz \
|
||||
| jq '.text' \
|
||||
| sed 's/"//g' \
|
||||
| ../local/tokenize_for_lm_training.py -t "char" \
|
||||
> lm_training_data/aidatatang_train_text
|
||||
|
||||
gunzip -c manifests/aidatatang_200zh/aidatatang_supervisions_dev.jsonl.gz \
|
||||
| jq '.text' \
|
||||
| sed 's/"//g' \
|
||||
| ../local/tokenize_for_lm_training.py -t "char" \
|
||||
> lm_dev_data/aidatatang_dev_text
|
||||
|
||||
gunzip -c manifests/aidatatang_200zh/aidatatang_supervisions_test.jsonl.gz \
|
||||
| jq '.text' \
|
||||
| sed 's/"//g' \
|
||||
| ../local/tokenize_for_lm_training.py -t "char" \
|
||||
> lm_test_data/aidatatang_test_text
|
||||
|
||||
log "aishell"
|
||||
gunzip -c manifests/aishell/aishell_supervisions_train.jsonl.gz \
|
||||
| jq '.text' \
|
||||
| sed 's/"//g' \
|
||||
| ../local/tokenize_for_lm_training.py -t "char" \
|
||||
> lm_training_data/aishell_train_text
|
||||
|
||||
gunzip -c manifests/aishell/aishell_supervisions_dev.jsonl.gz \
|
||||
| jq '.text' \
|
||||
| sed 's/"//g' \
|
||||
| ../local/tokenize_for_lm_training.py -t "char" \
|
||||
> lm_dev_data/aishell_dev_text
|
||||
|
||||
gunzip -c manifests/aishell/aishell_supervisions_test.jsonl.gz \
|
||||
| jq '.text' \
|
||||
| sed 's/"//g' \
|
||||
| ../local/tokenize_for_lm_training.py -t "char" \
|
||||
> lm_test_data/aishell_test_text
|
||||
|
||||
log "aishell2"
|
||||
gunzip -c manifests/aishell2/aishell2_supervisions_train.jsonl.gz \
|
||||
| jq '.text' \
|
||||
| sed 's/"//g' \
|
||||
| ../local/tokenize_for_lm_training.py -t "char" \
|
||||
> lm_training_data/aishell2_train_text
|
||||
|
||||
gunzip -c manifests/aishell2/aishell2_supervisions_dev.jsonl.gz \
|
||||
| jq '.text' \
|
||||
| sed 's/"//g' \
|
||||
| ../local/tokenize_for_lm_training.py -t "char" \
|
||||
> lm_dev_data/aishell2_dev_text
|
||||
|
||||
gunzip -c manifests/aishell2/aishell2_supervisions_test.jsonl.gz \
|
||||
| jq '.text' \
|
||||
| sed 's/"//g' \
|
||||
| ../local/tokenize_for_lm_training.py -t "char" \
|
||||
> lm_test_data/aishell2_test_text
|
||||
|
||||
log "aishell4"
|
||||
gunzip -c manifests/aishell4/aishell4_supervisions_train_L.jsonl.gz \
|
||||
| jq '.text' \
|
||||
| sed 's/"//g' \
|
||||
| ../local/tokenize_for_lm_training.py -t "char" \
|
||||
> lm_training_data/aishell4_train_L_text
|
||||
|
||||
gunzip -c manifests/aishell4/aishell4_supervisions_train_M.jsonl.gz \
|
||||
| jq '.text' \
|
||||
| sed 's/"//g' \
|
||||
| ../local/tokenize_for_lm_training.py -t "char" \
|
||||
> lm_training_data/aishell4_train_M_text
|
||||
|
||||
gunzip -c manifests/aishell4/aishell4_supervisions_train_S.jsonl.gz \
|
||||
| jq '.text' \
|
||||
| sed 's/"//g' \
|
||||
| ../local/tokenize_for_lm_training.py -t "char" \
|
||||
> lm_training_data/aishell4_train_S_text
|
||||
|
||||
gunzip -c manifests/aishell4/aishell4_supervisions_test.jsonl.gz \
|
||||
| jq '.text' \
|
||||
| sed 's/"//g' \
|
||||
| ../local/tokenize_for_lm_training.py -t "char" \
|
||||
> lm_test_data/aishell4_test_text
|
||||
|
||||
log "alimeeting"
|
||||
gunzip -c manifests/alimeeting/alimeeting-far_supervisions_train.jsonl.gz \
|
||||
| jq '.text' \
|
||||
| sed 's/"//g' \
|
||||
| ../local/tokenize_for_lm_training.py -t "char" \
|
||||
> lm_training_data/alimeeting-far_train_text
|
||||
|
||||
gunzip -c manifests/alimeeting/alimeeting-far_supervisions_test.jsonl.gz \
|
||||
| jq '.text' \
|
||||
| sed 's/"//g' \
|
||||
| ../local/tokenize_for_lm_training.py -t "char" \
|
||||
> lm_test_data/alimeeting-far_test_text
|
||||
|
||||
gunzip -c manifests/alimeeting/alimeeting-far_supervisions_eval.jsonl.gz \
|
||||
| jq '.text' \
|
||||
| sed 's/"//g' \
|
||||
| ../local/tokenize_for_lm_training.py -t "char" \
|
||||
> lm_dev_data/alimeeting-far_eval_text
|
||||
|
||||
log "kespeech"
|
||||
gunzip -c manifests/kespeech/kespeech-asr_supervisions_dev_phase1.jsonl.gz \
|
||||
| jq '.text' \
|
||||
| sed 's/"//g' \
|
||||
| ../local/tokenize_for_lm_training.py -t "char" \
|
||||
> lm_dev_data/kespeech_dev_phase1_text
|
||||
|
||||
gunzip -c manifests/kespeech/kespeech-asr_supervisions_dev_phase2.jsonl.gz \
|
||||
| jq '.text' \
|
||||
| sed 's/"//g' \
|
||||
| ../local/tokenize_for_lm_training.py -t "char" \
|
||||
> lm_dev_data/kespeech_dev_phase2_text
|
||||
|
||||
gunzip -c manifests/kespeech/kespeech-asr_supervisions_test.jsonl.gz \
|
||||
| jq '.text' \
|
||||
| sed 's/"//g' \
|
||||
| ../local/tokenize_for_lm_training.py -t "char" \
|
||||
> lm_test_data/kespeech_test_text
|
||||
|
||||
gunzip -c manifests/kespeech/kespeech-asr_supervisions_train_phase1.jsonl.gz \
|
||||
| jq '.text' \
|
||||
| sed 's/"//g' \
|
||||
| ../local/tokenize_for_lm_training.py -t "char" \
|
||||
> lm_training_data/kespeech_train_phase1_text
|
||||
|
||||
gunzip -c manifests/kespeech/kespeech-asr_supervisions_train_phase2.jsonl.gz \
|
||||
| jq '.text' \
|
||||
| sed 's/"//g' \
|
||||
| ../local/tokenize_for_lm_training.py -t "char" \
|
||||
> lm_training_data/kespeech_train_phase2_text
|
||||
|
||||
log "magicdata"
|
||||
gunzip -c manifests/magicdata/magicdata_supervisions_train.jsonl.gz \
|
||||
| jq '.text' \
|
||||
| sed 's/"//g' \
|
||||
| ../local/tokenize_for_lm_training.py -t "char" \
|
||||
> lm_training_data/magicdata_train_text
|
||||
|
||||
gunzip -c manifests/magicdata/magicdata_supervisions_test.jsonl.gz \
|
||||
| jq '.text' \
|
||||
| sed 's/"//g' \
|
||||
| ../local/tokenize_for_lm_training.py -t "char" \
|
||||
> lm_test_data/magicdata_test_text
|
||||
|
||||
gunzip -c manifests/magicdata/magicdata_supervisions_dev.jsonl.gz \
|
||||
| jq '.text' \
|
||||
| sed 's/"//g' \
|
||||
| ../local/tokenize_for_lm_training.py -t "char" \
|
||||
> lm_dev_data/magicdata_dev_text
|
||||
|
||||
log "stcmds"
|
||||
gunzip -c manifests/stcmds/stcmds_supervisions_train.jsonl.gz \
|
||||
| jq '.text' \
|
||||
| sed 's/"//g' \
|
||||
| ../local/tokenize_for_lm_training.py -t "char" \
|
||||
> lm_training_data/stcmds_train_text
|
||||
|
||||
log "primewords"
|
||||
gunzip -c manifests/primewords/primewords_supervisions_train.jsonl.gz \
|
||||
| jq '.text' \
|
||||
| sed 's/"//g' \
|
||||
| ../local/tokenize_for_lm_training.py -t "char" \
|
||||
> lm_training_data/primewords_train_text
|
||||
|
||||
log "thchs30"
|
||||
gunzip -c manifests/thchs30/thchs_30_supervisions_train.jsonl.gz \
|
||||
| jq '.text' \
|
||||
| sed 's/"//g' \
|
||||
| ../local/tokenize_for_lm_training.py -t "char" \
|
||||
> lm_training_data/thchs30_train_text
|
||||
|
||||
gunzip -c manifests/thchs30/thchs_30_supervisions_test.jsonl.gz \
|
||||
| jq '.text' \
|
||||
| sed 's/"//g' \
|
||||
| ../local/tokenize_for_lm_training.py -t "char" \
|
||||
> lm_test_data/thchs30_test_text
|
||||
|
||||
gunzip -c manifests/thchs30/thchs_30_supervisions_dev.jsonl.gz \
|
||||
| jq '.text' \
|
||||
| sed 's/"//g' \
|
||||
| ../local/tokenize_for_lm_training.py -t "char" \
|
||||
> lm_dev_data/thchs30_dev_text
|
||||
|
||||
log "wenetspeech"
|
||||
gunzip -c manifests/wenetspeech/wenetspeech_supervisions_L.jsonl.gz \
|
||||
| jq '.text' \
|
||||
| sed 's/"//g' \
|
||||
| ../local/tokenize_for_lm_training.py -t "char" \
|
||||
> lm_training_data/wenetspeech_L_text
|
||||
|
||||
gunzip -c manifests/wenetspeech/wenetspeech_supervisions_DEV.jsonl.gz \
|
||||
| jq '.text' \
|
||||
| sed 's/"//g' \
|
||||
| ../local/tokenize_for_lm_training.py -t "char" \
|
||||
> lm_dev_data/wenetspeech_DEV_text
|
||||
|
||||
gunzip -c manifests/wenetspeech/wenetspeech_supervisions_TEST_MEETING.jsonl.gz \
|
||||
| jq '.text' \
|
||||
| sed 's/"//g' \
|
||||
| ../local/tokenize_for_lm_training.py -t "char" \
|
||||
> lm_test_data/wenetspeech_TEST_MEETING_text
|
||||
|
||||
gunzip -c manifests/wenetspeech/wenetspeech_supervisions_TEST_NET.jsonl.gz \
|
||||
| jq '.text' \
|
||||
| sed 's/"//g' \
|
||||
| ../local/tokenize_for_lm_training.py -t "char" \
|
||||
> lm_test_data/wenetspeech_TEST_NET_text
|
||||
|
||||
for f in aidatatang_train_text aishell2_train_text aishell4_train_L_text aishell4_train_M_text aishell4_train_S_text aishell_train_text alimeeting-far_train_text kespeech_train_phase1_text kespeech_train_phase2_text magicdata_train_text primewords_train_text stcmds_train_text thchs30_train_text wenetspeech_L_text; do
|
||||
cat lm_training_data/$f >> lm_training_data/lm_training_text
|
||||
done
|
||||
|
||||
for subset in train dev test; do
|
||||
gunzip -c aishell/aishell_supervisions_${subset}.jsonl.gz \
|
||||
| jq '.text' \
|
||||
| sed 's/"//g' \
|
||||
| ../../local/tokenize_for_lm_training.py -t "char" \
|
||||
> aishell_${subset}_text
|
||||
for f in aidatatang_test_text aishell4_test_text alimeeting-far_test_text thchs30_test_text wenetspeech_TEST_NET_text aishell2_test_text aishell_test_text kespeech_test_text magicdata_test_text wenetspeech_TEST_MEETING_text; do
|
||||
cat lm_test_data/$f >> lm_test_data/lm_test_text
|
||||
done
|
||||
|
||||
for subset in train dev test; do
|
||||
gunzip -c aishell2/aishell2_supervisions_${subset}.jsonl.gz \
|
||||
| jq '.text' \
|
||||
| sed 's/"//g' \
|
||||
| ../../local/tokenize_for_lm_training.py -t "char" \
|
||||
> aishell2_${subset}_text
|
||||
for f in aidatatang_dev_text aishell_dev_text kespeech_dev_phase1_text thchs30_dev_text aishell2_dev_text alimeeting-far_eval_text kespeech_dev_phase2_text magicdata_dev_text wenetspeech_DEV_text; do
|
||||
cat lm_dev_data/$f >> lm_dev_data/lm_dev_text
|
||||
done
|
||||
|
||||
for subset in train_L train_M train_S test; do
|
||||
gunzip -c aishell4/aishell4_supervisions_${subset}.jsonl.gz \
|
||||
| jq '.text' \
|
||||
| sed 's/"//g' \
|
||||
| ../../local/tokenize_for_lm_training.py -t "char" \
|
||||
> aishell4_${subset}_text
|
||||
done
|
||||
|
||||
for subset in train test eval; do
|
||||
gunzip -c alimeeting/alimeeting-far_supervisions_${subset}.jsonl.gz \
|
||||
| jq '.text' \
|
||||
| sed 's/"//g' \
|
||||
| ../../local/tokenize_for_lm_training.py -t "char" \
|
||||
> alimeeting-far_${subset}_text
|
||||
done
|
||||
|
||||
for subset in dev_phase1 dev_phase2 test train_phase1 train_phase2; do
|
||||
gunzip -c kespeech/kespeech-asr_supervisions_${subset}.jsonl.gz \
|
||||
| jq '.text' \
|
||||
| sed 's/"//g' \
|
||||
| ../../local/tokenize_for_lm_training.py -t "char" \
|
||||
> kespeech_${subset}_text
|
||||
done
|
||||
|
||||
for subset in train test dev; do
|
||||
gunzip -c magicdata/magicdata_supervisions_${subset}.jsonl.gz \
|
||||
| jq '.text' \
|
||||
| sed 's/"//g' \
|
||||
| ../../local/tokenize_for_lm_training.py -t "char" \
|
||||
> magicdata_${subset}_text
|
||||
done
|
||||
|
||||
for subset in train ; do
|
||||
gunzip -c stcmds/stcmds_supervisions_${subset}.jsonl.gz \
|
||||
| jq '.text' \
|
||||
| sed 's/"//g' \
|
||||
| ../../local/tokenize_for_lm_training.py -t "char" \
|
||||
> stcmds_${subset}_text
|
||||
done
|
||||
|
||||
for subset in train ; do
|
||||
gunzip -c primewords/primewords_supervisions_${subset}.jsonl.gz \
|
||||
| jq '.text' \
|
||||
| sed 's/"//g' \
|
||||
| ../../local/tokenize_for_lm_training.py -t "char" \
|
||||
> primewords_${subset}_text
|
||||
done
|
||||
|
||||
for subset in train test dev ; do
|
||||
gunzip -c thchs30/thchs_30_supervisions_${subset}.jsonl.gz \
|
||||
| jq '.text' \
|
||||
| sed 's/"//g' \
|
||||
| ../../local/tokenize_for_lm_training.py -t "char" \
|
||||
> thchs30_${subset}_text
|
||||
done
|
||||
|
||||
for subset in L DEV TEST_MEETING TEST_NET ; do
|
||||
gunzip -c wenetspeech/wenetspeech_supervisions_${subset}.jsonl.gz \
|
||||
| jq '.text' \
|
||||
| sed 's/"//g' \
|
||||
| ../../local/tokenize_for_lm_training.py -t "char" \
|
||||
> wenetspeech_${subset}_text
|
||||
done
|
||||
|
||||
cat aidatatang_train_text aishell2_train_text aishell4_train_L_text \
|
||||
aishell4_train_M_text aishell4_train_S_text aishell_train_text \
|
||||
alimeeting-far_train_text kespeech_train_phase1_text kespeech_train_phase2_text \
|
||||
magicdata_train_text primewords_train_text stcmds_train_text \
|
||||
thchs30_train_text wenetspeech_L_text > lm_training_text
|
||||
cd ../
|
||||
Loading…
x
Reference in New Issue
Block a user