From 2639b739332614e19a307d545f6f76867d656f92 Mon Sep 17 00:00:00 2001 From: dohe0342 Date: Tue, 21 Feb 2023 14:35:51 +0900 Subject: [PATCH] from local --- egs/tedlium2/ASR/.prepare.sh.swp | Bin 4096 -> 20480 bytes egs/tedlium2/ASR/.prepare_ted3.sh.swp | Bin 4096 -> 16384 bytes egs/tedlium2/ASR/prepare.sh | 72 ++++++++------------------ 3 files changed, 23 insertions(+), 49 deletions(-) diff --git a/egs/tedlium2/ASR/.prepare.sh.swp b/egs/tedlium2/ASR/.prepare.sh.swp index 1b5b08a238b312664e6e3d2c805f711814c6413c..ab728ea203583d2292da6a41aa8d1c3772137da9 100644 GIT binary patch literal 20480 zcmeI3dyFJS9miWh0hJRmh#m>l?c;VAx2Jb!_qe;|2DshZyIa^7`;c6YbL^R^nQ6DD zyXo$^-SO_g7cU2@=s&0tA!z(ZOaO_G#27*UAV#>5ppqa46qMJ2->UBF?wOsQ zoh6(|(nCHwH(ige`qr zC9Uk7pHnQGTB=P`PPws~fsZ3?2es1twSmV_*~z@Wv@yGzVt%E|KeI7ISf5XQNthhk>!eNXhrI{ zj$F`eQczu02`))I4>IYhQR3-c>|a@y^9r7rQutdZ;?0q)PFJ+NX{4+g&!82_ezJ+k zMG_QL{480#k5X__?utteTtM71HI8e;Tw67qyscTT)2eaErWq-h*0-v%#Cj}SVK2~m zp+tsj#(!C5)1WNLERl|RS1nPpWrWOnZ`BZOK$0^%)7mE?W0lyu#FO zxBdJugLQYj?p9&@>N-Evtn;Ga>i!SK^&bgq^go!VtHxa~o32BN|2~%|J?SdwZ00I_ zAeznedJ_n0Uu_0KQuM|_a|!7ruIb-^*&{0GB$rd7I!?7hG1pa>)XXX=s|R6M=CYxi zY5|i1i-6zMylGoi2jwC&DILxvuLnUZR*RaB6BP;Viz)^!G^_?){S1M!qQLsZ_4N|N z<5x1ojk9gtL@5$TC#G((p*0GYH}9Q4-mgHAIU2xG=tJa$<2RYg3(Km6y{&iqMeXjx_e( zPZhaYCNX%S0HTUOP|>wjI`}z8iNc4hxHZmn0UA@??DCzk`_ddD1B6k-^_ui52sgdsao<7psb*N~l z6?$Z(>(J=V`Muf6iRtm_nf(I?*p=Oj(__;+r^fdW9yroRy3XrBz1)`}yimq3Ge5I9 zH^wq`cAlrOdv}tBvgVLt)yTV=X%O~r+pG}kSO#?^w7R#C9FnBE#Z>Af`>^0K3v|>X z)oaA26%(sp2it8sh-wsY#jYAy15u&ps4&a3X^9%tW}VdB6sp;&>+s%m>JX7oAh}vy zF}%TxyBuWJYxp-v!2w3ww+sy6@8AIcN_$_pHXt3utHZghqkWb-nO@u_bZK%H&DLr) z<_NImS+#AohBd};;U_s{Wo4zGAQ?p_T(Yd8aiUobb+DYs#cxE|jkr(XP{XB09y^sr z@wZ|YsP3S#DdLpTjRGc~?a-`gF{vHAmrM>HCU7xbG6>h7Z)X{0*Qhe7y22DVn-&E^9`KhzuGADYBF>D`6GM!kE1UD{O$U5K)E+Uh#~JHO*<#wk?CA zX`*mZgDJ7$>&LzG{Y0lFHDBu|lT-bKy10O0(+a|xB8?I23vW-OMO9t1qPiO1FV}0# z3Zq?Qw#zLtMX-<>rrXaNUh}LW7v11dF!Tbv(IdSb<#XJEeg3Ew(1 zAQ_*@3~oUxb%{(d!7RUDNp!_BZI{f>%`D7}%}i!T7kBR-pPSFl?pxS9Go787 znw=b<8lPU+xiB#^o$>r+w6CbG5Qmm3uuJq8)4Qlvbr;i+)#lA8F%~7wEmw0XxYWv0 zS}r)6(j=26l za4+~Gm;+OQ#r_lEqu?JKB-P7%%F?7#THkqfYbFjr^o7KGbHlL&ab7Z0KBWbfpj)xN;? zI9~t>!WW1JfFxKy^@=hqmu$6?Qf*5m7y4UHMcI=b&hqO;$7LrB=mh()!;Tdh#}3ra z3={7FgIz6Xj#|lSB^gd(&eStKodZ3QWVnn4KS_4goK7=**H*%ipZ&iz-dRS6Urq$x za)m6ZdO_1GegfYpvBg)}DF7RU$tkQZ4!m}39a0!@`(8V`*w~p+mB=}c@OqJd$dkl^ zH$e-jK`L)uEzi(e@(e}#|BlH6qs&z`sE17zrW2!1eP04~;F*h48_zr|+-+wbJ@$R0 zG{QCuwi~+G)Wjg+7STN_2vQreVHfpxI*WVyq8b#BxWU`P7^;ccZzC~hJxHiC`(P#D zlYsC~c$zjV1~+=C-t6lQKOKzhnx@Y(G%^10POiBGKZ(sSVfUs#9c9gJ^eG7qz5EQ_^C z46|>`!K%R1>c`t#=Gc-bFRQizTX!<@V4&!6SOxzl>tD2|;S_1U270n_Gm90@OMSOW z*WTLbZG+x?OZi<5w0q}Q$gM3`$gOcJBofwGA;qfUAAn~0Rt-0(rRUUaHJQYTsplN5 zT2$D=6+Tr69}Ots#qb%2zu3t#_qk*q0Cz?Fuh1D`O>Oz8g+HRmGM|oz%ihxrUVpHF zFyBJfNROlJs2WQbDoUxRz4kQ}ByXi8S?4`F;s5Zn5EkiEf2 z-E$}J`5Z41_(wtO`Q^7ifQW3C=Z7<%>DlUG*!gkO9|?8S-#7babH2rVU*~Lh*lEIH zQZ=+I>&@Dx!7voW0T2oak+?L5rm|0fX39|J7@7w`SE->-w$z^mXt;Cb*MxD8Z* z4z_`_!83^Y{|250N5S3TW?+L6unD}0*#9x`DEKY79fS-Wx0QL^R zRp27f1zteh|2%jO{1qGp_knMMWiSeQz%j)C?0n!Q@HqG$xCvNb0_+7l!CBx8WqPT$0o#BN6Y+vV!Lg|D~Bzr zYmpRVw{XbUEEJa^?t08D@v}$1)>(m#_@PCS7D@4)=PzyY_-^gu&Bkc$Qju6|&CIbG zViRIaPDEO3lxv>c{HoGo^%G%T7%1~PnhAzyf6rfZ!3O&_Qjj?Up_Xu5&!@I delta 16 YcmZozz}TR$;U)9L1FW0hF$?hk06Uomb^rhX diff --git a/egs/tedlium2/ASR/.prepare_ted3.sh.swp b/egs/tedlium2/ASR/.prepare_ted3.sh.swp index dba02ec972745e9de057f770004afa6a163f35ef..53d4ccd3cf61787b6fa8bca04c4da7d585e0ee45 100644 GIT binary patch literal 16384 zcmeI3TWlOx8ONuk+?%%IE=1`UuN^zBXJ&2Zx~(hX#7T^NBTh=AG+oW^p4}Zhx6YZ3 zy>Sw%R#hKpt4b9hC_I6XKth7j(w2uxAOWf>f#89Q!~<2~sf7Yck&uA$KWFC5%#S)xj>%fwj9A05JMN@; zXyt?%gej3BgSY$ZDt;FV6bf9a0z0&+$qBjCOG7*9{qH+UIt$R4NwQ4 z0?XhyI0z1a*RRvGpMYJ;D_K*a1Xc(mOQmdGKuzf=_~3Far*O>%br1j&$%iaKJM7D0mGM^cV0a@CtYqJPaNJcY~u~ z7JLMJ82lQCHD3j7Pyq+Q2f+z9VB*2)U-9iC-OV>HZ`os zGMhEaTQv>0!!6Hf_#TrPf^O9Iy(uzG`QnFJ_uZfqv8v2njWY*bnblow*dggbt~y+| zn{0q0uhZ>UQZ_Ex@shH+ncJq1bb&+%@O2k#7N6~BHL1zKyvp2H zGO;~UX8QL`?$lwwhnp^6Vz#5zkwk9Fx=bBhDy!7b!L^8g4jM z*sv)LEuNLMhE0^nr}de6nv`!i?JiLXD=%q4tT$9%qE9ojIPbU&>%oes>vyPaox{5E z*F49!8mu7}O48N39|j$ca=Aq_S)t|l-Kr#0+KWL^C$|z632$6fHEEf@=+iaHkSePR zT$#9uUTS(&C2~sMux_dprP8T6YxecVG*n(Wu2hg7u_rRIRc_iw?}@GSM#5%#CpxOR z?=?^iT=l(?N*fz(S{XERJkojd9GQU?wav&+>{`hv3kb^-NOP-O9@Rr;MX*HTUDvU_ zRhc-~%*+u>MpxZ}N!pPx<=!U7h2KcLZS3W;ZaS{HrxjXmmAiO5(6GY5f)`CkktWyd zmP$lrL~AV;k*?Q$FS5N3qjnSp71La`oQADiJT&~UWx9TYIo#ZvG?A-=^yOR|Z@R|b zPWRV2I4O>sPOlL8NV1d(JtNC&kgN|kDxnBDme&zCzjzcm4RL!>lXi0U$o$-?g=)wg zW^txZr76k^<=FboCrMP9k!P^4s)Brvk+DPrc9iB+y;2uMc3DI?um6HIsd}YB(oV~c4hHoB1 zy>b&pve0x=W@YKriNhjOsdTd`+Ah(_w#}*8@#>N7dnEo2eU}&yJQiu__1JFOz~(N$ z-_lBSI?ZuQk6aw3;(h(tqj z&&ZhE7L@i;-eD}7_27!>G?Ki=`Z+NyRMaz-=w_P+S;1o2h8J6?7t@Vn<|U60c=KC zJ@UhxAVQ<2?>PRN?X^f{urXaZr!4G6RHQ4jj$MD&0c9(yjF8@q%}h8}4$s=(x4jdp zYpQV5h6fW9IgYmLMk&U;^ovyIQ z{1Ao9l(uOQZb_aWjf;+VW4oy3w@N6O?Wf{-q_K3gsB?E_V?B~0AWfr)X&5qo!Y5DT z)mkoYKn#m4umil>ZabRB)&oC`X!*p_$)&?f^VOMCM~}{)Sg9_bK6!j;u{yV~JU_cI zyLj@@$+@M)>3G+To;B@t;;iMuHZf8x@ut;rBCJ?ZTfC~&EVS&X-KiM}yUf)|y~(Zg zy?PsaMaIp2`}b&CX{XuYp;@y%lX>T;X5qDZ5&s`SjQJ$uPZ9sC=llOcZ2wR2Ja`mr zfF@vI8vFrq{Y&6E@HucdI1TOuWnhB8Bc}g8cnaJPE`bej9vlNV0TKIOMm+xlcoN(P z?f?^@46X-%Lk#~5@GbC7umSb~6O4foFbw{RIR01Qi$KKp5wL&(c7V%>R5G8H47azab0|LUF;)u+|y}T?kL4y z@ev#KoLgd3TWvcnoNFb?Dh6t113iaTjNl*%h%j z&gbA~K5_Qqt=K7vz%Ccd9EeZD&^bcegOSXEts)sDs7OXCmCB`}vVmAy9W^2$)l$UB zO&%H1dW3$7kFLAcx-J59ea#9zY%}?^K9MRKC;MT8h0}X-R~t6BT-2uH;GpI^)1#&G zXf7GY3XLR5kF1)*re$Pe3T?%e$u}>0$&o5x# z#BrvBerj1*o1E!vG15%A^qhjyt#T?`7VK{(T%7t=l@7*P>)1FuK@b76+g`bzp{Cjq z^+cei;|j|zi4!42J6@|IHp(Ifh_IDc(}4o5xhQ2$j`IsgQ!!gXxwZgk!E&(Y9r9dGq^5$ER(F?CAj zJWiM6MKa)G#InPK1uqHAQ%7UQPaE>9Q@R*kM!3T{ccVPsru~Z()v#opGVqr1aO@Ce zuj|Rvo8DEKxks6)MBGQ*g)?dqIYn!}JnE70G)d>_;4~pLlDwGgkYTw}y2#@=Gr)5t zCPEb~WCsy9LVj5yHL6rMdHH|pgZE{IG_yRbIt9-(=#KYoWmQP%v8u!!E6tTD#VTxJ zM`8gHLc<1)^2*qmvVQRF?ndhBBY;PY zXn5k{XzJNi+-)YOh{nX;$yK9a*Xe?2d9&MLHfibTaxN2Q--+-%P>E$bT+aqA z*Y>u}n~oy!_5IRi>x1V^wv|2%Q}?}enDYzf%?9(t`7#8IRi)eRGWqRN)Vz%GlE}%p znHcGavZ>BF!E{a_<)F2Si`nzdy}bq}E0#wo@u@IQ8vJIwU" \ - > $lang_dir/transcript_tokens.txt - fi - - if [ ! -f $lang_dir/P.arpa ]; then - ./shared/make_kn_lm.py \ - -ngram-order 2 \ - -text $lang_dir/transcript_tokens.txt \ - -lm $lang_dir/P.arpa - fi - - if [ ! -f $lang_dir/P.fst.txt ]; then - python3 -m kaldilm \ - --read-symbol-table="$lang_dir/tokens.txt" \ - --disambig-symbol='#0' \ - --max-order=2 \ - $lang_dir/P.arpa > $lang_dir/P.fst.txt - fi - done -fi - -if [ $stage -le 8 ] && [ $stop_stage -ge 8 ]; then - log "Stage 8: Prepare G" + log "Stage 7: Prepare G" # We assume you have install kaldilm, if not, please install # it using: pip install kaldilm mkdir -p data/lm - if [ ! -f data/lm/G_3_gram.fst.txt ]; then + if [ ! -f data/lm/G_4_gram_small.fst.txt ]; then # It is used in building HLG python3 -m kaldilm \ - --read-symbol-table="data/lang_phone/words.txt" \ - --disambig-symbol='#0' \ - --max-order=3 \ - $dl_dir/lm/3-gram.pruned.1e-7.arpa > data/lm/G_3_gram.fst.txt - fi - - if [ ! -f data/lm/G_4_gram.fst.txt ]; then - # It is used for LM rescoring - python3 -m kaldilm \ - --read-symbol-table="data/lang_phone/words.txt" \ + --read-symbol-table="data/lang/words.txt" \ --disambig-symbol='#0' \ --max-order=4 \ - $dl_dir/lm/4-gram.arpa > data/lm/G_4_gram.fst.txt + --max-arpa-warnings=-1 \ + $dl_dir/lm/4gram_small.arpa > data/lm/G_4_gram_small.fst.txt + fi + + if [ ! -f data/lm/G_4_gram_big.fst.txt ]; then + # It is used for LM rescoring + python3 -m kaldilm \ + --read-symbol-table="data/lang/words.txt" \ + --disambig-symbol='#0' \ + --max-order=4 \ + --max-arpa-warnings=-1 \ + $dl_dir/lm/4gram_big.arpa > data/lm/G_4_gram_big.fst.txt fi fi -if [ $stage -le 9 ] && [ $stop_stage -ge 9 ]; then - log "Stage 9: Compile HLG" - ./local/compile_hlg.py --lang-dir data/lang_phone +if [ $stage -le 8 ] && [ $stop_stage -ge 8 ]; then + log "Stage 8: Compile HLG" for vocab_size in ${vocab_sizes[@]}; do lang_dir=data/lang_bpe_${vocab_size} - ./local/compile_hlg.py --lang-dir $lang_dir + + if [ ! -f $lang_dir/HLG.pt ]; then + ./local/compile_hlg.py \ + --lang-dir $lang_dir \ + --lm G_4_gram_small + fi done fi -