From 3debc7ff4d070d69604fa07c884abbc3b85526d2 Mon Sep 17 00:00:00 2001 From: dohe0342 Date: Wed, 24 May 2023 11:55:50 +0900 Subject: [PATCH] from local --- .../.data2vec_audio.py.swp | Bin 40960 -> 0 bytes .../data2vec_audio.py | 93 ------------------ 2 files changed, 93 deletions(-) delete mode 100644 egs/librispeech/ASR/pruned_transducer_stateless_d2v_v2/.data2vec_audio.py.swp diff --git a/egs/librispeech/ASR/pruned_transducer_stateless_d2v_v2/.data2vec_audio.py.swp b/egs/librispeech/ASR/pruned_transducer_stateless_d2v_v2/.data2vec_audio.py.swp deleted file mode 100644 index f4417218c7c9b07d8ead2df15f995f8a07053fb6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 40960 zcmeI53z%h9b>|O?fid`qP7p-Mu{x8!)l}bl0McDUVL#|5t<6Jh9mbmWdh6a(RoA+& zzUSWVs%{$t1WkzeNpuh&UlJdvs3bndXqXWvDo92#1|>wCs4*IiW`qgKBr}ThTl;bL z+2@>FU4Z7BN$Pyx|33C}?X_QP?Y;Isa}zJ#b5XdpJQ46bH3+_Uq*dQ^!e@_v|A&L1 z-dT($&N_QayjFWIAY3$PY`&;k+tKMPUE1rEo1I#OWO>q2Br?(V$tF>snQH$oQ z&E|Nt5RW$-bG=5~jiTD(`1TiGHs0+G+EKmI?^WAzeNc;fmAGH+N6jdXEA^>^m4j2` z^=iL5bug+`s)Kr?Q|_*0^IZSG&Vh9fz=1)3e)E4vmoOL(fpUe24VBD+bd%_TGTZY z&;u_4mxF0g0;hs!fCo9E}3izKaTKqM*2iywY4z2@NgR8)PFb>M#M=UUW349#<7jOfJK@aQ% z&jep#k>W1!Ch)7^WN;$*GK(9(0SS(Rj2*`9Peaa5WZ3nxcY(e{PQ;SR7f+{!G|vw1+0V9G(e1m3m`--Y>2sqTL}lj+*nB0rjeqou;EvU!G~? z^US-bgvL}7m1@7=ma3Q9skEx`(oFi#SlEs*5jqq#78d*Q%NTnG-12=(-wtZ`xHf)3ESN= zR!sFy%Y2narR(?!NyF6rXELy<%|k`Q@Goq%!>B!IMLmq5N-l1k?hVpbq$4INcufG$ z+5N84j@8DceXC|IZ9Hl4e*E2KQ(~#-?u;VKxQvCCncCU@^DBH*%9@ZXf8_BiS*6;G z4h$MSYCDb&$N*Gn(W)EWW;7Po8?BkmlVjn*YI6|H?CT9&`SXcY-~!24`kf>N*BoU0 z@+&k=i3~BR@o#!RNl9)#{plD4ST47tLzTl(uM^Yv%5mc`qsizP?&{~{D8K9dm2NcC z@APVm<+)C$=~zTF5n}a#tC|Sy$YYO;DN^N*>^w`Un>F!p%JOg`5XQ+P)#?aXg>@Sw zT-fh~{l$pf4h*7pEeZ|c)KJup8=bamJt}ed<63g1-RZST%U$DFBlCV7buhf8YPYvcYl_{4?5>8RVXs?1(Rc`cG5k~X0H`?`RxzZi< zRqIcUQe)wS%Rtv*GJ+LM0;AK=8wk+qtJ(TgHp3OQkBM)6rx~ zjg~20tzVgss!Y03TshR}FIKAcYPTQtO0J}4P9`aDBngC0N3r~oT3Ed7MtPGJ9M8D*JL%x^V8s<=`ojP{4B zy}BCP86Q-~OInStTg@r+J1L#|pPmLZrG0oG4^F)}s!i?ewC5WOWLIlew8V%pWNb?Q-1 zaxCSYg}|WMSy+fD!&=pwA!@ zHd@_IuOC8*b(01K_e}~)siI~+FJ`IoA!-B5XIsirjvxYA z&mGk`+O12hQw)32LL;W9SD-v<`f^{^T8_{%T~pDWR+qB~D}Ed?RYsEdia_{%TeGi}tr^p7t-@qg$mp zp`L|%_w3wt@e6m6N;49PO(w%$2kEko>{5eGD`96IS0?SaGw3lx_IpuOCg#4Bn{*=? z4x2E!9m7pDAbO1?cF~@F)|mkVcaux7fi!k@x+^`Ag~L*9G~A7aJ<`O|Sh%NME31LF z(T~IGJj68+BI9x(`u`p1f0vT{yFe2 za1*!&MBoCj6C4k|h|YdH_$_cP=mOE_%iw9?Vf6NIg1f*i;0@qfa0na({}wz891s2q zet!VG3<&RE0M?&%4*dVm0V@c}x05%lYg2?ArP3CtvnUKAK9rR-7%LUI5uPV1du-M@ z0-KI}D`Asso|34Zja+DGqNKX)MlMz1k>Vt{JmQIBsirYsSRD&h{(3zoC9?(+tw{YS zIgLYt7`|w!B0kXTmsZLLtG$va`V*t5dXv%S=ZuzHQMH{CYV*o_J{?jS7F{wH@0sv0 zV(mC1Vc~jy!>Ju&^PniR4EV{{&HR2hmS`wEhAHc8hoPRBo zgU~h;mR2-9<#^CCTnSYA<^y_X!j+-EHGDO5`^Xoa9_%&i5~JnXpk7Tem5!rmYQ#!a zXjGdlVAM;gX0W2NI@pK~m1GgdB(<4^4Mmbvx0M~G4p&UnZE~8gQENs0#ZKLE(RB$# zZRafI(hHfhBT=hbsYkWy%7`q$*lsJDgmz|iZBsZoG2s?5D1;5>_kBg7q%DtM6XG&I zLbtA~R)I8j#1ODi94nM%dh*K#dVqNy9GB)~rnfNH58#Y!iVO4+6=sYLO}{@D?fTpVT9(IqeAr3?=-2b3e9Eak+2GRYt> z+Fn^rJn>;yw)_NR&jV&K&4?mEVpD}^%AV; zP8ko)UP+Cayw&u)SnWo$nV;524?G2iylf z0p1OM8~g@%HJAfugR{VK;LGU#p9Fsl-U61vA@HlQc^+b8n zX$dMwwb@;)PO~L#obrieuKf&)MR_$Rwfvd28%u9X+*9pXjhMbE;xWQH^Ar-C=M6B#`d(dM=b< zem1e4*fbbb%J)DtZ!cJ8l$2!h(~r2;j%C3if!1wAgbQKFiY|L%|59y4jEreE6yvlN z$hr}Twgp)?0?{3X&r$gm$Vd9|7OS*+xo!lqxNZcpZUoZP765_mE$N<;@j_fkbuC{6 zDK<)2Hv&jwrLnhm2Ct>1dCt+JP+&wGvIXaztG=hum3~fJ>a#V0k(jL(Az%)?f~xrHSm0} z354Jj@C@)hboSf8N5IXX31+~l;G5{}?*WVbx9wy-BuDs_k~fx@f(kY>b0+F!`Nd8MGm6B3`Cgo9hjkzy`D5vVxZ0S+Fb8 zhFVQVMdD+m)b;le(hFiq(ZXkEl1TD)k)x)KYBy@tUWOu>g#3kBOR0RT5UuJ`RFVAp zQMI-x#&bbBj9J)kwC6je`4Iw(^^v4%f{(c1mI{-n>@_<=+{$J#JsUS}O!7^B*uf3^ zIa!~uJ2s`%Z=Jo&F4;g*I_&6OG`n1%&dO0Qg^rgR-O5t563-;R{I(^060@wvg72<* zt+)9T3wYZ`sTFHTSdm?AZ49r-b0vjXY0$LST^CuiM!HnyNysW=lS_SAqu%n7rK72u zkXSMWVIfsXW+7`)ki=;-7u7j~ke-v$`Kdw;ErhlZ4{QWm`8K?g6Jk|f*HRPmDJIpG zYHEHx2%G1-XH6CC|BILElzJ<`W~ZE7ow#~RGZa%qYEEwmNQ=oeeJQco#kGwjDWKv; zsx^`DgsULsn`+}qFxlH+)amIX4Qd_DSgWhwqnXHBuo4x3l=oT-O%qugPIh{9RF>l$ zvo@%FP0bQ7Z(>;DoHAM-v>UX- zRwY%>DrdEmK}jIk6W=l%F{U7DuT33kAiU(1bI-)HM^&sTiO%K%eda|bTaBu0wM7}}n-$qGSN8!M%8NY{Hml6pT=arrgVSqPyC#JpzJ3y3 zd~;Aiv#!8rm8k5ZDU)VlnHN)etP+ueU=zybT~WCc&s7qbi)c7eQEctAN>n-1R1{I% zGYv(Rsj!X`M?WLW{o*O!HR>;HTd9X6qsKzrAFZt(2ZV4o|+sfaN!nh|+ejw9H zXC}(2!z4)-dit5MLj?rtbAPc*1YXT2V@Z+|C1#o!VzFLw#Zs}=Mzq~3mjXG5KuD3t z1=L%@ieW08w}Y*MRq5;1g$T2L#1KQUsoo;53^i(QFu`**5-4IUJzCOG)_@rgzj{e6 z&qB6X_s>r3Pa@_+5G~@&ni^|W9orzfTC!s%oYQ0{Nm{`#Ij^Bxk+;1h98=A2I=n!x zOXmp3cDXH`>PsKv@cdAvF~Z0m8(_-c<{um=jfmmAyMZ0^194Es^)p_ICO1@;kjHFg zA*5qm*TWp6%TXi@DYgBi3gWp?>`6k_20gh1KI&qj(eLyz$do_Yc2ey2bb4&M_xgeG zAg|IWKTNXPby#9kFetXM)d^jI!7 zgmS5Uj44L5b4VHcj~sO!3d*Y3wn>jPo11BaF}HW94kBnN%+cZ^V>_@G!nBouGOx-L zrS|_(D}&!R`hVRhm0v)|e;*it3OEOx1U`ape>0G?{a1pm;34$+AA&D|H-k5UUk6u% zt>6^!59syxfOmjvz*%4lJR5uieg0$Mqu_VJ?|^H;E5HFT3$}sN!O7r9==uK#d=y*( z=E0?)1Rh4`|1z5K7B*WUtO0WJYg1NTw^ zw}ZEWDtIn99Xv#xd=sd;QS~BS$39~0k{no)?KMyGFz0i;V~94UYfRQfzRS^amjyv( z!nu!Qt_2oID%hYMbF?$!5bLlItRpU$>(N2Zt`ujb1!FnXCE99trBph1U5N5FDFTp z^TJ*bEe&TZrPB03rkdq#t>;&|iu>EJezxHl!Cj+38p*fU3~9w3#^kLQ?uU1{eB!Dz zh>@HX4HYpWmZ@xwYrc9Vp4oq`V3Da$0D3WV+1M|$ACVLkBC^gJ79^&V$?TBai{Ym-Vl z8k)4kHs4v`O9}E#aSf9Wt0SvSII?LqyzLkFn+%;kRjLbQLgXeHKZ+DBX}_s9ZBt47 z@@;!qyS7n>wr|&Tlc2H)kRsBx?0^j4!U z)tYf;#c^b0WXGV<JLkkj9V)q>91!b8^9=;mO>&(8Uz?V=?ZDnTAY{se-Wj?` zvrA5W7GG7HsEN!qXkb0A!AY_rDD}u?EPm(YhJuKj0+!VBeiAs4#O$*~&1>2?88#w& zJnpEDZD6txz}P6#g*0isR4;7B9KNX7O6PXO4KI9ZC^eO=V!5gpDwWf0hDPt~lk5WM zEedy%a47A8ExAgU(%U+81efYCm?Mv}IO&LAilVNZ$VuH4k&2?)F%_6Z)E&X_c4lN` zEKCy6S2c;QozNm;guic)`j-T=`wcdld?&3!Rs+mVfL>{P1ba+=vq|yJJ6CpIIO(=& zEDbLI|7IrRP8z{kP$;C0}Y z-~gzC-QX;6GI$&iTYyi4_kh=fE5Tv#GEf0qKmhJW_ZM4$KDZ3XTEH&?x$Eyc;1l3B za22=!oCy99z5gTNI&dD4yZknT`_cFR0Q^UAH7J9h1F{D24e%Op1$ZfV5!eDw1t)-; z(ftpDePAb$bN_!0-Unjv60jNEg}(nmAU6GD;56_p^nJOj@6})v_&$360U-D9JxCq= zJ-8i6eJGm@Qy;5-*_L2d55??T)wkO!D_O5m*Wn~qq|P5s`}av@R3Qo1@2%tlY2m8I zp)p1#?grtSH17hW^go7XvB=4+7!*W&d{G8Rp| zezK{z+LTgc3thMvVLg)^QzhXN5OXckYJ?NhP-B)(5-wZv@&xByD1tvC1j6w2!{u=? z%i5*+jYr}xT3e%cbo_`IZ*fH7Xt~{8*~nxr`vpRDg@wm*tvld09lf@aNnH{)aiQP8 zmyYyGa%@%}pC*YF$VbuR#a?4cG)sD-P`c z?6*h78VTes$#$nLtxz?)>9rB;q7)J%-KfyR#FG}4>O-)pYhU#31 zO6iD}UlaRNQml<<*qc?(#JiYL=7{*L^fioIOwTCnRFX&_AK9l9ms?A;+>&X*`Pyg3 z73qqAT{(N%h|{6-Tr=swvH6YTi`)aGUMjoBH_ImF96y2u;ZbK%BVVi=I=}w*( z|0M8+4cdlLjY?skoqR446%Tm=^=UrR^#j!xBSj|u`PHUO=#uPyq}@0G+US;Khe;0X zx8#^5E4C@uQFV`CrrC(d=q{JE)aC0!e7N_LU43)$U`+Ny7E$zYjd4uA5vp^BGUDmQ zVDIKm+kEmOi@!9OF7#}|Fd=XQ+be4KTW&?t)S97#Tb=avR9sw!X;q&FT;~*jzmthi zM9$#%4|T#0XE~Lu8!KZM?q=dnJT`8s{gCmL%@gjG_W%}di4^KUMlVD;lIay8(bD5mISjnD$lDo@gcn<2C z=v9@{dF5|%S5TB?F73=SsK1HBQA>1rwJiD{W!E+Mz}$i;;9NUr{Ij8f}2Dzj99(j}G!izF4g zoJu2I4!gi6?B1#GM~u}ieR_zrqO7{{FH+1lH&`-?Y8#J0qRL~FQbZkV^j59Mui$Aj zN9?6(7#%3Jx?&h+w~5XCh(2CdX8*xopl^!(|B_KO|2sPVjo<~~iQrS{^&bWw0#}3Q zgXe)?0Y5^AzY}~0yb;_0t^qFw=Ycc8gXr%c1MdZL{{L!F19M<2*aRLAzKcHpb?`NC zJNN*&5xg8sgE4R>I0Zb6e*XjTAdve2KMXzuejRKBCxdUJ>%RfWef~{wDL4as7hV4^ zz#ZUv&<1i3fZPRmA~*s3F}nVL1!52Id@us;Lf`+-;4R?IpaHglGl1OR|3PpCXo4kh zE_gEd1iJq%;QiqB;N{?Ca6EWCxSO>AvHx#@X;15obP&{9ZT$MYTxKux~S^QI$ zZAvccuo>I$M_jgrUT4r%w}F_Ne}rX&*UCU2+eb`4vqMcUc(mXAkgw}~A zedA(J#^f*2w@L0U&sF=iMSZW#W9OJl>{!jxHUxTb?uLIMuDCT&`zE4lv|l;%EvZYH6O{=knH-2b#oJ&0a|J5>Ut-GN~B_~ z>S$mSOn+w>G3{TAQcUkix{#QHrvDC!u}rX)yq^^9NRF=J&MYPe$dk{7IX62o>^Bf0BPn+_XvB zSl)@N)N)Lq;+qfb1adt!#n*C;)r0iAAdcPVZ0bW&R@LZ{kfl3!U?xSoP27@^3!7dF zOz~!e+^d@E?Hjs1*}(EPLT1lm$y>_{?1$<7oKiKe^aor#sbri{ws#2ST2x^&%%5vW zX6jhUOs**z*7Or*7E&{@mSl2iq#Liz5gEYAs}Z4E(F`bHgFR7c_8)q?bNz}bTW}4jEIQ9pF3xw+FYx;Y)+5cOYZjU zDUEcvqqx;LtaeB+61G7&Nv$HerdDtu<Lv_eoc5hO% zyjd12IxizqzaCpfGSFg=p65b&XO;|c^6CyP4*oQ0IyNz>0u)lD*i>NEvo3I6=wR3G!7&Fo7yW<25gfn>?1pwXn@gY&w{i_VlI0yv2#jgy&Y&88=zq#vHKSFN-82LV?8KZXb!PLQRgVE0yKY zyECzLV@4o(%43TyY3LsB4vbER_GLb^>PAxL!s32!6XE_|mqk({b;wlKTJ$Z*VRKTI zrOYuBSs;nS0b9e(DdlLmnH)yVwse-V)tj&i4m_6pS3R+Juq`w)@CBr{CT>ekn&?@!;CoTNZj2`R%fhM}^G;J>2G zCd~U6&2Cb`EV~ zVEWOZHM&2^si4I0A-P3WzX>D>E!4>4)2H1+b&fl^Rl=jP;Jk%IatFUL6fuQZW4z0v zy)P;qGg&V&$EqRmHV8dyE`Vux#Cf55W2216Ygtb*@pfW%W_GkVGAyr{lM8Rdvr?OE z!XkzPRjndKZ11j&o~-^lF9y&@Z3r?Pg=nd8zDF_4df z%Y<#6x<1dfVot+LTB99S)1F?%xdhg9C$NxXw**=ClGn+EF+y0A!^MpAA~neZo4l3A zjRdB18M973EvTDPnjNeg<(`R>dUvL*$;(@5bPw1gxA2Q(#;d{dY0dHOxH_U*xY+M^ z6fyTs$xj2WC)Z$ z@(dq{tgotUbp!Q|WPn@tMz~}+RH3%75~_HtU3JJ?)?;2E% z>GwZpp)`G=MUPYS={v4h8<;bc=qk#O$bN?Yzf|^Kw#M+K_eitfp3go==cqWM2sLXg zgl|?`bMWjCO0utH8~$Vmur*{N&S Yg-p4oUH$O`$&CoineK(+1;gL}1!|4j3IG5A diff --git a/egs/librispeech/ASR/pruned_transducer_stateless_d2v_v2/data2vec_audio.py b/egs/librispeech/ASR/pruned_transducer_stateless_d2v_v2/data2vec_audio.py index ed0159e23..021502339 100644 --- a/egs/librispeech/ASR/pruned_transducer_stateless_d2v_v2/data2vec_audio.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless_d2v_v2/data2vec_audio.py @@ -38,99 +38,6 @@ from convolution import ConvolutionModule logger = logging.getLogger().setLevel(logging.INFO) -class TransformerEncoderLoRA(TransformerEncoder): - def __init__(self, args: Wav2Vec2Config): - super().__init__(args) - self.lora = LoRAModule() - - def forward(self, x, padding_mask=None, layer=None, tgt_layer=None): - x, layer_results = self.extract_features_with_lora( - x, - padding_mask=padding_mask, - tgt_layer=tgt_layer - ) - - if self.layer_norm_first and layer is None: - x = self.layer_norm(x) - - return x, layer_results - - def extract_features_with_adapter( - self, - x, - padding_mask=None, - tgt_layer=None, - min_layer=0, - ): - - if padding_mask is not None: - x = index_put(x, padding_mask, 0) - - x_conv = self.pos_conv(x.transpose(1, 2)) - x_conv = x_conv.transpose(1, 2) - x = x + x_conv - - if not self.layer_norm_first: - x = self.layer_norm(x) - - # pad to the sequence length dimension - x, pad_length = pad_to_multiple( - x, self.required_seq_len_multiple, dim=-2, value=0 - ) - if pad_length > 0 and padding_mask is None: - padding_mask = x.new_zeros((x.size(0), x.size(1)), dtype=torch.bool) - padding_mask[:, -pad_length:] = True - else: - padding_mask, _ = pad_to_multiple( - padding_mask, self.required_seq_len_multiple, dim=-1, value=True - ) - x = F.dropout(x, p=self.dropout, training=self.training) - - # B x T x C -> T x B x C - x = x.transpose(0, 1) - - layer_results = [] - r = None - - for i, layer in enumerate(self.layers): - dropout_probability = np.random.random() if self.layerdrop > 0 else 1 - if not self.training or (dropout_probability > self.layerdrop): - x, (z, lr) = layer( - x, self_attn_padding_mask=padding_mask, need_weights=False, - ) - x_diff = self.lora(x, layer_idx=i) - - x += x_diff - - if i >= min_layer: - layer_results.append((x, z, lr)) - - if i == tgt_layer: - r = x - break - - if r is not None: - x = r - - # T x B x C -> B x T x C - x = x.transpose(0, 1) - - # undo paddding - if pad_length > 0: - x = x[:, :-pad_length] - - def undo_pad(a, b, c): - return ( - a[:-pad_length], - b[:-pad_length] if b is not None else b, - c[:-pad_length], - ) - - layer_results = [undo_pad(*u) for u in layer_results] - - return x, layer_results - - class TransformerEncoderAdapter(TransformerEncoder): def __init__(self, args: Wav2Vec2Config): super().__init__(args)