From c52b816fab2a1c90d589023f876f14bb383d7bfa Mon Sep 17 00:00:00 2001
From: dohe0342 <kimdohe1070@gmail.com>
Date: Wed, 4 Jan 2023 00:10:52 +0900
Subject: [PATCH] from local

---
 .../.conformer.py.swp                           | Bin 0 -> 4096 bytes
 .../.conformer.py.swp                           | Bin 0 -> 16384 bytes
 .../conformer.py                                |  11 ++++++++++-
 3 files changed, 10 insertions(+), 1 deletion(-)
 create mode 100644 egs/librispeech/ASR/pruned_transducer_stateless5/.conformer.py.swp
 create mode 100644 egs/librispeech/ASR/pruned_transducer_stateless_gtrans/.conformer.py.swp

diff --git a/egs/librispeech/ASR/pruned_transducer_stateless5/.conformer.py.swp b/egs/librispeech/ASR/pruned_transducer_stateless5/.conformer.py.swp
new file mode 100644
index 0000000000000000000000000000000000000000..bf23ca48f43109c923b4e4bec31758182daefd71
GIT binary patch
literal 4096
zcmeIxy$ZrG5C`xGf|DTn0)EUk^=oicTpS9b=vvbBw573Ek`{6C1solG5Agx5f=ekr
zfZT1l{Bs9Ch-dGXPJ3$EEUyM*5Bn+aw_5dz73O0A_GmnOU;U~@o`WpxJ3`J8qm@QW
zp%kk4=JFlLPpBi42njrtuop^kI2f)%PlqM}kO7}<);!T!48EIc5gR{~pk0%S6ciXU
z;S%FL>LX|?ITshvnFL6nM4-+_!@=jP_qv^StGO-7P>TdefCNZ@1W14cNPq-L;9mt^
DiJLW*

literal 0
HcmV?d00001

diff --git a/egs/librispeech/ASR/pruned_transducer_stateless_gtrans/.conformer.py.swp b/egs/librispeech/ASR/pruned_transducer_stateless_gtrans/.conformer.py.swp
new file mode 100644
index 0000000000000000000000000000000000000000..18357caebe34196c11a847edaf502a650a5a1aa6
GIT binary patch
literal 16384
zcmeHOU5q4E6)yfoL2xn16S>U9q#Js=duH}`q|x3VWMp@S%*?Xly3|xx-R>@@etPRx
zO;1C>#AMM2UY6hzl*AVwR4^uLM2#l+LPU)|tcjW^;Yr~|QT|25@7#Oqr+ar<5;ZZV
zD)V)B-E)7=Irp4<&$(68Y2LYbi0!F24O~|n#?y1(wQo6f*~O1;Twpk1pEu{X&k^aJ
zuKlRT6L;p2W$z8cjXR>S?uEAHO}4o|?XWQnqYW{zY~FBf-nBfh!F!_Nxt+)r1J3P!
zqrJM)7(_|H9W#!sKsbrbBU8jy%snoI*^{LWI}EyE<nyRL7)=gW-j^CEH87@uD~<WN
zS?TK2Q#Y_{wjCa$mN%saN)416C^b-OpwvLAfl>pd21*V5e`_Efe9-tTM7&Uod8huq
zVB-62eVMxa$OQeLn%>dn-<zPnrRmGM{5m~AI?g{doojk;g8rtaZ)o~2Cg^|H^mR?&
zr3Fg+e?!w9O@Cy9{<@~88PS^)^w%`q*6r`ONc~9re^t{zrs?0ApueK&OPc=D1pQ@A
zzeCg87pos>|1W8Js`rx<^uK9(IzP`((EqCG$8`T6zhu1s7d5@F>0h0o|3%a9*7Peb
z9dG~ViSa)(LD%9MpVaN2nV{?9@~hN9sew`hr3Ok3lo}{CP->vmK&gRJ1EmK3OB%3{
z5N|`@4^X0s=l^N`|H4&<aSC`CI0W1dTn;?_e#3YS_!8g(J>Ul54_6w-L%>1cCg4Wk
zdSDxHA@I8^4CA-J&w<B)2Y>)DfhFK(;6~s&;9B5~_Zi0Pzz=}$15W}E0}le90}cVt
zy%&4{n?M`56nO4(!}uoP0@ndQdk^S98+h?D!}toY3|s;H7&&MHd>HsO^4-q^3&5X{
zJD&nR4O|8M1j*|NI1UVey8+7ocK}xdFXOD>_rR0DNk9OHfC_L4@Ehp%8vyC}1nK#V
zi|I@B!zecGeiCe$!o7ztFgJ)<i_Pt*8E2~#_alxrp0mKZUT9I(W_?$ovNH-S-?ep@
zcxS>5+@QC>I$`Lc@~xI9c;2;@Y?+=l!iyFI9=RcURm0EcRqP-z{m@A~Zf@`>;9gc|
zHm{`maH4P!CUKFq<`{2h(hGeAiEr}24jphK`xEI%vvc`Kj_aFU&K<l2GPI(M;koAa
z-Fc;;&#jEKJui1il3aqh9XoTHNIJsu2OhL#b}c&&qjdN}2gGBSTXBL}kmN#p?se6j
z!@JBhq4(G{D~6oLsi~=KI%!&^e08?gcLh6icy*0+I9rPn&W3#+utb0fcJ`U9XZ_e#
zA}o(pXlAPm>`0%-eIBtmM0?IO6|c)|hF%i8VSuS++1oQrYdY(VxRUI^KteWjJ&$-o
zPYw$MZ=@JIJ4*~g5c4@})LEPPZjcDJX?c*#=Xj-u3C>BfZi3!D-i?W$m~X}7Eu}Xw
z@&@YJ;kN65i_VA@oh5SwW(Cd|g&O7^(u~ni${_@+z`&s`sA`M@Dra(FQDs&f^FXT1
zwgO_hM)eJnD!u-rswxPaDqfl*q)q~tUU$L-bfeH`voyU$HH-^O+)2Ih;|b=GDu)r|
z6!_VPDC_IK1&4=;-8j}urp)#AaZx9SO4SFQFekJHvpV4>)Z=!gM)BU36B4sM9`Fcn
zZaGS)MyfXYma0qT{5K3tT9Gy&vh8%O1CMoHhY81hHTtAVv<|Adkq}!b#Z(_p;22~Y
zK@ZKtF0+#;f-$g;72EwPNeM@am4kqJgPNunthE<mbs#T%m+EGfbVks4(#SA<sSiRv
z-x3?-J*k}M)w-11NSP~6Wj5ceB6krugoV-d)<S0Ea7Y0MCMP6c$xSrtkUAzikjsPT
z`Yuhk^lI|R$nEvd;on%P$gAR*q@jfalH?i%C#j?YqQ<;8B8GfxLvfNh=NKcSFkwRr
z^MtBWLFfj}Irc&fB3FH~Y$zFjd~3jM;#MhmeC;~N8ka7!SfHVgY?;-ywSJ0Ild*Tk
z2AVq2?Q%O+)=8FDSgO^An;YoZ{+dskES}NZE+b{-P14eAhSLJ7mN9qcrkipHN(XX+
zF}+o&O-p1=CwAhrU14)(sG=6qtdZQA)2jIpv|nd+^e&y&$|hWOC$N>(nB1kvFcEJI
zk~k-2y_~t9ryUk3OzreVDJ#<43v;1j5qKgNVwfPko2uSG71eAoQezqyq{A#_3UmWv
zF{R5!?WiXf<Sz;hY|p|Hy)S*WwQ@j(L5tWNBXG#83kV?P2B6=btz$>W`tJGzgpblU
z%z*{HVTq_Ql20yVX<$jkx;?@gyPvjO<ADgH8VdqLiL#1!c)4@X4Y(DlSVonAv&}hY
zvSe15><^QU$5)50=&LGwE#cZr*tXPd+m78$i}usKmS<skMc+~wf#JwKwHOsus?7}n
zl$JiFPin=`?W9bpe4|03$AhkjYixNyV_RO0ttA7Gr_B&PQfMsFd*6!t>2Euvw~MEl
zT-GG(0$VX6Q@I!(#4H2?zBbKPu>g|6=pqKokoS10*>=0G4W|iuiPht*hmBGW$PjA#
zt{^+c{uwyv<;ddNZOez(*pc?iN_%N-@xUruUSa!|m-a8NEiNzN?pD@bx{KYuxU|2<
zICeI0f>jI)wxAszitt?S5JL_`j@(W{%i9?o%Xq3#0YM?6f~oc@TU?!Dd)upvt2Oj-
zIFKPj1S4`I^n=jL8ejz5!aPHwA&qHat5ZuT_NHOv(rOOd3%fBC%Z+LDLsvhJ2MdkH
za5$`6c>lZ4>tWQxi{Da%qH%C>-+`so12c1YADx=qg>~GwMl9^a$Xu|9(@c<i7L{zp
zs7Wh6)eOQ|2E1}{@48@+Z`phht8kYTLnf!ffkG#wD?MpNMYSc5DW%5l<Pl921Ak1e
z3{$$%RP`g0VKRVvy^;NCfute8ts^o5X+SLEJtQn-oqpoQE-aB*NsRnY$d2|wP_=1R
z>?&h(&AC~InQ&n>!i}X4_VwYJi0#|9<#1mXt=ca9g(I$2eJ!xtFr%O9CskwmhDIWy
zM#l{rJlJG|5ylGU4a)z&e34;%h@ST%|4*OKK7m~Sao|DVW?&9@7I{A9|33o04txzb
z0knXfz-!3!e+fJVd<$@aBS00n26zs6{jY!%z^%abz&7A5<nm7d4*_2QjsQ)d0n~wK
zk=LIF0-z5Z20jW@fXjhDBF}#Y_%`qa@Ho%|y1-q)Zs1kq|E~Zq1E+z<ffK+nU>EQK
z;02rsJP(`(P6GD>KF|hs0&nA-;3?pTz$3sZ;ETXL00*XltAV$n%hSM5fs?=ra0{>-
z*aeWTuLa7h)Ih0$|49wdUS-By#HLDOO*{0lbaKR0ZcwSJW1KwT!ye}+gI2R%IC_(D
zNS)HsN=2rN@<gvz#jePYQ`TK$PCOd$ma2&OX6N>*3Xy1ZRZ61=#pZcq9YC6hs&%t2
zPW0@GL9M6RClaATK?Hyls3IysG(v}DY(U{9{<lh~^gy#i$Ba$cIOi5+J7kfKUwJ@U
zjBi^7J|K}gQftl5?M5I;aeQmLQd6NC_1ZLWNMM>co#dv;S}itZO5K{~6h*fXfptj?
z<YW=|5iH?=6FJ{*q;)z@qbm6fl-ho0Z5yPnvsrY7j25TH@?%~p^xXAksp@%aHP2sb
z&z?P~MD?2I8}nV%8tY9SxSrd0Q}cooaAZr$aPTjPwigOs8P(X<(dIFPT+;~Tkd0Hc
zi1S|@py413A2{IP+HrIbqq^u@15QRvP1T*$_Xw8mSGM$Wv4y*%YB=hnGGAlc;Uvfv
zZ5$8bC>>{wlrWeStHSYnyh22oE{ZB%+@mSXEndWq!#7*%D_=JT@OE<$g`HORD<^hc
zBtGVl(`ihrc#~Jjhiv6P71@UXEAmWlpIQyoT2qujxi&GHH%8gKK}9+5-Kn#g7@iaq
z`MbAxcVCTV^0e;Fexg|G;a>H9pE_9|L|9KN#<@(PQtVnkpePKGm8d@TtxUT~4!R<a
zQ~!W@VUKhz>Xhl=?5={<xHX##n$nLh)Y#1IF?Lf63%J*<tMNhAlVQkC@sT&B5}eCY
zZN-K`CFxb%9!S;Q2rH?ey#@7i-KTqu+-Y#bH<4<!)K@-K$|Yt>O(~5s{*YoF=NxsB
z_*%eq${okOL2(AtJ=(0-Au7&1W9&UXE#a_tMhUM<hB?l*9A<5~;Fk_(4b_xB+YD#z
zIEA1bPkEK}fpbWQy>!@jae{&W38tf^{4{DsPLDoQNxNUe9thRwb1WP=B3te<<qh;?
zK~Q>I<R(|FI-DtXq;$vvWWC6H(?lOy8seXkjkG4yDnCnmE4F56|4#>==mUv;y;oO{
z^)PYrP-U!NYMXbPpCHlD>+h(w_1xPr$x2not#i$*n1^O<rjLADYp9k@iWCp-Kts_*
zMyD+e&r;}2sXk_KS)nvPwdScQl~2tK>O;C}k7ogw?{49LzxB`+wfIpz?RZeMLAP|}
zhD>)VsFI%s-P48Iux-Uz5FjWEa0o?*Ky}DyLPk9AarhQF5Y4$ffa9Vy;Y`(f9OnKp
zK5FidW2zr_ICisM;b_gMsoxk82Ys~e8Io0eXkIdoamXC;f(p_GSp}LeTJI+KWit%>
Nb&(%9%s5BJzW_c&J~99R

literal 0
HcmV?d00001

diff --git a/egs/librispeech/ASR/pruned_transducer_stateless_gtrans/conformer.py b/egs/librispeech/ASR/pruned_transducer_stateless_gtrans/conformer.py
index 8d766dd37..3d5891ea7 100644
--- a/egs/librispeech/ASR/pruned_transducer_stateless_gtrans/conformer.py
+++ b/egs/librispeech/ASR/pruned_transducer_stateless_gtrans/conformer.py
@@ -135,6 +135,10 @@ class Conformer(EncoderInterface):
         )
         self._init_state: List[torch.Tensor] = [torch.empty(0)]
 
+        self.alpha = nn.Parameters(torch.rand(4))
+        self.sigmoid = nn.Sigmoid()
+        self.layer_norm = nn.LayerNorm(512)
+
     def forward(
         self, x: torch.Tensor, x_lens: torch.Tensor, warmup: float = 1.0
     ) -> Tuple[torch.Tensor, torch.Tensor]:
@@ -198,7 +202,12 @@ class Conformer(EncoderInterface):
         x = x.permute(1, 0, 2)  # (T, N, C) ->(N, T, C)
 
         layer_output = [x.permute(1, 0, 2) for x in layer_output]
-
+        x = self.layer_norm(1/4*(self.sigmoid(self.alpha[0])*layer_output[2] + \
+                                 self.sigmoid(self.alpha[1])*layer_output[5] + \
+                                 self.sigmoid(self.alpha[2])*layer_output[8] + \
+                                 self.sigmoid(self.alpha[3])*layer_output[11]
+                                )
+                            )
         return x, lengths
 
     @torch.jit.export