From 09701ed03a00a1cf99ca0150e0b73e268a877118 Mon Sep 17 00:00:00 2001
From: dohe0342 <kimdohe1070@gmail.com>
Date: Thu, 18 May 2023 16:56:48 +0900
Subject: [PATCH] from local

---
 .../.data2vec_audio.py.swp                    | Bin 45056 -> 49152 bytes
 .../data2vec_audio.py                         |  42 ++++++++++++++++++
 2 files changed, 42 insertions(+)

diff --git a/egs/librispeech/ASR/pruned_transducer_stateless_d2v_v2/.data2vec_audio.py.swp b/egs/librispeech/ASR/pruned_transducer_stateless_d2v_v2/.data2vec_audio.py.swp
index 26c8b1e144b355be06b35c9d9bdfe5dcfceb5222..ed9131ca5d241ec890b2b1e6e5082fbd3bc2f594 100644
GIT binary patch
delta 979
zcmXZaT}V@59Dwov+;nbjJ6mOosMI2))UcpYuxJKJQW2Vj5>&J0*g&I?8{6tEDqFj#
z=!8L$b<+wWa}LsrSrG10DT7k#q9mh~DAK6ZKAuAz_?`1|zIe}h-q@R*Sj^~dtk_+>
zqiDnG3So1Kd>%O%_<HwA$~~bNteVl;Z18A@oo?40{W@Fl-$6d6Tb)6ryU66T_vDM*
zvv8irJ(|n6Nj>8biJ}Yxc9AU7xQ$Z?U<*7b$6UBDWfPe|5^bo%W-P`yGw6nfJy?Z>
zaAVvml0Xa*tEf`Pz=L8;F|}9dLk9X(Z=wS?a2Y`~VJjy2oKNUSC$3{HN>Gexvq%q8
z=t3JBum(1a=BXmRdEA7(@M1cbiC_|0#BmP$P>mUr$TtijgCP9ak1~`ZA1NarfCP?V
z1q!hgOE8)vGB}@I2O>C*R;<_F`b*RH#%8m2D^#MTR?gG9cAC=<LQhSm?T2bB(-&Hf
zRI2^}KiMD>=2d#m;Q!ix6p`O}fkB)^A>7EtcS1kMLtI51DgrowDs03LA`c^pD`<fi
zCXCq#cc0Qxgb{)luPHo)0VEN}QB<KE3LN-F>{-m<3$o}(5;fR{Ww7BVp)cVqRMexu
zLO6wi1MeuUFYJp5!-oYZK!(cKP>)*FU^(6sdJKKIg9hwE1r|XM{}_*uz<D%5fdwDd
jQCMFAr*Q&NMBszI3TDYNf?>QwJHp1)vt9#b+iiaV$kUx9

delta 500
zcmXBRJxD?U6o%pBfBxK5h#Z16p(w178p0y7(IRMy3Zj%Cmn0E`vbb&;O>wu13>tC>
zL|sZ~32H59Xw-0$pvhWn=!FhEhi~|%bFx`OwyyWYf=iJFUuZNaj7#LFXMXc=s#x-T
zI(1gf;M&%uw4}>ZDY%xMr*&W1XACR-R2BVyKc%_DGwNQaND_T`R79#M;0(K1!6X9k
z!vZs!4v{*}kwF9#aHD1yxj_z#@IrwDHJgZvG<IyFN`w$VH=0(FSCmo074o=*hB&qm
zL90XL3r{HG0^=BjA8m`s9m*)-2&))_f`(b--mFq0tYaE&lgKw(c*Zeu*uWfqjocY;
zct9L6L@|P4SWz~J+~Ndj48R93dePtqFUWVX4v@h<LQu4iXrN+`B`lTq>X4~&omiXw
E1LXiwWdHyG

diff --git a/egs/librispeech/ASR/pruned_transducer_stateless_d2v_v2/data2vec_audio.py b/egs/librispeech/ASR/pruned_transducer_stateless_d2v_v2/data2vec_audio.py
index 0e8b4a95f..406433944 100644
--- a/egs/librispeech/ASR/pruned_transducer_stateless_d2v_v2/data2vec_audio.py
+++ b/egs/librispeech/ASR/pruned_transducer_stateless_d2v_v2/data2vec_audio.py
@@ -229,6 +229,48 @@ class TransformerEncoderAdapter(TransformerEncoder):
         return x, layer_results
 
 
+class LoRAModule(nn.Module):
+    """
+    Implements a residual adapter based on https://arxiv.org/pdf/1909.08478.pdf
+    modules similar to the original residual adapter except layernorm location (first -> last)
+    """
+    def __init__(
+        self,
+        embedding_dim: float = 768,
+        layer_num: int = 12,
+        proj_dim: float = 512,
+    ) -> None:
+        
+        super().__init__()
+
+        self.type = 'linear'
+        
+        def build_adapter(embedding_dim, proj_dim, type_=self.type):
+            if type_ == 'conv':
+                return ConvolutionModule(768, 31)
+            else:
+                return nn.Sequential(
+                    #nn.LayerNorm(embedding_dim),
+                    nn.Linear(embedding_dim, proj_dim),
+                    nn.ReLU(),
+                    nn.Linear(proj_dim, embedding_dim),
+                    nn.LayerNorm(embedding_dim),
+                )
+
+        self.adapter_layers = nn.ModuleList(
+            [build_adapter(embedding_dim, proj_dim, type_=self.type) for _ in range(layer_num)]
+        )
+    
+    def forward(self, x, layer_id=-1):
+        x = x.transpose(0, 1)
+        residual = x
+        x = self.adapter_layers[layer_id](x)
+        x = residual + x
+        x = x.transpose(0, 1)
+
+        return x
+
+
 class ResidualAdapterModule(nn.Module):
     """
     Implements a residual adapter based on https://arxiv.org/pdf/1909.08478.pdf