From c875d7c1c15c56bc5764f5c2447f0a15be8ee5b0 Mon Sep 17 00:00:00 2001
From: dohe0342 <kimdohe1070@gmail.com>
Date: Thu, 2 Feb 2023 13:48:52 +0900
Subject: [PATCH] from local

---
 .../ASR/conformer_ctc/.transformer.py.swp     | Bin 49152 -> 61440 bytes
 egs/aishell/ASR/conformer_ctc/transformer.py  | 124 ++++++++++++++++++
 2 files changed, 124 insertions(+)

diff --git a/egs/aishell/ASR/conformer_ctc/.transformer.py.swp b/egs/aishell/ASR/conformer_ctc/.transformer.py.swp
index a4911e7486070c7e7eee125ddf425609e944c93e..89e1f19e85894de52db9ae2209a5d070870f315e 100644
GIT binary patch
delta 6321
zcmbtYYit}>6~2z02hGEdlIGEr=z2@!*~%F|>Qs%919p;D6^fgJQx&D|WOn!N?!>b*
z>pQbfOjS*gXp;&8v=hmtQE5X0sI(x!NJx}NL50%(K>TT?sz|^eP$NN#5K=*?@SQue
zv$M9>iPe!l&wA$GbI(2ZobQ}7XU?>r`BVFjT>k_29^w1-euk@yvA=)fo&4<k*^VFV
zxZ2saF>LQT-}X@WWY_llH_`xHW6_O0-WL6cZ){Wid?>#EcES4%@$;MF`{(Ds*0r|t
zhrHwH#vOQcg@$AOhvqj|jkd`vs~9^5tOh!O|KK$ZxeR;&Tm!xbTmsa^IpgsqoV;5Q
z(y+~0A&0r?aLe(0)5#0&7P(&*;U&2<$nv~xdLGxLn66n;6tcRKFe*#lFjGIr4u5(=
zEBk)c8}92f<!Nh1cV(&1%z1qScMbIG{X=*4@5}NER<rpj%P;f1E2Z$Nu2Zm_60e%l
ztO#F7FA^;?naqSV9k1xhLlq$pM~ZUG<0glO`6*IxoW~pBY6G6mk9iQm^VmqPX4wVW
z0#nL|-Q!#5e<lwe88ZuvJ&hyy104g*Shw<9os9hscm}8ehk%2?e&BZC-FVJBo90J1
zKfNjZY4^JDoYp!2<LS<}fP~%+c_+!p5IUwT&Di%b(Fb+`uR=ftuz^XS3wRm6@+43M
z9tK8%PXcej2QC9eU=+}RYwH<%19%;{2UrKJ1^#^#3)m~z#utD=U;xMf|NIDJe+7O5
zJO=Cm{<n^?7k~?Z1dafkfWNGTGQc#j1^C+<#x4S<fjz)KRx|b$;4{G04#s{290fK2
zbL|0RGwqD62mS~bd<rN5yMec15wc1jpp$i?tL}kxKUJPJ9M?BIc!6j6vqsVMe4}c@
zOGbF6VtP|XzASL$!ZCBU^0)MK(<uyb%;&x<^JU#|gy)Nb;S0xerBRe_#ZWSAQS=Sd
zwn~mDXr9c6j=e3=ZMS6QO`8<Zvf1o#B3{^5k!zYplFO!N`o7dWVHbPTDrK@+?n+M2
z^@gI^L<s4*-1b)SOjxzIZs(^=kB_^K$Rtx~{m`{<4#?^0RYSGwN#z-3DbZ{SwFd2G
zLe>_nqs*r^v3O$O$`!0!%vesf<|~0Y)5p0LKF-DIs>tI^IION0tcnJs75N>AQJ<Hf
z&?Cw7N1OLgVuIAHkr95!ij;Z8^Wg%Pgg+^YcvN>>SsBnp)l19)QZsEh57by{a4<-t
z7wg5fl-!h9=(zBOl91&q(mc^7Xy|U{eQU<_E!Q!muxkd~yM%kPlNJY?R;PuCWmF3m
zMr1Lw*ihs<213Nqjp?Zw!vmQcqLM=x;<#6$-(o>QO42fFF4i)!V8eh7>B=`F&$y0y
zLDO+ADPcuwEi#^3-*UJwP0OKVNLjMsZ-FAB^cmI%ez;|o%cd-_g!8CHAB-X=J^aHh
zT(>uBiyc~HpI}=eC(EmsAnEw(4X0)I11WZ15UP|Y=xFq)5IEjH*$})O|MH$57T$OE
z){O=VjcB7DA0gVEOo%vTdMwktN-`+#Ix|B0hVRCti=lU$wkMJiZp|lRganyq;h$gJ
z8f09@p5-~$wNZH$c+qwd+c<xJwkkYg$SE<4G+3YtNu@AuO2NHawdzW+X{I^NP*Q*g
zd4i96sDg+k4om4uEzy8Rz{-X<Rjn8We-O*??vXfcCTh?jM48m`jJi%n)wLX7qg-`s
z>J&&~T^vbMmfNfsXa|g5(iNq#@a=DZ>^2Vb2`b8X!OeRpHj#^>A`dlQRAq+WJ(~-^
zGq+(C=OI5g8UA-}Yxu%gX9rb;;kV9do7FVGCLNs}Cn=NR@2_mT9Y!7I6DsBRa>~TR
z{JtvLz_gD?k0)VzN~4ta!_FtxZ)tv<<$DkCr@px@XrysWX#B&}Uhqdbz~VHCDqtik
zP!^=!=82SkFpzEwUeqee|2~EK74PH!q}Mzua&HU(@%u*J#(q4lsfzDS%?-c$rA+v%
zXSatJUfdG29EBOsW`u0hyW|c=h#DfRMunJq05?#&83ufxng-u93}|+|hs5<n@q<H)
zHxT~({D#e7E>+Uwv<;<C_~!Wo8`4k1Uyt*lma2~2+3@)T2M*NBe==P7#`rDazAq28
zb2v`-_?7^5mvGQnsPI8#_<{<lvQ;YU{E#S`HQOKNCr;4bPMqK{B5KQ|wSf?IX%v`o
zZ1bFII~*QGa#rUPWk@__**3Q-Rq4(|u8q4i?ZD<$A;Bt@xSe9L$R$lYOo17hIjWzO
zRI}#06>tLTQxHX64OA*oiOYTE?}=ct`>={G*y9!i6g@z}`#DuIV15PdmT>&=xw)JB
z8Xgu+!T&i-0qc0R+}`*E_Qw8ZIpMXrovO>Zhz8&+8UpHa>J97{(cRd<M}ZH})qDe(
z0{Ve9z-#Deegj+t0^lSt1Tf%rbT$`&lfW3T19%Tz&8xsOfCC%??p)7GY#)C73!Tl^
zfro)Dz{}`rJ`22uF6J6A2ROj{=wf~eyaGHA3<LXtUBFdr;9cM~;CbLV;0fRyaCXgn
zTgT<q)ZYbC5Yfa39=Ut{6dQC-X@@MW#eR{T2aZ&c1-&HSBW8^X`D6%5OC&`<bfggS
zTGUPtMhC3*>kYSu&_%f>jzkhhS0WIFM66UVWgV46Mu^hDf!o73AKN;ZN_&kZ;IveO
z5Iv`|UA##6i?jUx)HFmq?TM+CQ-2aAmhZ}9{g&v(ng~aw(uRihbrx9y+E5fVX#<O?
zCa!fIkX8A&O{KT4^hpuNh21ah4*Or~40zPq^u;?vz@J1{gA~J039yQflr4GAGCd^k
z_)Cdrl%%;-qHe7VcA#Zw7U>a_l2R`)eQpbM#vJCRIGwHuIZJC&KCRPiWEoX5CP|#6
zfo>JQq4gq4Go4xOwBDp8r(-`&haP#IVdC>nD%S^fqc5v_27ab2DDaU7d~)#Ep;6)?
ze(>nAC3K{apk7=@t&y)4OxjLQzFO;1&PN?Lb&O)h%8MY~oZp}>ZbigJ0ZOM*tpf6+
zULI8%z7!b6nxoFISueOtWI5t-X%&R0t0}vh=4eeu3NPKugH)ZU+m<6tIjEC6^fq}T
zX};9R$cJuZ?7-4eoBNbVfdTzS#1cQ<h^o=JRuY@`PeWVi*|hs;Tq}`jI}sB_9VX(>
zpx|3zhQvRwGizmwS&mxExc))zZfIDalwK;_`}j_!V#;wS))VJ|-%&KQj;;kP-+Ca#
zW~g+?6InD^B*PLDHS<G#Y=a|K(BNgLkSL3YuiQeR5loBvR_~tV4C|9TzK0KTd@>nW
z$&%ED1d7KfnSp-M7Uf)gE~MVbw$K_SR#i%<QB4V$qOg7~Z^&`=HE}{!S|BBA;gqDn
zPjzCfE=w&JB?zLls7_nS)*1~*o&0;%Mu}F5xN$dBTNuvT>!Z(x<TdndlgHmSGwlCd
Ci+VT!

delta 141
zcmZp8z}(QlERkdo=IN_osb|0d1Of~Uu?Dx3dlp_|-YD^kpOI@avp_Fn!sLkp;+r`H
zfACGbkg!RCC4iR=sCE$}L_@-6!Gd^3!#PY03{!wO0f?=Em<xzGf%rdA*#{tA55yoX
W3xOC6Cf{v(I{CnPht0=|tb75eFd|$4

diff --git a/egs/aishell/ASR/conformer_ctc/transformer.py b/egs/aishell/ASR/conformer_ctc/transformer.py
index 06485f3b5..ffb3cd8c3 100644
--- a/egs/aishell/ASR/conformer_ctc/transformer.py
+++ b/egs/aishell/ASR/conformer_ctc/transformer.py
@@ -381,6 +381,130 @@ class Transformer(nn.Module):
         return nll
 
 
+class TransformerEncoder(Module):
+    r"""TransformerEncoder is a stack of N encoder layers. Users can build the
+    BERT(https://arxiv.org/abs/1810.04805) model with corresponding parameters.
+
+    Args:
+        encoder_layer: an instance of the TransformerEncoderLayer() class (required).
+        num_layers: the number of sub-encoder-layers in the encoder (required).
+        norm: the layer normalization component (optional).
+        enable_nested_tensor: if True, input will automatically convert to nested tensor
+            (and convert back on output). This will improve the overall performance of
+            TransformerEncoder when padding rate is high. Default: ``True`` (enabled).
+
+    Examples::
+        >>> encoder_layer = nn.TransformerEncoderLayer(d_model=512, nhead=8)
+        >>> transformer_encoder = nn.TransformerEncoder(encoder_layer, num_layers=6)
+        >>> src = torch.rand(10, 32, 512)
+        >>> out = transformer_encoder(src)
+    """
+    __constants__ = ['norm']
+
+    def __init__(self, encoder_layer, num_layers, norm=None, enable_nested_tensor=True, mask_check=True):
+        super(TransformerEncoder, self).__init__()
+        self.layers = _get_clones(encoder_layer, num_layers)
+        self.num_layers = num_layers
+        self.norm = norm
+        self.enable_nested_tensor = enable_nested_tensor
+        self.mask_check = mask_check
+
+[docs]    def forward(self, src: Tensor, mask: Optional[Tensor] = None, src_key_padding_mask: Optional[Tensor] = None) -> Tensor:
+        r"""Pass the input through the encoder layers in turn.
+
+        Args:
+            src: the sequence to the encoder (required).
+            mask: the mask for the src sequence (optional).
+            src_key_padding_mask: the mask for the src keys per batch (optional).
+
+        Shape:
+            see the docs in Transformer class.
+        """
+        if src_key_padding_mask is not None:
+            _skpm_dtype = src_key_padding_mask.dtype
+            if _skpm_dtype != torch.bool and not torch.is_floating_point(src_key_padding_mask):
+                raise AssertionError(
+                    "only bool and floating types of key_padding_mask are supported")
+        output = src
+        convert_to_nested = False
+        first_layer = self.layers[0]
+        src_key_padding_mask_for_layers = src_key_padding_mask
+        why_not_sparsity_fast_path = ''
+        str_first_layer = "self.layers[0]"
+        if not isinstance(first_layer, torch.nn.TransformerEncoderLayer):
+            why_not_sparsity_fast_path = f"{str_first_layer} was not TransformerEncoderLayer"
+        elif first_layer.norm_first :
+            why_not_sparsity_fast_path = f"{str_first_layer}.norm_first was True"
+        elif first_layer.training:
+            why_not_sparsity_fast_path = f"{str_first_layer} was in training mode"
+        elif not first_layer.self_attn.batch_first:
+            why_not_sparsity_fast_path = f" {str_first_layer}.self_attn.batch_first was not True"
+        elif not first_layer.self_attn._qkv_same_embed_dim:
+            why_not_sparsity_fast_path = f"{str_first_layer}.self_attn._qkv_same_embed_dim was not True"
+        elif not first_layer.activation_relu_or_gelu:
+            why_not_sparsity_fast_path = f" {str_first_layer}.activation_relu_or_gelu was not True"
+        elif not (first_layer.norm1.eps == first_layer.norm2.eps) :
+            why_not_sparsity_fast_path = f"{str_first_layer}.norm1.eps was not equal to {str_first_layer}.norm2.eps"
+        elif not src.dim() == 3:
+            why_not_sparsity_fast_path = f"input not batched; expected src.dim() of 3 but got {src.dim()}"
+        elif not self.enable_nested_tensor:
+            why_not_sparsity_fast_path = "enable_nested_tensor was not True"
+        elif src_key_padding_mask is None:
+            why_not_sparsity_fast_path = "src_key_padding_mask was None"
+        elif (((not hasattr(self, "mask_check")) or self.mask_check)
+                and not torch._nested_tensor_from_mask_left_aligned(src, src_key_padding_mask.logical_not())):
+            why_not_sparsity_fast_path = "mask_check enabled, and src and src_key_padding_mask was not left aligned"
+        elif output.is_nested:
+            why_not_sparsity_fast_path = "NestedTensor input is not supported"
+        elif mask is not None:
+            why_not_sparsity_fast_path = "src_key_padding_mask and mask were both supplied"
+        elif first_layer.self_attn.num_heads % 2 == 1:
+            why_not_sparsity_fast_path = "num_head is odd"
+        elif torch.is_autocast_enabled():
+            why_not_sparsity_fast_path = "autocast is enabled"
+
+        if not why_not_sparsity_fast_path:
+            tensor_args = (
+                src,
+                first_layer.self_attn.in_proj_weight,
+                first_layer.self_attn.in_proj_bias,
+                first_layer.self_attn.out_proj.weight,
+                first_layer.self_attn.out_proj.bias,
+                first_layer.norm1.weight,
+                first_layer.norm1.bias,
+                first_layer.norm2.weight,
+                first_layer.norm2.bias,
+                first_layer.linear1.weight,
+                first_layer.linear1.bias,
+                first_layer.linear2.weight,
+                first_layer.linear2.bias,
+            )
+
+            if torch.overrides.has_torch_function(tensor_args):
+                why_not_sparsity_fast_path = "some Tensor argument has_torch_function"
+            elif not (src.is_cuda or 'cpu' in str(src.device)):
+                why_not_sparsity_fast_path = "src is neither CUDA nor CPU"
+            elif torch.is_grad_enabled() and any(x.requires_grad for x in tensor_args):
+                why_not_sparsity_fast_path = ("grad is enabled and at least one of query or the "
+                                              "input/output projection weights or biases requires_grad")
+
+            if (not why_not_sparsity_fast_path) and (src_key_padding_mask is not None):
+                convert_to_nested = True
+                output = torch._nested_tensor_from_mask(output, src_key_padding_mask.logical_not(), mask_check=False)
+                src_key_padding_mask_for_layers = None
+
+        for mod in self.layers:
+            output = mod(output, src_mask=mask, src_key_padding_mask=src_key_padding_mask_for_layers)
+
+        if convert_to_nested:
+            output = output.to_padded_tensor(0.)
+
+        if self.norm is not None:
+            output = self.norm(output)
+
+        return output
+
+
 class TransformerEncoderLayer(nn.Module):
     """
     Modified from torch.nn.TransformerEncoderLayer.