mirrors/icefall
1
0
Fork 0
mirror of https://github.com/k2-fsa/icefall.git synced 2025-12-11 06:55:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Replace MaxEig with Whiten with limit=5.0, and move it to end of ConformerEncoderLayer

Browse Source
This commit is contained in:
Daniel Povey 2022-10-16 11:36:12 +08:00
parent ef4650bc8e
commit 29d4e8ec6d
1 changed files with 6 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
egs/librispeech/ASR/pruned_transducer_stateless7/conformer.py
View File

@ -297,9 +297,10 @@ class ConformerEncoderLayer(nn.Module):
min_positive=0.45, max_positive=0.55, min_positive=0.45, max_positive=0.55,
max_abs=6.0, max_abs=6.0,
) )
self.max_eig = MaxEig( self.whiten = Whiten(num_groups=1,
d_model, channel_dim=-1, whitening_limit=5.0,
) prob=(0.025, 0.25),
grad_scale=0.01)
def forward( def forward(
@ -352,7 +353,7 @@ class ConformerEncoderLayer(nn.Module):
src = src + self.feed_forward3(src) src = src + self.feed_forward3(src)
src = self.norm_final(self.max_eig(self.balancer(src))) src = self.norm_final(self.balancer(src))
delta = src - src_orig delta = src - src_orig
bypass_scale = self.bypass_scale bypass_scale = self.bypass_scale
@ -360,7 +361,7 @@ class ConformerEncoderLayer(nn.Module):
bypass_scale = bypass_scale.clamp(min=0.1, max=1.0) bypass_scale = bypass_scale.clamp(min=0.1, max=1.0)
src = src_orig + delta * self.bypass_scale src = src_orig + delta * self.bypass_scale
return src return self.whiten(src)
class ConformerEncoder(nn.Module): class ConformerEncoder(nn.Module):