mirrors/icefall
1
0
Fork 0
mirror of https://github.com/k2-fsa/icefall.git synced 2025-12-11 06:55:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add balancer to ConvNeXt

Browse Source
This commit is contained in:
Daniel Povey 2022-12-21 18:41:05 +08:00
parent 11f68afa1f
commit 39e7c613c7
1 changed files with 10 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
egs/librispeech/ASR/pruned_transducer_stateless7/zipformer.py
View File

@ -1712,6 +1712,13 @@ class ConvNeXt(nn.Module):
kernel_size=1, kernel_size=1,
initial_scale=0.01) initial_scale=0.01)
self.out_balancer = ActivationBalancer(
channels, channel_dim=1,
min_positive=0.5, max_positive=0.5,
min_abs=0.25, max_abs=6.0,
)
def forward(self, x: Tensor) -> Tensor: def forward(self, x: Tensor) -> Tensor:
""" """
@ -1732,7 +1739,9 @@ class ConvNeXt(nn.Module):
mask = torch.rand((batch_size, 1, 1, 1), dtype=x.dtype, device=x.device) > layerdrop_rate mask = torch.rand((batch_size, 1, 1, 1), dtype=x.dtype, device=x.device) > layerdrop_rate
x = x * mask x = x * mask
return bypass + x x = bypass + x
x = self.out_balancer(x)
return x