mirrors/icefall
1
0
Fork 0
mirror of https://github.com/k2-fsa/icefall.git synced 2025-12-11 06:55:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Increase eps; make it added not applied as floor.

Browse Source
This commit is contained in:
Daniel Povey 2023-05-18 20:08:19 +08:00
parent 57a023902c
commit 769033c857
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
egs/libriheavy/LM/zipformer1/subformer.py
View File

@ -966,7 +966,7 @@ class LearnedDownsamplingModule(nn.Module):
attn_offset: Tensor,
indexes: Tensor,
weights: Tensor,
eps: float = 2.0e-04) -> Tensor:
eps: float = 1.0e-03) -> Tensor:
"""
Downsamples attn_offset and also modifies it to account for the weights in `weights`.
Args:
@ -989,7 +989,7 @@ class LearnedDownsamplingModule(nn.Module):
attn_offset = attn_offset.gather(dim=2, index=indexes.unsqueeze(1).expand(
batch_size, seq_len_reduced, seq_len_reduced))
# unsqueeze at position 1 so the extra cost relates to the source position.
attn_offset = attn_offset + weights.clamp(min=eps).log().unsqueeze(1)
attn_offset = attn_offset + (weights + eps).log().unsqueeze(1)
if self.weight_scale != 1.0:
attn_offset = attn_offset - math.log(self.weight_scale)