mirrors/icefall
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'scaled_adam_exp147' into scaled_adam_exp149

Browse Source
This commit is contained in:
Daniel Povey 2022-10-20 12:59:50 +08:00
commit d75d646dc4
2 changed files with 1 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
egs/librispeech/ASR/pruned_transducer_stateless7/model.py
View File

@ -19,12 +19,10 @@ import k2
import torch import torch
import torch.nn as nn import torch.nn as nn
from encoder_interface import EncoderInterface from encoder_interface import EncoderInterface
from scaling import random_clamp
from icefall.utils import add_sos from icefall.utils import add_sos
class Transducer(nn.Module): class Transducer(nn.Module):
"""It implements https://arxiv.org/pdf/1211.3711.pdf """It implements https://arxiv.org/pdf/1211.3711.pdf
"Sequence Transduction with Recurrent Neural Networks" "Sequence Transduction with Recurrent Neural Networks"
@ -142,12 +140,6 @@ class Transducer(nn.Module):
lm = self.simple_lm_proj(decoder_out) lm = self.simple_lm_proj(decoder_out)
am = self.simple_am_proj(encoder_out) am = self.simple_am_proj(encoder_out)
if self.training:
lm = random_clamp(lm, min=-8.0, max=2.0, prob=0.5,
reflect=0.1)
am = random_clamp(am, min=-5.0, max=5.0, prob=0.5,
reflect=0.1)
with torch.cuda.amp.autocast(enabled=False): with torch.cuda.amp.autocast(enabled=False):
simple_loss, (px_grad, py_grad) = k2.rnnt_loss_smoothed( simple_loss, (px_grad, py_grad) = k2.rnnt_loss_smoothed(
lm=lm.float(), lm=lm.float(),
@ -183,10 +175,6 @@ class Transducer(nn.Module):
# prior to do_rnnt_pruning (this is an optimization for speed). # prior to do_rnnt_pruning (this is an optimization for speed).
logits = self.joiner(am_pruned, lm_pruned, project_input=False) logits = self.joiner(am_pruned, lm_pruned, project_input=False)
if self.training:
logits = random_clamp(logits, -8.0, 2.0, prob=0.5,
reflect=0.1)
with torch.cuda.amp.autocast(enabled=False): with torch.cuda.amp.autocast(enabled=False):
pruned_loss = k2.rnnt_loss_pruned( pruned_loss = k2.rnnt_loss_pruned(
logits=logits.float(), logits=logits.float(),

3
egs/librispeech/ASR/pruned_transducer_stateless7/scaling.py
View File

@ -175,7 +175,6 @@ class RandomClampFunction(torch.autograd.Function):
ctx.reflect = reflect ctx.reflect = reflect
if reflect != 0.0: if reflect != 0.0:
ans = ans * (1.0 + reflect) - (x * reflect) ans = ans * (1.0 + reflect) - (x * reflect)
return ans return ans
@staticmethod @staticmethod
@ -185,7 +184,7 @@ class RandomClampFunction(torch.autograd.Function):
reflect = ctx.reflect reflect = ctx.reflect
if reflect != 0.0: if reflect != 0.0:
x_grad = x_grad * (1.0 + reflect) - (ans_grad * reflect) x_grad = x_grad * (1.0 + reflect) - (ans_grad * reflect)
return ans_grad * is_same.to(ans_grad.dtype), None, None, None, None return x_grad, None, None, None, None
def random_clamp(x: Tensor, def random_clamp(x: Tensor,
min: Optional[float] = None, min: Optional[float] = None,