mirrors/icefall
1
0
Fork 0
mirror of https://github.com/k2-fsa/icefall.git synced 2025-12-11 06:55:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix to Balancer to treat max-rms and min-rms losses separately, only max-rms loss scaled up

Browse Source
This commit is contained in:
Daniel Povey 2023-01-01 00:38:07 +08:00
parent 907d28ca2a
commit 8db0636f1d
1 changed files with 8 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
egs/librispeech/ASR/pruned_transducer_stateless7/scaling.py
View File

@ -1031,11 +1031,15 @@ class BalancerFunction(torch.autograd.Function):
m_loss = (m - m.clamp(min=min_mean, max=max_mean)).abs() m_loss = (m - m.clamp(min=min_mean, max=max_mean)).abs()
rms_clamped = rms.clamp(min=min_rms, max=max_rms) rms_clamped = rms.clamp(min=min_rms, max=max_rms)
r_loss = (rms_clamped / rms).log().abs() if min_rms <= 0:
r_loss_min = 0.0
else:
r_loss_min = (min_rms / rms).log().relu().sum()
r_loss_max = (rms / max_rms).log().relu().sum()
# put a much larger scale on the RMS loss, so that if both are violated we fix # put a much larger scale on the RMS-max-limit loss, so that if both it and the
# the RMS loss first. # m_loss are violated we fix the RMS loss first.
loss = (m_loss + 100.0 * r_loss).sum() loss = (m_loss + r_loss_min + 100.0 * r_loss_max).sum()
loss.backward() loss.backward()
loss_grad = x.grad loss_grad = x.grad