diff --git a/egs/librispeech/ASR/pruned_transducer_stateless7/optim.py b/egs/librispeech/ASR/pruned_transducer_stateless7/optim.py
index 8cda942a7..1f6b33b70 100644
--- a/egs/librispeech/ASR/pruned_transducer_stateless7/optim.py
+++ b/egs/librispeech/ASR/pruned_transducer_stateless7/optim.py
@@ -604,8 +604,15 @@ param_rms_smooth1: Smoothing proportion for parameter matrix, if assumed rank of
         mean = rms.mean()
         rms += eps + smooth * mean
         new_mean = (eps + (smooth + 1) * mean)
-        return rms / new_mean
+        ans = rms / new_mean
 
+        # Apply max_rms
+        max_rms = 5.0
+        ans.clamp_(max=max_eig*2)
+        ans /= ans.mean()
+        ans.clamp_(max=max_eig)
+        ans /= ans.mean()
+        return ans