Make max eig ratio 10

This commit is contained in:
Daniel Povey 2022-07-12 13:59:58 +08:00
parent 278358bb9f
commit cedfb5a377

View File

@ -960,9 +960,9 @@ param_rms_smooth1: Smoothing proportion for parameter matrix, if assumed rank of
new_mean = (eps + (smooth + 1) * mean) # mean of modified rms. new_mean = (eps + (smooth + 1) * mean) # mean of modified rms.
ans = rms / new_mean ans = rms / new_mean
if False: if True:
# Apply max_rms # Apply max_rms
max_rms = 5.0 max_rms = 10.0
ans.clamp_(max=max_rms*2) ans.clamp_(max=max_rms*2)
ans /= _mean(ans, exclude_dims=[0], keepdim=True) ans /= _mean(ans, exclude_dims=[0], keepdim=True)
ans.clamp_(max=max_rms) ans.clamp_(max=max_rms)