mirrors/icefall
1
0
Fork 0
mirror of https://github.com/k2-fsa/icefall.git synced 2025-09-07 16:14:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Change printing-prob, initial scales

Browse Source
This commit is contained in:
Daniel Povey 2022-04-25 14:02:43 +08:00
parent bb7cb82b04
commit aea116ea25
1 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
egs/librispeech/ASR/pruned2_knowledge/sampling.py
View File

@ -916,11 +916,11 @@ class KnowledgeBaseLookup(nn.Module):
super(KnowledgeBaseLookup, self).__init__() super(KnowledgeBaseLookup, self).__init__()
self.knowledge_base = knowledge_base # shared! self.knowledge_base = knowledge_base # shared!
self.in_proj = ScaledLinear(embedding_dim, M * N, self.in_proj = ScaledLinear(embedding_dim, M * N,
initial_scale=5.0) initial_scale=1.0)
# initial_scale = 4.0 because the knowlege_base activations are # initial_scale = 4.0 because the knowlege_base activations are
# quite small -- if we use our optimizer they'll have stddev <= 0.1. # quite small -- if we use our optimizer they'll have stddev <= 0.1.
self.out_proj = ScaledLinear(D, embedding_dim, self.out_proj = ScaledLinear(D, embedding_dim,
initial_scale = 10.0) initial_scale = 4.0)
self.M = M self.M = M
self.N = N self.N = N
self.K = K self.K = K
@ -938,7 +938,7 @@ class KnowledgeBaseLookup(nn.Module):
x = self.in_proj(x) # now (*, M*N) x = self.in_proj(x) # now (*, M*N)
x = x.reshape(*x.shape[:-1], self.N, self.M) # now (*, N, M) x = x.reshape(*x.shape[:-1], self.N, self.M) # now (*, N, M)
x = x.log_softmax(dim=-1) # now normalized logprobs, dim= (*, N, M) x = x.log_softmax(dim=-1) # now normalized logprobs, dim= (*, N, M)
if random.random() < 0.01: if random.random() < 0.001:
entropy = (x * x.exp()).sum(dim=-1).mean() entropy = (x * x.exp()).sum(dim=-1).mean()
print("Entropy = ", entropy) print("Entropy = ", entropy)
weights, indexes, = sample_combined(x, self.K, input_is_log=True) weights, indexes, = sample_combined(x, self.K, input_is_log=True)