mirrors/icefall
1
0
Fork 0
mirror of https://github.com/k2-fsa/icefall.git synced 2025-08-09 01:52:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #168 from huangruizhe/patch-1

Browse Source 
Update make_kn_lm.py
This commit is contained in:
Daniel Povey 2022-01-03 18:38:03 +08:00 committed by GitHub
commit 4314309f1e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 8 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
icefall/shared/make_kn_lm.py
View File

@ -165,7 +165,9 @@ class NgramCounts:
n1 += stat[1] n1 += stat[1]
n2 += stat[2] n2 += stat[2]
assert n1 + 2 * n2 > 0 assert n1 + 2 * n2 > 0
self.d.append(n1 * 1.0 / (n1 + 2 * n2)) self.d.append(max(0.1, n1 * 1.0) / (n1 + 2 * n2)) # We are doing this max(0.001, xxx) to avoid zero discounting constant D due to n1=0,
# which could happen if the number of symbols is small.
# Otherwise, zero discounting constant can cause division by zero in computing BOW.
def cal_f(self): def cal_f(self):
# f(a_z) is a probability distribution of word sequence a_z. # f(a_z) is a probability distribution of word sequence a_z.
@ -241,7 +243,11 @@ class NgramCounts:
for u in a_counts_for_hist.word_to_count.keys(): # Should be careful here: what is Z1 for u in a_counts_for_hist.word_to_count.keys(): # Should be careful here: what is Z1
sum_z1_f_z += _counts_for_hist.word_to_f[u] sum_z1_f_z += _counts_for_hist.word_to_f[u]
if sum_z1_f_z < 1:
# assert sum_z1_f_a_z < 1
counts_for_hist.word_to_bow[w] = (1.0 - sum_z1_f_a_z) / (1.0 - sum_z1_f_z) counts_for_hist.word_to_bow[w] = (1.0 - sum_z1_f_a_z) / (1.0 - sum_z1_f_z)
else:
counts_for_hist.word_to_bow[w] = None
def print_raw_counts(self, info_string): def print_raw_counts(self, info_string):
# these are useful for debug. # these are useful for debug.