mirrors/icefall
1
0
Fork 0
mirror of https://github.com/k2-fsa/icefall.git synced 2025-08-13 03:52:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
yifanyeung 2024-10-30 22:55:33 -07:00
parent 258e106904
commit 512c4831af
1 changed files with 12 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
egs/libriheavy/TTS/local/prepare_manifest.py
View File

@ -1,5 +1,5 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# Copyright 2023 Xiaomi Corp. (authors: Wei Kang) # Copyright 2024 Xiaomi Corp. (authors: Yifan Yang)
# #
# See ../../../../LICENSE for clarification regarding multiple authors # See ../../../../LICENSE for clarification regarding multiple authors
# #
@ -28,9 +28,10 @@ from tqdm import tqdm
from icefall.utils import str2bool from icefall.utils import str2bool
class TextNormlizer: class TextNormalizer:
def __init__(self): def __init__(self):
self.en_tn_model = EnNormalizer(cache_dir="/tmp/tn", overwrite_cache=False) self.en_tn_model = EnNormalizer(cache_dir="/tmp/tn", overwrite_cache=False)
self.table = str.maketrans("’‘，。；？！（）：-《》、“”【】", "'',.;?!(): <>/\"\"[]")
def __call__(self, cut): def __call__(self, cut):
text = cut["supervisions"][0]["custom"]["texts"][0] text = cut["supervisions"][0]["custom"]["texts"][0]
@ -40,8 +41,7 @@ class TextNormlizer:
text = re.sub(r"\([^\)]*\)", " ", text) text = re.sub(r"\([^\)]*\)", " ", text)
# Apply mappings # Apply mappings
table = str.maketrans("’‘，。；？！（）：-《》、“”【】", "'',.;?!(): <>/\"\"[]") text = text.translate(self.table)
text = text.translate(table)
# Remove extra spaces # Remove extra spaces
text = re.sub(r"\s+", " ", text).strip() text = re.sub(r"\s+", " ", text).strip()
@ -61,17 +61,14 @@ def main():
fname = Path(sys.argv[1]).name fname = Path(sys.argv[1]).name
oname = Path(sys.argv[2]) / fname oname = Path(sys.argv[2]) / fname
tn = TextNormlizer() tn = TextNormalizer()
with gzip.open(sys.argv[1], "r") as fin, ProcessPoolExecutor() as ex: with ProcessPoolExecutor() as ex:
futures = [] with gzip.open(sys.argv[1], "r") as fin:
cuts = [] cuts = (json.loads(line) for line in fin)
for line in tqdm(fin, desc="Distributing tasks"): results = ex.map(tn, cuts)
cut = json.loads(line)
futures.append(ex.submit(tn, cut))
with gzip.open(oname, "w") as fout: with gzip.open(oname, "w") as fout:
for future in tqdm(futures, desc="Processing"): for cut in tqdm(results, desc="Processing"):
cut = future.result()
fout.write((json.dumps(cut) + "\n").encode()) fout.write((json.dumps(cut) + "\n").encode())