From cbe012d54c7007bfbbb82e71a9f1184500fb0824 Mon Sep 17 00:00:00 2001 From: Yuekai Zhang Date: Fri, 22 Nov 2024 11:18:01 +0800 Subject: [PATCH 01/59] Valle Recipe for WenetSpeech4TTS, LibriTTS, LibriTTS-R (#1805) * add valle * update readme --- egs/libritts/TTS/README.md | 65 +- ...te_neural_codec_and_prepare_text_tokens.py | 1 + egs/libritts/TTS/prepare.sh | 43 +- egs/libritts/TTS/valle | 1 + egs/wenetspeech4tts/TTS/README.md | 72 + ...te_neural_codec_and_prepare_text_tokens.py | 609 ++++++ .../TTS/local/display_manifest_statistics.py | 53 + egs/wenetspeech4tts/TTS/prepare.sh | 100 + egs/wenetspeech4tts/TTS/shared | 1 + ...te_neural_codec_and_prepare_text_tokens.py | 1 + egs/wenetspeech4tts/TTS/valle/infer.py | 300 +++ egs/wenetspeech4tts/TTS/valle/optim.py | 1 + egs/wenetspeech4tts/TTS/valle/tokenizer.py | 111 ++ egs/wenetspeech4tts/TTS/valle/train.py | 1244 ++++++++++++ .../TTS/valle/tts_datamodule.py | 343 ++++ egs/wenetspeech4tts/TTS/valle/valle.py | 1745 +++++++++++++++++ 16 files changed, 4675 insertions(+), 15 deletions(-) create mode 120000 egs/libritts/TTS/local/compute_neural_codec_and_prepare_text_tokens.py create mode 120000 egs/libritts/TTS/valle create mode 100644 egs/wenetspeech4tts/TTS/README.md create mode 100755 egs/wenetspeech4tts/TTS/local/compute_neural_codec_and_prepare_text_tokens.py create mode 100755 egs/wenetspeech4tts/TTS/local/display_manifest_statistics.py create mode 100755 egs/wenetspeech4tts/TTS/prepare.sh create mode 120000 egs/wenetspeech4tts/TTS/shared create mode 120000 egs/wenetspeech4tts/TTS/valle/compute_neural_codec_and_prepare_text_tokens.py create mode 100644 egs/wenetspeech4tts/TTS/valle/infer.py create mode 120000 egs/wenetspeech4tts/TTS/valle/optim.py create mode 100644 egs/wenetspeech4tts/TTS/valle/tokenizer.py create mode 100755 egs/wenetspeech4tts/TTS/valle/train.py create mode 100644 egs/wenetspeech4tts/TTS/valle/tts_datamodule.py create mode 100644 egs/wenetspeech4tts/TTS/valle/valle.py diff --git a/egs/libritts/TTS/README.md b/egs/libritts/TTS/README.md index 4d4fb8580..67424a1ca 100644 --- a/egs/libritts/TTS/README.md +++ b/egs/libritts/TTS/README.md @@ -1,7 +1,7 @@ # Introduction -LibriTTS is a multi-speaker English corpus of approximately 585 hours of read English speech at 24kHz sampling rate, prepared by Heiga Zen with the assistance of Google Speech and Google Brain team members. -The LibriTTS corpus is designed for TTS research. It is derived from the original materials (mp3 audio files from LibriVox and text files from Project Gutenberg) of the LibriSpeech corpus. +LibriTTS is a multi-speaker English corpus of approximately 585 hours of read English speech at 24kHz sampling rate, prepared by Heiga Zen with the assistance of Google Speech and Google Brain team members. +The LibriTTS corpus is designed for TTS research. It is derived from the original materials (mp3 audio files from LibriVox and text files from Project Gutenberg) of the LibriSpeech corpus. The main differences from the LibriSpeech corpus are listed below: 1. The audio files are at 24kHz sampling rate. 2. The speech is split at sentence breaks. @@ -11,16 +11,16 @@ The main differences from the LibriSpeech corpus are listed below: For more information, refer to the paper "LibriTTS: A Corpus Derived from LibriSpeech for Text-to-Speech", Heiga Zen, Viet Dang, Rob Clark, Yu Zhang, Ron J. Weiss, Ye Jia, Zhifeng Chen, and Yonghui Wu, arXiv, 2019. If you use the LibriTTS corpus in your work, please cite this paper where it was introduced. > [!CAUTION] -> The next-gen Kaldi framework provides tools and models for generating high-quality, synthetic speech (Text-to-Speech, TTS). +> The next-gen Kaldi framework provides tools and models for generating high-quality, synthetic speech (Text-to-Speech, TTS). > While these recipes has the potential to advance various fields such as accessibility, language education, and AI-driven solutions, it also carries certain ethical and legal responsibilities. -> +> > By using this framework, you agree to the following: > 1. Legal and Ethical Use: You shall not use this framework, or any models derived from it, for any unlawful or unethical purposes. This includes, but is not limited to: Creating voice clones without the explicit, informed consent of the individual whose voice is being cloned. Engaging in any form of identity theft, impersonation, or fraud using cloned voices. Violating any local, national, or international laws regarding privacy, intellectual property, or personal data. -> +> > 2. Responsibility of Use: The users of this framework are solely responsible for ensuring that their use of voice cloning technologies complies with all applicable laws and ethical guidelines. We explicitly disclaim any liability for misuse of the technology. -> +> > 3. Attribution and Use of Open-Source Components: This project is provided under the Apache 2.0 license. Users must adhere to the terms of this license and provide appropriate attribution when required. -> +> > 4. No Warranty: This framework is provided “as-is,” without warranty of any kind, either express or implied. We do not guarantee that the use of this software will comply with legal requirements or that it will not infringe the rights of third parties. @@ -49,3 +49,54 @@ To inference, use: --epoch 400 \ --tokens data/tokens.txt ``` + +# [VALL-E](https://arxiv.org/abs/2301.02111) + +./valle contains the code for training VALL-E TTS model. + +Checkpoints and training logs can be found [here](https://huggingface.co/yuekai/vall-e_libritts). The demo of the model trained with libritts and [libritts-r](https://www.openslr.org/141/) is available [here](https://huggingface.co/spaces/yuekai/valle-libritts-demo). + +Preparation: + +``` +bash prepare.sh --start-stage 4 +``` + +The training command is given below: + +``` +world_size=8 +exp_dir=exp/valle + +## Train AR model +python3 valle/train.py --max-duration 320 --filter-min-duration 0.5 --filter-max-duration 14 --train-stage 1 \ + --num-buckets 6 --dtype "bfloat16" --save-every-n 1000 --valid-interval 2000 \ + --share-embedding true --norm-first true --add-prenet false \ + --decoder-dim 1024 --nhead 16 --num-decoder-layers 12 --prefix-mode 1 \ + --base-lr 0.03 --warmup-steps 200 --average-period 0 \ + --num-epochs 20 --start-epoch 1 --start-batch 0 --accumulate-grad-steps 1 \ + --exp-dir ${exp_dir} --world-size ${world_size} + +## Train NAR model +# cd ${exp_dir} +# ln -s ${exp_dir}/best-valid-loss.pt epoch-99.pt # --start-epoch 100=99+1 +# cd - +python3 valle/train.py --max-duration 160 --filter-min-duration 0.5 --filter-max-duration 14 --train-stage 2 \ + --num-buckets 6 --dtype "float32" --save-every-n 1000 --valid-interval 2000 \ + --share-embedding true --norm-first true --add-prenet false \ + --decoder-dim 1024 --nhead 16 --num-decoder-layers 12 --prefix-mode 1 \ + --base-lr 0.03 --warmup-steps 200 --average-period 0 \ + --num-epochs 40 --start-epoch 100 --start-batch 0 --accumulate-grad-steps 2 \ + --exp-dir ${exp_dir} --world-size ${world_size} +``` + +To inference, use: +``` +huggingface-cli login +huggingface-cli download --local-dir ${exp_dir} yuekai/vall-e_libritts +top_p=1.0 +python3 valle/infer.py --output-dir demos_epoch_${epoch}_avg_${avg}_top_p_${top_p} \ + --top-k -1 --temperature 1.0 \ + --text ./libritts.txt \ + --checkpoint ${exp_dir}/epoch-${epoch}-avg-${avg}.pt --top-p ${top_p} +``` diff --git a/egs/libritts/TTS/local/compute_neural_codec_and_prepare_text_tokens.py b/egs/libritts/TTS/local/compute_neural_codec_and_prepare_text_tokens.py new file mode 120000 index 000000000..68579ffd4 --- /dev/null +++ b/egs/libritts/TTS/local/compute_neural_codec_and_prepare_text_tokens.py @@ -0,0 +1 @@ +../../../wenetspeech4tts/TTS/local/compute_neural_codec_and_prepare_text_tokens.py \ No newline at end of file diff --git a/egs/libritts/TTS/prepare.sh b/egs/libritts/TTS/prepare.sh index 44016e6d2..1700e0737 100755 --- a/egs/libritts/TTS/prepare.sh +++ b/egs/libritts/TTS/prepare.sh @@ -32,7 +32,7 @@ if [ $stage -le -1 ] && [ $stop_stage -ge -1 ]; then cd vits/monotonic_align python setup.py build_ext --inplace cd ../../ - else + else log "monotonic_align lib already built" fi fi @@ -75,11 +75,11 @@ if [ $stage -le 2 ] && [ $stop_stage -ge 2 ]; then log "Stage 2: Compute Spectrogram for LibriTTS" mkdir -p data/spectrogram if [ ! -e data/spectrogram/.libritts.done ]; then - ./local/compute_spectrogram_libritts.py --sampling-rate $sampling_rate + ./local/compute_spectrogram_libritts.py --sampling-rate $sampling_rate touch data/spectrogram/.libritts.done fi - # Here we shuffle and combine the train-clean-100, train-clean-360 and + # Here we shuffle and combine the train-clean-100, train-clean-360 and # train-other-500 together to form the training set. if [ ! -f data/spectrogram/libritts_cuts_train-all-shuf.jsonl.gz ]; then cat <(gunzip -c data/spectrogram/libritts_cuts_train-clean-100.jsonl.gz) \ @@ -88,7 +88,7 @@ if [ $stage -le 2 ] && [ $stop_stage -ge 2 ]; then shuf | gzip -c > data/spectrogram/libritts_cuts_train-all-shuf.jsonl.gz fi - # Here we shuffle and combine the train-clean-100, train-clean-360 + # Here we shuffle and combine the train-clean-100, train-clean-360 # together to form the training set. if [ ! -f data/spectrogram/libritts_cuts_train-clean-460.jsonl.gz ]; then cat <(gunzip -c data/spectrogram/libritts_cuts_train-clean-100.jsonl.gz) \ @@ -108,10 +108,10 @@ if [ $stage -le 3 ] && [ $stop_stage -ge 3 ]; then log "Stage 3: Prepare phoneme tokens for LibriTTS" # We assume you have installed piper_phonemize and espnet_tts_frontend. # If not, please install them with: - # - piper_phonemize: + # - piper_phonemize: # refer to https://github.com/rhasspy/piper-phonemize, # could install the pre-built wheels from https://github.com/csukuangfj/piper-phonemize/releases/tag/2023.12.5 - # - espnet_tts_frontend: + # - espnet_tts_frontend: # `pip install espnet_tts_frontend`, refer to https://github.com/espnet/espnet_tts_frontend/ if [ ! -e data/spectrogram/.libritts_with_token.done ]; then ./local/prepare_tokens_libritts.py @@ -123,12 +123,39 @@ if [ $stage -le 4 ] && [ $stop_stage -ge 4 ]; then log "Stage 4: Generate token file" # We assume you have installed piper_phonemize and espnet_tts_frontend. # If not, please install them with: - # - piper_phonemize: + # - piper_phonemize: # refer to https://github.com/rhasspy/piper-phonemize, # could install the pre-built wheels from https://github.com/csukuangfj/piper-phonemize/releases/tag/2023.12.5 - # - espnet_tts_frontend: + # - espnet_tts_frontend: # `pip install espnet_tts_frontend`, refer to https://github.com/espnet/espnet_tts_frontend/ if [ ! -e data/tokens.txt ]; then ./local/prepare_token_file.py --tokens data/tokens.txt fi fi + +audio_feats_dir=data/tokenized +dataset_parts="--dataset-parts all" # debug "-p dev-clean -p test-clean" +if [ $stage -le 5 ] && [ $stop_stage -ge 5 ]; then + log "Stage 5: Tokenize/Fbank LibriTTS for valle" + mkdir -p ${audio_feats_dir} + if [ ! -e ${audio_feats_dir}/.libritts.tokenize.done ]; then + python3 ./local/compute_neural_codec_and_prepare_text_tokens.py --dataset-parts "${dataset_parts}" \ + --audio-extractor "Encodec" \ + --batch-duration 400 \ + --src-dir "data/manifests" \ + --output-dir "${audio_feats_dir}" + fi + touch ${audio_feats_dir}/.libritts.tokenize.done + + lhotse combine \ + ${audio_feats_dir}/libritts_cuts_train-clean-100.jsonl.gz \ + ${audio_feats_dir}/libritts_cuts_train-clean-360.jsonl.gz \ + ${audio_feats_dir}/libritts_cuts_train-other-500.jsonl.gz \ + ${audio_feats_dir}/cuts_train.jsonl.gz + lhotse copy \ + ${audio_feats_dir}/libritts_cuts_dev-clean.jsonl.gz \ + ${audio_feats_dir}/cuts_dev.jsonl.gz + lhotse copy \ + ${audio_feats_dir}/libritts_cuts_test-clean.jsonl.gz \ + ${audio_feats_dir}/cuts_test.jsonl.gz +fi diff --git a/egs/libritts/TTS/valle b/egs/libritts/TTS/valle new file mode 120000 index 000000000..c8fe8fdb0 --- /dev/null +++ b/egs/libritts/TTS/valle @@ -0,0 +1 @@ +../../wenetspeech4tts/TTS/valle/ \ No newline at end of file diff --git a/egs/wenetspeech4tts/TTS/README.md b/egs/wenetspeech4tts/TTS/README.md new file mode 100644 index 000000000..f35bb51c7 --- /dev/null +++ b/egs/wenetspeech4tts/TTS/README.md @@ -0,0 +1,72 @@ +# Introduction + +[**WenetSpeech4TTS**](https://huggingface.co/datasets/Wenetspeech4TTS/WenetSpeech4TTS) is a multi-domain **Mandarin** corpus derived from the open-sourced [WenetSpeech](https://arxiv.org/abs/2110.03370) dataset. + +> [!CAUTION] +> The next-gen Kaldi framework provides tools and models for generating high-quality, synthetic speech (Text-to-Speech, TTS). +> While these recipes has the potential to advance various fields such as accessibility, language education, and AI-driven solutions, it also carries certain ethical and legal responsibilities. +> +> By using this framework, you agree to the following: +> 1. Legal and Ethical Use: You shall not use this framework, or any models derived from it, for any unlawful or unethical purposes. This includes, but is not limited to: Creating voice clones without the explicit, informed consent of the individual whose voice is being cloned. Engaging in any form of identity theft, impersonation, or fraud using cloned voices. Violating any local, national, or international laws regarding privacy, intellectual property, or personal data. +> +> 2. Responsibility of Use: The users of this framework are solely responsible for ensuring that their use of voice cloning technologies complies with all applicable laws and ethical guidelines. We explicitly disclaim any liability for misuse of the technology. +> +> 3. Attribution and Use of Open-Source Components: This project is provided under the Apache 2.0 license. Users must adhere to the terms of this license and provide appropriate attribution when required. +> +> 4. No Warranty: This framework is provided “as-is,” without warranty of any kind, either express or implied. We do not guarantee that the use of this software will comply with legal requirements or that it will not infringe the rights of third parties. + + +# [VALL-E](https://arxiv.org/abs/2301.02111) + +./valle contains the code for training VALL-E TTS model. + +Checkpoints and training logs can be found [here](https://huggingface.co/yuekai/vall-e_wenetspeech4tts). The demo of the model trained with Wenetspeech4TTS Premium (945 hours) is available [here](https://huggingface.co/spaces/yuekai/valle_wenetspeech4tts_demo). + +Preparation: + +``` +bash prepare.sh +``` + +The training command is given below: + +``` +world_size=8 +exp_dir=exp/valle + +## Train AR model +python3 valle/train.py --max-duration 320 --filter-min-duration 0.5 --filter-max-duration 14 --train-stage 1 \ + --num-buckets 6 --dtype "bfloat16" --save-every-n 1000 --valid-interval 2000 \ + --share-embedding true --norm-first true --add-prenet false \ + --decoder-dim 1024 --nhead 16 --num-decoder-layers 12 --prefix-mode 1 \ + --base-lr 0.03 --warmup-steps 200 --average-period 0 \ + --num-epochs 20 --start-epoch 1 --start-batch 0 --accumulate-grad-steps 1 \ + --exp-dir ${exp_dir} --world-size ${world_size} + +## Train NAR model +# cd ${exp_dir} +# ln -s ${exp_dir}/best-valid-loss.pt epoch-99.pt # --start-epoch 100=99+1 +# cd - +python3 valle/train.py --max-duration 160 --filter-min-duration 0.5 --filter-max-duration 14 --train-stage 2 \ + --num-buckets 6 --dtype "float32" --save-every-n 1000 --valid-interval 2000 \ + --share-embedding true --norm-first true --add-prenet false \ + --decoder-dim 1024 --nhead 16 --num-decoder-layers 12 --prefix-mode 1 \ + --base-lr 0.03 --warmup-steps 200 --average-period 0 \ + --num-epochs 40 --start-epoch 100 --start-batch 0 --accumulate-grad-steps 2 \ + --exp-dir ${exp_dir} --world-size ${world_size} +``` + +To inference, use: +``` +huggingface-cli login +huggingface-cli download --local-dir ${exp_dir} yuekai/vall-e_wenetspeech4tts +top_p=1.0 +python3 valle/infer.py --output-dir demos_epoch_${epoch}_avg_${avg}_top_p_${top_p} \ + --top-k -1 --temperature 1.0 \ + --text ./aishell3.txt \ + --checkpoint ${exp_dir}/epoch-${epoch}-avg-${avg}.pt \ + --text-extractor pypinyin_initials_finals --top-p ${top_p} +``` + +# Credits +- [vall-e](https://github.com/lifeiteng/vall-e) diff --git a/egs/wenetspeech4tts/TTS/local/compute_neural_codec_and_prepare_text_tokens.py b/egs/wenetspeech4tts/TTS/local/compute_neural_codec_and_prepare_text_tokens.py new file mode 100755 index 000000000..5494bf340 --- /dev/null +++ b/egs/wenetspeech4tts/TTS/local/compute_neural_codec_and_prepare_text_tokens.py @@ -0,0 +1,609 @@ +#!/usr/bin/env python3 +# Copyright 2023 (authors: Feiteng Li) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Phonemize Text and EnCodec Audio. + +Usage example: + python3 ./local/compute_neural_codec_and_prepare_text_tokens.py --dataset-parts "${dataset_parts}" \ + --text-extractor ${text_extractor} \ + --audio-extractor ${audio_extractor} \ + --batch-duration 2500 --prefix "wenetspeech4tts" \ + --src-dir "data/manifests" --split 100 \ + --output-dir "${audio_feats_dir}/wenetspeech4tts_${dataset_parts}_split_100" + +""" +import argparse +import logging +import os +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any, Dict, List, Optional, Union + +import torch +import torch.multiprocessing +from encodec import EncodecModel +from encodec.utils import convert_audio +from lhotse import CutSet, NumpyHdf5Writer +from lhotse.features import FeatureExtractor +from lhotse.recipes.utils import read_manifests_if_cached +from lhotse.utils import Seconds, compute_num_frames +from phonemizer.backend import EspeakBackend +from phonemizer.backend.espeak.language_switch import LanguageSwitch +from phonemizer.backend.espeak.words_mismatch import WordMismatch +from phonemizer.punctuation import Punctuation +from phonemizer.separator import Separator +from tqdm.auto import tqdm + +from icefall.utils import get_executor + +try: + from pypinyin import Style, pinyin + from pypinyin.style._utils import get_finals, get_initials +except Exception: + pass + + +import re +from typing import Pattern + +import numpy as np +from k2 import SymbolTable + +# from valle.data import ( +# AudioTokenConfig, +# AudioTokenExtractor, +# TextTokenizer, +# tokenize_text, +# ) +# from valle.data.fbank import get_fbank_extractor +# from valle.utils import SymbolTable + +os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python" + + +# Torch's multithreaded behavior needs to be disabled or +# it wastes a lot of CPU and slow things down. +# Do this outside of main() in case it needs to take effect +# even when we are not invoking the main (e.g. when spawning subprocesses). +torch.set_num_threads(1) +torch.set_num_interop_threads(1) +torch.multiprocessing.set_sharing_strategy("file_system") + + +def get_args(): + parser = argparse.ArgumentParser() + + parser.add_argument( + "--src-dir", + type=Path, + default=Path("data/manifests"), + help="Path to the manifest files", + ) + parser.add_argument( + "--output-dir", + type=Path, + default=Path("data/tokenized"), + help="Path to the tokenized files", + ) + parser.add_argument( + "--text-extractor", + type=str, + default="espeak", + help="espeak or pypinyin or pypinyin_initials_finals", + ) + parser.add_argument( + "--audio-extractor", + type=str, + default="Encodec", + help="Encodec or Fbank", + ) + parser.add_argument( + "--dataset-parts", + type=str, + default="dev-clean test-clean", + help="Space separated dataset parts", + ) + parser.add_argument( + "--prefix", + type=str, + default="libritts", + help="prefix of the manifest file", + ) + parser.add_argument( + "--suffix", + type=str, + default="jsonl.gz", + help="suffix of the manifest file", + ) + parser.add_argument( + "--batch-duration", + type=float, + default=400.0, + help="The maximum number of audio seconds in a batch." + "Determines batch size dynamically.", + ) + parser.add_argument( + "--split", + type=int, + default=1, + help="Split the cut_set into multiple parts", + ) + + return parser.parse_args() + + +class PypinyinBackend: + """PypinyinBackend for Chinese. Most codes is referenced from espnet. + There are two types pinyin or initials_finals, one is + just like "ni1 hao3", the other is like "n i1 h ao3". + """ + + def __init__( + self, + backend="initials_finals", + punctuation_marks: Union[str, Pattern] = Punctuation.default_marks(), + ) -> None: + self.backend = backend + self.punctuation_marks = punctuation_marks + + def phonemize( + self, text: List[str], separator: Separator, strip=True, njobs=1 + ) -> List[str]: + assert isinstance(text, List) + phonemized = [] + for _text in text: + _text = re.sub(" +", " ", _text.strip()) + _text = _text.replace(" ", separator.word) + phones = [] + if self.backend == "pypinyin": + for n, py in enumerate( + pinyin(_text, style=Style.TONE3, neutral_tone_with_five=True) + ): + if all([c in self.punctuation_marks for c in py[0]]): + if len(phones): + assert phones[-1] == separator.syllable + phones.pop(-1) + + phones.extend(list(py[0])) + else: + phones.extend([py[0], separator.syllable]) + elif self.backend == "pypinyin_initials_finals": + for n, py in enumerate( + pinyin(_text, style=Style.TONE3, neutral_tone_with_five=True) + ): + if all([c in self.punctuation_marks for c in py[0]]): + if len(phones): + assert phones[-1] == separator.syllable + phones.pop(-1) + phones.extend(list(py[0])) + else: + if py[0][-1].isalnum(): + initial = get_initials(py[0], strict=False) + if py[0][-1].isdigit(): + final = get_finals(py[0][:-1], strict=False) + py[0][-1] + else: + final = get_finals(py[0], strict=False) + phones.extend( + [ + initial, + separator.phone, + final, + separator.syllable, + ] + ) + else: + assert ValueError + else: + raise NotImplementedError + phonemized.append( + "".join(phones).rstrip(f"{separator.word}{separator.syllable}") + ) + return phonemized + + +class TextTokenizer: + """Phonemize Text.""" + + def __init__( + self, + language="en-us", + backend="espeak", + separator=Separator(word="_", syllable="-", phone="|"), + preserve_punctuation=True, + punctuation_marks: Union[str, Pattern] = Punctuation.default_marks(), + with_stress: bool = False, + tie: Union[bool, str] = False, + language_switch: LanguageSwitch = "keep-flags", + words_mismatch: WordMismatch = "ignore", + ) -> None: + if backend == "espeak": + phonemizer = EspeakBackend( + language, + punctuation_marks=punctuation_marks, + preserve_punctuation=preserve_punctuation, + with_stress=with_stress, + tie=tie, + language_switch=language_switch, + words_mismatch=words_mismatch, + ) + elif backend in ["pypinyin", "pypinyin_initials_finals"]: + phonemizer = PypinyinBackend( + backend=backend, + punctuation_marks=punctuation_marks + separator.word, + ) + else: + raise NotImplementedError(f"{backend}") + + self.backend = phonemizer + self.separator = separator + + def to_list(self, phonemized: str) -> List[str]: + fields = [] + for word in phonemized.split(self.separator.word): + # "ɐ m|iː|n?" ɹ|ɪ|z|ɜː|v; h|ɪ|z. + pp = re.findall(r"\w+|[^\w\s]", word, re.UNICODE) + fields.extend( + [p for p in pp if p != self.separator.phone] + [self.separator.word] + ) + assert len("".join(fields[:-1])) == len(phonemized) - phonemized.count( + self.separator.phone + ) + return fields[:-1] + + def __call__(self, text, strip=True) -> List[List[str]]: + if isinstance(text, str): + text = [text] + + phonemized = self.backend.phonemize( + text, separator=self.separator, strip=strip, njobs=1 + ) + return [self.to_list(p) for p in phonemized] + + +def tokenize_text(tokenizer: TextTokenizer, text: str) -> List[str]: + phonemes = tokenizer([text.strip()]) + return phonemes[0] # k2symbols + + +def remove_encodec_weight_norm(model): + from encodec.modules import SConv1d + from encodec.modules.seanet import SConvTranspose1d, SEANetResnetBlock + from torch.nn.utils import remove_weight_norm + + encoder = model.encoder.model + for key in encoder._modules: + if isinstance(encoder._modules[key], SEANetResnetBlock): + remove_weight_norm(encoder._modules[key].shortcut.conv.conv) + block_modules = encoder._modules[key].block._modules + for skey in block_modules: + if isinstance(block_modules[skey], SConv1d): + remove_weight_norm(block_modules[skey].conv.conv) + elif isinstance(encoder._modules[key], SConv1d): + remove_weight_norm(encoder._modules[key].conv.conv) + + decoder = model.decoder.model + for key in decoder._modules: + if isinstance(decoder._modules[key], SEANetResnetBlock): + remove_weight_norm(decoder._modules[key].shortcut.conv.conv) + block_modules = decoder._modules[key].block._modules + for skey in block_modules: + if isinstance(block_modules[skey], SConv1d): + remove_weight_norm(block_modules[skey].conv.conv) + elif isinstance(decoder._modules[key], SConvTranspose1d): + remove_weight_norm(decoder._modules[key].convtr.convtr) + elif isinstance(decoder._modules[key], SConv1d): + remove_weight_norm(decoder._modules[key].conv.conv) + + +class AudioTokenizer: + """EnCodec audio.""" + + def __init__( + self, + device: Any = None, + ) -> None: + # Instantiate a pretrained EnCodec model + model = EncodecModel.encodec_model_24khz() + model.set_target_bandwidth(6.0) + remove_encodec_weight_norm(model) + + if not device: + device = torch.device("cpu") + if torch.cuda.is_available(): + device = torch.device("cuda:0") + + self._device = device + + self.codec = model.to(device) + self.sample_rate = model.sample_rate + self.channels = model.channels + + @property + def device(self): + return self._device + + def encode(self, wav: torch.Tensor) -> torch.Tensor: + return self.codec.encode(wav.to(self.device)) + + def decode(self, frames: torch.Tensor) -> torch.Tensor: + return self.codec.decode(frames) + + +@dataclass +class AudioTokenConfig: + frame_shift: Seconds = 320.0 / 24000 + num_quantizers: int = 8 + + def to_dict(self) -> Dict[str, Any]: + return asdict(self) + + @staticmethod + def from_dict(data: Dict[str, Any]) -> "AudioTokenConfig": + return AudioTokenConfig(**data) + + +class AudioTokenExtractor(FeatureExtractor): + name = "encodec" + config_type = AudioTokenConfig + + def __init__(self, config: Optional[Any] = None): + super(AudioTokenExtractor, self).__init__(config) + self.tokenizer = AudioTokenizer() + + def extract( + self, samples: Union[np.ndarray, torch.Tensor], sampling_rate: int + ) -> np.ndarray: + if not isinstance(samples, torch.Tensor): + samples = torch.from_numpy(samples) + if sampling_rate != self.tokenizer.sample_rate: + samples = convert_audio( + samples, + sampling_rate, + self.tokenizer.sample_rate, + self.tokenizer.channels, + ) + if len(samples.shape) == 2: + samples = samples.unsqueeze(0) + else: + raise ValueError() + + device = self.tokenizer.device + encoded_frames = self.tokenizer.encode(samples.detach().to(device)) + codes = encoded_frames[0][0] # [B, n_q, T] + if True: + duration = round(samples.shape[-1] / sampling_rate, ndigits=12) + expected_num_frames = compute_num_frames( + duration=duration, + frame_shift=self.frame_shift, + sampling_rate=sampling_rate, + ) + assert abs(codes.shape[-1] - expected_num_frames) <= 1 + codes = codes[..., :expected_num_frames] + return codes.cpu().squeeze(0).permute(1, 0).numpy() + + @property + def frame_shift(self) -> Seconds: + return self.config.frame_shift + + def feature_dim(self, sampling_rate: int) -> int: + return self.config.num_quantizers + + def pad_tensor_list(self, tensor_list, device, padding_value=0): + lengths = [tensor.shape[0] for tensor in tensor_list] + tensor_list = [torch.Tensor(t).to(device) for t in tensor_list] + padded_tensor = torch.nn.utils.rnn.pad_sequence( + tensor_list, batch_first=True, padding_value=padding_value + ) + return padded_tensor, lengths + + def extract_batch(self, samples, sampling_rate, lengths) -> np.ndarray: + samples = [wav.squeeze() for wav in samples] + device = self.tokenizer.device + samples, lengths = self.pad_tensor_list(samples, device) + samples = samples.unsqueeze(1) + + if not isinstance(samples, torch.Tensor): + samples = torch.from_numpy(samples) + if len(samples.shape) != 3: + raise ValueError() + if sampling_rate != self.tokenizer.sample_rate: + samples = [ + convert_audio( + wav, + sampling_rate, + self.tokenizer.sample_rate, + self.tokenizer.channels, + ) + for wav in samples + ] + samples = torch.stack(samples, 0) # convert samples from list to tensor + # Extract discrete codes from EnCodec + with torch.no_grad(): + encoded_frames = self.tokenizer.encode(samples.detach().to(device)) + encoded_frames = encoded_frames[0][0] # [B, n_q, T] + batch_codes = [] + for b, length in enumerate(lengths): + codes = encoded_frames[b] + duration = round(length / sampling_rate, ndigits=12) + expected_num_frames = compute_num_frames( + duration=duration, + frame_shift=self.frame_shift, + sampling_rate=sampling_rate, + ) + batch_codes.append(codes[..., :expected_num_frames]) + return [codes.cpu().permute(1, 0).numpy() for codes in batch_codes] + + +def main(): + args = get_args() + + dataset_parts = args.dataset_parts.replace("--dataset-parts", "").strip() + if dataset_parts == "all": # LibriTTS + dataset_parts = [ + "dev-clean", + "dev-other", + "test-clean", + "test-other", + "train-clean-100", + "train-clean-360", + "train-other-500", + ] + else: + dataset_parts = dataset_parts.replace("-p", "").strip().split(" ") + + assert len(dataset_parts) >= 1 + + manifests = read_manifests_if_cached( + dataset_parts=dataset_parts, + output_dir=args.src_dir, + prefix=args.prefix, + suffix=args.suffix, + types=["recordings", "supervisions", "cuts"], + ) + + text_tokenizer = None + if args.text_extractor: + text_tokenizer = TextTokenizer(backend=args.text_extractor) + + audio_extractor = None + if args.audio_extractor: + if args.audio_extractor == "Encodec": + audio_extractor = AudioTokenExtractor(AudioTokenConfig()) + else: + raise NotImplementedError(f"{args.audio_extractor}") + + Path(args.output_dir).mkdir(parents=True, exist_ok=True) + unique_symbols = set() + num_jobs = min(32, os.cpu_count()) + logging.info(f"dataset_parts: {dataset_parts} manifests {len(manifests)}") + + prefix = args.prefix + if prefix and not prefix.endswith("_"): + prefix = f"{prefix}_" + with get_executor() as ex: + for partition, m in manifests.items(): + logging.info( + f"Processing partition: {partition} CUDA: {torch.cuda.is_available()}" + ) + try: + cut_set = CutSet.from_manifests( + recordings=m["recordings"], + supervisions=m["supervisions"], + ) + except Exception: + cut_set = m["cuts"] + + # Split cut_set if split > 1 + split = 1 + if args.split > 1: + cut_sets = cut_set.split(args.split) + split = args.split + else: + cut_sets = [cut_set] + + for idx, part in enumerate(cut_sets): + if args.audio_extractor: + if args.audio_extractor == "Encodec": + storage_path = f"{args.output_dir}/{args.prefix}_encodec_{partition}_{idx if split > 1 else ''}" + else: + storage_path = f"{args.output_dir}/{args.prefix}_fbank_{partition}_{idx if split > 1 else ''}" + + if args.prefix.lower() in [ + "ljspeech", + "aishell", + "baker", + "wenetspeech4tts", + ]: + part = part.resample(24000) + assert args.prefix.lower() in [ + "ljspeech", + "aishell", + "baker", + "wenetspeech4tts", + "libritts", + "libritts-r", + ] + with torch.no_grad(): + if ( + torch.cuda.is_available() + and args.audio_extractor == "Encodec" + ): + part = part.compute_and_store_features_batch( + extractor=audio_extractor, + storage_path=storage_path, + num_workers=num_jobs, + batch_duration=args.batch_duration, + collate=False, + overwrite=True, + storage_type=NumpyHdf5Writer, + ) + else: + part = part.compute_and_store_features( + extractor=audio_extractor, + storage_path=storage_path, + num_jobs=num_jobs if ex is None else 64, + executor=ex, + storage_type=NumpyHdf5Writer, + ) + + # TextTokenizer + if args.text_extractor: + for c in tqdm(part): + if args.prefix == "ljspeech": + text = c.supervisions[0].custom["normalized_text"] + text = text.replace(""", '"').replace(""", '"') + phonemes = tokenize_text(text_tokenizer, text=text) + elif args.prefix in [ + "aishell", + "aishell2", + "wenetspeech4tts", + "libritts", + "libritts-r", + ]: + phonemes = tokenize_text( + text_tokenizer, text=c.supervisions[0].text + ) + if c.supervisions[0].custom is None: + c.supervisions[0].custom = {} + c.supervisions[0].normalized_text = c.supervisions[0].text + else: + raise NotImplementedError(f"{args.prefix}") + unique_symbols.update(phonemes) + c.tokens = phonemes + assert c.supervisions[ + 0 + ].normalized_text, "normalized_text is None" + + # Save each part with an index if split > 1 + cuts_filename = ( + f"{prefix}cuts_{partition}.{idx if split > 1 else ''}.{args.suffix}" + ) + part.to_file(f"{args.output_dir}/{cuts_filename}") + logging.info(f"Saved {cuts_filename}") + + if args.text_extractor: + unique_phonemes = SymbolTable() + for s in sorted(list(unique_symbols)): + unique_phonemes.add(s) + logging.info(f"{len(unique_symbols)} unique phonemes: {unique_symbols}") + + unique_phonemes_file = f"{args.output_dir}/unique_text_tokens.k2symbols" + unique_phonemes.to_file(unique_phonemes_file) + + +if __name__ == "__main__": + formatter = "%(asctime)s %(levelname)s [%(filename)s:%(lineno)d] %(message)s" + logging.basicConfig(format=formatter, level=logging.INFO) + main() diff --git a/egs/wenetspeech4tts/TTS/local/display_manifest_statistics.py b/egs/wenetspeech4tts/TTS/local/display_manifest_statistics.py new file mode 100755 index 000000000..f967dfd2b --- /dev/null +++ b/egs/wenetspeech4tts/TTS/local/display_manifest_statistics.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +# Copyright 2021 Xiaomi Corp. (authors: Fangjun Kuang) +# Copyright 2023 (authors: Feiteng Li) +# +# See ../../../../LICENSE for clarification regarding multiple authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This file displays duration statistics of utterances in the manifests. +You can use the displayed value to choose minimum/maximum duration +to remove short and long utterances during the training. +""" + +import argparse +from pathlib import Path + +from lhotse import load_manifest_lazy + + +def get_args(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--manifest-dir", + type=Path, + default=Path("data/tokenized"), + help="Path to the tokenized manifests.", + ) + return parser.parse_args() + + +def main(): + args = get_args() + manifest_dir = args.manifest_dir or Path("data/tokenized") + for part in ["train", "dev", "test"]: + print(f"## {part}") + cuts = load_manifest_lazy(manifest_dir / f"cuts_{part}.jsonl.gz") + cuts.describe() + print("\n") + + +if __name__ == "__main__": + main() diff --git a/egs/wenetspeech4tts/TTS/prepare.sh b/egs/wenetspeech4tts/TTS/prepare.sh new file mode 100755 index 000000000..54e140dbb --- /dev/null +++ b/egs/wenetspeech4tts/TTS/prepare.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash + +set -eou pipefail + +# fix segmentation fault reported in https://github.com/k2-fsa/icefall/issues/674 +export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python + +stage=1 +stop_stage=4 + +dl_dir=$PWD/download + +dataset_parts="Premium" # Basic for all 10k hours data, Premium for about 10% of the data + +text_extractor="pypinyin_initials_finals" # default is espeak for English +audio_extractor="Encodec" # or Fbank +audio_feats_dir=data/tokenized + +. shared/parse_options.sh || exit 1 + + +# All files generated by this script are saved in "data". +# You can safely remove "data" and rerun this script to regenerate it. +mkdir -p data +log() { + # This function is from espnet + local fname=${BASH_SOURCE[1]##*/} + echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*" +} + +if [ $stage -le 0 ] && [ $stop_stage -ge 0 ]; then + log "dl_dir: $dl_dir" + log "Stage 0: Download data" + huggingface-cli login + huggingface-cli download --repo-type dataset --local-dir $dl_dir Wenetspeech4TTS/WenetSpeech4TTS + + # Extract the downloaded data: + for folder in Standard Premium Basic; do + for file in "$dl_dir/$folder"/*.tar.gz; do + tar -xzvf "$file" -C "$dl_dir/$folder" + done + done +fi + +if [ $stage -le 1 ] && [ $stop_stage -ge 1 ]; then + log "Stage 1: Prepare wenetspeech4tts manifest" + # We assume that you have downloaded the wenetspeech4tts corpus + # to $dl_dir/wenetspeech4tts + mkdir -p data/manifests + if [ ! -e data/manifests/.wenetspeech4tts.done ]; then + lhotse prepare wenetspeech4tts $dl_dir data/manifests --dataset-parts "${dataset_parts}" + touch data/manifests/.wenetspeech4tts.done + fi +fi + + +if [ $stage -le 2 ] && [ $stop_stage -ge 2 ]; then + log "Stage 2: Tokenize/Fbank wenetspeech4tts" + mkdir -p ${audio_feats_dir} + if [ ! -e ${audio_feats_dir}/.wenetspeech4tts.tokenize.done ]; then + python3 ./local/compute_neural_codec_and_prepare_text_tokens.py --dataset-parts "${dataset_parts}" \ + --text-extractor ${text_extractor} \ + --audio-extractor ${audio_extractor} \ + --batch-duration 2500 --prefix "wenetspeech4tts" \ + --src-dir "data/manifests" \ + --split 100 \ + --output-dir "${audio_feats_dir}/wenetspeech4tts_${dataset_parts}_split_100" + cp ${audio_feats_dir}/wenetspeech4tts_${dataset_parts}_split_100/unique_text_tokens.k2symbols ${audio_feats_dir} + fi + touch ${audio_feats_dir}/.wenetspeech4tts.tokenize.done +fi + +if [ $stage -le 3 ] && [ $stop_stage -ge 3 ]; then + log "Stage 3: Combine features" + if [ ! -f ${audio_feats_dir}/wenetspeech4tts_cuts_${dataset_parts}.jsonl.gz ]; then + pieces=$(find ${audio_feats_dir}/wenetspeech4tts_${dataset_parts}_split_100 -name "*.jsonl.gz") + lhotse combine $pieces ${audio_feats_dir}/wenetspeech4tts_cuts_${dataset_parts}.jsonl.gz + fi +fi + +if [ $stage -le 4 ] && [ $stop_stage -ge 4 ]; then + log "Stage 4: Prepare wenetspeech4tts train/dev/test" + if [ ! -e ${audio_feats_dir}/.wenetspeech4tts.train.done ]; then + + lhotse subset --first 400 \ + ${audio_feats_dir}/wenetspeech4tts_cuts_${dataset_parts}.jsonl.gz \ + ${audio_feats_dir}/cuts_dev.jsonl.gz + + lhotse subset --last 400 \ + ${audio_feats_dir}/wenetspeech4tts_cuts_${dataset_parts}.jsonl.gz \ + ${audio_feats_dir}/cuts_test.jsonl.gz + + lhotse copy \ + ${audio_feats_dir}/wenetspeech4tts_cuts_${dataset_parts}.jsonl.gz \ + ${audio_feats_dir}/cuts_train.jsonl.gz + + touch ${audio_feats_dir}/.wenetspeech4tts.train.done + fi + python3 ./local/display_manifest_statistics.py --manifest-dir ${audio_feats_dir} +fi diff --git a/egs/wenetspeech4tts/TTS/shared b/egs/wenetspeech4tts/TTS/shared new file mode 120000 index 000000000..4c5e91438 --- /dev/null +++ b/egs/wenetspeech4tts/TTS/shared @@ -0,0 +1 @@ +../../../icefall/shared/ \ No newline at end of file diff --git a/egs/wenetspeech4tts/TTS/valle/compute_neural_codec_and_prepare_text_tokens.py b/egs/wenetspeech4tts/TTS/valle/compute_neural_codec_and_prepare_text_tokens.py new file mode 120000 index 000000000..e70ee319a --- /dev/null +++ b/egs/wenetspeech4tts/TTS/valle/compute_neural_codec_and_prepare_text_tokens.py @@ -0,0 +1 @@ +../local/compute_neural_codec_and_prepare_text_tokens.py \ No newline at end of file diff --git a/egs/wenetspeech4tts/TTS/valle/infer.py b/egs/wenetspeech4tts/TTS/valle/infer.py new file mode 100644 index 000000000..fd7ba9f21 --- /dev/null +++ b/egs/wenetspeech4tts/TTS/valle/infer.py @@ -0,0 +1,300 @@ +#!/usr/bin/env python3 +# Copyright 2023 (authors: Feiteng Li) +# Copyright 2024 (authors: Yuekai Zhang) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +This script is used to synthesize speech from text prompts and audio prompts. +Usage example: + python3 valle/infer.py --output-dir demos_epoch_${epoch}_avg_${avg} \ + --checkpoint=${exp_dir}/epoch-${epoch}-avg-${avg}.pt \ + --text-prompts "KNOT one point one five miles per hour." \ + --audio-prompts ./prompts/8463_294825_000043_000000.wav \ + --text "To get up and running quickly just follow the steps below." + + top_p=1.0 + python3 valle/infer.py --output-dir demos_epoch_${epoch}_avg_${avg}_top_p_${top_p} \ + --top-k -1 --temperature 1.0 \ + --text ./aishell3.txt \ + --checkpoint ${exp_dir}/epoch-${epoch}-avg-${avg}.pt \ + --text-extractor pypinyin_initials_finals --top-p ${top_p} + +""" +import argparse +import logging +import os +from pathlib import Path + +os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python" + +import torch +import torchaudio +from compute_neural_codec_and_prepare_text_tokens import ( + AudioTokenizer, + TextTokenizer, + tokenize_text, +) +from encodec.utils import convert_audio +from k2 import symbol_table +from tokenizer import get_text_token_collater +from valle import VALLE + +from icefall.utils import AttributeDict, str2bool + + +def get_args(): + parser = argparse.ArgumentParser() + + parser.add_argument( + "--text-prompts", + type=str, + default="", + help="Text prompts which are separated by |.", + ) + + parser.add_argument( + "--audio-prompts", + type=str, + default="", + help="Audio prompts which are separated by | and should be aligned with --text-prompts.", + ) + + parser.add_argument( + "--text", + type=str, + default="", + help="prompt text\t prompt audio\ttarget text\ttarget audio", + ) + + parser.add_argument( + "--text-extractor", + type=str, + default="espeak", + help="espeak or pypinyin or pypinyin_initials_finals", + ) + + parser.add_argument( + "--checkpoint", + type=str, + default="exp/vallf_nano_full/checkpoint-100000.pt", + help="Path to the saved checkpoint.", + ) + + parser.add_argument( + "--output-dir", + type=Path, + default=Path("infer/demo"), + help="Path to the tokenized files.", + ) + + parser.add_argument( + "--top-k", + type=int, + default=-100, + help="Whether AR Decoder do top_k(if > 0) sampling.", + ) + + parser.add_argument( + "--top-p", + type=float, + default=1.0, + help="Whether AR Decoder do top_p(if > 0) sampling.", + ) + + parser.add_argument( + "--temperature", + type=float, + default=1.0, + help="The temperature of AR Decoder top_k sampling.", + ) + + parser.add_argument( + "--continual", + type=str2bool, + default=False, + help="Do continual task.", + ) + + parser.add_argument( + "--repetition-aware-sampling", + type=str2bool, + default=False, + help="Whether AR Decoder do valle-2 repetition-aware sampling. https://arxiv.org/pdf/2406.05370", + ) + + return parser.parse_args() + + +def load_model(checkpoint, device): + if not checkpoint: + return None + + checkpoint = torch.load(checkpoint, map_location=device) + + params = AttributeDict(checkpoint) + model = VALLE( + params.decoder_dim, + params.nhead, + params.num_decoder_layers, + norm_first=params.norm_first, + add_prenet=params.add_prenet, + prefix_mode=params.prefix_mode, + share_embedding=params.share_embedding, + nar_scale_factor=params.scale_factor, + prepend_bos=params.prepend_bos, + num_quantizers=params.num_quantizers, + ) + + missing_keys, unexpected_keys = model.load_state_dict( + checkpoint["model"], strict=True + ) + assert not missing_keys + model.to(device) + model.eval() + + return model, params.text_tokens + + +def tokenize_audio(tokenizer: AudioTokenizer, audio_path: str): + # Load and pre-process the audio waveform + wav, sr = torchaudio.load(audio_path) + wav = convert_audio(wav, sr, tokenizer.sample_rate, tokenizer.channels) + wav = wav.unsqueeze(0) + + # Extract discrete codes from EnCodec + with torch.no_grad(): + encoded_frames = tokenizer.encode(wav) + return encoded_frames + + +@torch.no_grad() +def main(): + args = get_args() + text_tokenizer = TextTokenizer(backend=args.text_extractor) + device = torch.device("cpu") + if torch.cuda.is_available(): + device = torch.device("cuda", 0) + + model, text_tokens = load_model(args.checkpoint, device) + + text_collater = get_text_token_collater(text_tokens) + + audio_tokenizer = AudioTokenizer() + + Path(args.output_dir).mkdir(parents=True, exist_ok=True) + + text_prompts = " ".join(args.text_prompts.split("|")) + + audio_prompts = [] + if args.audio_prompts: + for n, audio_file in enumerate(args.audio_prompts.split("|")): + encoded_frames = tokenize_audio(audio_tokenizer, audio_file) + if False: + samples = audio_tokenizer.decode(encoded_frames) + torchaudio.save(f"{args.output_dir}/p{n}.wav", samples[0], 24000) + + audio_prompts.append(encoded_frames[0][0]) + + assert len(args.text_prompts.split("|")) == len(audio_prompts) + audio_prompts = torch.concat(audio_prompts, dim=-1).transpose(2, 1) + audio_prompts = audio_prompts.to(device) + + if os.path.isfile(args.text): # for demos + # https://github.com/lifeiteng/lifeiteng.github.com/blob/main/valle/prepare.py + with open(args.text) as f: + for line in f: + fields = line.strip().split(" ") + fields = [item for item in fields if item] + assert len(fields) == 4 + prompt_text, prompt_audio, text, audio_path = fields + logging.info(f"synthesize text: {text}") + text_tokens, text_tokens_lens = text_collater( + [ + tokenize_text( + text_tokenizer, text=f"{prompt_text} {text}".strip() + ) + ] + ) + _, enroll_x_lens = text_collater( + [tokenize_text(text_tokenizer, text=f"{prompt_text}".strip())] + ) + + audio_prompts = tokenize_audio(audio_tokenizer, prompt_audio) + audio_prompts = audio_prompts[0][0].transpose(2, 1).to(device) + + # synthesis + encoded_frames = model.inference( + text_tokens.to(device), + text_tokens_lens.to(device), + audio_prompts, + enroll_x_lens=enroll_x_lens, + top_k=args.top_k, + temperature=args.temperature, + top_p=args.top_p, + ras=args.repetition_aware_sampling, + ) + + samples = audio_tokenizer.decode( + [(encoded_frames.transpose(2, 1), None)] + ) + # store + # save audio path into args.output_dir + audio_path + audio_path = f"{args.output_dir}/{audio_path}" + # mkdir -p + os.makedirs(os.path.dirname(audio_path), exist_ok=True) + torchaudio.save(audio_path, samples[0].cpu(), 24000) + return + + for n, text in enumerate(args.text.split("|")): + logging.info(f"synthesize text: {text}") + text_tokens, text_tokens_lens = text_collater( + [tokenize_text(text_tokenizer, text=f"{text_prompts} {text}".strip())] + ) + + # synthesis + if args.continual: + assert text == "" + encoded_frames = model.continual( + text_tokens.to(device), + text_tokens_lens.to(device), + audio_prompts, + ) + else: + enroll_x_lens = None + if text_prompts: + _, enroll_x_lens = text_collater( + [tokenize_text(text_tokenizer, text=f"{text_prompts}".strip())] + ) + encoded_frames = model.inference( + text_tokens.to(device), + text_tokens_lens.to(device), + audio_prompts, + enroll_x_lens=enroll_x_lens, + top_k=args.top_k, + temperature=args.temperature, + top_p=args.top_p, + ras=args.repetition_aware_sampling, + ) + + if audio_prompts != []: + samples = audio_tokenizer.decode([(encoded_frames.transpose(2, 1), None)]) + # store + torchaudio.save(f"{args.output_dir}/{n}.wav", samples[0].cpu(), 24000) + else: # Transformer + pass + + +if __name__ == "__main__": + formatter = "%(asctime)s %(levelname)s [%(filename)s:%(lineno)d] %(message)s" + logging.basicConfig(format=formatter, level=logging.INFO) + main() diff --git a/egs/wenetspeech4tts/TTS/valle/optim.py b/egs/wenetspeech4tts/TTS/valle/optim.py new file mode 120000 index 000000000..5eaa3cffd --- /dev/null +++ b/egs/wenetspeech4tts/TTS/valle/optim.py @@ -0,0 +1 @@ +../../../librispeech/ASR/zipformer/optim.py \ No newline at end of file diff --git a/egs/wenetspeech4tts/TTS/valle/tokenizer.py b/egs/wenetspeech4tts/TTS/valle/tokenizer.py new file mode 100644 index 000000000..db4f00396 --- /dev/null +++ b/egs/wenetspeech4tts/TTS/valle/tokenizer.py @@ -0,0 +1,111 @@ +from pathlib import Path +from typing import List, Tuple + +import numpy as np +import torch +from k2 import SymbolTable + + +class TextTokenCollater: + """Collate list of text tokens + + Map sentences to integers. Sentences are padded to equal length. + Beginning and end-of-sequence symbols can be added. + + Example: + >>> token_collater = TextTokenCollater(text_tokens) + >>> tokens_batch, tokens_lens = token_collater(text) + + Returns: + tokens_batch: IntTensor of shape (B, L) + B: batch dimension, number of input sentences + L: length of the longest sentence + tokens_lens: IntTensor of shape (B,) + Length of each sentence after adding and + but before padding. + """ + + def __init__( + self, + text_tokens: List[str], + add_eos: bool = True, + add_bos: bool = True, + pad_symbol: str = "", + bos_symbol: str = "", + eos_symbol: str = "", + ): + self.pad_symbol = pad_symbol + + self.add_eos = add_eos + self.add_bos = add_bos + + self.bos_symbol = bos_symbol + self.eos_symbol = eos_symbol + + unique_tokens = ( + [pad_symbol] + + ([bos_symbol] if add_bos else []) + + ([eos_symbol] if add_eos else []) + + sorted(text_tokens) + ) + + self.token2idx = {token: idx for idx, token in enumerate(unique_tokens)} + self.idx2token = [token for token in unique_tokens] + + def index(self, tokens_list: List[str]) -> Tuple[torch.Tensor, torch.Tensor]: + seqs, seq_lens = [], [] + for tokens in tokens_list: + assert all([True if s in self.token2idx else False for s in tokens]) is True + seq = ( + ([self.bos_symbol] if self.add_bos else []) + + list(tokens) + + ([self.eos_symbol] if self.add_eos else []) + ) + seqs.append(seq) + seq_lens.append(len(seq)) + + max_len = max(seq_lens) + for k, (seq, seq_len) in enumerate(zip(seqs, seq_lens)): + seq.extend([self.pad_symbol] * (max_len - seq_len)) + + tokens = torch.from_numpy( + np.array( + [[self.token2idx[token] for token in seq] for seq in seqs], + dtype=np.int64, + ) + ) + tokens_lens = torch.IntTensor(seq_lens) + + return tokens, tokens_lens + + def __call__(self, texts: List[str]) -> Tuple[torch.Tensor, torch.Tensor]: + tokens_seqs = [[p for p in text] for text in texts] + max_len = len(max(tokens_seqs, key=len)) + + seqs = [ + ([self.bos_symbol] if self.add_bos else []) + + list(seq) + + ([self.eos_symbol] if self.add_eos else []) + + [self.pad_symbol] * (max_len - len(seq)) + for seq in tokens_seqs + ] + + tokens_batch = torch.from_numpy( + np.array( + [[self.token2idx[token] for token in seq] for seq in seqs], + dtype=np.int64, + ) + ) + + tokens_lens = torch.IntTensor( + [len(seq) + int(self.add_eos) + int(self.add_bos) for seq in tokens_seqs] + ) + + return tokens_batch, tokens_lens + + +def get_text_token_collater(text_tokens_file: str) -> TextTokenCollater: + text_tokens_path = Path(text_tokens_file) + unique_tokens = SymbolTable.from_file(text_tokens_path) + collater = TextTokenCollater(unique_tokens.symbols, add_bos=True, add_eos=True) + return collater diff --git a/egs/wenetspeech4tts/TTS/valle/train.py b/egs/wenetspeech4tts/TTS/valle/train.py new file mode 100755 index 000000000..fde209511 --- /dev/null +++ b/egs/wenetspeech4tts/TTS/valle/train.py @@ -0,0 +1,1244 @@ +#!/usr/bin/env python3 +# Copyright 2021-2022 Xiaomi Corp. (authors: Fangjun Kuang, +# Wei Kang, +# Mingshuang Luo) +# Copyright 2023 (authors: Feiteng Li) +# Copyright 2024 (authors: Yuekai Zhang) +# +# See ../../../../LICENSE for clarification regarding multiple authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Usage: +world_size=8 +exp_dir=exp/valle + +## Train AR model +python3 valle/train.py --max-duration 320 --filter-min-duration 0.5 --filter-max-duration 14 --train-stage 1 \ + --num-buckets 6 --dtype "bfloat16" --save-every-n 1000 --valid-interval 2000 \ + --share-embedding true --norm-first true --add-prenet false \ + --decoder-dim 1024 --nhead 16 --num-decoder-layers 12 --prefix-mode 1 \ + --base-lr 0.03 --warmup-steps 200 --average-period 0 \ + --num-epochs 20 --start-epoch 1 --start-batch 0 --accumulate-grad-steps 1 \ + --exp-dir ${exp_dir} --world-size ${world_size} + +## Train NAR model +# cd ${exp_dir} +# ln -s ${exp_dir}/best-valid-loss.pt epoch-99.pt # --start-epoch 100=99+1 +# cd - +python3 valle/train.py --max-duration 160 --filter-min-duration 0.5 --filter-max-duration 14 --train-stage 2 \ + --num-buckets 6 --dtype "float32" --save-every-n 1000 --valid-interval 2000 \ + --share-embedding true --norm-first true --add-prenet false \ + --decoder-dim 1024 --nhead 16 --num-decoder-layers 12 --prefix-mode 1 \ + --base-lr 0.03 --warmup-steps 200 --average-period 0 \ + --num-epochs 40 --start-epoch 100 --start-batch 0 --accumulate-grad-steps 2 \ + --exp-dir ${exp_dir} --world-size ${world_size} +""" + +import argparse +import copy +import logging +import os +import random +import warnings +from contextlib import nullcontext +from pathlib import Path +from shutil import copyfile +from typing import Any, Dict, Optional, Tuple, Union + +import torch +import torch.multiprocessing as mp +import torch.nn as nn +from lhotse import CutSet +from lhotse.cut import Cut +from lhotse.dataset.sampling.base import CutSampler +from lhotse.utils import fix_random_seed +from optim import Eden, ScaledAdam +from tokenizer import TextTokenCollater, get_text_token_collater +from torch import Tensor +from torch.cuda.amp import GradScaler +from torch.nn.parallel import DistributedDataParallel as DDP +from torch.utils.tensorboard import SummaryWriter +from tts_datamodule import TtsDataModule +from valle import VALLE + +from icefall.checkpoint import load_checkpoint, remove_checkpoints +from icefall.checkpoint import save_checkpoint as save_checkpoint_impl +from icefall.checkpoint import ( + save_checkpoint_with_global_batch_idx, + update_averaged_model, +) +from icefall.dist import cleanup_dist, setup_dist +from icefall.env import get_env_info +from icefall.hooks import register_inf_check_hooks +from icefall.utils import AttributeDict, MetricsTracker, setup_logger, str2bool + +LRSchedulerType = torch.optim.lr_scheduler._LRScheduler + + +def set_batch_count(model: Union[nn.Module, DDP], batch_count: float) -> None: + if isinstance(model, DDP): + # get underlying nn.Module + model = model.module + + for module in model.modules(): + if hasattr(module, "batch_count"): + module.batch_count = batch_count + + +def add_model_arguments(parser: argparse.ArgumentParser): + parser.add_argument( + "--decoder-dim", + type=int, + default=1024, + help="Embedding dimension in the decoder model.", + ) + parser.add_argument( + "--nhead", + type=int, + default=16, + help="Number of attention heads in the Decoder layers.", + ) + parser.add_argument( + "--num-decoder-layers", + type=int, + default=12, + help="Number of Decoder layers.", + ) + parser.add_argument( + "--scale-factor", + type=float, + default=1.0, + help="Model scale factor which will be assigned different meanings in different models.", + ) + parser.add_argument( + "--norm-first", + type=str2bool, + default=True, + help="Pre or Post Normalization.", + ) + parser.add_argument( + "--add-prenet", + type=str2bool, + default=False, + help="Whether add PreNet after Inputs.", + ) + + parser.add_argument( + "--prefix-mode", + type=int, + default=0, + help="The mode for how to prefix VALL-E NAR Decoder, " + "0: no prefix, 1: 0 to random, 2: random to random, 4: chunk of pre or post utterance.", + ) + parser.add_argument( + "--share-embedding", + type=str2bool, + default=True, + help="Share the parameters of the output projection layer with the parameters of the acoustic embedding.", + ) + parser.add_argument( + "--prepend-bos", + type=str2bool, + default=False, + help="Whether prepend to the acoustic tokens -> AR Decoder inputs.", + ) + parser.add_argument( + "--num-quantizers", + type=int, + default=8, + help="Number of Audio/Semantic quantization layers.", + ) + + +def get_parser(): + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + + parser.add_argument( + "--world-size", + type=int, + default=1, + help="Number of GPUs for DDP training.", + ) + + parser.add_argument( + "--master-port", + type=int, + default=12354, + help="Master port to use for DDP training.", + ) + + parser.add_argument( + "--tensorboard", + type=str2bool, + default=True, + help="Should various information be logged in tensorboard.", + ) + + parser.add_argument( + "--num-epochs", + type=int, + default=20, + help="Number of epochs to train.", + ) + + parser.add_argument( + "--start-epoch", + type=int, + default=1, + help="""Resume training from this epoch. It should be positive. + If larger than 1, it will load checkpoint from + exp-dir/epoch-{start_epoch-1}.pt + """, + ) + + parser.add_argument( + "--start-batch", + type=int, + default=0, + help="""If positive, --start-epoch is ignored and + it loads the checkpoint from exp-dir/checkpoint-{start_batch}.pt + """, + ) + + parser.add_argument( + "--exp-dir", + type=str, + default="exp/valle_dev", + help="""The experiment dir. + It specifies the directory where all training related + files, e.g., checkpoints, log, etc, are saved + """, + ) + + parser.add_argument( + "--text-tokens", + type=str, + default="data/tokenized/unique_text_tokens.k2symbols", + help="Path to the unique text tokens file", + ) + + parser.add_argument( + "--optimizer-name", + type=str, + default="ScaledAdam", + help="The optimizer.", + ) + parser.add_argument( + "--scheduler-name", + type=str, + default="Eden", + help="The scheduler.", + ) + parser.add_argument( + "--base-lr", type=float, default=0.05, help="The base learning rate." + ) + parser.add_argument( + "--warmup-steps", + type=int, + default=200, + help="""Number of steps that affects how rapidly the learning rate + decreases. We suggest not to change this.""", + ) + + parser.add_argument( + "--seed", + type=int, + default=42, + help="The seed for random generators intended for reproducibility", + ) + + parser.add_argument( + "--inf-check", + type=str2bool, + default=False, + help="Add hooks to check for infinite module outputs and gradients.", + ) + + parser.add_argument( + "--save-every-n", + type=int, + default=10000, + help="""Save checkpoint after processing this number of batches" + periodically. We save checkpoint to exp-dir/ whenever + params.batch_idx_train %% save_every_n == 0. The checkpoint filename + has the form: f'exp-dir/checkpoint-{params.batch_idx_train}.pt' + Note: It also saves checkpoint to `exp-dir/epoch-xxx.pt` at the + end of each epoch where `xxx` is the epoch number counting from 0. + """, + ) + parser.add_argument( + "--valid-interval", + type=int, + default=10000, + help="""Run validation if batch_idx %% valid_interval is 0.""", + ) + + parser.add_argument( + "--keep-last-k", + type=int, + default=20, + help="""Only keep this number of checkpoints on disk. + For instance, if it is 3, there are only 3 checkpoints + in the exp-dir with filenames `checkpoint-xxx.pt`. + It does not affect checkpoints with name `epoch-xxx.pt`. + """, + ) + + parser.add_argument( + "--average-period", + type=int, + default=0, + help="""Update the averaged model, namely `model_avg`, after processing + this number of batches. `model_avg` is a separate version of model, + in which each floating-point parameter is the average of all the + parameters from the start of training. Each time we take the average, + we do: `model_avg = model * (average_period / batch_idx_train) + + model_avg * ((batch_idx_train - average_period) / batch_idx_train)`. + """, + ) + + parser.add_argument( + "--accumulate-grad-steps", + type=int, + default=1, + help="""update gradient when batch_idx_train %% accumulate_grad_steps == 0. + """, + ) + + parser.add_argument( + "--dtype", + type=str, + default="float32", + help="Training dtype: float32 bfloat16 float16.", + ) + + parser.add_argument( + "--filter-min-duration", + type=float, + default=0.0, + help="Keep only utterances with duration > this.", + ) + parser.add_argument( + "--filter-max-duration", + type=float, + default=20.0, + help="Keep only utterances with duration < this.", + ) + + parser.add_argument( + "--train-stage", + type=int, + default=0, + help="""0: train all modules, For VALL-E, support 1: AR Decoder 2: NAR Decoder(s) + """, + ) + + parser.add_argument( + "--visualize", + type=str2bool, + default=False, + help="visualize model results in eval step.", + ) + + parser.add_argument( + "--oom-check", + type=str2bool, + default=False, + help="perform OOM check on dataloader batches before starting training.", + ) + + add_model_arguments(parser) + + return parser + + +def get_params() -> AttributeDict: + """Return a dict containing training parameters. + + All training related parameters that are not passed from the commandline + are saved in the variable `params`. + + Commandline options are merged into `params` after they are parsed, so + you can also access them via `params`. + + Explanation of options saved in `params`: + + - best_train_loss: Best training loss so far. It is used to select + the model that has the lowest training loss. It is + updated during the training. + + - best_valid_loss: Best validation loss so far. It is used to select + the model that has the lowest validation loss. It is + updated during the training. + + - best_train_epoch: It is the epoch that has the best training loss. + + - best_valid_epoch: It is the epoch that has the best validation loss. + + - batch_idx_train: Used to writing statistics to tensorboard. It + contains number of batches trained so far across + epochs. + + - log_interval: Print training loss if batch_idx % log_interval` is 0 + + - reset_interval: Reset statistics if batch_idx % reset_interval is 0 + + - valid_interval: Run validation if batch_idx % valid_interval is 0 + """ + params = AttributeDict( + { + "best_train_loss": float("inf"), + "best_valid_loss": float("inf"), + "best_train_epoch": -1, + "best_valid_epoch": -1, + "batch_idx_train": 0, + "log_interval": 100, + "reset_interval": 200, + "valid_interval": 10000, + "env_info": get_env_info(), + } + ) + + return params + + +def load_checkpoint_if_available( + params: AttributeDict, + model: nn.Module, + model_avg: nn.Module = None, + optimizer: Optional[torch.optim.Optimizer] = None, + scheduler: Optional[LRSchedulerType] = None, +) -> Optional[Dict[str, Any]]: + """Load checkpoint from file. + + If params.start_batch is positive, it will load the checkpoint from + `params.exp_dir/checkpoint-{params.start_batch}.pt`. Otherwise, if + params.start_epoch is larger than 1, it will load the checkpoint from + `params.start_epoch - 1`. + + Apart from loading state dict for `model` and `optimizer` it also updates + `best_train_epoch`, `best_train_loss`, `best_valid_epoch`, + and `best_valid_loss` in `params`. + + Args: + params: + The return value of :func:`get_params`. + model: + The training model. + model_avg: + The stored model averaged from the start of training. + optimizer: + The optimizer that we are using. + scheduler: + The scheduler that we are using. + Returns: + Return a dict containing previously saved training info. + """ + if params.start_batch > 0: + filename = params.exp_dir / f"checkpoint-{params.start_batch}.pt" + elif params.start_epoch > 1: + filename = params.exp_dir / f"epoch-{params.start_epoch-1}.pt" + else: + return None + + assert filename.is_file(), f"{filename} does not exist!" + + if isinstance(model, DDP): + raise ValueError("load_checkpoint before DDP") + + saved_params = load_checkpoint( + filename, + model=model, + model_avg=model_avg, + optimizer=optimizer, + scheduler=scheduler, + ) + + saved_stage = saved_params.get("train_stage", 0) + if params.train_stage != saved_stage: + # switch training stage + if params.train_stage and saved_stage: # switch between 1 and 2 + params.start_epoch = 1 + params.start_batch = 0 + else: + # switch between 0 and 1/2 + assert params.num_epochs >= params.start_epoch + params.batch_idx_train = saved_params["batch_idx_train"] + + for key in ["optimizer", "grad_scaler", "sampler"]: + if key in saved_params: + saved_params.pop(key) + + # when base on stage 0, we keep scheduler + if saved_stage != 0: + for key in ["scheduler"]: + if key in saved_params: + saved_params.pop(key) + + best_train_filename = params.exp_dir / "best-train-loss.pt" + if best_train_filename.is_file(): + copyfile( + src=best_train_filename, + dst=params.exp_dir / f"best-train-loss-stage{saved_stage}.pt", + ) + + best_valid_filename = params.exp_dir / "best-valid-loss.pt" + if best_valid_filename.is_file(): + copyfile( + src=best_valid_filename, + dst=params.exp_dir / f"best-valid-loss-stage{saved_stage}.pt", + ) + else: + + keys = [ + "best_train_epoch", + "best_valid_epoch", + "batch_idx_train", + "best_train_loss", + "best_valid_loss", + ] + for k in keys: + params[k] = saved_params[k] + + if params.start_batch > 0: + if "cur_epoch" in saved_params: + params["start_epoch"] = saved_params["cur_epoch"] + + return saved_params + + +def save_checkpoint( + params: AttributeDict, + model: Union[nn.Module, DDP], + model_avg: Optional[nn.Module] = None, + optimizer: Optional[torch.optim.Optimizer] = None, + scheduler: Optional[LRSchedulerType] = None, + sampler: Optional[CutSampler] = None, + scaler: Optional[GradScaler] = None, + rank: int = 0, +) -> None: + """Save model, optimizer, scheduler and training stats to file. + + Args: + params: + It is returned by :func:`get_params`. + model: + The training model. + model_avg: + The stored model averaged from the start of training. + optimizer: + The optimizer used in the training. + sampler: + The sampler for the training dataset. + scaler: + The scaler used for mix precision training. + """ + if rank != 0: + return + filename = params.exp_dir / f"epoch-{params.cur_epoch}.pt" + save_checkpoint_impl( + filename=filename, + model=model, + model_avg=model_avg, + params=params, + optimizer=optimizer, + scheduler=scheduler, + sampler=sampler, + scaler=scaler, + rank=rank, + ) + + if params.best_train_epoch == params.cur_epoch: + best_train_filename = params.exp_dir / "best-train-loss.pt" + copyfile(src=filename, dst=best_train_filename) + + if params.best_valid_epoch == params.cur_epoch: + best_valid_filename = params.exp_dir / "best-valid-loss.pt" + copyfile(src=filename, dst=best_valid_filename) + + +def prepare_input(batch: dict, tokenizer: TextTokenCollater, device: torch.device): + """Parse batch data""" + + features = batch["features"].to(device) + features_lens = batch["features_lens"].to(device) + if "tokens" not in batch: + raise NotImplementedError("Need to tokenize text") + # tokens = [] + # for c in batch["cuts"]: + # phonemes = tokenize_text( + # tokenizer, text=c.supervisions[0].text + # ) + # tokens.append(phonemes) + else: + tokens = batch["tokens"] + + text_tokens, text_tokens_lens = tokenizer(tokens) + text_tokens = text_tokens.to(device) + text_tokens_lens = text_tokens_lens.to(device) + + return features, features_lens, text_tokens, text_tokens_lens + + +def compute_loss( + params: AttributeDict, + model: Union[nn.Module, DDP], + tokenizer: TextTokenCollater, + batch: dict, + is_training: bool, +) -> Tuple[Tensor, MetricsTracker]: + """ + Compute transducer loss given the model and its inputs. + + Args: + params: + Parameters for training. See :func:`get_params`. + model: + The model for training. It is an instance of Zipformer in our case. + batch: + A batch of data. See `lhotse.dataset.K2SpeechRecognitionDataset()` + for the content in it. + is_training: + True for training. False for validation. When it is True, this + function enables autograd during computation; when it is False, it + disables autograd. + warmup: a floating point value which increases throughout training; + values >= 1.0 are fully warmed up and have all modules present. + """ + device = model.device if isinstance(model, DDP) else next(model.parameters()).device + ( + audio_features, + audio_features_lens, + text_tokens, + text_tokens_lens, + ) = prepare_input(batch, tokenizer, device) + # at entry, TextTokens is (N, P) + assert text_tokens.ndim == 2 + assert audio_features.ndim == 3 + + with torch.set_grad_enabled(is_training): + predicts, loss, metrics = model( + x=text_tokens, + x_lens=text_tokens_lens, + y=audio_features, + y_lens=audio_features_lens, + train_stage=params.train_stage, + ) + + assert loss.requires_grad == is_training + + info = MetricsTracker() + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + info["frames"] = (audio_features_lens).sum().item() + info["utterances"] = text_tokens.size(0) + + # Note: We use reduction=sum while computing the loss. + info["loss"] = loss.detach().cpu().item() + for metric in metrics: + info[metric] = metrics[metric].detach().cpu().item() + del metrics + + return predicts, loss, info + + +def compute_validation_loss( + params: AttributeDict, + model: Union[nn.Module, DDP], + tokenizer: TextTokenCollater, + valid_dl: torch.utils.data.DataLoader, + world_size: int = 1, +) -> MetricsTracker: + """Run the validation process.""" + tot_loss = MetricsTracker() + + for batch_idx, batch in enumerate(valid_dl): + predicts, loss, loss_info = compute_loss( + params=params, + model=model, + tokenizer=tokenizer, + batch=batch, + is_training=False, + ) + assert loss.requires_grad is False + tot_loss = tot_loss + loss_info + if world_size > 1: + tot_loss.reduce(loss.device) + loss_value = tot_loss["loss"] / tot_loss["frames"] + if loss_value < params.best_valid_loss: + params.best_valid_epoch = params.cur_epoch + params.best_valid_loss = loss_value + + if params.visualize: + output_dir = Path(f"{params.exp_dir}/eval/step-{params.batch_idx_train:06d}") + output_dir.mkdir(parents=True, exist_ok=True) + if isinstance(model, DDP): + model.module.visualize(predicts, batch, output_dir=output_dir) + else: + model.visualize(predicts, batch, output_dir=output_dir) + + return tot_loss + + +def train_one_epoch( + params: AttributeDict, + model: Union[nn.Module, DDP], + tokenizer: TextTokenCollater, + optimizer: torch.optim.Optimizer, + scheduler: LRSchedulerType, + train_dl: torch.utils.data.DataLoader, + valid_dl: torch.utils.data.DataLoader, + rng: random.Random, + scaler: GradScaler, + model_avg: Optional[nn.Module] = None, + tb_writer: Optional[SummaryWriter] = None, + world_size: int = 1, + rank: int = 0, +) -> None: + """Train the model for one epoch. + + The training loss from the mean of all frames is saved in + `params.train_loss`. It runs the validation process every + `params.valid_interval` batches. + + Args: + params: + It is returned by :func:`get_params`. + model: + The model for training. + optimizer: + The optimizer we are using. + scheduler: + The learning rate scheduler, we call step() every step. + train_dl: + Dataloader for the training dataset. + valid_dl: + Dataloader for the validation dataset. + rng: + Random for selecting. + scaler: + The scaler used for mix precision training. + model_avg: + The stored model averaged from the start of training. + tb_writer: + Writer to write log messages to tensorboard. + world_size: + Number of nodes in DDP training. If it is 1, DDP is disabled. + rank: + The rank of the node in DDP training. If no DDP is used, it should + be set to 0. + """ + model.train() + tot_loss = MetricsTracker() + iter_dl = iter(train_dl) + + dtype, enabled = torch.float32, False + if params.dtype in ["bfloat16", "bf16"]: + dtype, enabled = torch.bfloat16, True + elif params.dtype in ["float16", "fp16"]: + dtype, enabled = torch.float16, True + + batch_idx = 0 + while True: + try: + batch = next(iter_dl) + except StopIteration: + logging.info("Reaches end of dataloader.") + break + + batch_idx += 1 + + params.batch_idx_train += 1 + batch_size = len(batch["text"]) + + try: + with torch.cuda.amp.autocast(dtype=dtype, enabled=enabled): + _, loss, loss_info = compute_loss( + params=params, + model=model, + tokenizer=tokenizer, + batch=batch, + is_training=True, + ) + # summary stats + tot_loss = (tot_loss * (1 - 1 / params.reset_interval)) + loss_info * ( + 1 / params.reset_interval + ) + + # NOTE: We use reduction==sum and loss is computed over utterances + # in the batch and there is no normalization to it so far. + + scaler.scale(loss).backward() + if params.batch_idx_train >= params.accumulate_grad_steps: + if params.batch_idx_train % params.accumulate_grad_steps == 0: + if params.optimizer_name not in ["ScaledAdam", "Eve"]: + # Unscales the gradients of optimizer's assigned params in-place + scaler.unscale_(optimizer) + # Since the gradients of optimizer's assigned params are unscaled, clips as usual: + torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0) + + scaler.step(optimizer) + scaler.update() + optimizer.zero_grad() + + for k in range(params.accumulate_grad_steps): + if isinstance(scheduler, Eden): + scheduler.step_batch(params.batch_idx_train) + else: + scheduler.step() + + set_batch_count(model, params.batch_idx_train) + except: # noqa + display_and_save_batch(batch, params=params) + raise + + if params.average_period > 0: + if ( + params.batch_idx_train > 0 + and params.batch_idx_train % params.average_period == 0 + ): + # Perform Operation in rank 0 + if rank == 0: + update_averaged_model( + params=params, + model_cur=model, + model_avg=model_avg, + ) + + if ( + params.batch_idx_train > 0 + and params.batch_idx_train % params.save_every_n == 0 + ): + # Perform Operation in rank 0 + if rank == 0: + save_checkpoint_with_global_batch_idx( + out_dir=params.exp_dir, + global_batch_idx=params.batch_idx_train, + model=model, + model_avg=model_avg, + params=params, + optimizer=optimizer, + scheduler=scheduler, + sampler=train_dl.sampler, + scaler=scaler, + rank=rank, + ) + remove_checkpoints( + out_dir=params.exp_dir, + topk=params.keep_last_k, + rank=rank, + ) + + if batch_idx % 100 == 0 and params.dtype in ["float16", "fp16"]: + # If the grad scale was less than 1, try increasing it. The _growth_interval + # of the grad scaler is configurable, but we can't configure it to have different + # behavior depending on the current grad scale. + cur_grad_scale = scaler._scale.item() + if cur_grad_scale < 1.0 or (cur_grad_scale < 8.0 and batch_idx % 400 == 0): + scaler.update(cur_grad_scale * 2.0) + + if cur_grad_scale < 0.01: + logging.warning(f"Grad scale is small: {cur_grad_scale}") + if cur_grad_scale < 1.0e-05: + raise RuntimeError( + f"grad_scale is too small, exiting: {cur_grad_scale}" + ) + + if batch_idx % params.log_interval == 0: + cur_lr = scheduler.get_last_lr()[0] + cur_grad_scale = ( + scaler._scale.item() if params.dtype in ["float16", "fp16"] else 1.0 + ) + + logging.info( + f"Epoch {params.cur_epoch}, " + f"batch {batch_idx}, train_loss[{loss_info}], " + f"tot_loss[{tot_loss}], " + f"batch size: {batch_size}, " + f"lr: {cur_lr:.2e}" + + ( + f", grad_scale: {cur_grad_scale}" + if params.dtype in ["float16", "fp16"] + else "" + ) + ) + + if tb_writer is not None: + tb_writer.add_scalar( + "train/learning_rate", cur_lr, params.batch_idx_train + ) + loss_info.write_summary( + tb_writer, + "train/current_", + params.batch_idx_train, + ) + tot_loss.write_summary(tb_writer, "train/tot_", params.batch_idx_train) + tot_loss.write_summary(tb_writer, "train/tot_", params.batch_idx_train) + if params.dtype in ["float16", "fp16"]: + tb_writer.add_scalar( + "train/grad_scale", + cur_grad_scale, + params.batch_idx_train, + ) + + if params.batch_idx_train % params.valid_interval == 0: + # Calculate validation loss in Rank 0 + model.eval() + logging.info("Computing validation loss") + with torch.cuda.amp.autocast(dtype=dtype): + valid_info = compute_validation_loss( + params=params, + model=model, + tokenizer=tokenizer, + valid_dl=valid_dl, + world_size=world_size, + ) + logging.info(f"Epoch {params.cur_epoch}, validation: {valid_info}") + logging.info( + f"Maximum memory allocated so far is {torch.cuda.max_memory_allocated()//1000000}MB" + ) + + if tb_writer is not None: + valid_info.write_summary( + tb_writer, "train/valid_", params.batch_idx_train + ) + + model.train() + + loss_value = tot_loss["loss"] / tot_loss["frames"] + params.train_loss = loss_value + if params.train_loss < params.best_train_loss: + params.best_train_epoch = params.cur_epoch + params.best_train_loss = params.train_loss + + +def filter_short_and_long_utterances( + cuts: CutSet, min_duration: float, max_duration: float +) -> CutSet: + def remove_short_and_long_utt(c: Cut): + # Keep only utterances with duration between 0.6 second and 20 seconds + if c.duration < min_duration or c.duration > max_duration: + # logging.warning( + # f"Exclude cut with ID {c.id} from training. Duration: {c.duration}" + # ) + return False + return True + + cuts = cuts.filter(remove_short_and_long_utt) + + return cuts + + +def run(rank, world_size, args): + """ + Args: + rank: + It is a value between 0 and `world_size-1`, which is + passed automatically by `mp.spawn()` in :func:`main`. + The node with rank 0 is responsible for saving checkpoint. + world_size: + Number of GPUs for DDP training. + args: + The return value of get_parser().parse_args() + """ + params = get_params() + params.update(vars(args)) + + fix_random_seed(params.seed) + rng = random.Random(params.seed) + if world_size > 1: + setup_dist(rank, world_size, params.master_port) + + setup_logger(f"{params.exp_dir}/log/log-train") + logging.info("Training started") + + if args.tensorboard and rank == 0: + if params.train_stage: + tb_writer = SummaryWriter( + log_dir=f"{params.exp_dir}/tensorboard_stage{params.train_stage}" + ) + else: + tb_writer = SummaryWriter(log_dir=f"{params.exp_dir}/tensorboard") + else: + tb_writer = None + + device = torch.device("cpu") + if torch.cuda.is_available(): + device = torch.device("cuda", rank) + # https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices + torch.backends.cudnn.allow_tf32 = True + torch.backends.cuda.matmul.allow_tf32 = True + + logging.info(f"Device: {device}") + + tokenizer = get_text_token_collater(params.text_tokens) + logging.info(params) + + logging.info("About to create model") + + model = VALLE( + params.decoder_dim, + params.nhead, + params.num_decoder_layers, + norm_first=params.norm_first, + add_prenet=params.add_prenet, + prefix_mode=params.prefix_mode, + share_embedding=params.share_embedding, + nar_scale_factor=params.scale_factor, + prepend_bos=params.prepend_bos, + num_quantizers=params.num_quantizers, + ) + + with open(f"{params.exp_dir}/model.txt", "w") as f: + print(model) + print(model, file=f) + + num_param = sum([p.numel() for p in model.parameters()]) + logging.info(f"Number of model parameters: {num_param}") + + assert params.save_every_n >= params.average_period + model_avg: Optional[nn.Module] = None + if rank == 0 and params.average_period > 0: + # model_avg is only used with rank 0 + model_avg = copy.deepcopy(model).to(torch.float64) + + assert params.start_epoch > 0, params.start_epoch + checkpoints = load_checkpoint_if_available( + params=params, model=model, model_avg=model_avg + ) + + model.to(device) + if world_size > 1: + logging.info("Using DDP") + model = DDP(model, device_ids=[rank], find_unused_parameters=True) + + if params.train_stage: + _model = model.module if isinstance(model, DDP) else model + model_parameters = _model.stage_parameters(params.train_stage) + else: + model_parameters = model.parameters() + + if params.optimizer_name == "ScaledAdam": + optimizer = ScaledAdam( + model_parameters, + lr=params.base_lr, + clipping_scale=2.0, + ) + elif params.optimizer_name == "AdamW": + optimizer = torch.optim.AdamW( + model_parameters, + lr=params.base_lr, + betas=(0.9, 0.95), + weight_decay=1e-2, + eps=1e-8, + ) + elif params.optimizer_name == "Adam": + optimizer = torch.optim.Adam( + model_parameters, + lr=params.base_lr, + betas=(0.9, 0.95), + eps=1e-8, + ) + else: + raise NotImplementedError() + + scheduler = Eden(optimizer, 5000, 4, warmup_batches=params.warmup_steps) + optimizer.zero_grad() + + if checkpoints and "optimizer" in checkpoints: + logging.info("Loading optimizer state dict") + optimizer.load_state_dict(checkpoints["optimizer"]) + + if ( + checkpoints + and "scheduler" in checkpoints + and checkpoints["scheduler"] is not None + ): + logging.info("Loading scheduler state dict") + scheduler.load_state_dict(checkpoints["scheduler"]) + + if params.inf_check: + register_inf_check_hooks(model) + + if params.start_batch > 0 and checkpoints and "sampler" in checkpoints: + sampler_state_dict = checkpoints["sampler"] + else: + sampler_state_dict = None + + dataset = TtsDataModule(args) + train_cuts = dataset.train_cuts() + valid_cuts = dataset.dev_cuts() + + train_cuts = filter_short_and_long_utterances( + train_cuts, params.filter_min_duration, params.filter_max_duration + ) + valid_cuts = filter_short_and_long_utterances( + valid_cuts, params.filter_min_duration, params.filter_max_duration + ) + + train_dl = dataset.train_dataloaders( + train_cuts, sampler_state_dict=sampler_state_dict + ) + valid_dl = dataset.dev_dataloaders(valid_cuts) + + if params.oom_check: + scan_pessimistic_batches_for_oom( + model=model, + tokenizer=tokenizer, + train_dl=train_dl, + optimizer=optimizer, + params=params, + ) + + scaler = GradScaler(enabled=(params.dtype in ["fp16", "float16"]), init_scale=1.0) + if checkpoints and "grad_scaler" in checkpoints: + logging.info("Loading grad scaler state dict") + scaler.load_state_dict(checkpoints["grad_scaler"]) + + for epoch in range(params.start_epoch, params.num_epochs + 1): + if isinstance(scheduler, Eden): + scheduler.step_epoch(epoch - 1) + + fix_random_seed(params.seed + epoch - 1) + train_dl.sampler.set_epoch(epoch - 1) + + if tb_writer is not None: + tb_writer.add_scalar("train/epoch", epoch, params.batch_idx_train) + + params.cur_epoch = epoch + + train_one_epoch( + params=params, + model=model, + tokenizer=tokenizer, + model_avg=model_avg, + optimizer=optimizer, + scheduler=scheduler, + train_dl=train_dl, + valid_dl=valid_dl, + rng=rng, + scaler=scaler, + tb_writer=tb_writer, + world_size=world_size, + rank=rank, + ) + + save_checkpoint( + params=params, + model=model, + model_avg=model_avg, + optimizer=optimizer, + scheduler=scheduler, + sampler=train_dl.sampler, + scaler=scaler, + rank=rank, + ) + + logging.info("Done!") + + if world_size > 1: + torch.distributed.barrier() + cleanup_dist() + + +def display_and_save_batch( + batch: dict, + params: AttributeDict, +) -> None: + """Display the batch statistics and save the batch into disk. + + Args: + batch: + A batch of data. See `lhotse.dataset.K2SpeechRecognitionDataset()` + for the content in it. + params: + Parameters for training. See :func:`get_params`. + """ + from lhotse.utils import uuid4 + + filename = f"{params.exp_dir}/batch-{uuid4()}.pt" + logging.info(f"Saving batch to {filename}") + torch.save(batch, filename) + + +def scan_pessimistic_batches_for_oom( + model: Union[nn.Module, DDP], + tokenizer: TextTokenCollater, + train_dl: torch.utils.data.DataLoader, + optimizer: torch.optim.Optimizer, + params: AttributeDict, +): + from lhotse.dataset import find_pessimistic_batches + + logging.info( + "Sanity check -- see if any of the batches in epoch 1 would cause OOM." + ) + batches, crit_values = find_pessimistic_batches(train_dl.sampler) + + dtype = torch.float32 + if params.dtype in ["bfloat16", "bf16"]: + dtype = torch.bfloat16 + elif params.dtype in ["float16", "fp16"]: + dtype = torch.float16 + + for criterion, cuts in batches.items(): + batch = train_dl.dataset[cuts] + try: + with torch.cuda.amp.autocast(dtype=dtype): + _, loss, _ = compute_loss( + params=params, + model=model, + tokenizer=tokenizer, + batch=batch, + is_training=True, + ) + loss.backward() + optimizer.zero_grad() + except Exception as e: + if "CUDA out of memory" in str(e): + logging.error( + "Your GPU ran out of memory with the current " + "max_duration setting. We recommend decreasing " + "max_duration and trying again.\n" + f"Failing criterion: {criterion} " + f"(={crit_values[criterion]}) ..." + ) + display_and_save_batch(batch, params=params) + raise + logging.info( + f"Maximum memory allocated so far is {torch.cuda.max_memory_allocated()//1000000}MB" + ) + + +def main(): + parser = get_parser() + TtsDataModule.add_arguments(parser) + args = parser.parse_args() + args.exp_dir = Path(args.exp_dir) + + world_size = args.world_size + assert world_size >= 1 + if world_size > 1: + mp.spawn(run, args=(world_size, args), nprocs=world_size, join=True) + else: + run(rank=0, world_size=1, args=args) + + +torch.set_num_threads(1) +torch.set_num_interop_threads(1) + +if __name__ == "__main__": + main() diff --git a/egs/wenetspeech4tts/TTS/valle/tts_datamodule.py b/egs/wenetspeech4tts/TTS/valle/tts_datamodule.py new file mode 100644 index 000000000..8e34d06dc --- /dev/null +++ b/egs/wenetspeech4tts/TTS/valle/tts_datamodule.py @@ -0,0 +1,343 @@ +# Copyright 2021 Piotr Żelasko +# Copyright 2022-2024 Xiaomi Corporation (Authors: Mingshuang Luo, +# Zengwei Yao, +# Zengrui Jin,) +# Copyright 2023 (authors: Feiteng Li) +# Copyright 2024 (Author: Yuekai Zhang) +# See ../../../../LICENSE for clarification regarding multiple authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import argparse +import logging +from functools import lru_cache +from pathlib import Path +from typing import Any, Dict, Optional + +import torch +from lhotse import CutSet, Spectrogram, SpectrogramConfig, load_manifest_lazy +from lhotse.dataset import ( # noqa F401 for PrecomputedFeatures + CutConcatenate, + DynamicBucketingSampler, + PrecomputedFeatures, + SimpleCutSampler, + SpeechSynthesisDataset, +) +from lhotse.dataset.input_strategies import ( # noqa F401 For AudioSamples + AudioSamples, + OnTheFlyFeatures, +) +from lhotse.features.io import KaldiReader +from lhotse.utils import fix_random_seed +from torch.utils.data import DataLoader + +from icefall.utils import str2bool + + +class _SeedWorkers: + def __init__(self, seed: int): + self.seed = seed + + def __call__(self, worker_id: int): + fix_random_seed(self.seed + worker_id) + + +class TtsDataModule: + """ + DataModule for tts experiments. + It assumes there is always one train and valid dataloader, + but there can be multiple test dataloaders (e.g. LibriSpeech test-clean + and test-other). + + It contains all the common data pipeline modules used in TTS + experiments, e.g.: + - dynamic batch size, + - bucketing samplers, + - cut concatenation, + - on-the-fly feature extraction + + This class should be derived for specific corpora used in ASR tasks. + """ + + def __init__(self, args: argparse.Namespace): + self.args = args + + @classmethod + def add_arguments(cls, parser: argparse.ArgumentParser): + group = parser.add_argument_group( + title="TTS data related options", + description="These options are used for the preparation of " + "PyTorch DataLoaders from Lhotse CutSet's -- they control the " + "effective batch sizes, sampling strategies, applied data " + "augmentations, etc.", + ) + group.add_argument( + "--manifest-dir", + type=Path, + default=Path("data/tokenized"), + help="Path to directory with train/valid/test cuts.", + ) + group.add_argument( + "--speaker-embeds", + type=Path, + default=Path("exp/xvector_nnet_1a/"), + help="Path to directory with speaker embeddings.", + ) + group.add_argument( + "--max-duration", + type=int, + default=200.0, + help="Maximum pooled recordings duration (seconds) in a " + "single batch. You can reduce it if it causes CUDA OOM.", + ) + group.add_argument( + "--bucketing-sampler", + type=str2bool, + default=True, + help="When enabled, the batches will come from buckets of " + "similar duration (saves padding frames).", + ) + group.add_argument( + "--num-buckets", + type=int, + default=30, + help="The number of buckets for the DynamicBucketingSampler" + "(you might want to increase it for larger datasets).", + ) + + group.add_argument( + "--on-the-fly-feats", + type=str2bool, + default=False, + help="When enabled, use on-the-fly cut mixing and feature " + "extraction. Will drop existing precomputed feature manifests " + "if available.", + ) + group.add_argument( + "--shuffle", + type=str2bool, + default=True, + help="When enabled (=default), the examples will be " + "shuffled for each epoch.", + ) + group.add_argument( + "--drop-last", + type=str2bool, + default=True, + help="Whether to drop last batch. Used by sampler.", + ) + group.add_argument( + "--return-cuts", + type=str2bool, + default=True, + help="When enabled, each batch will have the " + "field: batch['cut'] with the cuts that " + "were used to construct it.", + ) + group.add_argument( + "--num-workers", + type=int, + default=4, + help="The number of training dataloader workers that " + "collect the batches.", + ) + + group.add_argument( + "--enable-spec-aug", + type=str2bool, + default=False, + help="When enabled, use SpecAugment for training dataset.", + ) + + group.add_argument( + "--input-strategy", + type=str, + default="PrecomputedFeatures", + help="AudioSamples or PrecomputedFeatures", + ) + + group.add_argument( + "--dataset", + type=str, + default="libritts", + help="--input-strategy PromptedPrecomputedFeatures needs dataset name to prepare prompts.", + ) + + parser.add_argument( + "--sampling-rate", + type=int, + default=24000, + help="""Audio sampling rate.""", + ) + + def train_dataloaders( + self, + cuts_train: CutSet, + sampler_state_dict: Optional[Dict[str, Any]] = None, + ) -> DataLoader: + """ + Args: + cuts_train: + CutSet for training. + sampler_state_dict: + The state dict for the training sampler. + """ + logging.info("About to create train dataset") + train = SpeechSynthesisDataset( + return_text=True, + return_tokens=True, + return_spk_ids=False, + feature_input_strategy=eval(self.args.input_strategy)(), + return_cuts=self.args.return_cuts, + ) + + if self.args.on_the_fly_feats: + raise NotImplementedError + + if self.args.bucketing_sampler: + logging.info("Using DynamicBucketingSampler.") + train_sampler = DynamicBucketingSampler( + cuts_train, + max_duration=self.args.max_duration, + shuffle=self.args.shuffle, + num_buckets=self.args.num_buckets, + buffer_size=self.args.num_buckets * 2000, + shuffle_buffer_size=self.args.num_buckets * 5000, + drop_last=self.args.drop_last, + ) + else: + logging.info("Using SimpleCutSampler.") + train_sampler = SimpleCutSampler( + cuts_train, + max_duration=self.args.max_duration, + shuffle=self.args.shuffle, + ) + logging.info("About to create train dataloader") + + if sampler_state_dict is not None: + logging.info("Loading sampler state dict") + train_sampler.load_state_dict(sampler_state_dict) + + # 'seed' is derived from the current random state, which will have + # previously been set in the main process. + seed = torch.randint(0, 100000, ()).item() + worker_init_fn = _SeedWorkers(seed) + + train_dl = DataLoader( + train, + sampler=train_sampler, + batch_size=None, + num_workers=self.args.num_workers, + persistent_workers=False, + worker_init_fn=worker_init_fn, + ) + + return train_dl + + def dev_dataloaders(self, cuts_valid: CutSet) -> DataLoader: + logging.info("About to create dev dataset") + if self.args.on_the_fly_feats: + raise NotImplementedError + else: + validate = SpeechSynthesisDataset( + return_text=True, + return_tokens=True, + return_spk_ids=False, + feature_input_strategy=eval(self.args.input_strategy)(), + return_cuts=self.args.return_cuts, + ) + dev_sampler = DynamicBucketingSampler( + cuts_valid, + max_duration=self.args.max_duration, + shuffle=False, + ) + logging.info("About to create valid dataloader") + dev_dl = DataLoader( + validate, + sampler=dev_sampler, + batch_size=None, + num_workers=self.args.num_workers, + persistent_workers=False, + ) + + return dev_dl + + def test_dataloaders(self, cuts: CutSet) -> DataLoader: + logging.info("About to create test dataset") + if self.args.on_the_fly_feats: + raise NotImplementedError + else: + test = SpeechSynthesisDataset( + return_text=True, + return_tokens=True, + return_spk_ids=False, + feature_input_strategy=eval(self.args.input_strategy)(), + return_cuts=self.args.return_cuts, + ) + test_sampler = DynamicBucketingSampler( + cuts, + max_duration=self.args.max_duration, + shuffle=False, + ) + logging.info("About to create test dataloader") + test_dl = DataLoader( + test, + batch_size=None, + sampler=test_sampler, + num_workers=self.args.num_workers, + ) + return test_dl + + @lru_cache() + def train_cuts(self) -> CutSet: + logging.info("About to get train cuts") + return load_manifest_lazy(self.args.manifest_dir / "cuts_train.jsonl.gz") + + @lru_cache() + def dev_cuts(self) -> CutSet: + logging.info("About to get dev cuts") + return load_manifest_lazy(self.args.manifest_dir / "cuts_dev.jsonl.gz") + + @lru_cache() + def test_cuts(self) -> CutSet: + logging.info("About to get test cuts") + return load_manifest_lazy(self.args.manifest_dir / "cuts_test.jsonl.gz") + + @lru_cache() + def dev_clean_cuts(self) -> CutSet: + logging.info("About to get dev-clean cuts") + return load_manifest_lazy( + self.args.manifest_dir / "libritts_cuts_dev-clean.jsonl.gz" + ) + + @lru_cache() + def dev_other_cuts(self) -> CutSet: + logging.info("About to get dev-other cuts") + return load_manifest_lazy( + self.args.manifest_dir / "libritts_cuts_dev-other.jsonl.gz" + ) + + @lru_cache() + def test_clean_cuts(self) -> CutSet: + logging.info("About to get test-clean cuts") + return load_manifest_lazy( + self.args.manifest_dir / "libritts_cuts_test-clean.jsonl.gz" + ) + + @lru_cache() + def test_other_cuts(self) -> CutSet: + logging.info("About to get test-other cuts") + return load_manifest_lazy( + self.args.manifest_dir / "libritts_cuts_test-other.jsonl.gz" + ) diff --git a/egs/wenetspeech4tts/TTS/valle/valle.py b/egs/wenetspeech4tts/TTS/valle/valle.py new file mode 100644 index 000000000..b2eb8ae69 --- /dev/null +++ b/egs/wenetspeech4tts/TTS/valle/valle.py @@ -0,0 +1,1745 @@ +# Copyright 2023 (authors: Feiteng Li) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy +import math +import numbers +import random +from functools import partial +from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple, Union + +import torch +import torch.nn as nn +from torch import Tensor +from torch.nn import Linear, Module +from torch.nn import functional as F +from torch.nn.init import constant_, xavier_normal_, xavier_uniform_ +from torch.nn.modules.linear import NonDynamicallyQuantizableLinear +from torch.nn.parameter import Parameter +from torchmetrics.classification import MulticlassAccuracy + +from icefall.utils import make_pad_mask + +NUM_TEXT_TOKENS = 5000 +NUM_AUDIO_TOKENS = 1024 # EnCodec RVQ bins + + +class PromptedFeatures: + def __init__(self, prompts, features): + self.prompts = prompts + self.features = features + + def to(self, device): + return PromptedFeatures(self.prompts.to(device), self.features.to(device)) + + def sum(self): + return self.features.sum() + + @property + def ndim(self): + return self.features.ndim + + @property + def data(self): + return (self.prompts, self.features) + + +class TokenEmbedding(nn.Module): + def __init__( + self, + dim_model: int, + vocab_size: int, + dropout: float = 0.0, + ): + super().__init__() + + self.vocab_size = vocab_size + self.dim_model = dim_model + + self.dropout = torch.nn.Dropout(p=dropout) + self.word_embeddings = nn.Embedding(self.vocab_size, self.dim_model) + + @property + def weight(self) -> torch.Tensor: + return self.word_embeddings.weight + + def embedding(self, index: int) -> torch.Tensor: + return self.word_embeddings.weight[index : index + 1] + + def forward(self, x: torch.Tensor): + X = self.word_embeddings(x) + X = self.dropout(X) + + return X + + +class SinePositionalEmbedding(nn.Module): + def __init__( + self, + dim_model: int, + dropout: float = 0.0, + scale: bool = False, + alpha: bool = False, + ): + super().__init__() + self.dim_model = dim_model + self.x_scale = math.sqrt(dim_model) if scale else 1.0 + self.alpha = nn.Parameter(torch.ones(1), requires_grad=alpha) + self.dropout = torch.nn.Dropout(p=dropout) + + self.reverse = False + self.pe = None + self.extend_pe(torch.tensor(0.0).expand(1, 4000)) + + def extend_pe(self, x): + """Reset the positional encodings.""" + if self.pe is not None: + if self.pe.size(1) >= x.size(1): + if self.pe.dtype != x.dtype or self.pe.device != x.device: + self.pe = self.pe.to(dtype=x.dtype, device=x.device) + return + pe = torch.zeros(x.size(1), self.dim_model) + if self.reverse: + position = torch.arange( + x.size(1) - 1, -1, -1.0, dtype=torch.float32 + ).unsqueeze(1) + else: + position = torch.arange(0, x.size(1), dtype=torch.float32).unsqueeze(1) + div_term = torch.exp( + torch.arange(0, self.dim_model, 2, dtype=torch.float32) + * -(math.log(10000.0) / self.dim_model) + ) + pe[:, 0::2] = torch.sin(position * div_term) + pe[:, 1::2] = torch.cos(position * div_term) + pe = pe.unsqueeze(0) + self.pe = pe.to(device=x.device, dtype=x.dtype).detach() + + def forward(self, x: torch.Tensor) -> torch.Tensor: + self.extend_pe(x) + output = x.unsqueeze(-1) if x.ndim == 2 else x + output = output * self.x_scale + self.alpha * self.pe[:, : x.size(1)] + return self.dropout(output) + + +class Transpose(nn.Identity): + """(N, T, D) -> (N, D, T)""" + + def forward(self, input: torch.Tensor) -> torch.Tensor: + return input.transpose(1, 2) + + +_shape_t = Union[int, List[int], torch.Size] + + +class MultiheadAttention(Module): + r"""Allows the model to jointly attend to information + from different representation subspaces as described in the paper: + `Attention Is All You Need `_. + + Multi-Head Attention is defined as: + + .. math:: + \text{MultiHead}(Q, K, V) = \text{Concat}(head_1,\dots,head_h)W^O + + where :math:`head_i = \text{Attention}(QW_i^Q, KW_i^K, VW_i^V)`. + + ``forward()`` will use a special optimized implementation if all of the following + conditions are met: + + - self attention is being computed (i.e., ``query``, ``key``, and ``value`` are the same tensor. This + restriction will be loosened in the future.) + - Either autograd is disabled (using ``torch.inference_mode`` or ``torch.no_grad``) or no tensor argument ``requires_grad`` + - training is disabled (using ``.eval()``) + - dropout is 0 + - ``add_bias_kv`` is ``False`` + - ``add_zero_attn`` is ``False`` + - ``batch_first`` is ``True`` and the input is batched + - ``kdim`` and ``vdim`` are equal to ``embed_dim`` + - at most one of ``key_padding_mask`` or ``attn_mask`` is passed + - if a `NestedTensor `_ is passed, neither ``key_padding_mask`` + nor ``attn_mask`` is passed + + If the optimized implementation is in use, a + `NestedTensor `_ can be passed for + ``query``/``key``/``value`` to represent padding more efficiently than using a + padding mask. In this case, a `NestedTensor `_ + will be returned, and an additional speedup proportional to the fraction of the input + that is padding can be expected. + + Args: + embed_dim: Total dimension of the model. + num_heads: Number of parallel attention heads. Note that ``embed_dim`` will be split + across ``num_heads`` (i.e. each head will have dimension ``embed_dim // num_heads``). + dropout: Dropout probability on ``attn_output_weights``. Default: ``0.0`` (no dropout). + bias: If specified, adds bias to input / output projection layers. Default: ``True``. + add_bias_kv: If specified, adds bias to the key and value sequences at dim=0. Default: ``False``. + add_zero_attn: If specified, adds a new batch of zeros to the key and value sequences at dim=1. + Default: ``False``. + kdim: Total number of features for keys. Default: ``None`` (uses ``kdim=embed_dim``). + vdim: Total number of features for values. Default: ``None`` (uses ``vdim=embed_dim``). + batch_first: If ``True``, then the input and output tensors are provided + as (batch, seq, feature). Default: ``False`` (seq, batch, feature). + + Examples:: + + >>> # xdoctest: +SKIP + >>> multihead_attn = nn.MultiheadAttention(embed_dim, num_heads) + >>> attn_output, attn_output_weights = multihead_attn(query, key, value) + + """ + __constants__ = ["batch_first"] + bias_k: Optional[torch.Tensor] + bias_v: Optional[torch.Tensor] + + def __init__( + self, + embed_dim, + num_heads, + dropout=0.0, + bias=True, + add_bias_kv=False, + add_zero_attn=False, + kdim=None, + vdim=None, + batch_first=False, + linear1_cls=Linear, + linear2_cls=Linear, + device=None, + dtype=None, + ) -> None: + factory_kwargs = {"device": device, "dtype": dtype} + super(MultiheadAttention, self).__init__() + self.embed_dim = embed_dim + self.kdim = kdim if kdim is not None else embed_dim + self.vdim = vdim if vdim is not None else embed_dim + self._qkv_same_embed_dim = self.kdim == embed_dim and self.vdim == embed_dim + + self.num_heads = num_heads + self.dropout = dropout + self.batch_first = batch_first + self.head_dim = embed_dim // num_heads + assert ( + self.head_dim * num_heads == self.embed_dim + ), "embed_dim must be divisible by num_heads" + + if add_bias_kv: + self.bias_k = Parameter(torch.empty((1, 1, embed_dim), **factory_kwargs)) + self.bias_v = Parameter(torch.empty((1, 1, embed_dim), **factory_kwargs)) + else: + self.bias_k = self.bias_v = None + + if linear1_cls == Linear: + if not self._qkv_same_embed_dim: + self.q_proj_weight = Parameter( + torch.empty((embed_dim, embed_dim), **factory_kwargs) + ) + self.k_proj_weight = Parameter( + torch.empty((embed_dim, self.kdim), **factory_kwargs) + ) + self.v_proj_weight = Parameter( + torch.empty((embed_dim, self.vdim), **factory_kwargs) + ) + self.register_parameter("in_proj_weight", None) + else: + self.in_proj_weight = Parameter( + torch.empty((3 * embed_dim, embed_dim), **factory_kwargs) + ) + self.register_parameter("q_proj_weight", None) + self.register_parameter("k_proj_weight", None) + self.register_parameter("v_proj_weight", None) + + if bias: + self.in_proj_bias = Parameter( + torch.empty(3 * embed_dim, **factory_kwargs) + ) + else: + self.register_parameter("in_proj_bias", None) + self.out_proj = NonDynamicallyQuantizableLinear( + embed_dim, embed_dim, bias=bias, **factory_kwargs + ) + + self._reset_parameters() + else: + if not self._qkv_same_embed_dim: + raise NotImplementedError + else: + self.in_proj_linear = linear1_cls( + embed_dim, 3 * embed_dim, bias=bias, **factory_kwargs + ) + self.in_proj_weight = self.in_proj_linear.weight + + self.register_parameter("q_proj_weight", None) + self.register_parameter("k_proj_weight", None) + self.register_parameter("v_proj_weight", None) + + if bias: + self.in_proj_bias = self.in_proj_linear.bias + else: + self.register_parameter("in_proj_bias", None) + + self.out_proj = linear2_cls( + embed_dim, embed_dim, bias=bias, **factory_kwargs + ) + + if self.bias_k is not None: + xavier_normal_(self.bias_k) + if self.bias_v is not None: + xavier_normal_(self.bias_v) + + self.add_zero_attn = add_zero_attn + + def _reset_parameters(self): + if self._qkv_same_embed_dim: + xavier_uniform_(self.in_proj_weight) + else: + xavier_uniform_(self.q_proj_weight) + xavier_uniform_(self.k_proj_weight) + xavier_uniform_(self.v_proj_weight) + + if self.in_proj_bias is not None: + constant_(self.in_proj_bias, 0.0) + constant_(self.out_proj.bias, 0.0) + + if self.bias_k is not None: + xavier_normal_(self.bias_k) + if self.bias_v is not None: + xavier_normal_(self.bias_v) + + def __setstate__(self, state): + # Support loading old MultiheadAttention checkpoints generated by v1.1.0 + if "_qkv_same_embed_dim" not in state: + state["_qkv_same_embed_dim"] = True + + super(MultiheadAttention, self).__setstate__(state) + + def forward( + self, + query: Tensor, + key: Tensor, + value: Tensor, + key_padding_mask: Optional[Tensor] = None, + need_weights: bool = True, + attn_mask: Optional[Tensor] = None, + average_attn_weights: bool = True, + ) -> Tuple[Tensor, Optional[Tensor]]: + r""" + Args: + query: Query embeddings of shape :math:`(L, E_q)` for unbatched input, :math:`(L, N, E_q)` when ``batch_first=False`` + or :math:`(N, L, E_q)` when ``batch_first=True``, where :math:`L` is the target sequence length, + :math:`N` is the batch size, and :math:`E_q` is the query embedding dimension ``embed_dim``. + Queries are compared against key-value pairs to produce the output. + See "Attention Is All You Need" for more details. + key: Key embeddings of shape :math:`(S, E_k)` for unbatched input, :math:`(S, N, E_k)` when ``batch_first=False`` + or :math:`(N, S, E_k)` when ``batch_first=True``, where :math:`S` is the source sequence length, + :math:`N` is the batch size, and :math:`E_k` is the key embedding dimension ``kdim``. + See "Attention Is All You Need" for more details. + value: Value embeddings of shape :math:`(S, E_v)` for unbatched input, :math:`(S, N, E_v)` when + ``batch_first=False`` or :math:`(N, S, E_v)` when ``batch_first=True``, where :math:`S` is the source + sequence length, :math:`N` is the batch size, and :math:`E_v` is the value embedding dimension ``vdim``. + See "Attention Is All You Need" for more details. + key_padding_mask: If specified, a mask of shape :math:`(N, S)` indicating which elements within ``key`` + to ignore for the purpose of attention (i.e. treat as "padding"). For unbatched `query`, shape should be :math:`(S)`. + Binary and byte masks are supported. + For a binary mask, a ``True`` value indicates that the corresponding ``key`` value will be ignored for + the purpose of attention. For a float mask, it will be directly added to the corresponding ``key`` value. + need_weights: If specified, returns ``attn_output_weights`` in addition to ``attn_outputs``. + Default: ``True``. + attn_mask: If specified, a 2D or 3D mask preventing attention to certain positions. Must be of shape + :math:`(L, S)` or :math:`(N\cdot\text{num\_heads}, L, S)`, where :math:`N` is the batch size, + :math:`L` is the target sequence length, and :math:`S` is the source sequence length. A 2D mask will be + broadcasted across the batch while a 3D mask allows for a different mask for each entry in the batch. + Binary, byte, and float masks are supported. For a binary mask, a ``True`` value indicates that the + corresponding position is not allowed to attend. For a byte mask, a non-zero value indicates that the + corresponding position is not allowed to attend. For a float mask, the mask values will be added to + the attention weight. + average_attn_weights: If true, indicates that the returned ``attn_weights`` should be averaged across + heads. Otherwise, ``attn_weights`` are provided separately per head. Note that this flag only has an + effect when ``need_weights=True``. Default: ``True`` (i.e. average weights across heads) + + Outputs: + - **attn_output** - Attention outputs of shape :math:`(L, E)` when input is unbatched, + :math:`(L, N, E)` when ``batch_first=False`` or :math:`(N, L, E)` when ``batch_first=True``, + where :math:`L` is the target sequence length, :math:`N` is the batch size, and :math:`E` is the + embedding dimension ``embed_dim``. + - **attn_output_weights** - Only returned when ``need_weights=True``. If ``average_attn_weights=True``, + returns attention weights averaged across heads of shape :math:`(L, S)` when input is unbatched or + :math:`(N, L, S)`, where :math:`N` is the batch size, :math:`L` is the target sequence length, and + :math:`S` is the source sequence length. If ``average_attn_weights=False``, returns attention weights per + head of shape :math:`(\text{num\_heads}, L, S)` when input is unbatched or :math:`(N, \text{num\_heads}, L, S)`. + + .. note:: + `batch_first` argument is ignored for unbatched inputs. + """ + is_batched = query.dim() == 3 + if key_padding_mask is not None: + _kpm_dtype = key_padding_mask.dtype + if _kpm_dtype != torch.bool and not torch.is_floating_point( + key_padding_mask + ): + raise AssertionError( + "only bool and floating types of key_padding_mask are supported" + ) + why_not_fast_path = "" + if not is_batched: + why_not_fast_path = ( + f"input not batched; expected query.dim() of 3 but got {query.dim()}" + ) + elif query is not key or key is not value: + # When lifting this restriction, don't forget to either + # enforce that the dtypes all match or test cases where + # they don't! + why_not_fast_path = "non-self attention was used (query, key, and value are not the same Tensor)" + elif self.in_proj_bias is not None and query.dtype != self.in_proj_bias.dtype: + why_not_fast_path = f"dtypes of query ({query.dtype}) and self.in_proj_bias ({self.in_proj_bias.dtype}) don't match" + elif ( + self.in_proj_weight is not None and query.dtype != self.in_proj_weight.dtype + ): + # this case will fail anyway, but at least they'll get a useful error message. + why_not_fast_path = f"dtypes of query ({query.dtype}) and self.in_proj_weight ({self.in_proj_weight.dtype}) don't match" + elif self.training: + why_not_fast_path = "training is enabled" + elif not self.batch_first: + why_not_fast_path = "batch_first was not True" + elif self.bias_k is not None: + why_not_fast_path = "self.bias_k was not None" + elif self.bias_v is not None: + why_not_fast_path = "self.bias_v was not None" + elif self.dropout: + why_not_fast_path = f"dropout was {self.dropout}, required zero" + elif self.add_zero_attn: + why_not_fast_path = "add_zero_attn was enabled" + elif not self._qkv_same_embed_dim: + why_not_fast_path = "_qkv_same_embed_dim was not True" + elif attn_mask is not None: + why_not_fast_path = "attn_mask was not None" + elif query.is_nested and key_padding_mask is not None: + why_not_fast_path = ( + "key_padding_mask is not supported with NestedTensor input" + ) + elif self.num_heads % 2 == 1: + why_not_fast_path = "num_heads is odd" + elif torch.is_autocast_enabled(): + why_not_fast_path = "autocast is enabled" + + if not why_not_fast_path: + tensor_args = ( + query, + key, + value, + self.in_proj_weight, + self.in_proj_bias, + self.out_proj.weight, + self.out_proj.bias, + ) + # We have to use list comprehensions below because TorchScript does not support + # generator expressions. + if torch.overrides.has_torch_function(tensor_args): + why_not_fast_path = "some Tensor argument has_torch_function" + elif not all( + [ + (x is None or x.is_cuda or "cpu" in str(x.device)) + for x in tensor_args + ] + ): + why_not_fast_path = "some Tensor argument is neither CUDA nor CPU" + elif torch.is_grad_enabled() and any( + [x is not None and x.requires_grad for x in tensor_args] + ): + why_not_fast_path = ( + "grad is enabled and at least one of query or the " + "input/output projection weights or biases requires_grad" + ) + if not why_not_fast_path: + return torch._native_multi_head_attention( + query, + key, + value, + self.embed_dim, + self.num_heads, + self.in_proj_weight, + self.in_proj_bias, + self.out_proj.weight, + self.out_proj.bias, + key_padding_mask if key_padding_mask is not None else attn_mask, + need_weights, + average_attn_weights, + 1 + if key_padding_mask is not None + else 0 + if attn_mask is not None + else None, + ) + + any_nested = query.is_nested or key.is_nested or value.is_nested + assert not any_nested, ( + "MultiheadAttention does not support NestedTensor outside of its fast path. " + + f"The fast path was not hit because {why_not_fast_path}" + ) + + if self.batch_first and is_batched: + # make sure that the transpose op does not affect the "is" property + if key is value: + if query is key: + query = key = value = query.transpose(1, 0) + else: + query, key = [x.transpose(1, 0) for x in (query, key)] + value = key + else: + query, key, value = [x.transpose(1, 0) for x in (query, key, value)] + + if not self._qkv_same_embed_dim: + attn_output, attn_output_weights = F.multi_head_attention_forward( + query, + key, + value, + self.embed_dim, + self.num_heads, + self.in_proj_weight, + self.in_proj_bias, + self.bias_k, + self.bias_v, + self.add_zero_attn, + self.dropout, + self.out_proj.weight, + self.out_proj.bias, + training=self.training, + key_padding_mask=key_padding_mask, + need_weights=need_weights, + attn_mask=attn_mask, + use_separate_proj_weight=True, + q_proj_weight=self.q_proj_weight, + k_proj_weight=self.k_proj_weight, + v_proj_weight=self.v_proj_weight, + average_attn_weights=average_attn_weights, + ) + else: + attn_output, attn_output_weights = F.multi_head_attention_forward( + query, + key, + value, + self.embed_dim, + self.num_heads, + self.in_proj_weight, + self.in_proj_bias, + self.bias_k, + self.bias_v, + self.add_zero_attn, + self.dropout, + self.out_proj.weight, + self.out_proj.bias, + training=self.training, + key_padding_mask=key_padding_mask, + need_weights=need_weights, + attn_mask=attn_mask, + average_attn_weights=average_attn_weights, + ) + if self.batch_first and is_batched: + return attn_output.transpose(1, 0), attn_output_weights + else: + return attn_output, attn_output_weights + + +class LayerNorm(nn.Module): + __constants__ = ["normalized_shape", "eps", "elementwise_affine"] + normalized_shape: Tuple[int, ...] + eps: float + elementwise_affine: bool + + def __init__( + self, + normalized_shape: _shape_t, + eps: float = 1e-5, + elementwise_affine: bool = True, + device=None, + dtype=None, + ) -> None: + factory_kwargs = {"device": device, "dtype": dtype} + super(LayerNorm, self).__init__() + if isinstance(normalized_shape, numbers.Integral): + # mypy error: incompatible types in assignment + normalized_shape = (normalized_shape,) # type: ignore[assignment] + self.normalized_shape = tuple(normalized_shape) # type: ignore[arg-type] + self.eps = eps + self.elementwise_affine = elementwise_affine + if self.elementwise_affine: + self.weight = nn.Parameter( + torch.empty(self.normalized_shape, **factory_kwargs) + ) + self.bias = nn.Parameter( + torch.empty(self.normalized_shape, **factory_kwargs) + ) + else: + self.register_parameter("weight", None) + self.register_parameter("bias", None) + + self.reset_parameters() + + def reset_parameters(self) -> None: + if self.elementwise_affine: + nn.init.ones_(self.weight) + nn.init.zeros_(self.bias) + + def forward(self, input: Tensor, embedding: Any = None) -> Tensor: + if isinstance(input, tuple): + input, embedding = input + return ( + F.layer_norm( + input, + self.normalized_shape, + self.weight, + self.bias, + self.eps, + ), + embedding, + ) + + assert embedding is None + return F.layer_norm( + input, self.normalized_shape, self.weight, self.bias, self.eps + ) + + def extra_repr(self) -> str: + return ( + "{normalized_shape}, eps={eps}, " + "elementwise_affine={elementwise_affine}".format(**self.__dict__) + ) + + +class AdaptiveLayerNorm(nn.Module): + r"""Adaptive Layer Normalization""" + + def __init__(self, d_model, norm) -> None: + super(AdaptiveLayerNorm, self).__init__() + self.project_layer = nn.Linear(d_model, 2 * d_model) + self.norm = norm + self.d_model = d_model + self.eps = self.norm.eps + + def forward(self, input: Tensor, embedding: Tensor = None) -> Tensor: + if isinstance(input, tuple): + input, embedding = input + weight, bias = torch.split( + self.project_layer(embedding), + split_size_or_sections=self.d_model, + dim=-1, + ) + return (weight * self.norm(input) + bias, embedding) + + weight, bias = torch.split( + self.project_layer(embedding), + split_size_or_sections=self.d_model, + dim=-1, + ) + return weight * self.norm(input) + bias + + +class TransformerEncoderLayer(nn.Module): + __constants__ = ["batch_first", "norm_first"] + + def __init__( + self, + d_model: int, + nhead: int, + dim_feedforward: int = 2048, + dropout: float = 0.1, + activation: Union[str, Callable[[Tensor], Tensor]] = F.relu, + batch_first: bool = False, + norm_first: bool = False, + device=None, + dtype=None, + linear1_self_attention_cls: nn.Module = nn.Linear, + linear2_self_attention_cls: nn.Module = nn.Linear, + linear1_feedforward_cls: nn.Module = nn.Linear, + linear2_feedforward_cls: nn.Module = nn.Linear, + layer_norm_cls: nn.Module = LayerNorm, + layer_norm_eps: float = 1e-5, + adaptive_layer_norm=False, + ) -> None: + factory_kwargs = {"device": device, "dtype": dtype} + super(TransformerEncoderLayer, self).__init__() + self.self_attn = MultiheadAttention( + d_model, + nhead, + dropout=dropout, + batch_first=batch_first, + linear1_cls=linear1_self_attention_cls, + linear2_cls=linear2_self_attention_cls, + **factory_kwargs, + ) + + # Implementation of Feedforward model + self.linear1 = linear1_feedforward_cls( + d_model, dim_feedforward, **factory_kwargs + ) + self.dropout = nn.Dropout(dropout) + self.linear2 = linear2_feedforward_cls( + dim_feedforward, d_model, **factory_kwargs + ) + + self.norm_first = norm_first + self.dropout1 = nn.Dropout(dropout) + self.dropout2 = nn.Dropout(dropout) + + # Legacy string support for activation function. + if isinstance(activation, str): + activation = _get_activation_fn(activation) + elif isinstance(activation, partial): + activation = activation(d_model) + # elif activation == BalancedDoubleSwish: + # activation = BalancedDoubleSwish(d_model) + + # # We can't test self.activation in forward() in TorchScript, + # # so stash some information about it instead. + # if activation is F.relu or isinstance(activation, torch.nn.ReLU): + # self.activation_relu_or_gelu = 1 + # elif activation is F.gelu or isinstance(activation, torch.nn.GELU): + # self.activation_relu_or_gelu = 2 + # else: + # self.activation_relu_or_gelu = 0 + self.activation = activation + + norm1 = layer_norm_cls(d_model, eps=layer_norm_eps, **factory_kwargs) + # if layer_norm_cls == IdentityNorm: + # norm2 = BalancedBasicNorm(d_model, eps=layer_norm_eps, **factory_kwargs) + # else: + if True: + norm2 = layer_norm_cls(d_model, eps=layer_norm_eps, **factory_kwargs) + + if adaptive_layer_norm: + self.norm1 = AdaptiveLayerNorm(d_model, norm1) + self.norm2 = AdaptiveLayerNorm(d_model, norm2) + else: + self.norm1 = norm1 + self.norm2 = norm2 + + def __setstate__(self, state): + super(TransformerEncoderLayer, self).__setstate__(state) + if not hasattr(self, "activation"): + self.activation = F.relu + + def forward( + self, + src: Tensor, + src_mask: Optional[Tensor] = None, + src_key_padding_mask: Optional[Tensor] = None, + ) -> Tensor: + r"""Pass the input through the encoder layer. + + Args: + src: the sequence to the encoder layer (required). + src_mask: the mask for the src sequence (optional). + src_key_padding_mask: the mask for the src keys per batch (optional). + + Shape: + see the docs in Transformer class. + """ + x, stage_embedding = src, None + is_src_tuple = False + if isinstance(src, tuple): + x, stage_embedding = src + is_src_tuple = True + + if src_key_padding_mask is not None: + _skpm_dtype = src_key_padding_mask.dtype + if _skpm_dtype != torch.bool and not torch.is_floating_point( + src_key_padding_mask + ): + raise AssertionError( + "only bool and floating types of key_padding_mask are supported" + ) + + if self.norm_first: + x = x + self._sa_block( + self.norm1(x, stage_embedding), + src_mask, + src_key_padding_mask, + ) + x = x + self._ff_block(self.norm2(x, stage_embedding)) + else: + x = self.norm1( + x + self._sa_block(x, src_mask, src_key_padding_mask), + stage_embedding, + ) + x = self.norm2(x + self._ff_block(x), stage_embedding) + + if is_src_tuple: + return (x, stage_embedding) + return x + + # self-attention block + def _sa_block( + self, + x: Tensor, + attn_mask: Optional[Tensor], + key_padding_mask: Optional[Tensor], + ) -> Tensor: + x = self.self_attn( + x, + x, + x, + attn_mask=attn_mask, + key_padding_mask=key_padding_mask, + need_weights=False, + )[0] + return self.dropout1(x) + + # feed forward block + def _ff_block(self, x: Tensor) -> Tensor: + x = self.linear2(self.dropout(self.activation(self.linear1(x)))) + return self.dropout2(x) + + +class TransformerEncoder(nn.Module): + r"""TransformerEncoder is a stack of N encoder layers. Users can build the + BERT(https://arxiv.org/abs/1810.04805) model with corresponding parameters. + + Args: + encoder_layer: an instance of the TransformerEncoderLayer() class (required). + num_layers: the number of sub-encoder-layers in the encoder (required). + norm: the layer normalization component (optional). + enable_nested_tensor: if True, input will automatically convert to nested tensor + (and convert back on output). This will improve the overall performance of + TransformerEncoder when padding rate is high. Default: ``True`` (enabled). + + Examples:: + >>> encoder_layer = TransformerEncoderLayer(d_model=512, nhead=8) + >>> transformer_encoder = TransformerEncoder(encoder_layer, num_layers=6) + >>> src = torch.rand(10, 32, 512) + >>> out = transformer_encoder(src) + """ + __constants__ = ["norm"] + + def __init__(self, encoder_layer, num_layers, norm=None): + super(TransformerEncoder, self).__init__() + self.layers = _get_clones(encoder_layer, num_layers) + self.num_layers = num_layers + self.norm = norm + + def forward( + self, + src: Tensor, + mask: Optional[Tensor] = None, + src_key_padding_mask: Optional[Tensor] = None, + return_layer_states: bool = False, + ) -> Tensor: + r"""Pass the input through the encoder layers in turn. + + Args: + src: the sequence to the encoder (required). + mask: the mask for the src sequence (optional). + src_key_padding_mask: the mask for the src keys per batch (optional). + return_layer_states: return layers' state (optional). + + Shape: + see the docs in Transformer class. + """ + if return_layer_states: + layer_states = [] # layers' output + output = src + for mod in self.layers: + output = mod( + output, + src_mask=mask, + src_key_padding_mask=src_key_padding_mask, + ) + layer_states.append(output[0]) + + if self.norm is not None: + output = self.norm(output) + + return layer_states, output + + output = src + for mod in self.layers: + output = mod( + output, src_mask=mask, src_key_padding_mask=src_key_padding_mask + ) + + if self.norm is not None: + output = self.norm(output) + + return output + + +def _get_clones(module, N): + return nn.ModuleList([copy.deepcopy(module) for i in range(N)]) + + +def _get_activation_fn(activation: str) -> Callable[[Tensor], Tensor]: + if activation == "relu": + return F.relu + elif activation == "gelu": + return F.gelu + + raise RuntimeError("activation should be relu/gelu, not {}".format(activation)) + + +class VALLE(nn.Module): + """It implements https://arxiv.org/abs/2301.02111 + "Neural Codec Language Models are Zero-Shot Text to Speech Synthesizers" + """ + + def __init__( + self, + d_model: int, + nhead: int, + num_layers: int, + norm_first: bool = True, + add_prenet: bool = False, + decoder_cls=TransformerEncoder, + decoder_layer_cls=TransformerEncoderLayer, + prefix_mode: int = 0, + share_embedding: bool = True, + nar_scale_factor: float = 1.0, + prepend_bos: bool = False, + num_quantizers: int = 8, + **kwargs, + ): + """ + Args: + d_model: + The number of expected features in the input (required). + nhead: + The number of heads in the multiheadattention models (required). + num_layers: + The number of sub-decoder-layers in the decoder (required). + """ + super().__init__() + nar_d_model = int(d_model * nar_scale_factor) + + self.ar_text_embedding = TokenEmbedding(d_model, NUM_TEXT_TOKENS) # W_x + self.nar_text_embedding = TokenEmbedding(nar_d_model, NUM_TEXT_TOKENS) + + # ID NUM_AUDIO_TOKENS -> PAD + # ID NUM_AUDIO_TOKENS + 1 -> BOS + self.ar_audio_prepend_bos = prepend_bos + self.ar_audio_embedding = TokenEmbedding( + d_model, NUM_AUDIO_TOKENS + 1 + int(prepend_bos) + ) + + # PreNet + if add_prenet: + self.ar_text_prenet = nn.Sequential( + Transpose(), + nn.Conv1d(d_model, d_model, kernel_size=5, padding="same"), + nn.BatchNorm1d(d_model), + nn.ReLU(), + nn.Dropout(0.5), + nn.Conv1d(d_model, d_model, kernel_size=5, padding="same"), + nn.BatchNorm1d(d_model), + nn.ReLU(), + nn.Dropout(0.5), + nn.Conv1d(d_model, d_model, kernel_size=5, padding="same"), + nn.BatchNorm1d(d_model), + nn.ReLU(), + nn.Dropout(0.5), + Transpose(), + nn.Linear(d_model, d_model), + ) + + self.ar_audio_prenet = nn.Sequential( + nn.Linear(d_model, 256), + nn.ReLU(), + nn.Dropout(0.25), + nn.Linear(256, 256), + nn.ReLU(), + nn.Dropout(0.25), + nn.Linear(256, d_model), + ) + else: + self.ar_text_prenet = nn.Identity() + self.ar_audio_prenet = nn.Identity() + + self.ar_text_position = SinePositionalEmbedding( + d_model, + dropout=0.1, + scale=False, + alpha=True, + ) + self.ar_audio_position = SinePositionalEmbedding( + d_model, + dropout=0.1, + scale=False, + alpha=True, + ) + + self.ar_decoder = decoder_cls( + decoder_layer_cls( + d_model, + nhead, + dim_feedforward=d_model * 4, + dropout=0.1, + batch_first=True, + norm_first=norm_first, + ), + num_layers=num_layers, + norm=LayerNorm(d_model) if norm_first else None, + ) + self.ar_predict_layer = nn.Linear(d_model, NUM_AUDIO_TOKENS + 1, bias=False) + + self.ar_accuracy_metric = MulticlassAccuracy( + NUM_AUDIO_TOKENS + 1, + top_k=10, + average="micro", + multidim_average="global", + ignore_index=NUM_AUDIO_TOKENS, + ) + + self.rng = random.Random(0) + self.num_heads = nhead + self.prefix_mode = prefix_mode + self.num_quantizers = num_quantizers + + assert num_quantizers >= 1 + if num_quantizers > 1: + self.nar_audio_embeddings = nn.ModuleList( + [TokenEmbedding(nar_d_model, NUM_AUDIO_TOKENS + 1)] + + [ + TokenEmbedding(nar_d_model, NUM_AUDIO_TOKENS) + for i in range(num_quantizers - 1) + ] + ) # W_a + + # PreNet + if add_prenet: + self.nar_text_prenet = nn.Sequential( + Transpose(), + nn.Conv1d(nar_d_model, nar_d_model, kernel_size=5, padding="same"), + nn.BatchNorm1d(nar_d_model), + nn.ReLU(), + nn.Dropout(0.5), + nn.Conv1d(nar_d_model, nar_d_model, kernel_size=5, padding="same"), + nn.BatchNorm1d(nar_d_model), + nn.ReLU(), + nn.Dropout(0.5), + nn.Conv1d(nar_d_model, nar_d_model, kernel_size=5, padding="same"), + nn.BatchNorm1d(nar_d_model), + nn.ReLU(), + nn.Dropout(0.5), + Transpose(), + nn.Linear(nar_d_model, nar_d_model), + ) + self.nar_audio_prenet = nn.Sequential( + nn.Linear(nar_d_model, 256), + nn.ReLU(), + nn.Dropout(0.25), + nn.Linear(256, 256), + nn.ReLU(), + nn.Dropout(0.25), + nn.Linear(256, nar_d_model), + ) + else: + self.nar_text_prenet = nn.Identity() + self.nar_audio_prenet = nn.Identity() + + self.nar_text_position = SinePositionalEmbedding( + nar_d_model, + dropout=0.0, + scale=False, + alpha=False, + ) + self.nar_audio_position = SinePositionalEmbedding( + nar_d_model, + dropout=0.1, + scale=False, + alpha=False, + ) + + self.nar_decoder = decoder_cls( + decoder_layer_cls( + nar_d_model, + int(nhead * nar_scale_factor), + dim_feedforward=nar_d_model * 4, + dropout=0.1, + batch_first=True, + norm_first=norm_first, + adaptive_layer_norm=True, + ), + num_layers=int(num_layers * nar_scale_factor), + norm=AdaptiveLayerNorm(nar_d_model, norm=nn.LayerNorm(nar_d_model)) + if norm_first + else None, + ) + self.nar_predict_layers = nn.ModuleList( + [ + nn.Linear(nar_d_model, NUM_AUDIO_TOKENS, bias=False) + for i in range(num_quantizers - 1) + ] + ) + self.nar_stage_embeddings = nn.ModuleList( + [TokenEmbedding(nar_d_model, 1) for i in range(num_quantizers - 1)] + ) + + if share_embedding: + # We share the parameters of the output projection layer with the parameters of the acoustic embedding Wa + # NOTE(Feiteng): In the experiment, this undermines accuracy + # self.ar_predict_layer.weight = self.ar_audio_embedding.weight + + # We also share the parameters of the acoustic embedding layer and the output prediction layer, + # which means the weights of the j-th prediction layer are the same as the (j + 1)-th acoustic embedding layer. + for j in range(0, num_quantizers - 2): + self.nar_predict_layers[j].weight = self.nar_audio_embeddings[ + j + 2 + ].weight + + self.nar_accuracy_metric = MulticlassAccuracy( + NUM_AUDIO_TOKENS + 1, + top_k=10, + average="micro", + multidim_average="global", + ignore_index=NUM_AUDIO_TOKENS, + ) + + def stage_parameters(self, stage: int = 1) -> Iterator[nn.Parameter]: + assert stage > 0 + if stage == 1: + for name, param in self.named_parameters(): + if name.startswith("ar_"): + print(f" AR parameter: {name}") + yield param + + if stage == 2: + for name, param in self.named_parameters(): + if name.startswith("nar_"): + print(f"NAR parameter: {name}") + yield param + + def stage_named_parameters( + self, stage: int = 1 + ) -> Iterator[Tuple[str, nn.Parameter]]: + assert stage > 0 + if stage == 1: + for pair in self.named_parameters(): + if pair[0].startswith("ar_"): + yield pair + + if stage == 2: + for pair in self.named_parameters(): + if pair[0].startswith("nar_"): + yield pair + + def pad_y_eos(self, y, y_mask_int, eos_id): + targets = F.pad(y, (0, 1), value=0) + eos_id * F.pad( + y_mask_int, (0, 1), value=1 + ) + # inputs, targets + if self.ar_audio_prepend_bos: + return ( + F.pad(targets[:, :-1], (1, 0), value=NUM_AUDIO_TOKENS + 1), + targets, + ) + + return targets[:, :-1], targets[:, 1:] + + def _prepare_prompts(self, y, y_lens, codes, nar_stage, y_prompts_codes): + # 5.1 For the NAR acoustic prompt tokens, we select a random segment waveform of 3 seconds + # from the same utterance. + # We implement this differently. + if self.prefix_mode == 0: + # no prefix + prefix_len = 0 + y_emb = self.nar_audio_embeddings[0](y) + for j in range(1, nar_stage): + # Formula (4) (5) + y_emb = y_emb + self.nar_audio_embeddings[j](codes[..., j]) + elif self.prefix_mode == 1: + # prefix at begining + int_low = (0.25 * y_lens.min()).type(torch.int64).item() + prefix_len = torch.randint(int_low, int_low * 2, size=()).item() + prefix_len = min(prefix_len, 225) # 24000/320 * 3s = 225 frames + + y_prompts = self.nar_audio_embeddings[0](y[:, :prefix_len]) + y_emb = self.nar_audio_embeddings[0](y[:, prefix_len:]) + for j in range(1, self.num_quantizers): + y_prompts += self.nar_audio_embeddings[j](codes[:, :prefix_len, j]) + if j < nar_stage: + y_emb += self.nar_audio_embeddings[j](codes[:, prefix_len:, j]) + y_emb = torch.concat([y_prompts, y_emb], axis=1) + elif self.prefix_mode in [2, 4]: + if self.prefix_mode == 2: + # random prefix + prefix_len = min(225, int(0.25 * y_lens.min().item())) + + y_prompts_codes = [] + for b in range(codes.shape[0]): + start = self.rng.randint(0, y_lens[b].item() - prefix_len) + y_prompts_codes.append( + torch.clone(codes[b, start : start + prefix_len]) + ) + codes[b, start : start + prefix_len, nar_stage] = NUM_AUDIO_TOKENS + y_prompts_codes = torch.stack(y_prompts_codes, dim=0) + else: + prefix_len = y_prompts_codes.shape[1] + + y_prompts = self.nar_audio_embeddings[0](y_prompts_codes[..., 0]) + y_emb = self.nar_audio_embeddings[0](y) + for j in range(1, self.num_quantizers): + y_prompts += self.nar_audio_embeddings[j](y_prompts_codes[..., j]) + if j < nar_stage: + y_emb += self.nar_audio_embeddings[j](codes[..., j]) + y_emb = torch.concat([y_prompts, y_emb], axis=1) + else: + raise ValueError + + return y_emb, prefix_len + + def forward( + self, + x: torch.Tensor, + x_lens: torch.Tensor, + y: Union[torch.Tensor, PromptedFeatures], + y_lens: Union[torch.Tensor, PromptedFeatures], + reduction: str = "sum", + train_stage: int = 0, + **kwargs, + ) -> Tuple[torch.Tensor, Union[torch.Tensor, None]]: + """ + Args: + x: + A 2-D tensor of shape (N, S). + x_lens: + A 1-D tensor of shape (N,). It contains the number of tokens in `x` + before padding. + y: + A 3-D tensor of shape (N, T, 8). + y_lens: + A 1-D tensor of shape (N,). It contains the number of tokens in `x` + before padding. + train_stage: + 0: AR & NAR modules, 1: AR modules, 2: NAR modules + Returns: + Return the predicted audio code matrix, cross-entropy loss and Top-10 accuracy. + """ + assert x.ndim == 2, x.shape + assert x_lens.ndim == 1, x_lens.shape + + y_prompts_codes = None + if isinstance(y, PromptedFeatures): + y_prompts_codes, y = y.data + prompts_len, y_lens = y_lens.data + assert prompts_len.min() == prompts_len.max() + assert self.prefix_mode == 4 + y_prompts_codes = y_prompts_codes.type(torch.int64) + + assert y.ndim == 3, y.shape + assert y_lens.ndim == 1, y_lens.shape + + # NOTE: x has been padded in TextTokenCollater + x_mask = make_pad_mask(x_lens).to(x.device) + y_mask = make_pad_mask(y_lens).to(y.device) + y_mask_int = y_mask.type(torch.int64) + + text = x + codes = y.type(torch.int64) * (1 - y_mask_int.unsqueeze(dim=-1)) + + y, targets = self.pad_y_eos(codes[..., 0], y_mask_int, eos_id=NUM_AUDIO_TOKENS) + + x_len = x_lens.max() + + metrics = {} + total_loss = 0.0 + + xy_padding_mask = torch.concat([x_mask, y_mask], dim=1) + if self.ar_audio_prepend_bos: + ar_xy_padding_mask = torch.concat( + [x_mask, F.pad(y_mask, (1, 0), value=False)], dim=1 + ) + else: + ar_xy_padding_mask = xy_padding_mask + # AR Decoder + if train_stage in [0, 1]: + x = self.ar_text_embedding(text) + x = self.ar_text_prenet(x) + x = self.ar_text_position(x) + + y_len = y_lens.max() + int(self.ar_audio_prepend_bos) + + x_attn_mask = F.pad( + torch.zeros((x_len, x_len), dtype=torch.bool, device=x.device), + (0, y_len), + value=True, + ) + y_attn_mask = F.pad( + torch.triu( + torch.ones(y_len, y_len, dtype=torch.bool, device=x.device), + diagonal=1, + ), + (x_len, 0), + value=False, + ) + xy_attn_mask = torch.concat([x_attn_mask, y_attn_mask], dim=0) + + # merge key padding and attention masks + bsz, src_len = x.shape[0], x_len + y_len + _xy_padding_mask = ( + ar_xy_padding_mask.view(bsz, 1, 1, src_len) + .expand(-1, self.num_heads, -1, -1) + .reshape(bsz * self.num_heads, 1, src_len) + ) + xy_attn_mask = xy_attn_mask.logical_or(_xy_padding_mask) + + new_attn_mask = torch.zeros_like(xy_attn_mask, dtype=x.dtype) + new_attn_mask.masked_fill_(xy_attn_mask, float("-inf")) + xy_attn_mask = new_attn_mask + + y_emb = self.ar_audio_embedding(y) + y_emb = self.ar_audio_prenet(y_emb) + y_pos = self.ar_audio_position(y_emb) + + xy_pos = torch.concat([x, y_pos], dim=1) + + xy_dec, _ = self.ar_decoder( + (xy_pos, None), + mask=xy_attn_mask, + # src_key_padding_mask=xy_padding_mask, + # is_causal=True, + ) + logits = self.ar_predict_layer(xy_dec[:, x_len:]).permute(0, 2, 1) + # loss + total_loss = F.cross_entropy(logits, targets, reduction=reduction) + + metrics["ArTop10Accuracy"] = self.ar_accuracy_metric( + logits.detach(), targets + ).item() * y_lens.sum().type(torch.float32) + + if self.num_quantizers == 1: + return ((x, codes), total_loss, metrics) + + # Non-AR Decoders + if self.ar_audio_prepend_bos: + y = y[:, 1:] + if train_stage in [0, 2]: + num_nar_layers = self.num_quantizers - 1 + nar_stage = self.rng.choices( + [_k for _k in range(1, self.num_quantizers)], + weights=[1.0 / num_nar_layers] * num_nar_layers, + k=1, + )[0] + + x = self.nar_text_embedding(text) + x = self.nar_text_prenet(x) + x = self.nar_text_position(x) + + y_emb, prefix_len = self._prepare_prompts( + y, y_lens, codes, nar_stage, y_prompts_codes + ) + + y_len = y_lens.max() + targets = codes[..., nar_stage] + NUM_AUDIO_TOKENS * y_mask_int + if self.prefix_mode in [2, 4]: + xy_padding_mask = torch.concat( + [ + x_mask, + F.pad(y_mask, (y_emb.shape[1] - y_len, 0), value=False), + ], + dim=1, + ) + elif self.prefix_mode == 1: + targets = targets[:, prefix_len:] + + y_pos = self.nar_audio_prenet(y_emb) + y_pos = self.nar_audio_position(y_pos) + xy_pos = torch.concat([x, y_pos], dim=1) + xy_dec, _ = self.nar_decoder( + (xy_pos, self.nar_stage_embeddings[nar_stage - 1].weight), + src_key_padding_mask=xy_padding_mask, + # is_causal=False, + ) + xy_dec = xy_dec[:, x_lens.max() + prefix_len :] + if self.prefix_mode == 4: + prefix_len = 0 # reset for Top10Accuracy metric + logits = self.nar_predict_layers[nar_stage - 1](xy_dec).permute(0, 2, 1) + + # loss + total_length = (y_lens).sum().type(torch.float32) + total_loss += F.cross_entropy( + logits, + targets, + ignore_index=NUM_AUDIO_TOKENS, + reduction=reduction, + ) * (total_length / (total_length - prefix_len * x.shape[0])) + metrics["NarTop10Accuracy"] = ( + self.nar_accuracy_metric( + F.pad( + logits.detach(), + (0, 0, 0, 1, 0, 0), + value=logits.min().cpu().item(), + ), + targets, + ).item() + * total_length + ) + + if train_stage == 0: + total_loss = total_loss / 2.0 + + return ((x, codes), total_loss, metrics) + + def inference( + self, + x: torch.Tensor, + x_lens: torch.Tensor, + y: torch.Tensor, + enroll_x_lens: torch.Tensor, + top_k: int = -100, + temperature: float = 1.0, + top_p: float = 1.0, + ras: bool = False, + ) -> torch.Tensor: + """ + Args: + x: + A 2-D tensor of shape (1, S). + x_lens: + A 1-D tensor of shape (1,). It contains the number of tokens in `x` + before padding. + y: + A 3-D tensor of shape (1, T, 8). + top_k: (`optional`) int + The number of highest probability tokens to keep for top-k-filtering. Default to -100. + temperature: (`optional`) float + The value used to module the next token probabilities. Must be strictly positive. Default to 1.0. + ras: (`optional`) bool + Whether to use repetition-aware sampling. Default to False. + Returns: + Return the predicted audio code matrix. + """ + assert x.ndim == 2, x.shape + assert x_lens.ndim == 1, x_lens.shape + assert y.ndim == 3, y.shape + assert y.shape[0] == 1, y.shape + + assert torch.all(x_lens > 0) + + # NOTE: x has been padded in TextTokenCollater + text = x + x = self.ar_text_embedding(text) + x = self.ar_text_prenet(x) + x = self.ar_text_position(x) + + text_len = x_lens.max() + prompts = y + prefix_len = y.shape[1] + + # AR Decoder + # TODO: Managing decoder steps avoid repetitive computation + y = prompts[..., 0] + if self.ar_audio_prepend_bos: + y = F.pad(y, (1, 0), value=NUM_AUDIO_TOKENS + 1) + + x_len = x_lens.max() + x_attn_mask = torch.zeros((x_len, x_len), dtype=torch.bool) + + while True: + y_emb = self.ar_audio_embedding(y) + y_emb = self.ar_audio_prenet(y_emb) + y_pos = self.ar_audio_position(y_emb) + xy_pos = torch.concat([x, y_pos], dim=1) + + y_len = y.shape[1] + x_attn_mask_pad = F.pad( + x_attn_mask, + (0, y_len), + value=True, + ) + y_attn_mask = F.pad( + torch.triu(torch.ones(y_len, y_len, dtype=torch.bool), diagonal=1), + (x_len, 0), + value=False, + ) + xy_attn_mask = torch.concat([x_attn_mask_pad, y_attn_mask], dim=0).to( + y.device + ) + + xy_dec, _ = self.ar_decoder( + (xy_pos, None), + mask=xy_attn_mask, + ) + logits = self.ar_predict_layer(xy_dec[:, -1]) + samples = topk_sampling( + logits, + top_k=top_k, + top_p=top_p, + temperature=temperature, + repetition_aware_sampling=ras, + preceding_tokens=y, + ) + + if ( + torch.argmax(logits, dim=-1)[0] == NUM_AUDIO_TOKENS + or samples[0, 0] == NUM_AUDIO_TOKENS + or (y.shape[1] - prompts.shape[1]) > x_lens.max() * 16 + ): + if prompts.shape[1] == y.shape[1]: + raise SyntaxError("well trained model shouldn't reach here.") + break + + y = torch.concat([y, samples], dim=1) + + codes = [y[:, prefix_len + int(self.ar_audio_prepend_bos) :]] + if self.num_quantizers == 1: + return torch.stack(codes, dim=-1) + + # Non-AR Decoders + y_emb = self.nar_audio_embeddings[0](y[:, int(self.ar_audio_prepend_bos) :]) + + if self.prefix_mode in [2, 4]: # Exclude enrolled_phonemes + enrolled_len = enroll_x_lens.max().item() + # SOS + Synthesis Text + EOS + text = torch.concat( + [ + text[:, :1], + text[:, enrolled_len - 1 :], + ], + dim=1, + ) + text_len = text_len - (enrolled_len - 2) + assert text.shape[0] == 1 + + x = self.nar_text_embedding(text) + x = self.nar_text_prenet(x) + x = self.nar_text_position(x) + + if self.prefix_mode == 0: + for i, (predict_layer, embedding_layer) in enumerate( + zip( + self.nar_predict_layers, + self.nar_audio_embeddings[1:], + ) + ): + y_pos = self.nar_audio_prenet(y_emb) + y_pos = self.nar_audio_position(y_pos) + xy_pos = torch.concat([x, y_pos], dim=1) + + xy_dec, _ = self.nar_decoder( + (xy_pos, self.nar_stage_embeddings[i].weight) + ) + logits = predict_layer(xy_dec[:, text_len + prefix_len :]) + + samples = torch.argmax(logits, dim=-1) + codes.append(samples) + + if i < self.num_quantizers - 2: + y_emb[:, :prefix_len] += embedding_layer(prompts[..., i + 1]) + y_emb[:, prefix_len:] += embedding_layer(samples) + else: + for j in range(1, self.num_quantizers): + y_emb[:, :prefix_len] += self.nar_audio_embeddings[j](prompts[..., j]) + + for i, (predict_layer, embedding_layer) in enumerate( + zip( + self.nar_predict_layers, + self.nar_audio_embeddings[1:], + ) + ): + y_pos = self.nar_audio_prenet(y_emb) + y_pos = self.nar_audio_position(y_pos) + xy_pos = torch.concat([x, y_pos], dim=1) + + xy_dec, _ = self.nar_decoder( + (xy_pos, self.nar_stage_embeddings[i].weight) + ) + logits = predict_layer(xy_dec[:, text_len + prefix_len :]) + + samples = torch.argmax(logits, dim=-1) + codes.append(samples) + + if i < self.num_quantizers - 2: + y_emb[:, prefix_len:] += embedding_layer(samples) + + assert len(codes) == self.num_quantizers + return torch.stack(codes, dim=-1) + + def continual( + self, + x: torch.Tensor, + x_lens: torch.Tensor, + y: torch.Tensor, + ) -> torch.Tensor: + """ + Args: + x: + A 2-D tensor of shape (1, S). + x_lens: + A 1-D tensor of shape (1,). It contains the number of tokens in `x` + before padding. + y: + A 3-D tensor of shape (1, T, 8). + Returns: + Return the predicted audio code matrix. + """ + assert x.ndim == 2, x.shape + assert x_lens.ndim == 1, x_lens.shape + assert y.ndim == 3, y.shape + assert y.shape[0] == 1, y.shape + + assert torch.all(x_lens > 0) + assert self.num_quantizers == 8 + + # NOTE: x has been padded in TextTokenCollater + text = x + x = self.ar_text_embedding(text) + x = self.ar_text_prenet(x) + x = self.ar_text_position(x) + + text_len = x_lens.max() + + prefix_len = min(int(y.shape[1] * 0.5), 3 * 75) + + # AR Decoder + prompts = y[:, :prefix_len] + + codes = [y[:, prefix_len:, 0]] + # Non-AR Decoders + x = self.nar_text_embedding(text) + x = self.nar_text_prenet(x) + x = self.nar_text_position(x) + + y_emb = self.nar_audio_embeddings[0](y[..., 0]) + + if self.prefix_mode == 0: + for i, (predict_layer, embedding_layer) in enumerate( + zip( + self.nar_predict_layers, + self.nar_audio_embeddings[1:], + ) + ): + y_pos = self.nar_audio_position(y_emb) + y_pos = self.nar_audio_prenet(y_pos) + xy_pos = torch.concat([x, y_pos], dim=1) + + xy_dec, _ = self.nar_decoder( + (xy_pos, self.nar_stage_embeddings[i].weight) + ) + logits = predict_layer(xy_dec[:, text_len + prefix_len :]) + + samples = torch.argmax(logits, dim=-1) + codes.append(samples) + + if i < 6: + y_emb[:, :prefix_len] += embedding_layer(prompts[..., i + 1]) + y_emb[:, prefix_len:] += embedding_layer(samples) + else: + for j in range(1, 8): + y_emb[:, :prefix_len] += self.nar_audio_embeddings[j](prompts[..., j]) + + for i, (predict_layer, embedding_layer) in enumerate( + zip( + self.nar_predict_layers, + self.nar_audio_embeddings[1:], + ) + ): + y_pos = self.nar_audio_prenet(y_emb) + y_pos = self.nar_audio_position(y_pos) + xy_pos = torch.concat([x, y_pos], dim=1) + + xy_dec, _ = self.nar_decoder( + (xy_pos, self.nar_stage_embeddings[i].weight) + ) + logits = predict_layer(xy_dec[:, text_len + prefix_len :]) + + samples = torch.argmax(logits, dim=-1) + codes.append(samples) + + if i < 6: + y_emb[:, prefix_len:] += embedding_layer(samples) + + assert len(codes) == 8 + return torch.stack(codes, dim=-1) + + +# https://github.com/microsoft/unilm/blob/master/xtune/src/transformers/modeling_utils.py +def top_k_top_p_filtering( + logits, top_k=0, top_p=1.0, filter_value=-float("Inf"), min_tokens_to_keep=1 +): + """Filter a distribution of logits using top-k and/or nucleus (top-p) filtering + Args: + logits: logits distribution shape (batch size, vocabulary size) + if top_k > 0: keep only top k tokens with highest probability (top-k filtering). + if top_p < 1.0: keep the top tokens with cumulative probability >= top_p (nucleus filtering). + Nucleus filtering is described in Holtzman et al. (http://arxiv.org/abs/1904.09751) + Make sure we keep at least min_tokens_to_keep per batch example in the output + From: https://gist.github.com/thomwolf/1a5a29f6962089e871b94cbd09daf317 + """ + if top_k > 0: + top_k = min(max(top_k, min_tokens_to_keep), logits.size(-1)) # Safety check + # Remove all tokens with a probability less than the last token of the top-k + indices_to_remove = logits < torch.topk(logits, top_k)[0][..., -1, None] + logits[indices_to_remove] = filter_value + + if top_p < 1.0: + sorted_logits, sorted_indices = torch.sort(logits, descending=True) + cumulative_probs = torch.cumsum(F.softmax(sorted_logits, dim=-1), dim=-1) + + # Remove tokens with cumulative probability above the threshold (token with 0 are kept) + sorted_indices_to_remove = cumulative_probs > top_p + if min_tokens_to_keep > 1: + # Keep at least min_tokens_to_keep (set to min_tokens_to_keep-1 because we add the first one below) + sorted_indices_to_remove[..., :min_tokens_to_keep] = 0 + # Shift the indices to the right to keep also the first token above the threshold + sorted_indices_to_remove[..., 1:] = sorted_indices_to_remove[..., :-1].clone() + sorted_indices_to_remove[..., 0] = 0 + + # scatter sorted tensors to original indexing + indices_to_remove = sorted_indices_to_remove.scatter( + 1, sorted_indices, sorted_indices_to_remove + ) + logits[indices_to_remove] = filter_value + return logits + + +def topk_sampling( + logits, + top_k=10, + top_p=1.0, + temperature=1.0, + repetition_aware_sampling=False, + preceding_tokens=None, +): + if temperature != 1.0: + logits = logits / temperature + # Top-p/top-k filtering + logits_filtered = top_k_top_p_filtering( + logits.clone(), top_k=top_k, top_p=top_p, min_tokens_to_keep=2 + ) + # Sample + probs = F.softmax(logits_filtered, dim=-1) + tokens = torch.multinomial(probs, num_samples=1) + + if repetition_aware_sampling: + window_size = 10 + threshold = 0.1 + # we first generate the target code ct′ + # by nucleus sampling with a pre-defined top-p value v. Then, we + # calculate the repetition ratio r of token ct′ + # in the preceding code sequence with a window size K. + # If the ratio r exceeds a pre-defined repetition threshold ratio tn, we replace the target code ct′ + # by + # random sampling from p(ct′ + # |x, c window_size: + preceding_tokens = preceding_tokens[:, -window_size:] + if preceding_tokens.shape[1] > 0: + for i, item in enumerate(preceding_tokens): + # check if the repeat ratio exceeds the threshold + if (item == tokens[i]).sum() / window_size > threshold: + # replace the target code ct′ by random sampling + probs = F.softmax(logits[i], dim=-1) + token_new = torch.multinomial(probs, num_samples=1) + tokens[i] = token_new + return tokens From 18fa6a0fecb16c4b825f87e5ace06e7e84b3a3ff Mon Sep 17 00:00:00 2001 From: Han Zhu Date: Fri, 29 Nov 2024 11:45:05 +0800 Subject: [PATCH 02/59] Fix LibriTTS prepare.sh (#1815) --- egs/libritts/TTS/prepare.sh | 2 +- egs/ljspeech/TTS/local/validate_manifest.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/egs/libritts/TTS/prepare.sh b/egs/libritts/TTS/prepare.sh index 1700e0737..a0a6d2ae1 100755 --- a/egs/libritts/TTS/prepare.sh +++ b/egs/libritts/TTS/prepare.sh @@ -84,7 +84,7 @@ if [ $stage -le 2 ] && [ $stop_stage -ge 2 ]; then if [ ! -f data/spectrogram/libritts_cuts_train-all-shuf.jsonl.gz ]; then cat <(gunzip -c data/spectrogram/libritts_cuts_train-clean-100.jsonl.gz) \ <(gunzip -c data/spectrogram/libritts_cuts_train-clean-360.jsonl.gz) \ - <(gunzip -c data/spectrogramlibritts_cuts_train-other-500.jsonl.gz) | \ + <(gunzip -c data/spectrogram/libritts_cuts_train-other-500.jsonl.gz) | \ shuf | gzip -c > data/spectrogram/libritts_cuts_train-all-shuf.jsonl.gz fi diff --git a/egs/ljspeech/TTS/local/validate_manifest.py b/egs/ljspeech/TTS/local/validate_manifest.py index 9535ba9f4..68159ae03 100755 --- a/egs/ljspeech/TTS/local/validate_manifest.py +++ b/egs/ljspeech/TTS/local/validate_manifest.py @@ -33,7 +33,6 @@ import argparse import logging from pathlib import Path -from compute_fbank_ljspeech import MyFbank from lhotse import CutSet, load_manifest_lazy from lhotse.dataset.speech_synthesis import validate_for_tts From a1ade8ecb77a78ff55d1bf41a918051b5f306731 Mon Sep 17 00:00:00 2001 From: zr_jin Date: Fri, 29 Nov 2024 16:36:02 +0800 Subject: [PATCH 03/59] fixed failed assertion in the `xbmu_ambo31` recipe (#1816) --- .../ASR/pruned_transducer_stateless7/train.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/egs/xbmu_amdo31/ASR/pruned_transducer_stateless7/train.py b/egs/xbmu_amdo31/ASR/pruned_transducer_stateless7/train.py index d24c27326..dd72551d9 100755 --- a/egs/xbmu_amdo31/ASR/pruned_transducer_stateless7/train.py +++ b/egs/xbmu_amdo31/ASR/pruned_transducer_stateless7/train.py @@ -974,7 +974,16 @@ def run(rank, world_size, args): logging.info("Using DDP") model = DDP(model, device_ids=[rank], find_unused_parameters=True) - optimizer = ScaledAdam(model.parameters(), lr=params.base_lr, clipping_scale=2.0) + parameters_names = [] + parameters_names.append( + [name_param_pair[0] for name_param_pair in model.named_parameters()] + ) + optimizer = ScaledAdam( + model.parameters(), + lr=params.base_lr, + clipping_scale=2.0, + parameters_names=parameters_names, + ) scheduler = Eden(optimizer, params.lr_batches, params.lr_epochs) From bdd0f85704a1c257a89811a1cece728425fe6e9b Mon Sep 17 00:00:00 2001 From: Han Zhu <1106766460@qq.com> Date: Thu, 5 Dec 2024 15:12:06 +0800 Subject: [PATCH 04/59] Fix the normalized_text in LibriTTS recipe (#1825) --- egs/libritts/TTS/local/prepare_tokens_libritts.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/egs/libritts/TTS/local/prepare_tokens_libritts.py b/egs/libritts/TTS/local/prepare_tokens_libritts.py index faeb611f5..cdc39ea6b 100755 --- a/egs/libritts/TTS/local/prepare_tokens_libritts.py +++ b/egs/libritts/TTS/local/prepare_tokens_libritts.py @@ -31,15 +31,6 @@ from piper_phonemize import phonemize_espeak from tqdm.auto import tqdm -def remove_punc_to_upper(text: str) -> str: - text = text.replace("‘", "'") - text = text.replace("’", "'") - tokens = set("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'") - s_list = [x.upper() if x in tokens else " " for x in text] - s = " ".join("".join(s_list).split()).strip() - return s - - def prepare_tokens_libritts(): output_dir = Path("data/spectrogram") prefix = "libritts" @@ -72,7 +63,7 @@ def prepare_tokens_libritts(): for t in tokens_list: tokens.extend(t) cut.tokens = tokens - cut.supervisions[0].normalized_text = remove_punc_to_upper(text) + cut.supervisions[0].normalized_text = text new_cuts.append(cut) From 6e6b022e413a49cc2cf1c14995db39656e0ad85b Mon Sep 17 00:00:00 2001 From: zr_jin Date: Fri, 6 Dec 2024 16:14:51 +0800 Subject: [PATCH 05/59] performed end to end testing to the VALL-E recipe (#1818) * added the missing ``visualize`` function * minor fixes --- ...te_neural_codec_and_prepare_text_tokens.py | 22 +++-- egs/wenetspeech4tts/TTS/valle/infer.py | 2 +- .../TTS/valle/requirements.txt | 2 + egs/wenetspeech4tts/TTS/valle/train.py | 9 +- egs/wenetspeech4tts/TTS/valle/valle.py | 85 +++++++++++++++++++ 5 files changed, 109 insertions(+), 11 deletions(-) create mode 100644 egs/wenetspeech4tts/TTS/valle/requirements.txt diff --git a/egs/wenetspeech4tts/TTS/local/compute_neural_codec_and_prepare_text_tokens.py b/egs/wenetspeech4tts/TTS/local/compute_neural_codec_and_prepare_text_tokens.py index 5494bf340..7de2c6202 100755 --- a/egs/wenetspeech4tts/TTS/local/compute_neural_codec_and_prepare_text_tokens.py +++ b/egs/wenetspeech4tts/TTS/local/compute_neural_codec_and_prepare_text_tokens.py @@ -516,9 +516,19 @@ def main(): for idx, part in enumerate(cut_sets): if args.audio_extractor: if args.audio_extractor == "Encodec": - storage_path = f"{args.output_dir}/{args.prefix}_encodec_{partition}_{idx if split > 1 else ''}" + if split > 1: + storage_path = f"{args.output_dir}/{args.prefix}_encodec_{partition}_{idx}" + else: + storage_path = ( + f"{args.output_dir}/{args.prefix}_encodec_{partition}" + ) else: - storage_path = f"{args.output_dir}/{args.prefix}_fbank_{partition}_{idx if split > 1 else ''}" + if split > 1: + storage_path = f"{args.output_dir}/{args.prefix}_fbank_{partition}_{idx}" + else: + storage_path = ( + f"{args.output_dir}/{args.prefix}_fbank_{partition}" + ) if args.prefix.lower() in [ "ljspeech", @@ -587,9 +597,11 @@ def main(): ].normalized_text, "normalized_text is None" # Save each part with an index if split > 1 - cuts_filename = ( - f"{prefix}cuts_{partition}.{idx if split > 1 else ''}.{args.suffix}" - ) + if split > 1: + cuts_filename = f"{prefix}cuts_{partition}.{idx}.{args.suffix}" + else: + cuts_filename = f"{prefix}cuts_{partition}.{args.suffix}" + part.to_file(f"{args.output_dir}/{cuts_filename}") logging.info(f"Saved {cuts_filename}") diff --git a/egs/wenetspeech4tts/TTS/valle/infer.py b/egs/wenetspeech4tts/TTS/valle/infer.py index fd7ba9f21..44a251c56 100644 --- a/egs/wenetspeech4tts/TTS/valle/infer.py +++ b/egs/wenetspeech4tts/TTS/valle/infer.py @@ -86,7 +86,7 @@ def get_args(): parser.add_argument( "--checkpoint", type=str, - default="exp/vallf_nano_full/checkpoint-100000.pt", + default="./valle/exp/checkpoint-100000.pt", help="Path to the saved checkpoint.", ) diff --git a/egs/wenetspeech4tts/TTS/valle/requirements.txt b/egs/wenetspeech4tts/TTS/valle/requirements.txt new file mode 100644 index 000000000..06958dbea --- /dev/null +++ b/egs/wenetspeech4tts/TTS/valle/requirements.txt @@ -0,0 +1,2 @@ +phonemizer==3.2.1 +git+https://github.com/facebookresearch/encodec.git \ No newline at end of file diff --git a/egs/wenetspeech4tts/TTS/valle/train.py b/egs/wenetspeech4tts/TTS/valle/train.py index fde209511..e9ec548f3 100755 --- a/egs/wenetspeech4tts/TTS/valle/train.py +++ b/egs/wenetspeech4tts/TTS/valle/train.py @@ -4,6 +4,7 @@ # Mingshuang Luo) # Copyright 2023 (authors: Feiteng Li) # Copyright 2024 (authors: Yuekai Zhang) +# Copyright 2024 Tsinghua University (authors: Zengrui Jin,) # # See ../../../../LICENSE for clarification regarding multiple authors # @@ -48,10 +49,8 @@ python3 valle/train.py --max-duration 160 --filter-min-duration 0.5 --filter-max import argparse import copy import logging -import os import random import warnings -from contextlib import nullcontext from pathlib import Path from shutil import copyfile from typing import Any, Dict, Optional, Tuple, Union @@ -216,7 +215,7 @@ def get_parser(): parser.add_argument( "--exp-dir", type=str, - default="exp/valle_dev", + default="./valle/exp", help="""The experiment dir. It specifies the directory where all training related files, e.g., checkpoints, log, etc, are saved @@ -686,9 +685,9 @@ def compute_validation_loss( output_dir = Path(f"{params.exp_dir}/eval/step-{params.batch_idx_train:06d}") output_dir.mkdir(parents=True, exist_ok=True) if isinstance(model, DDP): - model.module.visualize(predicts, batch, output_dir=output_dir) + model.module.visualize(predicts, batch, tokenizer, output_dir=output_dir) else: - model.visualize(predicts, batch, output_dir=output_dir) + model.visualize(predicts, batch, tokenizer, output_dir=output_dir) return tot_loss diff --git a/egs/wenetspeech4tts/TTS/valle/valle.py b/egs/wenetspeech4tts/TTS/valle/valle.py index b2eb8ae69..4bfa2b577 100644 --- a/egs/wenetspeech4tts/TTS/valle/valle.py +++ b/egs/wenetspeech4tts/TTS/valle/valle.py @@ -19,8 +19,11 @@ import random from functools import partial from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple, Union +import matplotlib.pyplot as plt +import numpy as np import torch import torch.nn as nn +from tokenizer import TextTokenCollater from torch import Tensor from torch.nn import Linear, Module from torch.nn import functional as F @@ -1658,6 +1661,88 @@ class VALLE(nn.Module): assert len(codes) == 8 return torch.stack(codes, dim=-1) + def visualize( + self, + predicts: Tuple[torch.Tensor], + batch: Dict[str, Union[List, torch.Tensor]], + tokenizer: TextTokenCollater, + output_dir: str, + limit: int = 4, + ) -> None: + audio_features = batch["features"].to("cpu").detach().numpy() + audio_features_lens = batch["features_lens"].to("cpu").detach().numpy() + + tokens = batch["tokens"] + text_tokens, text_tokens_lens = tokenizer(tokens) + assert text_tokens.ndim == 2 + + texts = batch["text"] + utt_ids = [cut.id for cut in batch["cut"]] + + encoder_outputs = predicts[0].to("cpu").type(torch.float32).detach().numpy() + decoder_outputs = predicts[1] + if isinstance(decoder_outputs, list): + decoder_outputs = decoder_outputs[-1] + decoder_outputs = decoder_outputs.to("cpu").type(torch.float32).detach().numpy() + + vmin, vmax = 0, 1024 # Encodec + if decoder_outputs.dtype == np.float32: + vmin, vmax = -6, 0 # Fbank + + num_figures = 3 + for b, (utt_id, text) in enumerate(zip(utt_ids[:limit], texts[:limit])): + _ = plt.figure(figsize=(14, 8 * num_figures)) + + S = text_tokens_lens[b] + T = audio_features_lens[b] + + # encoder + plt.subplot(num_figures, 1, 1) + plt.title(f"Text: {text}") + plt.imshow( + X=np.transpose(encoder_outputs[b]), + cmap=plt.get_cmap("jet"), + aspect="auto", + interpolation="nearest", + ) + plt.gca().invert_yaxis() + plt.axvline(x=S - 0.4, linewidth=2, color="r") + plt.xlabel("Encoder Output") + plt.colorbar() + + # decoder + plt.subplot(num_figures, 1, 2) + plt.imshow( + X=np.transpose(decoder_outputs[b]), + cmap=plt.get_cmap("jet"), + aspect="auto", + interpolation="nearest", + vmin=vmin, + vmax=vmax, + ) + plt.gca().invert_yaxis() + plt.axvline(x=T - 0.4, linewidth=2, color="r") + plt.xlabel("Decoder Output") + plt.colorbar() + + # target + plt.subplot(num_figures, 1, 3) + plt.imshow( + X=np.transpose(audio_features[b]), + cmap=plt.get_cmap("jet"), + aspect="auto", + interpolation="nearest", + vmin=vmin, + vmax=vmax, + ) + plt.gca().invert_yaxis() + plt.axvline(x=T - 0.4, linewidth=2, color="r") + plt.xlabel("Decoder Target") + plt.colorbar() + + plt.savefig(f"{output_dir}/{utt_id}.png") + plt.close() + # https://github.com/microsoft/unilm/blob/master/xtune/src/transformers/modeling_utils.py def top_k_top_p_filtering( From 1c4dd464a0dcba042ea0050c8b7a0416799025b3 Mon Sep 17 00:00:00 2001 From: zr_jin Date: Sun, 8 Dec 2024 03:18:15 +0800 Subject: [PATCH 06/59] Performed end to end testing on the matcha recipe (#1797) * minor fixes to the `ljspeech/matcha` recipe --- .github/scripts/ljspeech/TTS/run-matcha.sh | 2 +- egs/ljspeech/TTS/README.md | 4 +- egs/ljspeech/TTS/local/audio.py | 1 + .../TTS/local/compute_fbank_ljspeech.py | 91 +---- egs/ljspeech/TTS/local/fbank.py | 1 + .../TTS/matcha/compute_fbank_ljspeech.py | 1 - .../TTS/matcha/export_onnx_hifigan.py | 2 +- egs/ljspeech/TTS/matcha/fbank.py | 88 +++++ egs/ljspeech/TTS/matcha/infer.py | 328 ++++++++++++++++++ egs/ljspeech/TTS/matcha/inference.py | 199 ----------- .../TTS/matcha/models/components/decoder.py | 2 +- .../matcha/models/components/flow_matching.py | 2 +- .../matcha/models/components/text_encoder.py | 2 +- egs/ljspeech/TTS/matcha/models/matcha_tts.py | 8 +- .../TTS/matcha/monotonic_align/.gitignore | 2 +- .../TTS/matcha/monotonic_align/__init__.py | 5 +- .../TTS/matcha/monotonic_align/core.pyx | 2 - .../TTS/matcha/monotonic_align/setup.py | 30 +- egs/ljspeech/TTS/matcha/requirements.txt | 1 + egs/ljspeech/TTS/matcha/train.py | 13 +- egs/ljspeech/TTS/matcha/tts_datamodule.py | 15 +- egs/ljspeech/TTS/prepare.sh | 30 +- egs/ljspeech/TTS/vits/infer.py | 2 +- .../TTS/vits/monotonic_align/.gitignore | 3 + egs/ljspeech/TTS/vits/test_model.py | 1 - 25 files changed, 485 insertions(+), 350 deletions(-) create mode 120000 egs/ljspeech/TTS/local/audio.py create mode 120000 egs/ljspeech/TTS/local/fbank.py delete mode 120000 egs/ljspeech/TTS/matcha/compute_fbank_ljspeech.py create mode 100644 egs/ljspeech/TTS/matcha/fbank.py create mode 100755 egs/ljspeech/TTS/matcha/infer.py delete mode 100755 egs/ljspeech/TTS/matcha/inference.py create mode 100644 egs/ljspeech/TTS/vits/monotonic_align/.gitignore diff --git a/.github/scripts/ljspeech/TTS/run-matcha.sh b/.github/scripts/ljspeech/TTS/run-matcha.sh index 37e1bc320..0876cb47f 100755 --- a/.github/scripts/ljspeech/TTS/run-matcha.sh +++ b/.github/scripts/ljspeech/TTS/run-matcha.sh @@ -56,7 +56,7 @@ function infer() { curl -SL -O https://github.com/csukuangfj/models/raw/refs/heads/master/hifigan/generator_v1 - ./matcha/inference.py \ + ./matcha/infer.py \ --epoch 1 \ --exp-dir ./matcha/exp \ --tokens data/tokens.txt \ diff --git a/egs/ljspeech/TTS/README.md b/egs/ljspeech/TTS/README.md index 1cd6e8fd7..82850cd04 100644 --- a/egs/ljspeech/TTS/README.md +++ b/egs/ljspeech/TTS/README.md @@ -131,12 +131,12 @@ To inference, use: wget https://github.com/csukuangfj/models/raw/refs/heads/master/hifigan/generator_v1 -./matcha/inference \ +./matcha/synth.py \ --exp-dir ./matcha/exp-new-3 \ --epoch 4000 \ --tokens ./data/tokens.txt \ --vocoder ./generator_v1 \ - --input-text "how are you doing?" + --input-text "how are you doing?" \ --output-wav ./generated.wav ``` diff --git a/egs/ljspeech/TTS/local/audio.py b/egs/ljspeech/TTS/local/audio.py new file mode 120000 index 000000000..b70d91c92 --- /dev/null +++ b/egs/ljspeech/TTS/local/audio.py @@ -0,0 +1 @@ +../matcha/audio.py \ No newline at end of file diff --git a/egs/ljspeech/TTS/local/compute_fbank_ljspeech.py b/egs/ljspeech/TTS/local/compute_fbank_ljspeech.py index 5152ae675..296f9a4f4 100755 --- a/egs/ljspeech/TTS/local/compute_fbank_ljspeech.py +++ b/egs/ljspeech/TTS/local/compute_fbank_ljspeech.py @@ -27,102 +27,17 @@ The generated fbank features are saved in data/fbank. import argparse import logging import os -from dataclasses import dataclass from pathlib import Path -from typing import Union -import numpy as np import torch +from fbank import MatchaFbank, MatchaFbankConfig from lhotse import CutSet, LilcomChunkyWriter, load_manifest from lhotse.audio import RecordingSet -from lhotse.features.base import FeatureExtractor, register_extractor from lhotse.supervision import SupervisionSet -from lhotse.utils import Seconds, compute_num_frames -from matcha.audio import mel_spectrogram from icefall.utils import get_executor -@dataclass -class MyFbankConfig: - n_fft: int - n_mels: int - sampling_rate: int - hop_length: int - win_length: int - f_min: float - f_max: float - - -@register_extractor -class MyFbank(FeatureExtractor): - - name = "MyFbank" - config_type = MyFbankConfig - - def __init__(self, config): - super().__init__(config=config) - - @property - def device(self) -> Union[str, torch.device]: - return self.config.device - - def feature_dim(self, sampling_rate: int) -> int: - return self.config.n_mels - - def extract( - self, - samples: np.ndarray, - sampling_rate: int, - ) -> torch.Tensor: - # Check for sampling rate compatibility. - expected_sr = self.config.sampling_rate - assert sampling_rate == expected_sr, ( - f"Mismatched sampling rate: extractor expects {expected_sr}, " - f"got {sampling_rate}" - ) - samples = torch.from_numpy(samples) - assert samples.ndim == 2, samples.shape - assert samples.shape[0] == 1, samples.shape - - mel = ( - mel_spectrogram( - samples, - self.config.n_fft, - self.config.n_mels, - self.config.sampling_rate, - self.config.hop_length, - self.config.win_length, - self.config.f_min, - self.config.f_max, - center=False, - ) - .squeeze() - .t() - ) - - assert mel.ndim == 2, mel.shape - assert mel.shape[1] == self.config.n_mels, mel.shape - - num_frames = compute_num_frames( - samples.shape[1] / sampling_rate, self.frame_shift, sampling_rate - ) - - if mel.shape[0] > num_frames: - mel = mel[:num_frames] - elif mel.shape[0] < num_frames: - mel = mel.unsqueeze(0) - mel = torch.nn.functional.pad( - mel, (0, 0, 0, num_frames - mel.shape[1]), mode="replicate" - ).squeeze(0) - - return mel.numpy() - - @property - def frame_shift(self) -> Seconds: - return self.config.hop_length / self.config.sampling_rate - - def get_parser(): parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter @@ -149,7 +64,7 @@ def compute_fbank_ljspeech(num_jobs: int): logging.info(f"num_jobs: {num_jobs}") logging.info(f"src_dir: {src_dir}") logging.info(f"output_dir: {output_dir}") - config = MyFbankConfig( + config = MatchaFbankConfig( n_fft=1024, n_mels=80, sampling_rate=22050, @@ -170,7 +85,7 @@ def compute_fbank_ljspeech(num_jobs: int): src_dir / f"{prefix}_supervisions_{partition}.{suffix}", SupervisionSet ) - extractor = MyFbank(config) + extractor = MatchaFbank(config) with get_executor() as ex: # Initialize the executor only once. cuts_filename = f"{prefix}_cuts_{partition}.{suffix}" diff --git a/egs/ljspeech/TTS/local/fbank.py b/egs/ljspeech/TTS/local/fbank.py new file mode 120000 index 000000000..5bcf1fde5 --- /dev/null +++ b/egs/ljspeech/TTS/local/fbank.py @@ -0,0 +1 @@ +../matcha/fbank.py \ No newline at end of file diff --git a/egs/ljspeech/TTS/matcha/compute_fbank_ljspeech.py b/egs/ljspeech/TTS/matcha/compute_fbank_ljspeech.py deleted file mode 120000 index 85255ba0c..000000000 --- a/egs/ljspeech/TTS/matcha/compute_fbank_ljspeech.py +++ /dev/null @@ -1 +0,0 @@ -../local/compute_fbank_ljspeech.py \ No newline at end of file diff --git a/egs/ljspeech/TTS/matcha/export_onnx_hifigan.py b/egs/ljspeech/TTS/matcha/export_onnx_hifigan.py index 63d1fac20..5c96b3bc7 100755 --- a/egs/ljspeech/TTS/matcha/export_onnx_hifigan.py +++ b/egs/ljspeech/TTS/matcha/export_onnx_hifigan.py @@ -7,7 +7,7 @@ from typing import Any, Dict import onnx import torch -from inference import load_vocoder +from infer import load_vocoder def add_meta_data(filename: str, meta_data: Dict[str, Any]): diff --git a/egs/ljspeech/TTS/matcha/fbank.py b/egs/ljspeech/TTS/matcha/fbank.py new file mode 100644 index 000000000..d729fa425 --- /dev/null +++ b/egs/ljspeech/TTS/matcha/fbank.py @@ -0,0 +1,88 @@ +from dataclasses import dataclass +from typing import Union + +import numpy as np +import torch +from audio import mel_spectrogram +from lhotse.features.base import FeatureExtractor, register_extractor +from lhotse.utils import Seconds, compute_num_frames + + +@dataclass +class MatchaFbankConfig: + n_fft: int + n_mels: int + sampling_rate: int + hop_length: int + win_length: int + f_min: float + f_max: float + + +@register_extractor +class MatchaFbank(FeatureExtractor): + + name = "MatchaFbank" + config_type = MatchaFbankConfig + + def __init__(self, config): + super().__init__(config=config) + + @property + def device(self) -> Union[str, torch.device]: + return self.config.device + + def feature_dim(self, sampling_rate: int) -> int: + return self.config.n_mels + + def extract( + self, + samples: np.ndarray, + sampling_rate: int, + ) -> torch.Tensor: + # Check for sampling rate compatibility. + expected_sr = self.config.sampling_rate + assert sampling_rate == expected_sr, ( + f"Mismatched sampling rate: extractor expects {expected_sr}, " + f"got {sampling_rate}" + ) + samples = torch.from_numpy(samples) + assert samples.ndim == 2, samples.shape + assert samples.shape[0] == 1, samples.shape + + mel = ( + mel_spectrogram( + samples, + self.config.n_fft, + self.config.n_mels, + self.config.sampling_rate, + self.config.hop_length, + self.config.win_length, + self.config.f_min, + self.config.f_max, + center=False, + ) + .squeeze() + .t() + ) + + assert mel.ndim == 2, mel.shape + assert mel.shape[1] == self.config.n_mels, mel.shape + + num_frames = compute_num_frames( + samples.shape[1] / sampling_rate, self.frame_shift, sampling_rate + ) + + if mel.shape[0] > num_frames: + mel = mel[:num_frames] + elif mel.shape[0] < num_frames: + mel = mel.unsqueeze(0) + mel = torch.nn.functional.pad( + mel, (0, 0, 0, num_frames - mel.shape[1]), mode="replicate" + ).squeeze(0) + + return mel.numpy() + + @property + def frame_shift(self) -> Seconds: + return self.config.hop_length / self.config.sampling_rate diff --git a/egs/ljspeech/TTS/matcha/infer.py b/egs/ljspeech/TTS/matcha/infer.py new file mode 100755 index 000000000..0b221d5c5 --- /dev/null +++ b/egs/ljspeech/TTS/matcha/infer.py @@ -0,0 +1,328 @@ +#!/usr/bin/env python3 +# Copyright 2024 Xiaomi Corp. (authors: Fangjun Kuang) + +import argparse +import datetime as dt +import json +import logging +from pathlib import Path + +import soundfile as sf +import torch +import torch.nn as nn +from hifigan.config import v1, v2, v3 +from hifigan.denoiser import Denoiser +from hifigan.models import Generator as HiFiGAN +from tokenizer import Tokenizer +from train import get_model, get_params +from tts_datamodule import LJSpeechTtsDataModule + +from icefall.checkpoint import load_checkpoint +from icefall.utils import AttributeDict, setup_logger + + +def get_parser(): + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + + parser.add_argument( + "--epoch", + type=int, + default=4000, + help="""It specifies the checkpoint to use for decoding. + Note: Epoch counts from 1. + """, + ) + + parser.add_argument( + "--exp-dir", + type=Path, + default="matcha/exp", + help="""The experiment dir. + It specifies the directory where all training related + files, e.g., checkpoints, log, etc, are saved + """, + ) + + parser.add_argument( + "--vocoder", + type=Path, + default="./generator_v1", + help="Path to the vocoder", + ) + + parser.add_argument( + "--tokens", + type=Path, + default="data/tokens.txt", + ) + + parser.add_argument( + "--cmvn", + type=str, + default="data/fbank/cmvn.json", + help="""Path to vocabulary.""", + ) + + # The following arguments are used for inference on single text + parser.add_argument( + "--input-text", + type=str, + required=False, + help="The text to generate speech for", + ) + + parser.add_argument( + "--output-wav", + type=str, + required=False, + help="The filename of the wave to save the generated speech", + ) + + parser.add_argument( + "--sampling-rate", + type=int, + default=22050, + help="The sampling rate of the generated speech (default: 22050 for LJSpeech)", + ) + + return parser + + +def load_vocoder(checkpoint_path: Path) -> nn.Module: + checkpoint_path = str(checkpoint_path) + if checkpoint_path.endswith("v1"): + h = AttributeDict(v1) + elif checkpoint_path.endswith("v2"): + h = AttributeDict(v2) + elif checkpoint_path.endswith("v3"): + h = AttributeDict(v3) + else: + raise ValueError(f"supports only v1, v2, and v3, given {checkpoint_path}") + + hifigan = HiFiGAN(h).to("cpu") + hifigan.load_state_dict( + torch.load(checkpoint_path, map_location="cpu")["generator"] + ) + _ = hifigan.eval() + hifigan.remove_weight_norm() + return hifigan + + +def to_waveform( + mel: torch.Tensor, vocoder: nn.Module, denoiser: nn.Module +) -> torch.Tensor: + audio = vocoder(mel).clamp(-1, 1) + audio = denoiser(audio.squeeze(0), strength=0.00025).cpu().squeeze() + return audio.squeeze() + + +def process_text(text: str, tokenizer: Tokenizer, device: str = "cpu") -> dict: + x = tokenizer.texts_to_token_ids([text], add_sos=True, add_eos=True) + x = torch.tensor(x, dtype=torch.long, device=device) + x_lengths = torch.tensor([x.shape[-1]], dtype=torch.long, device=device) + return {"x_orig": text, "x": x, "x_lengths": x_lengths} + + +def synthesize( + model: nn.Module, + tokenizer: Tokenizer, + n_timesteps: int, + text: str, + length_scale: float, + temperature: float, + device: str = "cpu", + spks=None, +) -> dict: + text_processed = process_text(text=text, tokenizer=tokenizer, device=device) + start_t = dt.datetime.now() + output = model.synthesise( + text_processed["x"], + text_processed["x_lengths"], + n_timesteps=n_timesteps, + temperature=temperature, + spks=spks, + length_scale=length_scale, + ) + # merge everything to one dict + output.update({"start_t": start_t, **text_processed}) + return output + + +def infer_dataset( + dl: torch.utils.data.DataLoader, + params: AttributeDict, + model: nn.Module, + vocoder: nn.Module, + denoiser: nn.Module, + tokenizer: Tokenizer, +) -> None: + """Decode dataset. + The ground-truth and generated audio pairs will be saved to `params.save_wav_dir`. + + Args: + dl: + PyTorch's dataloader containing the dataset to decode. + params: + It is returned by :func:`get_params`. + model: + The neural model. + tokenizer: + Used to convert text to phonemes. + """ + + device = next(model.parameters()).device + num_cuts = 0 + log_interval = 5 + + try: + num_batches = len(dl) + except TypeError: + num_batches = "?" + + for batch_idx, batch in enumerate(dl): + batch_size = len(batch["tokens"]) + + texts = [c.supervisions[0].normalized_text for c in batch["cut"]] + + audio = batch["audio"] + audio_lens = batch["audio_lens"].tolist() + cut_ids = [cut.id for cut in batch["cut"]] + + for i in range(batch_size): + output = synthesize( + model=model, + tokenizer=tokenizer, + n_timesteps=params.n_timesteps, + text=texts[i], + length_scale=params.length_scale, + temperature=params.temperature, + device=device, + ) + output["waveform"] = to_waveform(output["mel"], vocoder, denoiser) + + sf.write( + file=params.save_wav_dir / f"{cut_ids[i]}_pred.wav", + data=output["waveform"], + samplerate=params.data_args.sampling_rate, + subtype="PCM_16", + ) + sf.write( + file=params.save_wav_dir / f"{cut_ids[i]}_gt.wav", + data=audio[i].numpy(), + samplerate=params.data_args.sampling_rate, + subtype="PCM_16", + ) + + num_cuts += batch_size + + if batch_idx % log_interval == 0: + batch_str = f"{batch_idx}/{num_batches}" + + logging.info(f"batch {batch_str}, cuts processed until now is {num_cuts}") + + +@torch.inference_mode() +def main(): + parser = get_parser() + LJSpeechTtsDataModule.add_arguments(parser) + args = parser.parse_args() + args.exp_dir = Path(args.exp_dir) + + params = get_params() + params.update(vars(args)) + + params.suffix = f"epoch-{params.epoch}" + + params.res_dir = params.exp_dir / "infer" / params.suffix + params.save_wav_dir = params.res_dir / "wav" + params.save_wav_dir.mkdir(parents=True, exist_ok=True) + + setup_logger(f"{params.res_dir}/log-infer-{params.suffix}") + logging.info("Infer started") + + device = torch.device("cpu") + if torch.cuda.is_available(): + device = torch.device("cuda", 0) + logging.info(f"Device: {device}") + + tokenizer = Tokenizer(params.tokens) + params.blank_id = tokenizer.pad_id + params.vocab_size = tokenizer.vocab_size + params.model_args.n_vocab = params.vocab_size + + with open(params.cmvn) as f: + stats = json.load(f) + params.data_args.data_statistics.mel_mean = stats["fbank_mean"] + params.data_args.data_statistics.mel_std = stats["fbank_std"] + + params.model_args.data_statistics.mel_mean = stats["fbank_mean"] + params.model_args.data_statistics.mel_std = stats["fbank_std"] + + # Number of ODE Solver steps + params.n_timesteps = 2 + + # Changes to the speaking rate + params.length_scale = 1.0 + + # Sampling temperature + params.temperature = 0.667 + logging.info(params) + + logging.info("About to create model") + model = get_model(params) + + load_checkpoint(f"{params.exp_dir}/epoch-{params.epoch}.pt", model) + model.to(device) + model.eval() + + # we need cut ids to organize tts results. + args.return_cuts = True + ljspeech = LJSpeechTtsDataModule(args) + + test_cuts = ljspeech.test_cuts() + test_dl = ljspeech.test_dataloaders(test_cuts) + + if not Path(params.vocoder).is_file(): + raise ValueError(f"{params.vocoder} does not exist") + + vocoder = load_vocoder(params.vocoder) + vocoder.to(device) + + denoiser = Denoiser(vocoder, mode="zeros") + denoiser.to(device) + + if params.input_text is not None and params.output_wav is not None: + logging.info("Synthesizing a single text") + output = synthesize( + model=model, + tokenizer=tokenizer, + n_timesteps=params.n_timesteps, + text=params.input_text, + length_scale=params.length_scale, + temperature=params.temperature, + device=device, + ) + output["waveform"] = to_waveform(output["mel"], vocoder, denoiser) + + sf.write( + file=params.output_wav, + data=output["waveform"], + samplerate=params.sampling_rate, + subtype="PCM_16", + ) + else: + logging.info("Decoding the test set") + infer_dataset( + dl=test_dl, + params=params, + model=model, + vocoder=vocoder, + denoiser=denoiser, + tokenizer=tokenizer, + ) + + +if __name__ == "__main__": + main() diff --git a/egs/ljspeech/TTS/matcha/inference.py b/egs/ljspeech/TTS/matcha/inference.py deleted file mode 100755 index 64abd8e50..000000000 --- a/egs/ljspeech/TTS/matcha/inference.py +++ /dev/null @@ -1,199 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2024 Xiaomi Corp. (authors: Fangjun Kuang) - -import argparse -import datetime as dt -import json -import logging -from pathlib import Path - -import soundfile as sf -import torch -from matcha.hifigan.config import v1, v2, v3 -from matcha.hifigan.denoiser import Denoiser -from matcha.hifigan.models import Generator as HiFiGAN -from tokenizer import Tokenizer -from train import get_model, get_params - -from icefall.checkpoint import load_checkpoint -from icefall.utils import AttributeDict - - -def get_parser(): - parser = argparse.ArgumentParser( - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - - parser.add_argument( - "--epoch", - type=int, - default=4000, - help="""It specifies the checkpoint to use for decoding. - Note: Epoch counts from 1. - """, - ) - - parser.add_argument( - "--exp-dir", - type=Path, - default="matcha/exp-new-3", - help="""The experiment dir. - It specifies the directory where all training related - files, e.g., checkpoints, log, etc, are saved - """, - ) - - parser.add_argument( - "--vocoder", - type=Path, - default="./generator_v1", - help="Path to the vocoder", - ) - - parser.add_argument( - "--tokens", - type=Path, - default="data/tokens.txt", - ) - - parser.add_argument( - "--cmvn", - type=str, - default="data/fbank/cmvn.json", - help="""Path to vocabulary.""", - ) - - parser.add_argument( - "--input-text", - type=str, - required=True, - help="The text to generate speech for", - ) - - parser.add_argument( - "--output-wav", - type=str, - required=True, - help="The filename of the wave to save the generated speech", - ) - - return parser - - -def load_vocoder(checkpoint_path): - checkpoint_path = str(checkpoint_path) - if checkpoint_path.endswith("v1"): - h = AttributeDict(v1) - elif checkpoint_path.endswith("v2"): - h = AttributeDict(v2) - elif checkpoint_path.endswith("v3"): - h = AttributeDict(v3) - else: - raise ValueError(f"supports only v1, v2, and v3, given {checkpoint_path}") - - hifigan = HiFiGAN(h).to("cpu") - hifigan.load_state_dict( - torch.load(checkpoint_path, map_location="cpu")["generator"] - ) - _ = hifigan.eval() - hifigan.remove_weight_norm() - return hifigan - - -def to_waveform(mel, vocoder, denoiser): - audio = vocoder(mel).clamp(-1, 1) - audio = denoiser(audio.squeeze(0), strength=0.00025).cpu().squeeze() - return audio.cpu().squeeze() - - -def process_text(text: str, tokenizer): - x = tokenizer.texts_to_token_ids([text], add_sos=True, add_eos=True) - x = torch.tensor(x, dtype=torch.long) - x_lengths = torch.tensor([x.shape[-1]], dtype=torch.long, device="cpu") - return {"x_orig": text, "x": x, "x_lengths": x_lengths} - - -def synthesise( - model, tokenizer, n_timesteps, text, length_scale, temperature, spks=None -): - text_processed = process_text(text, tokenizer) - start_t = dt.datetime.now() - output = model.synthesise( - text_processed["x"], - text_processed["x_lengths"], - n_timesteps=n_timesteps, - temperature=temperature, - spks=spks, - length_scale=length_scale, - ) - # merge everything to one dict - output.update({"start_t": start_t, **text_processed}) - return output - - -@torch.inference_mode() -def main(): - parser = get_parser() - args = parser.parse_args() - params = get_params() - - params.update(vars(args)) - - tokenizer = Tokenizer(params.tokens) - params.blank_id = tokenizer.pad_id - params.vocab_size = tokenizer.vocab_size - params.model_args.n_vocab = params.vocab_size - - with open(params.cmvn) as f: - stats = json.load(f) - params.data_args.data_statistics.mel_mean = stats["fbank_mean"] - params.data_args.data_statistics.mel_std = stats["fbank_std"] - - params.model_args.data_statistics.mel_mean = stats["fbank_mean"] - params.model_args.data_statistics.mel_std = stats["fbank_std"] - logging.info(params) - - logging.info("About to create model") - model = get_model(params) - - if not Path(f"{params.exp_dir}/epoch-{params.epoch}.pt").is_file(): - raise ValueError("{params.exp_dir}/epoch-{params.epoch}.pt does not exist") - - load_checkpoint(f"{params.exp_dir}/epoch-{params.epoch}.pt", model) - model.eval() - - if not Path(params.vocoder).is_file(): - raise ValueError(f"{params.vocoder} does not exist") - - vocoder = load_vocoder(params.vocoder) - denoiser = Denoiser(vocoder, mode="zeros") - - # Number of ODE Solver steps - n_timesteps = 2 - - # Changes to the speaking rate - length_scale = 1.0 - - # Sampling temperature - temperature = 0.667 - - output = synthesise( - model=model, - tokenizer=tokenizer, - n_timesteps=n_timesteps, - text=params.input_text, - length_scale=length_scale, - temperature=temperature, - ) - output["waveform"] = to_waveform(output["mel"], vocoder, denoiser) - - sf.write(params.output_wav, output["waveform"], 22050, "PCM_16") - - -if __name__ == "__main__": - formatter = "%(asctime)s %(levelname)s [%(filename)s:%(lineno)d] %(message)s" - - logging.basicConfig(format=formatter, level=logging.INFO) - torch.set_num_threads(1) - torch.set_num_interop_threads(1) - main() diff --git a/egs/ljspeech/TTS/matcha/models/components/decoder.py b/egs/ljspeech/TTS/matcha/models/components/decoder.py index 14d19f5d4..102d87713 100644 --- a/egs/ljspeech/TTS/matcha/models/components/decoder.py +++ b/egs/ljspeech/TTS/matcha/models/components/decoder.py @@ -7,7 +7,7 @@ import torch.nn.functional as F from conformer import ConformerBlock from diffusers.models.activations import get_activation from einops import pack, rearrange, repeat -from matcha.models.components.transformer import BasicTransformerBlock +from models.components.transformer import BasicTransformerBlock class SinusoidalPosEmb(torch.nn.Module): diff --git a/egs/ljspeech/TTS/matcha/models/components/flow_matching.py b/egs/ljspeech/TTS/matcha/models/components/flow_matching.py index 997689b1c..eb795ef32 100644 --- a/egs/ljspeech/TTS/matcha/models/components/flow_matching.py +++ b/egs/ljspeech/TTS/matcha/models/components/flow_matching.py @@ -2,7 +2,7 @@ from abc import ABC import torch import torch.nn.functional as F -from matcha.models.components.decoder import Decoder +from models.components.decoder import Decoder class BASECFM(torch.nn.Module, ABC): diff --git a/egs/ljspeech/TTS/matcha/models/components/text_encoder.py b/egs/ljspeech/TTS/matcha/models/components/text_encoder.py index ca77cba51..364ff1938 100644 --- a/egs/ljspeech/TTS/matcha/models/components/text_encoder.py +++ b/egs/ljspeech/TTS/matcha/models/components/text_encoder.py @@ -5,7 +5,7 @@ import math import torch import torch.nn as nn from einops import rearrange -from matcha.model import sequence_mask +from model import sequence_mask class LayerNorm(nn.Module): diff --git a/egs/ljspeech/TTS/matcha/models/matcha_tts.py b/egs/ljspeech/TTS/matcha/models/matcha_tts.py index 330d1dc47..fe0a72402 100644 --- a/egs/ljspeech/TTS/matcha/models/matcha_tts.py +++ b/egs/ljspeech/TTS/matcha/models/matcha_tts.py @@ -2,17 +2,17 @@ import datetime as dt import math import random -import matcha.monotonic_align as monotonic_align +import monotonic_align as monotonic_align import torch -from matcha.model import ( +from model import ( denormalize, duration_loss, fix_len_compatibility, generate_path, sequence_mask, ) -from matcha.models.components.flow_matching import CFM -from matcha.models.components.text_encoder import TextEncoder +from models.components.flow_matching import CFM +from models.components.text_encoder import TextEncoder class MatchaTTS(torch.nn.Module): # 🍵 diff --git a/egs/ljspeech/TTS/matcha/monotonic_align/.gitignore b/egs/ljspeech/TTS/matcha/monotonic_align/.gitignore index 28bdad6b8..3def4ae26 100644 --- a/egs/ljspeech/TTS/matcha/monotonic_align/.gitignore +++ b/egs/ljspeech/TTS/matcha/monotonic_align/.gitignore @@ -1,3 +1,3 @@ build core.c -*.so +*.so \ No newline at end of file diff --git a/egs/ljspeech/TTS/matcha/monotonic_align/__init__.py b/egs/ljspeech/TTS/matcha/monotonic_align/__init__.py index 5b26fe474..f87ae1bd3 100644 --- a/egs/ljspeech/TTS/matcha/monotonic_align/__init__.py +++ b/egs/ljspeech/TTS/matcha/monotonic_align/__init__.py @@ -1,8 +1,7 @@ -# Copied from -# https://github.com/shivammehta25/Matcha-TTS/blob/main/matcha/utils/monotonic_align/__init__.py import numpy as np import torch -from matcha.monotonic_align.core import maximum_path_c + +from .core import maximum_path_c def maximum_path(value, mask): diff --git a/egs/ljspeech/TTS/matcha/monotonic_align/core.pyx b/egs/ljspeech/TTS/matcha/monotonic_align/core.pyx index eabc7f273..091fcc3a5 100644 --- a/egs/ljspeech/TTS/matcha/monotonic_align/core.pyx +++ b/egs/ljspeech/TTS/matcha/monotonic_align/core.pyx @@ -1,5 +1,3 @@ -# Copied from -# https://github.com/shivammehta25/Matcha-TTS/blob/main/matcha/utils/monotonic_align/core.pyx import numpy as np cimport cython diff --git a/egs/ljspeech/TTS/matcha/monotonic_align/setup.py b/egs/ljspeech/TTS/matcha/monotonic_align/setup.py index df26c633e..beacf2e36 100644 --- a/egs/ljspeech/TTS/matcha/monotonic_align/setup.py +++ b/egs/ljspeech/TTS/matcha/monotonic_align/setup.py @@ -1,12 +1,30 @@ -# Copied from +# Modified from # https://github.com/shivammehta25/Matcha-TTS/blob/main/matcha/utils/monotonic_align/setup.py -from distutils.core import setup - -import numpy from Cython.Build import cythonize +from setuptools import Extension, setup +from setuptools.command.build_ext import build_ext as _build_ext + +class build_ext(_build_ext): + """Overwrite build_ext.""" + + def finalize_options(self): + """Prevent numpy from thinking it is still in its setup process.""" + _build_ext.finalize_options(self) + __builtins__.__NUMPY_SETUP__ = False + import numpy + + self.include_dirs.append(numpy.get_include()) + + +exts = [ + Extension( + name="core", + sources=["core.pyx"], + ) +] setup( name="monotonic_align", - ext_modules=cythonize("core.pyx"), - include_dirs=[numpy.get_include()], + ext_modules=cythonize(exts, language_level=3), + cmdclass={"build_ext": build_ext}, ) diff --git a/egs/ljspeech/TTS/matcha/requirements.txt b/egs/ljspeech/TTS/matcha/requirements.txt index 5aadc8984..d7829c1e1 100644 --- a/egs/ljspeech/TTS/matcha/requirements.txt +++ b/egs/ljspeech/TTS/matcha/requirements.txt @@ -1,3 +1,4 @@ conformer==0.3.2 diffusers # developed using version ==0.25.0 librosa +einops \ No newline at end of file diff --git a/egs/ljspeech/TTS/matcha/train.py b/egs/ljspeech/TTS/matcha/train.py index 5e713fdfd..31135f623 100755 --- a/egs/ljspeech/TTS/matcha/train.py +++ b/egs/ljspeech/TTS/matcha/train.py @@ -14,9 +14,9 @@ import torch import torch.multiprocessing as mp import torch.nn as nn from lhotse.utils import fix_random_seed -from matcha.model import fix_len_compatibility -from matcha.models.matcha_tts import MatchaTTS -from matcha.tokenizer import Tokenizer +from model import fix_len_compatibility +from models.matcha_tts import MatchaTTS +from tokenizer import Tokenizer from torch.cuda.amp import GradScaler, autocast from torch.nn.parallel import DistributedDataParallel as DDP from torch.optim import Optimizer @@ -150,7 +150,7 @@ def _get_data_params() -> AttributeDict: "n_spks": 1, "n_fft": 1024, "n_feats": 80, - "sample_rate": 22050, + "sampling_rate": 22050, "hop_length": 256, "win_length": 1024, "f_min": 0, @@ -445,11 +445,6 @@ def train_one_epoch( saved_bad_model = False - # used to track the stats over iterations in one epoch - tot_loss = MetricsTracker() - - saved_bad_model = False - def save_bad_model(suffix: str = ""): save_checkpoint( filename=params.exp_dir / f"bad-model{suffix}-{rank}.pt", diff --git a/egs/ljspeech/TTS/matcha/tts_datamodule.py b/egs/ljspeech/TTS/matcha/tts_datamodule.py index 8e37fc030..1e637b766 100644 --- a/egs/ljspeech/TTS/matcha/tts_datamodule.py +++ b/egs/ljspeech/TTS/matcha/tts_datamodule.py @@ -24,7 +24,7 @@ from pathlib import Path from typing import Any, Dict, Optional import torch -from compute_fbank_ljspeech import MyFbank, MyFbankConfig +from fbank import MatchaFbank, MatchaFbankConfig from lhotse import CutSet, load_manifest_lazy from lhotse.dataset import ( # noqa F401 for PrecomputedFeatures CutConcatenate, @@ -32,7 +32,6 @@ from lhotse.dataset import ( # noqa F401 for PrecomputedFeatures DynamicBucketingSampler, PrecomputedFeatures, SimpleCutSampler, - SpecAugment, SpeechSynthesisDataset, ) from lhotse.dataset.input_strategies import ( # noqa F401 For AudioSamples @@ -177,7 +176,7 @@ class LJSpeechTtsDataModule: if self.args.on_the_fly_feats: sampling_rate = 22050 - config = MyFbankConfig( + config = MatchaFbankConfig( n_fft=1024, n_mels=80, sampling_rate=sampling_rate, @@ -189,7 +188,7 @@ class LJSpeechTtsDataModule: train = SpeechSynthesisDataset( return_text=False, return_tokens=True, - feature_input_strategy=OnTheFlyFeatures(MyFbank(config)), + feature_input_strategy=OnTheFlyFeatures(MatchaFbank(config)), return_cuts=self.args.return_cuts, ) @@ -238,7 +237,7 @@ class LJSpeechTtsDataModule: logging.info("About to create dev dataset") if self.args.on_the_fly_feats: sampling_rate = 22050 - config = MyFbankConfig( + config = MatchaFbankConfig( n_fft=1024, n_mels=80, sampling_rate=sampling_rate, @@ -250,7 +249,7 @@ class LJSpeechTtsDataModule: validate = SpeechSynthesisDataset( return_text=False, return_tokens=True, - feature_input_strategy=OnTheFlyFeatures(MyFbank(config)), + feature_input_strategy=OnTheFlyFeatures(MatchaFbank(config)), return_cuts=self.args.return_cuts, ) else: @@ -282,7 +281,7 @@ class LJSpeechTtsDataModule: logging.info("About to create test dataset") if self.args.on_the_fly_feats: sampling_rate = 22050 - config = MyFbankConfig( + config = MatchaFbankConfig( n_fft=1024, n_mels=80, sampling_rate=sampling_rate, @@ -294,7 +293,7 @@ class LJSpeechTtsDataModule: test = SpeechSynthesisDataset( return_text=False, return_tokens=True, - feature_input_strategy=OnTheFlyFeatures(MyFbank(config)), + feature_input_strategy=OnTheFlyFeatures(MatchaFbank(config)), return_cuts=self.args.return_cuts, ) else: diff --git a/egs/ljspeech/TTS/prepare.sh b/egs/ljspeech/TTS/prepare.sh index 6f16f8d47..ec5062933 100755 --- a/egs/ljspeech/TTS/prepare.sh +++ b/egs/ljspeech/TTS/prepare.sh @@ -25,26 +25,16 @@ log() { log "dl_dir: $dl_dir" if [ $stage -le -1 ] && [ $stop_stage -ge -1 ]; then - log "Stage -1: build monotonic_align lib" - if [ ! -d vits/monotonic_align/build ]; then - cd vits/monotonic_align - python3 setup.py build_ext --inplace - cd ../../ - else - log "monotonic_align lib for vits already built" - fi - - if [ ! -f ./matcha/monotonic_align/core.cpython-38-x86_64-linux-gnu.so ]; then - pushd matcha/monotonic_align - python3 setup.py build - mv -v build/lib.*/matcha/monotonic_align/core.*.so . - rm -rf build - rm core.c - ls -lh - popd - else - log "monotonic_align lib for matcha-tts already built" - fi + log "Stage -1: build monotonic_align lib (used by vits and matcha recipes)" + for recipe in vits matcha; do + if [ ! -d $recipe/monotonic_align/build ]; then + cd $recipe/monotonic_align + python3 setup.py build_ext --inplace + cd ../../ + else + log "monotonic_align lib for $recipe already built" + fi + done fi if [ $stage -le 0 ] && [ $stop_stage -ge 0 ]; then diff --git a/egs/ljspeech/TTS/vits/infer.py b/egs/ljspeech/TTS/vits/infer.py index 7be76e315..cf1067dfc 100755 --- a/egs/ljspeech/TTS/vits/infer.py +++ b/egs/ljspeech/TTS/vits/infer.py @@ -234,7 +234,7 @@ def main(): logging.info(f"Number of parameters in discriminator: {num_param_d}") logging.info(f"Total number of parameters: {num_param_g + num_param_d}") - # we need cut ids to display recognition results. + # we need cut ids to organize tts results. args.return_cuts = True ljspeech = LJSpeechTtsDataModule(args) diff --git a/egs/ljspeech/TTS/vits/monotonic_align/.gitignore b/egs/ljspeech/TTS/vits/monotonic_align/.gitignore new file mode 100644 index 000000000..3def4ae26 --- /dev/null +++ b/egs/ljspeech/TTS/vits/monotonic_align/.gitignore @@ -0,0 +1,3 @@ +build +core.c +*.so \ No newline at end of file diff --git a/egs/ljspeech/TTS/vits/test_model.py b/egs/ljspeech/TTS/vits/test_model.py index 1de10f012..4faaa96a5 100755 --- a/egs/ljspeech/TTS/vits/test_model.py +++ b/egs/ljspeech/TTS/vits/test_model.py @@ -18,7 +18,6 @@ from tokenizer import Tokenizer from train import get_model, get_params -from vits import VITS def test_model_type(model_type): From 5c04f7bfb84a1f2f3b307d824a1355c9c8d30a20 Mon Sep 17 00:00:00 2001 From: goddamnVincent <84380030+goddamnVincent@users.noreply.github.com> Date: Sun, 8 Dec 2024 11:17:15 +0800 Subject: [PATCH 07/59] 'try to fix 'compute_fbank_kespeech_splits.py: error: unrecognized arguments: --speed-perturb true'' (#1812) --- .../ASR/local/compute_fbank_kespeech_dev_test.py | 12 +++++++++++- .../ASR/local/compute_fbank_kespeech_splits.py | 13 +++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/egs/multi_zh-hans/ASR/local/compute_fbank_kespeech_dev_test.py b/egs/multi_zh-hans/ASR/local/compute_fbank_kespeech_dev_test.py index 6f75dbfa4..5e169e894 100755 --- a/egs/multi_zh-hans/ASR/local/compute_fbank_kespeech_dev_test.py +++ b/egs/multi_zh-hans/ASR/local/compute_fbank_kespeech_dev_test.py @@ -52,13 +52,19 @@ def get_parser(): default=80, help="""The number of mel bins for Fbank""", ) - parser.add_argument( "--whisper-fbank", type=str2bool, default=False, help="Use WhisperFbank instead of Fbank. Default: False.", ) + parser.add_argument( + "--speed-perturb", + type=str2bool, + default=False, + help="Enable 0.9 and 1.1 speed perturbation for data augmentation. Default: False.", + ) + return parser @@ -104,6 +110,10 @@ def compute_fbank_kespeech_dev_test(args): keep_overlapping=False, min_duration=None ) + if args.speed_perturb: + cut_set = ( + cut_set + cut_set.perturb_speed(0.9) + cut_set.perturb_speed(1.1) + ) logging.info("Computing features") cut_set = cut_set.compute_and_store_features_batch( extractor=extractor, diff --git a/egs/multi_zh-hans/ASR/local/compute_fbank_kespeech_splits.py b/egs/multi_zh-hans/ASR/local/compute_fbank_kespeech_splits.py index c398411f6..6bb8af0d6 100755 --- a/egs/multi_zh-hans/ASR/local/compute_fbank_kespeech_splits.py +++ b/egs/multi_zh-hans/ASR/local/compute_fbank_kespeech_splits.py @@ -106,6 +106,14 @@ def get_parser(): default=False, help="Use WhisperFbank instead of Fbank. Default: False.", ) + + parser.add_argument( + "--speed-perturb", + type=str2bool, + default=False, + help="Enable 0.9 and 1.1 speed perturbation for data augmentation. Default: False.", + ) + return parser @@ -158,6 +166,11 @@ def compute_fbank_kespeech_splits(args): keep_overlapping=False, min_duration=None ) + if args.speed_perturb: + cut_set = ( + cut_set + cut_set.perturb_speed(0.9) + cut_set.perturb_speed(1.1) + ) + logging.info("Computing features") cut_set = cut_set.compute_and_store_features_batch( extractor=extractor, From d33f67817641b3911f91c2b76698b266290a5e01 Mon Sep 17 00:00:00 2001 From: zr_jin Date: Sun, 8 Dec 2024 16:37:24 +0800 Subject: [PATCH 08/59] fixed the formatting issue of PR#1812 (#1828) --- .../ASR/local/compute_fbank_kespeech_dev_test.py | 5 ++--- egs/multi_zh-hans/ASR/local/compute_fbank_kespeech_splits.py | 4 +--- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/egs/multi_zh-hans/ASR/local/compute_fbank_kespeech_dev_test.py b/egs/multi_zh-hans/ASR/local/compute_fbank_kespeech_dev_test.py index 5e169e894..2bbe28560 100755 --- a/egs/multi_zh-hans/ASR/local/compute_fbank_kespeech_dev_test.py +++ b/egs/multi_zh-hans/ASR/local/compute_fbank_kespeech_dev_test.py @@ -111,9 +111,8 @@ def compute_fbank_kespeech_dev_test(args): ) if args.speed_perturb: - cut_set = ( - cut_set + cut_set.perturb_speed(0.9) + cut_set.perturb_speed(1.1) - ) + cut_set = cut_set + cut_set.perturb_speed(0.9) + cut_set.perturb_speed(1.1) + logging.info("Computing features") cut_set = cut_set.compute_and_store_features_batch( extractor=extractor, diff --git a/egs/multi_zh-hans/ASR/local/compute_fbank_kespeech_splits.py b/egs/multi_zh-hans/ASR/local/compute_fbank_kespeech_splits.py index 6bb8af0d6..fe7f87337 100755 --- a/egs/multi_zh-hans/ASR/local/compute_fbank_kespeech_splits.py +++ b/egs/multi_zh-hans/ASR/local/compute_fbank_kespeech_splits.py @@ -167,9 +167,7 @@ def compute_fbank_kespeech_splits(args): ) if args.speed_perturb: - cut_set = ( - cut_set + cut_set.perturb_speed(0.9) + cut_set.perturb_speed(1.1) - ) + cut_set = cut_set + cut_set.perturb_speed(0.9) + cut_set.perturb_speed(1.1) logging.info("Computing features") cut_set = cut_set.compute_and_store_features_batch( From 32b7a449e7ed87efdf0a49f74b01c846e831c8a3 Mon Sep 17 00:00:00 2001 From: zr_jin Date: Sun, 8 Dec 2024 17:36:08 +0800 Subject: [PATCH 09/59] removed unnecessary type check (#1827) --- egs/wenetspeech4tts/TTS/valle/valle.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/egs/wenetspeech4tts/TTS/valle/valle.py b/egs/wenetspeech4tts/TTS/valle/valle.py index 4bfa2b577..772317428 100644 --- a/egs/wenetspeech4tts/TTS/valle/valle.py +++ b/egs/wenetspeech4tts/TTS/valle/valle.py @@ -1686,8 +1686,6 @@ class VALLE(nn.Module): decoder_outputs = decoder_outputs.to("cpu").type(torch.float32).detach().numpy() vmin, vmax = 0, 1024 # Encodec - if decoder_outputs.dtype == np.float32: - vmin, vmax = -6, 0 # Fbank num_figures = 3 for b, (utt_id, text) in enumerate(zip(utt_ids[:limit], texts[:limit])): From 08caa1e4e52f9c0684a91fcfce02487382fae45a Mon Sep 17 00:00:00 2001 From: zr_jin Date: Mon, 9 Dec 2024 22:59:29 +0800 Subject: [PATCH 10/59] minor fixes to the matcha recipe --- egs/ljspeech/TTS/matcha/train.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/egs/ljspeech/TTS/matcha/train.py b/egs/ljspeech/TTS/matcha/train.py index 31135f623..853042413 100755 --- a/egs/ljspeech/TTS/matcha/train.py +++ b/egs/ljspeech/TTS/matcha/train.py @@ -488,9 +488,10 @@ def train_one_epoch( loss = sum(losses.values()) - optimizer.zero_grad() scaler.scale(loss).backward() scaler.step(optimizer) + scaler.update() + optimizer.zero_grad() loss_info = MetricsTracker() loss_info["samples"] = batch_size From a43480af47896329e82917d89ceee65f15afbf25 Mon Sep 17 00:00:00 2001 From: zr_jin Date: Tue, 10 Dec 2024 11:15:49 +0800 Subject: [PATCH 11/59] fixed the not found python 3.8 env (#1830) --- .github/workflows/style_check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/style_check.yml b/.github/workflows/style_check.yml index 0681ece60..2a077fa91 100644 --- a/.github/workflows/style_check.yml +++ b/.github/workflows/style_check.yml @@ -36,7 +36,7 @@ jobs: strategy: matrix: os: [ubuntu-latest] - python-version: [3.8] + python-version: [3.10.15] fail-fast: false steps: From b7acf0f57b3ad03cb98752a6e57216dc539eee14 Mon Sep 17 00:00:00 2001 From: zr_jin Date: Wed, 11 Dec 2024 14:33:47 +0800 Subject: [PATCH 12/59] minor fixes --- egs/ljspeech/TTS/README.md | 2 +- egs/ljspeech/TTS/matcha/onnx_pretrained.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/egs/ljspeech/TTS/README.md b/egs/ljspeech/TTS/README.md index 82850cd04..39280437b 100644 --- a/egs/ljspeech/TTS/README.md +++ b/egs/ljspeech/TTS/README.md @@ -131,7 +131,7 @@ To inference, use: wget https://github.com/csukuangfj/models/raw/refs/heads/master/hifigan/generator_v1 -./matcha/synth.py \ +./matcha/infer.py \ --exp-dir ./matcha/exp-new-3 \ --epoch 4000 \ --tokens ./data/tokens.txt \ diff --git a/egs/ljspeech/TTS/matcha/onnx_pretrained.py b/egs/ljspeech/TTS/matcha/onnx_pretrained.py index be34343d3..4eff9a084 100755 --- a/egs/ljspeech/TTS/matcha/onnx_pretrained.py +++ b/egs/ljspeech/TTS/matcha/onnx_pretrained.py @@ -8,7 +8,7 @@ import logging import onnxruntime as ort import soundfile as sf import torch -from inference import load_vocoder +from infer import load_vocoder from tokenizer import Tokenizer From 3e4da5f78160d3dba3bdf97968bd7ceb8c11631f Mon Sep 17 00:00:00 2001 From: Li Peng Date: Mon, 16 Dec 2024 10:24:16 +0800 Subject: [PATCH 13/59] Replace deprecated pytorch methods (#1814) * Replace deprecated pytorch methods - torch.cuda.amp.GradScaler(...) => torch.amp.GradScaler("cuda", ...) - torch.cuda.amp.autocast(...) => torch.amp.autocast("cuda", ...) * Replace `with autocast(...)` with `with autocast("cuda", ...)` Co-authored-by: Li Peng --- .../ASR/pruned_transducer_stateless2/train.py | 8 ++++---- .../ASR/pruned_transducer_stateless2/train.py | 8 ++++---- .../ASR/pruned_transducer_stateless3/model.py | 4 ++-- .../ASR/pruned_transducer_stateless3/train.py | 8 ++++---- .../do_not_use_it_directly.py | 8 ++++---- .../ASR/pruned_transducer_stateless7/train.py | 8 ++++---- .../ASR/pruned_transducer_stateless7_bbpe/train.py | 8 ++++---- .../do_not_use_it_directly.py | 8 ++++---- .../pruned_transducer_stateless7_streaming/train.py | 8 ++++---- egs/aishell/ASR/whisper/train.py | 8 ++++---- egs/aishell/ASR/zipformer/train.py | 8 ++++---- egs/aishell/ASR/zipformer/train_bbpe.py | 8 ++++---- .../ASR/pruned_transducer_stateless5/train.py | 8 ++++---- .../ASR/pruned_transducer_stateless5/train.py | 8 ++++---- .../ASR/pruned_transducer_stateless2/train.py | 8 ++++---- .../ASR_v2/pruned_transducer_stateless7/train.py | 8 ++++---- egs/ami/ASR/pruned_transducer_stateless7/train.py | 8 ++++---- egs/ami/SURT/dprnn_zipformer/train.py | 6 +++--- egs/ami/SURT/dprnn_zipformer/train_adapt.py | 6 +++--- egs/audioset/AT/zipformer/train.py | 8 ++++---- .../ASR/pruned_transducer_stateless7/train.py | 8 ++++---- .../do_not_use_it_directly.py | 8 ++++---- .../finetune.py | 8 ++++---- .../pruned_transducer_stateless7_streaming/train.py | 8 ++++---- egs/commonvoice/ASR/zipformer/train.py | 8 ++++---- egs/commonvoice/ASR/zipformer/train_char.py | 8 ++++---- .../do_not_use_it_directly.py | 8 ++++---- .../pruned_transducer_stateless7_streaming/train.py | 8 ++++---- .../ASR/pruned_transducer_stateless2/train.py | 8 ++++---- egs/gigaspeech/ASR/zipformer/train.py | 8 ++++---- egs/gigaspeech/KWS/zipformer/finetune.py | 6 +++--- egs/gigaspeech/KWS/zipformer/train.py | 8 ++++---- .../pruned_transducer_stateless7_streaming/train.py | 8 ++++---- egs/ksponspeech/ASR/zipformer/train.py | 8 ++++---- egs/libricss/SURT/dprnn_zipformer/model.py | 4 ++-- egs/libricss/SURT/dprnn_zipformer/scaling.py | 6 +++--- egs/libricss/SURT/dprnn_zipformer/train.py | 6 +++--- egs/libricss/SURT/dprnn_zipformer/train_adapt.py | 6 +++--- egs/libriheavy/ASR/zipformer/train.py | 8 ++++---- .../ASR/zipformer_prompt_asr/model_baseline.py | 4 ++-- .../ASR/zipformer_prompt_asr/model_with_BERT.py | 4 ++-- egs/libriheavy/ASR/zipformer_prompt_asr/scaling.py | 10 +++++----- .../ASR/zipformer_prompt_asr/train_baseline.py | 8 ++++---- .../ASR/zipformer_prompt_asr/train_bert_encoder.py | 8 ++++---- egs/libriheavy/ASR/zipformer_prompt_asr/zipformer.py | 4 ++-- egs/librilight/SSL/zipformer/finetune.py | 8 ++++---- egs/librilight/SSL/zipformer/pretrain.py | 8 ++++---- egs/librispeech/ASR/conformer_ctc2/train.py | 8 ++++---- egs/librispeech/ASR/conformer_ctc3/train.py | 8 ++++---- .../ASR/conv_emformer_transducer_stateless/train.py | 8 ++++---- .../do_not_use_it_directly.py | 8 ++++---- .../ASR/conv_emformer_transducer_stateless2/train.py | 8 ++++---- .../ASR/lstm_transducer_stateless/model.py | 4 ++-- .../ASR/lstm_transducer_stateless/train.py | 8 ++++---- .../ASR/lstm_transducer_stateless2/model.py | 4 ++-- .../ASR/lstm_transducer_stateless2/train.py | 8 ++++---- .../ASR/lstm_transducer_stateless3/train.py | 8 ++++---- egs/librispeech/ASR/pruned2_knowledge/model.py | 4 ++-- egs/librispeech/ASR/pruned2_knowledge/sampling.py | 6 +++--- egs/librispeech/ASR/pruned2_knowledge/train.py | 8 ++++---- .../ASR/pruned_stateless_emformer_rnnt2/train.py | 8 ++++---- .../ASR/pruned_transducer_stateless2/model.py | 4 ++-- .../ASR/pruned_transducer_stateless2/train.py | 8 ++++---- .../ASR/pruned_transducer_stateless3/model.py | 4 ++-- .../ASR/pruned_transducer_stateless3/train.py | 8 ++++---- .../ASR/pruned_transducer_stateless4/train.py | 8 ++++---- .../ASR/pruned_transducer_stateless5/train.py | 8 ++++---- .../ASR/pruned_transducer_stateless6/model.py | 4 ++-- .../ASR/pruned_transducer_stateless6/train.py | 8 ++++---- .../ASR/pruned_transducer_stateless7/finetune.py | 8 ++++---- .../ASR/pruned_transducer_stateless7/model.py | 4 ++-- .../ASR/pruned_transducer_stateless7/scaling.py | 6 +++--- .../ASR/pruned_transducer_stateless7/train.py | 8 ++++---- .../ASR/pruned_transducer_stateless7/zipformer.py | 2 +- .../ASR/pruned_transducer_stateless7_ctc/model.py | 4 ++-- .../ASR/pruned_transducer_stateless7_ctc/train.py | 8 ++++---- .../ASR/pruned_transducer_stateless7_ctc_bs/model.py | 4 ++-- .../ASR/pruned_transducer_stateless7_ctc_bs/train.py | 8 ++++---- .../do_not_use_it_directly.py | 8 ++++---- .../pruned_transducer_stateless7_streaming/train.py | 8 ++++---- .../zipformer.py | 2 +- .../zipformer_for_ncnn_export_only.py | 2 +- .../train.py | 8 ++++---- .../ASR/pruned_transducer_stateless8/model.py | 4 ++-- .../ASR/pruned_transducer_stateless8/train.py | 8 ++++---- egs/librispeech/ASR/tiny_transducer_ctc/train.py | 8 ++++---- egs/librispeech/ASR/zipformer/finetune.py | 8 ++++---- egs/librispeech/ASR/zipformer/model.py | 4 ++-- egs/librispeech/ASR/zipformer/scaling.py | 10 +++++----- egs/librispeech/ASR/zipformer/train.py | 8 ++++---- egs/librispeech/ASR/zipformer/zipformer.py | 2 +- egs/librispeech/ASR/zipformer_adapter/train.py | 8 ++++---- egs/librispeech/ASR/zipformer_adapter/zipformer.py | 2 +- egs/librispeech/ASR/zipformer_ctc/train.py | 6 +++--- egs/librispeech/ASR/zipformer_lora/finetune.py | 8 ++++---- egs/librispeech/ASR/zipformer_lora/scaling.py | 10 +++++----- egs/librispeech/ASR/zipformer_lora/train.py | 8 ++++---- egs/librispeech/ASR/zipformer_lora/zipformer.py | 2 +- egs/librispeech/ASR/zipformer_mmi/train.py | 8 ++++---- egs/librispeech/SSL/hubert/finetune.py | 8 ++++---- egs/librispeech/SSL/hubert/finetune_ce.py | 8 ++++---- egs/librispeech/SSL/hubert/model.py | 4 ++-- egs/librispeech/SSL/hubert/pretrain.py | 8 ++++---- egs/librispeech/SSL/hubert/pretrain_ce.py | 8 ++++---- egs/librispeech/SSL/zipformer/finetune.py | 8 ++++---- egs/librispeech/SSL/zipformer/model.py | 4 ++-- egs/librispeech/SSL/zipformer/pretrain.py | 8 ++++---- egs/librispeech/SSL/zipformer/zipformer.py | 2 +- egs/librispeech/WSASR/conformer_ctc2/train.py | 8 ++++---- egs/librispeech/WSASR/conformer_ctc2/train_phone.py | 8 ++++---- egs/libritts/ASR/zipformer/train.py | 12 ++++++------ egs/libritts/CODEC/encodec/encodec.py | 6 +++--- egs/libritts/CODEC/encodec/train.py | 12 ++++++------ egs/libritts/TTS/vits/train.py | 12 ++++++------ egs/ljspeech/TTS/matcha/train.py | 6 +++--- egs/ljspeech/TTS/vits/train.py | 12 ++++++------ egs/ljspeech/TTS/vits/utils.py | 2 +- egs/ljspeech/TTS/vits/vits.py | 6 +++--- egs/mdcc/ASR/zipformer/train.py | 8 ++++---- egs/mgb2/ASR/pruned_transducer_stateless5/train.py | 8 ++++---- egs/multi_zh-hans/ASR/whisper/train.py | 8 ++++---- egs/multi_zh-hans/ASR/zipformer/train.py | 8 ++++---- egs/multi_zh_en/ASR/zipformer/train.py | 8 ++++---- .../ASR/zipformer/do_not_use_it_directly.py | 8 ++++---- egs/reazonspeech/ASR/zipformer/train.py | 8 ++++---- egs/speech_llm/ASR_LLM/whisper_llm_zh/train.py | 4 ++-- .../ASR/pruned_transducer_stateless2/train.py | 8 ++++---- egs/spgispeech/ASR/zipformer/train.py | 8 ++++---- .../ASR/pruned_transducer_stateless5/train.py | 8 ++++---- .../ASR/pruned_transducer_stateless7_bbpe/train.py | 8 ++++---- egs/tedlium3/ASR/conformer_ctc2/train.py | 8 ++++---- egs/tedlium3/ASR/zipformer/model.py | 4 ++-- egs/tedlium3/ASR/zipformer/train.py | 8 ++++---- egs/vctk/TTS/vits/train.py | 12 ++++++------ .../ASR/pruned_transducer_stateless2/finetune.py | 8 ++++---- .../ASR/pruned_transducer_stateless2/train.py | 8 ++++---- .../ASR/pruned_transducer_stateless5/train.py | 8 ++++---- egs/wenetspeech/ASR/whisper/train.py | 8 ++++---- egs/wenetspeech/ASR/zipformer/train.py | 8 ++++---- egs/wenetspeech/KWS/zipformer/finetune.py | 6 +++--- egs/wenetspeech/KWS/zipformer/train.py | 8 ++++---- egs/wenetspeech4tts/TTS/valle/train.py | 12 +++++++----- .../ASR/pruned_transducer_stateless5/train.py | 8 ++++---- .../ASR/pruned_transducer_stateless7/train.py | 8 ++++---- icefall/checkpoint.py | 2 +- icefall/rnn_lm/train.py | 4 ++-- icefall/transformer_lm/train.py | 4 ++-- 147 files changed, 520 insertions(+), 518 deletions(-) diff --git a/egs/aidatatang_200zh/ASR/pruned_transducer_stateless2/train.py b/egs/aidatatang_200zh/ASR/pruned_transducer_stateless2/train.py index fa809b768..9088378fa 100644 --- a/egs/aidatatang_200zh/ASR/pruned_transducer_stateless2/train.py +++ b/egs/aidatatang_200zh/ASR/pruned_transducer_stateless2/train.py @@ -67,7 +67,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, Eve from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -638,7 +638,7 @@ def train_one_epoch( params.batch_idx_train += 1 batch_size = len(batch["supervisions"]["text"]) - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -843,7 +843,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16) + scaler = GradScaler("cuda", enabled=params.use_fp16) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -912,7 +912,7 @@ def scan_pessimistic_batches_for_oom( # warmup = 0.0 is so that the derivs for the pruned loss stay zero # (i.e. are not remembered by the decaying-average in adam), because # we want to avoid these params being subject to shrinkage in adam. - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/aishell/ASR/pruned_transducer_stateless2/train.py b/egs/aishell/ASR/pruned_transducer_stateless2/train.py index 60f014c48..dda098e99 100755 --- a/egs/aishell/ASR/pruned_transducer_stateless2/train.py +++ b/egs/aishell/ASR/pruned_transducer_stateless2/train.py @@ -60,7 +60,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, Eve from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -688,7 +688,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -888,7 +888,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16) + scaler = GradScaler("cuda", enabled=params.use_fp16) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -989,7 +989,7 @@ def scan_pessimistic_batches_for_oom( # warmup = 0.0 is so that the derivs for the pruned loss stay zero # (i.e. are not remembered by the decaying-average in adam), because # we want to avoid these params being subject to shrinkage in adam. - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/aishell/ASR/pruned_transducer_stateless3/model.py b/egs/aishell/ASR/pruned_transducer_stateless3/model.py index a4dda0d6d..cafc9d1bb 100644 --- a/egs/aishell/ASR/pruned_transducer_stateless3/model.py +++ b/egs/aishell/ASR/pruned_transducer_stateless3/model.py @@ -184,7 +184,7 @@ class Transducer(nn.Module): lm = simple_lm_proj(decoder_out) am = simple_am_proj(encoder_out) - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): simple_loss, (px_grad, py_grad) = k2.rnnt_loss_smoothed( lm=lm.float(), am=am.float(), @@ -219,7 +219,7 @@ class Transducer(nn.Module): # prior to do_rnnt_pruning (this is an optimization for speed). logits = joiner(am_pruned, lm_pruned, project_input=False) - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): pruned_loss = k2.rnnt_loss_pruned( logits=logits.float(), symbols=y_padded, diff --git a/egs/aishell/ASR/pruned_transducer_stateless3/train.py b/egs/aishell/ASR/pruned_transducer_stateless3/train.py index 7c23041ca..bf60c4fad 100755 --- a/egs/aishell/ASR/pruned_transducer_stateless3/train.py +++ b/egs/aishell/ASR/pruned_transducer_stateless3/train.py @@ -79,7 +79,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, Eve from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -797,7 +797,7 @@ def train_one_epoch( aishell = is_aishell(batch["supervisions"]["cut"][0]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1096,7 +1096,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16) + scaler = GradScaler("cuda", enabled=params.use_fp16) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1202,7 +1202,7 @@ def scan_pessimistic_batches_for_oom( # warmup = 0.0 is so that the derivs for the pruned loss stay zero # (i.e. are not remembered by the decaying-average in adam), because # we want to avoid these params being subject to shrinkage in adam. - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/aishell/ASR/pruned_transducer_stateless7/do_not_use_it_directly.py b/egs/aishell/ASR/pruned_transducer_stateless7/do_not_use_it_directly.py index 058d0ff6b..9a9d92c20 100755 --- a/egs/aishell/ASR/pruned_transducer_stateless7/do_not_use_it_directly.py +++ b/egs/aishell/ASR/pruned_transducer_stateless7/do_not_use_it_directly.py @@ -74,7 +74,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, ScaledAdam from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer @@ -812,7 +812,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1107,7 +1107,7 @@ def run(rank, world_size, args): # params=params, # ) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1206,7 +1206,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/aishell/ASR/pruned_transducer_stateless7/train.py b/egs/aishell/ASR/pruned_transducer_stateless7/train.py index 2dc835f3b..ede2bd3e5 100755 --- a/egs/aishell/ASR/pruned_transducer_stateless7/train.py +++ b/egs/aishell/ASR/pruned_transducer_stateless7/train.py @@ -70,7 +70,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, ScaledAdam from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer @@ -809,7 +809,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1107,7 +1107,7 @@ def run(rank, world_size, args): # params=params, # ) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1206,7 +1206,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/aishell/ASR/pruned_transducer_stateless7_bbpe/train.py b/egs/aishell/ASR/pruned_transducer_stateless7_bbpe/train.py index 811269989..be48d6dde 100755 --- a/egs/aishell/ASR/pruned_transducer_stateless7_bbpe/train.py +++ b/egs/aishell/ASR/pruned_transducer_stateless7_bbpe/train.py @@ -64,7 +64,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, ScaledAdam from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer @@ -802,7 +802,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1102,7 +1102,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1202,7 +1202,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/aishell/ASR/pruned_transducer_stateless7_streaming/do_not_use_it_directly.py b/egs/aishell/ASR/pruned_transducer_stateless7_streaming/do_not_use_it_directly.py index 6653d9d9c..e3387e670 100755 --- a/egs/aishell/ASR/pruned_transducer_stateless7_streaming/do_not_use_it_directly.py +++ b/egs/aishell/ASR/pruned_transducer_stateless7_streaming/do_not_use_it_directly.py @@ -63,7 +63,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, ScaledAdam from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer_for_ncnn_export_only import Zipformer @@ -813,7 +813,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1105,7 +1105,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1205,7 +1205,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/aishell/ASR/pruned_transducer_stateless7_streaming/train.py b/egs/aishell/ASR/pruned_transducer_stateless7_streaming/train.py index f3b0f1e11..cba312214 100755 --- a/egs/aishell/ASR/pruned_transducer_stateless7_streaming/train.py +++ b/egs/aishell/ASR/pruned_transducer_stateless7_streaming/train.py @@ -63,7 +63,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, ScaledAdam from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer @@ -812,7 +812,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1104,7 +1104,7 @@ def run(rank, world_size, args): # params=params, # ) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1202,7 +1202,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/aishell/ASR/whisper/train.py b/egs/aishell/ASR/whisper/train.py index d77f8c270..e84dcf156 100755 --- a/egs/aishell/ASR/whisper/train.py +++ b/egs/aishell/ASR/whisper/train.py @@ -62,7 +62,7 @@ from lhotse.dataset.sampling.base import CutSampler from lhotse.utils import fix_random_seed from optim import Eden, ScaledAdam from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.functional import pad as pad_tensor from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -514,7 +514,7 @@ def compute_validation_loss( tot_loss = MetricsTracker() for batch_idx, batch in enumerate(valid_dl): - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, tokenizer=tokenizer, @@ -608,7 +608,7 @@ def train_one_epoch( ) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, tokenizer=tokenizer, @@ -812,7 +812,7 @@ def run(rank, world_size, args): train_dl = aishell.train_dataloaders(aishell.train_cuts()) valid_dl = aishell.valid_dataloaders(aishell.valid_cuts()) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) diff --git a/egs/aishell/ASR/zipformer/train.py b/egs/aishell/ASR/zipformer/train.py index cd253c597..ab568b20f 100755 --- a/egs/aishell/ASR/zipformer/train.py +++ b/egs/aishell/ASR/zipformer/train.py @@ -71,7 +71,7 @@ from optim import Eden, ScaledAdam from scaling import ScheduledFloat from subsampling import Conv2dSubsampling from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer2 @@ -910,7 +910,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1201,7 +1201,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1302,7 +1302,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/aishell/ASR/zipformer/train_bbpe.py b/egs/aishell/ASR/zipformer/train_bbpe.py index 46a5506db..2dac0cc64 100755 --- a/egs/aishell/ASR/zipformer/train_bbpe.py +++ b/egs/aishell/ASR/zipformer/train_bbpe.py @@ -61,7 +61,7 @@ from lhotse.cut import Cut from lhotse.utils import fix_random_seed from optim import Eden, ScaledAdam from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from train import ( @@ -495,7 +495,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -795,7 +795,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -895,7 +895,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/aishell2/ASR/pruned_transducer_stateless5/train.py b/egs/aishell2/ASR/pruned_transducer_stateless5/train.py index 8c7448d4c..772d9e6bf 100755 --- a/egs/aishell2/ASR/pruned_transducer_stateless5/train.py +++ b/egs/aishell2/ASR/pruned_transducer_stateless5/train.py @@ -75,7 +75,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, Eve from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -734,7 +734,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -963,7 +963,7 @@ def run(rank, world_size, args): warmup=0.0 if params.start_epoch == 1 else 1.0, ) - scaler = GradScaler(enabled=params.use_fp16) + scaler = GradScaler("cuda", enabled=params.use_fp16) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1062,7 +1062,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/aishell4/ASR/pruned_transducer_stateless5/train.py b/egs/aishell4/ASR/pruned_transducer_stateless5/train.py index a354f761e..0eb9271f5 100755 --- a/egs/aishell4/ASR/pruned_transducer_stateless5/train.py +++ b/egs/aishell4/ASR/pruned_transducer_stateless5/train.py @@ -68,7 +68,7 @@ from local.text_normalize import text_normalize from model import Transducer from optim import Eden, Eve from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -727,7 +727,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) # print(batch["supervisions"]) - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -963,7 +963,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16) + scaler = GradScaler("cuda", enabled=params.use_fp16) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1034,7 +1034,7 @@ def scan_pessimistic_batches_for_oom( # warmup = 0.0 is so that the derivs for the pruned loss stay zero # (i.e. are not remembered by the decaying-average in adam), because # we want to avoid these params being subject to shrinkage in adam. - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/alimeeting/ASR/pruned_transducer_stateless2/train.py b/egs/alimeeting/ASR/pruned_transducer_stateless2/train.py index 30154291d..2b1b6f9b4 100644 --- a/egs/alimeeting/ASR/pruned_transducer_stateless2/train.py +++ b/egs/alimeeting/ASR/pruned_transducer_stateless2/train.py @@ -67,7 +67,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, Eve from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -638,7 +638,7 @@ def train_one_epoch( params.batch_idx_train += 1 batch_size = len(batch["supervisions"]["text"]) - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -843,7 +843,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16) + scaler = GradScaler("cuda", enabled=params.use_fp16) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -912,7 +912,7 @@ def scan_pessimistic_batches_for_oom( # warmup = 0.0 is so that the derivs for the pruned loss stay zero # (i.e. are not remembered by the decaying-average in adam), because # we want to avoid these params being subject to shrinkage in adam. - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/alimeeting/ASR_v2/pruned_transducer_stateless7/train.py b/egs/alimeeting/ASR_v2/pruned_transducer_stateless7/train.py index 30879d8d2..e321deeb1 100755 --- a/egs/alimeeting/ASR_v2/pruned_transducer_stateless7/train.py +++ b/egs/alimeeting/ASR_v2/pruned_transducer_stateless7/train.py @@ -55,7 +55,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, ScaledAdam from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer @@ -782,7 +782,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1031,7 +1031,7 @@ def run(rank, world_size, args): # params=params, # ) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1127,7 +1127,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/ami/ASR/pruned_transducer_stateless7/train.py b/egs/ami/ASR/pruned_transducer_stateless7/train.py index d62cdadb7..97ebc5bcf 100755 --- a/egs/ami/ASR/pruned_transducer_stateless7/train.py +++ b/egs/ami/ASR/pruned_transducer_stateless7/train.py @@ -55,7 +55,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, ScaledAdam from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer @@ -773,7 +773,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1034,7 +1034,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1134,7 +1134,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/ami/SURT/dprnn_zipformer/train.py b/egs/ami/SURT/dprnn_zipformer/train.py index adc6a8495..9e77c0527 100755 --- a/egs/ami/SURT/dprnn_zipformer/train.py +++ b/egs/ami/SURT/dprnn_zipformer/train.py @@ -61,7 +61,7 @@ from model import SURT from optim import Eden, ScaledAdam from scaling import ScaledLinear, ScaledLSTM from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer @@ -1067,7 +1067,7 @@ def train_one_epoch( batch_size = batch["inputs"].shape[0] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1314,7 +1314,7 @@ def run(rank, world_size, args): ) valid_dl = ami.valid_dataloaders(dev_cuts) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) diff --git a/egs/ami/SURT/dprnn_zipformer/train_adapt.py b/egs/ami/SURT/dprnn_zipformer/train_adapt.py index ac5b0dadc..0647a7c78 100755 --- a/egs/ami/SURT/dprnn_zipformer/train_adapt.py +++ b/egs/ami/SURT/dprnn_zipformer/train_adapt.py @@ -61,7 +61,7 @@ from model import SURT from optim import Eden, ScaledAdam from scaling import ScaledLinear, ScaledLSTM from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer @@ -1058,7 +1058,7 @@ def train_one_epoch( batch_size = batch["inputs"].shape[0] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1305,7 +1305,7 @@ def run(rank, world_size, args): ) valid_dl = ami.valid_dataloaders(dev_cuts) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) diff --git a/egs/audioset/AT/zipformer/train.py b/egs/audioset/AT/zipformer/train.py index 67c703364..9532ed906 100644 --- a/egs/audioset/AT/zipformer/train.py +++ b/egs/audioset/AT/zipformer/train.py @@ -53,7 +53,7 @@ from optim import Eden, ScaledAdam from scaling import ScheduledFloat from subsampling import Conv2dSubsampling from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer2 @@ -799,7 +799,7 @@ def train_one_epoch( num_samples += batch_size try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1057,7 +1057,7 @@ def run(rank, world_size, args): valid_cuts = audioset.audioset_eval_cuts() valid_dl = audioset.valid_dataloaders(valid_cuts) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1148,7 +1148,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/commonvoice/ASR/pruned_transducer_stateless7/train.py b/egs/commonvoice/ASR/pruned_transducer_stateless7/train.py index 5e98084ec..486ab73df 100755 --- a/egs/commonvoice/ASR/pruned_transducer_stateless7/train.py +++ b/egs/commonvoice/ASR/pruned_transducer_stateless7/train.py @@ -66,7 +66,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, ScaledAdam from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer @@ -825,7 +825,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1120,7 +1120,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1220,7 +1220,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/commonvoice/ASR/pruned_transducer_stateless7_streaming/do_not_use_it_directly.py b/egs/commonvoice/ASR/pruned_transducer_stateless7_streaming/do_not_use_it_directly.py index aefe88f3f..fa241abe7 100755 --- a/egs/commonvoice/ASR/pruned_transducer_stateless7_streaming/do_not_use_it_directly.py +++ b/egs/commonvoice/ASR/pruned_transducer_stateless7_streaming/do_not_use_it_directly.py @@ -65,7 +65,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, ScaledAdam from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer_for_ncnn_export_only import Zipformer @@ -818,7 +818,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1109,7 +1109,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1209,7 +1209,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/commonvoice/ASR/pruned_transducer_stateless7_streaming/finetune.py b/egs/commonvoice/ASR/pruned_transducer_stateless7_streaming/finetune.py index 976004eca..8905dc617 100755 --- a/egs/commonvoice/ASR/pruned_transducer_stateless7_streaming/finetune.py +++ b/egs/commonvoice/ASR/pruned_transducer_stateless7_streaming/finetune.py @@ -68,7 +68,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, ScaledAdam from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer @@ -895,7 +895,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1193,7 +1193,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1293,7 +1293,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/commonvoice/ASR/pruned_transducer_stateless7_streaming/train.py b/egs/commonvoice/ASR/pruned_transducer_stateless7_streaming/train.py index 67e1a8133..8260c4985 100755 --- a/egs/commonvoice/ASR/pruned_transducer_stateless7_streaming/train.py +++ b/egs/commonvoice/ASR/pruned_transducer_stateless7_streaming/train.py @@ -65,7 +65,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, ScaledAdam from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer @@ -840,7 +840,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1137,7 +1137,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1237,7 +1237,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/commonvoice/ASR/zipformer/train.py b/egs/commonvoice/ASR/zipformer/train.py index 271014db0..c0219df19 100755 --- a/egs/commonvoice/ASR/zipformer/train.py +++ b/egs/commonvoice/ASR/zipformer/train.py @@ -75,7 +75,7 @@ from optim import Eden, ScaledAdam from scaling import ScheduledFloat from subsampling import Conv2dSubsampling from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer2 @@ -969,7 +969,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1265,7 +1265,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1365,7 +1365,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/commonvoice/ASR/zipformer/train_char.py b/egs/commonvoice/ASR/zipformer/train_char.py index 0aa7856cc..639e1067a 100755 --- a/egs/commonvoice/ASR/zipformer/train_char.py +++ b/egs/commonvoice/ASR/zipformer/train_char.py @@ -67,7 +67,7 @@ from lhotse.cut import Cut from lhotse.utils import fix_random_seed from optim import Eden, ScaledAdam from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from train import ( @@ -604,7 +604,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -784,7 +784,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, @@ -979,7 +979,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) diff --git a/egs/csj/ASR/pruned_transducer_stateless7_streaming/do_not_use_it_directly.py b/egs/csj/ASR/pruned_transducer_stateless7_streaming/do_not_use_it_directly.py index 6d256308c..661bfa6ca 100755 --- a/egs/csj/ASR/pruned_transducer_stateless7_streaming/do_not_use_it_directly.py +++ b/egs/csj/ASR/pruned_transducer_stateless7_streaming/do_not_use_it_directly.py @@ -67,7 +67,7 @@ from model import Transducer from optim import Eden, ScaledAdam from tokenizer import Tokenizer from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer_for_ncnn_export_only import Zipformer @@ -839,7 +839,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1146,7 +1146,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1246,7 +1246,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/csj/ASR/pruned_transducer_stateless7_streaming/train.py b/egs/csj/ASR/pruned_transducer_stateless7_streaming/train.py index ef7ea9013..8f07fc42f 100755 --- a/egs/csj/ASR/pruned_transducer_stateless7_streaming/train.py +++ b/egs/csj/ASR/pruned_transducer_stateless7_streaming/train.py @@ -67,7 +67,7 @@ from model import Transducer from optim import Eden, ScaledAdam from tokenizer import Tokenizer from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer @@ -838,7 +838,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1145,7 +1145,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1245,7 +1245,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/gigaspeech/ASR/pruned_transducer_stateless2/train.py b/egs/gigaspeech/ASR/pruned_transducer_stateless2/train.py index a7772b62f..e0e11fc70 100755 --- a/egs/gigaspeech/ASR/pruned_transducer_stateless2/train.py +++ b/egs/gigaspeech/ASR/pruned_transducer_stateless2/train.py @@ -64,7 +64,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, Eve from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -675,7 +675,7 @@ def train_one_epoch( params.batch_idx_train += 1 batch_size = len(batch["supervisions"]["text"]) - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -873,7 +873,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16) + scaler = GradScaler("cuda", enabled=params.use_fp16) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -944,7 +944,7 @@ def scan_pessimistic_batches_for_oom( # warmup = 0.0 is so that the derivs for the pruned loss stay zero # (i.e. are not remembered by the decaying-average in adam), because # we want to avoid these params being subject to shrinkage in adam. - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/gigaspeech/ASR/zipformer/train.py b/egs/gigaspeech/ASR/zipformer/train.py index 4c122effe..5092ef8cb 100755 --- a/egs/gigaspeech/ASR/zipformer/train.py +++ b/egs/gigaspeech/ASR/zipformer/train.py @@ -75,7 +75,7 @@ from optim import Eden, ScaledAdam from scaling import ScheduledFloat from subsampling import Conv2dSubsampling from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer2 @@ -958,7 +958,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1217,7 +1217,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1317,7 +1317,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/gigaspeech/KWS/zipformer/finetune.py b/egs/gigaspeech/KWS/zipformer/finetune.py index a7ba56127..49e8aef1a 100755 --- a/egs/gigaspeech/KWS/zipformer/finetune.py +++ b/egs/gigaspeech/KWS/zipformer/finetune.py @@ -73,7 +73,7 @@ from lhotse.utils import fix_random_seed from model import AsrModel from optim import Eden, ScaledAdam from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from train import ( @@ -291,7 +291,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -570,7 +570,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) diff --git a/egs/gigaspeech/KWS/zipformer/train.py b/egs/gigaspeech/KWS/zipformer/train.py index 39d8fc6cd..f2283cb03 100755 --- a/egs/gigaspeech/KWS/zipformer/train.py +++ b/egs/gigaspeech/KWS/zipformer/train.py @@ -75,7 +75,7 @@ from optim import Eden, ScaledAdam from scaling import ScheduledFloat from subsampling import Conv2dSubsampling from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer2 @@ -961,7 +961,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1220,7 +1220,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1320,7 +1320,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/ksponspeech/ASR/pruned_transducer_stateless7_streaming/train.py b/egs/ksponspeech/ASR/pruned_transducer_stateless7_streaming/train.py index bf50bf5ea..30d9f0e51 100755 --- a/egs/ksponspeech/ASR/pruned_transducer_stateless7_streaming/train.py +++ b/egs/ksponspeech/ASR/pruned_transducer_stateless7_streaming/train.py @@ -61,7 +61,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, ScaledAdam from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer @@ -805,7 +805,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1096,7 +1096,7 @@ def run(rank, world_size, args): # params=params, # ) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1196,7 +1196,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/ksponspeech/ASR/zipformer/train.py b/egs/ksponspeech/ASR/zipformer/train.py index 485ea69c9..5f6ee7cca 100755 --- a/egs/ksponspeech/ASR/zipformer/train.py +++ b/egs/ksponspeech/ASR/zipformer/train.py @@ -70,7 +70,7 @@ from optim import Eden, ScaledAdam from scaling import ScheduledFloat from subsampling import Conv2dSubsampling from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer2 @@ -942,7 +942,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1233,7 +1233,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1333,7 +1333,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/libricss/SURT/dprnn_zipformer/model.py b/egs/libricss/SURT/dprnn_zipformer/model.py index 688e1e78d..0e88357d1 100644 --- a/egs/libricss/SURT/dprnn_zipformer/model.py +++ b/egs/libricss/SURT/dprnn_zipformer/model.py @@ -140,7 +140,7 @@ class SURT(nn.Module): lm = self.simple_lm_proj(decoder_out) am = self.simple_am_proj(encoder_out) - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): simple_loss, (px_grad, py_grad) = k2.rnnt_loss_smoothed( lm=lm.float(), am=am.float(), @@ -175,7 +175,7 @@ class SURT(nn.Module): # prior to do_rnnt_pruning (this is an optimization for speed). logits = self.joiner(am_pruned, lm_pruned, project_input=False) - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): pruned_loss = k2.rnnt_loss_pruned( logits=logits.float(), symbols=y_padded, diff --git a/egs/libricss/SURT/dprnn_zipformer/scaling.py b/egs/libricss/SURT/dprnn_zipformer/scaling.py index 4040a7b89..d46cb224e 100644 --- a/egs/libricss/SURT/dprnn_zipformer/scaling.py +++ b/egs/libricss/SURT/dprnn_zipformer/scaling.py @@ -287,7 +287,7 @@ class SoftmaxFunction(torch.autograd.Function): @staticmethod def backward(ctx, ans_grad: Tensor): (ans,) = ctx.saved_tensors - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): ans_grad = ans_grad.to(torch.float32) ans = ans.to(torch.float32) x_grad = ans_grad * ans @@ -1065,7 +1065,7 @@ class WhiteningPenaltyFunction(torch.autograd.Function): def backward(ctx, x_grad: Tensor): (x_orig,) = ctx.saved_tensors with torch.enable_grad(): - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): x_detached = x_orig.to(torch.float32).detach() x_detached.requires_grad = True @@ -1263,7 +1263,7 @@ class MaxEig(torch.nn.Module): ): return _no_op(x) - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): eps = 1.0e-20 orig_x = x x = x.to(torch.float32) diff --git a/egs/libricss/SURT/dprnn_zipformer/train.py b/egs/libricss/SURT/dprnn_zipformer/train.py index 148cafd4b..33ea7c5a6 100755 --- a/egs/libricss/SURT/dprnn_zipformer/train.py +++ b/egs/libricss/SURT/dprnn_zipformer/train.py @@ -69,7 +69,7 @@ from model import SURT from optim import Eden, ScaledAdam from scaling import ScaledLSTM from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer @@ -1096,7 +1096,7 @@ def train_one_epoch( batch_size = batch["inputs"].shape[0] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1343,7 +1343,7 @@ def run(rank, world_size, args): train_dl_ov40 = libricss.train_dataloaders(train_cuts_ov40) valid_dl = libricss.valid_dataloaders(dev_cuts) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) diff --git a/egs/libricss/SURT/dprnn_zipformer/train_adapt.py b/egs/libricss/SURT/dprnn_zipformer/train_adapt.py index 8c37430ec..82b61baa0 100755 --- a/egs/libricss/SURT/dprnn_zipformer/train_adapt.py +++ b/egs/libricss/SURT/dprnn_zipformer/train_adapt.py @@ -67,7 +67,7 @@ from model import SURT from optim import Eden, ScaledAdam from scaling import ScaledLinear, ScaledLSTM from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer @@ -985,7 +985,7 @@ def train_one_epoch( batch_size = batch["inputs"].shape[0] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1237,7 +1237,7 @@ def run(rank, world_size, args): ) valid_dl = libricss.valid_dataloaders(dev_cuts) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) diff --git a/egs/libriheavy/ASR/zipformer/train.py b/egs/libriheavy/ASR/zipformer/train.py index 357e8a827..524273ec5 100644 --- a/egs/libriheavy/ASR/zipformer/train.py +++ b/egs/libriheavy/ASR/zipformer/train.py @@ -78,7 +78,7 @@ from scaling import ScheduledFloat from subsampling import Conv2dSubsampling from text_normalization import remove_punc_to_upper from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer2 @@ -958,7 +958,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1268,7 +1268,7 @@ def run(rank, world_size, args): # params=params, # ) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1367,7 +1367,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/libriheavy/ASR/zipformer_prompt_asr/model_baseline.py b/egs/libriheavy/ASR/zipformer_prompt_asr/model_baseline.py index 77b4057c4..66328bb89 100644 --- a/egs/libriheavy/ASR/zipformer_prompt_asr/model_baseline.py +++ b/egs/libriheavy/ASR/zipformer_prompt_asr/model_baseline.py @@ -186,7 +186,7 @@ class Transducer(nn.Module): lm = self.simple_lm_proj(decoder_out) am = self.simple_am_proj(encoder_out) - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): simple_loss, (px_grad, py_grad) = k2.rnnt_loss_smoothed( lm=lm.float(), am=am.float(), @@ -221,7 +221,7 @@ class Transducer(nn.Module): # prior to do_rnnt_pruning (this is an optimization for speed). logits = self.joiner(am_pruned, lm_pruned, project_input=False) - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): pruned_loss = k2.rnnt_loss_pruned( logits=logits.float(), symbols=y_padded, diff --git a/egs/libriheavy/ASR/zipformer_prompt_asr/model_with_BERT.py b/egs/libriheavy/ASR/zipformer_prompt_asr/model_with_BERT.py index 21c7b4fac..80fbf09f0 100644 --- a/egs/libriheavy/ASR/zipformer_prompt_asr/model_with_BERT.py +++ b/egs/libriheavy/ASR/zipformer_prompt_asr/model_with_BERT.py @@ -245,7 +245,7 @@ class PromptedTransducer(nn.Module): lm = self.simple_lm_proj(decoder_out) am = self.simple_am_proj(encoder_out) - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): simple_loss, (px_grad, py_grad) = k2.rnnt_loss_smoothed( lm=lm.float(), am=am.float(), @@ -287,7 +287,7 @@ class PromptedTransducer(nn.Module): logits = self.joiner(am_pruned, lm_pruned, context=context, project_input=False) - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): pruned_loss = k2.rnnt_loss_pruned( logits=logits.float(), symbols=y_padded, diff --git a/egs/libriheavy/ASR/zipformer_prompt_asr/scaling.py b/egs/libriheavy/ASR/zipformer_prompt_asr/scaling.py index 0e6764ba0..a260d828e 100644 --- a/egs/libriheavy/ASR/zipformer_prompt_asr/scaling.py +++ b/egs/libriheavy/ASR/zipformer_prompt_asr/scaling.py @@ -271,7 +271,7 @@ class SoftmaxFunction(torch.autograd.Function): @staticmethod def backward(ctx, ans_grad: Tensor): (ans,) = ctx.saved_tensors - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): ans_grad = ans_grad.to(torch.float32) ans = ans.to(torch.float32) x_grad = ans_grad * ans @@ -685,7 +685,7 @@ class BalancerFunction(torch.autograd.Function): try: with torch.enable_grad(): - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): x = x.to(torch.float32) x = x.detach() x.requires_grad = True @@ -940,7 +940,7 @@ class WhiteningPenaltyFunction(torch.autograd.Function): try: with torch.enable_grad(): - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): x_detached = x_orig.to(torch.float32).detach() x_detached.requires_grad = True @@ -1280,7 +1280,7 @@ class SwooshLFunction(torch.autograd.Function): coeff = -0.08 - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): with torch.enable_grad(): x = x.detach() x.requires_grad = True @@ -1351,7 +1351,7 @@ class SwooshRFunction(torch.autograd.Function): zero = torch.tensor(0.0, dtype=x.dtype, device=x.device) - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): with torch.enable_grad(): x = x.detach() x.requires_grad = True diff --git a/egs/libriheavy/ASR/zipformer_prompt_asr/train_baseline.py b/egs/libriheavy/ASR/zipformer_prompt_asr/train_baseline.py index 93f7e1248..bfca5a0db 100644 --- a/egs/libriheavy/ASR/zipformer_prompt_asr/train_baseline.py +++ b/egs/libriheavy/ASR/zipformer_prompt_asr/train_baseline.py @@ -89,7 +89,7 @@ from scaling import ScheduledFloat from subsampling import Conv2dSubsampling from text_normalization import train_text_normalization, upper_only_alpha from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer2 @@ -975,7 +975,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1271,7 +1271,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1371,7 +1371,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/libriheavy/ASR/zipformer_prompt_asr/train_bert_encoder.py b/egs/libriheavy/ASR/zipformer_prompt_asr/train_bert_encoder.py index 2a2c206aa..36c6d6464 100755 --- a/egs/libriheavy/ASR/zipformer_prompt_asr/train_bert_encoder.py +++ b/egs/libriheavy/ASR/zipformer_prompt_asr/train_bert_encoder.py @@ -103,7 +103,7 @@ from text_normalization import ( upper_only_alpha, ) from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer2 @@ -1321,7 +1321,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1647,7 +1647,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1749,7 +1749,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/libriheavy/ASR/zipformer_prompt_asr/zipformer.py b/egs/libriheavy/ASR/zipformer_prompt_asr/zipformer.py index d1cf90ffb..405c95acc 100644 --- a/egs/libriheavy/ASR/zipformer_prompt_asr/zipformer.py +++ b/egs/libriheavy/ASR/zipformer_prompt_asr/zipformer.py @@ -1561,7 +1561,7 @@ class RelPositionMultiheadAttentionWeights(nn.Module): (num_heads, batch_size, seq_len, seq_len) = attn_weights.shape with torch.no_grad(): - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): attn_weights = attn_weights.to(torch.float32) attn_weights_entropy = ( -((attn_weights + 1.0e-20).log() * attn_weights) @@ -1844,7 +1844,7 @@ class MultiheadAttentionWeights(nn.Module): (num_heads, batch_size, seq_len, seq_len) = attn_weights.shape with torch.no_grad(): - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): attn_weights = attn_weights.to(torch.float32) attn_weights_entropy = ( -((attn_weights + 1.0e-20).log() * attn_weights) diff --git a/egs/librilight/SSL/zipformer/finetune.py b/egs/librilight/SSL/zipformer/finetune.py index 50dbd5f2d..568096c6a 100644 --- a/egs/librilight/SSL/zipformer/finetune.py +++ b/egs/librilight/SSL/zipformer/finetune.py @@ -67,7 +67,7 @@ from lhotse.utils import fix_random_seed from model import AsrModel from optim import Eden, ScaledAdam from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -1116,7 +1116,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1407,7 +1407,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1505,7 +1505,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librilight/SSL/zipformer/pretrain.py b/egs/librilight/SSL/zipformer/pretrain.py index 5728dbe75..019f77ea3 100644 --- a/egs/librilight/SSL/zipformer/pretrain.py +++ b/egs/librilight/SSL/zipformer/pretrain.py @@ -57,7 +57,7 @@ from lhotse.utils import fix_random_seed from optim import Eden, ScaledAdam from ssl_datamodule import LibriLightDataModule from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -936,7 +936,7 @@ def train_one_epoch( batch_size = batch["kmeans"].shape[0] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1229,7 +1229,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1320,7 +1320,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/ASR/conformer_ctc2/train.py b/egs/librispeech/ASR/conformer_ctc2/train.py index c4a13b101..b0b5da1c0 100755 --- a/egs/librispeech/ASR/conformer_ctc2/train.py +++ b/egs/librispeech/ASR/conformer_ctc2/train.py @@ -65,7 +65,7 @@ from lhotse.dataset.sampling.base import CutSampler from lhotse.utils import fix_random_seed from optim import Eden, Eve from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -676,7 +676,7 @@ def train_one_epoch( params.batch_idx_train += 1 batch_size = len(batch["supervisions"]["text"]) - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -965,7 +965,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16) + scaler = GradScaler("cuda", enabled=params.use_fp16) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1036,7 +1036,7 @@ def scan_pessimistic_batches_for_oom( # warmup = 0.0 is so that the derivs for the pruned loss stay zero # (i.e. are not remembered by the decaying-average in adam), because # we want to avoid these params being subject to shrinkage in adam. - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/ASR/conformer_ctc3/train.py b/egs/librispeech/ASR/conformer_ctc3/train.py index a2f1125ca..7e819a2d8 100755 --- a/egs/librispeech/ASR/conformer_ctc3/train.py +++ b/egs/librispeech/ASR/conformer_ctc3/train.py @@ -76,7 +76,7 @@ from lhotse.utils import fix_random_seed from model import CTCModel from optim import Eden, Eve from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -743,7 +743,7 @@ def train_one_epoch( params.batch_idx_train += 1 batch_size = len(batch["supervisions"]["text"]) - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1004,7 +1004,7 @@ def run(rank, world_size, args): warmup=0.0 if params.start_epoch == 1 else 1.0, ) - scaler = GradScaler(enabled=params.use_fp16) + scaler = GradScaler("cuda", enabled=params.use_fp16) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1073,7 +1073,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/ASR/conv_emformer_transducer_stateless/train.py b/egs/librispeech/ASR/conv_emformer_transducer_stateless/train.py index ca21bd6bf..130a7c97f 100755 --- a/egs/librispeech/ASR/conv_emformer_transducer_stateless/train.py +++ b/egs/librispeech/ASR/conv_emformer_transducer_stateless/train.py @@ -80,7 +80,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, Eve from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -772,7 +772,7 @@ def train_one_epoch( params.batch_idx_train += 1 batch_size = len(batch["supervisions"]["text"]) - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1002,7 +1002,7 @@ def run(rank, world_size, args): warmup=0.0 if params.start_epoch == 1 else 1.0, ) - scaler = GradScaler(enabled=params.use_fp16) + scaler = GradScaler("cuda", enabled=params.use_fp16) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1071,7 +1071,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/ASR/conv_emformer_transducer_stateless2/do_not_use_it_directly.py b/egs/librispeech/ASR/conv_emformer_transducer_stateless2/do_not_use_it_directly.py index d614f0914..16ae4e4e2 100755 --- a/egs/librispeech/ASR/conv_emformer_transducer_stateless2/do_not_use_it_directly.py +++ b/egs/librispeech/ASR/conv_emformer_transducer_stateless2/do_not_use_it_directly.py @@ -80,7 +80,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, Eve from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -774,7 +774,7 @@ def train_one_epoch( params.batch_idx_train += 1 batch_size = len(batch["supervisions"]["text"]) - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1003,7 +1003,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16) + scaler = GradScaler("cuda", enabled=params.use_fp16) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1074,7 +1074,7 @@ def scan_pessimistic_batches_for_oom( # warmup = 0.0 is so that the derivs for the pruned loss stay zero # (i.e. are not remembered by the decaying-average in adam), because # we want to avoid these params being subject to shrinkage in adam. - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/ASR/conv_emformer_transducer_stateless2/train.py b/egs/librispeech/ASR/conv_emformer_transducer_stateless2/train.py index 23ddb6bec..28d094a76 100755 --- a/egs/librispeech/ASR/conv_emformer_transducer_stateless2/train.py +++ b/egs/librispeech/ASR/conv_emformer_transducer_stateless2/train.py @@ -80,7 +80,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, Eve from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -772,7 +772,7 @@ def train_one_epoch( params.batch_idx_train += 1 batch_size = len(batch["supervisions"]["text"]) - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1001,7 +1001,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16) + scaler = GradScaler("cuda", enabled=params.use_fp16) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1072,7 +1072,7 @@ def scan_pessimistic_batches_for_oom( # warmup = 0.0 is so that the derivs for the pruned loss stay zero # (i.e. are not remembered by the decaying-average in adam), because # we want to avoid these params being subject to shrinkage in adam. - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/ASR/lstm_transducer_stateless/model.py b/egs/librispeech/ASR/lstm_transducer_stateless/model.py index e7bad7ed8..1ec9a8fc6 100644 --- a/egs/librispeech/ASR/lstm_transducer_stateless/model.py +++ b/egs/librispeech/ASR/lstm_transducer_stateless/model.py @@ -156,7 +156,7 @@ class Transducer(nn.Module): lm = self.simple_lm_proj(decoder_out) am = self.simple_am_proj(encoder_out) - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): simple_loss, (px_grad, py_grad) = k2.rnnt_loss_smoothed( lm=lm.float(), am=am.float(), @@ -192,7 +192,7 @@ class Transducer(nn.Module): # prior to do_rnnt_pruning (this is an optimization for speed). logits = self.joiner(am_pruned, lm_pruned, project_input=False) - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): pruned_loss = k2.rnnt_loss_pruned( logits=logits.float(), symbols=y_padded, diff --git a/egs/librispeech/ASR/lstm_transducer_stateless/train.py b/egs/librispeech/ASR/lstm_transducer_stateless/train.py index feb81d500..1e50ce090 100755 --- a/egs/librispeech/ASR/lstm_transducer_stateless/train.py +++ b/egs/librispeech/ASR/lstm_transducer_stateless/train.py @@ -66,7 +66,7 @@ from lstm import RNN from model import Transducer from optim import Eden, Eve from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -763,7 +763,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1023,7 +1023,7 @@ def run(rank, world_size, args): warmup=0.0 if params.start_epoch == 1 else 1.0, ) - scaler = GradScaler(enabled=params.use_fp16) + scaler = GradScaler("cuda", enabled=params.use_fp16) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1092,7 +1092,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/ASR/lstm_transducer_stateless2/model.py b/egs/librispeech/ASR/lstm_transducer_stateless2/model.py index 4957d14b1..a758c550d 100644 --- a/egs/librispeech/ASR/lstm_transducer_stateless2/model.py +++ b/egs/librispeech/ASR/lstm_transducer_stateless2/model.py @@ -195,7 +195,7 @@ class Transducer(nn.Module): lm = simple_lm_proj(decoder_out) am = simple_am_proj(encoder_out) - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): simple_loss, (px_grad, py_grad) = k2.rnnt_loss_smoothed( lm=lm.float(), am=am.float(), @@ -231,7 +231,7 @@ class Transducer(nn.Module): # prior to do_rnnt_pruning (this is an optimization for speed). logits = joiner(am_pruned, lm_pruned, project_input=False) - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): pruned_loss = k2.rnnt_loss_pruned( logits=logits.float(), symbols=y_padded, diff --git a/egs/librispeech/ASR/lstm_transducer_stateless2/train.py b/egs/librispeech/ASR/lstm_transducer_stateless2/train.py index 4fc4fa7f8..4d4f3e132 100755 --- a/egs/librispeech/ASR/lstm_transducer_stateless2/train.py +++ b/egs/librispeech/ASR/lstm_transducer_stateless2/train.py @@ -74,7 +74,7 @@ from lstm import RNN from model import Transducer from optim import Eden, Eve from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -848,7 +848,7 @@ def train_one_epoch( libri = is_libri(batch["supervisions"]["cut"][0]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1176,7 +1176,7 @@ def run(rank, world_size, args): else: logging.info("Skip scan_pessimistic_batches_for_oom") - scaler = GradScaler(enabled=params.use_fp16) + scaler = GradScaler("cuda", enabled=params.use_fp16) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1247,7 +1247,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/ASR/lstm_transducer_stateless3/train.py b/egs/librispeech/ASR/lstm_transducer_stateless3/train.py index 2c1cef3a3..ae4cd1c6a 100755 --- a/egs/librispeech/ASR/lstm_transducer_stateless3/train.py +++ b/egs/librispeech/ASR/lstm_transducer_stateless3/train.py @@ -66,7 +66,7 @@ from lstm import RNN from model import Transducer from optim import Eden, Eve from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -793,7 +793,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1067,7 +1067,7 @@ def run(rank, world_size, args): warmup=0.0 if params.start_epoch == 1 else 1.0, ) - scaler = GradScaler(enabled=params.use_fp16) + scaler = GradScaler("cuda", enabled=params.use_fp16) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1136,7 +1136,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/ASR/pruned2_knowledge/model.py b/egs/librispeech/ASR/pruned2_knowledge/model.py index ca8c28af1..2ffea06e7 100644 --- a/egs/librispeech/ASR/pruned2_knowledge/model.py +++ b/egs/librispeech/ASR/pruned2_knowledge/model.py @@ -141,7 +141,7 @@ class Transducer(nn.Module): lm = self.simple_lm_proj(decoder_out) am = self.simple_am_proj(encoder_out) - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): simple_loss, (px_grad, py_grad) = k2.rnnt_loss_smoothed( lm=lm.float(), am=am.float(), @@ -176,7 +176,7 @@ class Transducer(nn.Module): # prior to do_rnnt_pruning (this is an optimization for speed). logits = self.joiner(am_pruned, lm_pruned, project_input=False) - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): pruned_loss = k2.rnnt_loss_pruned( logits=logits.float(), symbols=y_padded, diff --git a/egs/librispeech/ASR/pruned2_knowledge/sampling.py b/egs/librispeech/ASR/pruned2_knowledge/sampling.py index 5b595c76c..3d2fdd6d8 100644 --- a/egs/librispeech/ASR/pruned2_knowledge/sampling.py +++ b/egs/librispeech/ASR/pruned2_knowledge/sampling.py @@ -10,7 +10,7 @@ from typing import Optional, Tuple import torch from scaling import ScaledLinear from torch import Tensor, nn -from torch.cuda.amp import GradScaler, custom_bwd, custom_fwd +from torch.amp import GradScaler, custom_bwd, custom_fwd from torch_scheduled_sampling import sample_combined # The main exports of this file are the module KnowledgeBaseLookup and the @@ -330,14 +330,14 @@ def _test_knowledge_base_lookup_autocast(): optimizer = Eve(m.parameters(), lr=0.005, eps=1.0e-04) m = m.to(device) - scaler = GradScaler(enabled=True) + scaler = GradScaler("cuda", enabled=True) start = timeit.default_timer() for epoch in range(150): for n, (x, y) in enumerate(train_pairs): y_out = m(x) - with torch.cuda.amp.autocast(enabled=True): + with torch.amp.autocast("cuda", enabled=True): loss = ((y_out - y) ** 2).mean() * 100.0 if n % 10 == 0 and epoch % 10 == 0: print(f"Epoch {epoch}, batch {n}, loss {loss.item()}") diff --git a/egs/librispeech/ASR/pruned2_knowledge/train.py b/egs/librispeech/ASR/pruned2_knowledge/train.py index 931341cc4..8c117dd60 100755 --- a/egs/librispeech/ASR/pruned2_knowledge/train.py +++ b/egs/librispeech/ASR/pruned2_knowledge/train.py @@ -66,7 +66,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, Eve from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -650,7 +650,7 @@ def train_one_epoch( params.batch_idx_train += 1 batch_size = len(batch["supervisions"]["text"]) - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -868,7 +868,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16) + scaler = GradScaler("cuda", enabled=params.use_fp16) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -937,7 +937,7 @@ def scan_pessimistic_batches_for_oom( # warmup = 0.0 is so that the derivs for the pruned loss stay zero # (i.e. are not remembered by the decaying-average in adam), because # we want to avoid these params being subject to shrinkage in adam. - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/ASR/pruned_stateless_emformer_rnnt2/train.py b/egs/librispeech/ASR/pruned_stateless_emformer_rnnt2/train.py index 2b872f1d5..b25a84a6b 100755 --- a/egs/librispeech/ASR/pruned_stateless_emformer_rnnt2/train.py +++ b/egs/librispeech/ASR/pruned_stateless_emformer_rnnt2/train.py @@ -55,7 +55,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from noam import Noam from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -693,7 +693,7 @@ def train_one_epoch( params.batch_idx_train += 1 batch_size = len(batch["supervisions"]["text"]) - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -939,7 +939,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16) + scaler = GradScaler("cuda", enabled=params.use_fp16) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1004,7 +1004,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/ASR/pruned_transducer_stateless2/model.py b/egs/librispeech/ASR/pruned_transducer_stateless2/model.py index 272d06c37..59ed8310c 100644 --- a/egs/librispeech/ASR/pruned_transducer_stateless2/model.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless2/model.py @@ -157,7 +157,7 @@ class Transducer(nn.Module): lm = self.simple_lm_proj(decoder_out) am = self.simple_am_proj(encoder_out) - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): simple_loss, (px_grad, py_grad) = k2.rnnt_loss_smoothed( lm=lm.float(), am=am.float(), @@ -193,7 +193,7 @@ class Transducer(nn.Module): # prior to do_rnnt_pruning (this is an optimization for speed). logits = self.joiner(am_pruned, lm_pruned, project_input=False) - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): pruned_loss = k2.rnnt_loss_pruned( logits=logits.float(), symbols=y_padded, diff --git a/egs/librispeech/ASR/pruned_transducer_stateless2/train.py b/egs/librispeech/ASR/pruned_transducer_stateless2/train.py index 6c19f2cb0..e86ec8052 100755 --- a/egs/librispeech/ASR/pruned_transducer_stateless2/train.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless2/train.py @@ -78,7 +78,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, Eve from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -759,7 +759,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1000,7 +1000,7 @@ def run(rank, world_size, args): warmup=0.0 if params.start_epoch == 0 else 1.0, ) - scaler = GradScaler(enabled=params.use_fp16) + scaler = GradScaler("cuda", enabled=params.use_fp16) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1067,7 +1067,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/ASR/pruned_transducer_stateless3/model.py b/egs/librispeech/ASR/pruned_transducer_stateless3/model.py index d45f6dadc..0495c8a29 100644 --- a/egs/librispeech/ASR/pruned_transducer_stateless3/model.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless3/model.py @@ -195,7 +195,7 @@ class Transducer(nn.Module): lm = simple_lm_proj(decoder_out) am = simple_am_proj(encoder_out) - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): simple_loss, (px_grad, py_grad) = k2.rnnt_loss_smoothed( lm=lm.float(), am=am.float(), @@ -231,7 +231,7 @@ class Transducer(nn.Module): # prior to do_rnnt_pruning (this is an optimization for speed). logits = joiner(am_pruned, lm_pruned, project_input=False) - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): pruned_loss = k2.rnnt_loss_pruned( logits=logits.float(), symbols=y_padded, diff --git a/egs/librispeech/ASR/pruned_transducer_stateless3/train.py b/egs/librispeech/ASR/pruned_transducer_stateless3/train.py index fdafa5a87..8ef207518 100755 --- a/egs/librispeech/ASR/pruned_transducer_stateless3/train.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless3/train.py @@ -74,7 +74,7 @@ from librispeech import LibriSpeech from model import Transducer from optim import Eden, Eve from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -827,7 +827,7 @@ def train_one_epoch( libri = is_libri(batch["supervisions"]["cut"][0]) - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1126,7 +1126,7 @@ def run(rank, world_size, args): warmup=0.0 if params.start_epoch == 0 else 1.0, ) - scaler = GradScaler(enabled=params.use_fp16) + scaler = GradScaler("cuda", enabled=params.use_fp16) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1195,7 +1195,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/ASR/pruned_transducer_stateless4/train.py b/egs/librispeech/ASR/pruned_transducer_stateless4/train.py index 875b03f7f..b6682908b 100755 --- a/egs/librispeech/ASR/pruned_transducer_stateless4/train.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless4/train.py @@ -80,7 +80,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, Eve from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -789,7 +789,7 @@ def train_one_epoch( params.batch_idx_train += 1 batch_size = len(batch["supervisions"]["text"]) - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1047,7 +1047,7 @@ def run(rank, world_size, args): warmup=0.0 if params.start_epoch == 1 else 1.0, ) - scaler = GradScaler(enabled=params.use_fp16) + scaler = GradScaler("cuda", enabled=params.use_fp16) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1116,7 +1116,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/ASR/pruned_transducer_stateless5/train.py b/egs/librispeech/ASR/pruned_transducer_stateless5/train.py index 66dc5f991..2b559a27c 100755 --- a/egs/librispeech/ASR/pruned_transducer_stateless5/train.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless5/train.py @@ -68,7 +68,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, Eve from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -814,7 +814,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1078,7 +1078,7 @@ def run(rank, world_size, args): warmup=0.0 if params.start_epoch == 1 else 1.0, ) - scaler = GradScaler(enabled=params.use_fp16) + scaler = GradScaler("cuda", enabled=params.use_fp16) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1147,7 +1147,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/ASR/pruned_transducer_stateless6/model.py b/egs/librispeech/ASR/pruned_transducer_stateless6/model.py index daadb70c9..20b730a08 100644 --- a/egs/librispeech/ASR/pruned_transducer_stateless6/model.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless6/model.py @@ -185,7 +185,7 @@ class Transducer(nn.Module): lm = self.simple_lm_proj(decoder_out) am = self.simple_am_proj(encoder_out) - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): simple_loss, (px_grad, py_grad) = k2.rnnt_loss_smoothed( lm=lm.float(), am=am.float(), @@ -220,7 +220,7 @@ class Transducer(nn.Module): # prior to do_rnnt_pruning (this is an optimization for speed). logits = self.joiner(am_pruned, lm_pruned, project_input=False) - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): pruned_loss = k2.rnnt_loss_pruned( logits=logits.float(), symbols=y_padded, diff --git a/egs/librispeech/ASR/pruned_transducer_stateless6/train.py b/egs/librispeech/ASR/pruned_transducer_stateless6/train.py index 8f033cb9a..93663505a 100755 --- a/egs/librispeech/ASR/pruned_transducer_stateless6/train.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless6/train.py @@ -80,7 +80,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, Eve from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -781,7 +781,7 @@ def train_one_epoch( params.batch_idx_train += 1 batch_size = len(batch["supervisions"]["text"]) - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1039,7 +1039,7 @@ def run(rank, world_size, args): warmup=0.0 if params.start_epoch == 1 else 1.0, ) - scaler = GradScaler(enabled=params.use_fp16) + scaler = GradScaler("cuda", enabled=params.use_fp16) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1108,7 +1108,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/ASR/pruned_transducer_stateless7/finetune.py b/egs/librispeech/ASR/pruned_transducer_stateless7/finetune.py index e7546ec45..d29010a23 100755 --- a/egs/librispeech/ASR/pruned_transducer_stateless7/finetune.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless7/finetune.py @@ -66,7 +66,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, ScaledAdam from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer @@ -903,7 +903,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1219,7 +1219,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1319,7 +1319,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/ASR/pruned_transducer_stateless7/model.py b/egs/librispeech/ASR/pruned_transducer_stateless7/model.py index add0e6a18..49076b96f 100644 --- a/egs/librispeech/ASR/pruned_transducer_stateless7/model.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless7/model.py @@ -150,7 +150,7 @@ class Transducer(nn.Module): # if self.training and random.random() < 0.25: # am = penalize_abs_values_gt(am, 30.0, 1.0e-04) - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): simple_loss, (px_grad, py_grad) = k2.rnnt_loss_smoothed( lm=lm.float(), am=am.float(), @@ -185,7 +185,7 @@ class Transducer(nn.Module): # prior to do_rnnt_pruning (this is an optimization for speed). logits = self.joiner(am_pruned, lm_pruned, project_input=False) - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): pruned_loss = k2.rnnt_loss_pruned( logits=logits.float(), symbols=y_padded, diff --git a/egs/librispeech/ASR/pruned_transducer_stateless7/scaling.py b/egs/librispeech/ASR/pruned_transducer_stateless7/scaling.py index 30a737061..16d86fe2d 100644 --- a/egs/librispeech/ASR/pruned_transducer_stateless7/scaling.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless7/scaling.py @@ -289,7 +289,7 @@ class SoftmaxFunction(torch.autograd.Function): @staticmethod def backward(ctx, ans_grad: Tensor): (ans,) = ctx.saved_tensors - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): ans_grad = ans_grad.to(torch.float32) ans = ans.to(torch.float32) x_grad = ans_grad * ans @@ -669,7 +669,7 @@ class WhiteningPenaltyFunction(torch.autograd.Function): def backward(ctx, x_grad: Tensor): (x_orig,) = ctx.saved_tensors with torch.enable_grad(): - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): x_detached = x_orig.to(torch.float32).detach() x_detached.requires_grad = True @@ -867,7 +867,7 @@ class MaxEig(torch.nn.Module): ): return _no_op(x) - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): eps = 1.0e-20 orig_x = x x = x.to(torch.float32) diff --git a/egs/librispeech/ASR/pruned_transducer_stateless7/train.py b/egs/librispeech/ASR/pruned_transducer_stateless7/train.py index 436ec53b4..91fccd58d 100755 --- a/egs/librispeech/ASR/pruned_transducer_stateless7/train.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless7/train.py @@ -67,7 +67,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, ScaledAdam from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer @@ -809,7 +809,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1106,7 +1106,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1206,7 +1206,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/ASR/pruned_transducer_stateless7/zipformer.py b/egs/librispeech/ASR/pruned_transducer_stateless7/zipformer.py index cbde2a2e4..ebef7e977 100644 --- a/egs/librispeech/ASR/pruned_transducer_stateless7/zipformer.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless7/zipformer.py @@ -1421,7 +1421,7 @@ class RelPositionMultiheadAttention(nn.Module): bsz = n // num_heads with torch.no_grad(): - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): attn_weights = attn_weights.to(torch.float32) attn_output = attn_output.to(torch.float32) attn_weights_entropy = ( diff --git a/egs/librispeech/ASR/pruned_transducer_stateless7_ctc/model.py b/egs/librispeech/ASR/pruned_transducer_stateless7_ctc/model.py index a6e919e2f..0224c15d7 100644 --- a/egs/librispeech/ASR/pruned_transducer_stateless7_ctc/model.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless7_ctc/model.py @@ -150,7 +150,7 @@ class Transducer(nn.Module): lm = self.simple_lm_proj(decoder_out) am = self.simple_am_proj(encoder_out) - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): simple_loss, (px_grad, py_grad) = k2.rnnt_loss_smoothed( lm=lm.float(), am=am.float(), @@ -185,7 +185,7 @@ class Transducer(nn.Module): # prior to do_rnnt_pruning (this is an optimization for speed). logits = self.joiner(am_pruned, lm_pruned, project_input=False) - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): pruned_loss = k2.rnnt_loss_pruned( logits=logits.float(), symbols=y_padded, diff --git a/egs/librispeech/ASR/pruned_transducer_stateless7_ctc/train.py b/egs/librispeech/ASR/pruned_transducer_stateless7_ctc/train.py index b35e56abc..395b07b05 100755 --- a/egs/librispeech/ASR/pruned_transducer_stateless7_ctc/train.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless7_ctc/train.py @@ -67,7 +67,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, ScaledAdam from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer @@ -833,7 +833,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1128,7 +1128,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1228,7 +1228,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/ASR/pruned_transducer_stateless7_ctc_bs/model.py b/egs/librispeech/ASR/pruned_transducer_stateless7_ctc_bs/model.py index 0582b289f..4675697c1 100644 --- a/egs/librispeech/ASR/pruned_transducer_stateless7_ctc_bs/model.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless7_ctc_bs/model.py @@ -178,7 +178,7 @@ class Transducer(nn.Module): am = self.simple_am_proj(encoder_out_fr) lm = self.simple_lm_proj(decoder_out) - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): simple_loss, (px_grad, py_grad) = k2.rnnt_loss_smoothed( lm=lm.float(), am=am.float(), @@ -213,7 +213,7 @@ class Transducer(nn.Module): # prior to do_rnnt_pruning (this is an optimization for speed). logits = self.joiner(am_pruned, lm_pruned, project_input=False) - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): pruned_loss = k2.rnnt_loss_pruned( logits=logits.float(), symbols=y_padded, diff --git a/egs/librispeech/ASR/pruned_transducer_stateless7_ctc_bs/train.py b/egs/librispeech/ASR/pruned_transducer_stateless7_ctc_bs/train.py index c2d877a93..a431b278d 100755 --- a/egs/librispeech/ASR/pruned_transducer_stateless7_ctc_bs/train.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless7_ctc_bs/train.py @@ -63,7 +63,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, ScaledAdam from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer @@ -822,7 +822,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1118,7 +1118,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1217,7 +1217,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/ASR/pruned_transducer_stateless7_streaming/do_not_use_it_directly.py b/egs/librispeech/ASR/pruned_transducer_stateless7_streaming/do_not_use_it_directly.py index 8e239e322..dc3493425 100755 --- a/egs/librispeech/ASR/pruned_transducer_stateless7_streaming/do_not_use_it_directly.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless7_streaming/do_not_use_it_directly.py @@ -66,7 +66,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, ScaledAdam from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer_for_ncnn_export_only import Zipformer @@ -811,7 +811,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1106,7 +1106,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1206,7 +1206,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/ASR/pruned_transducer_stateless7_streaming/train.py b/egs/librispeech/ASR/pruned_transducer_stateless7_streaming/train.py index 8bd00bbef..a8f47d941 100755 --- a/egs/librispeech/ASR/pruned_transducer_stateless7_streaming/train.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless7_streaming/train.py @@ -66,7 +66,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, ScaledAdam from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer @@ -810,7 +810,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1124,7 +1124,7 @@ def run(rank, world_size, args): # params=params, # ) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1224,7 +1224,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/ASR/pruned_transducer_stateless7_streaming/zipformer.py b/egs/librispeech/ASR/pruned_transducer_stateless7_streaming/zipformer.py index c7e45564f..e3b8b3725 100644 --- a/egs/librispeech/ASR/pruned_transducer_stateless7_streaming/zipformer.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless7_streaming/zipformer.py @@ -2408,7 +2408,7 @@ class RelPositionMultiheadAttention(nn.Module): bsz = n // num_heads with torch.no_grad(): - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): attn_weights = attn_weights.to(torch.float32) attn_output = attn_output.to(torch.float32) attn_weights_entropy = ( diff --git a/egs/librispeech/ASR/pruned_transducer_stateless7_streaming/zipformer_for_ncnn_export_only.py b/egs/librispeech/ASR/pruned_transducer_stateless7_streaming/zipformer_for_ncnn_export_only.py index 5284ed627..ff23725b7 100644 --- a/egs/librispeech/ASR/pruned_transducer_stateless7_streaming/zipformer_for_ncnn_export_only.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless7_streaming/zipformer_for_ncnn_export_only.py @@ -2708,7 +2708,7 @@ class RelPositionMultiheadAttention(nn.Module): bsz = n // num_heads with torch.no_grad(): - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): attn_weights = attn_weights.to(torch.float32) attn_output = attn_output.to(torch.float32) attn_weights_entropy = ( diff --git a/egs/librispeech/ASR/pruned_transducer_stateless7_streaming_multi/train.py b/egs/librispeech/ASR/pruned_transducer_stateless7_streaming_multi/train.py index da5e144c9..4c8c239a1 100755 --- a/egs/librispeech/ASR/pruned_transducer_stateless7_streaming_multi/train.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless7_streaming_multi/train.py @@ -70,7 +70,7 @@ from librispeech import LibriSpeech from model import Transducer from optim import Eden, ScaledAdam from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer @@ -866,7 +866,7 @@ def train_one_epoch( libri = is_libri(batch["supervisions"]["cut"][0]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1218,7 +1218,7 @@ def run(rank, world_size, args): # params=params, # ) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1320,7 +1320,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/ASR/pruned_transducer_stateless8/model.py b/egs/librispeech/ASR/pruned_transducer_stateless8/model.py index 39a360796..c0b9113b7 100644 --- a/egs/librispeech/ASR/pruned_transducer_stateless8/model.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless8/model.py @@ -172,7 +172,7 @@ class Transducer(nn.Module): # if self.training and random.random() < 0.25: # am = penalize_abs_values_gt(am, 30.0, 1.0e-04) - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): simple_loss, (px_grad, py_grad) = k2.rnnt_loss_smoothed( lm=lm.float(), am=am.float(), @@ -207,7 +207,7 @@ class Transducer(nn.Module): # prior to do_rnnt_pruning (this is an optimization for speed). logits = joiner(am_pruned, lm_pruned, project_input=False) - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): pruned_loss = k2.rnnt_loss_pruned( logits=logits.float(), symbols=y_padded, diff --git a/egs/librispeech/ASR/pruned_transducer_stateless8/train.py b/egs/librispeech/ASR/pruned_transducer_stateless8/train.py index 646f30ca1..0ccef210e 100755 --- a/egs/librispeech/ASR/pruned_transducer_stateless8/train.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless8/train.py @@ -75,7 +75,7 @@ from librispeech import LibriSpeech from model import Transducer from optim import Eden, ScaledAdam from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer @@ -866,7 +866,7 @@ def train_one_epoch( libri = is_libri(batch["supervisions"]["cut"][0]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1219,7 +1219,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1321,7 +1321,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/ASR/tiny_transducer_ctc/train.py b/egs/librispeech/ASR/tiny_transducer_ctc/train.py index 1bfd071de..0536e89b3 100644 --- a/egs/librispeech/ASR/tiny_transducer_ctc/train.py +++ b/egs/librispeech/ASR/tiny_transducer_ctc/train.py @@ -51,7 +51,7 @@ from lhotse.dataset.sampling.base import CutSampler from lhotse.utils import fix_random_seed from model import Transducer from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.optim import AdamW from torch.optim.lr_scheduler import StepLR @@ -809,7 +809,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1092,7 +1092,7 @@ def run(rank, world_size, args): # params=params, # ) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1198,7 +1198,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/ASR/zipformer/finetune.py b/egs/librispeech/ASR/zipformer/finetune.py index 2ff631914..5da903d38 100755 --- a/egs/librispeech/ASR/zipformer/finetune.py +++ b/egs/librispeech/ASR/zipformer/finetune.py @@ -78,7 +78,7 @@ from optim import Eden, ScaledAdam from scaling import ScheduledFloat from subsampling import Conv2dSubsampling from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer2 @@ -1049,7 +1049,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1373,7 +1373,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1474,7 +1474,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/ASR/zipformer/model.py b/egs/librispeech/ASR/zipformer/model.py index c7dbe1e0a..b0bb7c7fe 100644 --- a/egs/librispeech/ASR/zipformer/model.py +++ b/egs/librispeech/ASR/zipformer/model.py @@ -285,7 +285,7 @@ class AsrModel(nn.Module): # if self.training and random.random() < 0.25: # am = penalize_abs_values_gt(am, 30.0, 1.0e-04) - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): simple_loss, (px_grad, py_grad) = k2.rnnt_loss_smoothed( lm=lm.float(), am=am.float(), @@ -320,7 +320,7 @@ class AsrModel(nn.Module): # prior to do_rnnt_pruning (this is an optimization for speed). logits = self.joiner(am_pruned, lm_pruned, project_input=False) - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): pruned_loss = k2.rnnt_loss_pruned( logits=logits.float(), symbols=y_padded, diff --git a/egs/librispeech/ASR/zipformer/scaling.py b/egs/librispeech/ASR/zipformer/scaling.py index d345c2931..46df86bf7 100644 --- a/egs/librispeech/ASR/zipformer/scaling.py +++ b/egs/librispeech/ASR/zipformer/scaling.py @@ -306,7 +306,7 @@ class SoftmaxFunction(torch.autograd.Function): @staticmethod def backward(ctx, ans_grad: Tensor): (ans,) = ctx.saved_tensors - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): ans_grad = ans_grad.to(torch.float32) ans = ans.to(torch.float32) x_grad = ans_grad * ans @@ -759,7 +759,7 @@ class BalancerFunction(torch.autograd.Function): try: with torch.enable_grad(): - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): x = x.to(torch.float32) x = x.detach() x.requires_grad = True @@ -1014,7 +1014,7 @@ class WhiteningPenaltyFunction(torch.autograd.Function): try: with torch.enable_grad(): - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): x_detached = x_orig.to(torch.float32).detach() x_detached.requires_grad = True @@ -1353,7 +1353,7 @@ class SwooshLFunction(torch.autograd.Function): coeff = -0.08 - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): with torch.enable_grad(): x = x.detach() x.requires_grad = True @@ -1430,7 +1430,7 @@ class SwooshRFunction(torch.autograd.Function): zero = torch.tensor(0.0, dtype=x.dtype, device=x.device) - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): with torch.enable_grad(): x = x.detach() x.requires_grad = True diff --git a/egs/librispeech/ASR/zipformer/train.py b/egs/librispeech/ASR/zipformer/train.py index c074c32ec..71d045ea0 100755 --- a/egs/librispeech/ASR/zipformer/train.py +++ b/egs/librispeech/ASR/zipformer/train.py @@ -79,7 +79,7 @@ from optim import Eden, ScaledAdam from scaling import ScheduledFloat from subsampling import Conv2dSubsampling from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer2 @@ -1101,7 +1101,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast( + with torch.amp.autocast("cuda", enabled=params.use_autocast, dtype=params.dtype ): loss, loss_info = compute_loss( @@ -1438,7 +1438,7 @@ def run(rank, world_size, args): spec_augment=spec_augment, ) - scaler = GradScaler(enabled=params.use_autocast, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_autocast, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1540,7 +1540,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast( + with torch.amp.autocast("cuda", enabled=params.use_autocast, dtype=params.dtype ): loss, _ = compute_loss( diff --git a/egs/librispeech/ASR/zipformer/zipformer.py b/egs/librispeech/ASR/zipformer/zipformer.py index 2a0ae0129..bdfd2175c 100644 --- a/egs/librispeech/ASR/zipformer/zipformer.py +++ b/egs/librispeech/ASR/zipformer/zipformer.py @@ -1873,7 +1873,7 @@ class RelPositionMultiheadAttentionWeights(nn.Module): (num_heads, batch_size, seq_len, seq_len) = attn_weights.shape with torch.no_grad(): - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): attn_weights = attn_weights.to(torch.float32) attn_weights_entropy = ( -((attn_weights + 1.0e-20).log() * attn_weights) diff --git a/egs/librispeech/ASR/zipformer_adapter/train.py b/egs/librispeech/ASR/zipformer_adapter/train.py index 3511590da..0207fc26e 100755 --- a/egs/librispeech/ASR/zipformer_adapter/train.py +++ b/egs/librispeech/ASR/zipformer_adapter/train.py @@ -67,7 +67,7 @@ from optim import Eden, ScaledAdam from scaling import ScheduledFloat from subsampling import Conv2dSubsampling from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer2 @@ -1052,7 +1052,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1397,7 +1397,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1498,7 +1498,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/ASR/zipformer_adapter/zipformer.py b/egs/librispeech/ASR/zipformer_adapter/zipformer.py index 8e2dfdd72..6224d136a 100644 --- a/egs/librispeech/ASR/zipformer_adapter/zipformer.py +++ b/egs/librispeech/ASR/zipformer_adapter/zipformer.py @@ -1916,7 +1916,7 @@ class RelPositionMultiheadAttentionWeights(nn.Module): (num_heads, batch_size, seq_len, seq_len) = attn_weights.shape with torch.no_grad(): - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): attn_weights = attn_weights.to(torch.float32) attn_weights_entropy = ( -((attn_weights + 1.0e-20).log() * attn_weights) diff --git a/egs/librispeech/ASR/zipformer_ctc/train.py b/egs/librispeech/ASR/zipformer_ctc/train.py index 60112a84e..dfe702d2f 100755 --- a/egs/librispeech/ASR/zipformer_ctc/train.py +++ b/egs/librispeech/ASR/zipformer_ctc/train.py @@ -46,7 +46,7 @@ from lhotse.utils import fix_random_seed from model import CTCModel from optim import Eden, LRScheduler, ScaledAdam from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.nn.utils import clip_grad_norm_ from torch.utils.tensorboard import SummaryWriter @@ -726,7 +726,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -987,7 +987,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) diff --git a/egs/librispeech/ASR/zipformer_lora/finetune.py b/egs/librispeech/ASR/zipformer_lora/finetune.py index 3f36f229f..53152971d 100755 --- a/egs/librispeech/ASR/zipformer_lora/finetune.py +++ b/egs/librispeech/ASR/zipformer_lora/finetune.py @@ -78,7 +78,7 @@ from optim import Eden, ScaledAdam from scaling import ScheduledFloat from subsampling import Conv2dSubsampling from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer2 @@ -1065,7 +1065,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1406,7 +1406,7 @@ def run(rank, world_size, args): # params=params, # ) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1507,7 +1507,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/ASR/zipformer_lora/scaling.py b/egs/librispeech/ASR/zipformer_lora/scaling.py index 8d7aa8027..a1e77fe0e 100644 --- a/egs/librispeech/ASR/zipformer_lora/scaling.py +++ b/egs/librispeech/ASR/zipformer_lora/scaling.py @@ -307,7 +307,7 @@ class SoftmaxFunction(torch.autograd.Function): @staticmethod def backward(ctx, ans_grad: Tensor): (ans,) = ctx.saved_tensors - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): ans_grad = ans_grad.to(torch.float32) ans = ans.to(torch.float32) x_grad = ans_grad * ans @@ -863,7 +863,7 @@ class BalancerFunction(torch.autograd.Function): try: with torch.enable_grad(): - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): x = x.to(torch.float32) x = x.detach() x.requires_grad = True @@ -1118,7 +1118,7 @@ class WhiteningPenaltyFunction(torch.autograd.Function): try: with torch.enable_grad(): - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): x_detached = x_orig.to(torch.float32).detach() x_detached.requires_grad = True @@ -1457,7 +1457,7 @@ class SwooshLFunction(torch.autograd.Function): coeff = -0.08 - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): with torch.enable_grad(): x = x.detach() x.requires_grad = True @@ -1534,7 +1534,7 @@ class SwooshRFunction(torch.autograd.Function): zero = torch.tensor(0.0, dtype=x.dtype, device=x.device) - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): with torch.enable_grad(): x = x.detach() x.requires_grad = True diff --git a/egs/librispeech/ASR/zipformer_lora/train.py b/egs/librispeech/ASR/zipformer_lora/train.py index 9ab214e86..592bc0fd4 100755 --- a/egs/librispeech/ASR/zipformer_lora/train.py +++ b/egs/librispeech/ASR/zipformer_lora/train.py @@ -76,7 +76,7 @@ from optim import Eden, ScaledAdam from scaling import ScheduledFloat from subsampling import Conv2dSubsampling from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer2 @@ -947,7 +947,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1252,7 +1252,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1352,7 +1352,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/ASR/zipformer_lora/zipformer.py b/egs/librispeech/ASR/zipformer_lora/zipformer.py index 43865609a..ece7c3df1 100644 --- a/egs/librispeech/ASR/zipformer_lora/zipformer.py +++ b/egs/librispeech/ASR/zipformer_lora/zipformer.py @@ -1905,7 +1905,7 @@ class RelPositionMultiheadAttentionWeights(nn.Module): (num_heads, batch_size, seq_len, seq_len) = attn_weights.shape with torch.no_grad(): - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): attn_weights = attn_weights.to(torch.float32) attn_weights_entropy = ( -((attn_weights + 1.0e-20).log() * attn_weights) diff --git a/egs/librispeech/ASR/zipformer_mmi/train.py b/egs/librispeech/ASR/zipformer_mmi/train.py index c1785a328..bed3cfa04 100755 --- a/egs/librispeech/ASR/zipformer_mmi/train.py +++ b/egs/librispeech/ASR/zipformer_mmi/train.py @@ -64,7 +64,7 @@ from lhotse.utils import fix_random_seed from model import CTCModel from optim import Eden, ScaledAdam from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer @@ -744,7 +744,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1037,7 +1037,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1138,7 +1138,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/SSL/hubert/finetune.py b/egs/librispeech/SSL/hubert/finetune.py index 17daa3c9d..9717d579d 100644 --- a/egs/librispeech/SSL/hubert/finetune.py +++ b/egs/librispeech/SSL/hubert/finetune.py @@ -66,7 +66,7 @@ from lhotse.utils import fix_random_seed from model import AsrModel from optim import Eden, ScaledAdam from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -816,7 +816,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1109,7 +1109,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1207,7 +1207,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/SSL/hubert/finetune_ce.py b/egs/librispeech/SSL/hubert/finetune_ce.py index 2723cc770..340aa4aa2 100644 --- a/egs/librispeech/SSL/hubert/finetune_ce.py +++ b/egs/librispeech/SSL/hubert/finetune_ce.py @@ -66,7 +66,7 @@ from lhotse.utils import fix_random_seed from model import AsrModel from optim import Eden, ScaledAdam from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -816,7 +816,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1109,7 +1109,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1207,7 +1207,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/SSL/hubert/model.py b/egs/librispeech/SSL/hubert/model.py index 46a968b69..b23fa32ea 100644 --- a/egs/librispeech/SSL/hubert/model.py +++ b/egs/librispeech/SSL/hubert/model.py @@ -221,7 +221,7 @@ class AsrModel(nn.Module): # if self.training and random.random() < 0.25: # am = penalize_abs_values_gt(am, 30.0, 1.0e-04) - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): simple_loss, (px_grad, py_grad) = k2.rnnt_loss_smoothed( lm=lm.float(), am=am.float(), @@ -256,7 +256,7 @@ class AsrModel(nn.Module): # prior to do_rnnt_pruning (this is an optimization for speed). logits = self.joiner(am_pruned, lm_pruned, project_input=False) - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): pruned_loss = k2.rnnt_loss_pruned( logits=logits.float(), symbols=y_padded, diff --git a/egs/librispeech/SSL/hubert/pretrain.py b/egs/librispeech/SSL/hubert/pretrain.py index f183d90fd..1868bf0a6 100644 --- a/egs/librispeech/SSL/hubert/pretrain.py +++ b/egs/librispeech/SSL/hubert/pretrain.py @@ -59,7 +59,7 @@ from lhotse.utils import fix_random_seed from optim import Eden, ScaledAdam from ssl_datamodule import LibriSpeechDataModule from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.functional import pad from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -644,7 +644,7 @@ def train_one_epoch( batch_size = batch["kmeans"].shape[0] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -945,7 +945,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1036,7 +1036,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/SSL/hubert/pretrain_ce.py b/egs/librispeech/SSL/hubert/pretrain_ce.py index 94948695d..97efd983b 100644 --- a/egs/librispeech/SSL/hubert/pretrain_ce.py +++ b/egs/librispeech/SSL/hubert/pretrain_ce.py @@ -59,7 +59,7 @@ from lhotse.utils import fix_random_seed from optim import Eden, ScaledAdam from ssl_datamodule import LibriSpeechDataModule from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.functional import pad from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -644,7 +644,7 @@ def train_one_epoch( batch_size = batch["kmeans"].shape[0] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -945,7 +945,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1036,7 +1036,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/SSL/zipformer/finetune.py b/egs/librispeech/SSL/zipformer/finetune.py index c907b41c5..6bfab9d00 100644 --- a/egs/librispeech/SSL/zipformer/finetune.py +++ b/egs/librispeech/SSL/zipformer/finetune.py @@ -66,7 +66,7 @@ from lhotse.utils import fix_random_seed from model import AsrModel from optim import Eden, ScaledAdam from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -1115,7 +1115,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1406,7 +1406,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1504,7 +1504,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/SSL/zipformer/model.py b/egs/librispeech/SSL/zipformer/model.py index 46a968b69..b23fa32ea 100644 --- a/egs/librispeech/SSL/zipformer/model.py +++ b/egs/librispeech/SSL/zipformer/model.py @@ -221,7 +221,7 @@ class AsrModel(nn.Module): # if self.training and random.random() < 0.25: # am = penalize_abs_values_gt(am, 30.0, 1.0e-04) - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): simple_loss, (px_grad, py_grad) = k2.rnnt_loss_smoothed( lm=lm.float(), am=am.float(), @@ -256,7 +256,7 @@ class AsrModel(nn.Module): # prior to do_rnnt_pruning (this is an optimization for speed). logits = self.joiner(am_pruned, lm_pruned, project_input=False) - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): pruned_loss = k2.rnnt_loss_pruned( logits=logits.float(), symbols=y_padded, diff --git a/egs/librispeech/SSL/zipformer/pretrain.py b/egs/librispeech/SSL/zipformer/pretrain.py index 937fb382e..767c3bacb 100644 --- a/egs/librispeech/SSL/zipformer/pretrain.py +++ b/egs/librispeech/SSL/zipformer/pretrain.py @@ -58,7 +58,7 @@ from lhotse.utils import fix_random_seed from optim import Eden, ScaledAdam from ssl_datamodule import LibriSpeechDataModule from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -944,7 +944,7 @@ def train_one_epoch( batch_size = batch["kmeans"].shape[0] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1243,7 +1243,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1334,7 +1334,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/SSL/zipformer/zipformer.py b/egs/librispeech/SSL/zipformer/zipformer.py index e9eff3357..7e9ccb51f 100644 --- a/egs/librispeech/SSL/zipformer/zipformer.py +++ b/egs/librispeech/SSL/zipformer/zipformer.py @@ -1849,7 +1849,7 @@ class RelPositionMultiheadAttentionWeights(nn.Module): (num_heads, batch_size, seq_len, seq_len) = attn_weights.shape with torch.no_grad(): - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): attn_weights = attn_weights.to(torch.float32) attn_weights_entropy = ( -((attn_weights + 1.0e-20).log() * attn_weights) diff --git a/egs/librispeech/WSASR/conformer_ctc2/train.py b/egs/librispeech/WSASR/conformer_ctc2/train.py index 82c68803f..fc7728562 100755 --- a/egs/librispeech/WSASR/conformer_ctc2/train.py +++ b/egs/librispeech/WSASR/conformer_ctc2/train.py @@ -62,7 +62,7 @@ from lhotse.dataset.sampling.base import CutSampler from lhotse.utils import fix_random_seed from optim import Eden, Eve from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -757,7 +757,7 @@ def train_one_epoch( params.batch_idx_train += 1 batch_size = len(batch["supervisions"]["text"]) - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1005,7 +1005,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16) + scaler = GradScaler("cuda", enabled=params.use_fp16) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1076,7 +1076,7 @@ def scan_pessimistic_batches_for_oom( # warmup = 0.0 is so that the derivs for the pruned loss stay zero # (i.e. are not remembered by the decaying-average in adam), because # we want to avoid these params being subject to shrinkage in adam. - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/WSASR/conformer_ctc2/train_phone.py b/egs/librispeech/WSASR/conformer_ctc2/train_phone.py index b276d0587..1c4bd50bf 100755 --- a/egs/librispeech/WSASR/conformer_ctc2/train_phone.py +++ b/egs/librispeech/WSASR/conformer_ctc2/train_phone.py @@ -62,7 +62,7 @@ from lhotse.dataset.sampling.base import CutSampler from lhotse.utils import fix_random_seed from optim import Eden, Eve from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -758,7 +758,7 @@ def train_one_epoch( params.batch_idx_train += 1 batch_size = len(batch["supervisions"]["text"]) - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1007,7 +1007,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16) + scaler = GradScaler("cuda", enabled=params.use_fp16) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1078,7 +1078,7 @@ def scan_pessimistic_batches_for_oom( # warmup = 0.0 is so that the derivs for the pruned loss stay zero # (i.e. are not remembered by the decaying-average in adam), because # we want to avoid these params being subject to shrinkage in adam. - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/libritts/ASR/zipformer/train.py b/egs/libritts/ASR/zipformer/train.py index 5485eaf0a..78e3330bd 100755 --- a/egs/libritts/ASR/zipformer/train.py +++ b/egs/libritts/ASR/zipformer/train.py @@ -80,7 +80,7 @@ from optim import Eden, ScaledAdam from scaling import ScheduledFloat from subsampling import Conv2dSubsampling from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer2 @@ -1049,8 +1049,8 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast( - enabled=params.use_autocast, dtype=params.dtype + with torch.amp.autocast( + "cuda", enabled=params.use_autocast, dtype=params.dtype ): loss, loss_info = compute_loss( params=params, @@ -1378,7 +1378,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_autocast, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_autocast, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1478,8 +1478,8 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast( - enabled=params.use_autocast, dtype=params.dtype + with torch.amp.autocast( + "cuda", enabled=params.use_autocast, dtype=params.dtype ): loss, _ = compute_loss( params=params, diff --git a/egs/libritts/CODEC/encodec/encodec.py b/egs/libritts/CODEC/encodec/encodec.py index f21d494b6..31fc4f126 100644 --- a/egs/libritts/CODEC/encodec/encodec.py +++ b/egs/libritts/CODEC/encodec/encodec.py @@ -29,7 +29,7 @@ from loss import ( WavReconstructionLoss, ) from torch import nn -from torch.cuda.amp import autocast +from torch.amp import autocast class Encodec(nn.Module): @@ -148,7 +148,7 @@ class Encodec(nn.Module): ) # calculate losses - with autocast(enabled=False): + with autocast("cuda", enabled=False): gen_stft_adv_loss = self.generator_adversarial_loss(outputs=y_hat) if self.multi_period_discriminator is not None: @@ -272,7 +272,7 @@ class Encodec(nn.Module): speech_hat.contiguous().detach(), ) # calculate losses - with autocast(enabled=False): + with autocast("cuda", enabled=False): ( disc_stft_real_adv_loss, disc_stft_fake_adv_loss, diff --git a/egs/libritts/CODEC/encodec/train.py b/egs/libritts/CODEC/encodec/train.py index a4f2eb7ab..31349df43 100755 --- a/egs/libritts/CODEC/encodec/train.py +++ b/egs/libritts/CODEC/encodec/train.py @@ -34,7 +34,7 @@ from encodec import Encodec from lhotse.utils import fix_random_seed from scheduler import WarmupCosineLrScheduler from torch import nn -from torch.cuda.amp import GradScaler, autocast +from torch.amp import GradScaler, autocast from torch.nn.parallel import DistributedDataParallel as DDP from torch.optim import Optimizer from torch.utils.tensorboard import SummaryWriter @@ -466,7 +466,7 @@ def train_one_epoch( loss_info["samples"] = batch_size try: - with autocast(enabled=params.use_fp16): + with autocast("cuda", enabled=params.use_fp16): d_weight = train_discriminator( params.lambda_adv, params.cur_epoch, @@ -502,7 +502,7 @@ def train_one_epoch( scaler.scale(disc_loss).backward() scaler.step(optimizer_d) - with autocast(enabled=params.use_fp16): + with autocast("cuda", enabled=params.use_fp16): g_weight = train_discriminator( params.lambda_adv, params.cur_epoch, @@ -846,7 +846,7 @@ def scan_pessimistic_batches_for_oom( ) = prepare_input(params, batch, device) try: # for discriminator - with autocast(enabled=params.use_fp16): + with autocast("cuda", enabled=params.use_fp16): ( disc_stft_real_adv_loss, disc_stft_fake_adv_loss, @@ -876,7 +876,7 @@ def scan_pessimistic_batches_for_oom( optimizer_d.zero_grad() loss_d.backward() # for generator - with autocast(enabled=params.use_fp16): + with autocast("cuda", enabled=params.use_fp16): ( commit_loss, gen_stft_adv_loss, @@ -1102,7 +1102,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) diff --git a/egs/libritts/TTS/vits/train.py b/egs/libritts/TTS/vits/train.py index 447fbcf5d..6803d6eb2 100755 --- a/egs/libritts/TTS/vits/train.py +++ b/egs/libritts/TTS/vits/train.py @@ -32,7 +32,7 @@ from lhotse.cut import Cut from lhotse.features.io import KaldiReader from lhotse.utils import fix_random_seed from tokenizer import Tokenizer -from torch.cuda.amp import GradScaler, autocast +from torch.amp import GradScaler, autocast from torch.nn.parallel import DistributedDataParallel as DDP from torch.optim import Optimizer from torch.utils.tensorboard import SummaryWriter @@ -456,7 +456,7 @@ def train_one_epoch( loss_info["samples"] = batch_size try: - with autocast(enabled=params.use_fp16): + with autocast("cuda", enabled=params.use_fp16): # forward discriminator loss_d, stats_d = model( text=tokens, @@ -475,7 +475,7 @@ def train_one_epoch( scaler.scale(loss_d).backward() scaler.step(optimizer_d) - with autocast(enabled=params.use_fp16): + with autocast("cuda", enabled=params.use_fp16): # forward generator loss_g, stats_g = model( text=tokens, @@ -748,7 +748,7 @@ def scan_pessimistic_batches_for_oom( ) = prepare_input(batch, tokenizer, device, train_speaker_map) try: # for discriminator - with autocast(enabled=params.use_fp16): + with autocast("cuda", enabled=params.use_fp16): loss_d, stats_d = model( text=tokens, text_lengths=tokens_lens, @@ -762,7 +762,7 @@ def scan_pessimistic_batches_for_oom( optimizer_d.zero_grad() loss_d.backward() # for generator - with autocast(enabled=params.use_fp16): + with autocast("cuda", enabled=params.use_fp16): loss_g, stats_g = model( text=tokens, text_lengths=tokens_lens, @@ -922,7 +922,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) diff --git a/egs/ljspeech/TTS/matcha/train.py b/egs/ljspeech/TTS/matcha/train.py index 853042413..a25cc8723 100755 --- a/egs/ljspeech/TTS/matcha/train.py +++ b/egs/ljspeech/TTS/matcha/train.py @@ -17,7 +17,7 @@ from lhotse.utils import fix_random_seed from model import fix_len_compatibility from models.matcha_tts import MatchaTTS from tokenizer import Tokenizer -from torch.cuda.amp import GradScaler, autocast +from torch.amp import GradScaler, autocast from torch.nn.parallel import DistributedDataParallel as DDP from torch.optim import Optimizer from torch.utils.tensorboard import SummaryWriter @@ -474,7 +474,7 @@ def train_one_epoch( tokens_lens, ) = prepare_input(batch, tokenizer, device, params) try: - with autocast(enabled=params.use_fp16): + with autocast("cuda", enabled=params.use_fp16): losses = get_losses( { "x": tokens, @@ -645,7 +645,7 @@ def run(rank, world_size, args): valid_cuts = ljspeech.valid_cuts() valid_dl = ljspeech.valid_dataloaders(valid_cuts) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) diff --git a/egs/ljspeech/TTS/vits/train.py b/egs/ljspeech/TTS/vits/train.py index 184ae79af..e9994319a 100755 --- a/egs/ljspeech/TTS/vits/train.py +++ b/egs/ljspeech/TTS/vits/train.py @@ -30,7 +30,7 @@ import torch.nn as nn from lhotse.cut import Cut from lhotse.utils import fix_random_seed from tokenizer import Tokenizer -from torch.cuda.amp import GradScaler, autocast +from torch.amp import GradScaler, autocast from torch.nn.parallel import DistributedDataParallel as DDP from torch.optim import Optimizer from torch.utils.tensorboard import SummaryWriter @@ -396,7 +396,7 @@ def train_one_epoch( loss_info["samples"] = batch_size try: - with autocast(enabled=params.use_fp16): + with autocast("cuda", enabled=params.use_fp16): # forward discriminator loss_d, stats_d = model( text=tokens, @@ -414,7 +414,7 @@ def train_one_epoch( scaler.scale(loss_d).backward() scaler.step(optimizer_d) - with autocast(enabled=params.use_fp16): + with autocast("cuda", enabled=params.use_fp16): # forward generator loss_g, stats_g = model( text=tokens, @@ -673,7 +673,7 @@ def scan_pessimistic_batches_for_oom( ) try: # for discriminator - with autocast(enabled=params.use_fp16): + with autocast("cuda", enabled=params.use_fp16): loss_d, stats_d = model( text=tokens, text_lengths=tokens_lens, @@ -686,7 +686,7 @@ def scan_pessimistic_batches_for_oom( optimizer_d.zero_grad() loss_d.backward() # for generator - with autocast(enabled=params.use_fp16): + with autocast("cuda", enabled=params.use_fp16): loss_g, stats_g = model( text=tokens, text_lengths=tokens_lens, @@ -838,7 +838,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) diff --git a/egs/ljspeech/TTS/vits/utils.py b/egs/ljspeech/TTS/vits/utils.py index 6a067f596..d51ff5f5c 100644 --- a/egs/ljspeech/TTS/vits/utils.py +++ b/egs/ljspeech/TTS/vits/utils.py @@ -23,7 +23,7 @@ import torch import torch.distributed as dist import torch.nn as nn from lhotse.dataset.sampling.base import CutSampler -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.optim import Optimizer from torch.utils.tensorboard import SummaryWriter diff --git a/egs/ljspeech/TTS/vits/vits.py b/egs/ljspeech/TTS/vits/vits.py index a1fabf9ad..6fd6d219b 100644 --- a/egs/ljspeech/TTS/vits/vits.py +++ b/egs/ljspeech/TTS/vits/vits.py @@ -25,7 +25,7 @@ from loss import ( KLDivergenceLoss, MelSpectrogramLoss, ) -from torch.cuda.amp import autocast +from torch.amp import autocast from utils import get_segments AVAILABLE_GENERATERS = { @@ -410,7 +410,7 @@ class VITS(nn.Module): p = self.discriminator(speech_) # calculate losses - with autocast(enabled=False): + with autocast("cuda", enabled=False): if not return_sample: mel_loss = self.mel_loss(speech_hat_, speech_) else: @@ -518,7 +518,7 @@ class VITS(nn.Module): p = self.discriminator(speech_) # calculate losses - with autocast(enabled=False): + with autocast("cuda", enabled=False): real_loss, fake_loss = self.discriminator_adv_loss(p_hat, p) loss = real_loss + fake_loss diff --git a/egs/mdcc/ASR/zipformer/train.py b/egs/mdcc/ASR/zipformer/train.py index 730db7718..22249286a 100755 --- a/egs/mdcc/ASR/zipformer/train.py +++ b/egs/mdcc/ASR/zipformer/train.py @@ -68,7 +68,7 @@ from optim import Eden, ScaledAdam from scaling import ScheduledFloat from subsampling import Conv2dSubsampling from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer2 @@ -906,7 +906,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1197,7 +1197,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1298,7 +1298,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/mgb2/ASR/pruned_transducer_stateless5/train.py b/egs/mgb2/ASR/pruned_transducer_stateless5/train.py index 48468cfbd..916ada475 100755 --- a/egs/mgb2/ASR/pruned_transducer_stateless5/train.py +++ b/egs/mgb2/ASR/pruned_transducer_stateless5/train.py @@ -66,7 +66,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, Eve from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.nn.utils import clip_grad_norm_ from torch.utils.tensorboard import SummaryWriter @@ -751,7 +751,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info, inf_flag = compute_loss( params=params, model=model, @@ -1012,7 +1012,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16) + scaler = GradScaler("cuda", enabled=params.use_fp16) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1115,7 +1115,7 @@ def scan_pessimistic_batches_for_oom( # warmup = 0.0 is so that the derivs for the pruned loss stay zero # (i.e. are not remembered by the decaying-average in adam), because # we want to avoid these params being subject to shrinkage in adam. - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _, _ = compute_loss( params=params, model=model, diff --git a/egs/multi_zh-hans/ASR/whisper/train.py b/egs/multi_zh-hans/ASR/whisper/train.py index fe2d950c1..1a11d01af 100755 --- a/egs/multi_zh-hans/ASR/whisper/train.py +++ b/egs/multi_zh-hans/ASR/whisper/train.py @@ -61,7 +61,7 @@ from lhotse.utils import fix_random_seed from multi_dataset import MultiDataset from optim import Eden, ScaledAdam from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.functional import pad as pad_tensor from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -566,7 +566,7 @@ def compute_validation_loss( tot_loss = MetricsTracker() for batch_idx, batch in enumerate(valid_dl): - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, tokenizer=tokenizer, @@ -675,7 +675,7 @@ def train_one_epoch( ) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, tokenizer=tokenizer, @@ -913,7 +913,7 @@ def run(rank, world_size, args): valid_cuts = multi_dataset.dev_cuts() valid_dl = data_module.valid_dataloaders(valid_cuts) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) diff --git a/egs/multi_zh-hans/ASR/zipformer/train.py b/egs/multi_zh-hans/ASR/zipformer/train.py index 3dbfc48eb..047253d5b 100755 --- a/egs/multi_zh-hans/ASR/zipformer/train.py +++ b/egs/multi_zh-hans/ASR/zipformer/train.py @@ -75,7 +75,7 @@ from optim import Eden, ScaledAdam from scaling import ScheduledFloat from subsampling import Conv2dSubsampling from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer2 @@ -987,7 +987,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1278,7 +1278,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1378,7 +1378,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/multi_zh_en/ASR/zipformer/train.py b/egs/multi_zh_en/ASR/zipformer/train.py index 04bb41214..9e64defa3 100755 --- a/egs/multi_zh_en/ASR/zipformer/train.py +++ b/egs/multi_zh_en/ASR/zipformer/train.py @@ -75,7 +75,7 @@ from optim import Eden, ScaledAdam from scaling import ScheduledFloat from subsampling import Conv2dSubsampling from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer2 @@ -969,7 +969,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1269,7 +1269,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1369,7 +1369,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/reazonspeech/ASR/zipformer/do_not_use_it_directly.py b/egs/reazonspeech/ASR/zipformer/do_not_use_it_directly.py index 072679cfc..c01e4d336 100755 --- a/egs/reazonspeech/ASR/zipformer/do_not_use_it_directly.py +++ b/egs/reazonspeech/ASR/zipformer/do_not_use_it_directly.py @@ -67,7 +67,7 @@ from model import Transducer from optim import Eden, ScaledAdam from tokenizer import Tokenizer from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer_for_ncnn_export_only import Zipformer @@ -822,7 +822,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1113,7 +1113,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1213,7 +1213,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/reazonspeech/ASR/zipformer/train.py b/egs/reazonspeech/ASR/zipformer/train.py index 30bd3efba..8829a18ca 100755 --- a/egs/reazonspeech/ASR/zipformer/train.py +++ b/egs/reazonspeech/ASR/zipformer/train.py @@ -74,7 +74,7 @@ from scaling import ScheduledFloat from subsampling import Conv2dSubsampling from tokenizer import Tokenizer from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer2 @@ -945,7 +945,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1235,7 +1235,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1335,7 +1335,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/speech_llm/ASR_LLM/whisper_llm_zh/train.py b/egs/speech_llm/ASR_LLM/whisper_llm_zh/train.py index 5f224c984..5de2cf2b0 100755 --- a/egs/speech_llm/ASR_LLM/whisper_llm_zh/train.py +++ b/egs/speech_llm/ASR_LLM/whisper_llm_zh/train.py @@ -451,7 +451,7 @@ def compute_validation_loss( tot_loss = MetricsTracker() for batch_idx, batch in enumerate(valid_dl): - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, tokenizer=tokenizer, @@ -566,7 +566,7 @@ def train_one_epoch( f"rm -rf {params.exp_dir}/epoch-{params.cur_epoch}-checkpoint-{batch_idx}" ) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, tokenizer=tokenizer, diff --git a/egs/spgispeech/ASR/pruned_transducer_stateless2/train.py b/egs/spgispeech/ASR/pruned_transducer_stateless2/train.py index a9146a0fe..1e55ada87 100755 --- a/egs/spgispeech/ASR/pruned_transducer_stateless2/train.py +++ b/egs/spgispeech/ASR/pruned_transducer_stateless2/train.py @@ -65,7 +65,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, Eve from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -649,7 +649,7 @@ def train_one_epoch( params.batch_idx_train += 1 batch_size = len(batch["supervisions"]["text"]) - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -857,7 +857,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16) + scaler = GradScaler("cuda", enabled=params.use_fp16) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -957,7 +957,7 @@ def scan_pessimistic_batches_for_oom( # warmup = 0.0 is so that the derivs for the pruned loss stay zero # (i.e. are not remembered by the decaying-average in adam), because # we want to avoid these params being subject to shrinkage in adam. - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/spgispeech/ASR/zipformer/train.py b/egs/spgispeech/ASR/zipformer/train.py index dfc21c968..319713b02 100755 --- a/egs/spgispeech/ASR/zipformer/train.py +++ b/egs/spgispeech/ASR/zipformer/train.py @@ -74,7 +74,7 @@ from optim import Eden, ScaledAdam from scaling import ScheduledFloat from subsampling import Conv2dSubsampling from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer2 @@ -946,7 +946,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1217,7 +1217,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1317,7 +1317,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/tal_csasr/ASR/pruned_transducer_stateless5/train.py b/egs/tal_csasr/ASR/pruned_transducer_stateless5/train.py index c0aedd725..c44e30b89 100755 --- a/egs/tal_csasr/ASR/pruned_transducer_stateless5/train.py +++ b/egs/tal_csasr/ASR/pruned_transducer_stateless5/train.py @@ -69,7 +69,7 @@ from local.tokenize_with_bpe_model import tokenize_by_bpe_model from model import Transducer from optim import Eden, Eve from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -726,7 +726,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) # print(batch["supervisions"]) - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -967,7 +967,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16) + scaler = GradScaler("cuda", enabled=params.use_fp16) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1039,7 +1039,7 @@ def scan_pessimistic_batches_for_oom( # warmup = 0.0 is so that the derivs for the pruned loss stay zero # (i.e. are not remembered by the decaying-average in adam), because # we want to avoid these params being subject to shrinkage in adam. - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/tal_csasr/ASR/pruned_transducer_stateless7_bbpe/train.py b/egs/tal_csasr/ASR/pruned_transducer_stateless7_bbpe/train.py index 2108266ec..dd9576d99 100755 --- a/egs/tal_csasr/ASR/pruned_transducer_stateless7_bbpe/train.py +++ b/egs/tal_csasr/ASR/pruned_transducer_stateless7_bbpe/train.py @@ -64,7 +64,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, ScaledAdam from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer @@ -801,7 +801,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1101,7 +1101,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1201,7 +1201,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/tedlium3/ASR/conformer_ctc2/train.py b/egs/tedlium3/ASR/conformer_ctc2/train.py index fc3e3b2d9..179dcf14a 100755 --- a/egs/tedlium3/ASR/conformer_ctc2/train.py +++ b/egs/tedlium3/ASR/conformer_ctc2/train.py @@ -57,7 +57,7 @@ from lhotse.dataset.sampling.base import CutSampler from lhotse.utils import fix_random_seed from local.convert_transcript_words_to_bpe_ids import convert_texts_into_ids from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -710,7 +710,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -941,7 +941,7 @@ def run(rank, world_size, args): warmup=0.0 if params.start_epoch == 1 else 1.0, ) - scaler = GradScaler(enabled=params.use_fp16) + scaler = GradScaler("cuda", enabled=params.use_fp16) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1011,7 +1011,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/tedlium3/ASR/zipformer/model.py b/egs/tedlium3/ASR/zipformer/model.py index 65b052ab9..0d9b395ed 100644 --- a/egs/tedlium3/ASR/zipformer/model.py +++ b/egs/tedlium3/ASR/zipformer/model.py @@ -173,7 +173,7 @@ class Transducer(nn.Module): # if self.training and random.random() < 0.25: # am = penalize_abs_values_gt(am, 30.0, 1.0e-04) - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): simple_loss, (px_grad, py_grad) = k2.rnnt_loss_smoothed( lm=lm.float(), am=am.float(), @@ -209,7 +209,7 @@ class Transducer(nn.Module): # prior to do_rnnt_pruning (this is an optimization for speed). logits = self.joiner(am_pruned, lm_pruned, project_input=False) - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast("cuda", enabled=False): pruned_loss = k2.rnnt_loss_pruned( logits=logits.float(), symbols=y_padded, diff --git a/egs/tedlium3/ASR/zipformer/train.py b/egs/tedlium3/ASR/zipformer/train.py index 14a44efb3..ffe876863 100755 --- a/egs/tedlium3/ASR/zipformer/train.py +++ b/egs/tedlium3/ASR/zipformer/train.py @@ -73,7 +73,7 @@ from optim import Eden, ScaledAdam from scaling import ScheduledFloat from subsampling import Conv2dSubsampling from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer2 @@ -911,7 +911,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1160,7 +1160,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1260,7 +1260,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/vctk/TTS/vits/train.py b/egs/vctk/TTS/vits/train.py index 4686de169..6249640d4 100755 --- a/egs/vctk/TTS/vits/train.py +++ b/egs/vctk/TTS/vits/train.py @@ -31,7 +31,7 @@ import torch.nn as nn from lhotse.cut import Cut from lhotse.utils import fix_random_seed from tokenizer import Tokenizer -from torch.cuda.amp import GradScaler, autocast +from torch.amp import GradScaler, autocast from torch.nn.parallel import DistributedDataParallel as DDP from torch.optim import Optimizer from torch.utils.tensorboard import SummaryWriter @@ -448,7 +448,7 @@ def train_one_epoch( loss_info["samples"] = batch_size try: - with autocast(enabled=params.use_fp16): + with autocast("cuda", enabled=params.use_fp16): # forward discriminator loss_d, stats_d = model( text=tokens, @@ -467,7 +467,7 @@ def train_one_epoch( scaler.scale(loss_d).backward() scaler.step(optimizer_d) - with autocast(enabled=params.use_fp16): + with autocast("cuda", enabled=params.use_fp16): # forward generator loss_g, stats_g = model( text=tokens, @@ -740,7 +740,7 @@ def scan_pessimistic_batches_for_oom( ) = prepare_input(batch, tokenizer, device, speaker_map) try: # for discriminator - with autocast(enabled=params.use_fp16): + with autocast("cuda", enabled=params.use_fp16): loss_d, stats_d = model( text=tokens, text_lengths=tokens_lens, @@ -754,7 +754,7 @@ def scan_pessimistic_batches_for_oom( optimizer_d.zero_grad() loss_d.backward() # for generator - with autocast(enabled=params.use_fp16): + with autocast("cuda", enabled=params.use_fp16): loss_g, stats_g = model( text=tokens, text_lengths=tokens_lens, @@ -910,7 +910,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) diff --git a/egs/wenetspeech/ASR/pruned_transducer_stateless2/finetune.py b/egs/wenetspeech/ASR/pruned_transducer_stateless2/finetune.py index c34f1593d..2fd6f6478 100755 --- a/egs/wenetspeech/ASR/pruned_transducer_stateless2/finetune.py +++ b/egs/wenetspeech/ASR/pruned_transducer_stateless2/finetune.py @@ -52,7 +52,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, Eve from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -718,7 +718,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -907,7 +907,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16) + scaler = GradScaler("cuda", enabled=params.use_fp16) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1005,7 +1005,7 @@ def scan_pessimistic_batches_for_oom( # warmup = 0.0 is so that the derivs for the pruned loss stay zero # (i.e. are not remembered by the decaying-average in adam), because # we want to avoid these params being subject to shrinkage in adam. - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/wenetspeech/ASR/pruned_transducer_stateless2/train.py b/egs/wenetspeech/ASR/pruned_transducer_stateless2/train.py index 49977e01b..c90f03f08 100644 --- a/egs/wenetspeech/ASR/pruned_transducer_stateless2/train.py +++ b/egs/wenetspeech/ASR/pruned_transducer_stateless2/train.py @@ -101,7 +101,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, Eve from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -687,7 +687,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -921,7 +921,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16) + scaler = GradScaler("cuda", enabled=params.use_fp16) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1019,7 +1019,7 @@ def scan_pessimistic_batches_for_oom( # warmup = 0.0 is so that the derivs for the pruned loss stay zero # (i.e. are not remembered by the decaying-average in adam), because # we want to avoid these params being subject to shrinkage in adam. - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/wenetspeech/ASR/pruned_transducer_stateless5/train.py b/egs/wenetspeech/ASR/pruned_transducer_stateless5/train.py index 931e699d9..7b05eca97 100755 --- a/egs/wenetspeech/ASR/pruned_transducer_stateless5/train.py +++ b/egs/wenetspeech/ASR/pruned_transducer_stateless5/train.py @@ -81,7 +81,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, Eve from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -796,7 +796,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1056,7 +1056,7 @@ def run(rank, world_size, args): warmup=0.0 if params.start_epoch == 1 else 1.0, ) - scaler = GradScaler(enabled=params.use_fp16) + scaler = GradScaler("cuda", enabled=params.use_fp16) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1158,7 +1158,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/wenetspeech/ASR/whisper/train.py b/egs/wenetspeech/ASR/whisper/train.py index 4e55fd6a8..c46a4d84c 100644 --- a/egs/wenetspeech/ASR/whisper/train.py +++ b/egs/wenetspeech/ASR/whisper/train.py @@ -61,7 +61,7 @@ from lhotse.dataset.sampling.base import CutSampler from lhotse.utils import fix_random_seed from optim import Eden, ScaledAdam from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.functional import pad as pad_tensor from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -513,7 +513,7 @@ def compute_validation_loss( tot_loss = MetricsTracker() for batch_idx, batch in enumerate(valid_dl): - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, tokenizer=tokenizer, @@ -621,7 +621,7 @@ def train_one_epoch( f"rm -rf {params.exp_dir}/epoch-{params.cur_epoch}-checkpoint-{batch_idx}" ) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, tokenizer=tokenizer, @@ -843,7 +843,7 @@ def run(rank, world_size, args): train_dl = wenetspeech.train_dataloaders(train_cuts) valid_dl = wenetspeech.valid_dataloaders(wenetspeech.valid_cuts()) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) diff --git a/egs/wenetspeech/ASR/zipformer/train.py b/egs/wenetspeech/ASR/zipformer/train.py index 25b16f632..b6d55447f 100755 --- a/egs/wenetspeech/ASR/zipformer/train.py +++ b/egs/wenetspeech/ASR/zipformer/train.py @@ -71,7 +71,7 @@ from optim import Eden, ScaledAdam from scaling import ScheduledFloat from subsampling import Conv2dSubsampling from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer2 @@ -910,7 +910,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1201,7 +1201,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1302,7 +1302,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/wenetspeech/KWS/zipformer/finetune.py b/egs/wenetspeech/KWS/zipformer/finetune.py index d19172b38..00db4309d 100755 --- a/egs/wenetspeech/KWS/zipformer/finetune.py +++ b/egs/wenetspeech/KWS/zipformer/finetune.py @@ -82,7 +82,7 @@ from lhotse.cut import Cut, CutSet from lhotse.utils import fix_random_seed from optim import Eden, ScaledAdam from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from train import ( @@ -414,7 +414,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -703,7 +703,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) diff --git a/egs/wenetspeech/KWS/zipformer/train.py b/egs/wenetspeech/KWS/zipformer/train.py index 40960c2ae..4dc30ad89 100755 --- a/egs/wenetspeech/KWS/zipformer/train.py +++ b/egs/wenetspeech/KWS/zipformer/train.py @@ -73,7 +73,7 @@ from optim import Eden, ScaledAdam from scaling import ScheduledFloat from subsampling import Conv2dSubsampling from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer2 @@ -967,7 +967,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1252,7 +1252,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1353,7 +1353,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/wenetspeech4tts/TTS/valle/train.py b/egs/wenetspeech4tts/TTS/valle/train.py index e9ec548f3..1c6972e93 100755 --- a/egs/wenetspeech4tts/TTS/valle/train.py +++ b/egs/wenetspeech4tts/TTS/valle/train.py @@ -65,7 +65,7 @@ from lhotse.utils import fix_random_seed from optim import Eden, ScaledAdam from tokenizer import TextTokenCollater, get_text_token_collater from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from tts_datamodule import TtsDataModule @@ -764,7 +764,7 @@ def train_one_epoch( batch_size = len(batch["text"]) try: - with torch.cuda.amp.autocast(dtype=dtype, enabled=enabled): + with torch.amp.autocast("cuda", dtype=dtype, enabled=enabled): _, loss, loss_info = compute_loss( params=params, model=model, @@ -897,7 +897,7 @@ def train_one_epoch( # Calculate validation loss in Rank 0 model.eval() logging.info("Computing validation loss") - with torch.cuda.amp.autocast(dtype=dtype): + with torch.amp.autocast("cuda", dtype=dtype): valid_info = compute_validation_loss( params=params, model=model, @@ -1102,7 +1102,9 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=(params.dtype in ["fp16", "float16"]), init_scale=1.0) + scaler = GradScaler( + "cuda", enabled=(params.dtype in ["fp16", "float16"]), init_scale=1.0 + ) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1196,7 +1198,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(dtype=dtype): + with torch.amp.autocast("cuda", dtype=dtype): _, loss, _ = compute_loss( params=params, model=model, diff --git a/egs/xbmu_amdo31/ASR/pruned_transducer_stateless5/train.py b/egs/xbmu_amdo31/ASR/pruned_transducer_stateless5/train.py index a6fa46b17..5c3000a57 100755 --- a/egs/xbmu_amdo31/ASR/pruned_transducer_stateless5/train.py +++ b/egs/xbmu_amdo31/ASR/pruned_transducer_stateless5/train.py @@ -68,7 +68,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, Eve from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -814,7 +814,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1072,7 +1072,7 @@ def run(rank, world_size, args): warmup=0.0 if params.start_epoch == 1 else 1.0, ) - scaler = GradScaler(enabled=params.use_fp16) + scaler = GradScaler("cuda", enabled=params.use_fp16) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1141,7 +1141,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/xbmu_amdo31/ASR/pruned_transducer_stateless7/train.py b/egs/xbmu_amdo31/ASR/pruned_transducer_stateless7/train.py index dd72551d9..a1b3be246 100755 --- a/egs/xbmu_amdo31/ASR/pruned_transducer_stateless7/train.py +++ b/egs/xbmu_amdo31/ASR/pruned_transducer_stateless7/train.py @@ -67,7 +67,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, ScaledAdam from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer @@ -785,7 +785,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1074,7 +1074,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1174,7 +1174,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/icefall/checkpoint.py b/icefall/checkpoint.py index d31ce1301..b3a0fb865 100644 --- a/icefall/checkpoint.py +++ b/icefall/checkpoint.py @@ -27,7 +27,7 @@ import torch import torch.nn as nn from lhotse.dataset.sampling.base import CutSampler from torch import Tensor -from torch.cuda.amp import GradScaler +from torch.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.optim import Optimizer diff --git a/icefall/rnn_lm/train.py b/icefall/rnn_lm/train.py index 0178b80bf..257cdb09a 100755 --- a/icefall/rnn_lm/train.py +++ b/icefall/rnn_lm/train.py @@ -401,7 +401,7 @@ def compute_validation_loss( for batch_idx, batch in enumerate(valid_dl): x, y, sentence_lengths = batch - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( model=model, x=x, @@ -470,7 +470,7 @@ def train_one_epoch( params.batch_idx_train += 1 x, y, sentence_lengths = batch batch_size = x.size(0) - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( model=model, x=x, diff --git a/icefall/transformer_lm/train.py b/icefall/transformer_lm/train.py index c36abfcdf..6faa63484 100644 --- a/icefall/transformer_lm/train.py +++ b/icefall/transformer_lm/train.py @@ -341,7 +341,7 @@ def compute_validation_loss( for batch_idx, batch in enumerate(valid_dl): x, y, sentence_lengths = batch - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( model=model, x=x, @@ -403,7 +403,7 @@ def train_one_epoch( params.batch_idx_train += 1 x, y, sentence_lengths = batch batch_size = x.size(0) - with torch.cuda.amp.autocast(enabled=params.use_fp16): + with torch.amp.autocast("cuda", enabled=params.use_fp16): loss, loss_info = compute_loss( model=model, x=x, From d4d4f281ecefad7b779def552975d2881620a724 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Wed, 18 Dec 2024 16:49:57 +0800 Subject: [PATCH 14/59] Revert "Replace deprecated pytorch methods (#1814)" (#1841) This reverts commit 3e4da5f78160d3dba3bdf97968bd7ceb8c11631f. --- .../ASR/pruned_transducer_stateless2/train.py | 8 ++++---- .../ASR/pruned_transducer_stateless2/train.py | 8 ++++---- .../ASR/pruned_transducer_stateless3/model.py | 4 ++-- .../ASR/pruned_transducer_stateless3/train.py | 8 ++++---- .../do_not_use_it_directly.py | 8 ++++---- .../ASR/pruned_transducer_stateless7/train.py | 8 ++++---- .../ASR/pruned_transducer_stateless7_bbpe/train.py | 8 ++++---- .../do_not_use_it_directly.py | 8 ++++---- .../pruned_transducer_stateless7_streaming/train.py | 8 ++++---- egs/aishell/ASR/whisper/train.py | 8 ++++---- egs/aishell/ASR/zipformer/train.py | 8 ++++---- egs/aishell/ASR/zipformer/train_bbpe.py | 8 ++++---- .../ASR/pruned_transducer_stateless5/train.py | 8 ++++---- .../ASR/pruned_transducer_stateless5/train.py | 8 ++++---- .../ASR/pruned_transducer_stateless2/train.py | 8 ++++---- .../ASR_v2/pruned_transducer_stateless7/train.py | 8 ++++---- egs/ami/ASR/pruned_transducer_stateless7/train.py | 8 ++++---- egs/ami/SURT/dprnn_zipformer/train.py | 6 +++--- egs/ami/SURT/dprnn_zipformer/train_adapt.py | 6 +++--- egs/audioset/AT/zipformer/train.py | 8 ++++---- .../ASR/pruned_transducer_stateless7/train.py | 8 ++++---- .../do_not_use_it_directly.py | 8 ++++---- .../finetune.py | 8 ++++---- .../pruned_transducer_stateless7_streaming/train.py | 8 ++++---- egs/commonvoice/ASR/zipformer/train.py | 8 ++++---- egs/commonvoice/ASR/zipformer/train_char.py | 8 ++++---- .../do_not_use_it_directly.py | 8 ++++---- .../pruned_transducer_stateless7_streaming/train.py | 8 ++++---- .../ASR/pruned_transducer_stateless2/train.py | 8 ++++---- egs/gigaspeech/ASR/zipformer/train.py | 8 ++++---- egs/gigaspeech/KWS/zipformer/finetune.py | 6 +++--- egs/gigaspeech/KWS/zipformer/train.py | 8 ++++---- .../pruned_transducer_stateless7_streaming/train.py | 8 ++++---- egs/ksponspeech/ASR/zipformer/train.py | 8 ++++---- egs/libricss/SURT/dprnn_zipformer/model.py | 4 ++-- egs/libricss/SURT/dprnn_zipformer/scaling.py | 6 +++--- egs/libricss/SURT/dprnn_zipformer/train.py | 6 +++--- egs/libricss/SURT/dprnn_zipformer/train_adapt.py | 6 +++--- egs/libriheavy/ASR/zipformer/train.py | 8 ++++---- .../ASR/zipformer_prompt_asr/model_baseline.py | 4 ++-- .../ASR/zipformer_prompt_asr/model_with_BERT.py | 4 ++-- egs/libriheavy/ASR/zipformer_prompt_asr/scaling.py | 10 +++++----- .../ASR/zipformer_prompt_asr/train_baseline.py | 8 ++++---- .../ASR/zipformer_prompt_asr/train_bert_encoder.py | 8 ++++---- egs/libriheavy/ASR/zipformer_prompt_asr/zipformer.py | 4 ++-- egs/librilight/SSL/zipformer/finetune.py | 8 ++++---- egs/librilight/SSL/zipformer/pretrain.py | 8 ++++---- egs/librispeech/ASR/conformer_ctc2/train.py | 8 ++++---- egs/librispeech/ASR/conformer_ctc3/train.py | 8 ++++---- .../ASR/conv_emformer_transducer_stateless/train.py | 8 ++++---- .../do_not_use_it_directly.py | 8 ++++---- .../ASR/conv_emformer_transducer_stateless2/train.py | 8 ++++---- .../ASR/lstm_transducer_stateless/model.py | 4 ++-- .../ASR/lstm_transducer_stateless/train.py | 8 ++++---- .../ASR/lstm_transducer_stateless2/model.py | 4 ++-- .../ASR/lstm_transducer_stateless2/train.py | 8 ++++---- .../ASR/lstm_transducer_stateless3/train.py | 8 ++++---- egs/librispeech/ASR/pruned2_knowledge/model.py | 4 ++-- egs/librispeech/ASR/pruned2_knowledge/sampling.py | 6 +++--- egs/librispeech/ASR/pruned2_knowledge/train.py | 8 ++++---- .../ASR/pruned_stateless_emformer_rnnt2/train.py | 8 ++++---- .../ASR/pruned_transducer_stateless2/model.py | 4 ++-- .../ASR/pruned_transducer_stateless2/train.py | 8 ++++---- .../ASR/pruned_transducer_stateless3/model.py | 4 ++-- .../ASR/pruned_transducer_stateless3/train.py | 8 ++++---- .../ASR/pruned_transducer_stateless4/train.py | 8 ++++---- .../ASR/pruned_transducer_stateless5/train.py | 8 ++++---- .../ASR/pruned_transducer_stateless6/model.py | 4 ++-- .../ASR/pruned_transducer_stateless6/train.py | 8 ++++---- .../ASR/pruned_transducer_stateless7/finetune.py | 8 ++++---- .../ASR/pruned_transducer_stateless7/model.py | 4 ++-- .../ASR/pruned_transducer_stateless7/scaling.py | 6 +++--- .../ASR/pruned_transducer_stateless7/train.py | 8 ++++---- .../ASR/pruned_transducer_stateless7/zipformer.py | 2 +- .../ASR/pruned_transducer_stateless7_ctc/model.py | 4 ++-- .../ASR/pruned_transducer_stateless7_ctc/train.py | 8 ++++---- .../ASR/pruned_transducer_stateless7_ctc_bs/model.py | 4 ++-- .../ASR/pruned_transducer_stateless7_ctc_bs/train.py | 8 ++++---- .../do_not_use_it_directly.py | 8 ++++---- .../pruned_transducer_stateless7_streaming/train.py | 8 ++++---- .../zipformer.py | 2 +- .../zipformer_for_ncnn_export_only.py | 2 +- .../train.py | 8 ++++---- .../ASR/pruned_transducer_stateless8/model.py | 4 ++-- .../ASR/pruned_transducer_stateless8/train.py | 8 ++++---- egs/librispeech/ASR/tiny_transducer_ctc/train.py | 8 ++++---- egs/librispeech/ASR/zipformer/finetune.py | 8 ++++---- egs/librispeech/ASR/zipformer/model.py | 4 ++-- egs/librispeech/ASR/zipformer/scaling.py | 10 +++++----- egs/librispeech/ASR/zipformer/train.py | 8 ++++---- egs/librispeech/ASR/zipformer/zipformer.py | 2 +- egs/librispeech/ASR/zipformer_adapter/train.py | 8 ++++---- egs/librispeech/ASR/zipformer_adapter/zipformer.py | 2 +- egs/librispeech/ASR/zipformer_ctc/train.py | 6 +++--- egs/librispeech/ASR/zipformer_lora/finetune.py | 8 ++++---- egs/librispeech/ASR/zipformer_lora/scaling.py | 10 +++++----- egs/librispeech/ASR/zipformer_lora/train.py | 8 ++++---- egs/librispeech/ASR/zipformer_lora/zipformer.py | 2 +- egs/librispeech/ASR/zipformer_mmi/train.py | 8 ++++---- egs/librispeech/SSL/hubert/finetune.py | 8 ++++---- egs/librispeech/SSL/hubert/finetune_ce.py | 8 ++++---- egs/librispeech/SSL/hubert/model.py | 4 ++-- egs/librispeech/SSL/hubert/pretrain.py | 8 ++++---- egs/librispeech/SSL/hubert/pretrain_ce.py | 8 ++++---- egs/librispeech/SSL/zipformer/finetune.py | 8 ++++---- egs/librispeech/SSL/zipformer/model.py | 4 ++-- egs/librispeech/SSL/zipformer/pretrain.py | 8 ++++---- egs/librispeech/SSL/zipformer/zipformer.py | 2 +- egs/librispeech/WSASR/conformer_ctc2/train.py | 8 ++++---- egs/librispeech/WSASR/conformer_ctc2/train_phone.py | 8 ++++---- egs/libritts/ASR/zipformer/train.py | 12 ++++++------ egs/libritts/CODEC/encodec/encodec.py | 6 +++--- egs/libritts/CODEC/encodec/train.py | 12 ++++++------ egs/libritts/TTS/vits/train.py | 12 ++++++------ egs/ljspeech/TTS/matcha/train.py | 6 +++--- egs/ljspeech/TTS/vits/train.py | 12 ++++++------ egs/ljspeech/TTS/vits/utils.py | 2 +- egs/ljspeech/TTS/vits/vits.py | 6 +++--- egs/mdcc/ASR/zipformer/train.py | 8 ++++---- egs/mgb2/ASR/pruned_transducer_stateless5/train.py | 8 ++++---- egs/multi_zh-hans/ASR/whisper/train.py | 8 ++++---- egs/multi_zh-hans/ASR/zipformer/train.py | 8 ++++---- egs/multi_zh_en/ASR/zipformer/train.py | 8 ++++---- .../ASR/zipformer/do_not_use_it_directly.py | 8 ++++---- egs/reazonspeech/ASR/zipformer/train.py | 8 ++++---- egs/speech_llm/ASR_LLM/whisper_llm_zh/train.py | 4 ++-- .../ASR/pruned_transducer_stateless2/train.py | 8 ++++---- egs/spgispeech/ASR/zipformer/train.py | 8 ++++---- .../ASR/pruned_transducer_stateless5/train.py | 8 ++++---- .../ASR/pruned_transducer_stateless7_bbpe/train.py | 8 ++++---- egs/tedlium3/ASR/conformer_ctc2/train.py | 8 ++++---- egs/tedlium3/ASR/zipformer/model.py | 4 ++-- egs/tedlium3/ASR/zipformer/train.py | 8 ++++---- egs/vctk/TTS/vits/train.py | 12 ++++++------ .../ASR/pruned_transducer_stateless2/finetune.py | 8 ++++---- .../ASR/pruned_transducer_stateless2/train.py | 8 ++++---- .../ASR/pruned_transducer_stateless5/train.py | 8 ++++---- egs/wenetspeech/ASR/whisper/train.py | 8 ++++---- egs/wenetspeech/ASR/zipformer/train.py | 8 ++++---- egs/wenetspeech/KWS/zipformer/finetune.py | 6 +++--- egs/wenetspeech/KWS/zipformer/train.py | 8 ++++---- egs/wenetspeech4tts/TTS/valle/train.py | 12 +++++------- .../ASR/pruned_transducer_stateless5/train.py | 8 ++++---- .../ASR/pruned_transducer_stateless7/train.py | 8 ++++---- icefall/checkpoint.py | 2 +- icefall/rnn_lm/train.py | 4 ++-- icefall/transformer_lm/train.py | 4 ++-- 147 files changed, 518 insertions(+), 520 deletions(-) diff --git a/egs/aidatatang_200zh/ASR/pruned_transducer_stateless2/train.py b/egs/aidatatang_200zh/ASR/pruned_transducer_stateless2/train.py index 9088378fa..fa809b768 100644 --- a/egs/aidatatang_200zh/ASR/pruned_transducer_stateless2/train.py +++ b/egs/aidatatang_200zh/ASR/pruned_transducer_stateless2/train.py @@ -67,7 +67,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, Eve from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -638,7 +638,7 @@ def train_one_epoch( params.batch_idx_train += 1 batch_size = len(batch["supervisions"]["text"]) - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -843,7 +843,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16) + scaler = GradScaler(enabled=params.use_fp16) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -912,7 +912,7 @@ def scan_pessimistic_batches_for_oom( # warmup = 0.0 is so that the derivs for the pruned loss stay zero # (i.e. are not remembered by the decaying-average in adam), because # we want to avoid these params being subject to shrinkage in adam. - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/aishell/ASR/pruned_transducer_stateless2/train.py b/egs/aishell/ASR/pruned_transducer_stateless2/train.py index dda098e99..60f014c48 100755 --- a/egs/aishell/ASR/pruned_transducer_stateless2/train.py +++ b/egs/aishell/ASR/pruned_transducer_stateless2/train.py @@ -60,7 +60,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, Eve from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -688,7 +688,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -888,7 +888,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16) + scaler = GradScaler(enabled=params.use_fp16) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -989,7 +989,7 @@ def scan_pessimistic_batches_for_oom( # warmup = 0.0 is so that the derivs for the pruned loss stay zero # (i.e. are not remembered by the decaying-average in adam), because # we want to avoid these params being subject to shrinkage in adam. - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/aishell/ASR/pruned_transducer_stateless3/model.py b/egs/aishell/ASR/pruned_transducer_stateless3/model.py index cafc9d1bb..a4dda0d6d 100644 --- a/egs/aishell/ASR/pruned_transducer_stateless3/model.py +++ b/egs/aishell/ASR/pruned_transducer_stateless3/model.py @@ -184,7 +184,7 @@ class Transducer(nn.Module): lm = simple_lm_proj(decoder_out) am = simple_am_proj(encoder_out) - with torch.amp.autocast("cuda", enabled=False): + with torch.cuda.amp.autocast(enabled=False): simple_loss, (px_grad, py_grad) = k2.rnnt_loss_smoothed( lm=lm.float(), am=am.float(), @@ -219,7 +219,7 @@ class Transducer(nn.Module): # prior to do_rnnt_pruning (this is an optimization for speed). logits = joiner(am_pruned, lm_pruned, project_input=False) - with torch.amp.autocast("cuda", enabled=False): + with torch.cuda.amp.autocast(enabled=False): pruned_loss = k2.rnnt_loss_pruned( logits=logits.float(), symbols=y_padded, diff --git a/egs/aishell/ASR/pruned_transducer_stateless3/train.py b/egs/aishell/ASR/pruned_transducer_stateless3/train.py index bf60c4fad..7c23041ca 100755 --- a/egs/aishell/ASR/pruned_transducer_stateless3/train.py +++ b/egs/aishell/ASR/pruned_transducer_stateless3/train.py @@ -79,7 +79,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, Eve from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -797,7 +797,7 @@ def train_one_epoch( aishell = is_aishell(batch["supervisions"]["cut"][0]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1096,7 +1096,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16) + scaler = GradScaler(enabled=params.use_fp16) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1202,7 +1202,7 @@ def scan_pessimistic_batches_for_oom( # warmup = 0.0 is so that the derivs for the pruned loss stay zero # (i.e. are not remembered by the decaying-average in adam), because # we want to avoid these params being subject to shrinkage in adam. - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/aishell/ASR/pruned_transducer_stateless7/do_not_use_it_directly.py b/egs/aishell/ASR/pruned_transducer_stateless7/do_not_use_it_directly.py index 9a9d92c20..058d0ff6b 100755 --- a/egs/aishell/ASR/pruned_transducer_stateless7/do_not_use_it_directly.py +++ b/egs/aishell/ASR/pruned_transducer_stateless7/do_not_use_it_directly.py @@ -74,7 +74,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, ScaledAdam from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer @@ -812,7 +812,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1107,7 +1107,7 @@ def run(rank, world_size, args): # params=params, # ) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1206,7 +1206,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/aishell/ASR/pruned_transducer_stateless7/train.py b/egs/aishell/ASR/pruned_transducer_stateless7/train.py index ede2bd3e5..2dc835f3b 100755 --- a/egs/aishell/ASR/pruned_transducer_stateless7/train.py +++ b/egs/aishell/ASR/pruned_transducer_stateless7/train.py @@ -70,7 +70,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, ScaledAdam from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer @@ -809,7 +809,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1107,7 +1107,7 @@ def run(rank, world_size, args): # params=params, # ) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1206,7 +1206,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/aishell/ASR/pruned_transducer_stateless7_bbpe/train.py b/egs/aishell/ASR/pruned_transducer_stateless7_bbpe/train.py index be48d6dde..811269989 100755 --- a/egs/aishell/ASR/pruned_transducer_stateless7_bbpe/train.py +++ b/egs/aishell/ASR/pruned_transducer_stateless7_bbpe/train.py @@ -64,7 +64,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, ScaledAdam from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer @@ -802,7 +802,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1102,7 +1102,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1202,7 +1202,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/aishell/ASR/pruned_transducer_stateless7_streaming/do_not_use_it_directly.py b/egs/aishell/ASR/pruned_transducer_stateless7_streaming/do_not_use_it_directly.py index e3387e670..6653d9d9c 100755 --- a/egs/aishell/ASR/pruned_transducer_stateless7_streaming/do_not_use_it_directly.py +++ b/egs/aishell/ASR/pruned_transducer_stateless7_streaming/do_not_use_it_directly.py @@ -63,7 +63,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, ScaledAdam from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer_for_ncnn_export_only import Zipformer @@ -813,7 +813,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1105,7 +1105,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1205,7 +1205,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/aishell/ASR/pruned_transducer_stateless7_streaming/train.py b/egs/aishell/ASR/pruned_transducer_stateless7_streaming/train.py index cba312214..f3b0f1e11 100755 --- a/egs/aishell/ASR/pruned_transducer_stateless7_streaming/train.py +++ b/egs/aishell/ASR/pruned_transducer_stateless7_streaming/train.py @@ -63,7 +63,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, ScaledAdam from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer @@ -812,7 +812,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1104,7 +1104,7 @@ def run(rank, world_size, args): # params=params, # ) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1202,7 +1202,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/aishell/ASR/whisper/train.py b/egs/aishell/ASR/whisper/train.py index e84dcf156..d77f8c270 100755 --- a/egs/aishell/ASR/whisper/train.py +++ b/egs/aishell/ASR/whisper/train.py @@ -62,7 +62,7 @@ from lhotse.dataset.sampling.base import CutSampler from lhotse.utils import fix_random_seed from optim import Eden, ScaledAdam from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.functional import pad as pad_tensor from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -514,7 +514,7 @@ def compute_validation_loss( tot_loss = MetricsTracker() for batch_idx, batch in enumerate(valid_dl): - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, tokenizer=tokenizer, @@ -608,7 +608,7 @@ def train_one_epoch( ) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, tokenizer=tokenizer, @@ -812,7 +812,7 @@ def run(rank, world_size, args): train_dl = aishell.train_dataloaders(aishell.train_cuts()) valid_dl = aishell.valid_dataloaders(aishell.valid_cuts()) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) diff --git a/egs/aishell/ASR/zipformer/train.py b/egs/aishell/ASR/zipformer/train.py index ab568b20f..cd253c597 100755 --- a/egs/aishell/ASR/zipformer/train.py +++ b/egs/aishell/ASR/zipformer/train.py @@ -71,7 +71,7 @@ from optim import Eden, ScaledAdam from scaling import ScheduledFloat from subsampling import Conv2dSubsampling from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer2 @@ -910,7 +910,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1201,7 +1201,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1302,7 +1302,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/aishell/ASR/zipformer/train_bbpe.py b/egs/aishell/ASR/zipformer/train_bbpe.py index 2dac0cc64..46a5506db 100755 --- a/egs/aishell/ASR/zipformer/train_bbpe.py +++ b/egs/aishell/ASR/zipformer/train_bbpe.py @@ -61,7 +61,7 @@ from lhotse.cut import Cut from lhotse.utils import fix_random_seed from optim import Eden, ScaledAdam from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from train import ( @@ -495,7 +495,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -795,7 +795,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -895,7 +895,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/aishell2/ASR/pruned_transducer_stateless5/train.py b/egs/aishell2/ASR/pruned_transducer_stateless5/train.py index 772d9e6bf..8c7448d4c 100755 --- a/egs/aishell2/ASR/pruned_transducer_stateless5/train.py +++ b/egs/aishell2/ASR/pruned_transducer_stateless5/train.py @@ -75,7 +75,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, Eve from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -734,7 +734,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -963,7 +963,7 @@ def run(rank, world_size, args): warmup=0.0 if params.start_epoch == 1 else 1.0, ) - scaler = GradScaler("cuda", enabled=params.use_fp16) + scaler = GradScaler(enabled=params.use_fp16) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1062,7 +1062,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/aishell4/ASR/pruned_transducer_stateless5/train.py b/egs/aishell4/ASR/pruned_transducer_stateless5/train.py index 0eb9271f5..a354f761e 100755 --- a/egs/aishell4/ASR/pruned_transducer_stateless5/train.py +++ b/egs/aishell4/ASR/pruned_transducer_stateless5/train.py @@ -68,7 +68,7 @@ from local.text_normalize import text_normalize from model import Transducer from optim import Eden, Eve from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -727,7 +727,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) # print(batch["supervisions"]) - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -963,7 +963,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16) + scaler = GradScaler(enabled=params.use_fp16) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1034,7 +1034,7 @@ def scan_pessimistic_batches_for_oom( # warmup = 0.0 is so that the derivs for the pruned loss stay zero # (i.e. are not remembered by the decaying-average in adam), because # we want to avoid these params being subject to shrinkage in adam. - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/alimeeting/ASR/pruned_transducer_stateless2/train.py b/egs/alimeeting/ASR/pruned_transducer_stateless2/train.py index 2b1b6f9b4..30154291d 100644 --- a/egs/alimeeting/ASR/pruned_transducer_stateless2/train.py +++ b/egs/alimeeting/ASR/pruned_transducer_stateless2/train.py @@ -67,7 +67,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, Eve from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -638,7 +638,7 @@ def train_one_epoch( params.batch_idx_train += 1 batch_size = len(batch["supervisions"]["text"]) - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -843,7 +843,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16) + scaler = GradScaler(enabled=params.use_fp16) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -912,7 +912,7 @@ def scan_pessimistic_batches_for_oom( # warmup = 0.0 is so that the derivs for the pruned loss stay zero # (i.e. are not remembered by the decaying-average in adam), because # we want to avoid these params being subject to shrinkage in adam. - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/alimeeting/ASR_v2/pruned_transducer_stateless7/train.py b/egs/alimeeting/ASR_v2/pruned_transducer_stateless7/train.py index e321deeb1..30879d8d2 100755 --- a/egs/alimeeting/ASR_v2/pruned_transducer_stateless7/train.py +++ b/egs/alimeeting/ASR_v2/pruned_transducer_stateless7/train.py @@ -55,7 +55,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, ScaledAdam from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer @@ -782,7 +782,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1031,7 +1031,7 @@ def run(rank, world_size, args): # params=params, # ) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1127,7 +1127,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/ami/ASR/pruned_transducer_stateless7/train.py b/egs/ami/ASR/pruned_transducer_stateless7/train.py index 97ebc5bcf..d62cdadb7 100755 --- a/egs/ami/ASR/pruned_transducer_stateless7/train.py +++ b/egs/ami/ASR/pruned_transducer_stateless7/train.py @@ -55,7 +55,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, ScaledAdam from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer @@ -773,7 +773,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1034,7 +1034,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1134,7 +1134,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/ami/SURT/dprnn_zipformer/train.py b/egs/ami/SURT/dprnn_zipformer/train.py index 9e77c0527..adc6a8495 100755 --- a/egs/ami/SURT/dprnn_zipformer/train.py +++ b/egs/ami/SURT/dprnn_zipformer/train.py @@ -61,7 +61,7 @@ from model import SURT from optim import Eden, ScaledAdam from scaling import ScaledLinear, ScaledLSTM from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer @@ -1067,7 +1067,7 @@ def train_one_epoch( batch_size = batch["inputs"].shape[0] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1314,7 +1314,7 @@ def run(rank, world_size, args): ) valid_dl = ami.valid_dataloaders(dev_cuts) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) diff --git a/egs/ami/SURT/dprnn_zipformer/train_adapt.py b/egs/ami/SURT/dprnn_zipformer/train_adapt.py index 0647a7c78..ac5b0dadc 100755 --- a/egs/ami/SURT/dprnn_zipformer/train_adapt.py +++ b/egs/ami/SURT/dprnn_zipformer/train_adapt.py @@ -61,7 +61,7 @@ from model import SURT from optim import Eden, ScaledAdam from scaling import ScaledLinear, ScaledLSTM from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer @@ -1058,7 +1058,7 @@ def train_one_epoch( batch_size = batch["inputs"].shape[0] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1305,7 +1305,7 @@ def run(rank, world_size, args): ) valid_dl = ami.valid_dataloaders(dev_cuts) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) diff --git a/egs/audioset/AT/zipformer/train.py b/egs/audioset/AT/zipformer/train.py index 9532ed906..67c703364 100644 --- a/egs/audioset/AT/zipformer/train.py +++ b/egs/audioset/AT/zipformer/train.py @@ -53,7 +53,7 @@ from optim import Eden, ScaledAdam from scaling import ScheduledFloat from subsampling import Conv2dSubsampling from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer2 @@ -799,7 +799,7 @@ def train_one_epoch( num_samples += batch_size try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1057,7 +1057,7 @@ def run(rank, world_size, args): valid_cuts = audioset.audioset_eval_cuts() valid_dl = audioset.valid_dataloaders(valid_cuts) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1148,7 +1148,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/commonvoice/ASR/pruned_transducer_stateless7/train.py b/egs/commonvoice/ASR/pruned_transducer_stateless7/train.py index 486ab73df..5e98084ec 100755 --- a/egs/commonvoice/ASR/pruned_transducer_stateless7/train.py +++ b/egs/commonvoice/ASR/pruned_transducer_stateless7/train.py @@ -66,7 +66,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, ScaledAdam from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer @@ -825,7 +825,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1120,7 +1120,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1220,7 +1220,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/commonvoice/ASR/pruned_transducer_stateless7_streaming/do_not_use_it_directly.py b/egs/commonvoice/ASR/pruned_transducer_stateless7_streaming/do_not_use_it_directly.py index fa241abe7..aefe88f3f 100755 --- a/egs/commonvoice/ASR/pruned_transducer_stateless7_streaming/do_not_use_it_directly.py +++ b/egs/commonvoice/ASR/pruned_transducer_stateless7_streaming/do_not_use_it_directly.py @@ -65,7 +65,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, ScaledAdam from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer_for_ncnn_export_only import Zipformer @@ -818,7 +818,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1109,7 +1109,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1209,7 +1209,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/commonvoice/ASR/pruned_transducer_stateless7_streaming/finetune.py b/egs/commonvoice/ASR/pruned_transducer_stateless7_streaming/finetune.py index 8905dc617..976004eca 100755 --- a/egs/commonvoice/ASR/pruned_transducer_stateless7_streaming/finetune.py +++ b/egs/commonvoice/ASR/pruned_transducer_stateless7_streaming/finetune.py @@ -68,7 +68,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, ScaledAdam from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer @@ -895,7 +895,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1193,7 +1193,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1293,7 +1293,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/commonvoice/ASR/pruned_transducer_stateless7_streaming/train.py b/egs/commonvoice/ASR/pruned_transducer_stateless7_streaming/train.py index 8260c4985..67e1a8133 100755 --- a/egs/commonvoice/ASR/pruned_transducer_stateless7_streaming/train.py +++ b/egs/commonvoice/ASR/pruned_transducer_stateless7_streaming/train.py @@ -65,7 +65,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, ScaledAdam from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer @@ -840,7 +840,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1137,7 +1137,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1237,7 +1237,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/commonvoice/ASR/zipformer/train.py b/egs/commonvoice/ASR/zipformer/train.py index c0219df19..271014db0 100755 --- a/egs/commonvoice/ASR/zipformer/train.py +++ b/egs/commonvoice/ASR/zipformer/train.py @@ -75,7 +75,7 @@ from optim import Eden, ScaledAdam from scaling import ScheduledFloat from subsampling import Conv2dSubsampling from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer2 @@ -969,7 +969,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1265,7 +1265,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1365,7 +1365,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/commonvoice/ASR/zipformer/train_char.py b/egs/commonvoice/ASR/zipformer/train_char.py index 639e1067a..0aa7856cc 100755 --- a/egs/commonvoice/ASR/zipformer/train_char.py +++ b/egs/commonvoice/ASR/zipformer/train_char.py @@ -67,7 +67,7 @@ from lhotse.cut import Cut from lhotse.utils import fix_random_seed from optim import Eden, ScaledAdam from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from train import ( @@ -604,7 +604,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -784,7 +784,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, @@ -979,7 +979,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) diff --git a/egs/csj/ASR/pruned_transducer_stateless7_streaming/do_not_use_it_directly.py b/egs/csj/ASR/pruned_transducer_stateless7_streaming/do_not_use_it_directly.py index 661bfa6ca..6d256308c 100755 --- a/egs/csj/ASR/pruned_transducer_stateless7_streaming/do_not_use_it_directly.py +++ b/egs/csj/ASR/pruned_transducer_stateless7_streaming/do_not_use_it_directly.py @@ -67,7 +67,7 @@ from model import Transducer from optim import Eden, ScaledAdam from tokenizer import Tokenizer from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer_for_ncnn_export_only import Zipformer @@ -839,7 +839,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1146,7 +1146,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1246,7 +1246,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/csj/ASR/pruned_transducer_stateless7_streaming/train.py b/egs/csj/ASR/pruned_transducer_stateless7_streaming/train.py index 8f07fc42f..ef7ea9013 100755 --- a/egs/csj/ASR/pruned_transducer_stateless7_streaming/train.py +++ b/egs/csj/ASR/pruned_transducer_stateless7_streaming/train.py @@ -67,7 +67,7 @@ from model import Transducer from optim import Eden, ScaledAdam from tokenizer import Tokenizer from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer @@ -838,7 +838,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1145,7 +1145,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1245,7 +1245,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/gigaspeech/ASR/pruned_transducer_stateless2/train.py b/egs/gigaspeech/ASR/pruned_transducer_stateless2/train.py index e0e11fc70..a7772b62f 100755 --- a/egs/gigaspeech/ASR/pruned_transducer_stateless2/train.py +++ b/egs/gigaspeech/ASR/pruned_transducer_stateless2/train.py @@ -64,7 +64,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, Eve from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -675,7 +675,7 @@ def train_one_epoch( params.batch_idx_train += 1 batch_size = len(batch["supervisions"]["text"]) - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -873,7 +873,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16) + scaler = GradScaler(enabled=params.use_fp16) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -944,7 +944,7 @@ def scan_pessimistic_batches_for_oom( # warmup = 0.0 is so that the derivs for the pruned loss stay zero # (i.e. are not remembered by the decaying-average in adam), because # we want to avoid these params being subject to shrinkage in adam. - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/gigaspeech/ASR/zipformer/train.py b/egs/gigaspeech/ASR/zipformer/train.py index 5092ef8cb..4c122effe 100755 --- a/egs/gigaspeech/ASR/zipformer/train.py +++ b/egs/gigaspeech/ASR/zipformer/train.py @@ -75,7 +75,7 @@ from optim import Eden, ScaledAdam from scaling import ScheduledFloat from subsampling import Conv2dSubsampling from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer2 @@ -958,7 +958,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1217,7 +1217,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1317,7 +1317,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/gigaspeech/KWS/zipformer/finetune.py b/egs/gigaspeech/KWS/zipformer/finetune.py index 49e8aef1a..a7ba56127 100755 --- a/egs/gigaspeech/KWS/zipformer/finetune.py +++ b/egs/gigaspeech/KWS/zipformer/finetune.py @@ -73,7 +73,7 @@ from lhotse.utils import fix_random_seed from model import AsrModel from optim import Eden, ScaledAdam from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from train import ( @@ -291,7 +291,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -570,7 +570,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) diff --git a/egs/gigaspeech/KWS/zipformer/train.py b/egs/gigaspeech/KWS/zipformer/train.py index f2283cb03..39d8fc6cd 100755 --- a/egs/gigaspeech/KWS/zipformer/train.py +++ b/egs/gigaspeech/KWS/zipformer/train.py @@ -75,7 +75,7 @@ from optim import Eden, ScaledAdam from scaling import ScheduledFloat from subsampling import Conv2dSubsampling from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer2 @@ -961,7 +961,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1220,7 +1220,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1320,7 +1320,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/ksponspeech/ASR/pruned_transducer_stateless7_streaming/train.py b/egs/ksponspeech/ASR/pruned_transducer_stateless7_streaming/train.py index 30d9f0e51..bf50bf5ea 100755 --- a/egs/ksponspeech/ASR/pruned_transducer_stateless7_streaming/train.py +++ b/egs/ksponspeech/ASR/pruned_transducer_stateless7_streaming/train.py @@ -61,7 +61,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, ScaledAdam from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer @@ -805,7 +805,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1096,7 +1096,7 @@ def run(rank, world_size, args): # params=params, # ) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1196,7 +1196,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/ksponspeech/ASR/zipformer/train.py b/egs/ksponspeech/ASR/zipformer/train.py index 5f6ee7cca..485ea69c9 100755 --- a/egs/ksponspeech/ASR/zipformer/train.py +++ b/egs/ksponspeech/ASR/zipformer/train.py @@ -70,7 +70,7 @@ from optim import Eden, ScaledAdam from scaling import ScheduledFloat from subsampling import Conv2dSubsampling from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer2 @@ -942,7 +942,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1233,7 +1233,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1333,7 +1333,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/libricss/SURT/dprnn_zipformer/model.py b/egs/libricss/SURT/dprnn_zipformer/model.py index 0e88357d1..688e1e78d 100644 --- a/egs/libricss/SURT/dprnn_zipformer/model.py +++ b/egs/libricss/SURT/dprnn_zipformer/model.py @@ -140,7 +140,7 @@ class SURT(nn.Module): lm = self.simple_lm_proj(decoder_out) am = self.simple_am_proj(encoder_out) - with torch.amp.autocast("cuda", enabled=False): + with torch.cuda.amp.autocast(enabled=False): simple_loss, (px_grad, py_grad) = k2.rnnt_loss_smoothed( lm=lm.float(), am=am.float(), @@ -175,7 +175,7 @@ class SURT(nn.Module): # prior to do_rnnt_pruning (this is an optimization for speed). logits = self.joiner(am_pruned, lm_pruned, project_input=False) - with torch.amp.autocast("cuda", enabled=False): + with torch.cuda.amp.autocast(enabled=False): pruned_loss = k2.rnnt_loss_pruned( logits=logits.float(), symbols=y_padded, diff --git a/egs/libricss/SURT/dprnn_zipformer/scaling.py b/egs/libricss/SURT/dprnn_zipformer/scaling.py index d46cb224e..4040a7b89 100644 --- a/egs/libricss/SURT/dprnn_zipformer/scaling.py +++ b/egs/libricss/SURT/dprnn_zipformer/scaling.py @@ -287,7 +287,7 @@ class SoftmaxFunction(torch.autograd.Function): @staticmethod def backward(ctx, ans_grad: Tensor): (ans,) = ctx.saved_tensors - with torch.amp.autocast("cuda", enabled=False): + with torch.cuda.amp.autocast(enabled=False): ans_grad = ans_grad.to(torch.float32) ans = ans.to(torch.float32) x_grad = ans_grad * ans @@ -1065,7 +1065,7 @@ class WhiteningPenaltyFunction(torch.autograd.Function): def backward(ctx, x_grad: Tensor): (x_orig,) = ctx.saved_tensors with torch.enable_grad(): - with torch.amp.autocast("cuda", enabled=False): + with torch.cuda.amp.autocast(enabled=False): x_detached = x_orig.to(torch.float32).detach() x_detached.requires_grad = True @@ -1263,7 +1263,7 @@ class MaxEig(torch.nn.Module): ): return _no_op(x) - with torch.amp.autocast("cuda", enabled=False): + with torch.cuda.amp.autocast(enabled=False): eps = 1.0e-20 orig_x = x x = x.to(torch.float32) diff --git a/egs/libricss/SURT/dprnn_zipformer/train.py b/egs/libricss/SURT/dprnn_zipformer/train.py index 33ea7c5a6..148cafd4b 100755 --- a/egs/libricss/SURT/dprnn_zipformer/train.py +++ b/egs/libricss/SURT/dprnn_zipformer/train.py @@ -69,7 +69,7 @@ from model import SURT from optim import Eden, ScaledAdam from scaling import ScaledLSTM from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer @@ -1096,7 +1096,7 @@ def train_one_epoch( batch_size = batch["inputs"].shape[0] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1343,7 +1343,7 @@ def run(rank, world_size, args): train_dl_ov40 = libricss.train_dataloaders(train_cuts_ov40) valid_dl = libricss.valid_dataloaders(dev_cuts) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) diff --git a/egs/libricss/SURT/dprnn_zipformer/train_adapt.py b/egs/libricss/SURT/dprnn_zipformer/train_adapt.py index 82b61baa0..8c37430ec 100755 --- a/egs/libricss/SURT/dprnn_zipformer/train_adapt.py +++ b/egs/libricss/SURT/dprnn_zipformer/train_adapt.py @@ -67,7 +67,7 @@ from model import SURT from optim import Eden, ScaledAdam from scaling import ScaledLinear, ScaledLSTM from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer @@ -985,7 +985,7 @@ def train_one_epoch( batch_size = batch["inputs"].shape[0] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1237,7 +1237,7 @@ def run(rank, world_size, args): ) valid_dl = libricss.valid_dataloaders(dev_cuts) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) diff --git a/egs/libriheavy/ASR/zipformer/train.py b/egs/libriheavy/ASR/zipformer/train.py index 524273ec5..357e8a827 100644 --- a/egs/libriheavy/ASR/zipformer/train.py +++ b/egs/libriheavy/ASR/zipformer/train.py @@ -78,7 +78,7 @@ from scaling import ScheduledFloat from subsampling import Conv2dSubsampling from text_normalization import remove_punc_to_upper from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer2 @@ -958,7 +958,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1268,7 +1268,7 @@ def run(rank, world_size, args): # params=params, # ) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1367,7 +1367,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/libriheavy/ASR/zipformer_prompt_asr/model_baseline.py b/egs/libriheavy/ASR/zipformer_prompt_asr/model_baseline.py index 66328bb89..77b4057c4 100644 --- a/egs/libriheavy/ASR/zipformer_prompt_asr/model_baseline.py +++ b/egs/libriheavy/ASR/zipformer_prompt_asr/model_baseline.py @@ -186,7 +186,7 @@ class Transducer(nn.Module): lm = self.simple_lm_proj(decoder_out) am = self.simple_am_proj(encoder_out) - with torch.amp.autocast("cuda", enabled=False): + with torch.cuda.amp.autocast(enabled=False): simple_loss, (px_grad, py_grad) = k2.rnnt_loss_smoothed( lm=lm.float(), am=am.float(), @@ -221,7 +221,7 @@ class Transducer(nn.Module): # prior to do_rnnt_pruning (this is an optimization for speed). logits = self.joiner(am_pruned, lm_pruned, project_input=False) - with torch.amp.autocast("cuda", enabled=False): + with torch.cuda.amp.autocast(enabled=False): pruned_loss = k2.rnnt_loss_pruned( logits=logits.float(), symbols=y_padded, diff --git a/egs/libriheavy/ASR/zipformer_prompt_asr/model_with_BERT.py b/egs/libriheavy/ASR/zipformer_prompt_asr/model_with_BERT.py index 80fbf09f0..21c7b4fac 100644 --- a/egs/libriheavy/ASR/zipformer_prompt_asr/model_with_BERT.py +++ b/egs/libriheavy/ASR/zipformer_prompt_asr/model_with_BERT.py @@ -245,7 +245,7 @@ class PromptedTransducer(nn.Module): lm = self.simple_lm_proj(decoder_out) am = self.simple_am_proj(encoder_out) - with torch.amp.autocast("cuda", enabled=False): + with torch.cuda.amp.autocast(enabled=False): simple_loss, (px_grad, py_grad) = k2.rnnt_loss_smoothed( lm=lm.float(), am=am.float(), @@ -287,7 +287,7 @@ class PromptedTransducer(nn.Module): logits = self.joiner(am_pruned, lm_pruned, context=context, project_input=False) - with torch.amp.autocast("cuda", enabled=False): + with torch.cuda.amp.autocast(enabled=False): pruned_loss = k2.rnnt_loss_pruned( logits=logits.float(), symbols=y_padded, diff --git a/egs/libriheavy/ASR/zipformer_prompt_asr/scaling.py b/egs/libriheavy/ASR/zipformer_prompt_asr/scaling.py index a260d828e..0e6764ba0 100644 --- a/egs/libriheavy/ASR/zipformer_prompt_asr/scaling.py +++ b/egs/libriheavy/ASR/zipformer_prompt_asr/scaling.py @@ -271,7 +271,7 @@ class SoftmaxFunction(torch.autograd.Function): @staticmethod def backward(ctx, ans_grad: Tensor): (ans,) = ctx.saved_tensors - with torch.amp.autocast("cuda", enabled=False): + with torch.cuda.amp.autocast(enabled=False): ans_grad = ans_grad.to(torch.float32) ans = ans.to(torch.float32) x_grad = ans_grad * ans @@ -685,7 +685,7 @@ class BalancerFunction(torch.autograd.Function): try: with torch.enable_grad(): - with torch.amp.autocast("cuda", enabled=False): + with torch.cuda.amp.autocast(enabled=False): x = x.to(torch.float32) x = x.detach() x.requires_grad = True @@ -940,7 +940,7 @@ class WhiteningPenaltyFunction(torch.autograd.Function): try: with torch.enable_grad(): - with torch.amp.autocast("cuda", enabled=False): + with torch.cuda.amp.autocast(enabled=False): x_detached = x_orig.to(torch.float32).detach() x_detached.requires_grad = True @@ -1280,7 +1280,7 @@ class SwooshLFunction(torch.autograd.Function): coeff = -0.08 - with torch.amp.autocast("cuda", enabled=False): + with torch.cuda.amp.autocast(enabled=False): with torch.enable_grad(): x = x.detach() x.requires_grad = True @@ -1351,7 +1351,7 @@ class SwooshRFunction(torch.autograd.Function): zero = torch.tensor(0.0, dtype=x.dtype, device=x.device) - with torch.amp.autocast("cuda", enabled=False): + with torch.cuda.amp.autocast(enabled=False): with torch.enable_grad(): x = x.detach() x.requires_grad = True diff --git a/egs/libriheavy/ASR/zipformer_prompt_asr/train_baseline.py b/egs/libriheavy/ASR/zipformer_prompt_asr/train_baseline.py index bfca5a0db..93f7e1248 100644 --- a/egs/libriheavy/ASR/zipformer_prompt_asr/train_baseline.py +++ b/egs/libriheavy/ASR/zipformer_prompt_asr/train_baseline.py @@ -89,7 +89,7 @@ from scaling import ScheduledFloat from subsampling import Conv2dSubsampling from text_normalization import train_text_normalization, upper_only_alpha from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer2 @@ -975,7 +975,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1271,7 +1271,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1371,7 +1371,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/libriheavy/ASR/zipformer_prompt_asr/train_bert_encoder.py b/egs/libriheavy/ASR/zipformer_prompt_asr/train_bert_encoder.py index 36c6d6464..2a2c206aa 100755 --- a/egs/libriheavy/ASR/zipformer_prompt_asr/train_bert_encoder.py +++ b/egs/libriheavy/ASR/zipformer_prompt_asr/train_bert_encoder.py @@ -103,7 +103,7 @@ from text_normalization import ( upper_only_alpha, ) from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer2 @@ -1321,7 +1321,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1647,7 +1647,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1749,7 +1749,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/libriheavy/ASR/zipformer_prompt_asr/zipformer.py b/egs/libriheavy/ASR/zipformer_prompt_asr/zipformer.py index 405c95acc..d1cf90ffb 100644 --- a/egs/libriheavy/ASR/zipformer_prompt_asr/zipformer.py +++ b/egs/libriheavy/ASR/zipformer_prompt_asr/zipformer.py @@ -1561,7 +1561,7 @@ class RelPositionMultiheadAttentionWeights(nn.Module): (num_heads, batch_size, seq_len, seq_len) = attn_weights.shape with torch.no_grad(): - with torch.amp.autocast("cuda", enabled=False): + with torch.cuda.amp.autocast(enabled=False): attn_weights = attn_weights.to(torch.float32) attn_weights_entropy = ( -((attn_weights + 1.0e-20).log() * attn_weights) @@ -1844,7 +1844,7 @@ class MultiheadAttentionWeights(nn.Module): (num_heads, batch_size, seq_len, seq_len) = attn_weights.shape with torch.no_grad(): - with torch.amp.autocast("cuda", enabled=False): + with torch.cuda.amp.autocast(enabled=False): attn_weights = attn_weights.to(torch.float32) attn_weights_entropy = ( -((attn_weights + 1.0e-20).log() * attn_weights) diff --git a/egs/librilight/SSL/zipformer/finetune.py b/egs/librilight/SSL/zipformer/finetune.py index 568096c6a..50dbd5f2d 100644 --- a/egs/librilight/SSL/zipformer/finetune.py +++ b/egs/librilight/SSL/zipformer/finetune.py @@ -67,7 +67,7 @@ from lhotse.utils import fix_random_seed from model import AsrModel from optim import Eden, ScaledAdam from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -1116,7 +1116,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1407,7 +1407,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1505,7 +1505,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librilight/SSL/zipformer/pretrain.py b/egs/librilight/SSL/zipformer/pretrain.py index 019f77ea3..5728dbe75 100644 --- a/egs/librilight/SSL/zipformer/pretrain.py +++ b/egs/librilight/SSL/zipformer/pretrain.py @@ -57,7 +57,7 @@ from lhotse.utils import fix_random_seed from optim import Eden, ScaledAdam from ssl_datamodule import LibriLightDataModule from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -936,7 +936,7 @@ def train_one_epoch( batch_size = batch["kmeans"].shape[0] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1229,7 +1229,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1320,7 +1320,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/ASR/conformer_ctc2/train.py b/egs/librispeech/ASR/conformer_ctc2/train.py index b0b5da1c0..c4a13b101 100755 --- a/egs/librispeech/ASR/conformer_ctc2/train.py +++ b/egs/librispeech/ASR/conformer_ctc2/train.py @@ -65,7 +65,7 @@ from lhotse.dataset.sampling.base import CutSampler from lhotse.utils import fix_random_seed from optim import Eden, Eve from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -676,7 +676,7 @@ def train_one_epoch( params.batch_idx_train += 1 batch_size = len(batch["supervisions"]["text"]) - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -965,7 +965,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16) + scaler = GradScaler(enabled=params.use_fp16) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1036,7 +1036,7 @@ def scan_pessimistic_batches_for_oom( # warmup = 0.0 is so that the derivs for the pruned loss stay zero # (i.e. are not remembered by the decaying-average in adam), because # we want to avoid these params being subject to shrinkage in adam. - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/ASR/conformer_ctc3/train.py b/egs/librispeech/ASR/conformer_ctc3/train.py index 7e819a2d8..a2f1125ca 100755 --- a/egs/librispeech/ASR/conformer_ctc3/train.py +++ b/egs/librispeech/ASR/conformer_ctc3/train.py @@ -76,7 +76,7 @@ from lhotse.utils import fix_random_seed from model import CTCModel from optim import Eden, Eve from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -743,7 +743,7 @@ def train_one_epoch( params.batch_idx_train += 1 batch_size = len(batch["supervisions"]["text"]) - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1004,7 +1004,7 @@ def run(rank, world_size, args): warmup=0.0 if params.start_epoch == 1 else 1.0, ) - scaler = GradScaler("cuda", enabled=params.use_fp16) + scaler = GradScaler(enabled=params.use_fp16) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1073,7 +1073,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/ASR/conv_emformer_transducer_stateless/train.py b/egs/librispeech/ASR/conv_emformer_transducer_stateless/train.py index 130a7c97f..ca21bd6bf 100755 --- a/egs/librispeech/ASR/conv_emformer_transducer_stateless/train.py +++ b/egs/librispeech/ASR/conv_emformer_transducer_stateless/train.py @@ -80,7 +80,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, Eve from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -772,7 +772,7 @@ def train_one_epoch( params.batch_idx_train += 1 batch_size = len(batch["supervisions"]["text"]) - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1002,7 +1002,7 @@ def run(rank, world_size, args): warmup=0.0 if params.start_epoch == 1 else 1.0, ) - scaler = GradScaler("cuda", enabled=params.use_fp16) + scaler = GradScaler(enabled=params.use_fp16) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1071,7 +1071,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/ASR/conv_emformer_transducer_stateless2/do_not_use_it_directly.py b/egs/librispeech/ASR/conv_emformer_transducer_stateless2/do_not_use_it_directly.py index 16ae4e4e2..d614f0914 100755 --- a/egs/librispeech/ASR/conv_emformer_transducer_stateless2/do_not_use_it_directly.py +++ b/egs/librispeech/ASR/conv_emformer_transducer_stateless2/do_not_use_it_directly.py @@ -80,7 +80,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, Eve from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -774,7 +774,7 @@ def train_one_epoch( params.batch_idx_train += 1 batch_size = len(batch["supervisions"]["text"]) - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1003,7 +1003,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16) + scaler = GradScaler(enabled=params.use_fp16) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1074,7 +1074,7 @@ def scan_pessimistic_batches_for_oom( # warmup = 0.0 is so that the derivs for the pruned loss stay zero # (i.e. are not remembered by the decaying-average in adam), because # we want to avoid these params being subject to shrinkage in adam. - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/ASR/conv_emformer_transducer_stateless2/train.py b/egs/librispeech/ASR/conv_emformer_transducer_stateless2/train.py index 28d094a76..23ddb6bec 100755 --- a/egs/librispeech/ASR/conv_emformer_transducer_stateless2/train.py +++ b/egs/librispeech/ASR/conv_emformer_transducer_stateless2/train.py @@ -80,7 +80,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, Eve from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -772,7 +772,7 @@ def train_one_epoch( params.batch_idx_train += 1 batch_size = len(batch["supervisions"]["text"]) - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1001,7 +1001,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16) + scaler = GradScaler(enabled=params.use_fp16) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1072,7 +1072,7 @@ def scan_pessimistic_batches_for_oom( # warmup = 0.0 is so that the derivs for the pruned loss stay zero # (i.e. are not remembered by the decaying-average in adam), because # we want to avoid these params being subject to shrinkage in adam. - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/ASR/lstm_transducer_stateless/model.py b/egs/librispeech/ASR/lstm_transducer_stateless/model.py index 1ec9a8fc6..e7bad7ed8 100644 --- a/egs/librispeech/ASR/lstm_transducer_stateless/model.py +++ b/egs/librispeech/ASR/lstm_transducer_stateless/model.py @@ -156,7 +156,7 @@ class Transducer(nn.Module): lm = self.simple_lm_proj(decoder_out) am = self.simple_am_proj(encoder_out) - with torch.amp.autocast("cuda", enabled=False): + with torch.cuda.amp.autocast(enabled=False): simple_loss, (px_grad, py_grad) = k2.rnnt_loss_smoothed( lm=lm.float(), am=am.float(), @@ -192,7 +192,7 @@ class Transducer(nn.Module): # prior to do_rnnt_pruning (this is an optimization for speed). logits = self.joiner(am_pruned, lm_pruned, project_input=False) - with torch.amp.autocast("cuda", enabled=False): + with torch.cuda.amp.autocast(enabled=False): pruned_loss = k2.rnnt_loss_pruned( logits=logits.float(), symbols=y_padded, diff --git a/egs/librispeech/ASR/lstm_transducer_stateless/train.py b/egs/librispeech/ASR/lstm_transducer_stateless/train.py index 1e50ce090..feb81d500 100755 --- a/egs/librispeech/ASR/lstm_transducer_stateless/train.py +++ b/egs/librispeech/ASR/lstm_transducer_stateless/train.py @@ -66,7 +66,7 @@ from lstm import RNN from model import Transducer from optim import Eden, Eve from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -763,7 +763,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1023,7 +1023,7 @@ def run(rank, world_size, args): warmup=0.0 if params.start_epoch == 1 else 1.0, ) - scaler = GradScaler("cuda", enabled=params.use_fp16) + scaler = GradScaler(enabled=params.use_fp16) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1092,7 +1092,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/ASR/lstm_transducer_stateless2/model.py b/egs/librispeech/ASR/lstm_transducer_stateless2/model.py index a758c550d..4957d14b1 100644 --- a/egs/librispeech/ASR/lstm_transducer_stateless2/model.py +++ b/egs/librispeech/ASR/lstm_transducer_stateless2/model.py @@ -195,7 +195,7 @@ class Transducer(nn.Module): lm = simple_lm_proj(decoder_out) am = simple_am_proj(encoder_out) - with torch.amp.autocast("cuda", enabled=False): + with torch.cuda.amp.autocast(enabled=False): simple_loss, (px_grad, py_grad) = k2.rnnt_loss_smoothed( lm=lm.float(), am=am.float(), @@ -231,7 +231,7 @@ class Transducer(nn.Module): # prior to do_rnnt_pruning (this is an optimization for speed). logits = joiner(am_pruned, lm_pruned, project_input=False) - with torch.amp.autocast("cuda", enabled=False): + with torch.cuda.amp.autocast(enabled=False): pruned_loss = k2.rnnt_loss_pruned( logits=logits.float(), symbols=y_padded, diff --git a/egs/librispeech/ASR/lstm_transducer_stateless2/train.py b/egs/librispeech/ASR/lstm_transducer_stateless2/train.py index 4d4f3e132..4fc4fa7f8 100755 --- a/egs/librispeech/ASR/lstm_transducer_stateless2/train.py +++ b/egs/librispeech/ASR/lstm_transducer_stateless2/train.py @@ -74,7 +74,7 @@ from lstm import RNN from model import Transducer from optim import Eden, Eve from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -848,7 +848,7 @@ def train_one_epoch( libri = is_libri(batch["supervisions"]["cut"][0]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1176,7 +1176,7 @@ def run(rank, world_size, args): else: logging.info("Skip scan_pessimistic_batches_for_oom") - scaler = GradScaler("cuda", enabled=params.use_fp16) + scaler = GradScaler(enabled=params.use_fp16) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1247,7 +1247,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/ASR/lstm_transducer_stateless3/train.py b/egs/librispeech/ASR/lstm_transducer_stateless3/train.py index ae4cd1c6a..2c1cef3a3 100755 --- a/egs/librispeech/ASR/lstm_transducer_stateless3/train.py +++ b/egs/librispeech/ASR/lstm_transducer_stateless3/train.py @@ -66,7 +66,7 @@ from lstm import RNN from model import Transducer from optim import Eden, Eve from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -793,7 +793,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1067,7 +1067,7 @@ def run(rank, world_size, args): warmup=0.0 if params.start_epoch == 1 else 1.0, ) - scaler = GradScaler("cuda", enabled=params.use_fp16) + scaler = GradScaler(enabled=params.use_fp16) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1136,7 +1136,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/ASR/pruned2_knowledge/model.py b/egs/librispeech/ASR/pruned2_knowledge/model.py index 2ffea06e7..ca8c28af1 100644 --- a/egs/librispeech/ASR/pruned2_knowledge/model.py +++ b/egs/librispeech/ASR/pruned2_knowledge/model.py @@ -141,7 +141,7 @@ class Transducer(nn.Module): lm = self.simple_lm_proj(decoder_out) am = self.simple_am_proj(encoder_out) - with torch.amp.autocast("cuda", enabled=False): + with torch.cuda.amp.autocast(enabled=False): simple_loss, (px_grad, py_grad) = k2.rnnt_loss_smoothed( lm=lm.float(), am=am.float(), @@ -176,7 +176,7 @@ class Transducer(nn.Module): # prior to do_rnnt_pruning (this is an optimization for speed). logits = self.joiner(am_pruned, lm_pruned, project_input=False) - with torch.amp.autocast("cuda", enabled=False): + with torch.cuda.amp.autocast(enabled=False): pruned_loss = k2.rnnt_loss_pruned( logits=logits.float(), symbols=y_padded, diff --git a/egs/librispeech/ASR/pruned2_knowledge/sampling.py b/egs/librispeech/ASR/pruned2_knowledge/sampling.py index 3d2fdd6d8..5b595c76c 100644 --- a/egs/librispeech/ASR/pruned2_knowledge/sampling.py +++ b/egs/librispeech/ASR/pruned2_knowledge/sampling.py @@ -10,7 +10,7 @@ from typing import Optional, Tuple import torch from scaling import ScaledLinear from torch import Tensor, nn -from torch.amp import GradScaler, custom_bwd, custom_fwd +from torch.cuda.amp import GradScaler, custom_bwd, custom_fwd from torch_scheduled_sampling import sample_combined # The main exports of this file are the module KnowledgeBaseLookup and the @@ -330,14 +330,14 @@ def _test_knowledge_base_lookup_autocast(): optimizer = Eve(m.parameters(), lr=0.005, eps=1.0e-04) m = m.to(device) - scaler = GradScaler("cuda", enabled=True) + scaler = GradScaler(enabled=True) start = timeit.default_timer() for epoch in range(150): for n, (x, y) in enumerate(train_pairs): y_out = m(x) - with torch.amp.autocast("cuda", enabled=True): + with torch.cuda.amp.autocast(enabled=True): loss = ((y_out - y) ** 2).mean() * 100.0 if n % 10 == 0 and epoch % 10 == 0: print(f"Epoch {epoch}, batch {n}, loss {loss.item()}") diff --git a/egs/librispeech/ASR/pruned2_knowledge/train.py b/egs/librispeech/ASR/pruned2_knowledge/train.py index 8c117dd60..931341cc4 100755 --- a/egs/librispeech/ASR/pruned2_knowledge/train.py +++ b/egs/librispeech/ASR/pruned2_knowledge/train.py @@ -66,7 +66,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, Eve from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -650,7 +650,7 @@ def train_one_epoch( params.batch_idx_train += 1 batch_size = len(batch["supervisions"]["text"]) - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -868,7 +868,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16) + scaler = GradScaler(enabled=params.use_fp16) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -937,7 +937,7 @@ def scan_pessimistic_batches_for_oom( # warmup = 0.0 is so that the derivs for the pruned loss stay zero # (i.e. are not remembered by the decaying-average in adam), because # we want to avoid these params being subject to shrinkage in adam. - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/ASR/pruned_stateless_emformer_rnnt2/train.py b/egs/librispeech/ASR/pruned_stateless_emformer_rnnt2/train.py index b25a84a6b..2b872f1d5 100755 --- a/egs/librispeech/ASR/pruned_stateless_emformer_rnnt2/train.py +++ b/egs/librispeech/ASR/pruned_stateless_emformer_rnnt2/train.py @@ -55,7 +55,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from noam import Noam from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -693,7 +693,7 @@ def train_one_epoch( params.batch_idx_train += 1 batch_size = len(batch["supervisions"]["text"]) - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -939,7 +939,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16) + scaler = GradScaler(enabled=params.use_fp16) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1004,7 +1004,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/ASR/pruned_transducer_stateless2/model.py b/egs/librispeech/ASR/pruned_transducer_stateless2/model.py index 59ed8310c..272d06c37 100644 --- a/egs/librispeech/ASR/pruned_transducer_stateless2/model.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless2/model.py @@ -157,7 +157,7 @@ class Transducer(nn.Module): lm = self.simple_lm_proj(decoder_out) am = self.simple_am_proj(encoder_out) - with torch.amp.autocast("cuda", enabled=False): + with torch.cuda.amp.autocast(enabled=False): simple_loss, (px_grad, py_grad) = k2.rnnt_loss_smoothed( lm=lm.float(), am=am.float(), @@ -193,7 +193,7 @@ class Transducer(nn.Module): # prior to do_rnnt_pruning (this is an optimization for speed). logits = self.joiner(am_pruned, lm_pruned, project_input=False) - with torch.amp.autocast("cuda", enabled=False): + with torch.cuda.amp.autocast(enabled=False): pruned_loss = k2.rnnt_loss_pruned( logits=logits.float(), symbols=y_padded, diff --git a/egs/librispeech/ASR/pruned_transducer_stateless2/train.py b/egs/librispeech/ASR/pruned_transducer_stateless2/train.py index e86ec8052..6c19f2cb0 100755 --- a/egs/librispeech/ASR/pruned_transducer_stateless2/train.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless2/train.py @@ -78,7 +78,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, Eve from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -759,7 +759,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1000,7 +1000,7 @@ def run(rank, world_size, args): warmup=0.0 if params.start_epoch == 0 else 1.0, ) - scaler = GradScaler("cuda", enabled=params.use_fp16) + scaler = GradScaler(enabled=params.use_fp16) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1067,7 +1067,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/ASR/pruned_transducer_stateless3/model.py b/egs/librispeech/ASR/pruned_transducer_stateless3/model.py index 0495c8a29..d45f6dadc 100644 --- a/egs/librispeech/ASR/pruned_transducer_stateless3/model.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless3/model.py @@ -195,7 +195,7 @@ class Transducer(nn.Module): lm = simple_lm_proj(decoder_out) am = simple_am_proj(encoder_out) - with torch.amp.autocast("cuda", enabled=False): + with torch.cuda.amp.autocast(enabled=False): simple_loss, (px_grad, py_grad) = k2.rnnt_loss_smoothed( lm=lm.float(), am=am.float(), @@ -231,7 +231,7 @@ class Transducer(nn.Module): # prior to do_rnnt_pruning (this is an optimization for speed). logits = joiner(am_pruned, lm_pruned, project_input=False) - with torch.amp.autocast("cuda", enabled=False): + with torch.cuda.amp.autocast(enabled=False): pruned_loss = k2.rnnt_loss_pruned( logits=logits.float(), symbols=y_padded, diff --git a/egs/librispeech/ASR/pruned_transducer_stateless3/train.py b/egs/librispeech/ASR/pruned_transducer_stateless3/train.py index 8ef207518..fdafa5a87 100755 --- a/egs/librispeech/ASR/pruned_transducer_stateless3/train.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless3/train.py @@ -74,7 +74,7 @@ from librispeech import LibriSpeech from model import Transducer from optim import Eden, Eve from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -827,7 +827,7 @@ def train_one_epoch( libri = is_libri(batch["supervisions"]["cut"][0]) - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1126,7 +1126,7 @@ def run(rank, world_size, args): warmup=0.0 if params.start_epoch == 0 else 1.0, ) - scaler = GradScaler("cuda", enabled=params.use_fp16) + scaler = GradScaler(enabled=params.use_fp16) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1195,7 +1195,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/ASR/pruned_transducer_stateless4/train.py b/egs/librispeech/ASR/pruned_transducer_stateless4/train.py index b6682908b..875b03f7f 100755 --- a/egs/librispeech/ASR/pruned_transducer_stateless4/train.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless4/train.py @@ -80,7 +80,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, Eve from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -789,7 +789,7 @@ def train_one_epoch( params.batch_idx_train += 1 batch_size = len(batch["supervisions"]["text"]) - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1047,7 +1047,7 @@ def run(rank, world_size, args): warmup=0.0 if params.start_epoch == 1 else 1.0, ) - scaler = GradScaler("cuda", enabled=params.use_fp16) + scaler = GradScaler(enabled=params.use_fp16) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1116,7 +1116,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/ASR/pruned_transducer_stateless5/train.py b/egs/librispeech/ASR/pruned_transducer_stateless5/train.py index 2b559a27c..66dc5f991 100755 --- a/egs/librispeech/ASR/pruned_transducer_stateless5/train.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless5/train.py @@ -68,7 +68,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, Eve from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -814,7 +814,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1078,7 +1078,7 @@ def run(rank, world_size, args): warmup=0.0 if params.start_epoch == 1 else 1.0, ) - scaler = GradScaler("cuda", enabled=params.use_fp16) + scaler = GradScaler(enabled=params.use_fp16) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1147,7 +1147,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/ASR/pruned_transducer_stateless6/model.py b/egs/librispeech/ASR/pruned_transducer_stateless6/model.py index 20b730a08..daadb70c9 100644 --- a/egs/librispeech/ASR/pruned_transducer_stateless6/model.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless6/model.py @@ -185,7 +185,7 @@ class Transducer(nn.Module): lm = self.simple_lm_proj(decoder_out) am = self.simple_am_proj(encoder_out) - with torch.amp.autocast("cuda", enabled=False): + with torch.cuda.amp.autocast(enabled=False): simple_loss, (px_grad, py_grad) = k2.rnnt_loss_smoothed( lm=lm.float(), am=am.float(), @@ -220,7 +220,7 @@ class Transducer(nn.Module): # prior to do_rnnt_pruning (this is an optimization for speed). logits = self.joiner(am_pruned, lm_pruned, project_input=False) - with torch.amp.autocast("cuda", enabled=False): + with torch.cuda.amp.autocast(enabled=False): pruned_loss = k2.rnnt_loss_pruned( logits=logits.float(), symbols=y_padded, diff --git a/egs/librispeech/ASR/pruned_transducer_stateless6/train.py b/egs/librispeech/ASR/pruned_transducer_stateless6/train.py index 93663505a..8f033cb9a 100755 --- a/egs/librispeech/ASR/pruned_transducer_stateless6/train.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless6/train.py @@ -80,7 +80,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, Eve from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -781,7 +781,7 @@ def train_one_epoch( params.batch_idx_train += 1 batch_size = len(batch["supervisions"]["text"]) - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1039,7 +1039,7 @@ def run(rank, world_size, args): warmup=0.0 if params.start_epoch == 1 else 1.0, ) - scaler = GradScaler("cuda", enabled=params.use_fp16) + scaler = GradScaler(enabled=params.use_fp16) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1108,7 +1108,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/ASR/pruned_transducer_stateless7/finetune.py b/egs/librispeech/ASR/pruned_transducer_stateless7/finetune.py index d29010a23..e7546ec45 100755 --- a/egs/librispeech/ASR/pruned_transducer_stateless7/finetune.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless7/finetune.py @@ -66,7 +66,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, ScaledAdam from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer @@ -903,7 +903,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1219,7 +1219,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1319,7 +1319,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/ASR/pruned_transducer_stateless7/model.py b/egs/librispeech/ASR/pruned_transducer_stateless7/model.py index 49076b96f..add0e6a18 100644 --- a/egs/librispeech/ASR/pruned_transducer_stateless7/model.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless7/model.py @@ -150,7 +150,7 @@ class Transducer(nn.Module): # if self.training and random.random() < 0.25: # am = penalize_abs_values_gt(am, 30.0, 1.0e-04) - with torch.amp.autocast("cuda", enabled=False): + with torch.cuda.amp.autocast(enabled=False): simple_loss, (px_grad, py_grad) = k2.rnnt_loss_smoothed( lm=lm.float(), am=am.float(), @@ -185,7 +185,7 @@ class Transducer(nn.Module): # prior to do_rnnt_pruning (this is an optimization for speed). logits = self.joiner(am_pruned, lm_pruned, project_input=False) - with torch.amp.autocast("cuda", enabled=False): + with torch.cuda.amp.autocast(enabled=False): pruned_loss = k2.rnnt_loss_pruned( logits=logits.float(), symbols=y_padded, diff --git a/egs/librispeech/ASR/pruned_transducer_stateless7/scaling.py b/egs/librispeech/ASR/pruned_transducer_stateless7/scaling.py index 16d86fe2d..30a737061 100644 --- a/egs/librispeech/ASR/pruned_transducer_stateless7/scaling.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless7/scaling.py @@ -289,7 +289,7 @@ class SoftmaxFunction(torch.autograd.Function): @staticmethod def backward(ctx, ans_grad: Tensor): (ans,) = ctx.saved_tensors - with torch.amp.autocast("cuda", enabled=False): + with torch.cuda.amp.autocast(enabled=False): ans_grad = ans_grad.to(torch.float32) ans = ans.to(torch.float32) x_grad = ans_grad * ans @@ -669,7 +669,7 @@ class WhiteningPenaltyFunction(torch.autograd.Function): def backward(ctx, x_grad: Tensor): (x_orig,) = ctx.saved_tensors with torch.enable_grad(): - with torch.amp.autocast("cuda", enabled=False): + with torch.cuda.amp.autocast(enabled=False): x_detached = x_orig.to(torch.float32).detach() x_detached.requires_grad = True @@ -867,7 +867,7 @@ class MaxEig(torch.nn.Module): ): return _no_op(x) - with torch.amp.autocast("cuda", enabled=False): + with torch.cuda.amp.autocast(enabled=False): eps = 1.0e-20 orig_x = x x = x.to(torch.float32) diff --git a/egs/librispeech/ASR/pruned_transducer_stateless7/train.py b/egs/librispeech/ASR/pruned_transducer_stateless7/train.py index 91fccd58d..436ec53b4 100755 --- a/egs/librispeech/ASR/pruned_transducer_stateless7/train.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless7/train.py @@ -67,7 +67,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, ScaledAdam from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer @@ -809,7 +809,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1106,7 +1106,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1206,7 +1206,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/ASR/pruned_transducer_stateless7/zipformer.py b/egs/librispeech/ASR/pruned_transducer_stateless7/zipformer.py index ebef7e977..cbde2a2e4 100644 --- a/egs/librispeech/ASR/pruned_transducer_stateless7/zipformer.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless7/zipformer.py @@ -1421,7 +1421,7 @@ class RelPositionMultiheadAttention(nn.Module): bsz = n // num_heads with torch.no_grad(): - with torch.amp.autocast("cuda", enabled=False): + with torch.cuda.amp.autocast(enabled=False): attn_weights = attn_weights.to(torch.float32) attn_output = attn_output.to(torch.float32) attn_weights_entropy = ( diff --git a/egs/librispeech/ASR/pruned_transducer_stateless7_ctc/model.py b/egs/librispeech/ASR/pruned_transducer_stateless7_ctc/model.py index 0224c15d7..a6e919e2f 100644 --- a/egs/librispeech/ASR/pruned_transducer_stateless7_ctc/model.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless7_ctc/model.py @@ -150,7 +150,7 @@ class Transducer(nn.Module): lm = self.simple_lm_proj(decoder_out) am = self.simple_am_proj(encoder_out) - with torch.amp.autocast("cuda", enabled=False): + with torch.cuda.amp.autocast(enabled=False): simple_loss, (px_grad, py_grad) = k2.rnnt_loss_smoothed( lm=lm.float(), am=am.float(), @@ -185,7 +185,7 @@ class Transducer(nn.Module): # prior to do_rnnt_pruning (this is an optimization for speed). logits = self.joiner(am_pruned, lm_pruned, project_input=False) - with torch.amp.autocast("cuda", enabled=False): + with torch.cuda.amp.autocast(enabled=False): pruned_loss = k2.rnnt_loss_pruned( logits=logits.float(), symbols=y_padded, diff --git a/egs/librispeech/ASR/pruned_transducer_stateless7_ctc/train.py b/egs/librispeech/ASR/pruned_transducer_stateless7_ctc/train.py index 395b07b05..b35e56abc 100755 --- a/egs/librispeech/ASR/pruned_transducer_stateless7_ctc/train.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless7_ctc/train.py @@ -67,7 +67,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, ScaledAdam from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer @@ -833,7 +833,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1128,7 +1128,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1228,7 +1228,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/ASR/pruned_transducer_stateless7_ctc_bs/model.py b/egs/librispeech/ASR/pruned_transducer_stateless7_ctc_bs/model.py index 4675697c1..0582b289f 100644 --- a/egs/librispeech/ASR/pruned_transducer_stateless7_ctc_bs/model.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless7_ctc_bs/model.py @@ -178,7 +178,7 @@ class Transducer(nn.Module): am = self.simple_am_proj(encoder_out_fr) lm = self.simple_lm_proj(decoder_out) - with torch.amp.autocast("cuda", enabled=False): + with torch.cuda.amp.autocast(enabled=False): simple_loss, (px_grad, py_grad) = k2.rnnt_loss_smoothed( lm=lm.float(), am=am.float(), @@ -213,7 +213,7 @@ class Transducer(nn.Module): # prior to do_rnnt_pruning (this is an optimization for speed). logits = self.joiner(am_pruned, lm_pruned, project_input=False) - with torch.amp.autocast("cuda", enabled=False): + with torch.cuda.amp.autocast(enabled=False): pruned_loss = k2.rnnt_loss_pruned( logits=logits.float(), symbols=y_padded, diff --git a/egs/librispeech/ASR/pruned_transducer_stateless7_ctc_bs/train.py b/egs/librispeech/ASR/pruned_transducer_stateless7_ctc_bs/train.py index a431b278d..c2d877a93 100755 --- a/egs/librispeech/ASR/pruned_transducer_stateless7_ctc_bs/train.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless7_ctc_bs/train.py @@ -63,7 +63,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, ScaledAdam from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer @@ -822,7 +822,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1118,7 +1118,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1217,7 +1217,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/ASR/pruned_transducer_stateless7_streaming/do_not_use_it_directly.py b/egs/librispeech/ASR/pruned_transducer_stateless7_streaming/do_not_use_it_directly.py index dc3493425..8e239e322 100755 --- a/egs/librispeech/ASR/pruned_transducer_stateless7_streaming/do_not_use_it_directly.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless7_streaming/do_not_use_it_directly.py @@ -66,7 +66,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, ScaledAdam from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer_for_ncnn_export_only import Zipformer @@ -811,7 +811,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1106,7 +1106,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1206,7 +1206,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/ASR/pruned_transducer_stateless7_streaming/train.py b/egs/librispeech/ASR/pruned_transducer_stateless7_streaming/train.py index a8f47d941..8bd00bbef 100755 --- a/egs/librispeech/ASR/pruned_transducer_stateless7_streaming/train.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless7_streaming/train.py @@ -66,7 +66,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, ScaledAdam from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer @@ -810,7 +810,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1124,7 +1124,7 @@ def run(rank, world_size, args): # params=params, # ) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1224,7 +1224,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/ASR/pruned_transducer_stateless7_streaming/zipformer.py b/egs/librispeech/ASR/pruned_transducer_stateless7_streaming/zipformer.py index e3b8b3725..c7e45564f 100644 --- a/egs/librispeech/ASR/pruned_transducer_stateless7_streaming/zipformer.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless7_streaming/zipformer.py @@ -2408,7 +2408,7 @@ class RelPositionMultiheadAttention(nn.Module): bsz = n // num_heads with torch.no_grad(): - with torch.amp.autocast("cuda", enabled=False): + with torch.cuda.amp.autocast(enabled=False): attn_weights = attn_weights.to(torch.float32) attn_output = attn_output.to(torch.float32) attn_weights_entropy = ( diff --git a/egs/librispeech/ASR/pruned_transducer_stateless7_streaming/zipformer_for_ncnn_export_only.py b/egs/librispeech/ASR/pruned_transducer_stateless7_streaming/zipformer_for_ncnn_export_only.py index ff23725b7..5284ed627 100644 --- a/egs/librispeech/ASR/pruned_transducer_stateless7_streaming/zipformer_for_ncnn_export_only.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless7_streaming/zipformer_for_ncnn_export_only.py @@ -2708,7 +2708,7 @@ class RelPositionMultiheadAttention(nn.Module): bsz = n // num_heads with torch.no_grad(): - with torch.amp.autocast("cuda", enabled=False): + with torch.cuda.amp.autocast(enabled=False): attn_weights = attn_weights.to(torch.float32) attn_output = attn_output.to(torch.float32) attn_weights_entropy = ( diff --git a/egs/librispeech/ASR/pruned_transducer_stateless7_streaming_multi/train.py b/egs/librispeech/ASR/pruned_transducer_stateless7_streaming_multi/train.py index 4c8c239a1..da5e144c9 100755 --- a/egs/librispeech/ASR/pruned_transducer_stateless7_streaming_multi/train.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless7_streaming_multi/train.py @@ -70,7 +70,7 @@ from librispeech import LibriSpeech from model import Transducer from optim import Eden, ScaledAdam from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer @@ -866,7 +866,7 @@ def train_one_epoch( libri = is_libri(batch["supervisions"]["cut"][0]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1218,7 +1218,7 @@ def run(rank, world_size, args): # params=params, # ) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1320,7 +1320,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/ASR/pruned_transducer_stateless8/model.py b/egs/librispeech/ASR/pruned_transducer_stateless8/model.py index c0b9113b7..39a360796 100644 --- a/egs/librispeech/ASR/pruned_transducer_stateless8/model.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless8/model.py @@ -172,7 +172,7 @@ class Transducer(nn.Module): # if self.training and random.random() < 0.25: # am = penalize_abs_values_gt(am, 30.0, 1.0e-04) - with torch.amp.autocast("cuda", enabled=False): + with torch.cuda.amp.autocast(enabled=False): simple_loss, (px_grad, py_grad) = k2.rnnt_loss_smoothed( lm=lm.float(), am=am.float(), @@ -207,7 +207,7 @@ class Transducer(nn.Module): # prior to do_rnnt_pruning (this is an optimization for speed). logits = joiner(am_pruned, lm_pruned, project_input=False) - with torch.amp.autocast("cuda", enabled=False): + with torch.cuda.amp.autocast(enabled=False): pruned_loss = k2.rnnt_loss_pruned( logits=logits.float(), symbols=y_padded, diff --git a/egs/librispeech/ASR/pruned_transducer_stateless8/train.py b/egs/librispeech/ASR/pruned_transducer_stateless8/train.py index 0ccef210e..646f30ca1 100755 --- a/egs/librispeech/ASR/pruned_transducer_stateless8/train.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless8/train.py @@ -75,7 +75,7 @@ from librispeech import LibriSpeech from model import Transducer from optim import Eden, ScaledAdam from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer @@ -866,7 +866,7 @@ def train_one_epoch( libri = is_libri(batch["supervisions"]["cut"][0]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1219,7 +1219,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1321,7 +1321,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/ASR/tiny_transducer_ctc/train.py b/egs/librispeech/ASR/tiny_transducer_ctc/train.py index 0536e89b3..1bfd071de 100644 --- a/egs/librispeech/ASR/tiny_transducer_ctc/train.py +++ b/egs/librispeech/ASR/tiny_transducer_ctc/train.py @@ -51,7 +51,7 @@ from lhotse.dataset.sampling.base import CutSampler from lhotse.utils import fix_random_seed from model import Transducer from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.optim import AdamW from torch.optim.lr_scheduler import StepLR @@ -809,7 +809,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1092,7 +1092,7 @@ def run(rank, world_size, args): # params=params, # ) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1198,7 +1198,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/ASR/zipformer/finetune.py b/egs/librispeech/ASR/zipformer/finetune.py index 5da903d38..2ff631914 100755 --- a/egs/librispeech/ASR/zipformer/finetune.py +++ b/egs/librispeech/ASR/zipformer/finetune.py @@ -78,7 +78,7 @@ from optim import Eden, ScaledAdam from scaling import ScheduledFloat from subsampling import Conv2dSubsampling from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer2 @@ -1049,7 +1049,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1373,7 +1373,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1474,7 +1474,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/ASR/zipformer/model.py b/egs/librispeech/ASR/zipformer/model.py index b0bb7c7fe..c7dbe1e0a 100644 --- a/egs/librispeech/ASR/zipformer/model.py +++ b/egs/librispeech/ASR/zipformer/model.py @@ -285,7 +285,7 @@ class AsrModel(nn.Module): # if self.training and random.random() < 0.25: # am = penalize_abs_values_gt(am, 30.0, 1.0e-04) - with torch.amp.autocast("cuda", enabled=False): + with torch.cuda.amp.autocast(enabled=False): simple_loss, (px_grad, py_grad) = k2.rnnt_loss_smoothed( lm=lm.float(), am=am.float(), @@ -320,7 +320,7 @@ class AsrModel(nn.Module): # prior to do_rnnt_pruning (this is an optimization for speed). logits = self.joiner(am_pruned, lm_pruned, project_input=False) - with torch.amp.autocast("cuda", enabled=False): + with torch.cuda.amp.autocast(enabled=False): pruned_loss = k2.rnnt_loss_pruned( logits=logits.float(), symbols=y_padded, diff --git a/egs/librispeech/ASR/zipformer/scaling.py b/egs/librispeech/ASR/zipformer/scaling.py index 46df86bf7..d345c2931 100644 --- a/egs/librispeech/ASR/zipformer/scaling.py +++ b/egs/librispeech/ASR/zipformer/scaling.py @@ -306,7 +306,7 @@ class SoftmaxFunction(torch.autograd.Function): @staticmethod def backward(ctx, ans_grad: Tensor): (ans,) = ctx.saved_tensors - with torch.amp.autocast("cuda", enabled=False): + with torch.cuda.amp.autocast(enabled=False): ans_grad = ans_grad.to(torch.float32) ans = ans.to(torch.float32) x_grad = ans_grad * ans @@ -759,7 +759,7 @@ class BalancerFunction(torch.autograd.Function): try: with torch.enable_grad(): - with torch.amp.autocast("cuda", enabled=False): + with torch.cuda.amp.autocast(enabled=False): x = x.to(torch.float32) x = x.detach() x.requires_grad = True @@ -1014,7 +1014,7 @@ class WhiteningPenaltyFunction(torch.autograd.Function): try: with torch.enable_grad(): - with torch.amp.autocast("cuda", enabled=False): + with torch.cuda.amp.autocast(enabled=False): x_detached = x_orig.to(torch.float32).detach() x_detached.requires_grad = True @@ -1353,7 +1353,7 @@ class SwooshLFunction(torch.autograd.Function): coeff = -0.08 - with torch.amp.autocast("cuda", enabled=False): + with torch.cuda.amp.autocast(enabled=False): with torch.enable_grad(): x = x.detach() x.requires_grad = True @@ -1430,7 +1430,7 @@ class SwooshRFunction(torch.autograd.Function): zero = torch.tensor(0.0, dtype=x.dtype, device=x.device) - with torch.amp.autocast("cuda", enabled=False): + with torch.cuda.amp.autocast(enabled=False): with torch.enable_grad(): x = x.detach() x.requires_grad = True diff --git a/egs/librispeech/ASR/zipformer/train.py b/egs/librispeech/ASR/zipformer/train.py index 71d045ea0..c074c32ec 100755 --- a/egs/librispeech/ASR/zipformer/train.py +++ b/egs/librispeech/ASR/zipformer/train.py @@ -79,7 +79,7 @@ from optim import Eden, ScaledAdam from scaling import ScheduledFloat from subsampling import Conv2dSubsampling from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer2 @@ -1101,7 +1101,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", + with torch.cuda.amp.autocast( enabled=params.use_autocast, dtype=params.dtype ): loss, loss_info = compute_loss( @@ -1438,7 +1438,7 @@ def run(rank, world_size, args): spec_augment=spec_augment, ) - scaler = GradScaler("cuda", enabled=params.use_autocast, init_scale=1.0) + scaler = GradScaler(enabled=params.use_autocast, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1540,7 +1540,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", + with torch.cuda.amp.autocast( enabled=params.use_autocast, dtype=params.dtype ): loss, _ = compute_loss( diff --git a/egs/librispeech/ASR/zipformer/zipformer.py b/egs/librispeech/ASR/zipformer/zipformer.py index bdfd2175c..2a0ae0129 100644 --- a/egs/librispeech/ASR/zipformer/zipformer.py +++ b/egs/librispeech/ASR/zipformer/zipformer.py @@ -1873,7 +1873,7 @@ class RelPositionMultiheadAttentionWeights(nn.Module): (num_heads, batch_size, seq_len, seq_len) = attn_weights.shape with torch.no_grad(): - with torch.amp.autocast("cuda", enabled=False): + with torch.cuda.amp.autocast(enabled=False): attn_weights = attn_weights.to(torch.float32) attn_weights_entropy = ( -((attn_weights + 1.0e-20).log() * attn_weights) diff --git a/egs/librispeech/ASR/zipformer_adapter/train.py b/egs/librispeech/ASR/zipformer_adapter/train.py index 0207fc26e..3511590da 100755 --- a/egs/librispeech/ASR/zipformer_adapter/train.py +++ b/egs/librispeech/ASR/zipformer_adapter/train.py @@ -67,7 +67,7 @@ from optim import Eden, ScaledAdam from scaling import ScheduledFloat from subsampling import Conv2dSubsampling from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer2 @@ -1052,7 +1052,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1397,7 +1397,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1498,7 +1498,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/ASR/zipformer_adapter/zipformer.py b/egs/librispeech/ASR/zipformer_adapter/zipformer.py index 6224d136a..8e2dfdd72 100644 --- a/egs/librispeech/ASR/zipformer_adapter/zipformer.py +++ b/egs/librispeech/ASR/zipformer_adapter/zipformer.py @@ -1916,7 +1916,7 @@ class RelPositionMultiheadAttentionWeights(nn.Module): (num_heads, batch_size, seq_len, seq_len) = attn_weights.shape with torch.no_grad(): - with torch.amp.autocast("cuda", enabled=False): + with torch.cuda.amp.autocast(enabled=False): attn_weights = attn_weights.to(torch.float32) attn_weights_entropy = ( -((attn_weights + 1.0e-20).log() * attn_weights) diff --git a/egs/librispeech/ASR/zipformer_ctc/train.py b/egs/librispeech/ASR/zipformer_ctc/train.py index dfe702d2f..60112a84e 100755 --- a/egs/librispeech/ASR/zipformer_ctc/train.py +++ b/egs/librispeech/ASR/zipformer_ctc/train.py @@ -46,7 +46,7 @@ from lhotse.utils import fix_random_seed from model import CTCModel from optim import Eden, LRScheduler, ScaledAdam from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.nn.utils import clip_grad_norm_ from torch.utils.tensorboard import SummaryWriter @@ -726,7 +726,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -987,7 +987,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) diff --git a/egs/librispeech/ASR/zipformer_lora/finetune.py b/egs/librispeech/ASR/zipformer_lora/finetune.py index 53152971d..3f36f229f 100755 --- a/egs/librispeech/ASR/zipformer_lora/finetune.py +++ b/egs/librispeech/ASR/zipformer_lora/finetune.py @@ -78,7 +78,7 @@ from optim import Eden, ScaledAdam from scaling import ScheduledFloat from subsampling import Conv2dSubsampling from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer2 @@ -1065,7 +1065,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1406,7 +1406,7 @@ def run(rank, world_size, args): # params=params, # ) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1507,7 +1507,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/ASR/zipformer_lora/scaling.py b/egs/librispeech/ASR/zipformer_lora/scaling.py index a1e77fe0e..8d7aa8027 100644 --- a/egs/librispeech/ASR/zipformer_lora/scaling.py +++ b/egs/librispeech/ASR/zipformer_lora/scaling.py @@ -307,7 +307,7 @@ class SoftmaxFunction(torch.autograd.Function): @staticmethod def backward(ctx, ans_grad: Tensor): (ans,) = ctx.saved_tensors - with torch.amp.autocast("cuda", enabled=False): + with torch.cuda.amp.autocast(enabled=False): ans_grad = ans_grad.to(torch.float32) ans = ans.to(torch.float32) x_grad = ans_grad * ans @@ -863,7 +863,7 @@ class BalancerFunction(torch.autograd.Function): try: with torch.enable_grad(): - with torch.amp.autocast("cuda", enabled=False): + with torch.cuda.amp.autocast(enabled=False): x = x.to(torch.float32) x = x.detach() x.requires_grad = True @@ -1118,7 +1118,7 @@ class WhiteningPenaltyFunction(torch.autograd.Function): try: with torch.enable_grad(): - with torch.amp.autocast("cuda", enabled=False): + with torch.cuda.amp.autocast(enabled=False): x_detached = x_orig.to(torch.float32).detach() x_detached.requires_grad = True @@ -1457,7 +1457,7 @@ class SwooshLFunction(torch.autograd.Function): coeff = -0.08 - with torch.amp.autocast("cuda", enabled=False): + with torch.cuda.amp.autocast(enabled=False): with torch.enable_grad(): x = x.detach() x.requires_grad = True @@ -1534,7 +1534,7 @@ class SwooshRFunction(torch.autograd.Function): zero = torch.tensor(0.0, dtype=x.dtype, device=x.device) - with torch.amp.autocast("cuda", enabled=False): + with torch.cuda.amp.autocast(enabled=False): with torch.enable_grad(): x = x.detach() x.requires_grad = True diff --git a/egs/librispeech/ASR/zipformer_lora/train.py b/egs/librispeech/ASR/zipformer_lora/train.py index 592bc0fd4..9ab214e86 100755 --- a/egs/librispeech/ASR/zipformer_lora/train.py +++ b/egs/librispeech/ASR/zipformer_lora/train.py @@ -76,7 +76,7 @@ from optim import Eden, ScaledAdam from scaling import ScheduledFloat from subsampling import Conv2dSubsampling from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer2 @@ -947,7 +947,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1252,7 +1252,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1352,7 +1352,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/ASR/zipformer_lora/zipformer.py b/egs/librispeech/ASR/zipformer_lora/zipformer.py index ece7c3df1..43865609a 100644 --- a/egs/librispeech/ASR/zipformer_lora/zipformer.py +++ b/egs/librispeech/ASR/zipformer_lora/zipformer.py @@ -1905,7 +1905,7 @@ class RelPositionMultiheadAttentionWeights(nn.Module): (num_heads, batch_size, seq_len, seq_len) = attn_weights.shape with torch.no_grad(): - with torch.amp.autocast("cuda", enabled=False): + with torch.cuda.amp.autocast(enabled=False): attn_weights = attn_weights.to(torch.float32) attn_weights_entropy = ( -((attn_weights + 1.0e-20).log() * attn_weights) diff --git a/egs/librispeech/ASR/zipformer_mmi/train.py b/egs/librispeech/ASR/zipformer_mmi/train.py index bed3cfa04..c1785a328 100755 --- a/egs/librispeech/ASR/zipformer_mmi/train.py +++ b/egs/librispeech/ASR/zipformer_mmi/train.py @@ -64,7 +64,7 @@ from lhotse.utils import fix_random_seed from model import CTCModel from optim import Eden, ScaledAdam from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer @@ -744,7 +744,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1037,7 +1037,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1138,7 +1138,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/SSL/hubert/finetune.py b/egs/librispeech/SSL/hubert/finetune.py index 9717d579d..17daa3c9d 100644 --- a/egs/librispeech/SSL/hubert/finetune.py +++ b/egs/librispeech/SSL/hubert/finetune.py @@ -66,7 +66,7 @@ from lhotse.utils import fix_random_seed from model import AsrModel from optim import Eden, ScaledAdam from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -816,7 +816,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1109,7 +1109,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1207,7 +1207,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/SSL/hubert/finetune_ce.py b/egs/librispeech/SSL/hubert/finetune_ce.py index 340aa4aa2..2723cc770 100644 --- a/egs/librispeech/SSL/hubert/finetune_ce.py +++ b/egs/librispeech/SSL/hubert/finetune_ce.py @@ -66,7 +66,7 @@ from lhotse.utils import fix_random_seed from model import AsrModel from optim import Eden, ScaledAdam from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -816,7 +816,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1109,7 +1109,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1207,7 +1207,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/SSL/hubert/model.py b/egs/librispeech/SSL/hubert/model.py index b23fa32ea..46a968b69 100644 --- a/egs/librispeech/SSL/hubert/model.py +++ b/egs/librispeech/SSL/hubert/model.py @@ -221,7 +221,7 @@ class AsrModel(nn.Module): # if self.training and random.random() < 0.25: # am = penalize_abs_values_gt(am, 30.0, 1.0e-04) - with torch.amp.autocast("cuda", enabled=False): + with torch.cuda.amp.autocast(enabled=False): simple_loss, (px_grad, py_grad) = k2.rnnt_loss_smoothed( lm=lm.float(), am=am.float(), @@ -256,7 +256,7 @@ class AsrModel(nn.Module): # prior to do_rnnt_pruning (this is an optimization for speed). logits = self.joiner(am_pruned, lm_pruned, project_input=False) - with torch.amp.autocast("cuda", enabled=False): + with torch.cuda.amp.autocast(enabled=False): pruned_loss = k2.rnnt_loss_pruned( logits=logits.float(), symbols=y_padded, diff --git a/egs/librispeech/SSL/hubert/pretrain.py b/egs/librispeech/SSL/hubert/pretrain.py index 1868bf0a6..f183d90fd 100644 --- a/egs/librispeech/SSL/hubert/pretrain.py +++ b/egs/librispeech/SSL/hubert/pretrain.py @@ -59,7 +59,7 @@ from lhotse.utils import fix_random_seed from optim import Eden, ScaledAdam from ssl_datamodule import LibriSpeechDataModule from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.functional import pad from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -644,7 +644,7 @@ def train_one_epoch( batch_size = batch["kmeans"].shape[0] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -945,7 +945,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1036,7 +1036,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/SSL/hubert/pretrain_ce.py b/egs/librispeech/SSL/hubert/pretrain_ce.py index 97efd983b..94948695d 100644 --- a/egs/librispeech/SSL/hubert/pretrain_ce.py +++ b/egs/librispeech/SSL/hubert/pretrain_ce.py @@ -59,7 +59,7 @@ from lhotse.utils import fix_random_seed from optim import Eden, ScaledAdam from ssl_datamodule import LibriSpeechDataModule from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.functional import pad from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -644,7 +644,7 @@ def train_one_epoch( batch_size = batch["kmeans"].shape[0] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -945,7 +945,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1036,7 +1036,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/SSL/zipformer/finetune.py b/egs/librispeech/SSL/zipformer/finetune.py index 6bfab9d00..c907b41c5 100644 --- a/egs/librispeech/SSL/zipformer/finetune.py +++ b/egs/librispeech/SSL/zipformer/finetune.py @@ -66,7 +66,7 @@ from lhotse.utils import fix_random_seed from model import AsrModel from optim import Eden, ScaledAdam from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -1115,7 +1115,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1406,7 +1406,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1504,7 +1504,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/SSL/zipformer/model.py b/egs/librispeech/SSL/zipformer/model.py index b23fa32ea..46a968b69 100644 --- a/egs/librispeech/SSL/zipformer/model.py +++ b/egs/librispeech/SSL/zipformer/model.py @@ -221,7 +221,7 @@ class AsrModel(nn.Module): # if self.training and random.random() < 0.25: # am = penalize_abs_values_gt(am, 30.0, 1.0e-04) - with torch.amp.autocast("cuda", enabled=False): + with torch.cuda.amp.autocast(enabled=False): simple_loss, (px_grad, py_grad) = k2.rnnt_loss_smoothed( lm=lm.float(), am=am.float(), @@ -256,7 +256,7 @@ class AsrModel(nn.Module): # prior to do_rnnt_pruning (this is an optimization for speed). logits = self.joiner(am_pruned, lm_pruned, project_input=False) - with torch.amp.autocast("cuda", enabled=False): + with torch.cuda.amp.autocast(enabled=False): pruned_loss = k2.rnnt_loss_pruned( logits=logits.float(), symbols=y_padded, diff --git a/egs/librispeech/SSL/zipformer/pretrain.py b/egs/librispeech/SSL/zipformer/pretrain.py index 767c3bacb..937fb382e 100644 --- a/egs/librispeech/SSL/zipformer/pretrain.py +++ b/egs/librispeech/SSL/zipformer/pretrain.py @@ -58,7 +58,7 @@ from lhotse.utils import fix_random_seed from optim import Eden, ScaledAdam from ssl_datamodule import LibriSpeechDataModule from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -944,7 +944,7 @@ def train_one_epoch( batch_size = batch["kmeans"].shape[0] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1243,7 +1243,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1334,7 +1334,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/SSL/zipformer/zipformer.py b/egs/librispeech/SSL/zipformer/zipformer.py index 7e9ccb51f..e9eff3357 100644 --- a/egs/librispeech/SSL/zipformer/zipformer.py +++ b/egs/librispeech/SSL/zipformer/zipformer.py @@ -1849,7 +1849,7 @@ class RelPositionMultiheadAttentionWeights(nn.Module): (num_heads, batch_size, seq_len, seq_len) = attn_weights.shape with torch.no_grad(): - with torch.amp.autocast("cuda", enabled=False): + with torch.cuda.amp.autocast(enabled=False): attn_weights = attn_weights.to(torch.float32) attn_weights_entropy = ( -((attn_weights + 1.0e-20).log() * attn_weights) diff --git a/egs/librispeech/WSASR/conformer_ctc2/train.py b/egs/librispeech/WSASR/conformer_ctc2/train.py index fc7728562..82c68803f 100755 --- a/egs/librispeech/WSASR/conformer_ctc2/train.py +++ b/egs/librispeech/WSASR/conformer_ctc2/train.py @@ -62,7 +62,7 @@ from lhotse.dataset.sampling.base import CutSampler from lhotse.utils import fix_random_seed from optim import Eden, Eve from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -757,7 +757,7 @@ def train_one_epoch( params.batch_idx_train += 1 batch_size = len(batch["supervisions"]["text"]) - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1005,7 +1005,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16) + scaler = GradScaler(enabled=params.use_fp16) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1076,7 +1076,7 @@ def scan_pessimistic_batches_for_oom( # warmup = 0.0 is so that the derivs for the pruned loss stay zero # (i.e. are not remembered by the decaying-average in adam), because # we want to avoid these params being subject to shrinkage in adam. - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/librispeech/WSASR/conformer_ctc2/train_phone.py b/egs/librispeech/WSASR/conformer_ctc2/train_phone.py index 1c4bd50bf..b276d0587 100755 --- a/egs/librispeech/WSASR/conformer_ctc2/train_phone.py +++ b/egs/librispeech/WSASR/conformer_ctc2/train_phone.py @@ -62,7 +62,7 @@ from lhotse.dataset.sampling.base import CutSampler from lhotse.utils import fix_random_seed from optim import Eden, Eve from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -758,7 +758,7 @@ def train_one_epoch( params.batch_idx_train += 1 batch_size = len(batch["supervisions"]["text"]) - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1007,7 +1007,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16) + scaler = GradScaler(enabled=params.use_fp16) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1078,7 +1078,7 @@ def scan_pessimistic_batches_for_oom( # warmup = 0.0 is so that the derivs for the pruned loss stay zero # (i.e. are not remembered by the decaying-average in adam), because # we want to avoid these params being subject to shrinkage in adam. - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/libritts/ASR/zipformer/train.py b/egs/libritts/ASR/zipformer/train.py index 78e3330bd..5485eaf0a 100755 --- a/egs/libritts/ASR/zipformer/train.py +++ b/egs/libritts/ASR/zipformer/train.py @@ -80,7 +80,7 @@ from optim import Eden, ScaledAdam from scaling import ScheduledFloat from subsampling import Conv2dSubsampling from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer2 @@ -1049,8 +1049,8 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast( - "cuda", enabled=params.use_autocast, dtype=params.dtype + with torch.cuda.amp.autocast( + enabled=params.use_autocast, dtype=params.dtype ): loss, loss_info = compute_loss( params=params, @@ -1378,7 +1378,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_autocast, init_scale=1.0) + scaler = GradScaler(enabled=params.use_autocast, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1478,8 +1478,8 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast( - "cuda", enabled=params.use_autocast, dtype=params.dtype + with torch.cuda.amp.autocast( + enabled=params.use_autocast, dtype=params.dtype ): loss, _ = compute_loss( params=params, diff --git a/egs/libritts/CODEC/encodec/encodec.py b/egs/libritts/CODEC/encodec/encodec.py index 31fc4f126..f21d494b6 100644 --- a/egs/libritts/CODEC/encodec/encodec.py +++ b/egs/libritts/CODEC/encodec/encodec.py @@ -29,7 +29,7 @@ from loss import ( WavReconstructionLoss, ) from torch import nn -from torch.amp import autocast +from torch.cuda.amp import autocast class Encodec(nn.Module): @@ -148,7 +148,7 @@ class Encodec(nn.Module): ) # calculate losses - with autocast("cuda", enabled=False): + with autocast(enabled=False): gen_stft_adv_loss = self.generator_adversarial_loss(outputs=y_hat) if self.multi_period_discriminator is not None: @@ -272,7 +272,7 @@ class Encodec(nn.Module): speech_hat.contiguous().detach(), ) # calculate losses - with autocast("cuda", enabled=False): + with autocast(enabled=False): ( disc_stft_real_adv_loss, disc_stft_fake_adv_loss, diff --git a/egs/libritts/CODEC/encodec/train.py b/egs/libritts/CODEC/encodec/train.py index 31349df43..a4f2eb7ab 100755 --- a/egs/libritts/CODEC/encodec/train.py +++ b/egs/libritts/CODEC/encodec/train.py @@ -34,7 +34,7 @@ from encodec import Encodec from lhotse.utils import fix_random_seed from scheduler import WarmupCosineLrScheduler from torch import nn -from torch.amp import GradScaler, autocast +from torch.cuda.amp import GradScaler, autocast from torch.nn.parallel import DistributedDataParallel as DDP from torch.optim import Optimizer from torch.utils.tensorboard import SummaryWriter @@ -466,7 +466,7 @@ def train_one_epoch( loss_info["samples"] = batch_size try: - with autocast("cuda", enabled=params.use_fp16): + with autocast(enabled=params.use_fp16): d_weight = train_discriminator( params.lambda_adv, params.cur_epoch, @@ -502,7 +502,7 @@ def train_one_epoch( scaler.scale(disc_loss).backward() scaler.step(optimizer_d) - with autocast("cuda", enabled=params.use_fp16): + with autocast(enabled=params.use_fp16): g_weight = train_discriminator( params.lambda_adv, params.cur_epoch, @@ -846,7 +846,7 @@ def scan_pessimistic_batches_for_oom( ) = prepare_input(params, batch, device) try: # for discriminator - with autocast("cuda", enabled=params.use_fp16): + with autocast(enabled=params.use_fp16): ( disc_stft_real_adv_loss, disc_stft_fake_adv_loss, @@ -876,7 +876,7 @@ def scan_pessimistic_batches_for_oom( optimizer_d.zero_grad() loss_d.backward() # for generator - with autocast("cuda", enabled=params.use_fp16): + with autocast(enabled=params.use_fp16): ( commit_loss, gen_stft_adv_loss, @@ -1102,7 +1102,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) diff --git a/egs/libritts/TTS/vits/train.py b/egs/libritts/TTS/vits/train.py index 6803d6eb2..447fbcf5d 100755 --- a/egs/libritts/TTS/vits/train.py +++ b/egs/libritts/TTS/vits/train.py @@ -32,7 +32,7 @@ from lhotse.cut import Cut from lhotse.features.io import KaldiReader from lhotse.utils import fix_random_seed from tokenizer import Tokenizer -from torch.amp import GradScaler, autocast +from torch.cuda.amp import GradScaler, autocast from torch.nn.parallel import DistributedDataParallel as DDP from torch.optim import Optimizer from torch.utils.tensorboard import SummaryWriter @@ -456,7 +456,7 @@ def train_one_epoch( loss_info["samples"] = batch_size try: - with autocast("cuda", enabled=params.use_fp16): + with autocast(enabled=params.use_fp16): # forward discriminator loss_d, stats_d = model( text=tokens, @@ -475,7 +475,7 @@ def train_one_epoch( scaler.scale(loss_d).backward() scaler.step(optimizer_d) - with autocast("cuda", enabled=params.use_fp16): + with autocast(enabled=params.use_fp16): # forward generator loss_g, stats_g = model( text=tokens, @@ -748,7 +748,7 @@ def scan_pessimistic_batches_for_oom( ) = prepare_input(batch, tokenizer, device, train_speaker_map) try: # for discriminator - with autocast("cuda", enabled=params.use_fp16): + with autocast(enabled=params.use_fp16): loss_d, stats_d = model( text=tokens, text_lengths=tokens_lens, @@ -762,7 +762,7 @@ def scan_pessimistic_batches_for_oom( optimizer_d.zero_grad() loss_d.backward() # for generator - with autocast("cuda", enabled=params.use_fp16): + with autocast(enabled=params.use_fp16): loss_g, stats_g = model( text=tokens, text_lengths=tokens_lens, @@ -922,7 +922,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) diff --git a/egs/ljspeech/TTS/matcha/train.py b/egs/ljspeech/TTS/matcha/train.py index a25cc8723..853042413 100755 --- a/egs/ljspeech/TTS/matcha/train.py +++ b/egs/ljspeech/TTS/matcha/train.py @@ -17,7 +17,7 @@ from lhotse.utils import fix_random_seed from model import fix_len_compatibility from models.matcha_tts import MatchaTTS from tokenizer import Tokenizer -from torch.amp import GradScaler, autocast +from torch.cuda.amp import GradScaler, autocast from torch.nn.parallel import DistributedDataParallel as DDP from torch.optim import Optimizer from torch.utils.tensorboard import SummaryWriter @@ -474,7 +474,7 @@ def train_one_epoch( tokens_lens, ) = prepare_input(batch, tokenizer, device, params) try: - with autocast("cuda", enabled=params.use_fp16): + with autocast(enabled=params.use_fp16): losses = get_losses( { "x": tokens, @@ -645,7 +645,7 @@ def run(rank, world_size, args): valid_cuts = ljspeech.valid_cuts() valid_dl = ljspeech.valid_dataloaders(valid_cuts) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) diff --git a/egs/ljspeech/TTS/vits/train.py b/egs/ljspeech/TTS/vits/train.py index e9994319a..184ae79af 100755 --- a/egs/ljspeech/TTS/vits/train.py +++ b/egs/ljspeech/TTS/vits/train.py @@ -30,7 +30,7 @@ import torch.nn as nn from lhotse.cut import Cut from lhotse.utils import fix_random_seed from tokenizer import Tokenizer -from torch.amp import GradScaler, autocast +from torch.cuda.amp import GradScaler, autocast from torch.nn.parallel import DistributedDataParallel as DDP from torch.optim import Optimizer from torch.utils.tensorboard import SummaryWriter @@ -396,7 +396,7 @@ def train_one_epoch( loss_info["samples"] = batch_size try: - with autocast("cuda", enabled=params.use_fp16): + with autocast(enabled=params.use_fp16): # forward discriminator loss_d, stats_d = model( text=tokens, @@ -414,7 +414,7 @@ def train_one_epoch( scaler.scale(loss_d).backward() scaler.step(optimizer_d) - with autocast("cuda", enabled=params.use_fp16): + with autocast(enabled=params.use_fp16): # forward generator loss_g, stats_g = model( text=tokens, @@ -673,7 +673,7 @@ def scan_pessimistic_batches_for_oom( ) try: # for discriminator - with autocast("cuda", enabled=params.use_fp16): + with autocast(enabled=params.use_fp16): loss_d, stats_d = model( text=tokens, text_lengths=tokens_lens, @@ -686,7 +686,7 @@ def scan_pessimistic_batches_for_oom( optimizer_d.zero_grad() loss_d.backward() # for generator - with autocast("cuda", enabled=params.use_fp16): + with autocast(enabled=params.use_fp16): loss_g, stats_g = model( text=tokens, text_lengths=tokens_lens, @@ -838,7 +838,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) diff --git a/egs/ljspeech/TTS/vits/utils.py b/egs/ljspeech/TTS/vits/utils.py index d51ff5f5c..6a067f596 100644 --- a/egs/ljspeech/TTS/vits/utils.py +++ b/egs/ljspeech/TTS/vits/utils.py @@ -23,7 +23,7 @@ import torch import torch.distributed as dist import torch.nn as nn from lhotse.dataset.sampling.base import CutSampler -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.optim import Optimizer from torch.utils.tensorboard import SummaryWriter diff --git a/egs/ljspeech/TTS/vits/vits.py b/egs/ljspeech/TTS/vits/vits.py index 6fd6d219b..a1fabf9ad 100644 --- a/egs/ljspeech/TTS/vits/vits.py +++ b/egs/ljspeech/TTS/vits/vits.py @@ -25,7 +25,7 @@ from loss import ( KLDivergenceLoss, MelSpectrogramLoss, ) -from torch.amp import autocast +from torch.cuda.amp import autocast from utils import get_segments AVAILABLE_GENERATERS = { @@ -410,7 +410,7 @@ class VITS(nn.Module): p = self.discriminator(speech_) # calculate losses - with autocast("cuda", enabled=False): + with autocast(enabled=False): if not return_sample: mel_loss = self.mel_loss(speech_hat_, speech_) else: @@ -518,7 +518,7 @@ class VITS(nn.Module): p = self.discriminator(speech_) # calculate losses - with autocast("cuda", enabled=False): + with autocast(enabled=False): real_loss, fake_loss = self.discriminator_adv_loss(p_hat, p) loss = real_loss + fake_loss diff --git a/egs/mdcc/ASR/zipformer/train.py b/egs/mdcc/ASR/zipformer/train.py index 22249286a..730db7718 100755 --- a/egs/mdcc/ASR/zipformer/train.py +++ b/egs/mdcc/ASR/zipformer/train.py @@ -68,7 +68,7 @@ from optim import Eden, ScaledAdam from scaling import ScheduledFloat from subsampling import Conv2dSubsampling from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer2 @@ -906,7 +906,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1197,7 +1197,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1298,7 +1298,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/mgb2/ASR/pruned_transducer_stateless5/train.py b/egs/mgb2/ASR/pruned_transducer_stateless5/train.py index 916ada475..48468cfbd 100755 --- a/egs/mgb2/ASR/pruned_transducer_stateless5/train.py +++ b/egs/mgb2/ASR/pruned_transducer_stateless5/train.py @@ -66,7 +66,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, Eve from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.nn.utils import clip_grad_norm_ from torch.utils.tensorboard import SummaryWriter @@ -751,7 +751,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info, inf_flag = compute_loss( params=params, model=model, @@ -1012,7 +1012,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16) + scaler = GradScaler(enabled=params.use_fp16) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1115,7 +1115,7 @@ def scan_pessimistic_batches_for_oom( # warmup = 0.0 is so that the derivs for the pruned loss stay zero # (i.e. are not remembered by the decaying-average in adam), because # we want to avoid these params being subject to shrinkage in adam. - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _, _ = compute_loss( params=params, model=model, diff --git a/egs/multi_zh-hans/ASR/whisper/train.py b/egs/multi_zh-hans/ASR/whisper/train.py index 1a11d01af..fe2d950c1 100755 --- a/egs/multi_zh-hans/ASR/whisper/train.py +++ b/egs/multi_zh-hans/ASR/whisper/train.py @@ -61,7 +61,7 @@ from lhotse.utils import fix_random_seed from multi_dataset import MultiDataset from optim import Eden, ScaledAdam from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.functional import pad as pad_tensor from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -566,7 +566,7 @@ def compute_validation_loss( tot_loss = MetricsTracker() for batch_idx, batch in enumerate(valid_dl): - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, tokenizer=tokenizer, @@ -675,7 +675,7 @@ def train_one_epoch( ) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, tokenizer=tokenizer, @@ -913,7 +913,7 @@ def run(rank, world_size, args): valid_cuts = multi_dataset.dev_cuts() valid_dl = data_module.valid_dataloaders(valid_cuts) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) diff --git a/egs/multi_zh-hans/ASR/zipformer/train.py b/egs/multi_zh-hans/ASR/zipformer/train.py index 047253d5b..3dbfc48eb 100755 --- a/egs/multi_zh-hans/ASR/zipformer/train.py +++ b/egs/multi_zh-hans/ASR/zipformer/train.py @@ -75,7 +75,7 @@ from optim import Eden, ScaledAdam from scaling import ScheduledFloat from subsampling import Conv2dSubsampling from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer2 @@ -987,7 +987,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1278,7 +1278,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1378,7 +1378,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/multi_zh_en/ASR/zipformer/train.py b/egs/multi_zh_en/ASR/zipformer/train.py index 9e64defa3..04bb41214 100755 --- a/egs/multi_zh_en/ASR/zipformer/train.py +++ b/egs/multi_zh_en/ASR/zipformer/train.py @@ -75,7 +75,7 @@ from optim import Eden, ScaledAdam from scaling import ScheduledFloat from subsampling import Conv2dSubsampling from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer2 @@ -969,7 +969,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1269,7 +1269,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1369,7 +1369,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/reazonspeech/ASR/zipformer/do_not_use_it_directly.py b/egs/reazonspeech/ASR/zipformer/do_not_use_it_directly.py index c01e4d336..072679cfc 100755 --- a/egs/reazonspeech/ASR/zipformer/do_not_use_it_directly.py +++ b/egs/reazonspeech/ASR/zipformer/do_not_use_it_directly.py @@ -67,7 +67,7 @@ from model import Transducer from optim import Eden, ScaledAdam from tokenizer import Tokenizer from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer_for_ncnn_export_only import Zipformer @@ -822,7 +822,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1113,7 +1113,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1213,7 +1213,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/reazonspeech/ASR/zipformer/train.py b/egs/reazonspeech/ASR/zipformer/train.py index 8829a18ca..30bd3efba 100755 --- a/egs/reazonspeech/ASR/zipformer/train.py +++ b/egs/reazonspeech/ASR/zipformer/train.py @@ -74,7 +74,7 @@ from scaling import ScheduledFloat from subsampling import Conv2dSubsampling from tokenizer import Tokenizer from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer2 @@ -945,7 +945,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1235,7 +1235,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1335,7 +1335,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/speech_llm/ASR_LLM/whisper_llm_zh/train.py b/egs/speech_llm/ASR_LLM/whisper_llm_zh/train.py index 5de2cf2b0..5f224c984 100755 --- a/egs/speech_llm/ASR_LLM/whisper_llm_zh/train.py +++ b/egs/speech_llm/ASR_LLM/whisper_llm_zh/train.py @@ -451,7 +451,7 @@ def compute_validation_loss( tot_loss = MetricsTracker() for batch_idx, batch in enumerate(valid_dl): - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, tokenizer=tokenizer, @@ -566,7 +566,7 @@ def train_one_epoch( f"rm -rf {params.exp_dir}/epoch-{params.cur_epoch}-checkpoint-{batch_idx}" ) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, tokenizer=tokenizer, diff --git a/egs/spgispeech/ASR/pruned_transducer_stateless2/train.py b/egs/spgispeech/ASR/pruned_transducer_stateless2/train.py index 1e55ada87..a9146a0fe 100755 --- a/egs/spgispeech/ASR/pruned_transducer_stateless2/train.py +++ b/egs/spgispeech/ASR/pruned_transducer_stateless2/train.py @@ -65,7 +65,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, Eve from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -649,7 +649,7 @@ def train_one_epoch( params.batch_idx_train += 1 batch_size = len(batch["supervisions"]["text"]) - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -857,7 +857,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16) + scaler = GradScaler(enabled=params.use_fp16) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -957,7 +957,7 @@ def scan_pessimistic_batches_for_oom( # warmup = 0.0 is so that the derivs for the pruned loss stay zero # (i.e. are not remembered by the decaying-average in adam), because # we want to avoid these params being subject to shrinkage in adam. - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/spgispeech/ASR/zipformer/train.py b/egs/spgispeech/ASR/zipformer/train.py index 319713b02..dfc21c968 100755 --- a/egs/spgispeech/ASR/zipformer/train.py +++ b/egs/spgispeech/ASR/zipformer/train.py @@ -74,7 +74,7 @@ from optim import Eden, ScaledAdam from scaling import ScheduledFloat from subsampling import Conv2dSubsampling from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer2 @@ -946,7 +946,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1217,7 +1217,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1317,7 +1317,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/tal_csasr/ASR/pruned_transducer_stateless5/train.py b/egs/tal_csasr/ASR/pruned_transducer_stateless5/train.py index c44e30b89..c0aedd725 100755 --- a/egs/tal_csasr/ASR/pruned_transducer_stateless5/train.py +++ b/egs/tal_csasr/ASR/pruned_transducer_stateless5/train.py @@ -69,7 +69,7 @@ from local.tokenize_with_bpe_model import tokenize_by_bpe_model from model import Transducer from optim import Eden, Eve from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -726,7 +726,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) # print(batch["supervisions"]) - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -967,7 +967,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16) + scaler = GradScaler(enabled=params.use_fp16) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1039,7 +1039,7 @@ def scan_pessimistic_batches_for_oom( # warmup = 0.0 is so that the derivs for the pruned loss stay zero # (i.e. are not remembered by the decaying-average in adam), because # we want to avoid these params being subject to shrinkage in adam. - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/tal_csasr/ASR/pruned_transducer_stateless7_bbpe/train.py b/egs/tal_csasr/ASR/pruned_transducer_stateless7_bbpe/train.py index dd9576d99..2108266ec 100755 --- a/egs/tal_csasr/ASR/pruned_transducer_stateless7_bbpe/train.py +++ b/egs/tal_csasr/ASR/pruned_transducer_stateless7_bbpe/train.py @@ -64,7 +64,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, ScaledAdam from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer @@ -801,7 +801,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1101,7 +1101,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1201,7 +1201,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/tedlium3/ASR/conformer_ctc2/train.py b/egs/tedlium3/ASR/conformer_ctc2/train.py index 179dcf14a..fc3e3b2d9 100755 --- a/egs/tedlium3/ASR/conformer_ctc2/train.py +++ b/egs/tedlium3/ASR/conformer_ctc2/train.py @@ -57,7 +57,7 @@ from lhotse.dataset.sampling.base import CutSampler from lhotse.utils import fix_random_seed from local.convert_transcript_words_to_bpe_ids import convert_texts_into_ids from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -710,7 +710,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -941,7 +941,7 @@ def run(rank, world_size, args): warmup=0.0 if params.start_epoch == 1 else 1.0, ) - scaler = GradScaler("cuda", enabled=params.use_fp16) + scaler = GradScaler(enabled=params.use_fp16) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1011,7 +1011,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/tedlium3/ASR/zipformer/model.py b/egs/tedlium3/ASR/zipformer/model.py index 0d9b395ed..65b052ab9 100644 --- a/egs/tedlium3/ASR/zipformer/model.py +++ b/egs/tedlium3/ASR/zipformer/model.py @@ -173,7 +173,7 @@ class Transducer(nn.Module): # if self.training and random.random() < 0.25: # am = penalize_abs_values_gt(am, 30.0, 1.0e-04) - with torch.amp.autocast("cuda", enabled=False): + with torch.cuda.amp.autocast(enabled=False): simple_loss, (px_grad, py_grad) = k2.rnnt_loss_smoothed( lm=lm.float(), am=am.float(), @@ -209,7 +209,7 @@ class Transducer(nn.Module): # prior to do_rnnt_pruning (this is an optimization for speed). logits = self.joiner(am_pruned, lm_pruned, project_input=False) - with torch.amp.autocast("cuda", enabled=False): + with torch.cuda.amp.autocast(enabled=False): pruned_loss = k2.rnnt_loss_pruned( logits=logits.float(), symbols=y_padded, diff --git a/egs/tedlium3/ASR/zipformer/train.py b/egs/tedlium3/ASR/zipformer/train.py index ffe876863..14a44efb3 100755 --- a/egs/tedlium3/ASR/zipformer/train.py +++ b/egs/tedlium3/ASR/zipformer/train.py @@ -73,7 +73,7 @@ from optim import Eden, ScaledAdam from scaling import ScheduledFloat from subsampling import Conv2dSubsampling from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer2 @@ -911,7 +911,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1160,7 +1160,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1260,7 +1260,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/vctk/TTS/vits/train.py b/egs/vctk/TTS/vits/train.py index 6249640d4..4686de169 100755 --- a/egs/vctk/TTS/vits/train.py +++ b/egs/vctk/TTS/vits/train.py @@ -31,7 +31,7 @@ import torch.nn as nn from lhotse.cut import Cut from lhotse.utils import fix_random_seed from tokenizer import Tokenizer -from torch.amp import GradScaler, autocast +from torch.cuda.amp import GradScaler, autocast from torch.nn.parallel import DistributedDataParallel as DDP from torch.optim import Optimizer from torch.utils.tensorboard import SummaryWriter @@ -448,7 +448,7 @@ def train_one_epoch( loss_info["samples"] = batch_size try: - with autocast("cuda", enabled=params.use_fp16): + with autocast(enabled=params.use_fp16): # forward discriminator loss_d, stats_d = model( text=tokens, @@ -467,7 +467,7 @@ def train_one_epoch( scaler.scale(loss_d).backward() scaler.step(optimizer_d) - with autocast("cuda", enabled=params.use_fp16): + with autocast(enabled=params.use_fp16): # forward generator loss_g, stats_g = model( text=tokens, @@ -740,7 +740,7 @@ def scan_pessimistic_batches_for_oom( ) = prepare_input(batch, tokenizer, device, speaker_map) try: # for discriminator - with autocast("cuda", enabled=params.use_fp16): + with autocast(enabled=params.use_fp16): loss_d, stats_d = model( text=tokens, text_lengths=tokens_lens, @@ -754,7 +754,7 @@ def scan_pessimistic_batches_for_oom( optimizer_d.zero_grad() loss_d.backward() # for generator - with autocast("cuda", enabled=params.use_fp16): + with autocast(enabled=params.use_fp16): loss_g, stats_g = model( text=tokens, text_lengths=tokens_lens, @@ -910,7 +910,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) diff --git a/egs/wenetspeech/ASR/pruned_transducer_stateless2/finetune.py b/egs/wenetspeech/ASR/pruned_transducer_stateless2/finetune.py index 2fd6f6478..c34f1593d 100755 --- a/egs/wenetspeech/ASR/pruned_transducer_stateless2/finetune.py +++ b/egs/wenetspeech/ASR/pruned_transducer_stateless2/finetune.py @@ -52,7 +52,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, Eve from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -718,7 +718,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -907,7 +907,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16) + scaler = GradScaler(enabled=params.use_fp16) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1005,7 +1005,7 @@ def scan_pessimistic_batches_for_oom( # warmup = 0.0 is so that the derivs for the pruned loss stay zero # (i.e. are not remembered by the decaying-average in adam), because # we want to avoid these params being subject to shrinkage in adam. - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/wenetspeech/ASR/pruned_transducer_stateless2/train.py b/egs/wenetspeech/ASR/pruned_transducer_stateless2/train.py index c90f03f08..49977e01b 100644 --- a/egs/wenetspeech/ASR/pruned_transducer_stateless2/train.py +++ b/egs/wenetspeech/ASR/pruned_transducer_stateless2/train.py @@ -101,7 +101,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, Eve from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -687,7 +687,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -921,7 +921,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16) + scaler = GradScaler(enabled=params.use_fp16) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1019,7 +1019,7 @@ def scan_pessimistic_batches_for_oom( # warmup = 0.0 is so that the derivs for the pruned loss stay zero # (i.e. are not remembered by the decaying-average in adam), because # we want to avoid these params being subject to shrinkage in adam. - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/wenetspeech/ASR/pruned_transducer_stateless5/train.py b/egs/wenetspeech/ASR/pruned_transducer_stateless5/train.py index 7b05eca97..931e699d9 100755 --- a/egs/wenetspeech/ASR/pruned_transducer_stateless5/train.py +++ b/egs/wenetspeech/ASR/pruned_transducer_stateless5/train.py @@ -81,7 +81,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, Eve from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -796,7 +796,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1056,7 +1056,7 @@ def run(rank, world_size, args): warmup=0.0 if params.start_epoch == 1 else 1.0, ) - scaler = GradScaler("cuda", enabled=params.use_fp16) + scaler = GradScaler(enabled=params.use_fp16) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1158,7 +1158,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/wenetspeech/ASR/whisper/train.py b/egs/wenetspeech/ASR/whisper/train.py index c46a4d84c..4e55fd6a8 100644 --- a/egs/wenetspeech/ASR/whisper/train.py +++ b/egs/wenetspeech/ASR/whisper/train.py @@ -61,7 +61,7 @@ from lhotse.dataset.sampling.base import CutSampler from lhotse.utils import fix_random_seed from optim import Eden, ScaledAdam from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.functional import pad as pad_tensor from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -513,7 +513,7 @@ def compute_validation_loss( tot_loss = MetricsTracker() for batch_idx, batch in enumerate(valid_dl): - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, tokenizer=tokenizer, @@ -621,7 +621,7 @@ def train_one_epoch( f"rm -rf {params.exp_dir}/epoch-{params.cur_epoch}-checkpoint-{batch_idx}" ) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, tokenizer=tokenizer, @@ -843,7 +843,7 @@ def run(rank, world_size, args): train_dl = wenetspeech.train_dataloaders(train_cuts) valid_dl = wenetspeech.valid_dataloaders(wenetspeech.valid_cuts()) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) diff --git a/egs/wenetspeech/ASR/zipformer/train.py b/egs/wenetspeech/ASR/zipformer/train.py index b6d55447f..25b16f632 100755 --- a/egs/wenetspeech/ASR/zipformer/train.py +++ b/egs/wenetspeech/ASR/zipformer/train.py @@ -71,7 +71,7 @@ from optim import Eden, ScaledAdam from scaling import ScheduledFloat from subsampling import Conv2dSubsampling from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer2 @@ -910,7 +910,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1201,7 +1201,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1302,7 +1302,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/wenetspeech/KWS/zipformer/finetune.py b/egs/wenetspeech/KWS/zipformer/finetune.py index 00db4309d..d19172b38 100755 --- a/egs/wenetspeech/KWS/zipformer/finetune.py +++ b/egs/wenetspeech/KWS/zipformer/finetune.py @@ -82,7 +82,7 @@ from lhotse.cut import Cut, CutSet from lhotse.utils import fix_random_seed from optim import Eden, ScaledAdam from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from train import ( @@ -414,7 +414,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -703,7 +703,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) diff --git a/egs/wenetspeech/KWS/zipformer/train.py b/egs/wenetspeech/KWS/zipformer/train.py index 4dc30ad89..40960c2ae 100755 --- a/egs/wenetspeech/KWS/zipformer/train.py +++ b/egs/wenetspeech/KWS/zipformer/train.py @@ -73,7 +73,7 @@ from optim import Eden, ScaledAdam from scaling import ScheduledFloat from subsampling import Conv2dSubsampling from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer2 @@ -967,7 +967,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1252,7 +1252,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1353,7 +1353,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/wenetspeech4tts/TTS/valle/train.py b/egs/wenetspeech4tts/TTS/valle/train.py index 1c6972e93..e9ec548f3 100755 --- a/egs/wenetspeech4tts/TTS/valle/train.py +++ b/egs/wenetspeech4tts/TTS/valle/train.py @@ -65,7 +65,7 @@ from lhotse.utils import fix_random_seed from optim import Eden, ScaledAdam from tokenizer import TextTokenCollater, get_text_token_collater from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from tts_datamodule import TtsDataModule @@ -764,7 +764,7 @@ def train_one_epoch( batch_size = len(batch["text"]) try: - with torch.amp.autocast("cuda", dtype=dtype, enabled=enabled): + with torch.cuda.amp.autocast(dtype=dtype, enabled=enabled): _, loss, loss_info = compute_loss( params=params, model=model, @@ -897,7 +897,7 @@ def train_one_epoch( # Calculate validation loss in Rank 0 model.eval() logging.info("Computing validation loss") - with torch.amp.autocast("cuda", dtype=dtype): + with torch.cuda.amp.autocast(dtype=dtype): valid_info = compute_validation_loss( params=params, model=model, @@ -1102,9 +1102,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler( - "cuda", enabled=(params.dtype in ["fp16", "float16"]), init_scale=1.0 - ) + scaler = GradScaler(enabled=(params.dtype in ["fp16", "float16"]), init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1198,7 +1196,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", dtype=dtype): + with torch.cuda.amp.autocast(dtype=dtype): _, loss, _ = compute_loss( params=params, model=model, diff --git a/egs/xbmu_amdo31/ASR/pruned_transducer_stateless5/train.py b/egs/xbmu_amdo31/ASR/pruned_transducer_stateless5/train.py index 5c3000a57..a6fa46b17 100755 --- a/egs/xbmu_amdo31/ASR/pruned_transducer_stateless5/train.py +++ b/egs/xbmu_amdo31/ASR/pruned_transducer_stateless5/train.py @@ -68,7 +68,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, Eve from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter @@ -814,7 +814,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1072,7 +1072,7 @@ def run(rank, world_size, args): warmup=0.0 if params.start_epoch == 1 else 1.0, ) - scaler = GradScaler("cuda", enabled=params.use_fp16) + scaler = GradScaler(enabled=params.use_fp16) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1141,7 +1141,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/egs/xbmu_amdo31/ASR/pruned_transducer_stateless7/train.py b/egs/xbmu_amdo31/ASR/pruned_transducer_stateless7/train.py index a1b3be246..dd72551d9 100755 --- a/egs/xbmu_amdo31/ASR/pruned_transducer_stateless7/train.py +++ b/egs/xbmu_amdo31/ASR/pruned_transducer_stateless7/train.py @@ -67,7 +67,7 @@ from lhotse.utils import fix_random_seed from model import Transducer from optim import Eden, ScaledAdam from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer @@ -785,7 +785,7 @@ def train_one_epoch( batch_size = len(batch["supervisions"]["text"]) try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( params=params, model=model, @@ -1074,7 +1074,7 @@ def run(rank, world_size, args): params=params, ) - scaler = GradScaler("cuda", enabled=params.use_fp16, init_scale=1.0) + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) if checkpoints and "grad_scaler" in checkpoints: logging.info("Loading grad scaler state dict") scaler.load_state_dict(checkpoints["grad_scaler"]) @@ -1174,7 +1174,7 @@ def scan_pessimistic_batches_for_oom( for criterion, cuts in batches.items(): batch = train_dl.dataset[cuts] try: - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, _ = compute_loss( params=params, model=model, diff --git a/icefall/checkpoint.py b/icefall/checkpoint.py index b3a0fb865..d31ce1301 100644 --- a/icefall/checkpoint.py +++ b/icefall/checkpoint.py @@ -27,7 +27,7 @@ import torch import torch.nn as nn from lhotse.dataset.sampling.base import CutSampler from torch import Tensor -from torch.amp import GradScaler +from torch.cuda.amp import GradScaler from torch.nn.parallel import DistributedDataParallel as DDP from torch.optim import Optimizer diff --git a/icefall/rnn_lm/train.py b/icefall/rnn_lm/train.py index 257cdb09a..0178b80bf 100755 --- a/icefall/rnn_lm/train.py +++ b/icefall/rnn_lm/train.py @@ -401,7 +401,7 @@ def compute_validation_loss( for batch_idx, batch in enumerate(valid_dl): x, y, sentence_lengths = batch - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( model=model, x=x, @@ -470,7 +470,7 @@ def train_one_epoch( params.batch_idx_train += 1 x, y, sentence_lengths = batch batch_size = x.size(0) - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( model=model, x=x, diff --git a/icefall/transformer_lm/train.py b/icefall/transformer_lm/train.py index 6faa63484..c36abfcdf 100644 --- a/icefall/transformer_lm/train.py +++ b/icefall/transformer_lm/train.py @@ -341,7 +341,7 @@ def compute_validation_loss( for batch_idx, batch in enumerate(valid_dl): x, y, sentence_lengths = batch - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( model=model, x=x, @@ -403,7 +403,7 @@ def train_one_epoch( params.batch_idx_train += 1 x, y, sentence_lengths = batch batch_size = x.size(0) - with torch.amp.autocast("cuda", enabled=params.use_fp16): + with torch.cuda.amp.autocast(enabled=params.use_fp16): loss, loss_info = compute_loss( model=model, x=x, From 92ed1708c0786e31f59628408869078b330faffa Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Wed, 18 Dec 2024 16:50:14 +0800 Subject: [PATCH 15/59] Add torch 1.13 and 2.0 to CI tests (#1840) --- .github/scripts/docker/generate_build_matrix.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/scripts/docker/generate_build_matrix.py b/.github/scripts/docker/generate_build_matrix.py index 9c53a38df..c5a1a54cb 100755 --- a/.github/scripts/docker/generate_build_matrix.py +++ b/.github/scripts/docker/generate_build_matrix.py @@ -45,13 +45,13 @@ def get_torchaudio_version(torch_version): def get_matrix(): k2_version = "1.24.4.dev20241029" kaldifeat_version = "1.25.5.dev20241029" - version = "20241029" + version = "20241218" # torchaudio 2.5.0 does not support python 3.13 python_version = ["3.8", "3.9", "3.10", "3.11", "3.12"] torch_version = [] - # torch_version += ["1.13.0", "1.13.1"] - # torch_version += ["2.0.0", "2.0.1"] + torch_version += ["1.13.0", "1.13.1"] + torch_version += ["2.0.0", "2.0.1"] # torch_version += ["2.1.0", "2.1.1", "2.1.2"] # torch_version += ["2.2.0", "2.2.1", "2.2.2"] # Test only torch >= 2.3.0 @@ -59,6 +59,7 @@ def get_matrix(): torch_version += ["2.4.0"] torch_version += ["2.4.1"] torch_version += ["2.5.0"] + torch_version += ["2.5.1"] matrix = [] for p in python_version: @@ -79,8 +80,12 @@ def get_matrix(): # torch>=2.5 requires python 3.10 continue - k2_version_2 = k2_version - kaldifeat_version_2 = kaldifeat_version + if t == "2.5.1": + k2_version_2 = "1.24.4.dev20241122" + kaldifeat_version_2 = "1.25.5.dev20241126" + else: + k2_version_2 = k2_version + kaldifeat_version_2 = kaldifeat_version matrix.append( { From ad966fb81d76c9b6780cac6844d9c4aa1782a46b Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Thu, 19 Dec 2024 15:19:41 +0800 Subject: [PATCH 16/59] Minor fixes to the onnx inference script for ljspeech matcha-tts. (#1838) --- .github/scripts/ljspeech/TTS/run-matcha.sh | 20 +++++++++++++------- egs/ljspeech/TTS/matcha/export_onnx.py | 2 +- egs/ljspeech/TTS/matcha/onnx_pretrained.py | 19 ++++++++++++++----- 3 files changed, 28 insertions(+), 13 deletions(-) diff --git a/.github/scripts/ljspeech/TTS/run-matcha.sh b/.github/scripts/ljspeech/TTS/run-matcha.sh index 0876cb47f..352d685a0 100755 --- a/.github/scripts/ljspeech/TTS/run-matcha.sh +++ b/.github/scripts/ljspeech/TTS/run-matcha.sh @@ -57,6 +57,7 @@ function infer() { curl -SL -O https://github.com/csukuangfj/models/raw/refs/heads/master/hifigan/generator_v1 ./matcha/infer.py \ + --num-buckets 2 \ --epoch 1 \ --exp-dir ./matcha/exp \ --tokens data/tokens.txt \ @@ -97,19 +98,23 @@ function export_onnx() { python3 ./matcha/export_onnx_hifigan.py else curl -SL -O https://huggingface.co/csukuangfj/icefall-tts-ljspeech-matcha-en-2024-10-28/resolve/main/exp/hifigan_v1.onnx + curl -SL -O https://huggingface.co/csukuangfj/icefall-tts-ljspeech-matcha-en-2024-10-28/resolve/main/exp/hifigan_v2.onnx + curl -SL -O https://huggingface.co/csukuangfj/icefall-tts-ljspeech-matcha-en-2024-10-28/resolve/main/exp/hifigan_v3.onnx fi ls -lh *.onnx - python3 ./matcha/onnx_pretrained.py \ - --acoustic-model ./model-steps-6.onnx \ - --vocoder ./hifigan_v1.onnx \ - --tokens ./data/tokens.txt \ - --input-text "how are you doing?" \ - --output-wav /icefall/generated-matcha-tts-steps-6-v1.wav + for v in v1 v2 v3; do + python3 ./matcha/onnx_pretrained.py \ + --acoustic-model ./model-steps-6.onnx \ + --vocoder ./hifigan_$v.onnx \ + --tokens ./data/tokens.txt \ + --input-text "how are you doing?" \ + --output-wav /icefall/generated-matcha-tts-steps-6-$v.wav + done ls -lh /icefall/*.wav - soxi /icefall/generated-matcha-tts-steps-6-v1.wav + soxi /icefall/generated-matcha-tts-steps-6-*.wav } prepare_data @@ -118,3 +123,4 @@ infer export_onnx rm -rfv generator_v* matcha/exp +git checkout . diff --git a/egs/ljspeech/TTS/matcha/export_onnx.py b/egs/ljspeech/TTS/matcha/export_onnx.py index 487ea2995..623517431 100755 --- a/egs/ljspeech/TTS/matcha/export_onnx.py +++ b/egs/ljspeech/TTS/matcha/export_onnx.py @@ -163,7 +163,7 @@ def main(): (x, x_lengths, temperature, length_scale), filename, opset_version=opset_version, - input_names=["x", "x_length", "temperature", "length_scale"], + input_names=["x", "x_length", "noise_scale", "length_scale"], output_names=["mel"], dynamic_axes={ "x": {0: "N", 1: "L"}, diff --git a/egs/ljspeech/TTS/matcha/onnx_pretrained.py b/egs/ljspeech/TTS/matcha/onnx_pretrained.py index 4eff9a084..6d92b16eb 100755 --- a/egs/ljspeech/TTS/matcha/onnx_pretrained.py +++ b/egs/ljspeech/TTS/matcha/onnx_pretrained.py @@ -89,6 +89,7 @@ class OnnxHifiGANModel: self.model.get_inputs()[0].name: x.numpy(), }, )[0] + # audio: (batch_size, num_samples) return torch.from_numpy(audio) @@ -97,19 +98,24 @@ class OnnxModel: def __init__( self, filename: str, + tokens: str, ): session_opts = ort.SessionOptions() session_opts.inter_op_num_threads = 1 session_opts.intra_op_num_threads = 2 self.session_opts = session_opts - self.tokenizer = Tokenizer("./data/tokens.txt") + self.tokenizer = Tokenizer(tokens) self.model = ort.InferenceSession( filename, sess_options=self.session_opts, providers=["CPUExecutionProvider"], ) + logging.info(f"{self.model.get_modelmeta().custom_metadata_map}") + metadata = self.model.get_modelmeta().custom_metadata_map + self.sample_rate = int(metadata["sample_rate"]) + for i in self.model.get_inputs(): print(i) @@ -138,6 +144,7 @@ class OnnxModel: self.model.get_inputs()[3].name: length_scale.numpy(), }, )[0] + # mel: (batch_size, feat_dim, num_frames) return torch.from_numpy(mel) @@ -147,7 +154,7 @@ def main(): params = get_parser().parse_args() logging.info(vars(params)) - model = OnnxModel(params.acoustic_model) + model = OnnxModel(params.acoustic_model, params.tokens) vocoder = OnnxHifiGANModel(params.vocoder) text = params.input_text x = model.tokenizer.texts_to_token_ids([text], add_sos=True, add_eos=True) @@ -164,15 +171,17 @@ def main(): print("audio", audio.shape) # (1, 1, num_samples) audio = audio.squeeze() + sample_rate = model.sample_rate + t = (end_t - start_t).total_seconds() t2 = (end_t2 - start_t2).total_seconds() - rtf_am = t * 22050 / audio.shape[-1] - rtf_vocoder = t2 * 22050 / audio.shape[-1] + rtf_am = t * sample_rate / audio.shape[-1] + rtf_vocoder = t2 * sample_rate / audio.shape[-1] print("RTF for acoustic model ", rtf_am) print("RTF for vocoder", rtf_vocoder) # skip denoiser - sf.write(params.output_wav, audio, 22050, "PCM_16") + sf.write(params.output_wav, audio, sample_rate, "PCM_16") logging.info(f"Saved to {params.output_wav}") From 57e9f2a8db43eaa62d8701ef456f8323e9bcb8ff Mon Sep 17 00:00:00 2001 From: Han Zhu <1106766460@qq.com> Date: Mon, 30 Dec 2024 15:27:05 +0800 Subject: [PATCH 17/59] Add the "rms-sort" diagnostics (#1851) --- icefall/diagnostics.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/icefall/diagnostics.py b/icefall/diagnostics.py index 37872f233..e5eaba619 100644 --- a/icefall/diagnostics.py +++ b/icefall/diagnostics.py @@ -63,12 +63,22 @@ def get_tensor_stats( "rms" -> square before summing, we'll take sqrt later "value" -> just sum x itself "max", "min" -> take the maximum or minimum [over all other dims but dim] instead of summing + "rms-sort" -> this is a bit different than the others, it's based on computing the + rms over the specified dim and returning percentiles of the result (11 of them). Returns: stats: a Tensor of shape (x.shape[dim],). count: an integer saying how many items were counted in each element of stats. """ + if stats_type == "rms-sort": + rms = (x**2).mean(dim=dim).sqrt() + rms = rms.flatten() + rms = rms.sort()[0] + rms = rms[(torch.arange(11) * rms.numel() // 10).clamp(max=rms.numel() - 1)] + count = 1.0 + return rms, count + count = x.numel() // x.shape[dim] if stats_type == "eigs": @@ -164,7 +174,17 @@ class TensorDiagnostic(object): for dim in range(ndim): this_dim_stats = self.stats[dim] if ndim > 1: - stats_types = ["abs", "max", "min", "positive", "value", "rms"] + # rms-sort is different from the others, it's based on summing over just this + # dim, then sorting and returning the percentiles. + stats_types = [ + "abs", + "max", + "min", + "positive", + "value", + "rms", + "rms-sort", + ] if x.shape[dim] <= self.opts.max_eig_dim: stats_types.append("eigs") else: From 48088cb80703e3e5a98a1cab7558669b77875233 Mon Sep 17 00:00:00 2001 From: Han Zhu <1106766460@qq.com> Date: Mon, 30 Dec 2024 15:30:02 +0800 Subject: [PATCH 18/59] Refactor optimizer (#1837) * Print indexes of largest grad --- egs/librispeech/ASR/zipformer/optim.py | 445 ++++++++++++------------- 1 file changed, 212 insertions(+), 233 deletions(-) diff --git a/egs/librispeech/ASR/zipformer/optim.py b/egs/librispeech/ASR/zipformer/optim.py index 8434fab13..8a1764651 100644 --- a/egs/librispeech/ASR/zipformer/optim.py +++ b/egs/librispeech/ASR/zipformer/optim.py @@ -121,6 +121,139 @@ class BatchedOptimizer(Optimizer): p.copy_(stacked_params[i]) +def basic_step(group, p, state, grad): + # computes basic Adam update using beta2 (dividing by gradient stddev) only. no momentum yet. + lr = group["lr"] + if p.numel() == p.shape[0]: + lr = lr * group["scalar_lr_scale"] + beta2 = group["betas"][1] + eps = group["eps"] + # p shape: (batch_size,) or (batch_size, 1, [1,..]) + try: + exp_avg_sq = state[ + "exp_avg_sq" + ] # shape: (batch_size,) or (batch_size, 1, [1,..]) + except KeyError: + exp_avg_sq = torch.zeros(*p.shape, device=p.device, dtype=torch.float) + state["exp_avg_sq"] = exp_avg_sq + + exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value=1 - beta2) + + # bias_correction2 is like in Adam. + # slower update at the start will help stability anyway. + bias_correction2 = 1 - beta2 ** (state["step"] + 1) + if bias_correction2 < 0.99: + # note: not in-place. + exp_avg_sq = exp_avg_sq * (1.0 / bias_correction2) + denom = exp_avg_sq.sqrt().add_(eps) + + return -lr * grad / denom + + +def scaling_step(group, p, state, grad): + delta = basic_step(group, p, state, grad) + if p.numel() == p.shape[0]: + return delta # there is no scaling for scalar parameters. (p.shape[0] is the batch of parameters.) + + step = state["step"] + size_update_period = group["size_update_period"] + + try: + param_rms = state["param_rms"] + scale_grads = state["scale_grads"] + scale_exp_avg_sq = state["scale_exp_avg_sq"] + except KeyError: + # we know p.ndim > 1 because we'd have returned above if not, so don't worry + # about the speial case of dim=[] that pytorch treats inconsistently. + param_rms = (p**2).mean(dim=list(range(1, p.ndim)), keepdim=True).sqrt() + param_rms = param_rms.to(torch.float) + scale_exp_avg_sq = torch.zeros_like(param_rms) + scale_grads = torch.zeros( + size_update_period, *param_rms.shape, dtype=torch.float, device=p.device + ) + state["param_rms"] = param_rms + state["scale_grads"] = scale_grads + state["scale_exp_avg_sq"] = scale_exp_avg_sq + + # on every step, update the gradient w.r.t. the scale of the parameter, we + # store these as a batch and periodically update the size (for speed only, to + # avoid too many operations). + scale_grads[step % size_update_period] = (p * grad).sum( + dim=list(range(1, p.ndim)), keepdim=True + ) + + # periodically recompute the value of param_rms. + if step % size_update_period == size_update_period - 1: + param_rms.copy_((p**2).mean(dim=list(range(1, p.ndim)), keepdim=True).sqrt()) + + param_min_rms = group["param_min_rms"] + + # scale the step size by param_rms. This is the most important "scaling" part of + # ScaledAdam + delta *= param_rms.clamp(min=param_min_rms) + + if step % size_update_period == size_update_period - 1 and step > 0: + # This block updates the size of parameter by adding a step ("delta") value in + # the direction of either shrinking or growing it. + beta2 = group["betas"][1] + size_lr = group["lr"] * group["scalar_lr_scale"] + param_max_rms = group["param_max_rms"] + eps = group["eps"] + batch_size = p.shape[0] + # correct beta2 for the size update period: we will have + # faster decay at this level. + beta2_corr = beta2**size_update_period + scale_exp_avg_sq.mul_(beta2_corr).add_( + (scale_grads**2).mean(dim=0), # mean over dim `size_update_period` + alpha=1 - beta2_corr, + ) # shape is (batch_size, 1, 1, ...) + + # The 1st time we reach here is when size_step == 1. + size_step = (step + 1) // size_update_period + bias_correction2 = 1 - beta2_corr**size_step + + denom = scale_exp_avg_sq.sqrt() + eps + + scale_step = ( + -size_lr * (bias_correction2**0.5) * scale_grads.sum(dim=0) / denom + ) + + is_too_small = param_rms < param_min_rms + + # when the param gets too small, just don't shrink it any further. + scale_step.masked_fill_(is_too_small, 0.0) + + # The following may help prevent instability: don't allow the scale step to be too large in + # either direction. + scale_step.clamp_(min=-0.1, max=0.1) + + # and ensure the parameter rms after update never exceeds param_max_rms. + # We have to look at the trained model for parameters at or around the + # param_max_rms, because sometimes they can indicate a problem with the + # topology or settings. + scale_step = torch.minimum(scale_step, (param_max_rms - param_rms) / param_rms) + + delta.add_(p * scale_step) + + return delta + + +def momentum_step(group, p, state, grad): + delta = scaling_step(group, p, state, grad) + beta1 = group["betas"][0] + try: + stored_delta = state["delta"] + except KeyError: + stored_delta = torch.zeros(*p.shape, device=p.device, dtype=torch.float) + state["delta"] = stored_delta + stored_delta.mul_(beta1) + stored_delta.add_(delta, alpha=(1 - beta1)) + # we don't bother doing the "bias correction" part of Adam for beta1 because this is just + # an edge effect that affects the first 10 or so batches; and the effect of not doing it + # is just to do a slower update for the first few batches, which will help stability. + return stored_delta + + class ScaledAdam(BatchedOptimizer): """ Implements 'Scaled Adam', a variant of Adam where we scale each parameter's update @@ -352,58 +485,26 @@ class ScaledAdam(BatchedOptimizer): raise RuntimeError( "ScaledAdam optimizer does not support sparse gradients" ) - # State initialization - if len(state) == 0: - self._init_state(group, p, state) - self._step_one_batch(group, p, state, clipping_scale) + try: + cur_step = state["step"] + except KeyError: + state["step"] = 0 + cur_step = 0 + + grad = ( + p.grad if clipping_scale == 1.0 else p.grad.mul_(clipping_scale) + ) + p += momentum_step(group, p.detach(), state, grad) + + if p.numel() == p.shape[0]: # scalar parameter + scalar_max = group["scalar_max"] + p.clamp_(min=-scalar_max, max=scalar_max) + + state["step"] = cur_step + 1 return loss - def _init_state(self, group: dict, p: Tensor, state: dict): - """ - Initializes state dict for parameter 'p'. Assumes that dim 0 of tensor p - is actually the batch dimension, corresponding to batched-together - parameters of a given shape. - - - Args: - group: Dict to look up configuration values. - p: The parameter that we are initializing the state for - state: Dict from string to whatever state we are initializing - """ - size_update_period = group["size_update_period"] - - state["step"] = 0 - - kwargs = {"device": p.device, "dtype": p.dtype} - - # 'delta' implements conventional momentum. There are - # several different kinds of update going on, so rather than - # compute "exp_avg" like in Adam, we store and decay a - # parameter-change "delta", which combines all forms of - # update. this is equivalent to how it's done in Adam, - # except for the first few steps. - state["delta"] = torch.zeros_like(p, memory_format=torch.preserve_format) - - batch_size = p.shape[0] - numel = p.numel() // batch_size - - if numel > 1: - # "param_rms" just periodically records the scalar root-mean-square value of - # the parameter tensor. - # it has a shape like (batch_size, 1, 1, 1, 1) - param_rms = (p**2).mean(dim=list(range(1, p.ndim)), keepdim=True).sqrt() - state["param_rms"] = param_rms - - state["scale_exp_avg_sq"] = torch.zeros_like(param_rms) - state["scale_grads"] = torch.zeros( - size_update_period, *param_rms.shape, **kwargs - ) - - # exp_avg_sq is the weighted sum of scaled gradients. as in Adam. - state["exp_avg_sq"] = torch.zeros_like(p, memory_format=torch.preserve_format) - def _get_clipping_scale( self, group: dict, tuples: List[Tuple[Tensor, dict, List[str]]] ) -> float: @@ -484,7 +585,7 @@ class ScaledAdam(BatchedOptimizer): ) first_state["num_clipped"] = 0 quartiles = " ".join(["%.3e" % x for x in quartiles]) - logging.warn( + logging.warning( f"Clipping_scale={clipping_scale}, grad-norm quartiles {quartiles}, " f"threshold={threshold:.3e}, percent-clipped={percent_clipped:.1f}" ) @@ -499,8 +600,8 @@ class ScaledAdam(BatchedOptimizer): ans = 0.0 if ans < 1.0: first_state["num_clipped"] += 1 - if ans < 0.1: - logging.warn( + if ans < 0.5: + logging.warning( f"Scaling gradients by {ans}, model_norm_threshold={model_norm_threshold}" ) if self.show_dominant_parameters: @@ -508,6 +609,7 @@ class ScaledAdam(BatchedOptimizer): self._show_gradient_dominating_parameter( tuples, tot_sumsq, group["scalar_lr_scale"] ) + self._show_param_with_unusual_grad(tuples) if ans == 0.0: for (p, state, param_names) in tuples: @@ -515,6 +617,55 @@ class ScaledAdam(BatchedOptimizer): return ans + def _show_param_with_unusual_grad( + self, + tuples: List[Tuple[Tensor, dict, List[str]]], + ): + """ + Print information about parameter which has the largest ratio of grad-on-this-batch + divided by normal grad size. + tuples: a list of tuples of (param, state, param_names) + where param is a batched set of parameters, + with a .grad (1st dim is batch dim) + and state is the state-dict where optimization parameters are kept. + param_names is a List[str] while each str is name for a parameter + in batched set of parameters "param". + """ + largest_ratio = 0.0 + largest_name = "" + # ratios_names is a list of 3-tuples: (grad_ratio, param_name, tensor) + ratios_names = [] + for (p, state, batch_param_names) in tuples: + dims = list(range(1, p.ndim)) + + def mean(x): + # workaround for bad interface of torch's "mean" for when dims is the empty list. + if len(dims) > 0: + return x.mean(dim=dims) + else: + return x + + grad_ratio = ( + (mean(p.grad**2) / state["exp_avg_sq"].mean(dim=dims)) + .sqrt() + .to("cpu") + ) + + ratios_names += zip( + grad_ratio.tolist(), batch_param_names, p.grad.unbind(dim=0) + ) + + ratios_names = sorted(ratios_names, reverse=True) + ratios_names = ratios_names[:10] + ratios_names = [ + (ratio, name, largest_index(tensor)) + for (ratio, name, tensor) in ratios_names + ] + + logging.warning( + f"Parameters with most larger-than-usual grads, with ratios, are: {ratios_names}" + ) + def _show_gradient_dominating_parameter( self, tuples: List[Tuple[Tensor, dict, List[str]]], @@ -572,7 +723,7 @@ class ScaledAdam(BatchedOptimizer): dominant_rms, dominant_grad, ) = sorted_by_proportion[dominant_param_name] - logging.warn( + logging.warning( f"Parameter dominating tot_sumsq {dominant_param_name}" f" with proportion {dominant_proportion:.2f}," f" where dominant_sumsq=(grad_sumsq*orig_rms_sq)" @@ -581,182 +732,11 @@ class ScaledAdam(BatchedOptimizer): f" orig_rms_sq={(dominant_rms**2).item():.3e}" ) - def _step_one_batch( - self, group: dict, p: Tensor, state: dict, clipping_scale: float - ): - """ - Do the step for one parameter, which is actually going to be a batch of - `real` parameters, with dim 0 as the batch dim. - Args: - group: dict to look up configuration values - p: parameter to update (actually multiple parameters stacked together - as a batch) - state: state-dict for p, to look up the optimizer state - """ - lr = group["lr"] - size_update_period = group["size_update_period"] - beta1 = group["betas"][0] - grad = p.grad - if clipping_scale != 1.0: - grad *= clipping_scale - step = state["step"] - delta = state["delta"] - - delta.mul_(beta1) - batch_size = p.shape[0] - numel = p.numel() // batch_size - if numel > 1: - # Update the size/scale of p, and set param_rms - scale_grads = state["scale_grads"] - scale_grads[step % size_update_period] = (p * grad).sum( - dim=list(range(1, p.ndim)), keepdim=True - ) - if step % size_update_period == size_update_period - 1: - param_rms = state["param_rms"] # shape: (batch_size, 1, 1, ..) - param_rms.copy_( - (p**2).mean(dim=list(range(1, p.ndim)), keepdim=True).sqrt() - ) - if step > 0: - # self._size_update() learns the overall scale on the - # parameter, by shrinking or expanding it. - self._size_update(group, scale_grads, p, state) - - if numel == 1: - # For parameters with 1 element we just use regular Adam. - # Updates delta. - self._step_scalar(group, p, state) - else: - self._step(group, p, state) - - state["step"] = step + 1 - - def _size_update( - self, group: dict, scale_grads: Tensor, p: Tensor, state: dict - ) -> None: - """ - Called only where p.numel() > 1, this updates the scale of the parameter. - If we imagine: p = underlying_param * scale.exp(), and we are doing - gradient descent on underlying param and on scale, this function does the update - on `scale`. - - Args: - group: dict to look up configuration values - scale_grads: a tensor of shape (size_update_period, batch_size, 1, 1,...) containing - grads w.r.t. the scales. - p: The parameter to update - state: The state-dict of p - """ - - param_rms = state["param_rms"] - beta1, beta2 = group["betas"] - size_lr = group["lr"] * group["scalar_lr_scale"] - param_min_rms = group["param_min_rms"] - param_max_rms = group["param_max_rms"] - eps = group["eps"] - step = state["step"] - batch_size = p.shape[0] - - size_update_period = scale_grads.shape[0] - # correct beta2 for the size update period: we will have - # faster decay at this level. - beta2_corr = beta2**size_update_period - - scale_exp_avg_sq = state["scale_exp_avg_sq"] # shape: (batch_size, 1, 1, ..) - scale_exp_avg_sq.mul_(beta2_corr).add_( - (scale_grads**2).mean(dim=0), # mean over dim `size_update_period` - alpha=1 - beta2_corr, - ) # shape is (batch_size, 1, 1, ...) - - # The 1st time we reach here is when size_step == 1. - size_step = (step + 1) // size_update_period - bias_correction2 = 1 - beta2_corr**size_step - # we don't bother with bias_correction1; this will help prevent divergence - # at the start of training. - - denom = scale_exp_avg_sq.sqrt() + eps - - scale_step = ( - -size_lr * (bias_correction2**0.5) * scale_grads.sum(dim=0) / denom - ) - - is_too_small = param_rms < param_min_rms - - # when the param gets too small, just don't shrink it any further. - scale_step.masked_fill_(is_too_small, 0.0) - - # and ensure the parameter rms after update never exceeds param_max_rms. - # We have to look at the trained model for parameters at or around the - # param_max_rms, because sometimes they can indicate a problem with the - # topology or settings. - scale_step = torch.minimum(scale_step, (param_max_rms - param_rms) / param_rms) - - delta = state["delta"] - # the factor of (1-beta1) relates to momentum. - delta.add_(p * scale_step, alpha=(1 - beta1)) - - def _step(self, group: dict, p: Tensor, state: dict): - """ - This function does the core update of self.step(), in the case where the members of - the batch have more than 1 element. - - Args: - group: A dict which will be used to look up configuration values - p: The parameter to be updated - grad: The grad of p - state: The state-dict corresponding to parameter p - - This function modifies p. - """ - grad = p.grad - lr = group["lr"] - beta1, beta2 = group["betas"] - eps = group["eps"] - param_min_rms = group["param_min_rms"] - step = state["step"] - - exp_avg_sq = state["exp_avg_sq"] - exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value=(1 - beta2)) - - this_step = state["step"] - (state["zero_step"] if "zero_step" in state else 0) - bias_correction2 = 1 - beta2 ** (this_step + 1) - if bias_correction2 < 0.99: - # note: not in-place. - exp_avg_sq = exp_avg_sq * (1.0 / bias_correction2) - - denom = exp_avg_sq.sqrt() - denom += eps - grad = grad / denom - - alpha = -lr * (1 - beta1) * state["param_rms"].clamp(min=param_min_rms) - - delta = state["delta"] - delta.add_(grad * alpha) - p.add_(delta) - - def _step_scalar(self, group: dict, p: Tensor, state: dict): - """ - A simplified form of the core update for scalar tensors, where we cannot get a good - estimate of the parameter rms. - """ - beta1, beta2 = group["betas"] - scalar_max = group["scalar_max"] - eps = group["eps"] - lr = group["lr"] * group["scalar_lr_scale"] - grad = p.grad - - exp_avg_sq = state["exp_avg_sq"] # shape: (batch_size,) - exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value=1 - beta2) - - # bias_correction2 is like in Adam. Don't bother with bias_correction1; - # slower update at the start will help stability anyway. - bias_correction2 = 1 - beta2 ** (state["step"] + 1) - denom = (exp_avg_sq / bias_correction2).sqrt() + eps - - delta = state["delta"] - delta.add_(grad / denom, alpha=-lr * (1 - beta1)) - p.clamp_(min=-scalar_max, max=scalar_max) - p.add_(delta) +def largest_index(x: Tensor): + x = x.contiguous() + argmax = x.abs().argmax().item() + return [(argmax // x.stride(i)) % x.size(i) for i in range(x.ndim)] class LRScheduler(object): @@ -787,9 +767,9 @@ class LRScheduler(object): is not the optimizer. """ return { - # the user might try to override the base_lr, so don't include this in the state. - # previously they were included. - # "base_lrs": self.base_lrs, + # the user might try to override the base_lr, so don't include this in the state. + # previously they were included. + # "base_lrs": self.base_lrs, "epoch": self.epoch, "batch": self.batch, } @@ -807,7 +787,6 @@ class LRScheduler(object): self.__dict__.update(state_dict) self.base_lrs = base_lrs - def get_last_lr(self) -> List[float]: """Return last computed learning rate by current scheduler. Will be a list of float.""" return self._last_lr @@ -853,7 +832,7 @@ class LRScheduler(object): def print_lr(self, is_verbose, group, lr): """Display the current learning rate.""" if is_verbose: - logging.warn( + logging.warning( f"Epoch={self.epoch}, batch={self.batch}: adjusting learning rate" f" of group {group} to {lr:.4e}." ) @@ -1184,7 +1163,7 @@ def _test_scaled_adam(hidden_dim: int): if iter == 0: optim = Eve(m.parameters(), lr=0.003) elif iter == 1: - optim = ScaledAdam(m.parameters(), lr=0.03, clipping_scale=2.0) + optim = ScaledAdam(m.named_parameters(), lr=0.03, clipping_scale=2.0) scheduler = Eden(optim, lr_batches=200, lr_epochs=5, verbose=False) start = timeit.default_timer() From a2b0f6057c41a1f0eff64ed95356d14751a0c791 Mon Sep 17 00:00:00 2001 From: Yifan Yang <64255737+yfyeung@users.noreply.github.com> Date: Tue, 31 Dec 2024 07:41:44 +0800 Subject: [PATCH 19/59] Small fix (#1853) --- egs/wenetspeech4tts/TTS/prepare.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/egs/wenetspeech4tts/TTS/prepare.sh b/egs/wenetspeech4tts/TTS/prepare.sh index 54e140dbb..3d7ffadb1 100755 --- a/egs/wenetspeech4tts/TTS/prepare.sh +++ b/egs/wenetspeech4tts/TTS/prepare.sh @@ -10,9 +10,9 @@ stop_stage=4 dl_dir=$PWD/download -dataset_parts="Premium" # Basic for all 10k hours data, Premium for about 10% of the data +dataset_parts="Premium" # Basic for all 7226 hours data, Premium for 945 hours subset. -text_extractor="pypinyin_initials_finals" # default is espeak for English +text_extractor="pypinyin_initials_finals" # default is espeak for English audio_extractor="Encodec" # or Fbank audio_feats_dir=data/tokenized @@ -63,7 +63,7 @@ if [ $stage -le 2 ] && [ $stop_stage -ge 2 ]; then --audio-extractor ${audio_extractor} \ --batch-duration 2500 --prefix "wenetspeech4tts" \ --src-dir "data/manifests" \ - --split 100 \ + --split 100 \ --output-dir "${audio_feats_dir}/wenetspeech4tts_${dataset_parts}_split_100" cp ${audio_feats_dir}/wenetspeech4tts_${dataset_parts}_split_100/unique_text_tokens.k2symbols ${audio_feats_dir} fi From df46a3eaf94c0089c485eb23b0e08bf2b63cb53a Mon Sep 17 00:00:00 2001 From: Han Zhu <1106766460@qq.com> Date: Tue, 31 Dec 2024 16:52:06 +0800 Subject: [PATCH 20/59] Warn instead of raising exceptions in inf-check (#1852) --- icefall/hooks.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/icefall/hooks.py b/icefall/hooks.py index 1c5bd2ae6..83f2750fa 100644 --- a/icefall/hooks.py +++ b/icefall/hooks.py @@ -40,9 +40,7 @@ def register_inf_check_hooks(model: nn.Module) -> None: def forward_hook(_module, _input, _output, _name=name): if isinstance(_output, Tensor): if not torch.isfinite(_output.to(torch.float32).sum()): - raise ValueError( - f"The sum of {_name}.output is not finite: {_output}" - ) + logging.warning(f"The sum of {_name}.output is not finite") elif isinstance(_output, tuple): for i, o in enumerate(_output): if isinstance(o, tuple): @@ -50,9 +48,7 @@ def register_inf_check_hooks(model: nn.Module) -> None: if not isinstance(o, Tensor): continue if not torch.isfinite(o.to(torch.float32).sum()): - raise ValueError( - f"The sum of {_name}.output[{i}] is not finite: {_output}" - ) + logging.warning(f"The sum of {_name}.output[{i}] is not finite") # default param _name is a way to capture the current value of the variable "name". def backward_hook(_module, _input, _output, _name=name): From bfffda5afb74b193068078c9b51db380ec005afe Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Tue, 31 Dec 2024 17:17:05 +0800 Subject: [PATCH 21/59] Add MatchaTTS for the Chinese dataset Baker (#1849) --- .github/scripts/baker_zh/TTS/run-matcha.sh | 167 ++++ .../scripts/docker/generate_build_matrix.py | 18 +- .github/scripts/ljspeech/TTS/run-matcha.sh | 2 +- .github/workflows/baker_zh.yml | 152 ++++ egs/baker_zh/TTS/.gitignore | 6 + egs/baker_zh/TTS/README.md | 146 ++++ egs/baker_zh/TTS/local/audio.py | 1 + .../TTS/local/compute_fbank_baker_zh.py | 110 +++ .../TTS/local/compute_fbank_statistics.py | 1 + .../TTS/local/convert_text_to_tokens.py | 121 +++ egs/baker_zh/TTS/local/fbank.py | 1 + egs/baker_zh/TTS/local/generate_tokens.py | 85 +++ egs/baker_zh/TTS/local/validate_manifest.py | 70 ++ egs/baker_zh/TTS/matcha/__init__.py | 0 egs/baker_zh/TTS/matcha/audio.py | 1 + egs/baker_zh/TTS/matcha/export_onnx.py | 207 +++++ .../TTS/matcha/export_onnx_hifigan.py | 1 + egs/baker_zh/TTS/matcha/fbank.py | 1 + egs/baker_zh/TTS/matcha/generate_lexicon.py | 42 + egs/baker_zh/TTS/matcha/hifigan | 1 + egs/baker_zh/TTS/matcha/infer.py | 342 +++++++++ egs/baker_zh/TTS/matcha/model.py | 1 + egs/baker_zh/TTS/matcha/models | 1 + egs/baker_zh/TTS/matcha/monotonic_align | 1 + egs/baker_zh/TTS/matcha/onnx_pretrained.py | 316 ++++++++ egs/baker_zh/TTS/matcha/tokenizer.py | 119 +++ egs/baker_zh/TTS/matcha/train.py | 717 ++++++++++++++++++ egs/baker_zh/TTS/matcha/tts_datamodule.py | 340 +++++++++ egs/baker_zh/TTS/matcha/utils.py | 1 + egs/baker_zh/TTS/prepare.sh | 151 ++++ egs/baker_zh/TTS/shared | 1 + egs/ljspeech/TTS/README.md | 2 +- egs/ljspeech/TTS/matcha/export_onnx.py | 11 +- egs/ljspeech/TTS/matcha/onnx_pretrained.py | 4 +- 34 files changed, 3128 insertions(+), 12 deletions(-) create mode 100755 .github/scripts/baker_zh/TTS/run-matcha.sh create mode 100644 .github/workflows/baker_zh.yml create mode 100644 egs/baker_zh/TTS/.gitignore create mode 100644 egs/baker_zh/TTS/README.md create mode 120000 egs/baker_zh/TTS/local/audio.py create mode 100755 egs/baker_zh/TTS/local/compute_fbank_baker_zh.py create mode 120000 egs/baker_zh/TTS/local/compute_fbank_statistics.py create mode 100755 egs/baker_zh/TTS/local/convert_text_to_tokens.py create mode 120000 egs/baker_zh/TTS/local/fbank.py create mode 100755 egs/baker_zh/TTS/local/generate_tokens.py create mode 100755 egs/baker_zh/TTS/local/validate_manifest.py create mode 100644 egs/baker_zh/TTS/matcha/__init__.py create mode 120000 egs/baker_zh/TTS/matcha/audio.py create mode 100755 egs/baker_zh/TTS/matcha/export_onnx.py create mode 120000 egs/baker_zh/TTS/matcha/export_onnx_hifigan.py create mode 120000 egs/baker_zh/TTS/matcha/fbank.py create mode 100755 egs/baker_zh/TTS/matcha/generate_lexicon.py create mode 120000 egs/baker_zh/TTS/matcha/hifigan create mode 100755 egs/baker_zh/TTS/matcha/infer.py create mode 120000 egs/baker_zh/TTS/matcha/model.py create mode 120000 egs/baker_zh/TTS/matcha/models create mode 120000 egs/baker_zh/TTS/matcha/monotonic_align create mode 100755 egs/baker_zh/TTS/matcha/onnx_pretrained.py create mode 100644 egs/baker_zh/TTS/matcha/tokenizer.py create mode 100755 egs/baker_zh/TTS/matcha/train.py create mode 100644 egs/baker_zh/TTS/matcha/tts_datamodule.py create mode 120000 egs/baker_zh/TTS/matcha/utils.py create mode 100755 egs/baker_zh/TTS/prepare.sh create mode 120000 egs/baker_zh/TTS/shared diff --git a/.github/scripts/baker_zh/TTS/run-matcha.sh b/.github/scripts/baker_zh/TTS/run-matcha.sh new file mode 100755 index 000000000..150f023ae --- /dev/null +++ b/.github/scripts/baker_zh/TTS/run-matcha.sh @@ -0,0 +1,167 @@ +#!/usr/bin/env bash + +set -ex + +apt-get update +apt-get install -y sox + +python3 -m pip install numba conformer==0.3.2 diffusers librosa +python3 -m pip install jieba + + +log() { + # This function is from espnet + local fname=${BASH_SOURCE[1]##*/} + echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*" +} + +cd egs/baker_zh/TTS + +sed -i.bak s/600/8/g ./prepare.sh +sed -i.bak s/"first 100"/"first 3"/g ./prepare.sh +sed -i.bak s/500/5/g ./prepare.sh +git diff + +function prepare_data() { + # We have created a subset of the data for testing + # + mkdir -p download + pushd download + wget -q https://huggingface.co/csukuangfj/tmp-files/resolve/main/BZNSYP-samples.tar.bz2 + tar xvf BZNSYP-samples.tar.bz2 + mv BZNSYP-samples BZNSYP + rm BZNSYP-samples.tar.bz2 + popd + + ./prepare.sh + tree . +} + +function train() { + pushd ./matcha + sed -i.bak s/1500/3/g ./train.py + git diff . + popd + + ./matcha/train.py \ + --exp-dir matcha/exp \ + --num-epochs 1 \ + --save-every-n 1 \ + --num-buckets 2 \ + --tokens data/tokens.txt \ + --max-duration 20 + + ls -lh matcha/exp +} + +function infer() { + curl -SL -O https://github.com/csukuangfj/models/raw/refs/heads/master/hifigan/generator_v2 + + ./matcha/infer.py \ + --num-buckets 2 \ + --epoch 1 \ + --exp-dir ./matcha/exp \ + --tokens data/tokens.txt \ + --cmvn ./data/fbank/cmvn.json \ + --vocoder ./generator_v2 \ + --input-text "当夜幕降临，星光点点，伴随着微风拂面，我在静谧中感受着时光的流转，思念如涟漪荡漾，梦境如画卷展开，我与自然融为一体，沉静在这片宁静的美丽之中，感受着生命的奇迹与温柔。" \ + --output-wav ./generated.wav + + ls -lh *.wav + soxi ./generated.wav + rm -v ./generated.wav + rm -v generator_v2 +} + +function export_onnx() { + pushd matcha/exp + curl -SL -O https://huggingface.co/csukuangfj/icefall-tts-baker-matcha-zh-2024-12-27/resolve/main/epoch-2000.pt + popd + + pushd data/fbank + rm -v *.json + curl -SL -O https://huggingface.co/csukuangfj/icefall-tts-baker-matcha-zh-2024-12-27/resolve/main/cmvn.json + popd + + ./matcha/export_onnx.py \ + --exp-dir ./matcha/exp \ + --epoch 2000 \ + --tokens ./data/tokens.txt \ + --cmvn ./data/fbank/cmvn.json + + ls -lh *.onnx + + if false; then + # The CI machine does not have enough memory to run it + # + curl -SL -O https://github.com/csukuangfj/models/raw/refs/heads/master/hifigan/generator_v1 + curl -SL -O https://github.com/csukuangfj/models/raw/refs/heads/master/hifigan/generator_v2 + curl -SL -O https://github.com/csukuangfj/models/raw/refs/heads/master/hifigan/generator_v3 + python3 ./matcha/export_onnx_hifigan.py + else + curl -SL -O https://huggingface.co/csukuangfj/icefall-tts-ljspeech-matcha-en-2024-10-28/resolve/main/exp/hifigan_v1.onnx + curl -SL -O https://huggingface.co/csukuangfj/icefall-tts-ljspeech-matcha-en-2024-10-28/resolve/main/exp/hifigan_v2.onnx + curl -SL -O https://huggingface.co/csukuangfj/icefall-tts-ljspeech-matcha-en-2024-10-28/resolve/main/exp/hifigan_v3.onnx + fi + + ls -lh *.onnx + + python3 ./matcha/generate_lexicon.py + + for v in v1 v2 v3; do + python3 ./matcha/onnx_pretrained.py \ + --acoustic-model ./model-steps-6.onnx \ + --vocoder ./hifigan_$v.onnx \ + --tokens ./data/tokens.txt \ + --lexicon ./lexicon.txt \ + --input-text "当夜幕降临，星光点点，伴随着微风拂面，我在静谧中感受着时光的流转，思念如涟漪荡漾，梦境如画卷展开，我与自然融为一体，沉静在这片宁静的美丽之中，感受着生命的奇迹与温柔。" \ + --output-wav /icefall/generated-matcha-tts-steps-6-$v.wav + done + + ls -lh /icefall/*.wav + soxi /icefall/generated-matcha-tts-steps-6-*.wav + cp ./model-steps-*.onnx /icefall + + d=matcha-icefall-zh-baker + mkdir $d + cp -v data/tokens.txt $d + cp -v lexicon.txt $d + cp model-steps-3.onnx $d + pushd $d + curl -SL -O https://github.com/csukuangfj/cppjieba/releases/download/sherpa-onnx-2024-04-19/dict.tar.bz2 + tar xvf dict.tar.bz2 + rm dict.tar.bz2 + + curl -SL -O https://huggingface.co/csukuangfj/icefall-tts-aishell3-vits-low-2024-04-06/resolve/main/data/date.fst + curl -SL -O https://huggingface.co/csukuangfj/icefall-tts-aishell3-vits-low-2024-04-06/resolve/main/data/number.fst + curl -SL -O https://huggingface.co/csukuangfj/icefall-tts-aishell3-vits-low-2024-04-06/resolve/main/data/phone.fst + +cat >README.md < + +The training command is given below: +```bash +python3 ./matcha/train.py \ + --exp-dir ./matcha/exp-1/ \ + --num-workers 4 \ + --world-size 1 \ + --num-epochs 2000 \ + --max-duration 1200 \ + --bucketing-sampler 1 \ + --start-epoch 1 +``` + +To inference, use: + +```bash +# Download Hifigan vocoder. We use Hifigan v2 below. You can select from v1, v2, or v3 + +wget https://github.com/csukuangfj/models/raw/refs/heads/master/hifigan/generator_v2 + +python3 ./matcha/infer.py \ + --epoch 2000 \ + --exp-dir ./matcha/exp-1 \ + --vocoder ./generator_v2 \ + --tokens ./data/tokens.txt \ + --cmvn ./data/fbank/cmvn.json \ + --input-text "当夜幕降临，星光点点，伴随着微风拂面，我在静谧中感受着时光的流转，思念如涟漪荡漾，梦境如画卷展开，我与自然融为一体，沉静在这片宁静的美丽之中，感受着生命的奇迹与温柔。" \ + --output-wav ./generated.wav +``` + +```bash +soxi ./generated.wav +``` + +prints: +``` +Input File : './generated.wav' +Channels : 1 +Sample Rate : 22050 +Precision : 16-bit +Duration : 00:00:17.31 = 381696 samples ~ 1298.29 CDDA sectors +File Size : 763k +Bit Rate : 353k +Sample Encoding: 16-bit Signed Integer PCM +``` + +https://github.com/user-attachments/assets/88d4e88f-ebc4-4f32-b216-16d46b966024 + + +To export the checkpoint to onnx: +```bash +python3 ./matcha/export_onnx.py \ + --exp-dir ./matcha/exp-1 \ + --epoch 2000 \ + --tokens ./data/tokens.txt \ + --cmvn ./data/fbank/cmvn.json +``` + +The above command generates the following files: +``` +-rw-r--r-- 1 kuangfangjun root 72M Dec 27 18:53 model-steps-2.onnx +-rw-r--r-- 1 kuangfangjun root 73M Dec 27 18:54 model-steps-3.onnx +-rw-r--r-- 1 kuangfangjun root 73M Dec 27 18:54 model-steps-4.onnx +-rw-r--r-- 1 kuangfangjun root 74M Dec 27 18:55 model-steps-5.onnx +-rw-r--r-- 1 kuangfangjun root 74M Dec 27 18:57 model-steps-6.onnx +``` + +where the 2 in `model-steps-2.onnx` means it uses 2 steps for the ODE solver. + +**HINT**: If you get the following error while running `export_onnx.py`: + +``` +torch.onnx.errors.UnsupportedOperatorError: Exporting the operator +'aten::scaled_dot_product_attention' to ONNX opset version 14 is not supported. +``` + +please use `torch>=2.2.0`. + +To export the Hifigan vocoder to onnx, please use: + +```bash +wget https://github.com/csukuangfj/models/raw/refs/heads/master/hifigan/generator_v1 +wget https://github.com/csukuangfj/models/raw/refs/heads/master/hifigan/generator_v2 +wget https://github.com/csukuangfj/models/raw/refs/heads/master/hifigan/generator_v3 + +python3 ./matcha/export_onnx_hifigan.py +``` + +The above command generates 3 files: + + - hifigan_v1.onnx + - hifigan_v2.onnx + - hifigan_v3.onnx + +**HINT**: You can download pre-exported hifigan ONNX models from + + +To use the generated onnx files to generate speech from text, please run: + +```bash + +# First, generate ./lexicon.txt +python3 ./matcha/generate_lexicon.py + +python3 ./matcha/onnx_pretrained.py \ + --acoustic-model ./model-steps-4.onnx \ + --vocoder ./hifigan_v2.onnx \ + --tokens ./data/tokens.txt \ + --lexicon ./lexicon.txt \ + --input-text "在一个阳光明媚的夏天，小马、小羊和小狗它们一块儿在广阔的草地上，嬉戏玩耍，这时小猴来了，还带着它心爱的足球活蹦乱跳地跑前、跑后教小马、小羊、小狗踢足球。" \ + --output-wav ./1.wav +``` + +```bash +soxi ./1.wav + +Input File : './1.wav' +Channels : 1 +Sample Rate : 22050 +Precision : 16-bit +Duration : 00:00:16.37 = 360960 samples ~ 1227.76 CDDA sectors +File Size : 722k +Bit Rate : 353k +Sample Encoding: 16-bit Signed Integer PCM +``` + +https://github.com/user-attachments/assets/578d04bb-fee8-47e5-9984-a868dcce610e + diff --git a/egs/baker_zh/TTS/local/audio.py b/egs/baker_zh/TTS/local/audio.py new file mode 120000 index 000000000..b70d91c92 --- /dev/null +++ b/egs/baker_zh/TTS/local/audio.py @@ -0,0 +1 @@ +../matcha/audio.py \ No newline at end of file diff --git a/egs/baker_zh/TTS/local/compute_fbank_baker_zh.py b/egs/baker_zh/TTS/local/compute_fbank_baker_zh.py new file mode 100755 index 000000000..0720158f2 --- /dev/null +++ b/egs/baker_zh/TTS/local/compute_fbank_baker_zh.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +# Copyright 2021-2023 Xiaomi Corp. (authors: Fangjun Kuang, +# Zengwei Yao) +# +# See ../../../../LICENSE for clarification regarding multiple authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +""" +This file computes fbank features of the baker-zh dataset. +It looks for manifests in the directory data/manifests. + +The generated fbank features are saved in data/fbank. +""" + +import argparse +import logging +import os +from pathlib import Path + +import torch +from fbank import MatchaFbank, MatchaFbankConfig +from lhotse import CutSet, LilcomChunkyWriter, load_manifest +from lhotse.audio import RecordingSet +from lhotse.supervision import SupervisionSet + +from icefall.utils import get_executor + + +def get_parser(): + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + + parser.add_argument( + "--num-jobs", + type=int, + default=4, + help="""It specifies the checkpoint to use for decoding. + Note: Epoch counts from 1. + """, + ) + return parser + + +def compute_fbank_baker_zh(num_jobs: int): + src_dir = Path("data/manifests") + output_dir = Path("data/fbank") + + if num_jobs < 1: + num_jobs = os.cpu_count() + + logging.info(f"num_jobs: {num_jobs}") + logging.info(f"src_dir: {src_dir}") + logging.info(f"output_dir: {output_dir}") + config = MatchaFbankConfig( + n_fft=1024, + n_mels=80, + sampling_rate=22050, + hop_length=256, + win_length=1024, + f_min=0, + f_max=8000, + ) + + prefix = "baker_zh" + suffix = "jsonl.gz" + + extractor = MatchaFbank(config) + + with get_executor() as ex: # Initialize the executor only once. + cuts_filename = f"{prefix}_cuts.{suffix}" + logging.info(f"Processing {cuts_filename}") + cut_set = load_manifest(src_dir / cuts_filename).resample(22050) + + cut_set = cut_set.compute_and_store_features( + extractor=extractor, + storage_path=f"{output_dir}/{prefix}_feats", + num_jobs=num_jobs if ex is None else 80, + executor=ex, + storage_type=LilcomChunkyWriter, + ) + + cut_set.to_file(output_dir / cuts_filename) + + +if __name__ == "__main__": + # Torch's multithreaded behavior needs to be disabled or + # it wastes a lot of CPU and slow things down. + # Do this outside of main() in case it needs to take effect + # even when we are not invoking the main (e.g. when spawning subprocesses). + torch.set_num_threads(1) + torch.set_num_interop_threads(1) + formatter = "%(asctime)s %(levelname)s [%(filename)s:%(lineno)d] %(message)s" + + logging.basicConfig(format=formatter, level=logging.INFO) + + args = get_parser().parse_args() + compute_fbank_baker_zh(args.num_jobs) diff --git a/egs/baker_zh/TTS/local/compute_fbank_statistics.py b/egs/baker_zh/TTS/local/compute_fbank_statistics.py new file mode 120000 index 000000000..fd1d8b52e --- /dev/null +++ b/egs/baker_zh/TTS/local/compute_fbank_statistics.py @@ -0,0 +1 @@ +../../../ljspeech/TTS/local/compute_fbank_statistics.py \ No newline at end of file diff --git a/egs/baker_zh/TTS/local/convert_text_to_tokens.py b/egs/baker_zh/TTS/local/convert_text_to_tokens.py new file mode 100755 index 000000000..bf59cb466 --- /dev/null +++ b/egs/baker_zh/TTS/local/convert_text_to_tokens.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 + +import argparse +import re +from typing import List + +import jieba +from lhotse import load_manifest +from pypinyin import Style, lazy_pinyin, load_phrases_dict + +load_phrases_dict( + { + "行长": [["hang2"], ["zhang3"]], + "银行行长": [["yin2"], ["hang2"], ["hang2"], ["zhang3"]], + } +) + +whiter_space_re = re.compile(r"\s+") + +punctuations_re = [ + (re.compile(x[0], re.IGNORECASE), x[1]) + for x in [ + ("，", ","), + ("。", "."), + ("！", "!"), + ("？", "?"), + ("“", '"'), + ("”", '"'), + ("‘", "'"), + ("’", "'"), + ("：", ":"), + ("、", ","), + ("Ｂ", "逼"), + ("Ｐ", "批"), + ] +] + + +def get_parser(): + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + parser.add_argument( + "--in-file", + type=str, + required=True, + help="Input cutset.", + ) + + parser.add_argument( + "--out-file", + type=str, + required=True, + help="Output cutset.", + ) + + return parser + + +def normalize_white_spaces(text): + return whiter_space_re.sub(" ", text) + + +def normalize_punctuations(text): + for regex, replacement in punctuations_re: + text = re.sub(regex, replacement, text) + return text + + +def split_text(text: str) -> List[str]: + """ + Example input: '你好呀，You are 一个好人。 去银行存钱？How about you?' + Example output: ['你好', '呀', ',', 'you are', '一个', '好人', '.', '去', '银行', '存钱', '?', 'how about you', '?'] + """ + text = text.lower() + text = normalize_white_spaces(text) + text = normalize_punctuations(text) + ans = [] + + for seg in jieba.cut(text): + if seg in ",.!?:\"'": + ans.append(seg) + elif seg == " " and len(ans) > 0: + if ord("a") <= ord(ans[-1][-1]) <= ord("z"): + ans[-1] += seg + elif ord("a") <= ord(seg[0]) <= ord("z"): + if len(ans) == 0: + ans.append(seg) + continue + + if ans[-1][-1] == " ": + ans[-1] += seg + continue + + ans.append(seg) + else: + ans.append(seg) + + ans = [s.strip() for s in ans] + return ans + + +def main(): + args = get_parser().parse_args() + cuts = load_manifest(args.in_file) + for c in cuts: + assert len(c.supervisions) == 1, (len(c.supervisions), c.supervisions) + text = c.supervisions[0].normalized_text + + text_list = split_text(text) + tokens = lazy_pinyin(text_list, style=Style.TONE3, tone_sandhi=True) + + c.tokens = tokens + + cuts.to_file(args.out_file) + + print(f"saved to {args.out_file}") + + +if __name__ == "__main__": + main() diff --git a/egs/baker_zh/TTS/local/fbank.py b/egs/baker_zh/TTS/local/fbank.py new file mode 120000 index 000000000..5bcf1fde5 --- /dev/null +++ b/egs/baker_zh/TTS/local/fbank.py @@ -0,0 +1 @@ +../matcha/fbank.py \ No newline at end of file diff --git a/egs/baker_zh/TTS/local/generate_tokens.py b/egs/baker_zh/TTS/local/generate_tokens.py new file mode 100755 index 000000000..b2abe1a71 --- /dev/null +++ b/egs/baker_zh/TTS/local/generate_tokens.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 + +""" +This file generates the file tokens.txt. + +Usage: + +python3 ./local/generate_tokens.py > data/tokens.txt +""" + + +import argparse +from typing import List + +import jieba +from pypinyin import Style, lazy_pinyin, pinyin_dict + + +def get_parser(): + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + parser.add_argument( + "--tokens", + type=str, + required=True, + help="Path to to save tokens.txt.", + ) + + return parser + + +def generate_token_list() -> List[str]: + token_set = set() + + word_dict = pinyin_dict.pinyin_dict + i = 0 + for key in word_dict: + if not (0x4E00 <= key <= 0x9FFF): + continue + + w = chr(key) + t = lazy_pinyin(w, style=Style.TONE3, tone_sandhi=True)[0] + token_set.add(t) + + no_digit = set() + for t in token_set: + if t[-1] not in "1234": + no_digit.add(t) + else: + no_digit.add(t[:-1]) + + no_digit.add("dei") + no_digit.add("tou") + no_digit.add("dia") + + for t in no_digit: + token_set.add(t) + for i in range(1, 5): + token_set.add(f"{t}{i}") + + ans = list(token_set) + ans.sort() + + punctuations = list(",.!?:\"'") + ans = punctuations + ans + + # use ID 0 for blank + # Use ID 1 of _ for padding + ans.insert(0, " ") + ans.insert(1, "_") # + + return ans + + +def main(): + args = get_parser().parse_args() + token_list = generate_token_list() + with open(args.tokens, "w", encoding="utf-8") as f: + for indx, token in enumerate(token_list): + f.write(f"{token} {indx}\n") + + +if __name__ == "__main__": + main() diff --git a/egs/baker_zh/TTS/local/validate_manifest.py b/egs/baker_zh/TTS/local/validate_manifest.py new file mode 100755 index 000000000..4e31028f7 --- /dev/null +++ b/egs/baker_zh/TTS/local/validate_manifest.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +# Copyright 2022-2023 Xiaomi Corp. (authors: Fangjun Kuang, +# Zengwei Yao) +# +# See ../../../../LICENSE for clarification regarding multiple authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +This script checks the following assumptions of the generated manifest: + +- Single supervision per cut + +We will add more checks later if needed. + +Usage example: + + python3 ./local/validate_manifest.py \ + ./data/spectrogram/baker_zh_cuts_all.jsonl.gz + +""" + +import argparse +import logging +from pathlib import Path + +from lhotse import CutSet, load_manifest_lazy +from lhotse.dataset.speech_synthesis import validate_for_tts + + +def get_args(): + parser = argparse.ArgumentParser() + + parser.add_argument( + "manifest", + type=Path, + help="Path to the manifest file", + ) + + return parser.parse_args() + + +def main(): + args = get_args() + + manifest = args.manifest + logging.info(f"Validating {manifest}") + + assert manifest.is_file(), f"{manifest} does not exist" + cut_set = load_manifest_lazy(manifest) + assert isinstance(cut_set, CutSet), type(cut_set) + + validate_for_tts(cut_set) + + +if __name__ == "__main__": + formatter = "%(asctime)s %(levelname)s [%(filename)s:%(lineno)d] %(message)s" + + logging.basicConfig(format=formatter, level=logging.INFO) + + main() diff --git a/egs/baker_zh/TTS/matcha/__init__.py b/egs/baker_zh/TTS/matcha/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/egs/baker_zh/TTS/matcha/audio.py b/egs/baker_zh/TTS/matcha/audio.py new file mode 120000 index 000000000..62d3959d6 --- /dev/null +++ b/egs/baker_zh/TTS/matcha/audio.py @@ -0,0 +1 @@ +../../../ljspeech/TTS/matcha/audio.py \ No newline at end of file diff --git a/egs/baker_zh/TTS/matcha/export_onnx.py b/egs/baker_zh/TTS/matcha/export_onnx.py new file mode 100755 index 000000000..28efbfe61 --- /dev/null +++ b/egs/baker_zh/TTS/matcha/export_onnx.py @@ -0,0 +1,207 @@ +#!/usr/bin/env python3 +# Copyright 2024 Xiaomi Corp. (authors: Fangjun Kuang) + +""" +This script exports a Matcha-TTS model to ONNX. +Note that the model outputs fbank. You need to use a vocoder to convert +it to audio. See also ./export_onnx_hifigan.py + +python3 ./matcha/export_onnx.py \ + --exp-dir ./matcha/exp-1 \ + --epoch 2000 \ + --tokens ./data/tokens.txt \ + --cmvn ./data/fbank/cmvn.json + +""" + +import argparse +import json +import logging +from pathlib import Path +from typing import Any, Dict + +import onnx +import torch +from tokenizer import Tokenizer +from train import get_model, get_params + +from icefall.checkpoint import load_checkpoint + + +def get_parser(): + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + + parser.add_argument( + "--epoch", + type=int, + default=2000, + help="""It specifies the checkpoint to use for decoding. + Note: Epoch counts from 1. + """, + ) + + parser.add_argument( + "--exp-dir", + type=Path, + default="matcha/exp-new-3", + help="""The experiment dir. + It specifies the directory where all training related + files, e.g., checkpoints, log, etc, are saved + """, + ) + + parser.add_argument( + "--tokens", + type=Path, + default="data/tokens.txt", + ) + + parser.add_argument( + "--cmvn", + type=str, + default="data/fbank/cmvn.json", + help="""Path to vocabulary.""", + ) + + return parser + + +def add_meta_data(filename: str, meta_data: Dict[str, Any]): + """Add meta data to an ONNX model. It is changed in-place. + + Args: + filename: + Filename of the ONNX model to be changed. + meta_data: + Key-value pairs. + """ + model = onnx.load(filename) + + while len(model.metadata_props): + model.metadata_props.pop() + + for key, value in meta_data.items(): + meta = model.metadata_props.add() + meta.key = key + meta.value = str(value) + + onnx.save(model, filename) + + +class ModelWrapper(torch.nn.Module): + def __init__(self, model, num_steps: int = 5): + super().__init__() + self.model = model + self.num_steps = num_steps + + def forward( + self, + x: torch.Tensor, + x_lengths: torch.Tensor, + noise_scale: torch.Tensor, + length_scale: torch.Tensor, + ) -> torch.Tensor: + """ + Args: : + x: (batch_size, num_tokens), torch.int64 + x_lengths: (batch_size,), torch.int64 + noise_scale: (1,), torch.float32 + length_scale (1,), torch.float32 + Returns: + audio: (batch_size, num_samples) + + """ + mel = self.model.synthesise( + x=x, + x_lengths=x_lengths, + n_timesteps=self.num_steps, + temperature=noise_scale, + length_scale=length_scale, + )["mel"] + # mel: (batch_size, feat_dim, num_frames) + + return mel + + +@torch.inference_mode() +def main(): + parser = get_parser() + args = parser.parse_args() + params = get_params() + + params.update(vars(args)) + + tokenizer = Tokenizer(params.tokens) + params.pad_id = tokenizer.pad_id + params.vocab_size = tokenizer.vocab_size + params.model_args.n_vocab = params.vocab_size + + with open(params.cmvn) as f: + stats = json.load(f) + params.data_args.data_statistics.mel_mean = stats["fbank_mean"] + params.data_args.data_statistics.mel_std = stats["fbank_std"] + + params.model_args.data_statistics.mel_mean = stats["fbank_mean"] + params.model_args.data_statistics.mel_std = stats["fbank_std"] + logging.info(params) + + logging.info("About to create model") + model = get_model(params) + load_checkpoint(f"{params.exp_dir}/epoch-{params.epoch}.pt", model) + + for num_steps in [2, 3, 4, 5, 6]: + logging.info(f"num_steps: {num_steps}") + wrapper = ModelWrapper(model, num_steps=num_steps) + wrapper.eval() + + # Use a large value so the rotary position embedding in the text + # encoder has a large initial length + x = torch.ones(1, 1000, dtype=torch.int64) + x_lengths = torch.tensor([x.shape[1]], dtype=torch.int64) + noise_scale = torch.tensor([1.0]) + length_scale = torch.tensor([1.0]) + + opset_version = 14 + filename = f"model-steps-{num_steps}.onnx" + torch.onnx.export( + wrapper, + (x, x_lengths, noise_scale, length_scale), + filename, + opset_version=opset_version, + input_names=["x", "x_length", "noise_scale", "length_scale"], + output_names=["mel"], + dynamic_axes={ + "x": {0: "N", 1: "L"}, + "x_length": {0: "N"}, + "mel": {0: "N", 2: "L"}, + }, + ) + + meta_data = { + "model_type": "matcha-tts", + "language": "Chinese", + "has_espeak": 0, + "n_speakers": 1, + "jieba": 1, + "sample_rate": 22050, + "version": 1, + "pad_id": params.pad_id, + "model_author": "icefall", + "maintainer": "k2-fsa", + "dataset": "baker-zh", + "use_eos_bos": 0, + "dataset_url": "https://www.data-baker.com/open_source.html", + "dataset_comment": "The dataset is for non-commercial use only.", + "num_ode_steps": num_steps, + } + add_meta_data(filename=filename, meta_data=meta_data) + print(meta_data) + + +if __name__ == "__main__": + formatter = "%(asctime)s %(levelname)s [%(filename)s:%(lineno)d] %(message)s" + + logging.basicConfig(format=formatter, level=logging.INFO) + main() diff --git a/egs/baker_zh/TTS/matcha/export_onnx_hifigan.py b/egs/baker_zh/TTS/matcha/export_onnx_hifigan.py new file mode 120000 index 000000000..d0b8af15b --- /dev/null +++ b/egs/baker_zh/TTS/matcha/export_onnx_hifigan.py @@ -0,0 +1 @@ +../../../ljspeech/TTS/matcha/export_onnx_hifigan.py \ No newline at end of file diff --git a/egs/baker_zh/TTS/matcha/fbank.py b/egs/baker_zh/TTS/matcha/fbank.py new file mode 120000 index 000000000..3cfb7fe3f --- /dev/null +++ b/egs/baker_zh/TTS/matcha/fbank.py @@ -0,0 +1 @@ +../../../ljspeech/TTS/matcha/fbank.py \ No newline at end of file diff --git a/egs/baker_zh/TTS/matcha/generate_lexicon.py b/egs/baker_zh/TTS/matcha/generate_lexicon.py new file mode 100755 index 000000000..f26f28e91 --- /dev/null +++ b/egs/baker_zh/TTS/matcha/generate_lexicon.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 + +import jieba +from pypinyin import Style, lazy_pinyin, load_phrases_dict, phrases_dict, pinyin_dict +from tokenizer import Tokenizer + +load_phrases_dict( + { + "行长": [["hang2"], ["zhang3"]], + "银行行长": [["yin2"], ["hang2"], ["hang2"], ["zhang3"]], + } +) + + +def main(): + filename = "lexicon.txt" + tokens = "./data/tokens.txt" + tokenizer = Tokenizer(tokens) + + word_dict = pinyin_dict.pinyin_dict + phrases = phrases_dict.phrases_dict + + i = 0 + with open(filename, "w", encoding="utf-8") as f: + for key in word_dict: + if not (0x4E00 <= key <= 0x9FFF): + continue + + w = chr(key) + tokens = lazy_pinyin(w, style=Style.TONE3, tone_sandhi=True)[0] + + f.write(f"{w} {tokens}\n") + + for key in phrases: + tokens = lazy_pinyin(key, style=Style.TONE3, tone_sandhi=True) + tokens = " ".join(tokens) + + f.write(f"{key} {tokens}\n") + + +if __name__ == "__main__": + main() diff --git a/egs/baker_zh/TTS/matcha/hifigan b/egs/baker_zh/TTS/matcha/hifigan new file mode 120000 index 000000000..c0a91072c --- /dev/null +++ b/egs/baker_zh/TTS/matcha/hifigan @@ -0,0 +1 @@ +../../../ljspeech/TTS/matcha/hifigan \ No newline at end of file diff --git a/egs/baker_zh/TTS/matcha/infer.py b/egs/baker_zh/TTS/matcha/infer.py new file mode 100755 index 000000000..b90c2fdbd --- /dev/null +++ b/egs/baker_zh/TTS/matcha/infer.py @@ -0,0 +1,342 @@ +#!/usr/bin/env python3 +# Copyright 2024 Xiaomi Corp. (authors: Fangjun Kuang) +""" +python3 ./matcha/infer.py \ + --epoch 2000 \ + --exp-dir ./matcha/exp-1 \ + --vocoder ./generator_v2 \ + --tokens ./data/tokens.txt \ + --cmvn ./data/fbank/cmvn.json \ + --input-text "当夜幕降临，星光点点，伴随着微风拂面，我在静谧中感受着时光的流转，思念如涟漪荡漾，梦境如画卷展开，我与自然融为一体，沉静在这片宁静的美丽之中，感受着生命的奇迹与温柔。" \ + --output-wav ./generated.wav +""" + +import argparse +import datetime as dt +import json +import logging +from pathlib import Path + +import soundfile as sf +import torch +import torch.nn as nn +from hifigan.config import v1, v2, v3 +from hifigan.denoiser import Denoiser +from hifigan.models import Generator as HiFiGAN +from local.convert_text_to_tokens import split_text +from pypinyin import Style, lazy_pinyin +from tokenizer import Tokenizer +from train import get_model, get_params +from tts_datamodule import BakerZhTtsDataModule + +from icefall.checkpoint import load_checkpoint +from icefall.utils import AttributeDict, setup_logger + + +def get_parser(): + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + + parser.add_argument( + "--epoch", + type=int, + default=4000, + help="""It specifies the checkpoint to use for decoding. + Note: Epoch counts from 1. + """, + ) + + parser.add_argument( + "--exp-dir", + type=Path, + default="matcha/exp", + help="""The experiment dir. + It specifies the directory where all training related + files, e.g., checkpoints, log, etc, are saved + """, + ) + + parser.add_argument( + "--vocoder", + type=Path, + default="./generator_v1", + help="Path to the vocoder", + ) + + parser.add_argument( + "--tokens", + type=Path, + default="data/tokens.txt", + ) + + parser.add_argument( + "--cmvn", + type=str, + default="data/fbank/cmvn.json", + help="""Path to vocabulary.""", + ) + + # The following arguments are used for inference on single text + parser.add_argument( + "--input-text", + type=str, + required=False, + help="The text to generate speech for", + ) + + parser.add_argument( + "--output-wav", + type=str, + required=False, + help="The filename of the wave to save the generated speech", + ) + + parser.add_argument( + "--sampling-rate", + type=int, + default=22050, + help="The sampling rate of the generated speech (default: 22050 for baker_zh)", + ) + + return parser + + +def load_vocoder(checkpoint_path: Path) -> nn.Module: + checkpoint_path = str(checkpoint_path) + if checkpoint_path.endswith("v1"): + h = AttributeDict(v1) + elif checkpoint_path.endswith("v2"): + h = AttributeDict(v2) + elif checkpoint_path.endswith("v3"): + h = AttributeDict(v3) + else: + raise ValueError(f"supports only v1, v2, and v3, given {checkpoint_path}") + + hifigan = HiFiGAN(h).to("cpu") + hifigan.load_state_dict( + torch.load(checkpoint_path, map_location="cpu")["generator"] + ) + _ = hifigan.eval() + hifigan.remove_weight_norm() + return hifigan + + +def to_waveform( + mel: torch.Tensor, vocoder: nn.Module, denoiser: nn.Module +) -> torch.Tensor: + audio = vocoder(mel).clamp(-1, 1) + audio = denoiser(audio.squeeze(0), strength=0.00025).cpu().squeeze() + return audio.squeeze() + + +def process_text(text: str, tokenizer: Tokenizer, device: str = "cpu") -> dict: + text = split_text(text) + tokens = lazy_pinyin(text, style=Style.TONE3, tone_sandhi=True) + + x = tokenizer.texts_to_token_ids([tokens]) + x = torch.tensor(x, dtype=torch.long, device=device) + x_lengths = torch.tensor([x.shape[-1]], dtype=torch.long, device=device) + return {"x_orig": text, "x": x, "x_lengths": x_lengths} + + +def synthesize( + model: nn.Module, + tokenizer: Tokenizer, + n_timesteps: int, + text: str, + length_scale: float, + temperature: float, + device: str = "cpu", + spks=None, +) -> dict: + text_processed = process_text(text=text, tokenizer=tokenizer, device=device) + start_t = dt.datetime.now() + output = model.synthesise( + text_processed["x"], + text_processed["x_lengths"], + n_timesteps=n_timesteps, + temperature=temperature, + spks=spks, + length_scale=length_scale, + ) + # merge everything to one dict + output.update({"start_t": start_t, **text_processed}) + return output + + +def infer_dataset( + dl: torch.utils.data.DataLoader, + params: AttributeDict, + model: nn.Module, + vocoder: nn.Module, + denoiser: nn.Module, + tokenizer: Tokenizer, +) -> None: + """Decode dataset. + The ground-truth and generated audio pairs will be saved to `params.save_wav_dir`. + + Args: + dl: + PyTorch's dataloader containing the dataset to decode. + params: + It is returned by :func:`get_params`. + model: + The neural model. + tokenizer: + Used to convert text to phonemes. + """ + + device = next(model.parameters()).device + num_cuts = 0 + log_interval = 5 + + try: + num_batches = len(dl) + except TypeError: + num_batches = "?" + + for batch_idx, batch in enumerate(dl): + batch_size = len(batch["tokens"]) + + texts = [c.supervisions[0].normalized_text for c in batch["cut"]] + + audio = batch["audio"] + audio_lens = batch["audio_lens"].tolist() + cut_ids = [cut.id for cut in batch["cut"]] + + for i in range(batch_size): + output = synthesize( + model=model, + tokenizer=tokenizer, + n_timesteps=params.n_timesteps, + text=texts[i], + length_scale=params.length_scale, + temperature=params.temperature, + device=device, + ) + output["waveform"] = to_waveform(output["mel"], vocoder, denoiser) + + sf.write( + file=params.save_wav_dir / f"{cut_ids[i]}_pred.wav", + data=output["waveform"], + samplerate=params.data_args.sampling_rate, + subtype="PCM_16", + ) + sf.write( + file=params.save_wav_dir / f"{cut_ids[i]}_gt.wav", + data=audio[i].numpy(), + samplerate=params.data_args.sampling_rate, + subtype="PCM_16", + ) + + num_cuts += batch_size + + if batch_idx % log_interval == 0: + batch_str = f"{batch_idx}/{num_batches}" + + logging.info(f"batch {batch_str}, cuts processed until now is {num_cuts}") + + +@torch.inference_mode() +def main(): + parser = get_parser() + BakerZhTtsDataModule.add_arguments(parser) + args = parser.parse_args() + args.exp_dir = Path(args.exp_dir) + + params = get_params() + params.update(vars(args)) + + params.suffix = f"epoch-{params.epoch}" + + params.res_dir = params.exp_dir / "infer" / params.suffix + params.save_wav_dir = params.res_dir / "wav" + params.save_wav_dir.mkdir(parents=True, exist_ok=True) + + setup_logger(f"{params.res_dir}/log-infer-{params.suffix}") + logging.info("Infer started") + + device = torch.device("cpu") + if torch.cuda.is_available(): + device = torch.device("cuda", 0) + logging.info(f"Device: {device}") + + tokenizer = Tokenizer(params.tokens) + params.vocab_size = tokenizer.vocab_size + params.model_args.n_vocab = params.vocab_size + + with open(params.cmvn) as f: + stats = json.load(f) + params.data_args.data_statistics.mel_mean = stats["fbank_mean"] + params.data_args.data_statistics.mel_std = stats["fbank_std"] + + params.model_args.data_statistics.mel_mean = stats["fbank_mean"] + params.model_args.data_statistics.mel_std = stats["fbank_std"] + + # Number of ODE Solver steps + params.n_timesteps = 2 + + # Changes to the speaking rate + params.length_scale = 1.0 + + # Sampling temperature + params.temperature = 0.667 + logging.info(params) + + logging.info("About to create model") + model = get_model(params) + + load_checkpoint(f"{params.exp_dir}/epoch-{params.epoch}.pt", model) + model.to(device) + model.eval() + + # we need cut ids to organize tts results. + args.return_cuts = True + baker_zh = BakerZhTtsDataModule(args) + + test_cuts = baker_zh.test_cuts() + test_dl = baker_zh.test_dataloaders(test_cuts) + + if not Path(params.vocoder).is_file(): + raise ValueError(f"{params.vocoder} does not exist") + + vocoder = load_vocoder(params.vocoder) + vocoder.to(device) + + denoiser = Denoiser(vocoder, mode="zeros") + denoiser.to(device) + + if params.input_text is not None and params.output_wav is not None: + logging.info("Synthesizing a single text") + output = synthesize( + model=model, + tokenizer=tokenizer, + n_timesteps=params.n_timesteps, + text=params.input_text, + length_scale=params.length_scale, + temperature=params.temperature, + device=device, + ) + output["waveform"] = to_waveform(output["mel"], vocoder, denoiser) + + sf.write( + file=params.output_wav, + data=output["waveform"], + samplerate=params.sampling_rate, + subtype="PCM_16", + ) + else: + logging.info("Decoding the test set") + infer_dataset( + dl=test_dl, + params=params, + model=model, + vocoder=vocoder, + denoiser=denoiser, + tokenizer=tokenizer, + ) + + +if __name__ == "__main__": + main() diff --git a/egs/baker_zh/TTS/matcha/model.py b/egs/baker_zh/TTS/matcha/model.py new file mode 120000 index 000000000..8a1b812a9 --- /dev/null +++ b/egs/baker_zh/TTS/matcha/model.py @@ -0,0 +1 @@ +../../../ljspeech/TTS/matcha/model.py \ No newline at end of file diff --git a/egs/baker_zh/TTS/matcha/models b/egs/baker_zh/TTS/matcha/models new file mode 120000 index 000000000..09a862665 --- /dev/null +++ b/egs/baker_zh/TTS/matcha/models @@ -0,0 +1 @@ +../../../ljspeech/TTS/matcha/models \ No newline at end of file diff --git a/egs/baker_zh/TTS/matcha/monotonic_align b/egs/baker_zh/TTS/matcha/monotonic_align new file mode 120000 index 000000000..d0a0dd6b5 --- /dev/null +++ b/egs/baker_zh/TTS/matcha/monotonic_align @@ -0,0 +1 @@ +../../../ljspeech/TTS/matcha/monotonic_align \ No newline at end of file diff --git a/egs/baker_zh/TTS/matcha/onnx_pretrained.py b/egs/baker_zh/TTS/matcha/onnx_pretrained.py new file mode 100755 index 000000000..f6b7f7cae --- /dev/null +++ b/egs/baker_zh/TTS/matcha/onnx_pretrained.py @@ -0,0 +1,316 @@ +#!/usr/bin/env python3 +# Copyright 2024 Xiaomi Corp. (authors: Fangjun Kuang) + +""" +python3 ./matcha/onnx_pretrained.py \ + --acoustic-model ./model-steps-4.onnx \ + --vocoder ./hifigan_v2.onnx \ + --tokens ./data/tokens.txt \ + --lexicon ./lexicon.txt \ + --input-text "当夜幕降临，星光点点，伴随着微风拂面，我在静谧中感受着时光的流转，思念如涟漪荡漾，梦境如画卷展开，我与自然融为一体，沉静在这片宁静的美丽之中，感受着生命的奇迹与温柔。" \ + --output-wav ./b.wav +""" + +import argparse +import datetime as dt +import logging +import re +from typing import Dict, List + +import jieba +import onnxruntime as ort +import soundfile as sf +import torch +from infer import load_vocoder +from utils import intersperse + + +def get_parser(): + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + + parser.add_argument( + "--acoustic-model", + type=str, + required=True, + help="Path to the acoustic model", + ) + + parser.add_argument( + "--tokens", + type=str, + required=True, + help="Path to the tokens.txt", + ) + + parser.add_argument( + "--lexicon", + type=str, + required=True, + help="Path to the lexicon.txt", + ) + + parser.add_argument( + "--vocoder", + type=str, + required=True, + help="Path to the vocoder", + ) + + parser.add_argument( + "--input-text", + type=str, + required=True, + help="The text to generate speech for", + ) + + parser.add_argument( + "--output-wav", + type=str, + required=True, + help="The filename of the wave to save the generated speech", + ) + + return parser + + +class OnnxHifiGANModel: + def __init__( + self, + filename: str, + ): + session_opts = ort.SessionOptions() + session_opts.inter_op_num_threads = 1 + session_opts.intra_op_num_threads = 1 + + self.session_opts = session_opts + self.model = ort.InferenceSession( + filename, + sess_options=self.session_opts, + providers=["CPUExecutionProvider"], + ) + + for i in self.model.get_inputs(): + print(i) + + print("-----") + + for i in self.model.get_outputs(): + print(i) + + def __call__(self, x: torch.tensor): + assert x.ndim == 3, x.shape + assert x.shape[0] == 1, x.shape + + audio = self.model.run( + [self.model.get_outputs()[0].name], + { + self.model.get_inputs()[0].name: x.numpy(), + }, + )[0] + # audio: (batch_size, num_samples) + + return torch.from_numpy(audio) + + +class OnnxModel: + def __init__( + self, + filename: str, + ): + session_opts = ort.SessionOptions() + session_opts.inter_op_num_threads = 1 + session_opts.intra_op_num_threads = 2 + + self.session_opts = session_opts + self.model = ort.InferenceSession( + filename, + sess_options=self.session_opts, + providers=["CPUExecutionProvider"], + ) + + logging.info(f"{self.model.get_modelmeta().custom_metadata_map}") + metadata = self.model.get_modelmeta().custom_metadata_map + self.sample_rate = int(metadata["sample_rate"]) + + for i in self.model.get_inputs(): + print(i) + + print("-----") + + for i in self.model.get_outputs(): + print(i) + + def __call__(self, x: torch.tensor): + assert x.ndim == 2, x.shape + assert x.shape[0] == 1, x.shape + + x_lengths = torch.tensor([x.shape[1]], dtype=torch.int64) + print("x_lengths", x_lengths) + print("x", x.shape) + + noise_scale = torch.tensor([1.0], dtype=torch.float32) + length_scale = torch.tensor([1.0], dtype=torch.float32) + + mel = self.model.run( + [self.model.get_outputs()[0].name], + { + self.model.get_inputs()[0].name: x.numpy(), + self.model.get_inputs()[1].name: x_lengths.numpy(), + self.model.get_inputs()[2].name: noise_scale.numpy(), + self.model.get_inputs()[3].name: length_scale.numpy(), + }, + )[0] + # mel: (batch_size, feat_dim, num_frames) + + return torch.from_numpy(mel) + + +def read_tokens(filename: str) -> Dict[str, int]: + token2id = dict() + with open(filename, encoding="utf-8") as f: + for line in f.readlines(): + info = line.rstrip().split() + if len(info) == 1: + # case of space + token = " " + idx = int(info[0]) + else: + token, idx = info[0], int(info[1]) + assert token not in token2id, token + token2id[token] = idx + return token2id + + +def read_lexicon(filename: str) -> Dict[str, List[str]]: + word2token = dict() + with open(filename, encoding="utf-8") as f: + for line in f.readlines(): + info = line.rstrip().split() + w = info[0] + tokens = info[1:] + word2token[w] = tokens + return word2token + + +def convert_word_to_tokens(word2tokens: Dict[str, List[str]], word: str) -> List[str]: + if word in word2tokens: + return word2tokens[word] + + if len(word) == 1: + return [] + + ans = [] + for w in word: + t = convert_word_to_tokens(word2tokens, w) + ans.extend(t) + return ans + + +def normalize_text(text): + whiter_space_re = re.compile(r"\s+") + + punctuations_re = [ + (re.compile(x[0], re.IGNORECASE), x[1]) + for x in [ + ("，", ","), + ("。", "."), + ("！", "!"), + ("？", "?"), + ("“", '"'), + ("”", '"'), + ("‘", "'"), + ("’", "'"), + ("：", ":"), + ("、", ","), + ] + ] + + for regex, replacement in punctuations_re: + text = re.sub(regex, replacement, text) + return text + + +@torch.no_grad() +def main(): + params = get_parser().parse_args() + logging.info(vars(params)) + token2id = read_tokens(params.tokens) + word2tokens = read_lexicon(params.lexicon) + + text = normalize_text(params.input_text) + seg = jieba.cut(text) + tokens = [] + for s in seg: + if s in token2id: + tokens.append(s) + continue + + t = convert_word_to_tokens(word2tokens, s) + if t: + tokens.extend(t) + + model = OnnxModel(params.acoustic_model) + vocoder = OnnxHifiGANModel(params.vocoder) + + x = [] + for t in tokens: + if t in token2id: + x.append(token2id[t]) + + x = intersperse(x, item=token2id["_"]) + + x = torch.tensor(x, dtype=torch.int64).unsqueeze(0) + + start_t = dt.datetime.now() + mel = model(x) + end_t = dt.datetime.now() + + start_t2 = dt.datetime.now() + audio = vocoder(mel) + end_t2 = dt.datetime.now() + + print("audio", audio.shape) # (1, 1, num_samples) + audio = audio.squeeze() + + sample_rate = model.sample_rate + + t = (end_t - start_t).total_seconds() + t2 = (end_t2 - start_t2).total_seconds() + rtf_am = t * sample_rate / audio.shape[-1] + rtf_vocoder = t2 * sample_rate / audio.shape[-1] + print("RTF for acoustic model ", rtf_am) + print("RTF for vocoder", rtf_vocoder) + + # skip denoiser + sf.write(params.output_wav, audio, sample_rate, "PCM_16") + logging.info(f"Saved to {params.output_wav}") + + +if __name__ == "__main__": + torch.set_num_threads(1) + torch.set_num_interop_threads(1) + + formatter = "%(asctime)s %(levelname)s [%(filename)s:%(lineno)d] %(message)s" + + logging.basicConfig(format=formatter, level=logging.INFO) + main() + +""" + +|HifiGAN |RTF |#Parameters (M)| +|----------|-----|---------------| +|v1 |0.818| 13.926 | +|v2 |0.101| 0.925 | +|v3 |0.118| 1.462 | + +|Num steps|Acoustic Model RTF| +|---------|------------------| +| 2 | 0.039 | +| 3 | 0.047 | +| 4 | 0.071 | +| 5 | 0.076 | +| 6 | 0.103 | + +""" diff --git a/egs/baker_zh/TTS/matcha/tokenizer.py b/egs/baker_zh/TTS/matcha/tokenizer.py new file mode 100644 index 000000000..dda82c29d --- /dev/null +++ b/egs/baker_zh/TTS/matcha/tokenizer.py @@ -0,0 +1,119 @@ +# Copyright 2024 Xiaomi Corp. (authors: Fangjun Kuang) + +import logging +from typing import Dict, List + +import tacotron_cleaner.cleaners + +try: + from piper_phonemize import phonemize_espeak +except Exception as ex: + raise RuntimeError( + f"{ex}\nPlease run\n" + "pip install piper_phonemize -f https://k2-fsa.github.io/icefall/piper_phonemize.html" + ) + +from utils import intersperse + + +# This tokenizer supports both English and Chinese. +# We assume you have used +# ../local/convert_text_to_tokens.py +# to process your text +class Tokenizer(object): + def __init__(self, tokens: str): + """ + Args: + tokens: the file that maps tokens to ids + """ + # Parse token file + self.token2id: Dict[str, int] = {} + with open(tokens, "r", encoding="utf-8") as f: + for line in f.readlines(): + info = line.rstrip().split() + if len(info) == 1: + # case of space + token = " " + id = int(info[0]) + else: + token, id = info[0], int(info[1]) + assert token not in self.token2id, token + self.token2id[token] = id + + # Refer to https://github.com/rhasspy/piper/blob/master/TRAINING.md + self.pad_id = self.token2id["_"] # padding + self.space_id = self.token2id[" "] # word separator (whitespace) + + self.vocab_size = len(self.token2id) + + def texts_to_token_ids( + self, + sentence_list: List[List[str]], + intersperse_blank: bool = True, + lang: str = "en-us", + ) -> List[List[int]]: + """ + Args: + sentence_list: + A list of sentences. + intersperse_blank: + Whether to intersperse blanks in the token sequence. + lang: + Language argument passed to phonemize_espeak(). + + Returns: + Return a list of token id list [utterance][token_id] + """ + token_ids_list = [] + + for sentence in sentence_list: + tokens_list = [] + for word in sentence: + if word in self.token2id: + tokens_list.append(word) + continue + + tmp_tokens_list = phonemize_espeak(word, lang) + for t in tmp_tokens_list: + tokens_list.extend(t) + + token_ids = [] + for t in tokens_list: + if t not in self.token2id: + logging.warning(f"Skip OOV {t} {sentence}") + continue + + if t == " " and len(token_ids) > 0 and token_ids[-1] == self.space_id: + continue + + token_ids.append(self.token2id[t]) + + if intersperse_blank: + token_ids = intersperse(token_ids, self.pad_id) + + token_ids_list.append(token_ids) + + return token_ids_list + + +def test_tokenizer(): + import jieba + from pypinyin import Style, lazy_pinyin + + tokenizer = Tokenizer("data/tokens.txt") + text1 = "今天is Monday, tomorrow is 星期二" + text2 = "你好吗? 我很好, how about you?" + + text1 = list(jieba.cut(text1)) + text2 = list(jieba.cut(text2)) + tokens1 = lazy_pinyin(text1, style=Style.TONE3, tone_sandhi=True) + tokens2 = lazy_pinyin(text2, style=Style.TONE3, tone_sandhi=True) + print(tokens1) + print(tokens2) + + ids = tokenizer.texts_to_token_ids([tokens1, tokens2]) + print(ids) + + +if __name__ == "__main__": + test_tokenizer() diff --git a/egs/baker_zh/TTS/matcha/train.py b/egs/baker_zh/TTS/matcha/train.py new file mode 100755 index 000000000..ed2ba49b9 --- /dev/null +++ b/egs/baker_zh/TTS/matcha/train.py @@ -0,0 +1,717 @@ +#!/usr/bin/env python3 +# Copyright 2024 Xiaomi Corp. (authors: Fangjun Kuang) + + +import argparse +import json +import logging +from pathlib import Path +from shutil import copyfile +from typing import Any, Dict, Optional, Union + +import k2 +import torch +import torch.multiprocessing as mp +import torch.nn as nn +from lhotse.utils import fix_random_seed +from model import fix_len_compatibility +from models.matcha_tts import MatchaTTS +from tokenizer import Tokenizer +from torch.cuda.amp import GradScaler, autocast +from torch.nn.parallel import DistributedDataParallel as DDP +from torch.optim import Optimizer +from torch.utils.tensorboard import SummaryWriter +from tts_datamodule import BakerZhTtsDataModule +from utils import MetricsTracker + +from icefall.checkpoint import load_checkpoint, save_checkpoint +from icefall.dist import cleanup_dist, setup_dist +from icefall.env import get_env_info +from icefall.utils import AttributeDict, setup_logger, str2bool + + +def get_parser(): + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + + parser.add_argument( + "--world-size", + type=int, + default=1, + help="Number of GPUs for DDP training.", + ) + + parser.add_argument( + "--master-port", + type=int, + default=12335, + help="Master port to use for DDP training.", + ) + + parser.add_argument( + "--tensorboard", + type=str2bool, + default=True, + help="Should various information be logged in tensorboard.", + ) + + parser.add_argument( + "--num-epochs", + type=int, + default=1000, + help="Number of epochs to train.", + ) + + parser.add_argument( + "--start-epoch", + type=int, + default=1, + help="""Resume training from this epoch. It should be positive. + If larger than 1, it will load checkpoint from + exp-dir/epoch-{start_epoch-1}.pt + """, + ) + + parser.add_argument( + "--exp-dir", + type=Path, + default="matcha/exp", + help="""The experiment dir. + It specifies the directory where all training related + files, e.g., checkpoints, log, etc, are saved + """, + ) + + parser.add_argument( + "--tokens", + type=str, + default="data/tokens.txt", + help="""Path to vocabulary.""", + ) + + parser.add_argument( + "--cmvn", + type=str, + default="data/fbank/cmvn.json", + help="""Path to vocabulary.""", + ) + + parser.add_argument( + "--seed", + type=int, + default=42, + help="The seed for random generators intended for reproducibility", + ) + + parser.add_argument( + "--save-every-n", + type=int, + default=10, + help="""Save checkpoint after processing this number of epochs" + periodically. We save checkpoint to exp-dir/ whenever + params.cur_epoch % save_every_n == 0. The checkpoint filename + has the form: f'exp-dir/epoch-{params.cur_epoch}.pt'. + Since it will take around 1000 epochs, we suggest using a large + save_every_n to save disk space. + """, + ) + + parser.add_argument( + "--use-fp16", + type=str2bool, + default=False, + help="Whether to use half precision training.", + ) + + return parser + + +def get_data_statistics(): + return AttributeDict( + { + "mel_mean": 0, + "mel_std": 1, + } + ) + + +def _get_data_params() -> AttributeDict: + params = AttributeDict( + { + "name": "baker-zh", + "train_filelist_path": "./filelists/ljs_audio_text_train_filelist.txt", + "valid_filelist_path": "./filelists/ljs_audio_text_val_filelist.txt", + # "batch_size": 64, + # "num_workers": 1, + # "pin_memory": False, + "cleaners": ["english_cleaners2"], + "add_blank": True, + "n_spks": 1, + "n_fft": 1024, + "n_feats": 80, + "sampling_rate": 22050, + "hop_length": 256, + "win_length": 1024, + "f_min": 0, + "f_max": 8000, + "seed": 1234, + "load_durations": False, + "data_statistics": get_data_statistics(), + } + ) + return params + + +def _get_model_params() -> AttributeDict: + n_feats = 80 + filter_channels_dp = 256 + encoder_params_p_dropout = 0.1 + params = AttributeDict( + { + "n_spks": 1, # for baker-zh. + "spk_emb_dim": 64, + "n_feats": n_feats, + "out_size": None, # or use 172 + "prior_loss": True, + "use_precomputed_durations": False, + "data_statistics": get_data_statistics(), + "encoder": AttributeDict( + { + "encoder_type": "RoPE Encoder", # not used + "encoder_params": AttributeDict( + { + "n_feats": n_feats, + "n_channels": 192, + "filter_channels": 768, + "filter_channels_dp": filter_channels_dp, + "n_heads": 2, + "n_layers": 6, + "kernel_size": 3, + "p_dropout": encoder_params_p_dropout, + "spk_emb_dim": 64, + "n_spks": 1, + "prenet": True, + } + ), + "duration_predictor_params": AttributeDict( + { + "filter_channels_dp": filter_channels_dp, + "kernel_size": 3, + "p_dropout": encoder_params_p_dropout, + } + ), + } + ), + "decoder": AttributeDict( + { + "channels": [256, 256], + "dropout": 0.05, + "attention_head_dim": 64, + "n_blocks": 1, + "num_mid_blocks": 2, + "num_heads": 2, + "act_fn": "snakebeta", + } + ), + "cfm": AttributeDict( + { + "name": "CFM", + "solver": "euler", + "sigma_min": 1e-4, + } + ), + "optimizer": AttributeDict( + { + "lr": 1e-4, + "weight_decay": 0.0, + } + ), + } + ) + + return params + + +def get_params(): + params = AttributeDict( + { + "model_args": _get_model_params(), + "data_args": _get_data_params(), + "best_train_loss": float("inf"), + "best_valid_loss": float("inf"), + "best_train_epoch": -1, + "best_valid_epoch": -1, + "batch_idx_train": -1, # 0 + "log_interval": 10, + "valid_interval": 1500, + "env_info": get_env_info(), + } + ) + return params + + +def get_model(params): + m = MatchaTTS(**params.model_args) + return m + + +def load_checkpoint_if_available( + params: AttributeDict, model: nn.Module +) -> Optional[Dict[str, Any]]: + """Load checkpoint from file. + + If params.start_epoch is larger than 1, it will load the checkpoint from + `params.start_epoch - 1`. + + Apart from loading state dict for `model` and `optimizer` it also updates + `best_train_epoch`, `best_train_loss`, `best_valid_epoch`, + and `best_valid_loss` in `params`. + + Args: + params: + The return value of :func:`get_params`. + model: + The training model. + Returns: + Return a dict containing previously saved training info. + """ + if params.start_epoch > 1: + filename = params.exp_dir / f"epoch-{params.start_epoch-1}.pt" + else: + return None + + assert filename.is_file(), f"{filename} does not exist!" + + saved_params = load_checkpoint(filename, model=model) + + keys = [ + "best_train_epoch", + "best_valid_epoch", + "batch_idx_train", + "best_train_loss", + "best_valid_loss", + ] + for k in keys: + params[k] = saved_params[k] + + return saved_params + + +def prepare_input(batch: dict, tokenizer: Tokenizer, device: torch.device, params): + """Parse batch data""" + mel_mean = params.data_args.data_statistics.mel_mean + mel_std_inv = 1 / params.data_args.data_statistics.mel_std + for i in range(batch["features"].shape[0]): + n = batch["features_lens"][i] + batch["features"][i : i + 1, :n, :] = ( + batch["features"][i : i + 1, :n, :] - mel_mean + ) * mel_std_inv + batch["features"][i : i + 1, n:, :] = 0 + + audio = batch["audio"].to(device) + features = batch["features"].to(device) + audio_lens = batch["audio_lens"].to(device) + features_lens = batch["features_lens"].to(device) + tokens = batch["tokens"] + + tokens = tokenizer.texts_to_token_ids(tokens, intersperse_blank=True) + tokens = k2.RaggedTensor(tokens) + row_splits = tokens.shape.row_splits(1) + tokens_lens = row_splits[1:] - row_splits[:-1] + tokens = tokens.to(device) + tokens_lens = tokens_lens.to(device) + # a tensor of shape (B, T) + tokens = tokens.pad(mode="constant", padding_value=tokenizer.pad_id) + + max_feature_length = fix_len_compatibility(features.shape[1]) + if max_feature_length > features.shape[1]: + pad = max_feature_length - features.shape[1] + features = torch.nn.functional.pad(features, (0, 0, 0, pad)) + + # features_lens[features_lens.argmax()] += pad + + return audio, audio_lens, features, features_lens.long(), tokens, tokens_lens.long() + + +def compute_validation_loss( + params: AttributeDict, + model: Union[nn.Module, DDP], + tokenizer: Tokenizer, + valid_dl: torch.utils.data.DataLoader, + world_size: int = 1, + rank: int = 0, +) -> MetricsTracker: + """Run the validation process.""" + model.eval() + device = model.device if isinstance(model, DDP) else next(model.parameters()).device + get_losses = model.module.get_losses if isinstance(model, DDP) else model.get_losses + + # used to summary the stats over iterations + tot_loss = MetricsTracker() + + with torch.no_grad(): + for batch_idx, batch in enumerate(valid_dl): + ( + audio, + audio_lens, + features, + features_lens, + tokens, + tokens_lens, + ) = prepare_input(batch, tokenizer, device, params) + + losses = get_losses( + { + "x": tokens, + "x_lengths": tokens_lens, + "y": features.permute(0, 2, 1), + "y_lengths": features_lens, + "spks": None, # should change it for multi-speakers + "durations": None, + } + ) + + batch_size = len(batch["tokens"]) + + loss_info = MetricsTracker() + loss_info["samples"] = batch_size + + s = 0 + + for key, value in losses.items(): + v = value.detach().item() + loss_info[key] = v * batch_size + s += v * batch_size + + loss_info["tot_loss"] = s + + # summary stats + tot_loss = tot_loss + loss_info + + if world_size > 1: + tot_loss.reduce(device) + + loss_value = tot_loss["tot_loss"] / tot_loss["samples"] + if loss_value < params.best_valid_loss: + params.best_valid_epoch = params.cur_epoch + params.best_valid_loss = loss_value + + return tot_loss + + +def train_one_epoch( + params: AttributeDict, + model: Union[nn.Module, DDP], + tokenizer: Tokenizer, + optimizer: Optimizer, + train_dl: torch.utils.data.DataLoader, + valid_dl: torch.utils.data.DataLoader, + scaler: GradScaler, + tb_writer: Optional[SummaryWriter] = None, + world_size: int = 1, + rank: int = 0, +) -> None: + """Train the model for one epoch. + + The training loss from the mean of all frames is saved in + `params.train_loss`. It runs the validation process every + `params.valid_interval` batches. + + Args: + params: + It is returned by :func:`get_params`. + model: + The model for training. + optimizer: + The optimizer. + train_dl: + Dataloader for the training dataset. + valid_dl: + Dataloader for the validation dataset. + scaler: + The scaler used for mix precision training. + tb_writer: + Writer to write log messages to tensorboard. + """ + model.train() + device = model.device if isinstance(model, DDP) else next(model.parameters()).device + get_losses = model.module.get_losses if isinstance(model, DDP) else model.get_losses + + # used to track the stats over iterations in one epoch + tot_loss = MetricsTracker() + + saved_bad_model = False + + def save_bad_model(suffix: str = ""): + save_checkpoint( + filename=params.exp_dir / f"bad-model{suffix}-{rank}.pt", + model=model, + params=params, + optimizer=optimizer, + scaler=scaler, + rank=0, + ) + + for batch_idx, batch in enumerate(train_dl): + params.batch_idx_train += 1 + # audio: (N, T), float32 + # features: (N, T, C), float32 + # audio_lens, (N,), int32 + # features_lens, (N,), int32 + # tokens: List[List[str]], len(tokens) == N + + batch_size = len(batch["tokens"]) + + ( + audio, + audio_lens, + features, + features_lens, + tokens, + tokens_lens, + ) = prepare_input(batch, tokenizer, device, params) + try: + with autocast(enabled=params.use_fp16): + losses = get_losses( + { + "x": tokens, + "x_lengths": tokens_lens, + "y": features.permute(0, 2, 1), + "y_lengths": features_lens, + "spks": None, # should change it for multi-speakers + "durations": None, + } + ) + + loss = sum(losses.values()) + + scaler.scale(loss).backward() + scaler.step(optimizer) + scaler.update() + optimizer.zero_grad() + + loss_info = MetricsTracker() + loss_info["samples"] = batch_size + + s = 0 + + for key, value in losses.items(): + v = value.detach().item() + loss_info[key] = v * batch_size + s += v * batch_size + + loss_info["tot_loss"] = s + + tot_loss = tot_loss + loss_info + except: # noqa + save_bad_model() + raise + + if params.batch_idx_train % 100 == 0 and params.use_fp16: + # If the grad scale was less than 1, try increasing it. + # The _growth_interval of the grad scaler is configurable, + # but we can't configure it to have different + # behavior depending on the current grad scale. + cur_grad_scale = scaler._scale.item() + + if cur_grad_scale < 8.0 or ( + cur_grad_scale < 32.0 and params.batch_idx_train % 400 == 0 + ): + scaler.update(cur_grad_scale * 2.0) + if cur_grad_scale < 0.01: + if not saved_bad_model: + save_bad_model(suffix="-first-warning") + saved_bad_model = True + logging.warning(f"Grad scale is small: {cur_grad_scale}") + if cur_grad_scale < 1.0e-05: + save_bad_model() + raise RuntimeError( + f"grad_scale is too small, exiting: {cur_grad_scale}" + ) + + if params.batch_idx_train % params.log_interval == 0: + cur_grad_scale = scaler._scale.item() if params.use_fp16 else 1.0 + + logging.info( + f"Epoch {params.cur_epoch}, batch {batch_idx}, " + f"global_batch_idx: {params.batch_idx_train}, " + f"batch size: {batch_size}, " + f"loss[{loss_info}], tot_loss[{tot_loss}], " + + (f"grad_scale: {scaler._scale.item()}" if params.use_fp16 else "") + ) + + if tb_writer is not None: + loss_info.write_summary( + tb_writer, "train/current_", params.batch_idx_train + ) + tot_loss.write_summary(tb_writer, "train/tot_", params.batch_idx_train) + if params.use_fp16: + tb_writer.add_scalar( + "train/grad_scale", cur_grad_scale, params.batch_idx_train + ) + + if params.batch_idx_train % params.valid_interval == 1: + logging.info("Computing validation loss") + valid_info = compute_validation_loss( + params=params, + model=model, + tokenizer=tokenizer, + valid_dl=valid_dl, + world_size=world_size, + rank=rank, + ) + model.train() + logging.info(f"Epoch {params.cur_epoch}, validation: {valid_info}") + logging.info( + "Maximum memory allocated so far is " + f"{torch.cuda.max_memory_allocated()//1000000}MB" + ) + if tb_writer is not None: + valid_info.write_summary( + tb_writer, "train/valid_", params.batch_idx_train + ) + + loss_value = tot_loss["tot_loss"] / tot_loss["samples"] + params.train_loss = loss_value + if params.train_loss < params.best_train_loss: + params.best_train_epoch = params.cur_epoch + params.best_train_loss = params.train_loss + + +def run(rank, world_size, args): + params = get_params() + params.update(vars(args)) + + fix_random_seed(params.seed) + if world_size > 1: + setup_dist(rank, world_size, params.master_port) + + setup_logger(f"{params.exp_dir}/log/log-train") + logging.info("Training started") + + if args.tensorboard and rank == 0: + tb_writer = SummaryWriter(log_dir=f"{params.exp_dir}/tensorboard") + else: + tb_writer = None + + device = torch.device("cpu") + if torch.cuda.is_available(): + device = torch.device("cuda", rank) + logging.info(f"Device: {device}") + + tokenizer = Tokenizer(params.tokens) + params.pad_id = tokenizer.pad_id + params.vocab_size = tokenizer.vocab_size + params.model_args.n_vocab = params.vocab_size + + with open(params.cmvn) as f: + stats = json.load(f) + params.data_args.data_statistics.mel_mean = stats["fbank_mean"] + params.data_args.data_statistics.mel_std = stats["fbank_std"] + + params.model_args.data_statistics.mel_mean = stats["fbank_mean"] + params.model_args.data_statistics.mel_std = stats["fbank_std"] + + logging.info(params) + print(params) + + logging.info("About to create model") + model = get_model(params) + + num_param = sum([p.numel() for p in model.parameters()]) + logging.info(f"Number of parameters: {num_param}") + + assert params.start_epoch > 0, params.start_epoch + checkpoints = load_checkpoint_if_available(params=params, model=model) + + model.to(device) + + if world_size > 1: + logging.info("Using DDP") + model = DDP(model, device_ids=[rank], find_unused_parameters=True) + + optimizer = torch.optim.Adam(model.parameters(), **params.model_args.optimizer) + + logging.info("About to create datamodule") + + baker_zh = BakerZhTtsDataModule(args) + + train_cuts = baker_zh.train_cuts() + train_dl = baker_zh.train_dataloaders(train_cuts) + + valid_cuts = baker_zh.valid_cuts() + valid_dl = baker_zh.valid_dataloaders(valid_cuts) + + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + if checkpoints and "grad_scaler" in checkpoints: + logging.info("Loading grad scaler state dict") + scaler.load_state_dict(checkpoints["grad_scaler"]) + + for epoch in range(params.start_epoch, params.num_epochs + 1): + logging.info(f"Start epoch {epoch}") + fix_random_seed(params.seed + epoch - 1) + if "sampler" in train_dl: + train_dl.sampler.set_epoch(epoch - 1) + + params.cur_epoch = epoch + + if tb_writer is not None: + tb_writer.add_scalar("train/epoch", epoch, params.batch_idx_train) + + train_one_epoch( + params=params, + model=model, + tokenizer=tokenizer, + optimizer=optimizer, + train_dl=train_dl, + valid_dl=valid_dl, + scaler=scaler, + tb_writer=tb_writer, + world_size=world_size, + rank=rank, + ) + + if epoch % params.save_every_n == 0 or epoch == params.num_epochs: + filename = params.exp_dir / f"epoch-{params.cur_epoch}.pt" + save_checkpoint( + filename=filename, + params=params, + model=model, + optimizer=optimizer, + scaler=scaler, + rank=rank, + ) + if rank == 0: + if params.best_train_epoch == params.cur_epoch: + best_train_filename = params.exp_dir / "best-train-loss.pt" + copyfile(src=filename, dst=best_train_filename) + + if params.best_valid_epoch == params.cur_epoch: + best_valid_filename = params.exp_dir / "best-valid-loss.pt" + copyfile(src=filename, dst=best_valid_filename) + + logging.info("Done!") + + if world_size > 1: + torch.distributed.barrier() + cleanup_dist() + + +def main(): + parser = get_parser() + BakerZhTtsDataModule.add_arguments(parser) + args = parser.parse_args() + + world_size = args.world_size + assert world_size >= 1 + if world_size > 1: + mp.spawn(run, args=(world_size, args), nprocs=world_size, join=True) + else: + run(rank=0, world_size=1, args=args) + + +if __name__ == "__main__": + torch.set_num_threads(1) + torch.set_num_interop_threads(1) + main() diff --git a/egs/baker_zh/TTS/matcha/tts_datamodule.py b/egs/baker_zh/TTS/matcha/tts_datamodule.py new file mode 100644 index 000000000..d2bdfb96c --- /dev/null +++ b/egs/baker_zh/TTS/matcha/tts_datamodule.py @@ -0,0 +1,340 @@ +# Copyright 2021 Piotr Żelasko +# Copyright 2022-2023 Xiaomi Corporation (Authors: Mingshuang Luo, +# Zengwei Yao) +# +# See ../../../../LICENSE for clarification regarding multiple authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import argparse +import logging +from functools import lru_cache +from pathlib import Path +from typing import Any, Dict, Optional + +import torch +from fbank import MatchaFbank, MatchaFbankConfig +from lhotse import CutSet, load_manifest_lazy +from lhotse.dataset import ( # noqa F401 for PrecomputedFeatures + CutConcatenate, + CutMix, + DynamicBucketingSampler, + PrecomputedFeatures, + SimpleCutSampler, + SpeechSynthesisDataset, +) +from lhotse.dataset.input_strategies import ( # noqa F401 For AudioSamples + AudioSamples, + OnTheFlyFeatures, +) +from lhotse.utils import fix_random_seed +from torch.utils.data import DataLoader + +from icefall.utils import str2bool + + +class _SeedWorkers: + def __init__(self, seed: int): + self.seed = seed + + def __call__(self, worker_id: int): + fix_random_seed(self.seed + worker_id) + + +class BakerZhTtsDataModule: + """ + DataModule for tts experiments. + It assumes there is always one train and valid dataloader, + but there can be multiple test dataloaders (e.g. LibriSpeech test-clean + and test-other). + + It contains all the common data pipeline modules used in ASR + experiments, e.g.: + - dynamic batch size, + - bucketing samplers, + - cut concatenation, + - on-the-fly feature extraction + + This class should be derived for specific corpora used in ASR tasks. + """ + + def __init__(self, args: argparse.Namespace): + self.args = args + + @classmethod + def add_arguments(cls, parser: argparse.ArgumentParser): + group = parser.add_argument_group( + title="TTS data related options", + description="These options are used for the preparation of " + "PyTorch DataLoaders from Lhotse CutSet's -- they control the " + "effective batch sizes, sampling strategies, applied data " + "augmentations, etc.", + ) + + group.add_argument( + "--manifest-dir", + type=Path, + default=Path("data/fbank"), + help="Path to directory with train/valid/test cuts.", + ) + group.add_argument( + "--max-duration", + type=int, + default=200.0, + help="Maximum pooled recordings duration (seconds) in a " + "single batch. You can reduce it if it causes CUDA OOM.", + ) + group.add_argument( + "--bucketing-sampler", + type=str2bool, + default=True, + help="When enabled, the batches will come from buckets of " + "similar duration (saves padding frames).", + ) + group.add_argument( + "--num-buckets", + type=int, + default=30, + help="The number of buckets for the DynamicBucketingSampler" + "(you might want to increase it for larger datasets).", + ) + + group.add_argument( + "--on-the-fly-feats", + type=str2bool, + default=False, + help="When enabled, use on-the-fly cut mixing and feature " + "extraction. Will drop existing precomputed feature manifests " + "if available.", + ) + group.add_argument( + "--shuffle", + type=str2bool, + default=True, + help="When enabled (=default), the examples will be " + "shuffled for each epoch.", + ) + group.add_argument( + "--drop-last", + type=str2bool, + default=True, + help="Whether to drop last batch. Used by sampler.", + ) + group.add_argument( + "--return-cuts", + type=str2bool, + default=False, + help="When enabled, each batch will have the " + "field: batch['cut'] with the cuts that " + "were used to construct it.", + ) + group.add_argument( + "--num-workers", + type=int, + default=2, + help="The number of training dataloader workers that " + "collect the batches.", + ) + + group.add_argument( + "--input-strategy", + type=str, + default="PrecomputedFeatures", + help="AudioSamples or PrecomputedFeatures", + ) + + def train_dataloaders( + self, + cuts_train: CutSet, + sampler_state_dict: Optional[Dict[str, Any]] = None, + ) -> DataLoader: + """ + Args: + cuts_train: + CutSet for training. + sampler_state_dict: + The state dict for the training sampler. + """ + logging.info("About to create train dataset") + train = SpeechSynthesisDataset( + return_text=False, + return_tokens=True, + feature_input_strategy=eval(self.args.input_strategy)(), + return_cuts=self.args.return_cuts, + ) + + if self.args.on_the_fly_feats: + sampling_rate = 22050 + config = MatchaFbankConfig( + n_fft=1024, + n_mels=80, + sampling_rate=sampling_rate, + hop_length=256, + win_length=1024, + f_min=0, + f_max=8000, + ) + train = SpeechSynthesisDataset( + return_text=False, + return_tokens=True, + feature_input_strategy=OnTheFlyFeatures(MatchaFbank(config)), + return_cuts=self.args.return_cuts, + ) + + if self.args.bucketing_sampler: + logging.info("Using DynamicBucketingSampler.") + train_sampler = DynamicBucketingSampler( + cuts_train, + max_duration=self.args.max_duration, + shuffle=self.args.shuffle, + num_buckets=self.args.num_buckets, + buffer_size=self.args.num_buckets * 2000, + shuffle_buffer_size=self.args.num_buckets * 5000, + drop_last=self.args.drop_last, + ) + else: + logging.info("Using SimpleCutSampler.") + train_sampler = SimpleCutSampler( + cuts_train, + max_duration=self.args.max_duration, + shuffle=self.args.shuffle, + ) + logging.info("About to create train dataloader") + + if sampler_state_dict is not None: + logging.info("Loading sampler state dict") + train_sampler.load_state_dict(sampler_state_dict) + + # 'seed' is derived from the current random state, which will have + # previously been set in the main process. + seed = torch.randint(0, 100000, ()).item() + worker_init_fn = _SeedWorkers(seed) + + train_dl = DataLoader( + train, + sampler=train_sampler, + batch_size=None, + num_workers=self.args.num_workers, + persistent_workers=True, + pin_memory=True, + worker_init_fn=worker_init_fn, + ) + + return train_dl + + def valid_dataloaders(self, cuts_valid: CutSet) -> DataLoader: + logging.info("About to create dev dataset") + if self.args.on_the_fly_feats: + sampling_rate = 22050 + config = MatchaFbankConfig( + n_fft=1024, + n_mels=80, + sampling_rate=sampling_rate, + hop_length=256, + win_length=1024, + f_min=0, + f_max=8000, + ) + validate = SpeechSynthesisDataset( + return_text=False, + return_tokens=True, + feature_input_strategy=OnTheFlyFeatures(MatchaFbank(config)), + return_cuts=self.args.return_cuts, + ) + else: + validate = SpeechSynthesisDataset( + return_text=False, + return_tokens=True, + feature_input_strategy=eval(self.args.input_strategy)(), + return_cuts=self.args.return_cuts, + ) + valid_sampler = DynamicBucketingSampler( + cuts_valid, + max_duration=self.args.max_duration, + num_buckets=self.args.num_buckets, + shuffle=False, + ) + logging.info("About to create valid dataloader") + valid_dl = DataLoader( + validate, + sampler=valid_sampler, + batch_size=None, + num_workers=2, + persistent_workers=True, + pin_memory=True, + ) + + return valid_dl + + def test_dataloaders(self, cuts: CutSet) -> DataLoader: + logging.info("About to create test dataset") + if self.args.on_the_fly_feats: + sampling_rate = 22050 + config = MatchaFbankConfig( + n_fft=1024, + n_mels=80, + sampling_rate=sampling_rate, + hop_length=256, + win_length=1024, + f_min=0, + f_max=8000, + ) + test = SpeechSynthesisDataset( + return_text=False, + return_tokens=True, + feature_input_strategy=OnTheFlyFeatures(MatchaFbank(config)), + return_cuts=self.args.return_cuts, + ) + else: + test = SpeechSynthesisDataset( + return_text=False, + return_tokens=True, + feature_input_strategy=eval(self.args.input_strategy)(), + return_cuts=self.args.return_cuts, + ) + test_sampler = DynamicBucketingSampler( + cuts, + max_duration=self.args.max_duration, + num_buckets=self.args.num_buckets, + shuffle=False, + ) + logging.info("About to create test dataloader") + test_dl = DataLoader( + test, + batch_size=None, + sampler=test_sampler, + num_workers=self.args.num_workers, + ) + return test_dl + + @lru_cache() + def train_cuts(self) -> CutSet: + logging.info("About to get train cuts") + return load_manifest_lazy( + self.args.manifest_dir / "baker_zh_cuts_train.jsonl.gz" + ) + + @lru_cache() + def valid_cuts(self) -> CutSet: + logging.info("About to get validation cuts") + return load_manifest_lazy( + self.args.manifest_dir / "baker_zh_cuts_valid.jsonl.gz" + ) + + @lru_cache() + def test_cuts(self) -> CutSet: + logging.info("About to get test cuts") + return load_manifest_lazy( + self.args.manifest_dir / "baker_zh_cuts_test.jsonl.gz" + ) diff --git a/egs/baker_zh/TTS/matcha/utils.py b/egs/baker_zh/TTS/matcha/utils.py new file mode 120000 index 000000000..ceaaea196 --- /dev/null +++ b/egs/baker_zh/TTS/matcha/utils.py @@ -0,0 +1 @@ +../../../ljspeech/TTS/matcha/utils.py \ No newline at end of file diff --git a/egs/baker_zh/TTS/prepare.sh b/egs/baker_zh/TTS/prepare.sh new file mode 100755 index 000000000..e15e3d850 --- /dev/null +++ b/egs/baker_zh/TTS/prepare.sh @@ -0,0 +1,151 @@ +#!/usr/bin/env bash + +# fix segmentation fault reported in https://github.com/k2-fsa/icefall/issues/674 +export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python + +set -eou pipefail + +stage=-1 +stop_stage=100 + +dl_dir=$PWD/download +mkdir -p $dl_dir + +. shared/parse_options.sh || exit 1 + +# All files generated by this script are saved in "data". +# You can safely remove "data" and rerun this script to regenerate it. +mkdir -p data + +log() { + # This function is from espnet + local fname=${BASH_SOURCE[1]##*/} + echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*" +} + +log "dl_dir: $dl_dir" + +if [ $stage -le -1 ] && [ $stop_stage -ge -1 ]; then + log "Stage -1: build monotonic_align lib (used by ./matcha)" + for recipe in matcha; do + if [ ! -d $recipe/monotonic_align/build ]; then + cd $recipe/monotonic_align + python3 setup.py build_ext --inplace + cd ../../ + else + log "monotonic_align lib for $recipe already built" + fi + done +fi + +if [ $stage -le 0 ] && [ $stop_stage -ge 0 ]; then + log "Stage 0: Download data" + + # The directory $dl_dir/BANSYP contains the following 3 directories + + # ls -lh $dl_dir/BZNSYP/ + # total 0 + # drwxr-xr-x 10002 kuangfangjun root 0 Jan 4 2019 PhoneLabeling + # drwxr-xr-x 3 kuangfangjun root 0 Jan 31 2019 ProsodyLabeling + # drwxr-xr-x 10003 kuangfangjun root 0 Aug 26 17:45 Wave + + # If you have trouble accessing huggingface.co, please use + # + # cd $dl_dir + # wget https://huggingface.co/openspeech/BZNSYP/resolve/main/BZNSYP.tar.bz2 + # tar xf BZNSYP.tar.bz2 + # cd .. + + # If you have pre-downloaded it to /path/to/BZNSYP, you can create a symlink + # + # ln -sfv /path/to/BZNSYP $dl_dir/BZNSYP + # + if [ ! -d $dl_dir/BZNSYP/Wave ]; then + lhotse download baker-zh $dl_dir + fi +fi + +if [ $stage -le 1 ] && [ $stop_stage -ge 1 ]; then + log "Stage 1: Prepare baker-zh manifest" + # We assume that you have downloaded the baker corpus + # to $dl_dir/BZNSYP + mkdir -p data/manifests + if [ ! -e data/manifests/.baker-zh.done ]; then + lhotse prepare baker-zh $dl_dir/BZNSYP data/manifests + touch data/manifests/.baker-zh.done + fi +fi + +if [ $stage -le 2 ] && [ $stop_stage -ge 2 ]; then + log "Stage 2: Generate tokens.txt" + if [ ! -e data/tokens.txt ]; then + python3 ./local/generate_tokens.py --tokens data/tokens.txt + fi +fi + +if [ $stage -le 3 ] && [ $stop_stage -ge 3 ]; then + log "Stage 3: Generate raw cutset" + if [ ! -e data/manifests/baker_zh_cuts_raw.jsonl.gz ]; then + lhotse cut simple \ + -r ./data/manifests/baker_zh_recordings_all.jsonl.gz \ + -s ./data/manifests/baker_zh_supervisions_all.jsonl.gz \ + ./data/manifests/baker_zh_cuts_raw.jsonl.gz + fi +fi + +if [ $stage -le 4 ] && [ $stop_stage -ge 4 ]; then + log "Stage 4: Convert text to tokens" + if [ ! -e data/manifests/baker_zh_cuts.jsonl.gz ]; then + python3 ./local/convert_text_to_tokens.py \ + --in-file ./data/manifests/baker_zh_cuts_raw.jsonl.gz \ + --out-file ./data/manifests/baker_zh_cuts.jsonl.gz + fi +fi + +if [ $stage -le 5 ] && [ $stop_stage -ge 5 ]; then + log "Stage 5: Generate fbank (used by ./matcha)" + mkdir -p data/fbank + if [ ! -e data/fbank/.baker-zh.done ]; then + ./local/compute_fbank_baker_zh.py + touch data/fbank/.baker-zh.done + fi + + if [ ! -e data/fbank/.baker-zh-validated.done ]; then + log "Validating data/fbank for baker-zh (used by ./matcha)" + python3 ./local/validate_manifest.py \ + data/fbank/baker_zh_cuts.jsonl.gz + touch data/fbank/.baker-zh-validated.done + fi +fi + +if [ $stage -le 6 ] && [ $stop_stage -ge 6 ]; then + log "Stage 6: Split the baker-zh cuts into train, valid and test sets (used by ./matcha)" + if [ ! -e data/fbank/.baker_zh_split.done ]; then + lhotse subset --last 600 \ + data/fbank/baker_zh_cuts.jsonl.gz \ + data/fbank/baker_zh_cuts_validtest.jsonl.gz + lhotse subset --first 100 \ + data/fbank/baker_zh_cuts_validtest.jsonl.gz \ + data/fbank/baker_zh_cuts_valid.jsonl.gz + lhotse subset --last 500 \ + data/fbank/baker_zh_cuts_validtest.jsonl.gz \ + data/fbank/baker_zh_cuts_test.jsonl.gz + + rm data/fbank/baker_zh_cuts_validtest.jsonl.gz + + n=$(( $(gunzip -c data/fbank/baker_zh_cuts.jsonl.gz | wc -l) - 600 )) + + lhotse subset --first $n \ + data/fbank/baker_zh_cuts.jsonl.gz \ + data/fbank/baker_zh_cuts_train.jsonl.gz + + touch data/fbank/.baker_zh_split.done + fi +fi + +if [ $stage -le 7 ] && [ $stop_stage -ge 7 ]; then + log "Stage 6: Compute fbank mean and std (used by ./matcha)" + if [ ! -f ./data/fbank/cmvn.json ]; then + ./local/compute_fbank_statistics.py ./data/fbank/baker_zh_cuts_train.jsonl.gz ./data/fbank/cmvn.json + fi +fi diff --git a/egs/baker_zh/TTS/shared b/egs/baker_zh/TTS/shared new file mode 120000 index 000000000..4cbd91a7e --- /dev/null +++ b/egs/baker_zh/TTS/shared @@ -0,0 +1 @@ +../../../icefall/shared \ No newline at end of file diff --git a/egs/ljspeech/TTS/README.md b/egs/ljspeech/TTS/README.md index 39280437b..c9cfc22fd 100644 --- a/egs/ljspeech/TTS/README.md +++ b/egs/ljspeech/TTS/README.md @@ -166,7 +166,7 @@ To export the checkpoint to onnx: --tokens ./data/tokens.txt ``` -The above command generate the following files: +The above command generates the following files: - model-steps-2.onnx - model-steps-3.onnx diff --git a/egs/ljspeech/TTS/matcha/export_onnx.py b/egs/ljspeech/TTS/matcha/export_onnx.py index 623517431..39709cc36 100755 --- a/egs/ljspeech/TTS/matcha/export_onnx.py +++ b/egs/ljspeech/TTS/matcha/export_onnx.py @@ -93,14 +93,14 @@ class ModelWrapper(torch.nn.Module): self, x: torch.Tensor, x_lengths: torch.Tensor, - temperature: torch.Tensor, + noise_scale: torch.Tensor, length_scale: torch.Tensor, ) -> torch.Tensor: """ Args: : x: (batch_size, num_tokens), torch.int64 x_lengths: (batch_size,), torch.int64 - temperature: (1,), torch.float32 + noise_scale: (1,), torch.float32 length_scale (1,), torch.float32 Returns: audio: (batch_size, num_samples) @@ -110,7 +110,7 @@ class ModelWrapper(torch.nn.Module): x=x, x_lengths=x_lengths, n_timesteps=self.num_steps, - temperature=temperature, + temperature=noise_scale, length_scale=length_scale, )["mel"] # mel: (batch_size, feat_dim, num_frames) @@ -127,7 +127,6 @@ def main(): params.update(vars(args)) tokenizer = Tokenizer(params.tokens) - params.blank_id = tokenizer.pad_id params.vocab_size = tokenizer.vocab_size params.model_args.n_vocab = params.vocab_size @@ -153,14 +152,14 @@ def main(): # encoder has a large initial length x = torch.ones(1, 1000, dtype=torch.int64) x_lengths = torch.tensor([x.shape[1]], dtype=torch.int64) - temperature = torch.tensor([1.0]) + noise_scale = torch.tensor([1.0]) length_scale = torch.tensor([1.0]) opset_version = 14 filename = f"model-steps-{num_steps}.onnx" torch.onnx.export( wrapper, - (x, x_lengths, temperature, length_scale), + (x, x_lengths, noise_scale, length_scale), filename, opset_version=opset_version, input_names=["x", "x_length", "noise_scale", "length_scale"], diff --git a/egs/ljspeech/TTS/matcha/onnx_pretrained.py b/egs/ljspeech/TTS/matcha/onnx_pretrained.py index 6d92b16eb..19e9b49cb 100755 --- a/egs/ljspeech/TTS/matcha/onnx_pretrained.py +++ b/egs/ljspeech/TTS/matcha/onnx_pretrained.py @@ -132,7 +132,7 @@ class OnnxModel: print("x_lengths", x_lengths) print("x", x.shape) - temperature = torch.tensor([1.0], dtype=torch.float32) + noise_scale = torch.tensor([1.0], dtype=torch.float32) length_scale = torch.tensor([1.0], dtype=torch.float32) mel = self.model.run( @@ -140,7 +140,7 @@ class OnnxModel: { self.model.get_inputs()[0].name: x.numpy(), self.model.get_inputs()[1].name: x_lengths.numpy(), - self.model.get_inputs()[2].name: temperature.numpy(), + self.model.get_inputs()[2].name: noise_scale.numpy(), self.model.get_inputs()[3].name: length_scale.numpy(), }, )[0] From 3b263539cd34fb14b53d72339bc7c095028f4578 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Thu, 2 Jan 2025 15:54:34 +0800 Subject: [PATCH 22/59] Publish MatchaTTS onnx models trained with LJSpeech to huggingface (#1854) --- .github/scripts/docker/Dockerfile | 2 +- .github/scripts/ljspeech/TTS/run-matcha.sh | 33 +++++++++- .github/workflows/ljspeech.yml | 74 +++++++++++++++++++++- egs/ljspeech/TTS/README.md | 9 +++ egs/ljspeech/TTS/matcha/export_onnx.py | 4 ++ 5 files changed, 118 insertions(+), 4 deletions(-) diff --git a/.github/scripts/docker/Dockerfile b/.github/scripts/docker/Dockerfile index 94e8d8e1e..cf0523401 100644 --- a/.github/scripts/docker/Dockerfile +++ b/.github/scripts/docker/Dockerfile @@ -49,7 +49,7 @@ RUN pip install --no-cache-dir \ kaldifst \ kaldilm \ librosa \ - matplotlib \ + "matplotlib<=3.9.4" \ multi_quantization \ numba \ "numpy<2.0" \ diff --git a/.github/scripts/ljspeech/TTS/run-matcha.sh b/.github/scripts/ljspeech/TTS/run-matcha.sh index 954dd5bd8..bfb37fb6d 100755 --- a/.github/scripts/ljspeech/TTS/run-matcha.sh +++ b/.github/scripts/ljspeech/TTS/run-matcha.sh @@ -77,7 +77,7 @@ function export_onnx() { popd pushd data/fbank - rm -v *.json + rm -fv *.json curl -SL -O https://huggingface.co/csukuangfj/icefall-tts-ljspeech-matcha-en-2024-10-28/resolve/main/data/cmvn.json popd @@ -115,6 +115,37 @@ function export_onnx() { ls -lh /icefall/*.wav soxi /icefall/generated-matcha-tts-steps-6-*.wav + + cp ./model-steps-*.onnx /icefall + + d=matcha-icefall-en_US-ljspeech + mkdir $d + cp -v data/tokens.txt $d + cp model-steps-3.onnx $d + pushd $d + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/espeak-ng-data.tar.bz2 + tar xf espeak-ng-data.tar.bz2 + rm espeak-ng-data.tar.bz2 + +cat >README.md <=2.2.0`. + To export the Hifigan vocoder to onnx, please use: diff --git a/egs/ljspeech/TTS/matcha/export_onnx.py b/egs/ljspeech/TTS/matcha/export_onnx.py index 39709cc36..3c653fbf1 100755 --- a/egs/ljspeech/TTS/matcha/export_onnx.py +++ b/egs/ljspeech/TTS/matcha/export_onnx.py @@ -176,12 +176,16 @@ def main(): "language": "English", "voice": "en-us", "has_espeak": 1, + "jieba": 0, "n_speakers": 1, "sample_rate": 22050, "version": 1, + "pad_id": tokenizer.pad_id, "model_author": "icefall", "maintainer": "k2-fsa", + "use_eos_bos": 1, "dataset": "LJ Speech", + "dataset_url": "https://keithito.com/LJ-Speech-Dataset/", "num_ode_steps": num_steps, } add_meta_data(filename=filename, meta_data=meta_data) From 3b6d54007b7b9d0f2ee28ced3d91caed773ae3c1 Mon Sep 17 00:00:00 2001 From: Seonuk Kim <49300300+snkii@users.noreply.github.com> Date: Mon, 6 Jan 2025 14:17:02 +0900 Subject: [PATCH 23/59] Update conformer.py (#1857) * Update conformer.py feedforward dimention -> feedforward dimension * Update conformer.py feedforward dimention -> feedforward dimension * Update conformer.py feedforward dimention -> feedforward dimension * Update conformer.py feedforward dimention -> feedforward dimension * Update conformer.py feedforward dimention -> feedforward dimension * Update conformer.py feedforward dimention -> feedforward dimension * Update conformer.py feedforward dimention -> feedforward dimension * Update conformer.py feedforward dimention -> feedforward dimension * Update conformer.py feedforward dimention -> feedforward dimension --- egs/librispeech/ASR/conformer_ctc/conformer.py | 2 +- egs/librispeech/ASR/conformer_ctc2/conformer.py | 2 +- egs/librispeech/ASR/conformer_mmi/conformer.py | 2 +- egs/librispeech/ASR/pruned2_knowledge/conformer.py | 2 +- egs/librispeech/ASR/pruned_transducer_stateless2/conformer.py | 2 +- egs/librispeech/ASR/pruned_transducer_stateless5/conformer.py | 2 +- egs/librispeech/ASR/pruned_transducer_stateless6/conformer.py | 2 +- egs/librispeech/ASR/streaming_conformer_ctc/conformer.py | 2 +- egs/librispeech/ASR/transducer_stateless/conformer.py | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/egs/librispeech/ASR/conformer_ctc/conformer.py b/egs/librispeech/ASR/conformer_ctc/conformer.py index a1cfe6e75..3ac60e32f 100644 --- a/egs/librispeech/ASR/conformer_ctc/conformer.py +++ b/egs/librispeech/ASR/conformer_ctc/conformer.py @@ -32,7 +32,7 @@ class Conformer(Transformer): subsampling_factor (int): subsampling factor of encoder (the convolution layers before transformers) d_model (int): attention dimension nhead (int): number of head - dim_feedforward (int): feedforward dimention + dim_feedforward (int): feedforward dimension num_encoder_layers (int): number of encoder layers num_decoder_layers (int): number of decoder layers dropout (float): dropout rate diff --git a/egs/librispeech/ASR/conformer_ctc2/conformer.py b/egs/librispeech/ASR/conformer_ctc2/conformer.py index 09f1eb000..02ea80a46 100644 --- a/egs/librispeech/ASR/conformer_ctc2/conformer.py +++ b/egs/librispeech/ASR/conformer_ctc2/conformer.py @@ -42,7 +42,7 @@ class Conformer(Transformer): subsampling_factor (int): subsampling factor of encoder (the convolution layers before transformers) d_model (int): attention dimension, also the output dimension nhead (int): number of head - dim_feedforward (int): feedforward dimention + dim_feedforward (int): feedforward dimension num_encoder_layers (int): number of encoder layers num_decoder_layers (int): number of decoder layers dropout (float): dropout rate diff --git a/egs/librispeech/ASR/conformer_mmi/conformer.py b/egs/librispeech/ASR/conformer_mmi/conformer.py index 53e48eb13..cffe3df28 100644 --- a/egs/librispeech/ASR/conformer_mmi/conformer.py +++ b/egs/librispeech/ASR/conformer_mmi/conformer.py @@ -33,7 +33,7 @@ class Conformer(Transformer): subsampling_factor (int): subsampling factor of encoder (the convolution layers before transformers) d_model (int): attention dimension nhead (int): number of head - dim_feedforward (int): feedforward dimention + dim_feedforward (int): feedforward dimension num_encoder_layers (int): number of encoder layers num_decoder_layers (int): number of decoder layers dropout (float): dropout rate diff --git a/egs/librispeech/ASR/pruned2_knowledge/conformer.py b/egs/librispeech/ASR/pruned2_knowledge/conformer.py index de367c234..69cc59756 100644 --- a/egs/librispeech/ASR/pruned2_knowledge/conformer.py +++ b/egs/librispeech/ASR/pruned2_knowledge/conformer.py @@ -42,7 +42,7 @@ class Conformer(EncoderInterface): subsampling_factor (int): subsampling factor of encoder (the convolution layers before transformers) d_model (int): attention dimension, also the output dimension nhead (int): number of head - dim_feedforward (int): feedforward dimention + dim_feedforward (int): feedforward dimension num_encoder_layers (int): number of encoder layers dropout (float): dropout rate layer_dropout (float): layer-dropout rate. diff --git a/egs/librispeech/ASR/pruned_transducer_stateless2/conformer.py b/egs/librispeech/ASR/pruned_transducer_stateless2/conformer.py index ab46e233b..85e61ebab 100644 --- a/egs/librispeech/ASR/pruned_transducer_stateless2/conformer.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless2/conformer.py @@ -42,7 +42,7 @@ class Conformer(EncoderInterface): subsampling_factor (int): subsampling factor of encoder (the convolution layers before transformers) d_model (int): attention dimension, also the output dimension nhead (int): number of head - dim_feedforward (int): feedforward dimention + dim_feedforward (int): feedforward dimension num_encoder_layers (int): number of encoder layers dropout (float): dropout rate layer_dropout (float): layer-dropout rate. diff --git a/egs/librispeech/ASR/pruned_transducer_stateless5/conformer.py b/egs/librispeech/ASR/pruned_transducer_stateless5/conformer.py index 8bbceec61..968ea4150 100644 --- a/egs/librispeech/ASR/pruned_transducer_stateless5/conformer.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless5/conformer.py @@ -42,7 +42,7 @@ class Conformer(EncoderInterface): subsampling_factor (int): subsampling factor of encoder (the convolution layers before transformers) d_model (int): attention dimension, also the output dimension nhead (int): number of head - dim_feedforward (int): feedforward dimention + dim_feedforward (int): feedforward dimension num_encoder_layers (int): number of encoder layers dropout (float): dropout rate layer_dropout (float): layer-dropout rate. diff --git a/egs/librispeech/ASR/pruned_transducer_stateless6/conformer.py b/egs/librispeech/ASR/pruned_transducer_stateless6/conformer.py index 0667e7f61..8c1529500 100644 --- a/egs/librispeech/ASR/pruned_transducer_stateless6/conformer.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless6/conformer.py @@ -42,7 +42,7 @@ class Conformer(EncoderInterface): subsampling_factor (int): subsampling factor of encoder (the convolution layers before transformers) d_model (int): attention dimension, also the output dimension nhead (int): number of head - dim_feedforward (int): feedforward dimention + dim_feedforward (int): feedforward dimension num_encoder_layers (int): number of encoder layers dropout (float): dropout rate layer_dropout (float): layer-dropout rate. diff --git a/egs/librispeech/ASR/streaming_conformer_ctc/conformer.py b/egs/librispeech/ASR/streaming_conformer_ctc/conformer.py index 0b982f4bf..72842cc28 100644 --- a/egs/librispeech/ASR/streaming_conformer_ctc/conformer.py +++ b/egs/librispeech/ASR/streaming_conformer_ctc/conformer.py @@ -69,7 +69,7 @@ class Conformer(Transformer): subsampling_factor (int): subsampling factor of encoder (the convolution layers before transformers) d_model (int): attention dimension nhead (int): number of head - dim_feedforward (int): feedforward dimention + dim_feedforward (int): feedforward dimension num_encoder_layers (int): number of encoder layers num_decoder_layers (int): number of decoder layers dropout (float): dropout rate diff --git a/egs/librispeech/ASR/transducer_stateless/conformer.py b/egs/librispeech/ASR/transducer_stateless/conformer.py index 90b722bde..9b11df673 100644 --- a/egs/librispeech/ASR/transducer_stateless/conformer.py +++ b/egs/librispeech/ASR/transducer_stateless/conformer.py @@ -35,7 +35,7 @@ class Conformer(Transformer): subsampling_factor (int): subsampling factor of encoder (the convolution layers before transformers) d_model (int): attention dimension nhead (int): number of head - dim_feedforward (int): feedforward dimention + dim_feedforward (int): feedforward dimension num_encoder_layers (int): number of encoder layers dropout (float): dropout rate cnn_module_kernel (int): Kernel size of convolution module From 8d602806c3b141a5a75ac7d165292dc3d13d19b8 Mon Sep 17 00:00:00 2001 From: Seonuk Kim <49300300+snkii@users.noreply.github.com> Date: Mon, 6 Jan 2025 18:31:13 +0900 Subject: [PATCH 24/59] Update conformer.py (#1859) * Update conformer.py feedforward dimention -> feedforward dimension * Update conformer.py feedforward dimention -> feedforward dimension * Update conformer.py feedforward dimention -> feedforward dimension * Update conformer.py feedforward dimention -> feedforward dimension * Update conformer.py feedforward dimention -> feedforward dimension * Update conformer.py feedforward dimention -> feedforward dimension * Update conformer.py feedforward dimention -> feedforward dimension * Update conformer.py feedforward dimention -> feedforward dimension * Update conformer.py feedforward dimention -> feedforward dimension * Update conformer.py Swich -? Swish --- egs/librispeech/ASR/conformer_ctc/conformer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/egs/librispeech/ASR/conformer_ctc/conformer.py b/egs/librispeech/ASR/conformer_ctc/conformer.py index 3ac60e32f..ea793ce2f 100644 --- a/egs/librispeech/ASR/conformer_ctc/conformer.py +++ b/egs/librispeech/ASR/conformer_ctc/conformer.py @@ -902,7 +902,7 @@ class Swish(torch.nn.Module): """Construct an Swish object.""" def forward(self, x: Tensor) -> Tensor: - """Return Swich activation function.""" + """Return Swish activation function.""" return x * torch.sigmoid(x) From ab911129097dfc5c944918efdac7dd0b2d96aca2 Mon Sep 17 00:00:00 2001 From: Han Zhu <1106766460@qq.com> Date: Thu, 9 Jan 2025 15:05:38 +0800 Subject: [PATCH 25/59] Improve infinity-check (#1862) 1. Attach the inf-check hooks if the grad scale is getting too small. 2. Add try-catch to avoid OOM in the inf-check hooks. 3. Set warmup_start=0.1 to reduce chances of divergence --- egs/librispeech/ASR/zipformer/train.py | 25 ++++++++++++++++++------- icefall/hooks.py | 26 ++++++++++++++++++-------- 2 files changed, 36 insertions(+), 15 deletions(-) diff --git a/egs/librispeech/ASR/zipformer/train.py b/egs/librispeech/ASR/zipformer/train.py index c074c32ec..f8864d58b 100755 --- a/egs/librispeech/ASR/zipformer/train.py +++ b/egs/librispeech/ASR/zipformer/train.py @@ -1165,23 +1165,34 @@ def train_one_epoch( rank=rank, ) - if batch_idx % 100 == 0 and params.use_autocast: - # If the grad scale was less than 1, try increasing it. The _growth_interval - # of the grad scaler is configurable, but we can't configure it to have different - # behavior depending on the current grad scale. + if params.use_autocast: cur_grad_scale = scaler._scale.item() - if cur_grad_scale < 8.0 or (cur_grad_scale < 32.0 and batch_idx % 400 == 0): - scaler.update(cur_grad_scale * 2.0) if cur_grad_scale < 0.01: if not saved_bad_model: save_bad_model(suffix="-first-warning") saved_bad_model = True + if not params.inf_check: + register_inf_check_hooks(model) logging.warning(f"Grad scale is small: {cur_grad_scale}") + if cur_grad_scale < 1.0e-05: save_bad_model() raise_grad_scale_is_too_small_error(cur_grad_scale) + # If the grad scale was less than 1, try increasing it. The _growth_interval + # of the grad scaler is configurable, but we can't configure it to have different + # behavior depending on the current grad scale. + if ( + batch_idx % 25 == 0 + and cur_grad_scale < 2.0 + or batch_idx % 100 == 0 + and cur_grad_scale < 8.0 + or batch_idx % 400 == 0 + and cur_grad_scale < 32.0 + ): + scaler.update(cur_grad_scale * 2.0) + if batch_idx % params.log_interval == 0: cur_lr = max(scheduler.get_last_lr()) cur_grad_scale = scaler._scale.item() if params.use_autocast else 1.0 @@ -1335,7 +1346,7 @@ def run(rank, world_size, args): clipping_scale=2.0, ) - scheduler = Eden(optimizer, params.lr_batches, params.lr_epochs) + scheduler = Eden(optimizer, params.lr_batches, params.lr_epochs, warmup_start=0.1) if checkpoints and "optimizer" in checkpoints: logging.info("Loading optimizer state dict") diff --git a/icefall/hooks.py b/icefall/hooks.py index 83f2750fa..85583acbe 100644 --- a/icefall/hooks.py +++ b/icefall/hooks.py @@ -39,24 +39,34 @@ def register_inf_check_hooks(model: nn.Module) -> None: # default param _name is a way to capture the current value of the variable "name". def forward_hook(_module, _input, _output, _name=name): if isinstance(_output, Tensor): - if not torch.isfinite(_output.to(torch.float32).sum()): - logging.warning(f"The sum of {_name}.output is not finite") + try: + if not torch.isfinite(_output.to(torch.float32).sum()): + logging.warning(f"The sum of {_name}.output is not finite") + except RuntimeError: # e.g. CUDA out of memory + pass elif isinstance(_output, tuple): for i, o in enumerate(_output): if isinstance(o, tuple): o = o[0] if not isinstance(o, Tensor): continue - if not torch.isfinite(o.to(torch.float32).sum()): - logging.warning(f"The sum of {_name}.output[{i}] is not finite") + try: + if not torch.isfinite(o.to(torch.float32).sum()): + logging.warning( + f"The sum of {_name}.output[{i}] is not finite" + ) + except RuntimeError: # e.g. CUDA out of memory + pass # default param _name is a way to capture the current value of the variable "name". def backward_hook(_module, _input, _output, _name=name): if isinstance(_output, Tensor): - if not torch.isfinite(_output.to(torch.float32).sum()): - logging.warning( - f"The sum of {_name}.grad is not finite" # ": {_output}" - ) + try: + if not torch.isfinite(_output.to(torch.float32).sum()): + logging.warning(f"The sum of {_name}.grad is not finite") + except RuntimeError: # e.g. CUDA out of memory + pass + elif isinstance(_output, tuple): for i, o in enumerate(_output): if isinstance(o, tuple): From 8ab0352e60384929ab0dd77d2daf835d7deb4723 Mon Sep 17 00:00:00 2001 From: zr_jin Date: Thu, 16 Jan 2025 17:36:09 +0800 Subject: [PATCH 26/59] Update style_check.yml (#1866) --- .github/workflows/style_check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/style_check.yml b/.github/workflows/style_check.yml index 2a077fa91..3c971e231 100644 --- a/.github/workflows/style_check.yml +++ b/.github/workflows/style_check.yml @@ -36,7 +36,7 @@ jobs: strategy: matrix: os: [ubuntu-latest] - python-version: [3.10.15] + python-version: ["3.10"] fail-fast: false steps: From 79074ef0d45bc288c5e50479469d6a16b019ad8c Mon Sep 17 00:00:00 2001 From: zr_jin Date: Thu, 16 Jan 2025 20:51:28 +0800 Subject: [PATCH 27/59] =?UTF-8?q?removed=20the=20erroneous=20=E2=80=98?= =?UTF-8?q?=E2=80=99continual''=20implementation=20(#1865)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- egs/wenetspeech4tts/TTS/valle/infer.py | 43 ++++-------- egs/wenetspeech4tts/TTS/valle/valle.py | 97 -------------------------- 2 files changed, 14 insertions(+), 126 deletions(-) diff --git a/egs/wenetspeech4tts/TTS/valle/infer.py b/egs/wenetspeech4tts/TTS/valle/infer.py index 44a251c56..d98abb731 100644 --- a/egs/wenetspeech4tts/TTS/valle/infer.py +++ b/egs/wenetspeech4tts/TTS/valle/infer.py @@ -118,13 +118,6 @@ def get_args(): help="The temperature of AR Decoder top_k sampling.", ) - parser.add_argument( - "--continual", - type=str2bool, - default=False, - help="Do continual task.", - ) - parser.add_argument( "--repetition-aware-sampling", type=str2bool, @@ -262,29 +255,21 @@ def main(): ) # synthesis - if args.continual: - assert text == "" - encoded_frames = model.continual( - text_tokens.to(device), - text_tokens_lens.to(device), - audio_prompts, - ) - else: - enroll_x_lens = None - if text_prompts: - _, enroll_x_lens = text_collater( - [tokenize_text(text_tokenizer, text=f"{text_prompts}".strip())] - ) - encoded_frames = model.inference( - text_tokens.to(device), - text_tokens_lens.to(device), - audio_prompts, - enroll_x_lens=enroll_x_lens, - top_k=args.top_k, - temperature=args.temperature, - top_p=args.top_p, - ras=args.repetition_aware_sampling, + enroll_x_lens = None + if text_prompts: + _, enroll_x_lens = text_collater( + [tokenize_text(text_tokenizer, text=f"{text_prompts}".strip())] ) + encoded_frames = model.inference( + text_tokens.to(device), + text_tokens_lens.to(device), + audio_prompts, + enroll_x_lens=enroll_x_lens, + top_k=args.top_k, + temperature=args.temperature, + top_p=args.top_p, + ras=args.repetition_aware_sampling, + ) if audio_prompts != []: samples = audio_tokenizer.decode([(encoded_frames.transpose(2, 1), None)]) diff --git a/egs/wenetspeech4tts/TTS/valle/valle.py b/egs/wenetspeech4tts/TTS/valle/valle.py index 772317428..8f9b8fc3d 100644 --- a/egs/wenetspeech4tts/TTS/valle/valle.py +++ b/egs/wenetspeech4tts/TTS/valle/valle.py @@ -1564,103 +1564,6 @@ class VALLE(nn.Module): assert len(codes) == self.num_quantizers return torch.stack(codes, dim=-1) - def continual( - self, - x: torch.Tensor, - x_lens: torch.Tensor, - y: torch.Tensor, - ) -> torch.Tensor: - """ - Args: - x: - A 2-D tensor of shape (1, S). - x_lens: - A 1-D tensor of shape (1,). It contains the number of tokens in `x` - before padding. - y: - A 3-D tensor of shape (1, T, 8). - Returns: - Return the predicted audio code matrix. - """ - assert x.ndim == 2, x.shape - assert x_lens.ndim == 1, x_lens.shape - assert y.ndim == 3, y.shape - assert y.shape[0] == 1, y.shape - - assert torch.all(x_lens > 0) - assert self.num_quantizers == 8 - - # NOTE: x has been padded in TextTokenCollater - text = x - x = self.ar_text_embedding(text) - x = self.ar_text_prenet(x) - x = self.ar_text_position(x) - - text_len = x_lens.max() - - prefix_len = min(int(y.shape[1] * 0.5), 3 * 75) - - # AR Decoder - prompts = y[:, :prefix_len] - - codes = [y[:, prefix_len:, 0]] - # Non-AR Decoders - x = self.nar_text_embedding(text) - x = self.nar_text_prenet(x) - x = self.nar_text_position(x) - - y_emb = self.nar_audio_embeddings[0](y[..., 0]) - - if self.prefix_mode == 0: - for i, (predict_layer, embedding_layer) in enumerate( - zip( - self.nar_predict_layers, - self.nar_audio_embeddings[1:], - ) - ): - y_pos = self.nar_audio_position(y_emb) - y_pos = self.nar_audio_prenet(y_pos) - xy_pos = torch.concat([x, y_pos], dim=1) - - xy_dec, _ = self.nar_decoder( - (xy_pos, self.nar_stage_embeddings[i].weight) - ) - logits = predict_layer(xy_dec[:, text_len + prefix_len :]) - - samples = torch.argmax(logits, dim=-1) - codes.append(samples) - - if i < 6: - y_emb[:, :prefix_len] += embedding_layer(prompts[..., i + 1]) - y_emb[:, prefix_len:] += embedding_layer(samples) - else: - for j in range(1, 8): - y_emb[:, :prefix_len] += self.nar_audio_embeddings[j](prompts[..., j]) - - for i, (predict_layer, embedding_layer) in enumerate( - zip( - self.nar_predict_layers, - self.nar_audio_embeddings[1:], - ) - ): - y_pos = self.nar_audio_prenet(y_emb) - y_pos = self.nar_audio_position(y_pos) - xy_pos = torch.concat([x, y_pos], dim=1) - - xy_dec, _ = self.nar_decoder( - (xy_pos, self.nar_stage_embeddings[i].weight) - ) - logits = predict_layer(xy_dec[:, text_len + prefix_len :]) - - samples = torch.argmax(logits, dim=-1) - codes.append(samples) - - if i < 6: - y_emb[:, prefix_len:] += embedding_layer(samples) - - assert len(codes) == 8 - return torch.stack(codes, dim=-1) - def visualize( self, predicts: Tuple[torch.Tensor], From 39c466e8023b237c2a0599205f072595ffa95838 Mon Sep 17 00:00:00 2001 From: zr_jin Date: Tue, 21 Jan 2025 11:04:11 +0800 Subject: [PATCH 28/59] Update shared (#1868) --- egs/fluent_speech_commands/SLU/shared | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/egs/fluent_speech_commands/SLU/shared b/egs/fluent_speech_commands/SLU/shared index 9115c7e17..4cbd91a7e 120000 --- a/egs/fluent_speech_commands/SLU/shared +++ b/egs/fluent_speech_commands/SLU/shared @@ -1 +1 @@ -../../../icefall/shared/ +../../../icefall/shared \ No newline at end of file From dd5d7e358bf6a49eb2a31dff2dd2b943f792c1fc Mon Sep 17 00:00:00 2001 From: Yuekai Zhang Date: Mon, 27 Jan 2025 16:33:02 +0800 Subject: [PATCH 29/59] F5-TTS Training Recipe for WenetSpeech4TTS (#1846) * add f5 * add infer * add dit * add README * update pretrained checkpoint usage --------- Co-authored-by: yuekaiz Co-authored-by: yuekaiz Co-authored-by: yuekaiz Co-authored-by: zr_jin --- egs/ljspeech/TTS/matcha/fbank.py | 5 +- egs/wenetspeech4tts/TTS/README.md | 66 +- .../TTS/f5-tts/generate_averaged_model.py | 173 ++ egs/wenetspeech4tts/TTS/f5-tts/infer.py | 364 +++ .../TTS/f5-tts/model/README.md | 3 + egs/wenetspeech4tts/TTS/f5-tts/model/cfm.py | 326 +++ egs/wenetspeech4tts/TTS/f5-tts/model/dit.py | 210 ++ .../TTS/f5-tts/model/modules.py | 728 +++++ egs/wenetspeech4tts/TTS/f5-tts/model/utils.py | 206 ++ .../TTS/f5-tts/speech_synthesis.py | 104 + egs/wenetspeech4tts/TTS/f5-tts/train.py | 1178 ++++++++ .../TTS/f5-tts/tts_datamodule.py | 306 ++ egs/wenetspeech4tts/TTS/f5-tts/utils.py | 1 + egs/wenetspeech4tts/TTS/f5-tts/vocab.txt | 2545 +++++++++++++++++ egs/wenetspeech4tts/TTS/local/audio.py | 122 + .../TTS/local/compute_mel_feat.py | 218 ++ egs/wenetspeech4tts/TTS/local/compute_wer.sh | 26 + egs/wenetspeech4tts/TTS/local/fbank.py | 1 + .../TTS/local/offline-decode-files.py | 495 ++++ egs/wenetspeech4tts/TTS/prepare.sh | 41 + 20 files changed, 7115 insertions(+), 3 deletions(-) create mode 100644 egs/wenetspeech4tts/TTS/f5-tts/generate_averaged_model.py create mode 100644 egs/wenetspeech4tts/TTS/f5-tts/infer.py create mode 100644 egs/wenetspeech4tts/TTS/f5-tts/model/README.md create mode 100644 egs/wenetspeech4tts/TTS/f5-tts/model/cfm.py create mode 100644 egs/wenetspeech4tts/TTS/f5-tts/model/dit.py create mode 100644 egs/wenetspeech4tts/TTS/f5-tts/model/modules.py create mode 100644 egs/wenetspeech4tts/TTS/f5-tts/model/utils.py create mode 100644 egs/wenetspeech4tts/TTS/f5-tts/speech_synthesis.py create mode 100755 egs/wenetspeech4tts/TTS/f5-tts/train.py create mode 100644 egs/wenetspeech4tts/TTS/f5-tts/tts_datamodule.py create mode 120000 egs/wenetspeech4tts/TTS/f5-tts/utils.py create mode 100644 egs/wenetspeech4tts/TTS/f5-tts/vocab.txt create mode 100644 egs/wenetspeech4tts/TTS/local/audio.py create mode 100755 egs/wenetspeech4tts/TTS/local/compute_mel_feat.py create mode 100644 egs/wenetspeech4tts/TTS/local/compute_wer.sh create mode 120000 egs/wenetspeech4tts/TTS/local/fbank.py create mode 100755 egs/wenetspeech4tts/TTS/local/offline-decode-files.py diff --git a/egs/ljspeech/TTS/matcha/fbank.py b/egs/ljspeech/TTS/matcha/fbank.py index d729fa425..cc94a301f 100644 --- a/egs/ljspeech/TTS/matcha/fbank.py +++ b/egs/ljspeech/TTS/matcha/fbank.py @@ -17,6 +17,7 @@ class MatchaFbankConfig: win_length: int f_min: float f_max: float + device: str = "cuda" @register_extractor @@ -46,7 +47,7 @@ class MatchaFbank(FeatureExtractor): f"Mismatched sampling rate: extractor expects {expected_sr}, " f"got {sampling_rate}" ) - samples = torch.from_numpy(samples) + samples = torch.from_numpy(samples).to(self.device) assert samples.ndim == 2, samples.shape assert samples.shape[0] == 1, samples.shape @@ -81,7 +82,7 @@ class MatchaFbank(FeatureExtractor): mel, (0, 0, 0, num_frames - mel.shape[1]), mode="replicate" ).squeeze(0) - return mel.numpy() + return mel.cpu().numpy() @property def frame_shift(self) -> Seconds: diff --git a/egs/wenetspeech4tts/TTS/README.md b/egs/wenetspeech4tts/TTS/README.md index f35bb51c7..d47687acd 100644 --- a/egs/wenetspeech4tts/TTS/README.md +++ b/egs/wenetspeech4tts/TTS/README.md @@ -68,5 +68,69 @@ python3 valle/infer.py --output-dir demos_epoch_${epoch}_avg_${avg}_top_p_${top_ --text-extractor pypinyin_initials_finals --top-p ${top_p} ``` +# [F5-TTS](https://arxiv.org/abs/2410.06885) + +./f5-tts contains the code for training F5-TTS model. + +Generated samples and training logs of wenetspeech basic 7k hours data can be found [here](https://huggingface.co/yuekai/f5-tts-small-wenetspeech4tts-basic/tensorboard). + +Preparation: + +``` +bash prepare.sh --stage 5 --stop_stage 6 +``` +(Note: To compatiable with F5-TTS official checkpoint, we direclty use `vocab.txt` from [here.](https://github.com/SWivid/F5-TTS/blob/129014c5b43f135b0100d49a0c6804dd4cf673e1/data/Emilia_ZH_EN_pinyin/vocab.txt) To generate your own `vocab.txt`, you may refer to [the script](https://github.com/SWivid/F5-TTS/blob/main/src/f5_tts/train/datasets/prepare_emilia.py).) + +The training command is given below: + +``` +# docker: ghcr.io/swivid/f5-tts:main +# pip install k2==1.24.4.dev20241030+cuda12.4.torch2.4.0 -f https://k2-fsa.github.io/k2/cuda.html +# pip install kaldialign lhotse tensorboard bigvganinference sentencepiece + +world_size=8 +exp_dir=exp/f5-tts-small +python3 f5-tts/train.py --max-duration 700 --filter-min-duration 0.5 --filter-max-duration 20 \ + --num-buckets 6 --dtype "bfloat16" --save-every-n 5000 --valid-interval 10000 \ + --base-lr 7.5e-5 --warmup-steps 20000 --num-epochs 60 \ + --num-decoder-layers 18 --nhead 12 --decoder-dim 768 \ + --exp-dir ${exp_dir} --world-size ${world_size} +``` + +To inference with Icefall Wenetspeech4TTS trained F5-Small, use: +``` +huggingface-cli login +huggingface-cli download --local-dir seed_tts_eval yuekai/seed_tts_eval --repo-type dataset +huggingface-cli download --local-dir ${exp_dir} yuekai/f5-tts-small-wenetspeech4tts-basic +huggingface-cli download nvidia/bigvgan_v2_24khz_100band_256x --local-dir bigvgan_v2_24khz_100band_256x + +manifest=./seed_tts_eval/seedtts_testset/zh/meta.lst +model_path=f5-tts-small-wenetspeech4tts-basic/epoch-56-avg-14.pt +# skip +python3 f5-tts/generate_averaged_model.py \ + --epoch 56 \ + --avg 14 --decoder-dim 768 --nhead 12 --num-decoder-layers 18 \ + --exp-dir exp/f5_small + + +accelerate launch f5-tts/infer.py --nfe 16 --model-path $model_path --manifest-file $manifest --output-dir $output_dir --decoder-dim 768 --nhead 12 --num-decoder-layers 18 +bash local/compute_wer.sh $output_dir $manifest +``` + +To inference with official Emilia trained F5-Base, use: +``` +huggingface-cli login +huggingface-cli download --local-dir seed_tts_eval yuekai/seed_tts_eval --repo-type dataset +huggingface-cli download --local-dir F5-TTS SWivid/F5-TTS +huggingface-cli download nvidia/bigvgan_v2_24khz_100band_256x --local-dir bigvgan_v2_24khz_100band_256x + +manifest=./seed_tts_eval/seedtts_testset/zh/meta.lst +model_path=./F5-TTS/F5TTS_Base_bigvgan/model_1250000.pt + +accelerate launch f5-tts/infer.py --nfe 16 --model-path $model_path --manifest-file $manifest --output-dir $output_dir +bash local/compute_wer.sh $output_dir $manifest +``` + # Credits -- [vall-e](https://github.com/lifeiteng/vall-e) +- [VALL-E](https://github.com/lifeiteng/vall-e) +- [F5-TTS](https://github.com/SWivid/F5-TTS) diff --git a/egs/wenetspeech4tts/TTS/f5-tts/generate_averaged_model.py b/egs/wenetspeech4tts/TTS/f5-tts/generate_averaged_model.py new file mode 100644 index 000000000..f02358553 --- /dev/null +++ b/egs/wenetspeech4tts/TTS/f5-tts/generate_averaged_model.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +# +# Copyright 2021-2022 Xiaomi Corporation (Author: Yifan Yang) +# Copyright 2024 Yuekai Zhang +# +# See ../../../../LICENSE for clarification regarding multiple authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Usage: +(1) use the checkpoint exp_dir/epoch-xxx.pt +python3 bin/generate_averaged_model.py \ + --epoch 40 \ + --avg 5 \ + --exp-dir ${exp_dir} + +It will generate a file `epoch-28-avg-15.pt` in the given `exp_dir`. +You can later load it by `torch.load("epoch-28-avg-15.pt")`. +""" + + +import argparse +from pathlib import Path + +import k2 +import torch +from train import add_model_arguments, get_model + +from icefall.checkpoint import ( + average_checkpoints, + average_checkpoints_with_averaged_model, + find_checkpoints, +) +from icefall.utils import AttributeDict + + +def get_parser(): + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + + parser.add_argument( + "--epoch", + type=int, + default=30, + help="""It specifies the checkpoint to use for decoding. + Note: Epoch counts from 1. + You can specify --avg to use more checkpoints for model averaging.""", + ) + + parser.add_argument( + "--iter", + type=int, + default=0, + help="""If positive, --epoch is ignored and it + will use the checkpoint exp_dir/checkpoint-iter.pt. + You can specify --avg to use more checkpoints for model averaging. + """, + ) + + parser.add_argument( + "--avg", + type=int, + default=9, + help="Number of checkpoints to average. Automatically select " + "consecutive checkpoints before the checkpoint specified by " + "'--epoch' and '--iter'", + ) + + parser.add_argument( + "--exp-dir", + type=str, + default="zipformer/exp", + help="The experiment dir", + ) + add_model_arguments(parser) + return parser + + +@torch.no_grad() +def main(): + parser = get_parser() + + args = parser.parse_args() + args.exp_dir = Path(args.exp_dir) + + params = AttributeDict() + params.update(vars(args)) + + if params.iter > 0: + params.suffix = f"checkpoint-{params.iter}-avg-{params.avg}" + else: + params.suffix = f"epoch-{params.epoch}-avg-{params.avg}" + + print("Script started") + + device = torch.device("cpu") + print(f"Device: {device}") + + print("About to create model") + filename = f"{params.exp_dir}/epoch-{params.epoch}.pt" + checkpoint = torch.load(filename, map_location=device) + args = AttributeDict(checkpoint) + model = get_model(args) + + if params.iter > 0: + # TODO FIX ME + filenames = find_checkpoints(params.exp_dir, iteration=-params.iter)[ + : params.avg + 1 + ] + if len(filenames) == 0: + raise ValueError( + f"No checkpoints found for --iter {params.iter}, --avg {params.avg}" + ) + elif len(filenames) < params.avg + 1: + raise ValueError( + f"Not enough checkpoints ({len(filenames)}) found for" + f" --iter {params.iter}, --avg {params.avg}" + ) + filename_start = filenames[-1] + filename_end = filenames[0] + print( + "Calculating the averaged model over iteration checkpoints" + f" from {filename_start} (excluded) to {filename_end}" + ) + model.to(device) + model.load_state_dict( + average_checkpoints_with_averaged_model( + filename_start=filename_start, + filename_end=filename_end, + device=device, + ) + ) + filename = params.exp_dir / f"checkpoint-{params.iter}-avg-{params.avg}.pt" + torch.save({"model": model.state_dict()}, filename) + else: + assert params.avg > 0, params.avg + start = params.epoch - params.avg + assert start >= 1, start + filename_start = f"{params.exp_dir}/epoch-{start}.pt" + filename_end = f"{params.exp_dir}/epoch-{params.epoch}.pt" + print( + f"Calculating the averaged model over epoch range from " + f"{start} (excluded) to {params.epoch}" + ) + filenames = [ + f"{params.exp_dir}/epoch-{i}.pt" for i in range(start, params.epoch + 1) + ] + model.to(device) + model.load_state_dict(average_checkpoints(filenames, device=device)) + + filename = params.exp_dir / f"epoch-{params.epoch}-avg-{params.avg}.pt" + checkpoint["model"] = model.state_dict() + torch.save(checkpoint, filename) + + num_param = sum([p.numel() for p in model.parameters()]) + print(f"Number of model parameters: {num_param}") + + print("Done!") + + +if __name__ == "__main__": + main() diff --git a/egs/wenetspeech4tts/TTS/f5-tts/infer.py b/egs/wenetspeech4tts/TTS/f5-tts/infer.py new file mode 100644 index 000000000..02e5f0f4d --- /dev/null +++ b/egs/wenetspeech4tts/TTS/f5-tts/infer.py @@ -0,0 +1,364 @@ +#!/usr/bin/env python3 +# Modified from https://github.com/SWivid/F5-TTS/blob/main/src/f5_tts/eval/eval_infer_batch.py +""" +Usage: +# docker: ghcr.io/swivid/f5-tts:main +# pip install k2==1.24.4.dev20241030+cuda12.4.torch2.4.0 -f https://k2-fsa.github.io/k2/cuda.html +# pip install kaldialign lhotse tensorboard bigvganinference sentencepiece sherpa-onnx +# huggingface-cli download nvidia/bigvgan_v2_24khz_100band_256x --local-dir bigvgan_v2_24khz_100band_256x +manifest=/path/seed_tts_eval/seedtts_testset/zh/meta.lst +python3 f5-tts/generate_averaged_model.py \ + --epoch 56 \ + --avg 14 --decoder-dim 768 --nhead 12 --num-decoder-layers 18 \ + --exp-dir exp/f5_small +accelerate launch f5-tts/infer.py --nfe 16 --model-path $model_path --manifest-file $manifest --output-dir $output_dir --decoder-dim 768 --nhead 12 --num-decoder-layers 18 +bash local/compute_wer.sh $output_dir $manifest +""" +import argparse +import logging +import math +import os +import random +import time +from pathlib import Path + +import torch +import torch.nn.functional as F +import torchaudio +from accelerate import Accelerator +from bigvganinference import BigVGANInference +from model.cfm import CFM +from model.dit import DiT +from model.modules import MelSpec +from model.utils import convert_char_to_pinyin +from tqdm import tqdm +from train import ( + add_model_arguments, + get_model, + get_tokenizer, + load_F5_TTS_pretrained_checkpoint, +) + +from icefall.checkpoint import load_checkpoint + + +def get_parser(): + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + + parser.add_argument( + "--tokens", + type=str, + default="f5-tts/vocab.txt", + help="Path to the unique text tokens file", + ) + + parser.add_argument( + "--model-path", + type=str, + default="/home/yuekaiz/HF/F5-TTS/F5TTS_Base_bigvgan/model_1250000.pt", + help="Path to the unique text tokens file", + ) + + parser.add_argument( + "--seed", + type=int, + default=0, + help="The seed for random generators intended for reproducibility", + ) + + parser.add_argument( + "--nfe", + type=int, + default=16, + help="The number of steps for the neural ODE", + ) + + parser.add_argument( + "--manifest-file", + type=str, + default="/path/seed_tts_eval/seedtts_testset/zh/meta.lst", + help="The manifest file in seed_tts_eval format", + ) + + parser.add_argument( + "--output-dir", + type=Path, + default="results", + help="The output directory to save the generated wavs", + ) + + parser.add_argument("-ss", "--swaysampling", default=-1, type=float) + add_model_arguments(parser) + return parser.parse_args() + + +def get_inference_prompt( + metainfo, + speed=1.0, + tokenizer="pinyin", + polyphone=True, + target_sample_rate=24000, + n_fft=1024, + win_length=1024, + n_mel_channels=100, + hop_length=256, + mel_spec_type="bigvgan", + target_rms=0.1, + use_truth_duration=False, + infer_batch_size=1, + num_buckets=200, + min_secs=3, + max_secs=40, +): + prompts_all = [] + + min_tokens = min_secs * target_sample_rate // hop_length + max_tokens = max_secs * target_sample_rate // hop_length + + batch_accum = [0] * num_buckets + utts, ref_rms_list, ref_mels, ref_mel_lens, total_mel_lens, final_text_list = ( + [[] for _ in range(num_buckets)] for _ in range(6) + ) + + mel_spectrogram = MelSpec( + n_fft=n_fft, + hop_length=hop_length, + win_length=win_length, + n_mel_channels=n_mel_channels, + target_sample_rate=target_sample_rate, + mel_spec_type=mel_spec_type, + ) + + for utt, prompt_text, prompt_wav, gt_text, gt_wav in tqdm( + metainfo, desc="Processing prompts..." + ): + # Audio + ref_audio, ref_sr = torchaudio.load(prompt_wav) + ref_rms = torch.sqrt(torch.mean(torch.square(ref_audio))) + if ref_rms < target_rms: + ref_audio = ref_audio * target_rms / ref_rms + assert ( + ref_audio.shape[-1] > 5000 + ), f"Empty prompt wav: {prompt_wav}, or torchaudio backend issue." + if ref_sr != target_sample_rate: + resampler = torchaudio.transforms.Resample(ref_sr, target_sample_rate) + ref_audio = resampler(ref_audio) + + # Text + if len(prompt_text[-1].encode("utf-8")) == 1: + prompt_text = prompt_text + " " + text = [prompt_text + gt_text] + if tokenizer == "pinyin": + text_list = convert_char_to_pinyin(text, polyphone=polyphone) + else: + text_list = text + + # Duration, mel frame length + ref_mel_len = ref_audio.shape[-1] // hop_length + if use_truth_duration: + gt_audio, gt_sr = torchaudio.load(gt_wav) + if gt_sr != target_sample_rate: + resampler = torchaudio.transforms.Resample(gt_sr, target_sample_rate) + gt_audio = resampler(gt_audio) + total_mel_len = ref_mel_len + int(gt_audio.shape[-1] / hop_length / speed) + + # # test vocoder resynthesis + # ref_audio = gt_audio + else: + ref_text_len = len(prompt_text.encode("utf-8")) + gen_text_len = len(gt_text.encode("utf-8")) + total_mel_len = ref_mel_len + int( + ref_mel_len / ref_text_len * gen_text_len / speed + ) + + # to mel spectrogram + ref_mel = mel_spectrogram(ref_audio) + ref_mel = ref_mel.squeeze(0) + + # deal with batch + assert infer_batch_size > 0, "infer_batch_size should be greater than 0." + assert ( + min_tokens <= total_mel_len <= max_tokens + ), f"Audio {utt} has duration {total_mel_len*hop_length//target_sample_rate}s out of range [{min_secs}, {max_secs}]." + bucket_i = math.floor( + (total_mel_len - min_tokens) / (max_tokens - min_tokens + 1) * num_buckets + ) + + utts[bucket_i].append(utt) + ref_rms_list[bucket_i].append(ref_rms) + ref_mels[bucket_i].append(ref_mel) + ref_mel_lens[bucket_i].append(ref_mel_len) + total_mel_lens[bucket_i].append(total_mel_len) + final_text_list[bucket_i].extend(text_list) + + batch_accum[bucket_i] += total_mel_len + + if batch_accum[bucket_i] >= infer_batch_size: + prompts_all.append( + ( + utts[bucket_i], + ref_rms_list[bucket_i], + padded_mel_batch(ref_mels[bucket_i]), + ref_mel_lens[bucket_i], + total_mel_lens[bucket_i], + final_text_list[bucket_i], + ) + ) + batch_accum[bucket_i] = 0 + ( + utts[bucket_i], + ref_rms_list[bucket_i], + ref_mels[bucket_i], + ref_mel_lens[bucket_i], + total_mel_lens[bucket_i], + final_text_list[bucket_i], + ) = ( + [], + [], + [], + [], + [], + [], + ) + + # add residual + for bucket_i, bucket_frames in enumerate(batch_accum): + if bucket_frames > 0: + prompts_all.append( + ( + utts[bucket_i], + ref_rms_list[bucket_i], + padded_mel_batch(ref_mels[bucket_i]), + ref_mel_lens[bucket_i], + total_mel_lens[bucket_i], + final_text_list[bucket_i], + ) + ) + # not only leave easy work for last workers + random.seed(666) + random.shuffle(prompts_all) + + return prompts_all + + +def padded_mel_batch(ref_mels): + max_mel_length = torch.LongTensor([mel.shape[-1] for mel in ref_mels]).amax() + padded_ref_mels = [] + for mel in ref_mels: + padded_ref_mel = F.pad(mel, (0, max_mel_length - mel.shape[-1]), value=0) + padded_ref_mels.append(padded_ref_mel) + padded_ref_mels = torch.stack(padded_ref_mels) + padded_ref_mels = padded_ref_mels.permute(0, 2, 1) + return padded_ref_mels + + +def get_seedtts_testset_metainfo(metalst): + f = open(metalst) + lines = f.readlines() + f.close() + metainfo = [] + for line in lines: + assert len(line.strip().split("|")) == 4 + utt, prompt_text, prompt_wav, gt_text = line.strip().split("|") + utt = Path(utt).stem + gt_wav = os.path.join(os.path.dirname(metalst), "wavs", utt + ".wav") + if not os.path.isabs(prompt_wav): + prompt_wav = os.path.join(os.path.dirname(metalst), prompt_wav) + metainfo.append((utt, prompt_text, prompt_wav, gt_text, gt_wav)) + return metainfo + + +def main(): + args = get_parser() + + accelerator = Accelerator() + device = f"cuda:{accelerator.process_index}" + + metainfo = get_seedtts_testset_metainfo(args.manifest_file) + prompts_all = get_inference_prompt( + metainfo, + speed=1.0, + tokenizer="pinyin", + target_sample_rate=24_000, + n_mel_channels=100, + hop_length=256, + mel_spec_type="bigvgan", + target_rms=0.1, + use_truth_duration=False, + infer_batch_size=1, + ) + + vocoder = BigVGANInference.from_pretrained( + "./bigvgan_v2_24khz_100band_256x", use_cuda_kernel=False + ) + vocoder = vocoder.eval().to(device) + + model = get_model(args).eval().to(device) + checkpoint = torch.load(args.model_path, map_location="cpu") + if "ema_model_state_dict" in checkpoint or "model_state_dict" in checkpoint: + model = load_F5_TTS_pretrained_checkpoint(model, args.model_path) + else: + _ = load_checkpoint( + args.model_path, + model=model, + ) + + os.makedirs(args.output_dir, exist_ok=True) + + accelerator.wait_for_everyone() + start = time.time() + + with accelerator.split_between_processes(prompts_all) as prompts: + for prompt in tqdm(prompts, disable=not accelerator.is_local_main_process): + ( + utts, + ref_rms_list, + ref_mels, + ref_mel_lens, + total_mel_lens, + final_text_list, + ) = prompt + ref_mels = ref_mels.to(device) + ref_mel_lens = torch.tensor(ref_mel_lens, dtype=torch.long).to(device) + total_mel_lens = torch.tensor(total_mel_lens, dtype=torch.long).to(device) + + # Inference + with torch.inference_mode(): + generated, _ = model.sample( + cond=ref_mels, + text=final_text_list, + duration=total_mel_lens, + lens=ref_mel_lens, + steps=args.nfe, + cfg_strength=2.0, + sway_sampling_coef=args.swaysampling, + no_ref_audio=False, + seed=args.seed, + ) + for i, gen in enumerate(generated): + gen = gen[ref_mel_lens[i] : total_mel_lens[i], :].unsqueeze(0) + gen_mel_spec = gen.permute(0, 2, 1).to(torch.float32) + + generated_wave = vocoder(gen_mel_spec).squeeze(0).cpu() + target_rms = 0.1 + target_sample_rate = 24_000 + if ref_rms_list[i] < target_rms: + generated_wave = generated_wave * ref_rms_list[i] / target_rms + torchaudio.save( + f"{args.output_dir}/{utts[i]}.wav", + generated_wave, + target_sample_rate, + ) + + accelerator.wait_for_everyone() + if accelerator.is_main_process: + timediff = time.time() - start + print(f"Done batch inference in {timediff / 60 :.2f} minutes.") + + +if __name__ == "__main__": + formatter = "%(asctime)s %(levelname)s [%(filename)s:%(lineno)d] %(message)s" + logging.basicConfig(format=formatter, level=logging.INFO) + main() diff --git a/egs/wenetspeech4tts/TTS/f5-tts/model/README.md b/egs/wenetspeech4tts/TTS/f5-tts/model/README.md new file mode 100644 index 000000000..e4a7e2a7c --- /dev/null +++ b/egs/wenetspeech4tts/TTS/f5-tts/model/README.md @@ -0,0 +1,3 @@ +# Introduction +Files in this folder are copied from +https://github.com/SWivid/F5-TTS/tree/main/src/f5_tts/model diff --git a/egs/wenetspeech4tts/TTS/f5-tts/model/cfm.py b/egs/wenetspeech4tts/TTS/f5-tts/model/cfm.py new file mode 100644 index 000000000..349c7220e --- /dev/null +++ b/egs/wenetspeech4tts/TTS/f5-tts/model/cfm.py @@ -0,0 +1,326 @@ +""" +ein notation: +b - batch +n - sequence +nt - text sequence +nw - raw wave length +d - dimension +""" + +from __future__ import annotations + +from random import random +from typing import Callable + +import torch +import torch.nn.functional as F +from model.modules import MelSpec +from model.utils import ( + default, + exists, + lens_to_mask, + list_str_to_idx, + list_str_to_tensor, + mask_from_frac_lengths, +) +from torch import nn +from torch.nn.utils.rnn import pad_sequence +from torchdiffeq import odeint + + +class CFM(nn.Module): + def __init__( + self, + transformer: nn.Module, + sigma=0.0, + odeint_kwargs: dict = dict( + # atol = 1e-5, + # rtol = 1e-5, + method="euler" # 'midpoint' + ), + audio_drop_prob=0.3, + cond_drop_prob=0.2, + num_channels=None, + mel_spec_module: nn.Module | None = None, + mel_spec_kwargs: dict = dict(), + frac_lengths_mask: tuple[float, float] = (0.7, 1.0), + vocab_char_map: dict[str:int] | None = None, + ): + super().__init__() + + self.frac_lengths_mask = frac_lengths_mask + + # mel spec + self.mel_spec = default(mel_spec_module, MelSpec(**mel_spec_kwargs)) + num_channels = default(num_channels, self.mel_spec.n_mel_channels) + self.num_channels = num_channels + + # classifier-free guidance + self.audio_drop_prob = audio_drop_prob + self.cond_drop_prob = cond_drop_prob + + # transformer + self.transformer = transformer + dim = transformer.dim + self.dim = dim + + # conditional flow related + self.sigma = sigma + + # sampling related + self.odeint_kwargs = odeint_kwargs + + # vocab map for tokenization + self.vocab_char_map = vocab_char_map + + @property + def device(self): + return next(self.parameters()).device + + @torch.no_grad() + def sample( + self, + cond: float["b n d"] | float["b nw"], # noqa: F722 + text: int["b nt"] | list[str], # noqa: F722 + duration: int | int["b"], # noqa: F821 + *, + lens: int["b"] | None = None, # noqa: F821 + steps=32, + cfg_strength=1.0, + sway_sampling_coef=None, + seed: int | None = None, + max_duration=4096, + vocoder: Callable[[float["b d n"]], float["b nw"]] | None = None, # noqa: F722 + no_ref_audio=False, + duplicate_test=False, + t_inter=0.1, + edit_mask=None, + ): + self.eval() + # raw wave + + if cond.ndim == 2: + cond = self.mel_spec(cond) + cond = cond.permute(0, 2, 1) + assert cond.shape[-1] == self.num_channels + + cond = cond.to(next(self.parameters()).dtype) + + batch, cond_seq_len, device = *cond.shape[:2], cond.device + if not exists(lens): + lens = torch.full((batch,), cond_seq_len, device=device, dtype=torch.long) + + # text + + if isinstance(text, list): + if exists(self.vocab_char_map): + text = list_str_to_idx(text, self.vocab_char_map).to(device) + else: + text = list_str_to_tensor(text).to(device) + assert text.shape[0] == batch + + if exists(text): + text_lens = (text != -1).sum(dim=-1) + lens = torch.maximum( + text_lens, lens + ) # make sure lengths are at least those of the text characters + + # duration + + cond_mask = lens_to_mask(lens) + if edit_mask is not None: + cond_mask = cond_mask & edit_mask + + if isinstance(duration, int): + duration = torch.full((batch,), duration, device=device, dtype=torch.long) + + duration = torch.maximum( + lens + 1, duration + ) # just add one token so something is generated + duration = duration.clamp(max=max_duration) + max_duration = duration.amax() + + # duplicate test corner for inner time step oberservation + if duplicate_test: + test_cond = F.pad( + cond, (0, 0, cond_seq_len, max_duration - 2 * cond_seq_len), value=0.0 + ) + + cond = F.pad(cond, (0, 0, 0, max_duration - cond_seq_len), value=0.0) + cond_mask = F.pad( + cond_mask, (0, max_duration - cond_mask.shape[-1]), value=False + ) + cond_mask = cond_mask.unsqueeze(-1) + step_cond = torch.where( + cond_mask, cond, torch.zeros_like(cond) + ) # allow direct control (cut cond audio) with lens passed in + + if batch > 1: + mask = lens_to_mask(duration) + else: # save memory and speed up, as single inference need no mask currently + mask = None + + # test for no ref audio + if no_ref_audio: + cond = torch.zeros_like(cond) + + # neural ode + + def fn(t, x): + # at each step, conditioning is fixed + # step_cond = torch.where(cond_mask, cond, torch.zeros_like(cond)) + + # predict flow + pred = self.transformer( + x=x, + cond=step_cond, + text=text, + time=t, + mask=mask, + drop_audio_cond=False, + drop_text=False, + ) + if cfg_strength < 1e-5: + return pred + + null_pred = self.transformer( + x=x, + cond=step_cond, + text=text, + time=t, + mask=mask, + drop_audio_cond=True, + drop_text=True, + ) + return pred + (pred - null_pred) * cfg_strength + + # noise input + # to make sure batch inference result is same with different batch size, and for sure single inference + # still some difference maybe due to convolutional layers + y0 = [] + for dur in duration: + if exists(seed): + torch.manual_seed(seed) + y0.append( + torch.randn( + dur, self.num_channels, device=self.device, dtype=step_cond.dtype + ) + ) + y0 = pad_sequence(y0, padding_value=0, batch_first=True) + + t_start = 0 + + # duplicate test corner for inner time step oberservation + if duplicate_test: + t_start = t_inter + y0 = (1 - t_start) * y0 + t_start * test_cond + steps = int(steps * (1 - t_start)) + + t = torch.linspace( + t_start, 1, steps + 1, device=self.device, dtype=step_cond.dtype + ) + if sway_sampling_coef is not None: + t = t + sway_sampling_coef * (torch.cos(torch.pi / 2 * t) - 1 + t) + + trajectory = odeint(fn, y0, t, **self.odeint_kwargs) + + sampled = trajectory[-1] + out = sampled + out = torch.where(cond_mask, cond, out) + + if exists(vocoder): + out = out.permute(0, 2, 1) + out = vocoder(out) + + return out, trajectory + + def forward( + self, + inp: float["b n d"] | float["b nw"], # mel or raw wave # noqa: F722 + text: int["b nt"] | list[str], # noqa: F722 + *, + lens: int["b"] | None = None, # noqa: F821 + noise_scheduler: str | None = None, + ): + # handle raw wave + if inp.ndim == 2: + inp = self.mel_spec(inp) + inp = inp.permute(0, 2, 1) + assert inp.shape[-1] == self.num_channels + + batch, seq_len, dtype, device, _σ1 = ( + *inp.shape[:2], + inp.dtype, + self.device, + self.sigma, + ) + + # handle text as string + if isinstance(text, list): + if exists(self.vocab_char_map): + text = list_str_to_idx(text, self.vocab_char_map).to(device) + else: + text = list_str_to_tensor(text).to(device) + assert text.shape[0] == batch + + # lens and mask + if not exists(lens): + lens = torch.full((batch,), seq_len, device=device) + + mask = lens_to_mask( + lens, length=seq_len + ) # useless here, as collate_fn will pad to max length in batch + + # get a random span to mask out for training conditionally + frac_lengths = ( + torch.zeros((batch,), device=self.device) + .float() + .uniform_(*self.frac_lengths_mask) + ) + rand_span_mask = mask_from_frac_lengths(lens, frac_lengths) + + if exists(mask): + rand_span_mask &= mask + + # mel is x1 + x1 = inp + + # x0 is gaussian noise + x0 = torch.randn_like(x1) + + # time step + time = torch.rand((batch,), dtype=dtype, device=self.device) + # TODO. noise_scheduler + + # sample xt (φ_t(x) in the paper) + t = time.unsqueeze(-1).unsqueeze(-1) + φ = (1 - t) * x0 + t * x1 + flow = x1 - x0 + + # only predict what is within the random mask span for infilling + cond = torch.where(rand_span_mask[..., None], torch.zeros_like(x1), x1) + + # transformer and cfg training with a drop rate + drop_audio_cond = random() < self.audio_drop_prob # p_drop in voicebox paper + if random() < self.cond_drop_prob: # p_uncond in voicebox paper + drop_audio_cond = True + drop_text = True + else: + drop_text = False + + # if want rigourously mask out padding, record in collate_fn in dataset.py, and pass in here + # adding mask will use more memory, thus also need to adjust batchsampler with scaled down threshold for long sequences + pred = self.transformer( + x=φ, + cond=cond, + text=text, + time=time, + drop_audio_cond=drop_audio_cond, + drop_text=drop_text, + ) + + # flow matching loss + loss = F.mse_loss(pred, flow, reduction="none") + loss = loss[rand_span_mask] + + return loss.mean(), cond, pred diff --git a/egs/wenetspeech4tts/TTS/f5-tts/model/dit.py b/egs/wenetspeech4tts/TTS/f5-tts/model/dit.py new file mode 100644 index 000000000..966fabfdd --- /dev/null +++ b/egs/wenetspeech4tts/TTS/f5-tts/model/dit.py @@ -0,0 +1,210 @@ +""" +ein notation: +b - batch +n - sequence +nt - text sequence +nw - raw wave length +d - dimension +""" + +from __future__ import annotations + +import torch +import torch.nn.functional as F +from model.modules import ( + AdaLayerNormZero_Final, + ConvNeXtV2Block, + ConvPositionEmbedding, + DiTBlock, + TimestepEmbedding, + get_pos_embed_indices, + precompute_freqs_cis, +) +from torch import nn +from x_transformers.x_transformers import RotaryEmbedding + +# Text embedding + + +class TextEmbedding(nn.Module): + def __init__(self, text_num_embeds, text_dim, conv_layers=0, conv_mult=2): + super().__init__() + self.text_embed = nn.Embedding( + text_num_embeds + 1, text_dim + ) # use 0 as filler token + + if conv_layers > 0: + self.extra_modeling = True + self.precompute_max_pos = 4096 # ~44s of 24khz audio + self.register_buffer( + "freqs_cis", + precompute_freqs_cis(text_dim, self.precompute_max_pos), + persistent=False, + ) + self.text_blocks = nn.Sequential( + *[ + ConvNeXtV2Block(text_dim, text_dim * conv_mult) + for _ in range(conv_layers) + ] + ) + else: + self.extra_modeling = False + + def forward(self, text: int["b nt"], seq_len, drop_text=False): # noqa: F722 + text = ( + text + 1 + ) # use 0 as filler token. preprocess of batch pad -1, see list_str_to_idx() + text = text[ + :, :seq_len + ] # curtail if character tokens are more than the mel spec tokens + batch, text_len = text.shape[0], text.shape[1] + text = F.pad(text, (0, seq_len - text_len), value=0) + + if drop_text: # cfg for text + text = torch.zeros_like(text) + + text = self.text_embed(text) # b n -> b n d + + # possible extra modeling + if self.extra_modeling: + # sinus pos emb + batch_start = torch.zeros((batch,), dtype=torch.long) + pos_idx = get_pos_embed_indices( + batch_start, seq_len, max_pos=self.precompute_max_pos + ) + text_pos_embed = self.freqs_cis[pos_idx] + text = text + text_pos_embed + + # convnextv2 blocks + text = self.text_blocks(text) + + return text + + +# noised input audio and context mixing embedding + + +class InputEmbedding(nn.Module): + def __init__(self, mel_dim, text_dim, out_dim): + super().__init__() + self.proj = nn.Linear(mel_dim * 2 + text_dim, out_dim) + self.conv_pos_embed = ConvPositionEmbedding(dim=out_dim) + + def forward( + self, + x: float["b n d"], # noqa: F722 + cond: float["b n d"], # noqa: F722 + text_embed: float["b n d"], # noqa: F722 + drop_audio_cond=False, + ): + if drop_audio_cond: # cfg for cond audio + cond = torch.zeros_like(cond) + + x = self.proj(torch.cat((x, cond, text_embed), dim=-1)) + x = self.conv_pos_embed(x) + x + return x + + +# Transformer backbone using DiT blocks + + +class DiT(nn.Module): + def __init__( + self, + *, + dim, + depth=8, + heads=8, + dim_head=64, + dropout=0.1, + ff_mult=4, + mel_dim=100, + text_num_embeds=256, + text_dim=None, + conv_layers=0, + long_skip_connection=False, + checkpoint_activations=False, + ): + super().__init__() + + self.time_embed = TimestepEmbedding(dim) + if text_dim is None: + text_dim = mel_dim + self.text_embed = TextEmbedding( + text_num_embeds, text_dim, conv_layers=conv_layers + ) + self.input_embed = InputEmbedding(mel_dim, text_dim, dim) + + self.rotary_embed = RotaryEmbedding(dim_head) + + self.dim = dim + self.depth = depth + + self.transformer_blocks = nn.ModuleList( + [ + DiTBlock( + dim=dim, + heads=heads, + dim_head=dim_head, + ff_mult=ff_mult, + dropout=dropout, + ) + for _ in range(depth) + ] + ) + self.long_skip_connection = ( + nn.Linear(dim * 2, dim, bias=False) if long_skip_connection else None + ) + + self.norm_out = AdaLayerNormZero_Final(dim) # final modulation + self.proj_out = nn.Linear(dim, mel_dim) + + self.checkpoint_activations = checkpoint_activations + + def ckpt_wrapper(self, module): + # https://github.com/chuanyangjin/fast-DiT/blob/main/models.py + def ckpt_forward(*inputs): + outputs = module(*inputs) + return outputs + + return ckpt_forward + + def forward( + self, + x: float["b n d"], # nosied input audio # noqa: F722 + cond: float["b n d"], # masked cond audio # noqa: F722 + text: int["b nt"], # text # noqa: F722 + time: float["b"] | float[""], # time step # noqa: F821 F722 + drop_audio_cond, # cfg for cond audio + drop_text, # cfg for text + mask: bool["b n"] | None = None, # noqa: F722 + ): + batch, seq_len = x.shape[0], x.shape[1] + if time.ndim == 0: + time = time.repeat(batch) + + # t: conditioning time, c: context (text + masked cond audio), x: noised input audio + t = self.time_embed(time) + text_embed = self.text_embed(text, seq_len, drop_text=drop_text) + x = self.input_embed(x, cond, text_embed, drop_audio_cond=drop_audio_cond) + + rope = self.rotary_embed.forward_from_seq_len(seq_len) + + if self.long_skip_connection is not None: + residual = x + + for block in self.transformer_blocks: + if self.checkpoint_activations: + x = torch.utils.checkpoint.checkpoint( + self.ckpt_wrapper(block), x, t, mask, rope + ) + else: + x = block(x, t, mask=mask, rope=rope) + + if self.long_skip_connection is not None: + x = self.long_skip_connection(torch.cat((x, residual), dim=-1)) + + x = self.norm_out(x, t) + output = self.proj_out(x) + + return output diff --git a/egs/wenetspeech4tts/TTS/f5-tts/model/modules.py b/egs/wenetspeech4tts/TTS/f5-tts/model/modules.py new file mode 100644 index 000000000..05299d419 --- /dev/null +++ b/egs/wenetspeech4tts/TTS/f5-tts/model/modules.py @@ -0,0 +1,728 @@ +""" +ein notation: +b - batch +n - sequence +nt - text sequence +nw - raw wave length +d - dimension +""" + +from __future__ import annotations + +import math +from typing import Optional + +import torch +import torch.nn.functional as F +import torchaudio +from librosa.filters import mel as librosa_mel_fn +from torch import nn +from x_transformers.x_transformers import apply_rotary_pos_emb + +# raw wav to mel spec + + +mel_basis_cache = {} +hann_window_cache = {} + + +def get_bigvgan_mel_spectrogram( + waveform, + n_fft=1024, + n_mel_channels=100, + target_sample_rate=24000, + hop_length=256, + win_length=1024, + fmin=0, + fmax=None, + center=False, +): # Copy from https://github.com/NVIDIA/BigVGAN/tree/main + device = waveform.device + key = f"{n_fft}_{n_mel_channels}_{target_sample_rate}_{hop_length}_{win_length}_{fmin}_{fmax}_{device}" + + if key not in mel_basis_cache: + mel = librosa_mel_fn( + sr=target_sample_rate, + n_fft=n_fft, + n_mels=n_mel_channels, + fmin=fmin, + fmax=fmax, + ) + mel_basis_cache[key] = ( + torch.from_numpy(mel).float().to(device) + ) # TODO: why they need .float()? + hann_window_cache[key] = torch.hann_window(win_length).to(device) + + mel_basis = mel_basis_cache[key] + hann_window = hann_window_cache[key] + + padding = (n_fft - hop_length) // 2 + waveform = torch.nn.functional.pad( + waveform.unsqueeze(1), (padding, padding), mode="reflect" + ).squeeze(1) + + spec = torch.stft( + waveform, + n_fft, + hop_length=hop_length, + win_length=win_length, + window=hann_window, + center=center, + pad_mode="reflect", + normalized=False, + onesided=True, + return_complex=True, + ) + spec = torch.sqrt(torch.view_as_real(spec).pow(2).sum(-1) + 1e-9) + + mel_spec = torch.matmul(mel_basis, spec) + mel_spec = torch.log(torch.clamp(mel_spec, min=1e-5)) + + return mel_spec + + +def get_vocos_mel_spectrogram( + waveform, + n_fft=1024, + n_mel_channels=100, + target_sample_rate=24000, + hop_length=256, + win_length=1024, +): + mel_stft = torchaudio.transforms.MelSpectrogram( + sample_rate=target_sample_rate, + n_fft=n_fft, + win_length=win_length, + hop_length=hop_length, + n_mels=n_mel_channels, + power=1, + center=True, + normalized=False, + norm=None, + ).to(waveform.device) + if len(waveform.shape) == 3: + waveform = waveform.squeeze(1) # 'b 1 nw -> b nw' + + assert len(waveform.shape) == 2 + + mel = mel_stft(waveform) + mel = mel.clamp(min=1e-5).log() + return mel + + +class MelSpec(nn.Module): + def __init__( + self, + n_fft=1024, + hop_length=256, + win_length=1024, + n_mel_channels=100, + target_sample_rate=24_000, + mel_spec_type="vocos", + ): + super().__init__() + assert mel_spec_type in ["vocos", "bigvgan"], print( + "We only support two extract mel backend: vocos or bigvgan" + ) + + self.n_fft = n_fft + self.hop_length = hop_length + self.win_length = win_length + self.n_mel_channels = n_mel_channels + self.target_sample_rate = target_sample_rate + + if mel_spec_type == "vocos": + self.extractor = get_vocos_mel_spectrogram + elif mel_spec_type == "bigvgan": + self.extractor = get_bigvgan_mel_spectrogram + + self.register_buffer("dummy", torch.tensor(0), persistent=False) + + def forward(self, wav): + if self.dummy.device != wav.device: + self.to(wav.device) + + mel = self.extractor( + waveform=wav, + n_fft=self.n_fft, + n_mel_channels=self.n_mel_channels, + target_sample_rate=self.target_sample_rate, + hop_length=self.hop_length, + win_length=self.win_length, + ) + + return mel + + +# sinusoidal position embedding + + +class SinusPositionEmbedding(nn.Module): + def __init__(self, dim): + super().__init__() + self.dim = dim + + def forward(self, x, scale=1000): + device = x.device + half_dim = self.dim // 2 + emb = math.log(10000) / (half_dim - 1) + emb = torch.exp(torch.arange(half_dim, device=device).float() * -emb) + emb = scale * x.unsqueeze(1) * emb.unsqueeze(0) + emb = torch.cat((emb.sin(), emb.cos()), dim=-1) + return emb + + +# convolutional position embedding + + +class ConvPositionEmbedding(nn.Module): + def __init__(self, dim, kernel_size=31, groups=16): + super().__init__() + assert kernel_size % 2 != 0 + self.conv1d = nn.Sequential( + nn.Conv1d(dim, dim, kernel_size, groups=groups, padding=kernel_size // 2), + nn.Mish(), + nn.Conv1d(dim, dim, kernel_size, groups=groups, padding=kernel_size // 2), + nn.Mish(), + ) + + def forward(self, x: float["b n d"], mask: bool["b n"] | None = None): # noqa: F722 + if mask is not None: + mask = mask[..., None] + x = x.masked_fill(~mask, 0.0) + + x = x.permute(0, 2, 1) + x = self.conv1d(x) + out = x.permute(0, 2, 1) + + if mask is not None: + out = out.masked_fill(~mask, 0.0) + + return out + + +# rotary positional embedding related + + +def precompute_freqs_cis( + dim: int, end: int, theta: float = 10000.0, theta_rescale_factor=1.0 +): + # proposed by reddit user bloc97, to rescale rotary embeddings to longer sequence length without fine-tuning + # has some connection to NTK literature + # https://www.reddit.com/r/LocalLLaMA/comments/14lz7j5/ntkaware_scaled_rope_allows_llama_models_to_have/ + # https://github.com/lucidrains/rotary-embedding-torch/blob/main/rotary_embedding_torch/rotary_embedding_torch.py + theta *= theta_rescale_factor ** (dim / (dim - 2)) + freqs = 1.0 / (theta ** (torch.arange(0, dim, 2)[: (dim // 2)].float() / dim)) + t = torch.arange(end, device=freqs.device) # type: ignore + freqs = torch.outer(t, freqs).float() # type: ignore + freqs_cos = torch.cos(freqs) # real part + freqs_sin = torch.sin(freqs) # imaginary part + return torch.cat([freqs_cos, freqs_sin], dim=-1) + + +def get_pos_embed_indices(start, length, max_pos, scale=1.0): + # length = length if isinstance(length, int) else length.max() + scale = scale * torch.ones_like( + start, dtype=torch.float32 + ) # in case scale is a scalar + pos = ( + start.unsqueeze(1) + + ( + torch.arange(length, device=start.device, dtype=torch.float32).unsqueeze(0) + * scale.unsqueeze(1) + ).long() + ) + # avoid extra long error. + pos = torch.where(pos < max_pos, pos, max_pos - 1) + return pos + + +# Global Response Normalization layer (Instance Normalization ?) + + +class GRN(nn.Module): + def __init__(self, dim): + super().__init__() + self.gamma = nn.Parameter(torch.zeros(1, 1, dim)) + self.beta = nn.Parameter(torch.zeros(1, 1, dim)) + + def forward(self, x): + Gx = torch.norm(x, p=2, dim=1, keepdim=True) + Nx = Gx / (Gx.mean(dim=-1, keepdim=True) + 1e-6) + return self.gamma * (x * Nx) + self.beta + x + + +# ConvNeXt-V2 Block https://github.com/facebookresearch/ConvNeXt-V2/blob/main/models/convnextv2.py +# ref: https://github.com/bfs18/e2_tts/blob/main/rfwave/modules.py#L108 + + +class ConvNeXtV2Block(nn.Module): + def __init__( + self, + dim: int, + intermediate_dim: int, + dilation: int = 1, + ): + super().__init__() + padding = (dilation * (7 - 1)) // 2 + self.dwconv = nn.Conv1d( + dim, dim, kernel_size=7, padding=padding, groups=dim, dilation=dilation + ) # depthwise conv + self.norm = nn.LayerNorm(dim, eps=1e-6) + self.pwconv1 = nn.Linear( + dim, intermediate_dim + ) # pointwise/1x1 convs, implemented with linear layers + self.act = nn.GELU() + self.grn = GRN(intermediate_dim) + self.pwconv2 = nn.Linear(intermediate_dim, dim) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + residual = x + x = x.transpose(1, 2) # b n d -> b d n + x = self.dwconv(x) + x = x.transpose(1, 2) # b d n -> b n d + x = self.norm(x) + x = self.pwconv1(x) + x = self.act(x) + x = self.grn(x) + x = self.pwconv2(x) + return residual + x + + +# AdaLayerNormZero +# return with modulated x for attn input, and params for later mlp modulation + + +class AdaLayerNormZero(nn.Module): + def __init__(self, dim): + super().__init__() + + self.silu = nn.SiLU() + self.linear = nn.Linear(dim, dim * 6) + + self.norm = nn.LayerNorm(dim, elementwise_affine=False, eps=1e-6) + + def forward(self, x, emb=None): + emb = self.linear(self.silu(emb)) + shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = torch.chunk( + emb, 6, dim=1 + ) + + x = self.norm(x) * (1 + scale_msa[:, None]) + shift_msa[:, None] + return x, gate_msa, shift_mlp, scale_mlp, gate_mlp + + +# AdaLayerNormZero for final layer +# return only with modulated x for attn input, cuz no more mlp modulation + + +class AdaLayerNormZero_Final(nn.Module): + def __init__(self, dim): + super().__init__() + + self.silu = nn.SiLU() + self.linear = nn.Linear(dim, dim * 2) + + self.norm = nn.LayerNorm(dim, elementwise_affine=False, eps=1e-6) + + def forward(self, x, emb): + emb = self.linear(self.silu(emb)) + scale, shift = torch.chunk(emb, 2, dim=1) + + x = self.norm(x) * (1 + scale)[:, None, :] + shift[:, None, :] + return x + + +# FeedForward + + +class FeedForward(nn.Module): + def __init__( + self, dim, dim_out=None, mult=4, dropout=0.0, approximate: str = "none" + ): + super().__init__() + inner_dim = int(dim * mult) + dim_out = dim_out if dim_out is not None else dim + + activation = nn.GELU(approximate=approximate) + project_in = nn.Sequential(nn.Linear(dim, inner_dim), activation) + self.ff = nn.Sequential( + project_in, nn.Dropout(dropout), nn.Linear(inner_dim, dim_out) + ) + + def forward(self, x): + return self.ff(x) + + +# Attention with possible joint part +# modified from diffusers/src/diffusers/models/attention_processor.py + + +class Attention(nn.Module): + def __init__( + self, + processor: JointAttnProcessor | AttnProcessor, + dim: int, + heads: int = 8, + dim_head: int = 64, + dropout: float = 0.0, + context_dim: Optional[int] = None, # if not None -> joint attention + context_pre_only=None, + ): + super().__init__() + + if not hasattr(F, "scaled_dot_product_attention"): + raise ImportError( + "Attention equires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0." + ) + + self.processor = processor + + self.dim = dim + self.heads = heads + self.inner_dim = dim_head * heads + self.dropout = dropout + + self.context_dim = context_dim + self.context_pre_only = context_pre_only + + self.to_q = nn.Linear(dim, self.inner_dim) + self.to_k = nn.Linear(dim, self.inner_dim) + self.to_v = nn.Linear(dim, self.inner_dim) + + if self.context_dim is not None: + self.to_k_c = nn.Linear(context_dim, self.inner_dim) + self.to_v_c = nn.Linear(context_dim, self.inner_dim) + if self.context_pre_only is not None: + self.to_q_c = nn.Linear(context_dim, self.inner_dim) + + self.to_out = nn.ModuleList([]) + self.to_out.append(nn.Linear(self.inner_dim, dim)) + self.to_out.append(nn.Dropout(dropout)) + + if self.context_pre_only is not None and not self.context_pre_only: + self.to_out_c = nn.Linear(self.inner_dim, dim) + + def forward( + self, + x: float["b n d"], # noised input x # noqa: F722 + c: float["b n d"] = None, # context c # noqa: F722 + mask: bool["b n"] | None = None, # noqa: F722 + rope=None, # rotary position embedding for x + c_rope=None, # rotary position embedding for c + ) -> torch.Tensor: + if c is not None: + return self.processor(self, x, c=c, mask=mask, rope=rope, c_rope=c_rope) + else: + return self.processor(self, x, mask=mask, rope=rope) + + +# Attention processor + + +class AttnProcessor: + def __init__(self): + pass + + def __call__( + self, + attn: Attention, + x: float["b n d"], # noised input x # noqa: F722 + mask: bool["b n"] | None = None, # noqa: F722 + rope=None, # rotary position embedding + ) -> torch.FloatTensor: + batch_size = x.shape[0] + + # `sample` projections. + query = attn.to_q(x) + key = attn.to_k(x) + value = attn.to_v(x) + + # apply rotary position embedding + if rope is not None: + freqs, xpos_scale = rope + q_xpos_scale, k_xpos_scale = ( + (xpos_scale, xpos_scale**-1.0) + if xpos_scale is not None + else (1.0, 1.0) + ) + + query = apply_rotary_pos_emb(query, freqs, q_xpos_scale) + key = apply_rotary_pos_emb(key, freqs, k_xpos_scale) + + # attention + inner_dim = key.shape[-1] + head_dim = inner_dim // attn.heads + query = query.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) + key = key.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) + value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) + + # mask. e.g. inference got a batch with different target durations, mask out the padding + if mask is not None: + attn_mask = mask + attn_mask = attn_mask.unsqueeze(1).unsqueeze(1) # 'b n -> b 1 1 n' + attn_mask = attn_mask.expand( + batch_size, attn.heads, query.shape[-2], key.shape[-2] + ) + else: + attn_mask = None + + x = F.scaled_dot_product_attention( + query, key, value, attn_mask=attn_mask, dropout_p=0.0, is_causal=False + ) + x = x.transpose(1, 2).reshape(batch_size, -1, attn.heads * head_dim) + x = x.to(query.dtype) + + # linear proj + x = attn.to_out[0](x) + # dropout + x = attn.to_out[1](x) + + if mask is not None: + mask = mask.unsqueeze(-1) + x = x.masked_fill(~mask, 0.0) + + return x + + +# Joint Attention processor for MM-DiT +# modified from diffusers/src/diffusers/models/attention_processor.py + + +class JointAttnProcessor: + def __init__(self): + pass + + def __call__( + self, + attn: Attention, + x: float["b n d"], # noised input x # noqa: F722 + c: float["b nt d"] = None, # context c, here text # noqa: F722 + mask: bool["b n"] | None = None, # noqa: F722 + rope=None, # rotary position embedding for x + c_rope=None, # rotary position embedding for c + ) -> torch.FloatTensor: + residual = x + + batch_size = c.shape[0] + + # `sample` projections. + query = attn.to_q(x) + key = attn.to_k(x) + value = attn.to_v(x) + + # `context` projections. + c_query = attn.to_q_c(c) + c_key = attn.to_k_c(c) + c_value = attn.to_v_c(c) + + # apply rope for context and noised input independently + if rope is not None: + freqs, xpos_scale = rope + q_xpos_scale, k_xpos_scale = ( + (xpos_scale, xpos_scale**-1.0) + if xpos_scale is not None + else (1.0, 1.0) + ) + query = apply_rotary_pos_emb(query, freqs, q_xpos_scale) + key = apply_rotary_pos_emb(key, freqs, k_xpos_scale) + if c_rope is not None: + freqs, xpos_scale = c_rope + q_xpos_scale, k_xpos_scale = ( + (xpos_scale, xpos_scale**-1.0) + if xpos_scale is not None + else (1.0, 1.0) + ) + c_query = apply_rotary_pos_emb(c_query, freqs, q_xpos_scale) + c_key = apply_rotary_pos_emb(c_key, freqs, k_xpos_scale) + + # attention + query = torch.cat([query, c_query], dim=1) + key = torch.cat([key, c_key], dim=1) + value = torch.cat([value, c_value], dim=1) + + inner_dim = key.shape[-1] + head_dim = inner_dim // attn.heads + query = query.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) + key = key.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) + value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) + + # mask. e.g. inference got a batch with different target durations, mask out the padding + if mask is not None: + attn_mask = F.pad(mask, (0, c.shape[1]), value=True) # no mask for c (text) + attn_mask = attn_mask.unsqueeze(1).unsqueeze(1) # 'b n -> b 1 1 n' + attn_mask = attn_mask.expand( + batch_size, attn.heads, query.shape[-2], key.shape[-2] + ) + else: + attn_mask = None + + x = F.scaled_dot_product_attention( + query, key, value, attn_mask=attn_mask, dropout_p=0.0, is_causal=False + ) + x = x.transpose(1, 2).reshape(batch_size, -1, attn.heads * head_dim) + x = x.to(query.dtype) + + # Split the attention outputs. + x, c = ( + x[:, : residual.shape[1]], + x[:, residual.shape[1] :], + ) + + # linear proj + x = attn.to_out[0](x) + # dropout + x = attn.to_out[1](x) + if not attn.context_pre_only: + c = attn.to_out_c(c) + + if mask is not None: + mask = mask.unsqueeze(-1) + x = x.masked_fill(~mask, 0.0) + # c = c.masked_fill(~mask, 0.) # no mask for c (text) + + return x, c + + +# DiT Block + + +class DiTBlock(nn.Module): + def __init__(self, dim, heads, dim_head, ff_mult=4, dropout=0.1): + super().__init__() + + self.attn_norm = AdaLayerNormZero(dim) + self.attn = Attention( + processor=AttnProcessor(), + dim=dim, + heads=heads, + dim_head=dim_head, + dropout=dropout, + ) + + self.ff_norm = nn.LayerNorm(dim, elementwise_affine=False, eps=1e-6) + self.ff = FeedForward( + dim=dim, mult=ff_mult, dropout=dropout, approximate="tanh" + ) + + def forward(self, x, t, mask=None, rope=None): # x: noised input, t: time embedding + # pre-norm & modulation for attention input + norm, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.attn_norm(x, emb=t) + + # attention + attn_output = self.attn(x=norm, mask=mask, rope=rope) + + # process attention output for input x + x = x + gate_msa.unsqueeze(1) * attn_output + + norm = self.ff_norm(x) * (1 + scale_mlp[:, None]) + shift_mlp[:, None] + ff_output = self.ff(norm) + x = x + gate_mlp.unsqueeze(1) * ff_output + + return x + + +# MMDiT Block https://arxiv.org/abs/2403.03206 + + +class MMDiTBlock(nn.Module): + r""" + modified from diffusers/src/diffusers/models/attention.py + + notes. + _c: context related. text, cond, etc. (left part in sd3 fig2.b) + _x: noised input related. (right part) + context_pre_only: last layer only do prenorm + modulation cuz no more ffn + """ + + def __init__( + self, dim, heads, dim_head, ff_mult=4, dropout=0.1, context_pre_only=False + ): + super().__init__() + + self.context_pre_only = context_pre_only + + self.attn_norm_c = ( + AdaLayerNormZero_Final(dim) if context_pre_only else AdaLayerNormZero(dim) + ) + self.attn_norm_x = AdaLayerNormZero(dim) + self.attn = Attention( + processor=JointAttnProcessor(), + dim=dim, + heads=heads, + dim_head=dim_head, + dropout=dropout, + context_dim=dim, + context_pre_only=context_pre_only, + ) + + if not context_pre_only: + self.ff_norm_c = nn.LayerNorm(dim, elementwise_affine=False, eps=1e-6) + self.ff_c = FeedForward( + dim=dim, mult=ff_mult, dropout=dropout, approximate="tanh" + ) + else: + self.ff_norm_c = None + self.ff_c = None + self.ff_norm_x = nn.LayerNorm(dim, elementwise_affine=False, eps=1e-6) + self.ff_x = FeedForward( + dim=dim, mult=ff_mult, dropout=dropout, approximate="tanh" + ) + + def forward( + self, x, c, t, mask=None, rope=None, c_rope=None + ): # x: noised input, c: context, t: time embedding + # pre-norm & modulation for attention input + if self.context_pre_only: + norm_c = self.attn_norm_c(c, t) + else: + norm_c, c_gate_msa, c_shift_mlp, c_scale_mlp, c_gate_mlp = self.attn_norm_c( + c, emb=t + ) + norm_x, x_gate_msa, x_shift_mlp, x_scale_mlp, x_gate_mlp = self.attn_norm_x( + x, emb=t + ) + + # attention + x_attn_output, c_attn_output = self.attn( + x=norm_x, c=norm_c, mask=mask, rope=rope, c_rope=c_rope + ) + + # process attention output for context c + if self.context_pre_only: + c = None + else: # if not last layer + c = c + c_gate_msa.unsqueeze(1) * c_attn_output + + norm_c = ( + self.ff_norm_c(c) * (1 + c_scale_mlp[:, None]) + c_shift_mlp[:, None] + ) + c_ff_output = self.ff_c(norm_c) + c = c + c_gate_mlp.unsqueeze(1) * c_ff_output + + # process attention output for input x + x = x + x_gate_msa.unsqueeze(1) * x_attn_output + + norm_x = self.ff_norm_x(x) * (1 + x_scale_mlp[:, None]) + x_shift_mlp[:, None] + x_ff_output = self.ff_x(norm_x) + x = x + x_gate_mlp.unsqueeze(1) * x_ff_output + + return c, x + + +# time step conditioning embedding + + +class TimestepEmbedding(nn.Module): + def __init__(self, dim, freq_embed_dim=256): + super().__init__() + self.time_embed = SinusPositionEmbedding(freq_embed_dim) + self.time_mlp = nn.Sequential( + nn.Linear(freq_embed_dim, dim), nn.SiLU(), nn.Linear(dim, dim) + ) + + def forward(self, timestep: float["b"]): # noqa: F821 + time_hidden = self.time_embed(timestep) + time_hidden = time_hidden.to(timestep.dtype) + time = self.time_mlp(time_hidden) # b d + return time diff --git a/egs/wenetspeech4tts/TTS/f5-tts/model/utils.py b/egs/wenetspeech4tts/TTS/f5-tts/model/utils.py new file mode 100644 index 000000000..fae5fadb6 --- /dev/null +++ b/egs/wenetspeech4tts/TTS/f5-tts/model/utils.py @@ -0,0 +1,206 @@ +from __future__ import annotations + +import os +import random +from collections import defaultdict +from importlib.resources import files + +import jieba +import torch +from pypinyin import Style, lazy_pinyin +from torch.nn.utils.rnn import pad_sequence + +# seed everything + + +def seed_everything(seed=0): + random.seed(seed) + os.environ["PYTHONHASHSEED"] = str(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.benchmark = False + + +# helpers + + +def exists(v): + return v is not None + + +def default(v, d): + return v if exists(v) else d + + +# tensor helpers + + +def lens_to_mask( + t: int["b"], length: int | None = None # noqa: F722 F821 +) -> bool["b n"]: # noqa: F722 F821 + if not exists(length): + length = t.amax() + + seq = torch.arange(length, device=t.device) + return seq[None, :] < t[:, None] + + +def mask_from_start_end_indices( + seq_len: int["b"], start: int["b"], end: int["b"] # noqa: F722 F821 +): + max_seq_len = seq_len.max().item() + seq = torch.arange(max_seq_len, device=start.device).long() + start_mask = seq[None, :] >= start[:, None] + end_mask = seq[None, :] < end[:, None] + return start_mask & end_mask + + +def mask_from_frac_lengths( + seq_len: int["b"], frac_lengths: float["b"] # noqa: F722 F821 +): + lengths = (frac_lengths * seq_len).long() + max_start = seq_len - lengths + + rand = torch.rand_like(frac_lengths) + start = (max_start * rand).long().clamp(min=0) + end = start + lengths + + return mask_from_start_end_indices(seq_len, start, end) + + +def maybe_masked_mean( + t: float["b n d"], mask: bool["b n"] = None # noqa: F722 F821 +) -> float["b d"]: # noqa: F722 F821 + if not exists(mask): + return t.mean(dim=1) + + t = torch.where(mask[:, :, None], t, torch.tensor(0.0, device=t.device)) + num = t.sum(dim=1) + den = mask.float().sum(dim=1) + + return num / den.clamp(min=1.0) + + +# simple utf-8 tokenizer, since paper went character based +def list_str_to_tensor(text: list[str], padding_value=-1) -> int["b nt"]: # noqa: F722 + list_tensors = [torch.tensor([*bytes(t, "UTF-8")]) for t in text] # ByT5 style + text = pad_sequence(list_tensors, padding_value=padding_value, batch_first=True) + return text + + +# char tokenizer, based on custom dataset's extracted .txt file +def list_str_to_idx( + text: list[str] | list[list[str]], + vocab_char_map: dict[str, int], # {char: idx} + padding_value=-1, +) -> int["b nt"]: # noqa: F722 + list_idx_tensors = [ + torch.tensor([vocab_char_map.get(c, 0) for c in t]) for t in text + ] # pinyin or char style + text = pad_sequence(list_idx_tensors, padding_value=padding_value, batch_first=True) + return text + + +# Get tokenizer + + +def get_tokenizer(dataset_name, tokenizer: str = "pinyin"): + """ + tokenizer - "pinyin" do g2p for only chinese characters, need .txt vocab_file + - "char" for char-wise tokenizer, need .txt vocab_file + - "byte" for utf-8 tokenizer + - "custom" if you're directly passing in a path to the vocab.txt you want to use + vocab_size - if use "pinyin", all available pinyin types, common alphabets (also those with accent) and symbols + - if use "char", derived from unfiltered character & symbol counts of custom dataset + - if use "byte", set to 256 (unicode byte range) + """ + if tokenizer in ["pinyin", "char"]: + tokenizer_path = os.path.join( + files("f5_tts").joinpath("../../data"), + f"{dataset_name}_{tokenizer}/vocab.txt", + ) + with open(tokenizer_path, "r", encoding="utf-8") as f: + vocab_char_map = {} + for i, char in enumerate(f): + vocab_char_map[char[:-1]] = i + vocab_size = len(vocab_char_map) + assert ( + vocab_char_map[" "] == 0 + ), "make sure space is of idx 0 in vocab.txt, cuz 0 is used for unknown char" + + elif tokenizer == "byte": + vocab_char_map = None + vocab_size = 256 + + elif tokenizer == "custom": + with open(dataset_name, "r", encoding="utf-8") as f: + vocab_char_map = {} + for i, char in enumerate(f): + vocab_char_map[char[:-1]] = i + vocab_size = len(vocab_char_map) + + return vocab_char_map, vocab_size + + +# convert char to pinyin + +jieba.initialize() +print("Word segmentation module jieba initialized.\n") + + +def convert_char_to_pinyin(text_list, polyphone=True): + final_text_list = [] + custom_trans = str.maketrans( + {";": ",", "“": '"', "”": '"', "‘": "'", "’": "'"} + ) # add custom trans here, to address oov + + def is_chinese(c): + return "\u3100" <= c <= "\u9fff" # common chinese characters + + for text in text_list: + char_list = [] + text = text.translate(custom_trans) + for seg in jieba.cut(text): + seg_byte_len = len(bytes(seg, "UTF-8")) + if seg_byte_len == len(seg): # if pure alphabets and symbols + if char_list and seg_byte_len > 1 and char_list[-1] not in " :'\"": + char_list.append(" ") + char_list.extend(seg) + elif polyphone and seg_byte_len == 3 * len( + seg + ): # if pure east asian characters + seg_ = lazy_pinyin(seg, style=Style.TONE3, tone_sandhi=True) + for i, c in enumerate(seg): + if is_chinese(c): + char_list.append(" ") + char_list.append(seg_[i]) + else: # if mixed characters, alphabets and symbols + for c in seg: + if ord(c) < 256: + char_list.extend(c) + elif is_chinese(c): + char_list.append(" ") + char_list.extend( + lazy_pinyin(c, style=Style.TONE3, tone_sandhi=True) + ) + else: + char_list.append(c) + final_text_list.append(char_list) + + return final_text_list + + +# filter func for dirty data with many repetitions + + +def repetition_found(text, length=2, tolerance=10): + pattern_count = defaultdict(int) + for i in range(len(text) - length + 1): + pattern = text[i : i + length] + pattern_count[pattern] += 1 + for pattern, count in pattern_count.items(): + if count > tolerance: + return True + return False diff --git a/egs/wenetspeech4tts/TTS/f5-tts/speech_synthesis.py b/egs/wenetspeech4tts/TTS/f5-tts/speech_synthesis.py new file mode 100644 index 000000000..57f677fcb --- /dev/null +++ b/egs/wenetspeech4tts/TTS/f5-tts/speech_synthesis.py @@ -0,0 +1,104 @@ +from typing import Callable, Dict, List, Sequence, Union + +import torch +from lhotse import validate +from lhotse.cut import CutSet +from lhotse.dataset.collation import collate_audio +from lhotse.dataset.input_strategies import BatchIO, PrecomputedFeatures +from lhotse.utils import ifnone + + +class SpeechSynthesisDataset(torch.utils.data.Dataset): + """ + The PyTorch Dataset for the speech synthesis task. + Each item in this dataset is a dict of: + + .. code-block:: + + { + 'audio': (B x NumSamples) float tensor + 'features': (B x NumFrames x NumFeatures) float tensor + 'audio_lens': (B, ) int tensor + 'features_lens': (B, ) int tensor + 'text': List[str] of len B # when return_text=True + 'tokens': List[List[str]] # when return_tokens=True + 'speakers': List[str] of len B # when return_spk_ids=True + 'cut': List of Cuts # when return_cuts=True + } + """ + + def __init__( + self, + cut_transforms: List[Callable[[CutSet], CutSet]] = None, + feature_input_strategy: BatchIO = PrecomputedFeatures(), + feature_transforms: Union[Sequence[Callable], Callable] = None, + return_text: bool = True, + return_tokens: bool = False, + return_spk_ids: bool = False, + return_cuts: bool = False, + ) -> None: + super().__init__() + + self.cut_transforms = ifnone(cut_transforms, []) + self.feature_input_strategy = feature_input_strategy + + self.return_text = return_text + self.return_tokens = return_tokens + self.return_spk_ids = return_spk_ids + self.return_cuts = return_cuts + + if feature_transforms is None: + feature_transforms = [] + elif not isinstance(feature_transforms, Sequence): + feature_transforms = [feature_transforms] + + assert all( + isinstance(transform, Callable) for transform in feature_transforms + ), "Feature transforms must be Callable" + self.feature_transforms = feature_transforms + + def __getitem__(self, cuts: CutSet) -> Dict[str, torch.Tensor]: + validate_for_tts(cuts) + + for transform in self.cut_transforms: + cuts = transform(cuts) + + # audio, audio_lens = collate_audio(cuts) + features, features_lens = self.feature_input_strategy(cuts) + + for transform in self.feature_transforms: + features = transform(features) + + batch = { + # "audio": audio, + "features": features, + # "audio_lens": audio_lens, + "features_lens": features_lens, + } + + if self.return_text: + # use normalized text + # text = [cut.supervisions[0].normalized_text for cut in cuts] + text = [cut.supervisions[0].text for cut in cuts] + batch["text"] = text + + if self.return_tokens: + # tokens = [cut.tokens for cut in cuts] + tokens = [cut.supervisions[0].custom["tokens"]["text"] for cut in cuts] + batch["tokens"] = tokens + + if self.return_spk_ids: + batch["speakers"] = [cut.supervisions[0].speaker for cut in cuts] + + if self.return_cuts: + batch["cut"] = [cut for cut in cuts] + + return batch + + +def validate_for_tts(cuts: CutSet) -> None: + validate(cuts) + for cut in cuts: + assert ( + len(cut.supervisions) == 1 + ), "Only the Cuts with single supervision are supported." diff --git a/egs/wenetspeech4tts/TTS/f5-tts/train.py b/egs/wenetspeech4tts/TTS/f5-tts/train.py new file mode 100755 index 000000000..37dcf531e --- /dev/null +++ b/egs/wenetspeech4tts/TTS/f5-tts/train.py @@ -0,0 +1,1178 @@ +#!/usr/bin/env python3 +# Copyright 2021-2022 Xiaomi Corp. (authors: Fangjun Kuang, +# Wei Kang, +# Mingshuang Luo) +# Copyright 2023 (authors: Feiteng Li) +# Copyright 2024 (authors: Yuekai Zhang) +# +# See ../../../../LICENSE for clarification regarding multiple authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Usage: +# docker: ghcr.io/swivid/f5-tts:main +# pip install k2==1.24.4.dev20241030+cuda12.4.torch2.4.0 -f https://k2-fsa.github.io/k2/cuda.html +# pip install kaldialign lhotse tensorboard bigvganinference sentencepiece + +world_size=8 +exp_dir=exp/f5-tts-small +python3 f5-tts/train.py --max-duration 700 --filter-min-duration 0.5 --filter-max-duration 20 \ + --num-buckets 6 --dtype "bfloat16" --save-every-n 5000 --valid-interval 10000 \ + --base-lr 7.5e-5 --warmup-steps 20000 --num-epochs 60 \ + --num-decoder-layers 18 --nhead 12 --decoder-dim 768 \ + --exp-dir ${exp_dir} --world-size ${world_size} +""" + +import argparse +import copy +import logging +import os +import random +import warnings +from contextlib import nullcontext +from pathlib import Path +from shutil import copyfile +from typing import Any, Dict, Optional, Tuple, Union + +import torch +import torch.multiprocessing as mp +import torch.nn as nn +from lhotse import CutSet +from lhotse.cut import Cut +from lhotse.dataset.sampling.base import CutSampler +from lhotse.utils import fix_random_seed +from model.cfm import CFM +from model.dit import DiT +from model.utils import convert_char_to_pinyin +from torch import Tensor +from torch.amp import GradScaler +from torch.nn.parallel import DistributedDataParallel as DDP +from torch.optim.lr_scheduler import LinearLR, SequentialLR +from torch.utils.tensorboard import SummaryWriter +from tts_datamodule import TtsDataModule +from utils import MetricsTracker + +from icefall.checkpoint import load_checkpoint, remove_checkpoints +from icefall.checkpoint import save_checkpoint as save_checkpoint_impl +from icefall.checkpoint import ( + save_checkpoint_with_global_batch_idx, + update_averaged_model, +) +from icefall.dist import cleanup_dist, setup_dist +from icefall.env import get_env_info +from icefall.hooks import register_inf_check_hooks +from icefall.utils import AttributeDict, setup_logger, str2bool # MetricsTracker + +LRSchedulerType = torch.optim.lr_scheduler._LRScheduler + + +def set_batch_count(model: Union[nn.Module, DDP], batch_count: float) -> None: + if isinstance(model, DDP): + # get underlying nn.Module + model = model.module + + for module in model.modules(): + if hasattr(module, "batch_count"): + module.batch_count = batch_count + + +def add_model_arguments(parser: argparse.ArgumentParser): + parser.add_argument( + "--decoder-dim", + type=int, + default=1024, + help="Embedding dimension in the decoder model.", + ) + + parser.add_argument( + "--nhead", + type=int, + default=16, + help="Number of attention heads in the Decoder layers.", + ) + + parser.add_argument( + "--num-decoder-layers", + type=int, + default=22, + help="Number of Decoder layers.", + ) + + +def get_parser(): + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + + parser.add_argument( + "--world-size", + type=int, + default=1, + help="Number of GPUs for DDP training.", + ) + + parser.add_argument( + "--master-port", + type=int, + default=12354, + help="Master port to use for DDP training.", + ) + + parser.add_argument( + "--tensorboard", + type=str2bool, + default=True, + help="Should various information be logged in tensorboard.", + ) + + parser.add_argument( + "--num-epochs", + type=int, + default=20, + help="Number of epochs to train.", + ) + + parser.add_argument( + "--start-epoch", + type=int, + default=1, + help="""Resume training from this epoch. It should be positive. + If larger than 1, it will load checkpoint from + exp-dir/epoch-{start_epoch-1}.pt + """, + ) + + parser.add_argument( + "--start-batch", + type=int, + default=0, + help="""If positive, --start-epoch is ignored and + it loads the checkpoint from exp-dir/checkpoint-{start_batch}.pt + """, + ) + + parser.add_argument( + "--exp-dir", + type=Path, + default="exp/f5", + help="""The experiment dir. + It specifies the directory where all training related + files, e.g., checkpoints, log, etc, are saved + """, + ) + + parser.add_argument( + "--tokens", + type=str, + default="f5-tts/vocab.txt", + help="Path to the unique text tokens file", + ) + + parser.add_argument( + "--pretrained-model-path", + type=str, + default=None, + help="Path to file", + ) + + parser.add_argument( + "--optimizer-name", + type=str, + default="AdamW", + help="The optimizer.", + ) + parser.add_argument( + "--base-lr", type=float, default=0.05, help="The base learning rate." + ) + parser.add_argument( + "--warmup-steps", + type=int, + default=200, + help="""Number of steps that affects how rapidly the learning rate + decreases. We suggest not to change this.""", + ) + + parser.add_argument( + "--decay-steps", + type=int, + default=1000000, + help="""Number of steps that affects how rapidly the learning rate + decreases. We suggest not to change this.""", + ) + + parser.add_argument( + "--seed", + type=int, + default=42, + help="The seed for random generators intended for reproducibility", + ) + + parser.add_argument( + "--inf-check", + type=str2bool, + default=False, + help="Add hooks to check for infinite module outputs and gradients.", + ) + + parser.add_argument( + "--save-every-n", + type=int, + default=10000, + help="""Save checkpoint after processing this number of batches" + periodically. We save checkpoint to exp-dir/ whenever + params.batch_idx_train %% save_every_n == 0. The checkpoint filename + has the form: f'exp-dir/checkpoint-{params.batch_idx_train}.pt' + Note: It also saves checkpoint to `exp-dir/epoch-xxx.pt` at the + end of each epoch where `xxx` is the epoch number counting from 0. + """, + ) + parser.add_argument( + "--valid-interval", + type=int, + default=10000, + help="""Run validation if batch_idx %% valid_interval is 0.""", + ) + + parser.add_argument( + "--keep-last-k", + type=int, + default=20, + help="""Only keep this number of checkpoints on disk. + For instance, if it is 3, there are only 3 checkpoints + in the exp-dir with filenames `checkpoint-xxx.pt`. + It does not affect checkpoints with name `epoch-xxx.pt`. + """, + ) + + parser.add_argument( + "--average-period", + type=int, + default=0, + help="""Update the averaged model, namely `model_avg`, after processing + this number of batches. `model_avg` is a separate version of model, + in which each floating-point parameter is the average of all the + parameters from the start of training. Each time we take the average, + we do: `model_avg = model * (average_period / batch_idx_train) + + model_avg * ((batch_idx_train - average_period) / batch_idx_train)`. + """, + ) + + parser.add_argument( + "--accumulate-grad-steps", + type=int, + default=1, + help="""update gradient when batch_idx_train %% accumulate_grad_steps == 0. + """, + ) + + parser.add_argument( + "--dtype", + type=str, + default="bfloat16", + help="Training dtype: float32 bfloat16 float16.", + ) + + parser.add_argument( + "--filter-min-duration", + type=float, + default=0.0, + help="Keep only utterances with duration > this.", + ) + + parser.add_argument( + "--filter-max-duration", + type=float, + default=20.0, + help="Keep only utterances with duration < this.", + ) + + parser.add_argument( + "--oom-check", + type=str2bool, + default=False, + help="perform OOM check on dataloader batches before starting training.", + ) + + add_model_arguments(parser) + + return parser + + +def get_params() -> AttributeDict: + """Return a dict containing training parameters. + + All training related parameters that are not passed from the commandline + are saved in the variable `params`. + + Commandline options are merged into `params` after they are parsed, so + you can also access them via `params`. + + Explanation of options saved in `params`: + + - best_train_loss: Best training loss so far. It is used to select + the model that has the lowest training loss. It is + updated during the training. + + - best_valid_loss: Best validation loss so far. It is used to select + the model that has the lowest validation loss. It is + updated during the training. + + - best_train_epoch: It is the epoch that has the best training loss. + + - best_valid_epoch: It is the epoch that has the best validation loss. + + - batch_idx_train: Used to writing statistics to tensorboard. It + contains number of batches trained so far across + epochs. + + - log_interval: Print training loss if batch_idx % log_interval` is 0 + + - reset_interval: Reset statistics if batch_idx % reset_interval is 0 + + - valid_interval: Run validation if batch_idx % valid_interval is 0 + """ + params = AttributeDict( + { + "best_train_loss": float("inf"), + "best_valid_loss": float("inf"), + "best_train_epoch": -1, + "best_valid_epoch": -1, + "batch_idx_train": 0, + "log_interval": 100, + "reset_interval": 200, + "valid_interval": 10000, + "env_info": get_env_info(), + } + ) + + return params + + +def get_tokenizer(vocab_file_path: str): + """ + tokenizer - "pinyin" do g2p for only chinese characters, need .txt vocab_file + - "char" for char-wise tokenizer, need .txt vocab_file + - "byte" for utf-8 tokenizer + - "custom" if you're directly passing in a path to the vocab.txt you want to use + vocab_size - if use "pinyin", all available pinyin types, common alphabets (also those with accent) and symbols + - if use "char", derived from unfiltered character & symbol counts of custom dataset + - if use "byte", set to 256 (unicode byte range) + """ + with open(vocab_file_path, "r", encoding="utf-8") as f: + vocab_char_map = {} + for i, char in enumerate(f): + vocab_char_map[char[:-1]] = i + vocab_size = len(vocab_char_map) + + return vocab_char_map, vocab_size + + +def get_model(params): + vocab_char_map, vocab_size = get_tokenizer(params.tokens) + # bigvgan 100 dim features + n_mel_channels = 100 + n_fft = 1024 + sampling_rate = 24_000 + hop_length = 256 + win_length = 1024 + + model_cfg = { + "dim": params.decoder_dim, + "depth": params.num_decoder_layers, + "heads": params.nhead, + "ff_mult": 2, + "text_dim": 512, + "conv_layers": 4, + "checkpoint_activations": False, + } + model = CFM( + transformer=DiT( + **model_cfg, text_num_embeds=vocab_size, mel_dim=n_mel_channels + ), + mel_spec_kwargs=dict( + n_fft=n_fft, + hop_length=hop_length, + win_length=win_length, + n_mel_channels=n_mel_channels, + target_sample_rate=sampling_rate, + mel_spec_type="bigvgan", + ), + odeint_kwargs=dict( + method="euler", + ), + vocab_char_map=vocab_char_map, + ) + return model + + +def load_F5_TTS_pretrained_checkpoint( + model, ckpt_path, device: str = "cpu", dtype=torch.float32 +): + checkpoint = torch.load(ckpt_path, map_location=device, weights_only=True) + if "ema_model_state_dict" in checkpoint: + checkpoint["model_state_dict"] = { + k.replace("ema_model.", ""): v + for k, v in checkpoint["ema_model_state_dict"].items() + if k not in ["initted", "step"] + } + + # patch for backward compatibility, 305e3ea + for key in [ + "mel_spec.mel_stft.mel_scale.fb", + "mel_spec.mel_stft.spectrogram.window", + ]: + if key in checkpoint["model_state_dict"]: + del checkpoint["model_state_dict"][key] + model.load_state_dict(checkpoint["model_state_dict"]) + return model + + +def load_checkpoint_if_available( + params: AttributeDict, + model: nn.Module, + model_avg: nn.Module = None, + optimizer: Optional[torch.optim.Optimizer] = None, + scheduler: Optional[LRSchedulerType] = None, +) -> Optional[Dict[str, Any]]: + """Load checkpoint from file. + + If params.start_batch is positive, it will load the checkpoint from + `params.exp_dir/checkpoint-{params.start_batch}.pt`. Otherwise, if + params.start_epoch is larger than 1, it will load the checkpoint from + `params.start_epoch - 1`. + + Apart from loading state dict for `model` and `optimizer` it also updates + `best_train_epoch`, `best_train_loss`, `best_valid_epoch`, + and `best_valid_loss` in `params`. + + Args: + params: + The return value of :func:`get_params`. + model: + The training model. + model_avg: + The stored model averaged from the start of training. + optimizer: + The optimizer that we are using. + scheduler: + The scheduler that we are using. + Returns: + Return a dict containing previously saved training info. + """ + if params.start_batch > 0: + filename = params.exp_dir / f"checkpoint-{params.start_batch}.pt" + elif params.start_epoch > 1: + filename = params.exp_dir / f"epoch-{params.start_epoch-1}.pt" + else: + return None + + assert filename.is_file(), f"{filename} does not exist!" + + if isinstance(model, DDP): + raise ValueError("load_checkpoint before DDP") + + saved_params = load_checkpoint( + filename, + model=model, + model_avg=model_avg, + optimizer=optimizer, + scheduler=scheduler, + ) + + keys = [ + "best_train_epoch", + "best_valid_epoch", + "batch_idx_train", + "best_train_loss", + "best_valid_loss", + ] + for k in keys: + params[k] = saved_params[k] + + if params.start_batch > 0: + if "cur_epoch" in saved_params: + params["start_epoch"] = saved_params["cur_epoch"] + + return saved_params + + +def save_checkpoint( + params: AttributeDict, + model: Union[nn.Module, DDP], + model_avg: Optional[nn.Module] = None, + optimizer: Optional[torch.optim.Optimizer] = None, + scheduler: Optional[LRSchedulerType] = None, + sampler: Optional[CutSampler] = None, + scaler: Optional[GradScaler] = None, + rank: int = 0, +) -> None: + """Save model, optimizer, scheduler and training stats to file. + + Args: + params: + It is returned by :func:`get_params`. + model: + The training model. + model_avg: + The stored model averaged from the start of training. + optimizer: + The optimizer used in the training. + sampler: + The sampler for the training dataset. + scaler: + The scaler used for mix precision training. + """ + if rank != 0: + return + filename = params.exp_dir / f"epoch-{params.cur_epoch}.pt" + save_checkpoint_impl( + filename=filename, + model=model, + model_avg=model_avg, + params=params, + optimizer=optimizer, + scheduler=scheduler, + sampler=sampler, + scaler=scaler, + rank=rank, + ) + + if params.best_train_epoch == params.cur_epoch: + best_train_filename = params.exp_dir / "best-train-loss.pt" + copyfile(src=filename, dst=best_train_filename) + + if params.best_valid_epoch == params.cur_epoch: + best_valid_filename = params.exp_dir / "best-valid-loss.pt" + copyfile(src=filename, dst=best_valid_filename) + + +def prepare_input(batch: dict, device: torch.device): + """Parse batch data""" + text_inputs = batch["text"] + # texts.extend(convert_char_to_pinyin([text], polyphone=true)) + text_inputs = convert_char_to_pinyin(text_inputs, polyphone=True) + + mel_spec = batch["features"] + mel_lengths = batch["features_lens"] + return text_inputs, mel_spec.to(device), mel_lengths.to(device) + + +def compute_loss( + params: AttributeDict, + model: Union[nn.Module, DDP], + tokenizer, + batch: dict, + is_training: bool, +) -> Tuple[Tensor, MetricsTracker]: + """ + Compute transducer loss given the model and its inputs. + + Args: + params: + Parameters for training. See :func:`get_params`. + model: + The model for training. It is an instance of Zipformer in our case. + batch: + A batch of data. See `lhotse.dataset.K2SpeechRecognitionDataset()` + for the content in it. + is_training: + True for training. False for validation. When it is True, this + function enables autograd during computation; when it is False, it + disables autograd. + warmup: a floating point value which increases throughout training; + values >= 1.0 are fully warmed up and have all modules present. + """ + device = model.device if isinstance(model, DDP) else next(model.parameters()).device + (text_inputs, mel_spec, mel_lengths) = prepare_input(batch, device=device) + # at entry, TextTokens is (N, P) + + with torch.set_grad_enabled(is_training): + loss, cond, pred = model(mel_spec, text=text_inputs, lens=mel_lengths) + assert loss.requires_grad == is_training + + info = MetricsTracker() + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + info["samples"] = mel_lengths.size(0) + + info["loss"] = loss.detach().cpu().item() * info["samples"] + + return loss, info + + +def compute_validation_loss( + params: AttributeDict, + model: Union[nn.Module, DDP], + tokenizer, + valid_dl: torch.utils.data.DataLoader, + world_size: int = 1, +) -> MetricsTracker: + """Run the validation process.""" + tot_loss = MetricsTracker() + + for batch_idx, batch in enumerate(valid_dl): + loss, loss_info = compute_loss( + params=params, + model=model, + tokenizer=tokenizer, + batch=batch, + is_training=False, + ) + assert loss.requires_grad is False + tot_loss = tot_loss + loss_info + if world_size > 1: + tot_loss.reduce(loss.device) + loss_value = tot_loss["loss"] / tot_loss["samples"] + if loss_value < params.best_valid_loss: + params.best_valid_epoch = params.cur_epoch + params.best_valid_loss = loss_value + + return tot_loss + + +def train_one_epoch( + params: AttributeDict, + model: Union[nn.Module, DDP], + tokenizer, + optimizer: torch.optim.Optimizer, + scheduler: LRSchedulerType, + train_dl: torch.utils.data.DataLoader, + valid_dl: torch.utils.data.DataLoader, + rng: random.Random, + scaler: GradScaler, + model_avg: Optional[nn.Module] = None, + tb_writer: Optional[SummaryWriter] = None, + world_size: int = 1, + rank: int = 0, +) -> None: + """Train the model for one epoch. + + The training loss from the mean of all frames is saved in + `params.train_loss`. It runs the validation process every + `params.valid_interval` batches. + + Args: + params: + It is returned by :func:`get_params`. + model: + The model for training. + optimizer: + The optimizer we are using. + scheduler: + The learning rate scheduler, we call step() every step. + train_dl: + Dataloader for the training dataset. + valid_dl: + Dataloader for the validation dataset. + rng: + Random for selecting. + scaler: + The scaler used for mix precision training. + model_avg: + The stored model averaged from the start of training. + tb_writer: + Writer to write log messages to tensorboard. + world_size: + Number of nodes in DDP training. If it is 1, DDP is disabled. + rank: + The rank of the node in DDP training. If no DDP is used, it should + be set to 0. + """ + model.train() + tot_loss = MetricsTracker() + iter_dl = iter(train_dl) + + dtype, enabled = torch.float32, False + if params.dtype in ["bfloat16", "bf16"]: + dtype, enabled = torch.bfloat16, True + elif params.dtype in ["float16", "fp16"]: + dtype, enabled = torch.float16, True + + batch_idx = 0 + while True: + try: + batch = next(iter_dl) + except StopIteration: + logging.info("Reaches end of dataloader.") + break + + batch_idx += 1 + + params.batch_idx_train += 1 + batch_size = len(batch["text"]) + + try: + with torch.amp.autocast("cuda", dtype=dtype, enabled=enabled): + loss, loss_info = compute_loss( + params=params, + model=model, + tokenizer=tokenizer, + batch=batch, + is_training=True, + ) + + # summary stats + tot_loss = (tot_loss * (1 - 1 / params.reset_interval)) + loss_info * ( + 1 / params.reset_interval + ) + + # NOTE: We use reduction==sum and loss is computed over utterances + # in the batch and there is no normalization to it so far. + + scaler.scale(loss).backward() + if params.batch_idx_train >= params.accumulate_grad_steps: + if params.batch_idx_train % params.accumulate_grad_steps == 0: + + # Unscales the gradients of optimizer's assigned params in-place + scaler.unscale_(optimizer) + # Since the gradients of optimizer's assigned params are unscaled, clips as usual: + torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0) + + scaler.step(optimizer) + scaler.update() + optimizer.zero_grad() + # loss.backward() + # optimizer.step() + + for k in range(params.accumulate_grad_steps): + scheduler.step() + + set_batch_count(model, params.batch_idx_train) + except: # noqa + display_and_save_batch(batch, params=params) + raise + + if params.average_period > 0: + if ( + params.batch_idx_train > 0 + and params.batch_idx_train % params.average_period == 0 + ): + # Perform Operation in rank 0 + if rank == 0: + update_averaged_model( + params=params, + model_cur=model, + model_avg=model_avg, + ) + + if ( + params.batch_idx_train > 0 + and params.batch_idx_train % params.save_every_n == 0 + ): + # Perform Operation in rank 0 + if rank == 0: + save_checkpoint_with_global_batch_idx( + out_dir=params.exp_dir, + global_batch_idx=params.batch_idx_train, + model=model, + model_avg=model_avg, + params=params, + optimizer=optimizer, + scheduler=scheduler, + sampler=train_dl.sampler, + scaler=scaler, + rank=rank, + ) + remove_checkpoints( + out_dir=params.exp_dir, + topk=params.keep_last_k, + rank=rank, + ) + + if batch_idx % 100 == 0 and params.dtype in ["float16", "fp16"]: + # If the grad scale was less than 1, try increasing it. The _growth_interval + # of the grad scaler is configurable, but we can't configure it to have different + # behavior depending on the current grad scale. + cur_grad_scale = scaler._scale.item() + if cur_grad_scale < 1.0 or (cur_grad_scale < 8.0 and batch_idx % 400 == 0): + scaler.update(cur_grad_scale * 2.0) + + if cur_grad_scale < 0.01: + logging.warning(f"Grad scale is small: {cur_grad_scale}") + if cur_grad_scale < 1.0e-05: + raise RuntimeError( + f"grad_scale is too small, exiting: {cur_grad_scale}" + ) + + if batch_idx % params.log_interval == 0: + cur_lr = scheduler.get_last_lr()[0] + cur_grad_scale = ( + scaler._scale.item() if params.dtype in ["float16", "fp16"] else 1.0 + ) + + logging.info( + f"Epoch {params.cur_epoch}, " + f"batch {batch_idx}, train_loss[{loss_info}], " + f"batch size: {batch_size}, " + f"lr: {cur_lr:.2e}" + + ( + f", grad_scale: {cur_grad_scale}" + if params.dtype in ["float16", "fp16"] + else "" + ) + ) + + if tb_writer is not None: + tb_writer.add_scalar( + "train/learning_rate", cur_lr, params.batch_idx_train + ) + loss_info.write_summary( + tb_writer, + "train/current_", + params.batch_idx_train, + ) + tot_loss.write_summary(tb_writer, "train/tot_", params.batch_idx_train) + tot_loss.write_summary(tb_writer, "train/tot_", params.batch_idx_train) + if params.dtype in ["float16", "fp16"]: + tb_writer.add_scalar( + "train/grad_scale", + cur_grad_scale, + params.batch_idx_train, + ) + + if params.batch_idx_train % params.valid_interval == 0: + # Calculate validation loss in Rank 0 + model.eval() + logging.info("Computing validation loss") + with torch.amp.autocast("cuda", dtype=dtype): + valid_info = compute_validation_loss( + params=params, + model=model, + tokenizer=tokenizer, + valid_dl=valid_dl, + world_size=world_size, + ) + logging.info(f"Epoch {params.cur_epoch}, validation: {valid_info}") + logging.info( + f"Maximum memory allocated so far is {torch.cuda.max_memory_allocated()//1000000}MB" + ) + + if tb_writer is not None: + valid_info.write_summary( + tb_writer, "train/valid_", params.batch_idx_train + ) + + model.train() + + loss_value = tot_loss["loss"] / tot_loss["samples"] + params.train_loss = loss_value + if params.train_loss < params.best_train_loss: + params.best_train_epoch = params.cur_epoch + params.best_train_loss = params.train_loss + + +def filter_short_and_long_utterances( + cuts: CutSet, min_duration: float, max_duration: float +) -> CutSet: + def remove_short_and_long_utt(c: Cut): + # Keep only utterances with duration between 0.6 second and 20 seconds + if c.duration < min_duration or c.duration > max_duration: + # logging.warning( + # f"Exclude cut with ID {c.id} from training. Duration: {c.duration}" + # ) + return False + return True + + cuts = cuts.filter(remove_short_and_long_utt) + + return cuts + + +def run(rank, world_size, args): + """ + Args: + rank: + It is a value between 0 and `world_size-1`, which is + passed automatically by `mp.spawn()` in :func:`main`. + The node with rank 0 is responsible for saving checkpoint. + world_size: + Number of GPUs for DDP training. + args: + The return value of get_parser().parse_args() + """ + params = get_params() + params.update(vars(args)) + + fix_random_seed(params.seed) + rng = random.Random(params.seed) + if world_size > 1: + setup_dist(rank, world_size, params.master_port) + + setup_logger(f"{params.exp_dir}/log/log-train") + logging.info("Training started") + + if args.tensorboard and rank == 0: + tb_writer = SummaryWriter(log_dir=f"{params.exp_dir}/tensorboard") + else: + tb_writer = None + + device = torch.device("cpu") + if torch.cuda.is_available(): + device = torch.device("cuda", rank) + # https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices + torch.backends.cudnn.allow_tf32 = True + torch.backends.cuda.matmul.allow_tf32 = True + + logging.info(f"Device: {device}") + tokenizer = get_tokenizer(params.tokens) + logging.info(params) + + logging.info("About to create model") + + model = get_model(params) + + if params.pretrained_model_path: + checkpoint = torch.load(params.pretrained_model_path, map_location="cpu") + if "ema_model_state_dict" in checkpoint or "model_state_dict" in checkpoint: + model = load_F5_TTS_pretrained_checkpoint( + model, params.pretrained_model_path + ) + else: + _ = load_checkpoint( + params.pretrained_model_path, + model=model, + ) + + model = model.to(device) + + with open(f"{params.exp_dir}/model.txt", "w") as f: + print(model) + print(model, file=f) + + num_param = sum([p.numel() for p in model.parameters()]) + logging.info(f"Number of model parameters: {num_param}") + + assert params.save_every_n >= params.average_period + model_avg: Optional[nn.Module] = None + if rank == 0 and params.average_period > 0: + # model_avg is only used with rank 0 + model_avg = copy.deepcopy(model).to(torch.float64) + + assert params.start_epoch > 0, params.start_epoch + + checkpoints = load_checkpoint_if_available( + params=params, model=model, model_avg=model_avg + ) + + model.to(device) + if world_size > 1: + logging.info("Using DDP") + model = DDP(model, device_ids=[rank], find_unused_parameters=False) + + model_parameters = model.parameters() + + optimizer = torch.optim.AdamW( + model_parameters, + lr=params.base_lr, + betas=(0.9, 0.95), + weight_decay=1e-2, + eps=1e-8, + ) + + warmup_scheduler = LinearLR( + optimizer, start_factor=1e-8, end_factor=1.0, total_iters=params.warmup_steps + ) + decay_scheduler = LinearLR( + optimizer, start_factor=1.0, end_factor=1e-8, total_iters=params.decay_steps + ) + scheduler = SequentialLR( + optimizer, + schedulers=[warmup_scheduler, decay_scheduler], + milestones=[params.warmup_steps], + ) + + optimizer.zero_grad() + + if checkpoints and "optimizer" in checkpoints: + logging.info("Loading optimizer state dict") + optimizer.load_state_dict(checkpoints["optimizer"]) + + if ( + checkpoints + and "scheduler" in checkpoints + and checkpoints["scheduler"] is not None + ): + logging.info("Loading scheduler state dict") + scheduler.load_state_dict(checkpoints["scheduler"]) + + if params.inf_check: + register_inf_check_hooks(model) + + if params.start_batch > 0 and checkpoints and "sampler" in checkpoints: + sampler_state_dict = checkpoints["sampler"] + else: + sampler_state_dict = None + + dataset = TtsDataModule(args) + train_cuts = dataset.train_cuts() + valid_cuts = dataset.valid_cuts() + + train_cuts = filter_short_and_long_utterances( + train_cuts, params.filter_min_duration, params.filter_max_duration + ) + valid_cuts = filter_short_and_long_utterances( + valid_cuts, params.filter_min_duration, params.filter_max_duration + ) + + train_dl = dataset.train_dataloaders( + train_cuts, sampler_state_dict=sampler_state_dict + ) + valid_dl = dataset.valid_dataloaders(valid_cuts) + + if params.oom_check: + scan_pessimistic_batches_for_oom( + model=model, + tokenizer=tokenizer, + train_dl=train_dl, + optimizer=optimizer, + params=params, + ) + + scaler = GradScaler( + "cuda", enabled=(params.dtype in ["fp16", "float16"]), init_scale=1.0 + ) + if checkpoints and "grad_scaler" in checkpoints: + logging.info("Loading grad scaler state dict") + scaler.load_state_dict(checkpoints["grad_scaler"]) + + for epoch in range(params.start_epoch, params.num_epochs + 1): + + fix_random_seed(params.seed + epoch - 1) + train_dl.sampler.set_epoch(epoch - 1) + + if tb_writer is not None: + tb_writer.add_scalar("train/epoch", epoch, params.batch_idx_train) + + params.cur_epoch = epoch + + train_one_epoch( + params=params, + model=model, + tokenizer=tokenizer, + model_avg=model_avg, + optimizer=optimizer, + scheduler=scheduler, + train_dl=train_dl, + valid_dl=valid_dl, + rng=rng, + scaler=scaler, + tb_writer=tb_writer, + world_size=world_size, + rank=rank, + ) + + save_checkpoint( + params=params, + model=model, + model_avg=model_avg, + optimizer=optimizer, + scheduler=scheduler, + sampler=train_dl.sampler, + scaler=scaler, + rank=rank, + ) + + logging.info("Done!") + + if world_size > 1: + torch.distributed.barrier() + cleanup_dist() + + +def display_and_save_batch( + batch: dict, + params: AttributeDict, +) -> None: + """Display the batch statistics and save the batch into disk. + + Args: + batch: + A batch of data. See `lhotse.dataset.K2SpeechRecognitionDataset()` + for the content in it. + params: + Parameters for training. See :func:`get_params`. + """ + from lhotse.utils import uuid4 + + filename = f"{params.exp_dir}/batch-{uuid4()}.pt" + logging.info(f"Saving batch to {filename}") + torch.save(batch, filename) + + +def scan_pessimistic_batches_for_oom( + model: Union[nn.Module, DDP], + tokenizer, + train_dl: torch.utils.data.DataLoader, + optimizer: torch.optim.Optimizer, + params: AttributeDict, +): + from lhotse.dataset import find_pessimistic_batches + + logging.info( + "Sanity check -- see if any of the batches in epoch 1 would cause OOM." + ) + batches, crit_values = find_pessimistic_batches(train_dl.sampler) + dtype = torch.float32 + if params.dtype in ["bfloat16", "bf16"]: + dtype = torch.bfloat16 + elif params.dtype in ["float16", "fp16"]: + dtype = torch.float16 + + for criterion, cuts in batches.items(): + batch = train_dl.dataset[cuts] + print(batch.keys()) + try: + with torch.amp.autocast("cuda", dtype=dtype): + loss, loss_info = compute_loss( + params=params, + model=model, + tokenizer=tokenizer, + batch=batch, + is_training=True, + ) + loss.backward(retain_graph=True) + optimizer.zero_grad() + except Exception as e: + if "CUDA out of memory" in str(e): + logging.error( + "Your GPU ran out of memory with the current " + "max_duration setting. We recommend decreasing " + "max_duration and trying again.\n" + f"Failing criterion: {criterion} " + f"(={crit_values[criterion]}) ..." + ) + display_and_save_batch(batch, params=params) + raise + logging.info( + f"Maximum memory allocated so far is {torch.cuda.max_memory_allocated()//1000000}MB" + ) + + +def main(): + parser = get_parser() + TtsDataModule.add_arguments(parser) + args = parser.parse_args() + + world_size = args.world_size + assert world_size >= 1 + if world_size > 1: + mp.spawn(run, args=(world_size, args), nprocs=world_size, join=True) + else: + run(rank=0, world_size=1, args=args) + + +torch.set_num_threads(1) +torch.set_num_interop_threads(1) + +if __name__ == "__main__": + main() diff --git a/egs/wenetspeech4tts/TTS/f5-tts/tts_datamodule.py b/egs/wenetspeech4tts/TTS/f5-tts/tts_datamodule.py new file mode 100644 index 000000000..80ba17318 --- /dev/null +++ b/egs/wenetspeech4tts/TTS/f5-tts/tts_datamodule.py @@ -0,0 +1,306 @@ +# Copyright 2021 Piotr Żelasko +# Copyright 2022-2023 Xiaomi Corporation (Authors: Mingshuang Luo, +# Zengwei Yao) +# +# See ../../../../LICENSE for clarification regarding multiple authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import argparse +import logging +from functools import lru_cache +from pathlib import Path +from typing import Any, Dict, Optional + +import torch +from lhotse import CutSet, load_manifest_lazy +from lhotse.dataset import ( # noqa F401 for PrecomputedFeatures; SpeechSynthesisDataset, + CutConcatenate, + CutMix, + DynamicBucketingSampler, + PrecomputedFeatures, + SimpleCutSampler, +) +from lhotse.dataset.input_strategies import ( # noqa F401 For AudioSamples + AudioSamples, + OnTheFlyFeatures, +) +from lhotse.utils import fix_random_seed +from speech_synthesis import SpeechSynthesisDataset # noqa F401 +from torch.utils.data import DataLoader + +from icefall.utils import str2bool + + +class _SeedWorkers: + def __init__(self, seed: int): + self.seed = seed + + def __call__(self, worker_id: int): + fix_random_seed(self.seed + worker_id) + + +class TtsDataModule: + """ + DataModule for tts experiments. + It assumes there is always one train and valid dataloader, + but there can be multiple test dataloaders (e.g. LibriSpeech test-clean + and test-other). + + It contains all the common data pipeline modules used in ASR + experiments, e.g.: + - dynamic batch size, + - bucketing samplers, + - cut concatenation, + - on-the-fly feature extraction + + This class should be derived for specific corpora used in ASR tasks. + """ + + def __init__(self, args: argparse.Namespace): + self.args = args + + @classmethod + def add_arguments(cls, parser: argparse.ArgumentParser): + group = parser.add_argument_group( + title="TTS data related options", + description="These options are used for the preparation of " + "PyTorch DataLoaders from Lhotse CutSet's -- they control the " + "effective batch sizes, sampling strategies, applied data " + "augmentations, etc.", + ) + + group.add_argument( + "--manifest-dir", + type=Path, + default=Path("data/fbank"), + help="Path to directory with train/valid/test cuts.", + ) + group.add_argument( + "--max-duration", + type=int, + default=200.0, + help="Maximum pooled recordings duration (seconds) in a " + "single batch. You can reduce it if it causes CUDA OOM.", + ) + group.add_argument( + "--bucketing-sampler", + type=str2bool, + default=True, + help="When enabled, the batches will come from buckets of " + "similar duration (saves padding frames).", + ) + group.add_argument( + "--num-buckets", + type=int, + default=30, + help="The number of buckets for the DynamicBucketingSampler" + "(you might want to increase it for larger datasets).", + ) + + group.add_argument( + "--on-the-fly-feats", + type=str2bool, + default=False, + help="When enabled, use on-the-fly cut mixing and feature " + "extraction. Will drop existing precomputed feature manifests " + "if available.", + ) + group.add_argument( + "--shuffle", + type=str2bool, + default=True, + help="When enabled (=default), the examples will be " + "shuffled for each epoch.", + ) + group.add_argument( + "--drop-last", + type=str2bool, + default=True, + help="Whether to drop last batch. Used by sampler.", + ) + group.add_argument( + "--return-cuts", + type=str2bool, + default=False, + help="When enabled, each batch will have the " + "field: batch['cut'] with the cuts that " + "were used to construct it.", + ) + group.add_argument( + "--num-workers", + type=int, + default=2, + help="The number of training dataloader workers that " + "collect the batches.", + ) + + group.add_argument( + "--input-strategy", + type=str, + default="PrecomputedFeatures", + help="AudioSamples or PrecomputedFeatures", + ) + parser.add_argument( + "--prefix", + type=str, + default="wenetspeech4tts", + help="prefix of the manifest file", + ) + + def train_dataloaders( + self, + cuts_train: CutSet, + sampler_state_dict: Optional[Dict[str, Any]] = None, + ) -> DataLoader: + """ + Args: + cuts_train: + CutSet for training. + sampler_state_dict: + The state dict for the training sampler. + """ + logging.info("About to create train dataset") + train = SpeechSynthesisDataset( + return_text=True, + return_tokens=False, + feature_input_strategy=eval(self.args.input_strategy)(), + return_cuts=self.args.return_cuts, + ) + + if self.args.on_the_fly_feats: + raise NotImplementedError( + "On-the-fly feature extraction is not implemented yet." + ) + + if self.args.bucketing_sampler: + logging.info("Using DynamicBucketingSampler.") + train_sampler = DynamicBucketingSampler( + cuts_train, + max_duration=self.args.max_duration, + shuffle=self.args.shuffle, + num_buckets=self.args.num_buckets, + buffer_size=self.args.num_buckets * 2000, + shuffle_buffer_size=self.args.num_buckets * 5000, + drop_last=self.args.drop_last, + ) + else: + logging.info("Using SimpleCutSampler.") + train_sampler = SimpleCutSampler( + cuts_train, + max_duration=self.args.max_duration, + shuffle=self.args.shuffle, + ) + logging.info("About to create train dataloader") + + if sampler_state_dict is not None: + logging.info("Loading sampler state dict") + train_sampler.load_state_dict(sampler_state_dict) + + # 'seed' is derived from the current random state, which will have + # previously been set in the main process. + seed = torch.randint(0, 100000, ()).item() + worker_init_fn = _SeedWorkers(seed) + + train_dl = DataLoader( + train, + sampler=train_sampler, + batch_size=None, + num_workers=self.args.num_workers, + persistent_workers=True, + pin_memory=True, + worker_init_fn=worker_init_fn, + ) + + return train_dl + + def valid_dataloaders(self, cuts_valid: CutSet) -> DataLoader: + logging.info("About to create dev dataset") + if self.args.on_the_fly_feats: + raise NotImplementedError( + "On-the-fly feature extraction is not implemented yet." + ) + else: + validate = SpeechSynthesisDataset( + return_text=True, + return_tokens=False, + feature_input_strategy=eval(self.args.input_strategy)(), + return_cuts=self.args.return_cuts, + ) + valid_sampler = DynamicBucketingSampler( + cuts_valid, + max_duration=self.args.max_duration, + num_buckets=self.args.num_buckets, + shuffle=False, + ) + logging.info("About to create valid dataloader") + valid_dl = DataLoader( + validate, + sampler=valid_sampler, + batch_size=None, + num_workers=2, + persistent_workers=True, + pin_memory=True, + ) + + return valid_dl + + def test_dataloaders(self, cuts: CutSet) -> DataLoader: + logging.info("About to create test dataset") + if self.args.on_the_fly_feats: + raise NotImplementedError( + "On-the-fly feature extraction is not implemented yet." + ) + else: + test = SpeechSynthesisDataset( + return_text=True, + return_tokens=False, + feature_input_strategy=eval(self.args.input_strategy)(), + return_cuts=self.args.return_cuts, + ) + test_sampler = DynamicBucketingSampler( + cuts, + max_duration=self.args.max_duration, + num_buckets=self.args.num_buckets, + shuffle=False, + ) + logging.info("About to create test dataloader") + test_dl = DataLoader( + test, + batch_size=None, + sampler=test_sampler, + num_workers=self.args.num_workers, + ) + return test_dl + + @lru_cache() + def train_cuts(self) -> CutSet: + logging.info("About to get train cuts") + return load_manifest_lazy( + self.args.manifest_dir / f"{self.args.prefix}_cuts_train.jsonl.gz" + ) + + @lru_cache() + def valid_cuts(self) -> CutSet: + logging.info("About to get validation cuts") + return load_manifest_lazy( + self.args.manifest_dir / f"{self.args.prefix}_cuts_valid.jsonl.gz" + ) + + @lru_cache() + def test_cuts(self) -> CutSet: + logging.info("About to get test cuts") + return load_manifest_lazy( + self.args.manifest_dir / f"{self.args.prefix}_cuts_test.jsonl.gz" + ) diff --git a/egs/wenetspeech4tts/TTS/f5-tts/utils.py b/egs/wenetspeech4tts/TTS/f5-tts/utils.py new file mode 120000 index 000000000..ceaaea196 --- /dev/null +++ b/egs/wenetspeech4tts/TTS/f5-tts/utils.py @@ -0,0 +1 @@ +../../../ljspeech/TTS/matcha/utils.py \ No newline at end of file diff --git a/egs/wenetspeech4tts/TTS/f5-tts/vocab.txt b/egs/wenetspeech4tts/TTS/f5-tts/vocab.txt new file mode 100644 index 000000000..93f8b48b2 --- /dev/null +++ b/egs/wenetspeech4tts/TTS/f5-tts/vocab.txt @@ -0,0 +1,2545 @@ + +! +" +# +$ +% +& +' +( +) +* ++ +, +- +. +/ +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +: +; += +> +? +@ +A +B +C +D +E +F +G +H +I +J +K +L +M +N +O +P +Q +R +S +T +U +V +W +X +Y +Z +[ +\ +] +_ +a +a1 +ai1 +ai2 +ai3 +ai4 +an1 +an3 +an4 +ang1 +ang2 +ang4 +ao1 +ao2 +ao3 +ao4 +b +ba +ba1 +ba2 +ba3 +ba4 +bai1 +bai2 +bai3 +bai4 +ban1 +ban2 +ban3 +ban4 +bang1 +bang2 +bang3 +bang4 +bao1 +bao2 +bao3 +bao4 +bei +bei1 +bei2 +bei3 +bei4 +ben1 +ben2 +ben3 +ben4 +beng +beng1 +beng2 +beng3 +beng4 +bi1 +bi2 +bi3 +bi4 +bian1 +bian2 +bian3 +bian4 +biao1 +biao2 +biao3 +bie1 +bie2 +bie3 +bie4 +bin1 +bin4 +bing1 +bing2 +bing3 +bing4 +bo +bo1 +bo2 +bo3 +bo4 +bu2 +bu3 +bu4 +c +ca1 +cai1 +cai2 +cai3 +cai4 +can1 +can2 +can3 +can4 +cang1 +cang2 +cao1 +cao2 +cao3 +ce4 +cen1 +cen2 +ceng1 +ceng2 +ceng4 +cha1 +cha2 +cha3 +cha4 +chai1 +chai2 +chan1 +chan2 +chan3 +chan4 +chang1 +chang2 +chang3 +chang4 +chao1 +chao2 +chao3 +che1 +che2 +che3 +che4 +chen1 +chen2 +chen3 +chen4 +cheng1 +cheng2 +cheng3 +cheng4 +chi1 +chi2 +chi3 +chi4 +chong1 +chong2 +chong3 +chong4 +chou1 +chou2 +chou3 +chou4 +chu1 +chu2 +chu3 +chu4 +chua1 +chuai1 +chuai2 +chuai3 +chuai4 +chuan1 +chuan2 +chuan3 +chuan4 +chuang1 +chuang2 +chuang3 +chuang4 +chui1 +chui2 +chun1 +chun2 +chun3 +chuo1 +chuo4 +ci1 +ci2 +ci3 +ci4 +cong1 +cong2 +cou4 +cu1 +cu4 +cuan1 +cuan2 +cuan4 +cui1 +cui3 +cui4 +cun1 +cun2 +cun4 +cuo1 +cuo2 +cuo4 +d +da +da1 +da2 +da3 +da4 +dai1 +dai2 +dai3 +dai4 +dan1 +dan2 +dan3 +dan4 +dang1 +dang2 +dang3 +dang4 +dao1 +dao2 +dao3 +dao4 +de +de1 +de2 +dei3 +den4 +deng1 +deng2 +deng3 +deng4 +di1 +di2 +di3 +di4 +dia3 +dian1 +dian2 +dian3 +dian4 +diao1 +diao3 +diao4 +die1 +die2 +die4 +ding1 +ding2 +ding3 +ding4 +diu1 +dong1 +dong3 +dong4 +dou1 +dou2 +dou3 +dou4 +du1 +du2 +du3 +du4 +duan1 +duan2 +duan3 +duan4 +dui1 +dui4 +dun1 +dun3 +dun4 +duo1 +duo2 +duo3 +duo4 +e +e1 +e2 +e3 +e4 +ei2 +en1 +en4 +er +er2 +er3 +er4 +f +fa1 +fa2 +fa3 +fa4 +fan1 +fan2 +fan3 +fan4 +fang1 +fang2 +fang3 +fang4 +fei1 +fei2 +fei3 +fei4 +fen1 +fen2 +fen3 +fen4 +feng1 +feng2 +feng3 +feng4 +fo2 +fou2 +fou3 +fu1 +fu2 +fu3 +fu4 +g +ga1 +ga2 +ga3 +ga4 +gai1 +gai2 +gai3 +gai4 +gan1 +gan2 +gan3 +gan4 +gang1 +gang2 +gang3 +gang4 +gao1 +gao2 +gao3 +gao4 +ge1 +ge2 +ge3 +ge4 +gei2 +gei3 +gen1 +gen2 +gen3 +gen4 +geng1 +geng3 +geng4 +gong1 +gong3 +gong4 +gou1 +gou2 +gou3 +gou4 +gu +gu1 +gu2 +gu3 +gu4 +gua1 +gua2 +gua3 +gua4 +guai1 +guai2 +guai3 +guai4 +guan1 +guan2 +guan3 +guan4 +guang1 +guang2 +guang3 +guang4 +gui1 +gui2 +gui3 +gui4 +gun3 +gun4 +guo1 +guo2 +guo3 +guo4 +h +ha1 +ha2 +ha3 +hai1 +hai2 +hai3 +hai4 +han1 +han2 +han3 +han4 +hang1 +hang2 +hang4 +hao1 +hao2 +hao3 +hao4 +he1 +he2 +he4 +hei1 +hen2 +hen3 +hen4 +heng1 +heng2 +heng4 +hong1 +hong2 +hong3 +hong4 +hou1 +hou2 +hou3 +hou4 +hu1 +hu2 +hu3 +hu4 +hua1 +hua2 +hua4 +huai2 +huai4 +huan1 +huan2 +huan3 +huan4 +huang1 +huang2 +huang3 +huang4 +hui1 +hui2 +hui3 +hui4 +hun1 +hun2 +hun4 +huo +huo1 +huo2 +huo3 +huo4 +i +j +ji1 +ji2 +ji3 +ji4 +jia +jia1 +jia2 +jia3 +jia4 +jian1 +jian2 +jian3 +jian4 +jiang1 +jiang2 +jiang3 +jiang4 +jiao1 +jiao2 +jiao3 +jiao4 +jie1 +jie2 +jie3 +jie4 +jin1 +jin2 +jin3 +jin4 +jing1 +jing2 +jing3 +jing4 +jiong3 +jiu1 +jiu2 +jiu3 +jiu4 +ju1 +ju2 +ju3 +ju4 +juan1 +juan2 +juan3 +juan4 +jue1 +jue2 +jue4 +jun1 +jun4 +k +ka1 +ka2 +ka3 +kai1 +kai2 +kai3 +kai4 +kan1 +kan2 +kan3 +kan4 +kang1 +kang2 +kang4 +kao1 +kao2 +kao3 +kao4 +ke1 +ke2 +ke3 +ke4 +ken3 +keng1 +kong1 +kong3 +kong4 +kou1 +kou2 +kou3 +kou4 +ku1 +ku2 +ku3 +ku4 +kua1 +kua3 +kua4 +kuai3 +kuai4 +kuan1 +kuan2 +kuan3 +kuang1 +kuang2 +kuang4 +kui1 +kui2 +kui3 +kui4 +kun1 +kun3 +kun4 +kuo4 +l +la +la1 +la2 +la3 +la4 +lai2 +lai4 +lan2 +lan3 +lan4 +lang1 +lang2 +lang3 +lang4 +lao1 +lao2 +lao3 +lao4 +le +le1 +le4 +lei +lei1 +lei2 +lei3 +lei4 +leng1 +leng2 +leng3 +leng4 +li +li1 +li2 +li3 +li4 +lia3 +lian2 +lian3 +lian4 +liang2 +liang3 +liang4 +liao1 +liao2 +liao3 +liao4 +lie1 +lie2 +lie3 +lie4 +lin1 +lin2 +lin3 +lin4 +ling2 +ling3 +ling4 +liu1 +liu2 +liu3 +liu4 +long1 +long2 +long3 +long4 +lou1 +lou2 +lou3 +lou4 +lu1 +lu2 +lu3 +lu4 +luan2 +luan3 +luan4 +lun1 +lun2 +lun4 +luo1 +luo2 +luo3 +luo4 +lv2 +lv3 +lv4 +lve3 +lve4 +m +ma +ma1 +ma2 +ma3 +ma4 +mai2 +mai3 +mai4 +man1 +man2 +man3 +man4 +mang2 +mang3 +mao1 +mao2 +mao3 +mao4 +me +mei2 +mei3 +mei4 +men +men1 +men2 +men4 +meng +meng1 +meng2 +meng3 +meng4 +mi1 +mi2 +mi3 +mi4 +mian2 +mian3 +mian4 +miao1 +miao2 +miao3 +miao4 +mie1 +mie4 +min2 +min3 +ming2 +ming3 +ming4 +miu4 +mo1 +mo2 +mo3 +mo4 +mou1 +mou2 +mou3 +mu2 +mu3 +mu4 +n +n2 +na1 +na2 +na3 +na4 +nai2 +nai3 +nai4 +nan1 +nan2 +nan3 +nan4 +nang1 +nang2 +nang3 +nao1 +nao2 +nao3 +nao4 +ne +ne2 +ne4 +nei3 +nei4 +nen4 +neng2 +ni1 +ni2 +ni3 +ni4 +nian1 +nian2 +nian3 +nian4 +niang2 +niang4 +niao2 +niao3 +niao4 +nie1 +nie4 +nin2 +ning2 +ning3 +ning4 +niu1 +niu2 +niu3 +niu4 +nong2 +nong4 +nou4 +nu2 +nu3 +nu4 +nuan3 +nuo2 +nuo4 +nv2 +nv3 +nve4 +o +o1 +o2 +ou1 +ou2 +ou3 +ou4 +p +pa1 +pa2 +pa4 +pai1 +pai2 +pai3 +pai4 +pan1 +pan2 +pan4 +pang1 +pang2 +pang4 +pao1 +pao2 +pao3 +pao4 +pei1 +pei2 +pei4 +pen1 +pen2 +pen4 +peng1 +peng2 +peng3 +peng4 +pi1 +pi2 +pi3 +pi4 +pian1 +pian2 +pian4 +piao1 +piao2 +piao3 +piao4 +pie1 +pie2 +pie3 +pin1 +pin2 +pin3 +pin4 +ping1 +ping2 +po1 +po2 +po3 +po4 +pou1 +pu1 +pu2 +pu3 +pu4 +q +qi1 +qi2 +qi3 +qi4 +qia1 +qia3 +qia4 +qian1 +qian2 +qian3 +qian4 +qiang1 +qiang2 +qiang3 +qiang4 +qiao1 +qiao2 +qiao3 +qiao4 +qie1 +qie2 +qie3 +qie4 +qin1 +qin2 +qin3 +qin4 +qing1 +qing2 +qing3 +qing4 +qiong1 +qiong2 +qiu1 +qiu2 +qiu3 +qu1 +qu2 +qu3 +qu4 +quan1 +quan2 +quan3 +quan4 +que1 +que2 +que4 +qun2 +r +ran2 +ran3 +rang1 +rang2 +rang3 +rang4 +rao2 +rao3 +rao4 +re2 +re3 +re4 +ren2 +ren3 +ren4 +reng1 +reng2 +ri4 +rong1 +rong2 +rong3 +rou2 +rou4 +ru2 +ru3 +ru4 +ruan2 +ruan3 +rui3 +rui4 +run4 +ruo4 +s +sa1 +sa2 +sa3 +sa4 +sai1 +sai4 +san1 +san2 +san3 +san4 +sang1 +sang3 +sang4 +sao1 +sao2 +sao3 +sao4 +se4 +sen1 +seng1 +sha1 +sha2 +sha3 +sha4 +shai1 +shai2 +shai3 +shai4 +shan1 +shan3 +shan4 +shang +shang1 +shang3 +shang4 +shao1 +shao2 +shao3 +shao4 +she1 +she2 +she3 +she4 +shei2 +shen1 +shen2 +shen3 +shen4 +sheng1 +sheng2 +sheng3 +sheng4 +shi +shi1 +shi2 +shi3 +shi4 +shou1 +shou2 +shou3 +shou4 +shu1 +shu2 +shu3 +shu4 +shua1 +shua2 +shua3 +shua4 +shuai1 +shuai3 +shuai4 +shuan1 +shuan4 +shuang1 +shuang3 +shui2 +shui3 +shui4 +shun3 +shun4 +shuo1 +shuo4 +si1 +si2 +si3 +si4 +song1 +song3 +song4 +sou1 +sou3 +sou4 +su1 +su2 +su4 +suan1 +suan4 +sui1 +sui2 +sui3 +sui4 +sun1 +sun3 +suo +suo1 +suo2 +suo3 +t +ta1 +ta2 +ta3 +ta4 +tai1 +tai2 +tai4 +tan1 +tan2 +tan3 +tan4 +tang1 +tang2 +tang3 +tang4 +tao1 +tao2 +tao3 +tao4 +te4 +teng2 +ti1 +ti2 +ti3 +ti4 +tian1 +tian2 +tian3 +tiao1 +tiao2 +tiao3 +tiao4 +tie1 +tie2 +tie3 +tie4 +ting1 +ting2 +ting3 +tong1 +tong2 +tong3 +tong4 +tou +tou1 +tou2 +tou4 +tu1 +tu2 +tu3 +tu4 +tuan1 +tuan2 +tui1 +tui2 +tui3 +tui4 +tun1 +tun2 +tun4 +tuo1 +tuo2 +tuo3 +tuo4 +u +v +w +wa +wa1 +wa2 +wa3 +wa4 +wai1 +wai3 +wai4 +wan1 +wan2 +wan3 +wan4 +wang1 +wang2 +wang3 +wang4 +wei1 +wei2 +wei3 +wei4 +wen1 +wen2 +wen3 +wen4 +weng1 +weng4 +wo1 +wo2 +wo3 +wo4 +wu1 +wu2 +wu3 +wu4 +x +xi1 +xi2 +xi3 +xi4 +xia1 +xia2 +xia4 +xian1 +xian2 +xian3 +xian4 +xiang1 +xiang2 +xiang3 +xiang4 +xiao1 +xiao2 +xiao3 +xiao4 +xie1 +xie2 +xie3 +xie4 +xin1 +xin2 +xin4 +xing1 +xing2 +xing3 +xing4 +xiong1 +xiong2 +xiu1 +xiu3 +xiu4 +xu +xu1 +xu2 +xu3 +xu4 +xuan1 +xuan2 +xuan3 +xuan4 +xue1 +xue2 +xue3 +xue4 +xun1 +xun2 +xun4 +y +ya +ya1 +ya2 +ya3 +ya4 +yan1 +yan2 +yan3 +yan4 +yang1 +yang2 +yang3 +yang4 +yao1 +yao2 +yao3 +yao4 +ye1 +ye2 +ye3 +ye4 +yi +yi1 +yi2 +yi3 +yi4 +yin1 +yin2 +yin3 +yin4 +ying1 +ying2 +ying3 +ying4 +yo1 +yong1 +yong2 +yong3 +yong4 +you1 +you2 +you3 +you4 +yu1 +yu2 +yu3 +yu4 +yuan1 +yuan2 +yuan3 +yuan4 +yue1 +yue4 +yun1 +yun2 +yun3 +yun4 +z +za1 +za2 +za3 +zai1 +zai3 +zai4 +zan1 +zan2 +zan3 +zan4 +zang1 +zang4 +zao1 +zao2 +zao3 +zao4 +ze2 +ze4 +zei2 +zen3 +zeng1 +zeng4 +zha1 +zha2 +zha3 +zha4 +zhai1 +zhai2 +zhai3 +zhai4 +zhan1 +zhan2 +zhan3 +zhan4 +zhang1 +zhang2 +zhang3 +zhang4 +zhao1 +zhao2 +zhao3 +zhao4 +zhe +zhe1 +zhe2 +zhe3 +zhe4 +zhen1 +zhen2 +zhen3 +zhen4 +zheng1 +zheng2 +zheng3 +zheng4 +zhi1 +zhi2 +zhi3 +zhi4 +zhong1 +zhong2 +zhong3 +zhong4 +zhou1 +zhou2 +zhou3 +zhou4 +zhu1 +zhu2 +zhu3 +zhu4 +zhua1 +zhua2 +zhua3 +zhuai1 +zhuai3 +zhuai4 +zhuan1 +zhuan2 +zhuan3 +zhuan4 +zhuang1 +zhuang4 +zhui1 +zhui4 +zhun1 +zhun2 +zhun3 +zhuo1 +zhuo2 +zi +zi1 +zi2 +zi3 +zi4 +zong1 +zong2 +zong3 +zong4 +zou1 +zou2 +zou3 +zou4 +zu1 +zu2 +zu3 +zuan1 +zuan3 +zuan4 +zui2 +zui3 +zui4 +zun1 +zuo +zuo1 +zuo2 +zuo3 +zuo4 +{ +~ +¡ +¢ +£ +¥ +§ +¨ +© +« +® +¯ +° +± +² +³ +´ +µ +· +¹ +º +» +¼ +½ +¾ +¿ +À +Á + +à +Ä +Å +Æ +Ç +È +É +Ê +Í +Î +Ñ +Ó +Ö +× +Ø +Ú +Ü +Ý +Þ +ß +à +á +â +ã +ä +å +æ +ç +è +é +ê +ë +ì +í +î +ï +ð +ñ +ò +ó +ô +õ +ö +ø +ù +ú +û +ü +ý +Ā +ā +ă +ą +ć +Č +č +Đ +đ +ē +ė +ę +ě +ĝ +ğ +ħ +ī +į +İ +ı +Ł +ł +ń +ņ +ň +ŋ +Ō +ō +ő +œ +ř +Ś +ś +Ş +ş +Š +š +Ť +ť +ũ +ū +ź +Ż +ż +Ž +ž +ơ +ư +ǎ +ǐ +ǒ +ǔ +ǚ +ș +ț +ɑ +ɔ +ɕ +ə +ɛ +ɜ +ɡ +ɣ +ɪ +ɫ +ɴ +ɹ +ɾ +ʃ +ʊ +ʌ +ʒ +ʔ +ʰ +ʷ +ʻ +ʾ +ʿ +ˈ +ː +˙ +˜ +ˢ +́ +̅ +Α +Β +Δ +Ε +Θ +Κ +Λ +Μ +Ξ +Π +Σ +Τ +Φ +Χ +Ψ +Ω +ά +έ +ή +ί +α +β +γ +δ +ε +ζ +η +θ +ι +κ +λ +μ +ν +ξ +ο +π +ρ +ς +σ +τ +υ +φ +χ +ψ +ω +ϊ +ό +ύ +ώ +ϕ +ϵ +Ё +А +Б +В +Г +Д +Е +Ж +З +И +Й +К +Л +М +Н +О +П +Р +С +Т +У +Ф +Х +Ц +Ч +Ш +Щ +Ы +Ь +Э +Ю +Я +а +б +в +г +д +е +ж +з +и +й +к +л +м +н +о +п +р +с +т +у +ф +х +ц +ч +ш +щ +ъ +ы +ь +э +ю +я +ё +і +ְ +ִ +ֵ +ֶ +ַ +ָ +ֹ +ּ +־ +ׁ +א +ב +ג +ד +ה +ו +ז +ח +ט +י +כ +ל +ם +מ +ן +נ +ס +ע +פ +ק +ר +ש +ת +أ +ب +ة +ت +ج +ح +د +ر +ز +س +ص +ط +ع +ق +ك +ل +م +ن +ه +و +ي +َ +ُ +ِ +ْ +ก +ข +ง +จ +ต +ท +น +ป +ย +ร +ว +ส +ห +อ +ฮ +ั +า +ี +ึ +โ +ใ +ไ +่ +้ +์ +ḍ +Ḥ +ḥ +ṁ +ṃ +ṅ +ṇ +Ṛ +ṛ +Ṣ +ṣ +Ṭ +ṭ +ạ +ả +Ấ +ấ +ầ +ậ +ắ +ằ +ẻ +ẽ +ế +ề +ể +ễ +ệ +ị +ọ +ỏ +ố +ồ +ộ +ớ +ờ +ở +ụ +ủ +ứ +ữ +ἀ +ἁ +Ἀ +ἐ +ἔ +ἰ +ἱ +ὀ +ὁ +ὐ +ὲ +ὸ +ᾶ +᾽ +ῆ +ῇ +ῶ +‎ +‑ +‒ +– +— +― +‖ +† +‡ +• +… +‧ +‬ +′ +″ +⁄ +⁡ +⁰ +⁴ +⁵ +⁶ +⁷ +⁸ +⁹ +₁ +₂ +₃ +€ +₱ +₹ +₽ +℃ +ℏ +ℓ +№ +ℝ +™ +⅓ +⅔ +⅛ +→ +∂ +∈ +∑ +− +∗ +√ +∞ +∫ +≈ +≠ +≡ +≤ +≥ +⋅ +⋯ +█ +♪ +⟨ +⟩ +、 +。 +《 +》 +「 +」 +【 +】 +あ +う +え +お +か +が +き +ぎ +く +ぐ +け +げ +こ +ご +さ +し +じ +す +ず +せ +ぜ +そ +ぞ +た +だ +ち +っ +つ +で +と +ど +な +に +ね +の +は +ば +ひ +ぶ +へ +べ +ま +み +む +め +も +ゃ +や +ゆ +ょ +よ +ら +り +る +れ +ろ +わ +を +ん +ァ +ア +ィ +イ +ウ +ェ +エ +オ +カ +ガ +キ +ク +ケ +ゲ +コ +ゴ +サ +ザ +シ +ジ +ス +ズ +セ +ゾ +タ +ダ +チ +ッ +ツ +テ +デ +ト +ド +ナ +ニ +ネ +ノ +バ +パ +ビ +ピ +フ +プ +ヘ +ベ +ペ +ホ +ボ +ポ +マ +ミ +ム +メ +モ +ャ +ヤ +ュ +ユ +ョ +ヨ +ラ +リ +ル +レ +ロ +ワ +ン +・ +ー +ㄋ +ㄍ +ㄎ +ㄏ +ㄓ +ㄕ +ㄚ +ㄜ +ㄟ +ㄤ +ㄥ +ㄧ +ㄱ +ㄴ +ㄷ +ㄹ +ㅁ +ㅂ +ㅅ +ㅈ +ㅍ +ㅎ +ㅏ +ㅓ +ㅗ +ㅜ +ㅡ +ㅣ +㗎 +가 +각 +간 +갈 +감 +갑 +갓 +갔 +강 +같 +개 +거 +건 +걸 +겁 +것 +겉 +게 +겠 +겨 +결 +겼 +경 +계 +고 +곤 +골 +곱 +공 +과 +관 +광 +교 +구 +국 +굴 +귀 +귄 +그 +근 +글 +금 +기 +긴 +길 +까 +깍 +깔 +깜 +깨 +께 +꼬 +꼭 +꽃 +꾸 +꿔 +끔 +끗 +끝 +끼 +나 +난 +날 +남 +납 +내 +냐 +냥 +너 +넘 +넣 +네 +녁 +년 +녕 +노 +녹 +놀 +누 +눈 +느 +는 +늘 +니 +님 +닙 +다 +닥 +단 +달 +닭 +당 +대 +더 +덕 +던 +덥 +데 +도 +독 +동 +돼 +됐 +되 +된 +될 +두 +둑 +둥 +드 +들 +등 +디 +따 +딱 +딸 +땅 +때 +떤 +떨 +떻 +또 +똑 +뚱 +뛰 +뜻 +띠 +라 +락 +란 +람 +랍 +랑 +래 +랜 +러 +런 +럼 +렇 +레 +려 +력 +렵 +렸 +로 +록 +롬 +루 +르 +른 +를 +름 +릉 +리 +릴 +림 +마 +막 +만 +많 +말 +맑 +맙 +맛 +매 +머 +먹 +멍 +메 +면 +명 +몇 +모 +목 +몸 +못 +무 +문 +물 +뭐 +뭘 +미 +민 +밌 +밑 +바 +박 +밖 +반 +받 +발 +밤 +밥 +방 +배 +백 +밸 +뱀 +버 +번 +벌 +벚 +베 +벼 +벽 +별 +병 +보 +복 +본 +볼 +봐 +봤 +부 +분 +불 +비 +빔 +빛 +빠 +빨 +뼈 +뽀 +뿅 +쁘 +사 +산 +살 +삼 +샀 +상 +새 +색 +생 +서 +선 +설 +섭 +섰 +성 +세 +셔 +션 +셨 +소 +속 +손 +송 +수 +숙 +순 +술 +숫 +숭 +숲 +쉬 +쉽 +스 +슨 +습 +슷 +시 +식 +신 +실 +싫 +심 +십 +싶 +싸 +써 +쓰 +쓴 +씌 +씨 +씩 +씬 +아 +악 +안 +않 +알 +야 +약 +얀 +양 +얘 +어 +언 +얼 +엄 +업 +없 +었 +엉 +에 +여 +역 +연 +염 +엽 +영 +옆 +예 +옛 +오 +온 +올 +옷 +옹 +와 +왔 +왜 +요 +욕 +용 +우 +운 +울 +웃 +워 +원 +월 +웠 +위 +윙 +유 +육 +윤 +으 +은 +을 +음 +응 +의 +이 +익 +인 +일 +읽 +임 +입 +있 +자 +작 +잔 +잖 +잘 +잡 +잤 +장 +재 +저 +전 +점 +정 +제 +져 +졌 +조 +족 +좀 +종 +좋 +죠 +주 +준 +줄 +중 +줘 +즈 +즐 +즘 +지 +진 +집 +짜 +짝 +쩌 +쪼 +쪽 +쫌 +쭈 +쯔 +찌 +찍 +차 +착 +찾 +책 +처 +천 +철 +체 +쳐 +쳤 +초 +촌 +추 +출 +춤 +춥 +춰 +치 +친 +칠 +침 +칩 +칼 +커 +켓 +코 +콩 +쿠 +퀴 +크 +큰 +큽 +키 +킨 +타 +태 +터 +턴 +털 +테 +토 +통 +투 +트 +특 +튼 +틀 +티 +팀 +파 +팔 +패 +페 +펜 +펭 +평 +포 +폭 +표 +품 +풍 +프 +플 +피 +필 +하 +학 +한 +할 +함 +합 +항 +해 +햇 +했 +행 +허 +험 +형 +혜 +호 +혼 +홀 +화 +회 +획 +후 +휴 +흐 +흔 +희 +히 +힘 +ﷺ +ﷻ +！ +， +？ +� +𠮶 diff --git a/egs/wenetspeech4tts/TTS/local/audio.py b/egs/wenetspeech4tts/TTS/local/audio.py new file mode 100644 index 000000000..b643e3de0 --- /dev/null +++ b/egs/wenetspeech4tts/TTS/local/audio.py @@ -0,0 +1,122 @@ +# Copyright (c) 2024 NVIDIA CORPORATION. +# Licensed under the MIT license. + +# Adapted from https://github.com/jik876/hifi-gan under the MIT license. +# LICENSE is in incl_licenses directory. + +import math +import os +import pathlib +import random +from typing import List, Optional, Tuple + +import librosa +import numpy as np +import torch +import torch.utils.data +from librosa.filters import mel as librosa_mel_fn +from tqdm import tqdm + +# from env import AttrDict + +MAX_WAV_VALUE = 32767.0 # NOTE: 32768.0 -1 to prevent int16 overflow (results in popping sound in corner cases) + + +def dynamic_range_compression(x, C=1, clip_val=1e-5): + return np.log(np.clip(x, a_min=clip_val, a_max=None) * C) + + +def dynamic_range_decompression(x, C=1): + return np.exp(x) / C + + +def dynamic_range_compression_torch(x, C=1, clip_val=1e-5): + return torch.log(torch.clamp(x, min=clip_val) * C) + + +def dynamic_range_decompression_torch(x, C=1): + return torch.exp(x) / C + + +def spectral_normalize_torch(magnitudes): + return dynamic_range_compression_torch(magnitudes) + + +def spectral_de_normalize_torch(magnitudes): + return dynamic_range_decompression_torch(magnitudes) + + +mel_basis_cache = {} +hann_window_cache = {} + + +def mel_spectrogram( + y: torch.Tensor, + n_fft: int = 1024, + num_mels: int = 100, + sampling_rate: int = 24_000, + hop_size: int = 256, + win_size: int = 1024, + fmin: int = 0, + fmax: int = None, + center: bool = False, +) -> torch.Tensor: + """ + Calculate the mel spectrogram of an input signal. + This function uses slaney norm for the librosa mel filterbank (using librosa.filters.mel) and uses Hann window for STFT (using torch.stft). + + Args: + y (torch.Tensor): Input signal. + n_fft (int): FFT size. + num_mels (int): Number of mel bins. + sampling_rate (int): Sampling rate of the input signal. + hop_size (int): Hop size for STFT. + win_size (int): Window size for STFT. + fmin (int): Minimum frequency for mel filterbank. + fmax (int): Maximum frequency for mel filterbank. If None, defaults to half the sampling rate (fmax = sr / 2.0) inside librosa_mel_fn + center (bool): Whether to pad the input to center the frames. Default is False. + + Returns: + torch.Tensor: Mel spectrogram. + """ + if torch.min(y) < -1.0: + print(f"[WARNING] Min value of input waveform signal is {torch.min(y)}") + if torch.max(y) > 1.0: + print(f"[WARNING] Max value of input waveform signal is {torch.max(y)}") + + device = y.device + key = f"{n_fft}_{num_mels}_{sampling_rate}_{hop_size}_{win_size}_{fmin}_{fmax}_{device}" + + if key not in mel_basis_cache: + mel = librosa_mel_fn( + sr=sampling_rate, n_fft=n_fft, n_mels=num_mels, fmin=fmin, fmax=fmax + ) + mel_basis_cache[key] = torch.from_numpy(mel).float().to(device) + hann_window_cache[key] = torch.hann_window(win_size).to(device) + + mel_basis = mel_basis_cache[key] + hann_window = hann_window_cache[key] + + padding = (n_fft - hop_size) // 2 + y = torch.nn.functional.pad( + y.unsqueeze(1), (padding, padding), mode="reflect" + ).squeeze(1) + + spec = torch.stft( + y, + n_fft, + hop_length=hop_size, + win_length=win_size, + window=hann_window, + center=center, + pad_mode="reflect", + normalized=False, + onesided=True, + return_complex=True, + ) + spec = torch.sqrt(torch.view_as_real(spec).pow(2).sum(-1) + 1e-9) + + mel_spec = torch.matmul(mel_basis, spec) + mel_spec = spectral_normalize_torch(mel_spec) + + return mel_spec diff --git a/egs/wenetspeech4tts/TTS/local/compute_mel_feat.py b/egs/wenetspeech4tts/TTS/local/compute_mel_feat.py new file mode 100755 index 000000000..5292c75ad --- /dev/null +++ b/egs/wenetspeech4tts/TTS/local/compute_mel_feat.py @@ -0,0 +1,218 @@ +#!/usr/bin/env python3 +# Copyright 2021-2023 Xiaomi Corp. (authors: Fangjun Kuang, +# Zengwei Yao) +# +# See ../../../../LICENSE for clarification regarding multiple authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +""" +This file computes fbank features of the LJSpeech dataset. +It looks for manifests in the directory data/manifests. + +The generated fbank features are saved in data/fbank. +""" + +import argparse +import logging +import os +from pathlib import Path + +import torch +from fbank import MatchaFbank, MatchaFbankConfig +from lhotse import CutSet, LilcomChunkyWriter +from lhotse.recipes.utils import read_manifests_if_cached + +from icefall.utils import get_executor + + +def get_parser(): + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + + parser.add_argument( + "--num-jobs", + type=int, + default=1, + help="""It specifies the checkpoint to use for decoding. + Note: Epoch counts from 1. + """, + ) + + parser.add_argument( + "--src-dir", + type=Path, + default=Path("data/manifests"), + help="Path to the manifest files", + ) + + parser.add_argument( + "--output-dir", + type=Path, + default=Path("data/fbank"), + help="Path to the tokenized files", + ) + + parser.add_argument( + "--dataset-parts", + type=str, + default="Basic", + help="Space separated dataset parts", + ) + + parser.add_argument( + "--prefix", + type=str, + default="wenetspeech4tts", + help="prefix of the manifest file", + ) + + parser.add_argument( + "--suffix", + type=str, + default="jsonl.gz", + help="suffix of the manifest file", + ) + + parser.add_argument( + "--split", + type=int, + default=100, + help="Split the cut_set into multiple parts", + ) + + parser.add_argument( + "--resample-to-24kHz", + default=True, + help="Resample the audio to 24kHz", + ) + + parser.add_argument( + "--extractor", + type=str, + choices=["bigvgan", "hifigan"], + default="bigvgan", + help="The type of extractor to use", + ) + return parser + + +def compute_fbank(args): + src_dir = Path(args.src_dir) + output_dir = Path(args.output_dir) + Path(args.output_dir).mkdir(parents=True, exist_ok=True) + + num_jobs = min(args.num_jobs, os.cpu_count()) + dataset_parts = args.dataset_parts.replace("--dataset-parts", "").strip().split(" ") + + logging.info(f"num_jobs: {num_jobs}") + logging.info(f"src_dir: {src_dir}") + logging.info(f"output_dir: {output_dir}") + logging.info(f"dataset_parts: {dataset_parts}") + if args.extractor == "bigvgan": + config = MatchaFbankConfig( + n_fft=1024, + n_mels=100, + sampling_rate=24_000, + hop_length=256, + win_length=1024, + f_min=0, + f_max=None, + ) + elif args.extractor == "hifigan": + config = MatchaFbankConfig( + n_fft=1024, + n_mels=80, + sampling_rate=22050, + hop_length=256, + win_length=1024, + f_min=0, + f_max=8000, + ) + else: + raise NotImplementedError(f"Extractor {args.extractor} is not implemented") + + extractor = MatchaFbank(config) + + manifests = read_manifests_if_cached( + dataset_parts=dataset_parts, + output_dir=args.src_dir, + prefix=args.prefix, + suffix=args.suffix, + types=["recordings", "supervisions", "cuts"], + ) + + with get_executor() as ex: + for partition, m in manifests.items(): + logging.info( + f"Processing partition: {partition} CUDA: {torch.cuda.is_available()}" + ) + try: + cut_set = CutSet.from_manifests( + recordings=m["recordings"], + supervisions=m["supervisions"], + ) + except Exception: + cut_set = m["cuts"] + + if args.split > 1: + cut_sets = cut_set.split(args.split) + else: + cut_sets = [cut_set] + + for idx, part in enumerate(cut_sets): + if args.split > 1: + storage_path = f"{args.output_dir}/{args.prefix}_{args.extractor}_{partition}_{idx}" + else: + storage_path = ( + f"{args.output_dir}/{args.prefix}_{args.extractor}_{partition}" + ) + + if args.resample_to_24kHz: + part = part.resample(24000) + + with torch.no_grad(): + part = part.compute_and_store_features( + extractor=extractor, + storage_path=storage_path, + num_jobs=num_jobs if ex is None else 64, + executor=ex, + storage_type=LilcomChunkyWriter, + ) + + if args.split > 1: + cuts_filename = ( + f"{args.prefix}_cuts_{partition}.{idx}.{args.suffix}" + ) + else: + cuts_filename = f"{args.prefix}_cuts_{partition}.{args.suffix}" + + part.to_file(f"{args.output_dir}/{cuts_filename}") + logging.info(f"Saved {cuts_filename}") + + +if __name__ == "__main__": + # Torch's multithreaded behavior needs to be disabled or + # it wastes a lot of CPU and slow things down. + # Do this outside of main() in case it needs to take effect + # even when we are not invoking the main (e.g. when spawning subprocesses). + torch.set_num_threads(1) + torch.set_num_interop_threads(1) + formatter = "%(asctime)s %(levelname)s [%(filename)s:%(lineno)d] %(message)s" + + logging.basicConfig(format=formatter, level=logging.INFO) + + args = get_parser().parse_args() + compute_fbank(args) diff --git a/egs/wenetspeech4tts/TTS/local/compute_wer.sh b/egs/wenetspeech4tts/TTS/local/compute_wer.sh new file mode 100644 index 000000000..283546383 --- /dev/null +++ b/egs/wenetspeech4tts/TTS/local/compute_wer.sh @@ -0,0 +1,26 @@ +wav_dir=$1 +wav_files=$(ls $wav_dir/*.wav) +# if wav_files is empty, then exit +if [ -z "$wav_files" ]; then + exit 1 +fi +label_file=$2 +model_path=local/sherpa-onnx-paraformer-zh-2023-09-14 + +if [ ! -d $model_path ]; then + pip install sherpa-onnx + wget -nc https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-paraformer-zh-2023-09-14.tar.bz2 + tar xvf sherpa-onnx-paraformer-zh-2023-09-14.tar.bz2 -C local +fi + +python3 local/offline-decode-files.py \ + --tokens=$model_path/tokens.txt \ + --paraformer=$model_path/model.int8.onnx \ + --num-threads=2 \ + --decoding-method=greedy_search \ + --debug=false \ + --sample-rate=24000 \ + --log-dir $wav_dir \ + --feature-dim=80 \ + --label $label_file \ + $wav_files diff --git a/egs/wenetspeech4tts/TTS/local/fbank.py b/egs/wenetspeech4tts/TTS/local/fbank.py new file mode 120000 index 000000000..3cfb7fe3f --- /dev/null +++ b/egs/wenetspeech4tts/TTS/local/fbank.py @@ -0,0 +1 @@ +../../../ljspeech/TTS/matcha/fbank.py \ No newline at end of file diff --git a/egs/wenetspeech4tts/TTS/local/offline-decode-files.py b/egs/wenetspeech4tts/TTS/local/offline-decode-files.py new file mode 100755 index 000000000..fa6cbdb3e --- /dev/null +++ b/egs/wenetspeech4tts/TTS/local/offline-decode-files.py @@ -0,0 +1,495 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2023 by manyeyes +# Copyright (c) 2023 Xiaomi Corporation + +""" +This file demonstrates how to use sherpa-onnx Python API to transcribe +file(s) with a non-streaming model. + +(1) For paraformer + + ./python-api-examples/offline-decode-files.py \ + --tokens=/path/to/tokens.txt \ + --paraformer=/path/to/paraformer.onnx \ + --num-threads=2 \ + --decoding-method=greedy_search \ + --debug=false \ + --sample-rate=16000 \ + --feature-dim=80 \ + /path/to/0.wav \ + /path/to/1.wav + +(2) For transducer models from icefall + + ./python-api-examples/offline-decode-files.py \ + --tokens=/path/to/tokens.txt \ + --encoder=/path/to/encoder.onnx \ + --decoder=/path/to/decoder.onnx \ + --joiner=/path/to/joiner.onnx \ + --num-threads=2 \ + --decoding-method=greedy_search \ + --debug=false \ + --sample-rate=16000 \ + --feature-dim=80 \ + /path/to/0.wav \ + /path/to/1.wav + +(3) For CTC models from NeMo + +python3 ./python-api-examples/offline-decode-files.py \ + --tokens=./sherpa-onnx-nemo-ctc-en-citrinet-512/tokens.txt \ + --nemo-ctc=./sherpa-onnx-nemo-ctc-en-citrinet-512/model.onnx \ + --num-threads=2 \ + --decoding-method=greedy_search \ + --debug=false \ + ./sherpa-onnx-nemo-ctc-en-citrinet-512/test_wavs/0.wav \ + ./sherpa-onnx-nemo-ctc-en-citrinet-512/test_wavs/1.wav \ + ./sherpa-onnx-nemo-ctc-en-citrinet-512/test_wavs/8k.wav + +(4) For Whisper models + +python3 ./python-api-examples/offline-decode-files.py \ + --whisper-encoder=./sherpa-onnx-whisper-base.en/base.en-encoder.int8.onnx \ + --whisper-decoder=./sherpa-onnx-whisper-base.en/base.en-decoder.int8.onnx \ + --tokens=./sherpa-onnx-whisper-base.en/base.en-tokens.txt \ + --whisper-task=transcribe \ + --num-threads=1 \ + ./sherpa-onnx-whisper-base.en/test_wavs/0.wav \ + ./sherpa-onnx-whisper-base.en/test_wavs/1.wav \ + ./sherpa-onnx-whisper-base.en/test_wavs/8k.wav + +(5) For CTC models from WeNet + +python3 ./python-api-examples/offline-decode-files.py \ + --wenet-ctc=./sherpa-onnx-zh-wenet-wenetspeech/model.onnx \ + --tokens=./sherpa-onnx-zh-wenet-wenetspeech/tokens.txt \ + ./sherpa-onnx-zh-wenet-wenetspeech/test_wavs/0.wav \ + ./sherpa-onnx-zh-wenet-wenetspeech/test_wavs/1.wav \ + ./sherpa-onnx-zh-wenet-wenetspeech/test_wavs/8k.wav + +(6) For tdnn models of the yesno recipe from icefall + +python3 ./python-api-examples/offline-decode-files.py \ + --sample-rate=8000 \ + --feature-dim=23 \ + --tdnn-model=./sherpa-onnx-tdnn-yesno/model-epoch-14-avg-2.onnx \ + --tokens=./sherpa-onnx-tdnn-yesno/tokens.txt \ + ./sherpa-onnx-tdnn-yesno/test_wavs/0_0_0_1_0_0_0_1.wav \ + ./sherpa-onnx-tdnn-yesno/test_wavs/0_0_1_0_0_0_1_0.wav \ + ./sherpa-onnx-tdnn-yesno/test_wavs/0_0_1_0_0_1_1_1.wav + +Please refer to +https://k2-fsa.github.io/sherpa/onnx/index.html +to install sherpa-onnx and to download non-streaming pre-trained models +used in this file. +""" +import argparse +import time +import wave +from pathlib import Path +from typing import List, Tuple + +import numpy as np +import sherpa_onnx +import soundfile as sf + + +def get_args(): + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + + parser.add_argument( + "--tokens", + type=str, + help="Path to tokens.txt", + ) + + parser.add_argument( + "--hotwords-file", + type=str, + default="", + help=""" + The file containing hotwords, one words/phrases per line, like + HELLO WORLD + 你好世界 + """, + ) + + parser.add_argument( + "--hotwords-score", + type=float, + default=1.5, + help=""" + The hotword score of each token for biasing word/phrase. Used only if + --hotwords-file is given. + """, + ) + + parser.add_argument( + "--modeling-unit", + type=str, + default="", + help=""" + The modeling unit of the model, valid values are cjkchar, bpe, cjkchar+bpe. + Used only when hotwords-file is given. + """, + ) + + parser.add_argument( + "--bpe-vocab", + type=str, + default="", + help=""" + The path to the bpe vocabulary, the bpe vocabulary is generated by + sentencepiece, you can also export the bpe vocabulary through a bpe model + by `scripts/export_bpe_vocab.py`. Used only when hotwords-file is given + and modeling-unit is bpe or cjkchar+bpe. + """, + ) + + parser.add_argument( + "--encoder", + default="", + type=str, + help="Path to the encoder model", + ) + + parser.add_argument( + "--decoder", + default="", + type=str, + help="Path to the decoder model", + ) + + parser.add_argument( + "--joiner", + default="", + type=str, + help="Path to the joiner model", + ) + + parser.add_argument( + "--paraformer", + default="", + type=str, + help="Path to the model.onnx from Paraformer", + ) + + parser.add_argument( + "--nemo-ctc", + default="", + type=str, + help="Path to the model.onnx from NeMo CTC", + ) + + parser.add_argument( + "--wenet-ctc", + default="", + type=str, + help="Path to the model.onnx from WeNet CTC", + ) + + parser.add_argument( + "--tdnn-model", + default="", + type=str, + help="Path to the model.onnx for the tdnn model of the yesno recipe", + ) + + parser.add_argument( + "--num-threads", + type=int, + default=1, + help="Number of threads for neural network computation", + ) + + parser.add_argument( + "--whisper-encoder", + default="", + type=str, + help="Path to whisper encoder model", + ) + + parser.add_argument( + "--whisper-decoder", + default="", + type=str, + help="Path to whisper decoder model", + ) + + parser.add_argument( + "--whisper-language", + default="", + type=str, + help="""It specifies the spoken language in the input audio file. + Example values: en, fr, de, zh, jp. + Available languages for multilingual models can be found at + https://github.com/openai/whisper/blob/main/whisper/tokenizer.py#L10 + If not specified, we infer the language from the input audio file. + """, + ) + + parser.add_argument( + "--whisper-task", + default="transcribe", + choices=["transcribe", "translate"], + type=str, + help="""For multilingual models, if you specify translate, the output + will be in English. + """, + ) + + parser.add_argument( + "--whisper-tail-paddings", + default=-1, + type=int, + help="""Number of tail padding frames. + We have removed the 30-second constraint from whisper, so you need to + choose the amount of tail padding frames by yourself. + Use -1 to use a default value for tail padding. + """, + ) + + parser.add_argument( + "--blank-penalty", + type=float, + default=0.0, + help=""" + The penalty applied on blank symbol during decoding. + Note: It is a positive value that would be applied to logits like + this `logits[:, 0] -= blank_penalty` (suppose logits.shape is + [batch_size, vocab] and blank id is 0). + """, + ) + + parser.add_argument( + "--decoding-method", + type=str, + default="greedy_search", + help="Valid values are greedy_search and modified_beam_search", + ) + parser.add_argument( + "--debug", + type=bool, + default=False, + help="True to show debug messages", + ) + + parser.add_argument( + "--sample-rate", + type=int, + default=16000, + help="""Sample rate of the feature extractor. Must match the one + expected by the model. Note: The input sound files can have a + different sample rate from this argument.""", + ) + + parser.add_argument( + "--feature-dim", + type=int, + default=80, + help="Feature dimension. Must match the one expected by the model", + ) + + parser.add_argument( + "sound_files", + type=str, + nargs="+", + help="The input sound file(s) to decode. Each file must be of WAVE" + "format with a single channel, and each sample has 16-bit, " + "i.e., int16_t. " + "The sample rate of the file can be arbitrary and does not need to " + "be 16 kHz", + ) + + parser.add_argument( + "--name", + type=str, + default="", + help="The directory containing the input sound files to decode", + ) + + parser.add_argument( + "--log-dir", + type=str, + default="", + help="The directory containing the input sound files to decode", + ) + + parser.add_argument( + "--label", + type=str, + default=None, + help="wav_base_name label", + ) + return parser.parse_args() + + +def assert_file_exists(filename: str): + assert Path(filename).is_file(), ( + f"{filename} does not exist!\n" + "Please refer to " + "https://k2-fsa.github.io/sherpa/onnx/pretrained_models/index.html to download it" + ) + + +def read_wave(wave_filename: str) -> Tuple[np.ndarray, int]: + """ + Args: + wave_filename: + Path to a wave file. It should be single channel and can be of type + 32-bit floating point PCM. Its sample rate does not need to be 24kHz. + + Returns: + Return a tuple containing: + - A 1-D array of dtype np.float32 containing the samples, + which are normalized to the range [-1, 1]. + - Sample rate of the wave file. + """ + + samples, sample_rate = sf.read(wave_filename, dtype="float32") + assert ( + samples.ndim == 1 + ), f"Expected single channel, but got {samples.ndim} channels." + + samples_float32 = samples.astype(np.float32) + + return samples_float32, sample_rate + + +def normalize_text_alimeeting(text: str) -> str: + """ + Text normalization similar to M2MeT challenge baseline. + See: https://github.com/yufan-aslp/AliMeeting/blob/main/asr/local/text_normalize.pl + """ + import re + + text = text.replace(" ", "") + text = text.replace("", "") + text = text.replace("<%>", "") + text = text.replace("<->", "") + text = text.replace("<$>", "") + text = text.replace("<#>", "") + text = text.replace("<_>", "") + text = text.replace("", "") + text = text.replace("`", "") + text = text.replace("&", "") + text = text.replace(",", "") + if re.search("[a-zA-Z]", text): + text = text.upper() + text = text.replace("Ａ", "A") + text = text.replace("ａ", "A") + text = text.replace("ｂ", "B") + text = text.replace("ｃ", "C") + text = text.replace("ｋ", "K") + text = text.replace("ｔ", "T") + text = text.replace("，", "") + text = text.replace("丶", "") + text = text.replace("。", "") + text = text.replace("、", "") + text = text.replace("？", "") + return text + + +def main(): + args = get_args() + assert_file_exists(args.tokens) + assert args.num_threads > 0, args.num_threads + + assert len(args.nemo_ctc) == 0, args.nemo_ctc + assert len(args.wenet_ctc) == 0, args.wenet_ctc + assert len(args.whisper_encoder) == 0, args.whisper_encoder + assert len(args.whisper_decoder) == 0, args.whisper_decoder + assert len(args.tdnn_model) == 0, args.tdnn_model + + assert_file_exists(args.paraformer) + + recognizer = sherpa_onnx.OfflineRecognizer.from_paraformer( + paraformer=args.paraformer, + tokens=args.tokens, + num_threads=args.num_threads, + sample_rate=args.sample_rate, + feature_dim=args.feature_dim, + decoding_method=args.decoding_method, + debug=args.debug, + ) + + print("Started!") + start_time = time.time() + + streams, results = [], [] + total_duration = 0 + + for i, wave_filename in enumerate(args.sound_files): + assert_file_exists(wave_filename) + samples, sample_rate = read_wave(wave_filename) + duration = len(samples) / sample_rate + total_duration += duration + s = recognizer.create_stream() + s.accept_waveform(sample_rate, samples) + + streams.append(s) + if i % 10 == 0: + recognizer.decode_streams(streams) + results += [s.result.text for s in streams] + streams = [] + print(f"Processed {i} files") + # process the last batch + if streams: + recognizer.decode_streams(streams) + results += [s.result.text for s in streams] + end_time = time.time() + print("Done!") + + results_dict = {} + for wave_filename, result in zip(args.sound_files, results): + print(f"{wave_filename}\n{result}") + print("-" * 10) + wave_basename = Path(wave_filename).stem + results_dict[wave_basename] = result + + elapsed_seconds = end_time - start_time + rtf = elapsed_seconds / total_duration + print(f"num_threads: {args.num_threads}") + print(f"decoding_method: {args.decoding_method}") + print(f"Wave duration: {total_duration:.3f} s") + print(f"Elapsed time: {elapsed_seconds:.3f} s") + print( + f"Real time factor (RTF): {elapsed_seconds:.3f}/{total_duration:.3f} = {rtf:.3f}" + ) + if args.label: + from icefall.utils import store_transcripts, write_error_stats + + labels_dict = {} + with open(args.label, "r") as f: + for line in f: + # fields = line.strip().split(" ") + # fields = [item for item in fields if item] + # assert len(fields) == 4 + # prompt_text, prompt_audio, text, audio_path = fields + + fields = line.strip().split("|") + fields = [item for item in fields if item] + assert len(fields) == 4 + audio_path, prompt_text, prompt_audio, text = fields + labels_dict[Path(audio_path).stem] = normalize_text_alimeeting(text) + + final_results = [] + for key, value in results_dict.items(): + final_results.append((key, labels_dict[key], value)) + + store_transcripts( + filename=f"{args.log_dir}/recogs-{args.name}.txt", texts=final_results + ) + with open(f"{args.log_dir}/errs-{args.name}.txt", "w") as f: + write_error_stats(f, "test-set", final_results, enable_log=True) + + with open(f"{args.log_dir}/errs-{args.name}.txt", "r") as f: + print(f.readline()) # WER + print(f.readline()) # Detailed errors + + +if __name__ == "__main__": + main() diff --git a/egs/wenetspeech4tts/TTS/prepare.sh b/egs/wenetspeech4tts/TTS/prepare.sh index 3d7ffadb1..cf86b3fa5 100755 --- a/egs/wenetspeech4tts/TTS/prepare.sh +++ b/egs/wenetspeech4tts/TTS/prepare.sh @@ -98,3 +98,44 @@ if [ $stage -le 4 ] && [ $stop_stage -ge 4 ]; then fi python3 ./local/display_manifest_statistics.py --manifest-dir ${audio_feats_dir} fi + +subset="Basic" +prefix="wenetspeech4tts" +if [ $stage -le 5 ] && [ $stop_stage -ge 5 ]; then + log "Stage 5: Generate fbank (used by ./f5-tts)" + mkdir -p data/fbank + if [ ! -e data/fbank/.${prefix}.done ]; then + ./local/compute_mel_feat.py --dataset-parts $subset --split 100 + touch data/fbank/.${prefix}.done + fi +fi + +if [ $stage -le 6 ] && [ $stop_stage -ge 6 ]; then + log "Stage 7: Split the ${prefix} cuts into train, valid and test sets (used by ./f5-tts)" + if [ ! -f data/fbank/${prefix}_cuts_${subset}.jsonl.gz ]; then + echo "Combining ${prefix} cuts" + pieces=$(find data/fbank/ -name "${prefix}_cuts_${subset}.*.jsonl.gz") + lhotse combine $pieces data/fbank/${prefix}_cuts_${subset}.jsonl.gz + fi + if [ ! -e data/fbank/.${prefix}_split.done ]; then + echo "Splitting ${prefix} cuts into train, valid and test sets" + + lhotse subset --last 800 \ + data/fbank/${prefix}_cuts_${subset}.jsonl.gz \ + data/fbank/${prefix}_cuts_validtest.jsonl.gz + lhotse subset --first 400 \ + data/fbank/${prefix}_cuts_validtest.jsonl.gz \ + data/fbank/${prefix}_cuts_valid.jsonl.gz + lhotse subset --last 400 \ + data/fbank/${prefix}_cuts_validtest.jsonl.gz \ + data/fbank/${prefix}_cuts_test.jsonl.gz + + rm data/fbank/${prefix}_cuts_validtest.jsonl.gz + + n=$(( $(gunzip -c data/fbank/${prefix}_cuts_${subset}.jsonl.gz | wc -l) - 800 )) + lhotse subset --first $n \ + data/fbank/${prefix}_cuts_${subset}.jsonl.gz \ + data/fbank/${prefix}_cuts_train.jsonl.gz + touch data/fbank/.${prefix}_split.done + fi +fi From 0855b0338aa8c191ae8f5b401188aa0877275b14 Mon Sep 17 00:00:00 2001 From: Machiko Bailey <53164945+baileyeet@users.noreply.github.com> Date: Mon, 3 Feb 2025 12:33:09 -0500 Subject: [PATCH 30/59] Merge japanese-to-english multilingual branch (#1860) * add streaming support to reazonresearch * update README for streaming * Update RESULTS.md * add onnx decode --------- Co-authored-by: root Co-authored-by: Fangjun Kuang Co-authored-by: root Co-authored-by: zr_jin --- .github/workflows/style_check.yml | 2 +- egs/multi_ja_en/ASR/README.md | 17 + egs/multi_ja_en/ASR/RESULTS.md | 53 + .../ASR/local/compute_fbank_reazonspeech.py | 146 ++ .../ASR/local/display_manifest_statistics.py | 58 + egs/multi_ja_en/ASR/local/prepare_char.py | 1 + .../ASR/local/prepare_for_bpe_model.py | 66 + egs/multi_ja_en/ASR/local/prepare_lang.py | 1 + .../ASR/local/prepare_lang_bbpe.py | 268 +++ .../ASR/local/prepare_lang_char.py | 75 + egs/multi_ja_en/ASR/local/prepare_words.py | 1 + egs/multi_ja_en/ASR/local/text2segments.py | 95 ++ egs/multi_ja_en/ASR/local/text2token.py | 177 ++ egs/multi_ja_en/ASR/local/train_bbpe_model.py | 114 ++ .../ASR/local/utils/asr_datamodule.py | 355 ++++ egs/multi_ja_en/ASR/local/utils/tokenizer.py | 252 +++ .../ASR/local/validate_bpe_lexicon.py | 1 + .../ASR/local/validate_manifest.py | 96 ++ egs/multi_ja_en/ASR/prepare.sh | 185 +++ egs/multi_ja_en/ASR/shared | 1 + .../ASR/zipformer/asr_datamodule.py | 1 + egs/multi_ja_en/ASR/zipformer/beam_search.py | 1 + egs/multi_ja_en/ASR/zipformer/ctc_decode.py | 1 + egs/multi_ja_en/ASR/zipformer/decode.py | 792 +++++++++ .../ASR/zipformer/decode_stream.py | 1 + egs/multi_ja_en/ASR/zipformer/decoder.py | 1 + .../ASR/zipformer/do_not_use_it_directly.py | 1261 ++++++++++++++ .../ASR/zipformer/encoder_interface.py | 1 + egs/multi_ja_en/ASR/zipformer/export-onnx.py | 1 + egs/multi_ja_en/ASR/zipformer/export.py | 1 + .../ASR/zipformer/generate_averaged_model.py | 1 + egs/multi_ja_en/ASR/zipformer/joiner.py | 1 + egs/multi_ja_en/ASR/zipformer/model.py | 1 + .../ASR/zipformer/multi_dataset.py | 143 ++ egs/multi_ja_en/ASR/zipformer/my_profile.py | 1 + egs/multi_ja_en/ASR/zipformer/onnx_decode.py | 1 + .../ASR/zipformer/onnx_pretrained.py | 1 + egs/multi_ja_en/ASR/zipformer/optim.py | 1 + egs/multi_ja_en/ASR/zipformer/pretrained.py | 1 + egs/multi_ja_en/ASR/zipformer/scaling.py | 1 + .../ASR/zipformer/scaling_converter.py | 1 + .../ASR/zipformer/streaming_beam_search.py | 1 + .../ASR/zipformer/streaming_decode.py | 935 +++++++++++ egs/multi_ja_en/ASR/zipformer/subsampling.py | 1 + egs/multi_ja_en/ASR/zipformer/test_scaling.py | 1 + .../ASR/zipformer/test_subsampling.py | 1 + egs/multi_ja_en/ASR/zipformer/tokenizer.py | 1 + egs/multi_ja_en/ASR/zipformer/train.py | 1462 +++++++++++++++++ egs/multi_ja_en/ASR/zipformer/zipformer.py | 1 + egs/reazonspeech/ASR/RESULTS.md | 38 + egs/reazonspeech/ASR/local/utils/tokenizer.py | 1 - .../ASR/zipformer/streaming_decode.py | 507 ++++-- icefall/__init__.py | 1 + icefall/utils.py | 24 + 54 files changed, 7048 insertions(+), 104 deletions(-) create mode 100644 egs/multi_ja_en/ASR/README.md create mode 100644 egs/multi_ja_en/ASR/RESULTS.md create mode 100644 egs/multi_ja_en/ASR/local/compute_fbank_reazonspeech.py create mode 100644 egs/multi_ja_en/ASR/local/display_manifest_statistics.py create mode 120000 egs/multi_ja_en/ASR/local/prepare_char.py create mode 100755 egs/multi_ja_en/ASR/local/prepare_for_bpe_model.py create mode 120000 egs/multi_ja_en/ASR/local/prepare_lang.py create mode 100755 egs/multi_ja_en/ASR/local/prepare_lang_bbpe.py create mode 100644 egs/multi_ja_en/ASR/local/prepare_lang_char.py create mode 120000 egs/multi_ja_en/ASR/local/prepare_words.py create mode 100644 egs/multi_ja_en/ASR/local/text2segments.py create mode 100755 egs/multi_ja_en/ASR/local/text2token.py create mode 100755 egs/multi_ja_en/ASR/local/train_bbpe_model.py create mode 100644 egs/multi_ja_en/ASR/local/utils/asr_datamodule.py create mode 100644 egs/multi_ja_en/ASR/local/utils/tokenizer.py create mode 120000 egs/multi_ja_en/ASR/local/validate_bpe_lexicon.py create mode 100644 egs/multi_ja_en/ASR/local/validate_manifest.py create mode 100755 egs/multi_ja_en/ASR/prepare.sh create mode 120000 egs/multi_ja_en/ASR/shared create mode 120000 egs/multi_ja_en/ASR/zipformer/asr_datamodule.py create mode 120000 egs/multi_ja_en/ASR/zipformer/beam_search.py create mode 120000 egs/multi_ja_en/ASR/zipformer/ctc_decode.py create mode 100755 egs/multi_ja_en/ASR/zipformer/decode.py create mode 120000 egs/multi_ja_en/ASR/zipformer/decode_stream.py create mode 120000 egs/multi_ja_en/ASR/zipformer/decoder.py create mode 100755 egs/multi_ja_en/ASR/zipformer/do_not_use_it_directly.py create mode 120000 egs/multi_ja_en/ASR/zipformer/encoder_interface.py create mode 120000 egs/multi_ja_en/ASR/zipformer/export-onnx.py create mode 120000 egs/multi_ja_en/ASR/zipformer/export.py create mode 120000 egs/multi_ja_en/ASR/zipformer/generate_averaged_model.py create mode 120000 egs/multi_ja_en/ASR/zipformer/joiner.py create mode 120000 egs/multi_ja_en/ASR/zipformer/model.py create mode 100644 egs/multi_ja_en/ASR/zipformer/multi_dataset.py create mode 120000 egs/multi_ja_en/ASR/zipformer/my_profile.py create mode 120000 egs/multi_ja_en/ASR/zipformer/onnx_decode.py create mode 120000 egs/multi_ja_en/ASR/zipformer/onnx_pretrained.py create mode 120000 egs/multi_ja_en/ASR/zipformer/optim.py create mode 120000 egs/multi_ja_en/ASR/zipformer/pretrained.py create mode 120000 egs/multi_ja_en/ASR/zipformer/scaling.py create mode 120000 egs/multi_ja_en/ASR/zipformer/scaling_converter.py create mode 120000 egs/multi_ja_en/ASR/zipformer/streaming_beam_search.py create mode 100755 egs/multi_ja_en/ASR/zipformer/streaming_decode.py create mode 120000 egs/multi_ja_en/ASR/zipformer/subsampling.py create mode 120000 egs/multi_ja_en/ASR/zipformer/test_scaling.py create mode 120000 egs/multi_ja_en/ASR/zipformer/test_subsampling.py create mode 120000 egs/multi_ja_en/ASR/zipformer/tokenizer.py create mode 100755 egs/multi_ja_en/ASR/zipformer/train.py create mode 120000 egs/multi_ja_en/ASR/zipformer/zipformer.py diff --git a/.github/workflows/style_check.yml b/.github/workflows/style_check.yml index 3c971e231..908c3cc43 100644 --- a/.github/workflows/style_check.yml +++ b/.github/workflows/style_check.yml @@ -69,7 +69,7 @@ jobs: working-directory: ${{github.workspace}} run: | black --check --diff . - + - name: Run isort shell: bash working-directory: ${{github.workspace}} diff --git a/egs/multi_ja_en/ASR/README.md b/egs/multi_ja_en/ASR/README.md new file mode 100644 index 000000000..09964a4ab --- /dev/null +++ b/egs/multi_ja_en/ASR/README.md @@ -0,0 +1,17 @@ +# Introduction + +A bilingual Japanese-English ASR model that utilizes ReazonSpeech, developed by the developers of ReazonSpeech. + +**ReazonSpeech** is an open-source dataset that contains a diverse set of natural Japanese speech, collected from terrestrial television streams. It contains more than 35,000 hours of audio. + + +# Included Training Sets + +1. LibriSpeech (English) +2. ReazonSpeech (Japanese) + +|Datset| Number of hours| URL| +|---|---:|---| +|**TOTAL**|35,960|---| +|LibriSpeech|960|https://www.openslr.org/12/| +|ReazonSpeech (all) |35,000|https://huggingface.co/datasets/reazon-research/reazonspeech| diff --git a/egs/multi_ja_en/ASR/RESULTS.md b/egs/multi_ja_en/ASR/RESULTS.md new file mode 100644 index 000000000..c1bbb9ac5 --- /dev/null +++ b/egs/multi_ja_en/ASR/RESULTS.md @@ -0,0 +1,53 @@ +## Results + +### Zipformer + +#### Non-streaming + +The training command is: + +```shell +./zipformer/train.py \ + --bilingual 1 \ + --world-size 4 \ + --num-epochs 30 \ + --start-epoch 1 \ + --use-fp16 1 \ + --exp-dir zipformer/exp \ + --max-duration 600 +``` + +The decoding command is: + +```shell +./zipformer/decode.py \ + --epoch 28 \ + --avg 15 \ + --exp-dir ./zipformer/exp \ + --max-duration 600 \ + --decoding-method greedy_search +``` + +To export the model with onnx: + +```shell +./zipformer/export-onnx.py --tokens data/lang_bbpe_2000/tokens.txt --use-averaged-model 0 --epoch 35 --avg 1 --exp-dir zipformer/exp --num-encoder-layers "2,2,3,4,3,2" --downsampling-factor "1,2,4,8,4,2" --feedforward-dim "512,768,1024,1536,1024,768" --num-heads "4,4,4,8,4,4" --encoder-dim "192,256,384,512,384,256" --query-head-dim 32 --value-head-dim 12 --pos-head-dim 4 --pos-dim 48 --encoder-unmasked-dim "192,192,256,256,256,192" --cnn-module-kernel "31,31,15,15,15,31" --decoder-dim 512 --joiner-dim 512 --causal False --chunk-size "16,32,64,-1" --left-context-frames "64,128,256,-1" --fp16 True +``` +Word Error Rates (WERs) listed below: + +| Datasets | ReazonSpeech | ReazonSpeech | LibriSpeech | LibriSpeech | +|----------------------|--------------|---------------|--------------------|-------------------| +| Zipformer WER (%) | dev | test | test-clean | test-other | +| greedy_search | 5.9 | 4.07 | 3.46 | 8.35 | +| modified_beam_search | 4.87 | 3.61 | 3.28 | 8.07 | +| fast_beam_search | 41.04 | 36.59 | 16.14 | 22.0 | + + +Character Error Rates (CERs) for Japanese listed below: +| Decoding Method | In-Distribution CER | JSUT | CommonVoice | TEDx | +| :------------------: | :-----------------: | :--: | :---------: | :---: | +| greedy search | 12.56 | 6.93 | 9.75 | 9.67 | +| modified beam search | 11.59 | 6.97 | 9.55 | 9.51 | + +Pre-trained model can be found here: https://huggingface.co/reazon-research/reazonspeech-k2-v2-ja-en/tree/main + diff --git a/egs/multi_ja_en/ASR/local/compute_fbank_reazonspeech.py b/egs/multi_ja_en/ASR/local/compute_fbank_reazonspeech.py new file mode 100644 index 000000000..af7841406 --- /dev/null +++ b/egs/multi_ja_en/ASR/local/compute_fbank_reazonspeech.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 +# Copyright 2023 The University of Electro-Communications (Author: Teo Wen Shen) # noqa +# +# See ../../../../LICENSE for clarification regarding multiple authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import argparse +import logging +import os +from pathlib import Path +from typing import List, Tuple + +import torch + +# fmt: off +from lhotse import ( # See the following for why LilcomChunkyWriter is preferred; https://github.com/k2-fsa/icefall/pull/404; https://github.com/lhotse-speech/lhotse/pull/527 + CutSet, + Fbank, + FbankConfig, + LilcomChunkyWriter, + RecordingSet, + SupervisionSet, +) + +# fmt: on + +# Torch's multithreaded behavior needs to be disabled or +# it wastes a lot of CPU and slow things down. +# Do this outside of main() in case it needs to take effect +# even when we are not invoking the main (e.g. when spawning subprocesses). +torch.set_num_threads(1) +torch.set_num_interop_threads(1) + +RNG_SEED = 42 +concat_params = {"gap": 1.0, "maxlen": 10.0} + + +def make_cutset_blueprints( + manifest_dir: Path, +) -> List[Tuple[str, CutSet]]: + cut_sets = [] + + # Create test dataset + logging.info("Creating test cuts.") + cut_sets.append( + ( + "test", + CutSet.from_manifests( + recordings=RecordingSet.from_file( + manifest_dir / "reazonspeech_recordings_test.jsonl.gz" + ), + supervisions=SupervisionSet.from_file( + manifest_dir / "reazonspeech_supervisions_test.jsonl.gz" + ), + ), + ) + ) + + # Create dev dataset + logging.info("Creating dev cuts.") + cut_sets.append( + ( + "dev", + CutSet.from_manifests( + recordings=RecordingSet.from_file( + manifest_dir / "reazonspeech_recordings_dev.jsonl.gz" + ), + supervisions=SupervisionSet.from_file( + manifest_dir / "reazonspeech_supervisions_dev.jsonl.gz" + ), + ), + ) + ) + + # Create train dataset + logging.info("Creating train cuts.") + cut_sets.append( + ( + "train", + CutSet.from_manifests( + recordings=RecordingSet.from_file( + manifest_dir / "reazonspeech_recordings_train.jsonl.gz" + ), + supervisions=SupervisionSet.from_file( + manifest_dir / "reazonspeech_supervisions_train.jsonl.gz" + ), + ), + ) + ) + return cut_sets + + +def get_args(): + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument("-m", "--manifest-dir", type=Path) + return parser.parse_args() + + +def main(): + args = get_args() + + extractor = Fbank(FbankConfig(num_mel_bins=80)) + num_jobs = min(16, os.cpu_count()) + + formatter = "%(asctime)s %(levelname)s [%(filename)s:%(lineno)d] %(message)s" + + logging.basicConfig(format=formatter, level=logging.INFO) + + if (args.manifest_dir / ".reazonspeech-fbank.done").exists(): + logging.info( + "Previous fbank computed for ReazonSpeech found. " + f"Delete {args.manifest_dir / '.reazonspeech-fbank.done'} to allow recomputing fbank." + ) + return + else: + cut_sets = make_cutset_blueprints(args.manifest_dir) + for part, cut_set in cut_sets: + logging.info(f"Processing {part}") + cut_set = cut_set.compute_and_store_features( + extractor=extractor, + num_jobs=num_jobs, + storage_path=(args.manifest_dir / f"feats_{part}").as_posix(), + storage_type=LilcomChunkyWriter, + ) + cut_set.to_file(args.manifest_dir / f"reazonspeech_cuts_{part}.jsonl.gz") + + logging.info("All fbank computed for ReazonSpeech.") + (args.manifest_dir / ".reazonspeech-fbank.done").touch() + + +if __name__ == "__main__": + main() diff --git a/egs/multi_ja_en/ASR/local/display_manifest_statistics.py b/egs/multi_ja_en/ASR/local/display_manifest_statistics.py new file mode 100644 index 000000000..ace1dd73f --- /dev/null +++ b/egs/multi_ja_en/ASR/local/display_manifest_statistics.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +# Copyright 2021 Xiaomi Corp. (authors: Fangjun Kuang) +# 2022 The University of Electro-Communications (author: Teo Wen Shen) # noqa +# +# See ../../../../LICENSE for clarification regarding multiple authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +from pathlib import Path + +from lhotse import CutSet, load_manifest + +ARGPARSE_DESCRIPTION = """ +This file displays duration statistics of utterances in a manifest. +You can use the displayed value to choose minimum/maximum duration +to remove short and long utterances during the training. + +See the function `remove_short_and_long_utt()` in +pruned_transducer_stateless5/train.py for usage. +""" + + +def get_parser(): + parser = argparse.ArgumentParser( + description=ARGPARSE_DESCRIPTION, + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + + parser.add_argument("--manifest-dir", type=Path, help="Path to cutset manifests") + + return parser.parse_args() + + +def main(): + args = get_parser() + + for part in ["train", "dev"]: + path = args.manifest_dir / f"reazonspeech_cuts_{part}.jsonl.gz" + cuts: CutSet = load_manifest(path) + + print("\n---------------------------------\n") + print(path.name + ":") + cuts.describe() + + +if __name__ == "__main__": + main() diff --git a/egs/multi_ja_en/ASR/local/prepare_char.py b/egs/multi_ja_en/ASR/local/prepare_char.py new file mode 120000 index 000000000..42743b544 --- /dev/null +++ b/egs/multi_ja_en/ASR/local/prepare_char.py @@ -0,0 +1 @@ +../../../aishell/ASR/local/prepare_char.py \ No newline at end of file diff --git a/egs/multi_ja_en/ASR/local/prepare_for_bpe_model.py b/egs/multi_ja_en/ASR/local/prepare_for_bpe_model.py new file mode 100755 index 000000000..27832ad1b --- /dev/null +++ b/egs/multi_ja_en/ASR/local/prepare_for_bpe_model.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +# Copyright 2023 Xiaomi Corp. (authors: Zengrui Jin) +# +# See ../../../../LICENSE for clarification regarding multiple authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script tokenizes the training transcript by CJK characters +# and saves the result to transcript_chars.txt, which is used +# to train the BPE model later. + +import argparse +import re +from pathlib import Path + +from tqdm.auto import tqdm + +from icefall.utils import tokenize_by_ja_char + + +def get_args(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--lang-dir", + type=str, + help="""Output directory. + The generated transcript_chars.txt is saved to this directory. + """, + ) + + parser.add_argument( + "--text", + type=str, + help="Training transcript.", + ) + + return parser.parse_args() + + +def main(): + args = get_args() + lang_dir = Path(args.lang_dir) + text = Path(args.text) + + assert lang_dir.exists() and text.exists(), f"{lang_dir} or {text} does not exist!" + + transcript_path = lang_dir / "transcript_chars.txt" + + with open(text, "r", encoding="utf-8") as fin: + with open(transcript_path, "w+", encoding="utf-8") as fout: + for line in tqdm(fin): + fout.write(tokenize_by_ja_char(line) + "\n") + + +if __name__ == "__main__": + main() diff --git a/egs/multi_ja_en/ASR/local/prepare_lang.py b/egs/multi_ja_en/ASR/local/prepare_lang.py new file mode 120000 index 000000000..747f2ab39 --- /dev/null +++ b/egs/multi_ja_en/ASR/local/prepare_lang.py @@ -0,0 +1 @@ +../../../librispeech/ASR/local/prepare_lang.py \ No newline at end of file diff --git a/egs/multi_ja_en/ASR/local/prepare_lang_bbpe.py b/egs/multi_ja_en/ASR/local/prepare_lang_bbpe.py new file mode 100755 index 000000000..6134710ad --- /dev/null +++ b/egs/multi_ja_en/ASR/local/prepare_lang_bbpe.py @@ -0,0 +1,268 @@ +#!/usr/bin/env python3 +# Copyright 2021 Xiaomi Corp. (authors: Fangjun Kuang +# Wei Kang) +# +# See ../../../../LICENSE for clarification regarding multiple authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +""" + +This script takes as input `lang_dir`, which should contain:: + + - lang_dir/bbpe.model, + - lang_dir/words.txt + +and generates the following files in the directory `lang_dir`: + + - lexicon.txt + - lexicon_disambig.txt + - L.pt + - L_disambig.pt + - tokens.txt +""" + +import argparse +import re +from pathlib import Path +from typing import Dict, List, Tuple + +import k2 +import sentencepiece as spm +import torch +from prepare_lang import ( + Lexicon, + add_disambig_symbols, + add_self_loops, + write_lexicon, + write_mapping, +) + +from icefall.byte_utils import byte_encode +from icefall.utils import str2bool, tokenize_by_ja_char + + +def lexicon_to_fst_no_sil( + lexicon: Lexicon, + token2id: Dict[str, int], + word2id: Dict[str, int], + need_self_loops: bool = False, +) -> k2.Fsa: + """Convert a lexicon to an FST (in k2 format). + + Args: + lexicon: + The input lexicon. See also :func:`read_lexicon` + token2id: + A dict mapping tokens to IDs. + word2id: + A dict mapping words to IDs. + need_self_loops: + If True, add self-loop to states with non-epsilon output symbols + on at least one arc out of the state. The input label for this + self loop is `token2id["#0"]` and the output label is `word2id["#0"]`. + Returns: + Return an instance of `k2.Fsa` representing the given lexicon. + """ + loop_state = 0 # words enter and leave from here + next_state = 1 # the next un-allocated state, will be incremented as we go + + arcs = [] + + # The blank symbol is defined in local/train_bpe_model.py + assert token2id[""] == 0 + assert word2id[""] == 0 + + eps = 0 + + for word, pieces in lexicon: + assert len(pieces) > 0, f"{word} has no pronunciations" + cur_state = loop_state + + word = word2id[word] + pieces = [token2id[i] for i in pieces] + + for i in range(len(pieces) - 1): + w = word if i == 0 else eps + arcs.append([cur_state, next_state, pieces[i], w, 0]) + + cur_state = next_state + next_state += 1 + + # now for the last piece of this word + i = len(pieces) - 1 + w = word if i == 0 else eps + arcs.append([cur_state, loop_state, pieces[i], w, 0]) + + if need_self_loops: + disambig_token = token2id["#0"] + disambig_word = word2id["#0"] + arcs = add_self_loops( + arcs, + disambig_token=disambig_token, + disambig_word=disambig_word, + ) + + final_state = next_state + arcs.append([loop_state, final_state, -1, -1, 0]) + arcs.append([final_state]) + + arcs = sorted(arcs, key=lambda arc: arc[0]) + arcs = [[str(i) for i in arc] for arc in arcs] + arcs = [" ".join(arc) for arc in arcs] + arcs = "\n".join(arcs) + + fsa = k2.Fsa.from_str(arcs, acceptor=False) + return fsa + + +def generate_lexicon( + model_file: str, words: List[str], oov: str +) -> Tuple[Lexicon, Dict[str, int]]: + """Generate a lexicon from a BPE model. + + Args: + model_file: + Path to a sentencepiece model. + words: + A list of strings representing words. + oov: + The out of vocabulary word in lexicon. + Returns: + Return a tuple with two elements: + - A dict whose keys are words and values are the corresponding + word pieces. + - A dict representing the token symbol, mapping from tokens to IDs. + """ + sp = spm.SentencePieceProcessor() + sp.load(str(model_file)) + + # Convert word to word piece IDs instead of word piece strings + # to avoid OOV tokens. + encode_words = [byte_encode(tokenize_by_ja_char(w)) for w in words] + words_pieces_ids: List[List[int]] = sp.encode(encode_words, out_type=int) + + # Now convert word piece IDs back to word piece strings. + words_pieces: List[List[str]] = [sp.id_to_piece(ids) for ids in words_pieces_ids] + + lexicon = [] + for word, pieces in zip(words, words_pieces): + lexicon.append((word, pieces)) + + lexicon.append((oov, ["▁", sp.id_to_piece(sp.unk_id())])) + + token2id: Dict[str, int] = {sp.id_to_piece(i): i for i in range(sp.vocab_size())} + + return lexicon, token2id + + +def get_args(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--lang-dir", + type=str, + help="""Input and output directory. + It should contain the bpe.model and words.txt + """, + ) + + parser.add_argument( + "--oov", + type=str, + default="", + help="The out of vocabulary word in lexicon.", + ) + + parser.add_argument( + "--debug", + type=str2bool, + default=False, + help="""True for debugging, which will generate + a visualization of the lexicon FST. + + Caution: If your lexicon contains hundreds of thousands + of lines, please set it to False! + + See "test/test_bpe_lexicon.py" for usage. + """, + ) + + return parser.parse_args() + + +def main(): + args = get_args() + lang_dir = Path(args.lang_dir) + model_file = lang_dir / "bbpe.model" + + word_sym_table = k2.SymbolTable.from_file(lang_dir / "words.txt") + + words = word_sym_table.symbols + + excluded = ["", "!SIL", "", args.oov, "#0", "", ""] + + for w in excluded: + if w in words: + words.remove(w) + + lexicon, token_sym_table = generate_lexicon(model_file, words, args.oov) + + lexicon_disambig, max_disambig = add_disambig_symbols(lexicon) + + next_token_id = max(token_sym_table.values()) + 1 + for i in range(max_disambig + 1): + disambig = f"#{i}" + assert disambig not in token_sym_table + token_sym_table[disambig] = next_token_id + next_token_id += 1 + + word_sym_table.add("#0") + word_sym_table.add("") + word_sym_table.add("") + + write_mapping(lang_dir / "tokens.txt", token_sym_table) + + write_lexicon(lang_dir / "lexicon.txt", lexicon) + write_lexicon(lang_dir / "lexicon_disambig.txt", lexicon_disambig) + + L = lexicon_to_fst_no_sil( + lexicon, + token2id=token_sym_table, + word2id=word_sym_table, + ) + + L_disambig = lexicon_to_fst_no_sil( + lexicon_disambig, + token2id=token_sym_table, + word2id=word_sym_table, + need_self_loops=True, + ) + torch.save(L.as_dict(), lang_dir / "L.pt") + torch.save(L_disambig.as_dict(), lang_dir / "L_disambig.pt") + + if args.debug: + labels_sym = k2.SymbolTable.from_file(lang_dir / "tokens.txt") + aux_labels_sym = k2.SymbolTable.from_file(lang_dir / "words.txt") + + L.labels_sym = labels_sym + L.aux_labels_sym = aux_labels_sym + L.draw(f"{lang_dir / 'L.svg'}", title="L.pt") + + L_disambig.labels_sym = labels_sym + L_disambig.aux_labels_sym = aux_labels_sym + L_disambig.draw(f"{lang_dir / 'L_disambig.svg'}", title="L_disambig.pt") + + +if __name__ == "__main__": + main() diff --git a/egs/multi_ja_en/ASR/local/prepare_lang_char.py b/egs/multi_ja_en/ASR/local/prepare_lang_char.py new file mode 100644 index 000000000..19c5f4a31 --- /dev/null +++ b/egs/multi_ja_en/ASR/local/prepare_lang_char.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +# Copyright 2022 The University of Electro-Communications (Author: Teo Wen Shen) # noqa +# +# See ../../../../LICENSE for clarification regarding multiple authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import argparse +import logging +from pathlib import Path + +from lhotse import CutSet + + +def get_args(): + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + + parser.add_argument( + "train_cut", metavar="train-cut", type=Path, help="Path to the train cut" + ) + + parser.add_argument( + "--lang-dir", + type=Path, + default=Path("data/lang_char"), + help=( + "Name of lang dir. " + "If not set, this will default to lang_char_{trans-mode}" + ), + ) + + return parser.parse_args() + + +def main(): + args = get_args() + logging.basicConfig( + format=("%(asctime)s %(levelname)s [%(filename)s:%(lineno)d] %(message)s"), + level=logging.INFO, + ) + + sysdef_string = set(["", "", "", " "]) + + token_set = set() + logging.info(f"Creating vocabulary from {args.train_cut}.") + train_cut: CutSet = CutSet.from_file(args.train_cut) + for cut in train_cut: + for sup in cut.supervisions: + token_set.update(sup.text) + + token_set = [""] + sorted(token_set - sysdef_string) + ["", ""] + args.lang_dir.mkdir(parents=True, exist_ok=True) + (args.lang_dir / "tokens.txt").write_text( + "\n".join(f"{t}\t{i}" for i, t in enumerate(token_set)) + ) + + (args.lang_dir / "lang_type").write_text("char") + logging.info("Done.") + + +if __name__ == "__main__": + main() diff --git a/egs/multi_ja_en/ASR/local/prepare_words.py b/egs/multi_ja_en/ASR/local/prepare_words.py new file mode 120000 index 000000000..ef2b4eaf3 --- /dev/null +++ b/egs/multi_ja_en/ASR/local/prepare_words.py @@ -0,0 +1 @@ +../../../aishell2/ASR/local/prepare_words.py \ No newline at end of file diff --git a/egs/multi_ja_en/ASR/local/text2segments.py b/egs/multi_ja_en/ASR/local/text2segments.py new file mode 100644 index 000000000..e0f3a15c4 --- /dev/null +++ b/egs/multi_ja_en/ASR/local/text2segments.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright 2021 Xiaomi Corp. (authors: Mingshuang Luo) +# 2022 Xiaomi Corp. (authors: Weiji Zhuang) +# +# See ../../../../LICENSE for clarification regarding multiple authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +""" +This script takes as input "text", which refers to the transcript file: + - text +and generates the output file with word segmentation implemented using MeCab: + - text_words_segmentation +""" + +import argparse +from multiprocessing import Pool + +import MeCab +from tqdm import tqdm + + +def get_parser(): + parser = argparse.ArgumentParser( + description="Japanese Word Segmentation for text", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument( + "--num-process", + "-n", + default=20, + type=int, + help="the number of processes", + ) + parser.add_argument( + "--input-file", + "-i", + default="data/lang_char/text", + type=str, + help="the input text file", + ) + parser.add_argument( + "--output-file", + "-o", + default="data/lang_char/text_words_segmentation", + type=str, + help="the text implemented with word segmentation using MeCab", + ) + + return parser + + +def cut(lines): + if lines is not None: + mecab = MeCab.Tagger("-Owakati") # Use '-Owakati' option for word segmentation + segmented_line = mecab.parse(lines).strip() + return segmented_line.split() # Return as a list of words + else: + return None + + +def main(): + parser = get_parser() + args = parser.parse_args() + + num_process = args.num_process + input_file = args.input_file + output_file = args.output_file + + with open(input_file, "r", encoding="utf-8") as fr: + lines = fr.readlines() + + with Pool(processes=num_process) as p: + new_lines = list(tqdm(p.imap(cut, lines), total=len(lines))) + + with open(output_file, "w", encoding="utf-8") as fw: + for line in new_lines: + fw.write(" ".join(line) + "\n") + + +if __name__ == "__main__": + main() diff --git a/egs/multi_ja_en/ASR/local/text2token.py b/egs/multi_ja_en/ASR/local/text2token.py new file mode 100755 index 000000000..ce64847c9 --- /dev/null +++ b/egs/multi_ja_en/ASR/local/text2token.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python3 +# Copyright 2017 Johns Hopkins University (authors: Shinji Watanabe) +# 2022 Xiaomi Corp. (authors: Mingshuang Luo) +# +# See ../../../../LICENSE for clarification regarding multiple authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import argparse +import codecs +import re +import sys +from typing import List + +from romkan import to_roma # Replace with python-romkan v0.2.1 + +is_python2 = sys.version_info[0] == 2 + + +def exist_or_not(i, match_pos): + start_pos = None + end_pos = None + for pos in match_pos: + if pos[0] <= i < pos[1]: + start_pos = pos[0] + end_pos = pos[1] + break + + return start_pos, end_pos + + +def get_parser(): + parser = argparse.ArgumentParser( + description="convert raw text to tokenized text", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument( + "--nchar", + "-n", + default=1, + type=int, + help="number of characters to split, i.e., \ + aabb -> a a b b with -n 1 and aa bb with -n 2", + ) + parser.add_argument( + "--skip-ncols", "-s", default=0, type=int, help="skip first n columns" + ) + parser.add_argument("--space", default="", type=str, help="space symbol") + parser.add_argument( + "--non-lang-syms", + "-l", + default=None, + type=str, + help="list of non-linguistic symbols, e.g., etc.", + ) + parser.add_argument("text", type=str, default=False, nargs="?", help="input text") + parser.add_argument( + "--trans_type", + "-t", + type=str, + default="char", + choices=["char", "romaji"], + help="Transcript type. char/romaji", + ) + return parser + + +def token2id( + texts, token_table, token_type: str = "romaji", oov: str = "" +) -> List[List[int]]: + """Convert token to id. + Args: + texts: + The input texts, it refers to the Japanese text here. + token_table: + The token table is built based on "data/lang_xxx/token.txt" + token_type: + The type of token, such as "romaji". + oov: + Out of vocabulary token. When a word(token) in the transcript + does not exist in the token list, it is replaced with `oov`. + + Returns: + The list of ids for the input texts. + """ + if texts is None: + raise ValueError("texts can't be None!") + else: + oov_id = token_table[oov] + ids: List[List[int]] = [] + for text in texts: + chars_list = list(str(text)) + if token_type == "romaji": + text = [to_roma(c) for c in chars_list] + sub_ids = [ + token_table[txt] if txt in token_table else oov_id for txt in text + ] + ids.append(sub_ids) + return ids + + +def main(): + parser = get_parser() + args = parser.parse_args() + + rs = [] + if args.non_lang_syms is not None: + with codecs.open(args.non_lang_syms, "r", encoding="utf-8") as f: + nls = [x.rstrip() for x in f.readlines()] + rs = [re.compile(re.escape(x)) for x in nls] + + if args.text: + f = codecs.open(args.text, encoding="utf-8") + else: + f = codecs.getreader("utf-8")(sys.stdin if is_python2 else sys.stdin.buffer) + + sys.stdout = codecs.getwriter("utf-8")( + sys.stdout if is_python2 else sys.stdout.buffer + ) + line = f.readline() + n = args.nchar + while line: + x = line.split() + print(" ".join(x[: args.skip_ncols]), end=" ") + a = " ".join(x[args.skip_ncols :]) # noqa E203 + + # get all matched positions + match_pos = [] + for r in rs: + i = 0 + while i >= 0: + m = r.search(a, i) + if m: + match_pos.append([m.start(), m.end()]) + i = m.end() + else: + break + if len(match_pos) > 0: + chars = [] + i = 0 + while i < len(a): + start_pos, end_pos = exist_or_not(i, match_pos) + if start_pos is not None: + chars.append(a[start_pos:end_pos]) + i = end_pos + else: + chars.append(a[i]) + i += 1 + a = chars + + if args.trans_type == "romaji": + a = [to_roma(c) for c in list(str(a))] + + a = [a[j : j + n] for j in range(0, len(a), n)] # noqa E203 + + a_flat = [] + for z in a: + a_flat.append("".join(z)) + + a_chars = "".join(a_flat) + print(a_chars) + line = f.readline() + + +if __name__ == "__main__": + main() diff --git a/egs/multi_ja_en/ASR/local/train_bbpe_model.py b/egs/multi_ja_en/ASR/local/train_bbpe_model.py new file mode 100755 index 000000000..d104f2717 --- /dev/null +++ b/egs/multi_ja_en/ASR/local/train_bbpe_model.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +# Copyright 2023 Xiaomi Corp. (authors: Wei Kang) +# +# See ../../../../LICENSE for clarification regarding multiple authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# You can install sentencepiece via: +# +# pip install sentencepiece +# +# Due to an issue reported in +# https://github.com/google/sentencepiece/pull/642#issuecomment-857972030 +# +# Please install a version >=0.1.96 + +import argparse +import re +import shutil +import tempfile +from pathlib import Path + +import sentencepiece as spm + +from icefall import byte_encode +from icefall.utils import tokenize_by_ja_char + + +def get_args(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--lang-dir", + type=str, + help="""Input and output directory. + The generated bpe.model is saved to this directory. + """, + ) + + parser.add_argument( + "--transcript", + type=str, + help="Training transcript.", + ) + + parser.add_argument( + "--vocab-size", + type=int, + help="Vocabulary size for BPE training", + ) + + return parser.parse_args() + + +def _convert_to_bchar(in_path: str, out_path: str): + with open(out_path, "w") as f: + for line in open(in_path, "r").readlines(): + f.write(byte_encode(tokenize_by_ja_char(line)) + "\n") + + +def main(): + args = get_args() + vocab_size = args.vocab_size + lang_dir = Path(args.lang_dir) + + model_type = "unigram" + + model_prefix = f"{lang_dir}/{model_type}_{vocab_size}" + model_file = Path(model_prefix + ".model") + if model_file.is_file(): + print(f"{model_file} exists - skipping") + return + + character_coverage = 1.0 + input_sentence_size = 100000000 + + user_defined_symbols = ["", ""] + unk_id = len(user_defined_symbols) + # Note: unk_id is fixed to 2. + # If you change it, you should also change other + # places that are using it. + + temp = tempfile.NamedTemporaryFile() + train_text = temp.name + + _convert_to_bchar(args.transcript, train_text) + + spm.SentencePieceTrainer.train( + input=train_text, + vocab_size=vocab_size, + model_type=model_type, + model_prefix=model_prefix, + input_sentence_size=input_sentence_size, + character_coverage=character_coverage, + user_defined_symbols=user_defined_symbols, + unk_id=unk_id, + bos_id=-1, + eos_id=-1, + ) + + shutil.copyfile(model_file, f"{lang_dir}/bbpe.model") + + +if __name__ == "__main__": + main() diff --git a/egs/multi_ja_en/ASR/local/utils/asr_datamodule.py b/egs/multi_ja_en/ASR/local/utils/asr_datamodule.py new file mode 100644 index 000000000..be18e65c1 --- /dev/null +++ b/egs/multi_ja_en/ASR/local/utils/asr_datamodule.py @@ -0,0 +1,355 @@ +# Copyright 2021 Piotr Żelasko +# Copyright 2022 Xiaomi Corporation (Author: Mingshuang Luo) +# +# See ../../../../LICENSE for clarification regarding multiple authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import argparse +import inspect +import logging +from functools import lru_cache +from pathlib import Path +from typing import Any, Dict, List, Optional + +from lhotse import CutSet, Fbank, FbankConfig, load_manifest, load_manifest_lazy +from lhotse.dataset import ( + CutConcatenate, + CutMix, + DynamicBucketingSampler, + K2SpeechRecognitionDataset, + PrecomputedFeatures, + SimpleCutSampler, + SpecAugment, +) +from lhotse.dataset.input_strategies import OnTheFlyFeatures +from torch.utils.data import DataLoader + +from icefall.utils import str2bool + + +class ReazonSpeechAsrDataModule: + """ + DataModule for k2 ASR experiments. + It assumes there is always one train and valid dataloader, + but there can be multiple test dataloaders (e.g. LibriSpeech test-clean + and test-other). + It contains all the common data pipeline modules used in ASR + experiments, e.g.: + - dynamic batch size, + - bucketing samplers, + - cut concatenation, + - augmentation, + - on-the-fly feature extraction + This class should be derived for specific corpora used in ASR tasks. + """ + + def __init__(self, args: argparse.Namespace): + self.args = args + + @classmethod + def add_arguments(cls, parser: argparse.ArgumentParser): + group = parser.add_argument_group( + title="ASR data related options", + description="These options are used for the preparation of " + "PyTorch DataLoaders from Lhotse CutSet's -- they control the " + "effective batch sizes, sampling strategies, applied data " + "augmentations, etc.", + ) + group.add_argument( + "--manifest-dir", + type=Path, + default=Path("data/fbank"), + help="Path to directory with train/dev/test cuts.", + ) + group.add_argument( + "--max-duration", + type=int, + default=200.0, + help="Maximum pooled recordings duration (seconds) in a " + "single batch. You can reduce it if it causes CUDA OOM.", + ) + group.add_argument( + "--bucketing-sampler", + type=str2bool, + default=True, + help="When enabled, the batches will come from buckets of " + "similar duration (saves padding frames).", + ) + group.add_argument( + "--num-buckets", + type=int, + default=30, + help="The number of buckets for the DynamicBucketingSampler" + "(you might want to increase it for larger datasets).", + ) + group.add_argument( + "--concatenate-cuts", + type=str2bool, + default=False, + help="When enabled, utterances (cuts) will be concatenated " + "to minimize the amount of padding.", + ) + group.add_argument( + "--duration-factor", + type=float, + default=1.0, + help="Determines the maximum duration of a concatenated cut " + "relative to the duration of the longest cut in a batch.", + ) + group.add_argument( + "--gap", + type=float, + default=1.0, + help="The amount of padding (in seconds) inserted between " + "concatenated cuts. This padding is filled with noise when " + "noise augmentation is used.", + ) + group.add_argument( + "--on-the-fly-feats", + type=str2bool, + default=False, + help="When enabled, use on-the-fly cut mixing and feature " + "extraction. Will drop existing precomputed feature manifests " + "if available.", + ) + group.add_argument( + "--shuffle", + type=str2bool, + default=True, + help="When enabled (=default), the examples will be " + "shuffled for each epoch.", + ) + group.add_argument( + "--drop-last", + type=str2bool, + default=True, + help="Whether to drop last batch. Used by sampler.", + ) + group.add_argument( + "--return-cuts", + type=str2bool, + default=False, + help="When enabled, each batch will have the " + "field: batch['supervisions']['cut'] with the cuts that " + "were used to construct it.", + ) + + group.add_argument( + "--num-workers", + type=int, + default=2, + help="The number of training dataloader workers that " + "collect the batches.", + ) + + group.add_argument( + "--enable-spec-aug", + type=str2bool, + default=True, + help="When enabled, use SpecAugment for training dataset.", + ) + + group.add_argument( + "--spec-aug-time-warp-factor", + type=int, + default=80, + help="Used only when --enable-spec-aug is True. " + "It specifies the factor for time warping in SpecAugment. " + "Larger values mean more warping. " + "A value less than 1 means to disable time warp.", + ) + + group.add_argument( + "--enable-musan", + type=str2bool, + default=False, + help="When enabled, select noise from MUSAN and mix it" + "with training dataset. ", + ) + + def train_dataloaders( + self, cuts_train: CutSet, sampler_state_dict: Optional[Dict[str, Any]] = None + ) -> DataLoader: + """ + Args: + cuts_train: + CutSet for training. + sampler_state_dict: + The state dict for the training sampler. + """ + + transforms = [] + input_transforms = [] + + if self.args.enable_spec_aug: + logging.info("Enable SpecAugment") + logging.info(f"Time warp factor: {self.args.spec_aug_time_warp_factor}") + # Set the value of num_frame_masks according to Lhotse's version. + # In different Lhotse's versions, the default of num_frame_masks is + # different. + num_frame_masks = 10 + num_frame_masks_parameter = inspect.signature( + SpecAugment.__init__ + ).parameters["num_frame_masks"] + if num_frame_masks_parameter.default == 1: + num_frame_masks = 2 + logging.info(f"Num frame mask: {num_frame_masks}") + input_transforms.append( + SpecAugment( + time_warp_factor=self.args.spec_aug_time_warp_factor, + num_frame_masks=num_frame_masks, + features_mask_size=27, + num_feature_masks=2, + frames_mask_size=100, + ) + ) + else: + logging.info("Disable SpecAugment") + + logging.info("About to create train dataset") + train = K2SpeechRecognitionDataset( + cut_transforms=transforms, + input_transforms=input_transforms, + return_cuts=self.args.return_cuts, + ) + + if self.args.on_the_fly_feats: + # NOTE: the PerturbSpeed transform should be added only if we + # remove it from data prep stage. + # Add on-the-fly speed perturbation; since originally it would + # have increased epoch size by 3, we will apply prob 2/3 and use + # 3x more epochs. + # Speed perturbation probably should come first before + # concatenation, but in principle the transforms order doesn't have + # to be strict (e.g. could be randomized) + # transforms = [PerturbSpeed(factors=[0.9, 1.1], p=2/3)] + transforms # noqa + # Drop feats to be on the safe side. + train = K2SpeechRecognitionDataset( + cut_transforms=transforms, + input_strategy=OnTheFlyFeatures(Fbank(FbankConfig(num_mel_bins=80))), + input_transforms=input_transforms, + return_cuts=self.args.return_cuts, + ) + + if self.args.bucketing_sampler: + logging.info("Using DynamicBucketingSampler.") + train_sampler = DynamicBucketingSampler( + cuts_train, + max_duration=self.args.max_duration, + shuffle=self.args.shuffle, + num_buckets=self.args.num_buckets, + drop_last=self.args.drop_last, + ) + else: + logging.info("Using SimpleCutSampler.") + train_sampler = SimpleCutSampler( + cuts_train, + max_duration=self.args.max_duration, + shuffle=self.args.shuffle, + ) + logging.info("About to create train dataloader") + + if sampler_state_dict is not None: + logging.info("Loading sampler state dict") + train_sampler.load_state_dict(sampler_state_dict) + + train_dl = DataLoader( + train, + sampler=train_sampler, + batch_size=None, + num_workers=self.args.num_workers, + persistent_workers=False, + ) + + return train_dl + + def valid_dataloaders(self, cuts_valid: CutSet) -> DataLoader: + transforms = [] + if self.args.concatenate_cuts: + transforms = [ + CutConcatenate( + duration_factor=self.args.duration_factor, gap=self.args.gap + ) + ] + transforms + + logging.info("About to create dev dataset") + if self.args.on_the_fly_feats: + validate = K2SpeechRecognitionDataset( + cut_transforms=transforms, + input_strategy=OnTheFlyFeatures(Fbank(FbankConfig(num_mel_bins=80))), + return_cuts=self.args.return_cuts, + ) + else: + validate = K2SpeechRecognitionDataset( + cut_transforms=transforms, + return_cuts=self.args.return_cuts, + ) + valid_sampler = DynamicBucketingSampler( + cuts_valid, + max_duration=self.args.max_duration, + shuffle=False, + ) + logging.info("About to create dev dataloader") + valid_dl = DataLoader( + validate, + sampler=valid_sampler, + batch_size=None, + num_workers=2, + persistent_workers=False, + ) + + return valid_dl + + def test_dataloaders(self, cuts: CutSet) -> DataLoader: + logging.info("About to create test dataset") + test = K2SpeechRecognitionDataset( + input_strategy=OnTheFlyFeatures(Fbank(FbankConfig(num_mel_bins=80))) + if self.args.on_the_fly_feats + else PrecomputedFeatures(), + return_cuts=self.args.return_cuts, + ) + sampler = DynamicBucketingSampler( + cuts, + max_duration=self.args.max_duration, + shuffle=False, + ) + test_dl = DataLoader( + test, + batch_size=None, + sampler=sampler, + num_workers=self.args.num_workers, + ) + return test_dl + + @lru_cache() + def train_cuts(self) -> CutSet: + logging.info("About to get train cuts") + return load_manifest_lazy( + self.args.manifest_dir / "reazonspeech_cuts_train.jsonl.gz" + ) + + @lru_cache() + def valid_cuts(self) -> CutSet: + logging.info("About to get dev cuts") + return load_manifest_lazy( + self.args.manifest_dir / "reazonspeech_cuts_dev.jsonl.gz" + ) + + @lru_cache() + def test_cuts(self) -> List[CutSet]: + logging.info("About to get test cuts") + return load_manifest_lazy( + self.args.manifest_dir / "reazonspeech_cuts_test.jsonl.gz" + ) diff --git a/egs/multi_ja_en/ASR/local/utils/tokenizer.py b/egs/multi_ja_en/ASR/local/utils/tokenizer.py new file mode 100644 index 000000000..ba71cff89 --- /dev/null +++ b/egs/multi_ja_en/ASR/local/utils/tokenizer.py @@ -0,0 +1,252 @@ +import argparse +from pathlib import Path +from typing import Callable, List, Union + +import sentencepiece as spm +from k2 import SymbolTable + + +class Tokenizer: + text2word: Callable[[str], List[str]] + + @staticmethod + def add_arguments(parser: argparse.ArgumentParser): + group = parser.add_argument_group(title="Lang related options") + group.add_argument("--lang", type=Path, help="Path to lang directory.") + + group.add_argument( + "--lang-type", + type=str, + default=None, + help=( + "Either 'bpe' or 'char'. If not provided, it expects lang_dir/lang_type to exists. " + "Note: 'bpe' directly loads sentencepiece.SentencePieceProcessor" + ), + ) + + @staticmethod + def Load(lang_dir: Path, lang_type="", oov=""): + + if not lang_type: + assert (lang_dir / "lang_type").exists(), "lang_type not specified." + lang_type = (lang_dir / "lang_type").read_text().strip() + + tokenizer = None + + if lang_type == "bpe": + assert ( + lang_dir / "bpe.model" + ).exists(), f"No BPE .model could be found in {lang_dir}." + tokenizer = spm.SentencePieceProcessor() + tokenizer.Load(str(lang_dir / "bpe.model")) + elif lang_type == "char": + tokenizer = CharTokenizer(lang_dir, oov=oov) + else: + raise NotImplementedError(f"{lang_type} not supported at the moment.") + + return tokenizer + + load = Load + + def PieceToId(self, piece: str) -> int: + raise NotImplementedError( + "You need to implement this function in the child class." + ) + + piece_to_id = PieceToId + + def IdToPiece(self, id: int) -> str: + raise NotImplementedError( + "You need to implement this function in the child class." + ) + + id_to_piece = IdToPiece + + def GetPieceSize(self) -> int: + raise NotImplementedError( + "You need to implement this function in the child class." + ) + + get_piece_size = GetPieceSize + + def __len__(self) -> int: + return self.get_piece_size() + + def EncodeAsIdsBatch(self, input: List[str]) -> List[List[int]]: + raise NotImplementedError( + "You need to implement this function in the child class." + ) + + def EncodeAsPiecesBatch(self, input: List[str]) -> List[List[str]]: + raise NotImplementedError( + "You need to implement this function in the child class." + ) + + def EncodeAsIds(self, input: str) -> List[int]: + return self.EncodeAsIdsBatch([input])[0] + + def EncodeAsPieces(self, input: str) -> List[str]: + return self.EncodeAsPiecesBatch([input])[0] + + def Encode( + self, input: Union[str, List[str]], out_type=int + ) -> Union[List, List[List]]: + if not input: + return [] + + if isinstance(input, list): + if out_type is int: + return self.EncodeAsIdsBatch(input) + if out_type is str: + return self.EncodeAsPiecesBatch(input) + + if out_type is int: + return self.EncodeAsIds(input) + if out_type is str: + return self.EncodeAsPieces(input) + + encode = Encode + + def DecodeIdsBatch(self, input: List[List[int]]) -> List[str]: + raise NotImplementedError( + "You need to implement this function in the child class." + ) + + def DecodePiecesBatch(self, input: List[List[str]]) -> List[str]: + raise NotImplementedError( + "You need to implement this function in the child class." + ) + + def DecodeIds(self, input: List[int]) -> str: + return self.DecodeIdsBatch([input])[0] + + def DecodePieces(self, input: List[str]) -> str: + return self.DecodePiecesBatch([input])[0] + + def Decode( + self, + input: Union[int, List[int], List[str], List[List[int]], List[List[str]]], + ) -> Union[List[str], str]: + + if not input: + return "" + + if isinstance(input, int): + return self.id_to_piece(input) + elif isinstance(input, str): + raise TypeError( + "Unlike spm.SentencePieceProcessor, cannot decode from type str." + ) + + if isinstance(input[0], list): + if not input[0] or isinstance(input[0][0], int): + return self.DecodeIdsBatch(input) + + if isinstance(input[0][0], str): + return self.DecodePiecesBatch(input) + + if isinstance(input[0], int): + return self.DecodeIds(input) + if isinstance(input[0], str): + return self.DecodePieces(input) + + raise RuntimeError("Unknown input type") + + decode = Decode + + def SplitBatch(self, input: List[str]) -> List[List[str]]: + raise NotImplementedError( + "You need to implement this function in the child class." + ) + + def Split(self, input: Union[List[str], str]) -> Union[List[List[str]], List[str]]: + if isinstance(input, list): + return self.SplitBatch(input) + elif isinstance(input, str): + return self.SplitBatch([input])[0] + raise RuntimeError("Unknown input type") + + split = Split + + +class CharTokenizer(Tokenizer): + def __init__(self, lang_dir: Path, oov="", sep=""): + assert ( + lang_dir / "tokens.txt" + ).exists(), f"tokens.txt could not be found in {lang_dir}." + token_table = SymbolTable.from_file(lang_dir / "tokens.txt") + assert ( + "#0" not in token_table + ), "This tokenizer does not support disambig symbols." + self._id2sym = token_table._id2sym + self._sym2id = token_table._sym2id + self.oov = oov + self.oov_id = self._sym2id[oov] + self.sep = sep + if self.sep: + self.text2word = lambda x: x.split(self.sep) + else: + self.text2word = lambda x: list(x.replace(" ", "")) + + def piece_to_id(self, piece: str) -> int: + try: + return self._sym2id[piece] + except KeyError: + return self.oov_id + + def id_to_piece(self, id: int) -> str: + return self._id2sym[id] + + def get_piece_size(self) -> int: + return len(self._sym2id) + + def EncodeAsIdsBatch(self, input: List[str]) -> List[List[int]]: + return [[self.piece_to_id(i) for i in self.text2word(text)] for text in input] + + def EncodeAsPiecesBatch(self, input: List[str]) -> List[List[str]]: + return [ + [i if i in self._sym2id else self.oov for i in self.text2word(text)] + for text in input + ] + + def DecodeIdsBatch(self, input: List[List[int]]) -> List[str]: + return [self.sep.join(self.id_to_piece(i) for i in text) for text in input] + + def DecodePiecesBatch(self, input: List[List[str]]) -> List[str]: + return [self.sep.join(text) for text in input] + + def SplitBatch(self, input: List[str]) -> List[List[str]]: + return [self.text2word(text) for text in input] + + +def test_CharTokenizer(): + test_single_string = "こんにちは" + test_multiple_string = [ + "今日はいい天気ですよね", + "諏訪湖は綺麗でしょう", + "这在词表外", + "分かち 書き に し た 文章 です", + "", + ] + test_empty_string = "" + sp = Tokenizer.load(Path("lang_char"), "char", oov="") + splitter = sp.split + print(sp.encode(test_single_string, out_type=str)) + print(sp.encode(test_single_string, out_type=int)) + print(sp.encode(test_multiple_string, out_type=str)) + print(sp.encode(test_multiple_string, out_type=int)) + print(sp.encode(test_empty_string, out_type=str)) + print(sp.encode(test_empty_string, out_type=int)) + print(sp.decode(sp.encode(test_single_string, out_type=str))) + print(sp.decode(sp.encode(test_single_string, out_type=int))) + print(sp.decode(sp.encode(test_multiple_string, out_type=str))) + print(sp.decode(sp.encode(test_multiple_string, out_type=int))) + print(sp.decode(sp.encode(test_empty_string, out_type=str))) + print(sp.decode(sp.encode(test_empty_string, out_type=int))) + print(splitter(test_single_string)) + print(splitter(test_multiple_string)) + print(splitter(test_empty_string)) + + +if __name__ == "__main__": + test_CharTokenizer() diff --git a/egs/multi_ja_en/ASR/local/validate_bpe_lexicon.py b/egs/multi_ja_en/ASR/local/validate_bpe_lexicon.py new file mode 120000 index 000000000..721bb48e7 --- /dev/null +++ b/egs/multi_ja_en/ASR/local/validate_bpe_lexicon.py @@ -0,0 +1 @@ +../../../librispeech/ASR/local/validate_bpe_lexicon.py \ No newline at end of file diff --git a/egs/multi_ja_en/ASR/local/validate_manifest.py b/egs/multi_ja_en/ASR/local/validate_manifest.py new file mode 100644 index 000000000..7f67c64b6 --- /dev/null +++ b/egs/multi_ja_en/ASR/local/validate_manifest.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +# Copyright 2022 Xiaomi Corp. (authors: Fangjun Kuang) +# +# See ../../../../LICENSE for clarification regarding multiple authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +This script checks the following assumptions of the generated manifest: + +- Single supervision per cut +- Supervision time bounds are within cut time bounds + +We will add more checks later if needed. + +Usage example: + + python3 ./local/validate_manifest.py \ + ./data/fbank/librispeech_cuts_train-clean-100.jsonl.gz + +""" + +import argparse +import logging +from pathlib import Path + +from lhotse import CutSet, load_manifest +from lhotse.cut import Cut + + +def get_args(): + parser = argparse.ArgumentParser() + + parser.add_argument( + "--manifest", + type=Path, + help="Path to the manifest file", + ) + + return parser.parse_args() + + +def validate_one_supervision_per_cut(c: Cut): + if len(c.supervisions) != 1: + raise ValueError(f"{c.id} has {len(c.supervisions)} supervisions") + + +def validate_supervision_and_cut_time_bounds(c: Cut): + s = c.supervisions[0] + + # Removed because when the cuts were trimmed from supervisions, + # the start time of the supervision can be lesser than cut start time. + # https://github.com/lhotse-speech/lhotse/issues/813 + # if s.start < c.start: + # raise ValueError( + # f"{c.id}: Supervision start time {s.start} is less " + # f"than cut start time {c.start}" + # ) + + if s.end > c.end: + raise ValueError( + f"{c.id}: Supervision end time {s.end} is larger " + f"than cut end time {c.end}" + ) + + +def main(): + args = get_args() + + manifest = Path(args.manifest) + logging.info(f"Validating {manifest}") + + assert manifest.is_file(), f"{manifest} does not exist" + cut_set = load_manifest(manifest) + assert isinstance(cut_set, CutSet) + + for c in cut_set: + validate_one_supervision_per_cut(c) + validate_supervision_and_cut_time_bounds(c) + + +if __name__ == "__main__": + formatter = "%(asctime)s %(levelname)s [%(filename)s:%(lineno)d] %(message)s" + + logging.basicConfig(format=formatter, level=logging.INFO) + + main() diff --git a/egs/multi_ja_en/ASR/prepare.sh b/egs/multi_ja_en/ASR/prepare.sh new file mode 100755 index 000000000..7a6a63418 --- /dev/null +++ b/egs/multi_ja_en/ASR/prepare.sh @@ -0,0 +1,185 @@ +#!/usr/bin/env bash + +# fix segmentation fault reported in https://github.com/k2-fsa/icefall/issues/674 +export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python + +set -eou pipefail + +stage=-1 +stop_stage=100 + +dl_dir=$PWD/download + +. shared/parse_options.sh || exit 1 + +vocab_sizes=( + 2000 +) + +# All files generated by this script are saved in "data". +# You can safely remove "data" and rerun this script to regenerate it. +mkdir -p data + +log() { + # This function is from espnet + local fname=${BASH_SOURCE[1]##*/} + echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*" +} + +log "dl_dir: $dl_dir" + +log "Dataset: musan" +if [ $stage -le 1 ] && [ $stop_stage -ge 1 ]; then + log "Stage 1: Soft link fbank of musan" + mkdir -p data/fbank + if [ -e ../../librispeech/ASR/data/fbank/.musan.done ]; then + cd data/fbank + ln -svf $(realpath ../../../../librispeech/ASR/data/fbank/musan_feats) . + ln -svf $(realpath ../../../../librispeech/ASR/data/fbank/musan_cuts.jsonl.gz) . + cd ../.. + else + log "Abort! Please run ../../librispeech/ASR/prepare.sh --stage 4 --stop-stage 4" + exit 1 + fi +fi + +log "Dataset: LibriSpeech" +if [ $stage -le 2 ] && [ $stop_stage -ge 2 ]; then + log "Stage 1: Soft link fbank of LibriSpeech" + mkdir -p data/fbank + if [ -e ../../librispeech/ASR/data/fbank/.librispeech.done ]; then + cd data/fbank + ln -svf $(realpath ../../../../librispeech/ASR/data/fbank/librispeech_cuts*) . + ln -svf $(realpath ../../../../librispeech/ASR/data/fbank/librispeech_feats*) . + cd ../.. + else + log "Abort! Please run ../../librispeech/ASR/prepare.sh --stage 1 --stop-stage 1 and ../../librispeech/ASR/prepare.sh --stage 3 --stop-stage 3" + exit 1 + fi +fi + +log "Dataset: ReazonSpeech" +if [ $stage -le 3 ] && [ $stop_stage -ge 3 ]; then + log "Stage 2: Soft link fbank of ReazonSpeech" + mkdir -p data/fbank + if [ -e ../../reazonspeech/ASR/data/manifests/.reazonspeech.done ]; then + cd data/fbank + ln -svf $(realpath ../../../../reazonspeech/ASR/data/manifests/reazonspeech_cuts*) . + cd .. + mkdir -p manifests + cd manifests + ln -svf $(realpath ../../../../reazonspeech/ASR/data/manifests/feats_*) . + cd ../.. + else + log "Abort! Please run ../../reazonspeech/ASR/prepare.sh --stage 0 --stop-stage 2" + exit 1 + fi +fi + +# New Stage 3: Prepare char based lang for ReazonSpeech +if [ $stage -le 3 ] && [ $stop_stage -ge 3 ]; then + lang_char_dir=data/lang_char + log "Stage 3: Prepare char based lang for ReazonSpeech" + mkdir -p $lang_char_dir + + # Prepare text + if [ ! -f $lang_char_dir/text ]; then + gunzip -c ../../reazonspeech/ASR/data/manifests/reazonspeech_supervisions_train.jsonl.gz \ + | jq '.text' | sed 's/"//g' \ + | ./local/text2token.py -t "char" > $lang_char_dir/text + fi + + # jp word segmentation for text + if [ ! -f $lang_char_dir/text_words_segmentation ]; then + python3 ./local/text2segments.py \ + --input-file $lang_char_dir/text \ + --output-file $lang_char_dir/text_words_segmentation + fi + + cat $lang_char_dir/text_words_segmentation | sed 's/ /\n/g' \ + | sort -u | sed '/^$/d' | uniq > $lang_char_dir/words_no_ids.txt + + if [ ! -f $lang_char_dir/words.txt ]; then + python3 ./local/prepare_words.py \ + --input-file $lang_char_dir/words_no_ids.txt \ + --output-file $lang_char_dir/words.txt + fi + + if [ ! -f $lang_char_dir/L_disambig.pt ]; then + python3 ./local/prepare_char.py --lang-dir data/lang_char + fi +fi + +if [ $stage -le 4 ] && [ $stop_stage -ge 4 ]; then + log "Stage 4: Prepare Byte BPE based lang" + mkdir -p data/fbank + if [ ! -d ../../reazonspeech/ASR/data/lang_char ] && [ ! -d ./data/lang_char ]; then + log "Abort! Please run ../../reazonspeech/ASR/prepare.sh --stage 3 --stop-stage 3" + exit 1 + fi + + if [ ! -d ../../librispeech/ASR/data/lang_bpe_500 ] && [ ! -d ./data/lang_bpe_500 ]; then + log "Abort! Please run ../../librispeech/ASR/prepare.sh --stage 5 --stop-stage 5" + exit 1 + fi + + cd data/ + # if [ ! -d ./lang_char ]; then + # ln -svf $(realpath ../../../reazonspeech/ASR/data/lang_char) . + # fi + if [ ! -d ./lang_bpe_500 ]; then + ln -svf $(realpath ../../../librispeech/ASR/data/lang_bpe_500) . + fi + cd ../ + + for vocab_size in ${vocab_sizes[@]}; do + lang_dir=data/lang_bbpe_${vocab_size} + mkdir -p $lang_dir + + cat data/lang_char/text data/lang_bpe_500/transcript_words.txt \ + > $lang_dir/text + + if [ ! -f $lang_dir/transcript_chars.txt ]; then + ./local/prepare_for_bpe_model.py \ + --lang-dir ./$lang_dir \ + --text $lang_dir/text + fi + + if [ ! -f $lang_dir/text_words_segmentation ]; then + python3 ./local/text2segments.py \ + --input-file ./data/lang_char/text \ + --output-file $lang_dir/text_words_segmentation + + cat ./data/lang_bpe_500/transcript_words.txt \ + >> $lang_dir/text_words_segmentation + fi + + cat $lang_dir/text_words_segmentation | sed 's/ /\n/g' \ + | sort -u | sed '/^$/d' | uniq > $lang_dir/words_no_ids.txt + + if [ ! -f $lang_dir/words.txt ]; then + python3 ./local/prepare_words.py \ + --input-file $lang_dir/words_no_ids.txt \ + --output-file $lang_dir/words.txt + fi + + if [ ! -f $lang_dir/bbpe.model ]; then + ./local/train_bbpe_model.py \ + --lang-dir $lang_dir \ + --vocab-size $vocab_size \ + --transcript $lang_dir/text + fi + + if [ ! -f $lang_dir/L_disambig.pt ]; then + ./local/prepare_lang_bbpe.py --lang-dir $lang_dir + + log "Validating $lang_dir/lexicon.txt" + ln -svf $(realpath ../../multi_zh_en/ASR/local/validate_bpe_lexicon.py) local/ + ./local/validate_bpe_lexicon.py \ + --lexicon $lang_dir/lexicon.txt \ + --bpe-model $lang_dir/bbpe.model + fi + done +fi + +log "prepare.sh: PREPARATION DONE" diff --git a/egs/multi_ja_en/ASR/shared b/egs/multi_ja_en/ASR/shared new file mode 120000 index 000000000..4c5e91438 --- /dev/null +++ b/egs/multi_ja_en/ASR/shared @@ -0,0 +1 @@ +../../../icefall/shared/ \ No newline at end of file diff --git a/egs/multi_ja_en/ASR/zipformer/asr_datamodule.py b/egs/multi_ja_en/ASR/zipformer/asr_datamodule.py new file mode 120000 index 000000000..a48591198 --- /dev/null +++ b/egs/multi_ja_en/ASR/zipformer/asr_datamodule.py @@ -0,0 +1 @@ +../local/utils/asr_datamodule.py \ No newline at end of file diff --git a/egs/multi_ja_en/ASR/zipformer/beam_search.py b/egs/multi_ja_en/ASR/zipformer/beam_search.py new file mode 120000 index 000000000..8e2c0a65c --- /dev/null +++ b/egs/multi_ja_en/ASR/zipformer/beam_search.py @@ -0,0 +1 @@ +../../../librispeech/ASR/zipformer/beam_search.py \ No newline at end of file diff --git a/egs/multi_ja_en/ASR/zipformer/ctc_decode.py b/egs/multi_ja_en/ASR/zipformer/ctc_decode.py new file mode 120000 index 000000000..faa8bd562 --- /dev/null +++ b/egs/multi_ja_en/ASR/zipformer/ctc_decode.py @@ -0,0 +1 @@ +../../../librispeech/ASR/zipformer/ctc_decode.py \ No newline at end of file diff --git a/egs/multi_ja_en/ASR/zipformer/decode.py b/egs/multi_ja_en/ASR/zipformer/decode.py new file mode 100755 index 000000000..26ce3e018 --- /dev/null +++ b/egs/multi_ja_en/ASR/zipformer/decode.py @@ -0,0 +1,792 @@ +#!/usr/bin/env python3 +# +# Copyright 2021-2023 Xiaomi Corporation (Author: Fangjun Kuang, +# Zengwei Yao) +# +# See ../../../../LICENSE for clarification regarding multiple authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Usage: +(1) greedy search +./zipformer/decode.py \ + --epoch 28 \ + --avg 15 \ + --exp-dir ./zipformer/exp \ + --max-duration 600 \ + --decoding-method greedy_search + +(2) beam search (not recommended) +./zipformer/decode.py \ + --epoch 28 \ + --avg 15 \ + --exp-dir ./zipformer/exp \ + --max-duration 600 \ + --decoding-method beam_search \ + --beam-size 4 + +(3) modified beam search +./zipformer/decode.py \ + --epoch 28 \ + --avg 15 \ + --exp-dir ./zipformer/exp \ + --max-duration 600 \ + --decoding-method modified_beam_search \ + --beam-size 4 + +(4) fast beam search (one best) +./zipformer/decode.py \ + --epoch 28 \ + --avg 15 \ + --exp-dir ./zipformer/exp \ + --max-duration 600 \ + --decoding-method fast_beam_search \ + --beam 20.0 \ + --max-contexts 8 \ + --max-states 64 +""" + +import argparse +import logging +import math +import re +from collections import defaultdict +from pathlib import Path +from typing import Dict, List, Optional, Tuple + +import k2 +import sentencepiece as spm +import torch +import torch.nn as nn +from asr_datamodule import ReazonSpeechAsrDataModule +from beam_search import ( + beam_search, + fast_beam_search_nbest, + fast_beam_search_nbest_LG, + fast_beam_search_nbest_oracle, + fast_beam_search_one_best, + greedy_search, + greedy_search_batch, + modified_beam_search, +) +from lhotse.cut import Cut +from multi_dataset import MultiDataset +from train import add_model_arguments, get_model, get_params + +from icefall import byte_encode, smart_byte_decode +from icefall.checkpoint import ( + average_checkpoints, + average_checkpoints_with_averaged_model, + find_checkpoints, + load_checkpoint, +) +from icefall.lexicon import Lexicon +from icefall.utils import ( + AttributeDict, + setup_logger, + store_transcripts, + str2bool, + tokenize_by_ja_char, + write_error_stats, +) + +LOG_EPS = math.log(1e-10) + + +def get_parser(): + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + + parser.add_argument( + "--epoch", + type=int, + default=30, + help="""It specifies the checkpoint to use for decoding. + Note: Epoch counts from 1. + You can specify --avg to use more checkpoints for model averaging.""", + ) + + parser.add_argument( + "--iter", + type=int, + default=0, + help="""If positive, --epoch is ignored and it + will use the checkpoint exp_dir/checkpoint-iter.pt. + You can specify --avg to use more checkpoints for model averaging. + """, + ) + + parser.add_argument( + "--avg", + type=int, + default=15, + help="Number of checkpoints to average. Automatically select " + "consecutive checkpoints before the checkpoint specified by " + "'--epoch' and '--iter'", + ) + + parser.add_argument( + "--use-averaged-model", + type=str2bool, + default=True, + help="Whether to load averaged model. Currently it only supports " + "using --epoch. If True, it would decode with the averaged model " + "over the epoch range from `epoch-avg` (excluded) to `epoch`." + "Actually only the models with epoch number of `epoch-avg` and " + "`epoch` are loaded for averaging. ", + ) + + parser.add_argument( + "--exp-dir", + type=str, + default="zipformer/exp", + help="The experiment dir", + ) + + parser.add_argument( + "--bpe-model", + type=str, + default="data/lang_bbpe_2000/bbpe.model", + help="Path to the BPE model", + ) + + parser.add_argument( + "--lang-dir", + type=Path, + default="data/lang_bbpe_2000", + help="The lang dir containing word table and LG graph", + ) + + parser.add_argument( + "--decoding-method", + type=str, + default="greedy_search", + help="""Possible values are: + - greedy_search + - beam_search + - modified_beam_search + - fast_beam_search + - fast_beam_search_nbest + - fast_beam_search_nbest_oracle + - fast_beam_search_nbest_LG + If you use fast_beam_search_nbest_LG, you have to specify + `--lang-dir`, which should contain `LG.pt`. + """, + ) + + parser.add_argument( + "--beam-size", + type=int, + default=4, + help="""An integer indicating how many candidates we will keep for each + frame. Used only when --decoding-method is beam_search or + modified_beam_search.""", + ) + + parser.add_argument( + "--beam", + type=float, + default=20.0, + help="""A floating point value to calculate the cutoff score during beam + search (i.e., `cutoff = max-score - beam`), which is the same as the + `beam` in Kaldi. + Used only when --decoding-method is fast_beam_search, + fast_beam_search_nbest, fast_beam_search_nbest_LG, + and fast_beam_search_nbest_oracle + """, + ) + + parser.add_argument( + "--ngram-lm-scale", + type=float, + default=0.01, + help=""" + Used only when --decoding_method is fast_beam_search_nbest_LG. + It specifies the scale for n-gram LM scores. + """, + ) + + parser.add_argument( + "--max-contexts", + type=int, + default=8, + help="""Used only when --decoding-method is + fast_beam_search, fast_beam_search_nbest, fast_beam_search_nbest_LG, + and fast_beam_search_nbest_oracle""", + ) + + parser.add_argument( + "--max-states", + type=int, + default=64, + help="""Used only when --decoding-method is + fast_beam_search, fast_beam_search_nbest, fast_beam_search_nbest_LG, + and fast_beam_search_nbest_oracle""", + ) + + parser.add_argument( + "--context-size", + type=int, + default=2, + help="The context size in the decoder. 1 means bigram; " "2 means tri-gram", + ) + parser.add_argument( + "--max-sym-per-frame", + type=int, + default=1, + help="""Maximum number of symbols per frame. + Used only when --decoding_method is greedy_search""", + ) + + parser.add_argument( + "--num-paths", + type=int, + default=200, + help="""Number of paths for nbest decoding. + Used only when the decoding method is fast_beam_search_nbest, + fast_beam_search_nbest_LG, and fast_beam_search_nbest_oracle""", + ) + + parser.add_argument( + "--nbest-scale", + type=float, + default=0.5, + help="""Scale applied to lattice scores when computing nbest paths. + Used only when the decoding method is fast_beam_search_nbest, + fast_beam_search_nbest_LG, and fast_beam_search_nbest_oracle""", + ) + + add_model_arguments(parser) + + return parser + + +def decode_one_batch( + params: AttributeDict, + model: nn.Module, + sp: spm.SentencePieceProcessor, + batch: dict, + word_table: Optional[k2.SymbolTable] = None, + decoding_graph: Optional[k2.Fsa] = None, +) -> Dict[str, List[List[str]]]: + """Decode one batch and return the result in a dict. The dict has the + following format: + + - key: It indicates the setting used for decoding. For example, + if greedy_search is used, it would be "greedy_search" + If beam search with a beam size of 7 is used, it would be + "beam_7" + - value: It contains the decoding result. `len(value)` equals to + batch size. `value[i]` is the decoding result for the i-th + utterance in the given batch. + Args: + params: + It's the return value of :func:`get_params`. + model: + The neural model. + sp: + The BPE model. + batch: + It is the return value from iterating + `lhotse.dataset.K2SpeechRecognitionDataset`. See its documentation + for the format of the `batch`. + word_table: + The word symbol table. + decoding_graph: + The decoding graph. Can be either a `k2.trivial_graph` or HLG, Used + only when --decoding_method is fast_beam_search, fast_beam_search_nbest, + fast_beam_search_nbest_oracle, and fast_beam_search_nbest_LG. + Returns: + Return the decoding result. See above description for the format of + the returned dict. + """ + device = next(model.parameters()).device + feature = batch["inputs"] + assert feature.ndim == 3 + + feature = feature.to(device) + # at entry, feature is (N, T, C) + + supervisions = batch["supervisions"] + feature_lens = supervisions["num_frames"].to(device) + + if params.causal: + # this seems to cause insertions at the end of the utterance if used with zipformer. + pad_len = 30 + feature_lens += pad_len + feature = torch.nn.functional.pad( + feature, + pad=(0, 0, 0, pad_len), + value=LOG_EPS, + ) + + encoder_out, encoder_out_lens = model.forward_encoder(feature, feature_lens) + + hyps = [] + + if params.decoding_method == "fast_beam_search": + hyp_tokens = fast_beam_search_one_best( + model=model, + decoding_graph=decoding_graph, + encoder_out=encoder_out, + encoder_out_lens=encoder_out_lens, + beam=params.beam, + max_contexts=params.max_contexts, + max_states=params.max_states, + ) + for hyp in sp.decode(hyp_tokens): + hyps.append(smart_byte_decode(hyp).split()) + elif params.decoding_method == "fast_beam_search_nbest_LG": + hyp_tokens = fast_beam_search_nbest_LG( + model=model, + decoding_graph=decoding_graph, + encoder_out=encoder_out, + encoder_out_lens=encoder_out_lens, + beam=params.beam, + max_contexts=params.max_contexts, + max_states=params.max_states, + num_paths=params.num_paths, + nbest_scale=params.nbest_scale, + ) + for hyp in hyp_tokens: + hyps.append([word_table[i] for i in hyp]) + elif params.decoding_method == "fast_beam_search_nbest": + hyp_tokens = fast_beam_search_nbest( + model=model, + decoding_graph=decoding_graph, + encoder_out=encoder_out, + encoder_out_lens=encoder_out_lens, + beam=params.beam, + max_contexts=params.max_contexts, + max_states=params.max_states, + num_paths=params.num_paths, + nbest_scale=params.nbest_scale, + ) + for hyp in sp.decode(hyp_tokens): + hyps.append(smart_byte_decode(hyp).split()) + elif params.decoding_method == "fast_beam_search_nbest_oracle": + hyp_tokens = fast_beam_search_nbest_oracle( + model=model, + decoding_graph=decoding_graph, + encoder_out=encoder_out, + encoder_out_lens=encoder_out_lens, + beam=params.beam, + max_contexts=params.max_contexts, + max_states=params.max_states, + num_paths=params.num_paths, + ref_texts=sp.encode(byte_encode(tokenize_by_ja_char(supervisions["text"]))), + nbest_scale=params.nbest_scale, + ) + for hyp in sp.decode(hyp_tokens): + hyps.append(smart_byte_decode(hyp).split()) + elif params.decoding_method == "greedy_search" and params.max_sym_per_frame == 1: + hyp_tokens = greedy_search_batch( + model=model, + encoder_out=encoder_out, + encoder_out_lens=encoder_out_lens, + ) + for hyp in sp.decode(hyp_tokens): + hyps.append(smart_byte_decode(hyp).split()) + elif params.decoding_method == "modified_beam_search": + hyp_tokens = modified_beam_search( + model=model, + encoder_out=encoder_out, + encoder_out_lens=encoder_out_lens, + beam=params.beam_size, + ) + for hyp in sp.decode(hyp_tokens): + hyps.append(smart_byte_decode(hyp).split()) + else: + batch_size = encoder_out.size(0) + + for i in range(batch_size): + # fmt: off + encoder_out_i = encoder_out[i:i+1, :encoder_out_lens[i]] + # fmt: on + if params.decoding_method == "greedy_search": + hyp = greedy_search( + model=model, + encoder_out=encoder_out_i, + max_sym_per_frame=params.max_sym_per_frame, + ) + elif params.decoding_method == "beam_search": + hyp = beam_search( + model=model, + encoder_out=encoder_out_i, + beam=params.beam_size, + ) + else: + raise ValueError( + f"Unsupported decoding method: {params.decoding_method}" + ) + hyps.append(smart_byte_decode(sp.decode(hyp)).split()) + if params.decoding_method == "greedy_search": + return {"greedy_search": hyps} + elif "fast_beam_search" in params.decoding_method: + key = f"beam_{params.beam}_" + key += f"max_contexts_{params.max_contexts}_" + key += f"max_states_{params.max_states}" + if "nbest" in params.decoding_method: + key += f"_num_paths_{params.num_paths}_" + key += f"nbest_scale_{params.nbest_scale}" + if "LG" in params.decoding_method: + key += f"_ngram_lm_scale_{params.ngram_lm_scale}" + + return {key: hyps} + else: + return {f"beam_size_{params.beam_size}": hyps} + + +def decode_dataset( + dl: torch.utils.data.DataLoader, + params: AttributeDict, + model: nn.Module, + sp: spm.SentencePieceProcessor, + word_table: Optional[k2.SymbolTable] = None, + decoding_graph: Optional[k2.Fsa] = None, +) -> Dict[str, List[Tuple[str, List[str], List[str]]]]: + """Decode dataset. + + Args: + dl: + PyTorch's dataloader containing the dataset to decode. + params: + It is returned by :func:`get_params`. + model: + The neural model. + sp: + The BPE model. + word_table: + The word symbol table. + decoding_graph: + The decoding graph. Can be either a `k2.trivial_graph` or HLG, Used + only when --decoding_method is fast_beam_search, fast_beam_search_nbest, + fast_beam_search_nbest_oracle, and fast_beam_search_nbest_LG. + Returns: + Return a dict, whose key may be "greedy_search" if greedy search + is used, or it may be "beam_7" if beam size of 7 is used. + Its value is a list of tuples. Each tuple contains two elements: + The first is the reference transcript, and the second is the + predicted result. + """ + num_cuts = 0 + + try: + num_batches = len(dl) + except TypeError: + num_batches = "?" + + if params.decoding_method == "greedy_search": + log_interval = 50 + else: + log_interval = 20 + + results = defaultdict(list) + for batch_idx, batch in enumerate(dl): + texts = batch["supervisions"]["text"] + texts = [tokenize_by_ja_char(str(text)).split() for text in texts] + # print(texts) + # exit() + cut_ids = [cut.id for cut in batch["supervisions"]["cut"]] + + hyps_dict = decode_one_batch( + params=params, + model=model, + sp=sp, + decoding_graph=decoding_graph, + word_table=word_table, + batch=batch, + ) + + for name, hyps in hyps_dict.items(): + this_batch = [] + assert len(hyps) == len(texts) + for cut_id, hyp_words, ref_text in zip(cut_ids, hyps, texts): + this_batch.append((cut_id, ref_text, hyp_words)) + + results[name].extend(this_batch) + + num_cuts += len(texts) + + if batch_idx % log_interval == 0: + batch_str = f"{batch_idx}/{num_batches}" + + logging.info(f"batch {batch_str}, cuts processed until now is {num_cuts}") + return results + + +def save_results( + params: AttributeDict, + test_set_name: str, + results_dict: Dict[str, List[Tuple[str, List[str], List[str]]]], +): + test_set_wers = dict() + for key, results in results_dict.items(): + recog_path = ( + params.res_dir / f"recogs-{test_set_name}-{key}-{params.suffix}.txt" + ) + results = sorted(results) + store_transcripts(filename=recog_path, texts=results) + logging.info(f"The transcripts are stored in {recog_path}") + + # The following prints out WERs, per-word error statistics and aligned + # ref/hyp pairs. + errs_filename = ( + params.res_dir / f"errs-{test_set_name}-{key}-{params.suffix}.txt" + ) + with open(errs_filename, "w") as f: + wer = write_error_stats( + f, f"{test_set_name}-{key}", results, enable_log=True + ) + test_set_wers[key] = wer + + logging.info("Wrote detailed error stats to {}".format(errs_filename)) + + test_set_wers = sorted(test_set_wers.items(), key=lambda x: x[1]) + errs_info = ( + params.res_dir / f"wer-summary-{test_set_name}-{key}-{params.suffix}.txt" + ) + with open(errs_info, "w") as f: + print("settings\tWER", file=f) + for key, val in test_set_wers: + print("{}\t{}".format(key, val), file=f) + + s = "\nFor {}, WER of different settings are:\n".format(test_set_name) + note = "\tbest for {}".format(test_set_name) + for key, val in test_set_wers: + s += "{}\t{}{}\n".format(key, val, note) + note = "" + logging.info(s) + + +@torch.no_grad() +def main(): + parser = get_parser() + ReazonSpeechAsrDataModule.add_arguments(parser) + args = parser.parse_args() + args.exp_dir = Path(args.exp_dir) + + params = get_params() + params.update(vars(args)) + + assert params.decoding_method in ( + "greedy_search", + "beam_search", + "fast_beam_search", + "fast_beam_search_nbest", + "fast_beam_search_nbest_LG", + "fast_beam_search_nbest_oracle", + "modified_beam_search", + ) + params.res_dir = params.exp_dir / params.decoding_method + + if params.iter > 0: + params.suffix = f"iter-{params.iter}-avg-{params.avg}" + else: + params.suffix = f"epoch-{params.epoch}-avg-{params.avg}" + + if params.causal: + assert ( + "," not in params.chunk_size + ), "chunk_size should be one value in decoding." + assert ( + "," not in params.left_context_frames + ), "left_context_frames should be one value in decoding." + params.suffix += f"-chunk-{params.chunk_size}" + params.suffix += f"-left-context-{params.left_context_frames}" + + if "fast_beam_search" in params.decoding_method: + params.suffix += f"-beam-{params.beam}" + params.suffix += f"-max-contexts-{params.max_contexts}" + params.suffix += f"-max-states-{params.max_states}" + if "nbest" in params.decoding_method: + params.suffix += f"-nbest-scale-{params.nbest_scale}" + params.suffix += f"-num-paths-{params.num_paths}" + if "LG" in params.decoding_method: + params.suffix += f"-ngram-lm-scale-{params.ngram_lm_scale}" + elif "beam_search" in params.decoding_method: + params.suffix += f"-{params.decoding_method}-beam-size-{params.beam_size}" + else: + params.suffix += f"-context-{params.context_size}" + params.suffix += f"-max-sym-per-frame-{params.max_sym_per_frame}" + + if params.use_averaged_model: + params.suffix += "-use-averaged-model" + + setup_logger(f"{params.res_dir}/log-decode-{params.suffix}") + logging.info("Decoding started") + + device = torch.device("cpu") + if torch.cuda.is_available(): + device = torch.device("cuda", 0) + + logging.info(f"Device: {device}") + + sp = spm.SentencePieceProcessor() + sp.load(params.bpe_model) + + # and are defined in local/train_bpe_model.py + params.blank_id = sp.piece_to_id("") + params.unk_id = sp.piece_to_id("") + params.vocab_size = sp.get_piece_size() + + logging.info(params) + + logging.info("About to create model") + model = get_model(params) + + if not params.use_averaged_model: + if params.iter > 0: + filenames = find_checkpoints(params.exp_dir, iteration=-params.iter)[ + : params.avg + ] + if len(filenames) == 0: + raise ValueError( + f"No checkpoints found for" + f" --iter {params.iter}, --avg {params.avg}" + ) + elif len(filenames) < params.avg: + raise ValueError( + f"Not enough checkpoints ({len(filenames)}) found for" + f" --iter {params.iter}, --avg {params.avg}" + ) + logging.info(f"averaging {filenames}") + model.to(device) + model.load_state_dict(average_checkpoints(filenames, device=device)) + elif params.avg == 1: + load_checkpoint(f"{params.exp_dir}/epoch-{params.epoch}.pt", model) + else: + start = params.epoch - params.avg + 1 + filenames = [] + for i in range(start, params.epoch + 1): + if i >= 1: + filenames.append(f"{params.exp_dir}/epoch-{i}.pt") + logging.info(f"averaging {filenames}") + model.to(device) + model.load_state_dict(average_checkpoints(filenames, device=device)) + else: + if params.iter > 0: + filenames = find_checkpoints(params.exp_dir, iteration=-params.iter)[ + : params.avg + 1 + ] + if len(filenames) == 0: + raise ValueError( + f"No checkpoints found for" + f" --iter {params.iter}, --avg {params.avg}" + ) + elif len(filenames) < params.avg + 1: + raise ValueError( + f"Not enough checkpoints ({len(filenames)}) found for" + f" --iter {params.iter}, --avg {params.avg}" + ) + filename_start = filenames[-1] + filename_end = filenames[0] + logging.info( + "Calculating the averaged model over iteration checkpoints" + f" from {filename_start} (excluded) to {filename_end}" + ) + model.to(device) + model.load_state_dict( + average_checkpoints_with_averaged_model( + filename_start=filename_start, + filename_end=filename_end, + device=device, + ) + ) + else: + assert params.avg > 0, params.avg + start = params.epoch - params.avg + assert start >= 1, start + filename_start = f"{params.exp_dir}/epoch-{start}.pt" + filename_end = f"{params.exp_dir}/epoch-{params.epoch}.pt" + logging.info( + f"Calculating the averaged model over epoch range from " + f"{start} (excluded) to {params.epoch}" + ) + model.to(device) + model.load_state_dict( + average_checkpoints_with_averaged_model( + filename_start=filename_start, + filename_end=filename_end, + device=device, + ) + ) + + model.to(device) + model.eval() + + if "fast_beam_search" in params.decoding_method: + if params.decoding_method == "fast_beam_search_nbest_LG": + lexicon = Lexicon(params.lang_dir) + word_table = lexicon.word_table + lg_filename = params.lang_dir / "LG.pt" + logging.info(f"Loading {lg_filename}") + decoding_graph = k2.Fsa.from_dict( + torch.load(lg_filename, map_location=device) + ) + decoding_graph.scores *= params.ngram_lm_scale + else: + word_table = None + decoding_graph = k2.trivial_graph(params.vocab_size - 1, device=device) + else: + decoding_graph = None + word_table = None + + num_param = sum([p.numel() for p in model.parameters()]) + logging.info(f"Number of model parameters: {num_param}") + + # we need cut ids to display recognition results. + args.return_cuts = True + data_module = ReazonSpeechAsrDataModule(args) + multi_dataset = MultiDataset(args) + + def remove_short_utt(c: Cut): + T = ((c.num_frames - 7) // 2 + 1) // 2 + if T <= 0: + logging.warning( + f"Excluding cut with ID: {c.id} from decoding, num_frames: {c.num_frames}" + ) + return T > 0 + + test_sets_cuts = multi_dataset.test_cuts() + + test_sets = test_sets_cuts.keys() + test_dl = [ + data_module.test_dataloaders(test_sets_cuts[cuts_name].filter(remove_short_utt)) + for cuts_name in test_sets + ] + + for test_set, test_dl in zip(test_sets, test_dl): + logging.info(f"Start decoding test set: {test_set}") + + results_dict = decode_dataset( + dl=test_dl, + params=params, + model=model, + sp=sp, + word_table=word_table, + decoding_graph=decoding_graph, + ) + + save_results( + params=params, + test_set_name=test_set, + results_dict=results_dict, + ) + + logging.info("Done!") + + +if __name__ == "__main__": + main() diff --git a/egs/multi_ja_en/ASR/zipformer/decode_stream.py b/egs/multi_ja_en/ASR/zipformer/decode_stream.py new file mode 120000 index 000000000..b8d8ddfc4 --- /dev/null +++ b/egs/multi_ja_en/ASR/zipformer/decode_stream.py @@ -0,0 +1 @@ +../../../librispeech/ASR/zipformer/decode_stream.py \ No newline at end of file diff --git a/egs/multi_ja_en/ASR/zipformer/decoder.py b/egs/multi_ja_en/ASR/zipformer/decoder.py new file mode 120000 index 000000000..5a8018680 --- /dev/null +++ b/egs/multi_ja_en/ASR/zipformer/decoder.py @@ -0,0 +1 @@ +../../../librispeech/ASR/zipformer/decoder.py \ No newline at end of file diff --git a/egs/multi_ja_en/ASR/zipformer/do_not_use_it_directly.py b/egs/multi_ja_en/ASR/zipformer/do_not_use_it_directly.py new file mode 100755 index 000000000..072679cfc --- /dev/null +++ b/egs/multi_ja_en/ASR/zipformer/do_not_use_it_directly.py @@ -0,0 +1,1261 @@ +#!/usr/bin/env python3 +# Copyright 2021-2022 Xiaomi Corp. (authors: Fangjun Kuang, +# Wei Kang, +# Mingshuang Luo,) +# Zengwei Yao) +# +# See ../../../../LICENSE for clarification regarding multiple authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Usage: + +export CUDA_VISIBLE_DEVICES="0,1,2,3" + +./pruned_transducer_stateless7_streaming/train.py \ + --world-size 4 \ + --num-epochs 30 \ + --start-epoch 1 \ + --exp-dir pruned_transducer_stateless7_streaming/exp \ + --lang data/lang_char \ + --max-duration 300 + +# For mix precision training: + +./pruned_transducer_stateless7_streaming/train.py \ + --world-size 4 \ + --num-epochs 30 \ + --start-epoch 1 \ + --use-fp16 1 \ + --exp-dir pruned_transducer_stateless7_streaming/exp \ + --lang data/lang_char \ + --max-duration 550 +""" + + +import argparse +import copy +import logging +import math +import warnings +from pathlib import Path +from shutil import copyfile +from typing import Any, Dict, Optional, Tuple, Union + +import k2 +import optim +import torch +import torch.multiprocessing as mp +import torch.nn as nn +from asr_datamodule import ReazonSpeechAsrDataModule +from decoder import Decoder +from joiner import Joiner +from lhotse.cut import Cut +from lhotse.dataset.sampling.base import CutSampler +from lhotse.utils import fix_random_seed +from model import Transducer +from optim import Eden, ScaledAdam +from tokenizer import Tokenizer +from torch import Tensor +from torch.cuda.amp import GradScaler +from torch.nn.parallel import DistributedDataParallel as DDP +from torch.utils.tensorboard import SummaryWriter +from zipformer_for_ncnn_export_only import Zipformer + +from icefall import diagnostics +from icefall.checkpoint import load_checkpoint, remove_checkpoints +from icefall.checkpoint import save_checkpoint as save_checkpoint_impl +from icefall.checkpoint import ( + save_checkpoint_with_global_batch_idx, + update_averaged_model, +) +from icefall.dist import cleanup_dist, setup_dist +from icefall.env import get_env_info +from icefall.hooks import register_inf_check_hooks +from icefall.utils import AttributeDict, MetricsTracker, setup_logger, str2bool + +LRSchedulerType = Union[torch.optim.lr_scheduler._LRScheduler, optim.LRScheduler] +LOG_EPS = math.log(1e-10) + + +def set_batch_count(model: Union[nn.Module, DDP], batch_count: float) -> None: + if isinstance(model, DDP): + # get underlying nn.Module + model = model.module + for module in model.modules(): + if hasattr(module, "batch_count"): + module.batch_count = batch_count + + +def add_model_arguments(parser: argparse.ArgumentParser): + parser.add_argument( + "--num-encoder-layers", + type=str, + default="2,4,3,2,4", + help="Number of zipformer encoder layers, comma separated.", + ) + + parser.add_argument( + "--feedforward-dims", + type=str, + default="1024,1024,2048,2048,1024", + help="Feedforward dimension of the zipformer encoder layers, comma separated.", + ) + + parser.add_argument( + "--nhead", + type=str, + default="8,8,8,8,8", + help="Number of attention heads in the zipformer encoder layers.", + ) + + parser.add_argument( + "--encoder-dims", + type=str, + default="384,384,384,384,384", + help="Embedding dimension in the 2 blocks of zipformer encoder layers, comma separated", + ) + + parser.add_argument( + "--attention-dims", + type=str, + default="192,192,192,192,192", + help="""Attention dimension in the 2 blocks of zipformer encoder layers, comma separated; + not the same as embedding dimension.""", + ) + + parser.add_argument( + "--encoder-unmasked-dims", + type=str, + default="256,256,256,256,256", + help="Unmasked dimensions in the encoders, relates to augmentation during training. " + "Must be <= each of encoder_dims. Empirically, less than 256 seems to make performance " + " worse.", + ) + + parser.add_argument( + "--zipformer-downsampling-factors", + type=str, + default="1,2,4,8,2", + help="Downsampling factor for each stack of encoder layers.", + ) + + parser.add_argument( + "--cnn-module-kernels", + type=str, + default="31,31,31,31,31", + help="Sizes of kernels in convolution modules", + ) + + parser.add_argument( + "--decoder-dim", + type=int, + default=512, + help="Embedding dimension in the decoder model.", + ) + + parser.add_argument( + "--joiner-dim", + type=int, + default=512, + help="""Dimension used in the joiner model. + Outputs from the encoder and decoder model are projected + to this dimension before adding. + """, + ) + + parser.add_argument( + "--short-chunk-size", + type=int, + default=50, + help="""Chunk length of dynamic training, the chunk size would be either + max sequence length of current batch or uniformly sampled from (1, short_chunk_size). + """, + ) + + parser.add_argument( + "--num-left-chunks", + type=int, + default=4, + help="How many left context can be seen in chunks when calculating attention.", + ) + + parser.add_argument( + "--decode-chunk-len", + type=int, + default=32, + help="The chunk size for decoding (in frames before subsampling)", + ) + + +def get_parser(): + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + + parser.add_argument( + "--world-size", + type=int, + default=1, + help="Number of GPUs for DDP training.", + ) + + parser.add_argument( + "--master-port", + type=int, + default=12354, + help="Master port to use for DDP training.", + ) + + parser.add_argument( + "--tensorboard", + type=str2bool, + default=True, + help="Should various information be logged in tensorboard.", + ) + + parser.add_argument( + "--num-epochs", + type=int, + default=30, + help="Number of epochs to train.", + ) + + parser.add_argument( + "--start-epoch", + type=int, + default=1, + help="""Resume training from this epoch. It should be positive. + If larger than 1, it will load checkpoint from + exp-dir/epoch-{start_epoch-1}.pt + """, + ) + + parser.add_argument( + "--start-batch", + type=int, + default=0, + help="""If positive, --start-epoch is ignored and + it loads the checkpoint from exp-dir/checkpoint-{start_batch}.pt + """, + ) + + parser.add_argument( + "--exp-dir", + type=Path, + default="pruned_transducer_stateless7_streaming/exp", + help="""The experiment dir. + It specifies the directory where all training related + files, e.g., checkpoints, log, etc, are saved + """, + ) + + parser.add_argument( + "--base-lr", type=float, default=0.05, help="The base learning rate." + ) + + parser.add_argument( + "--lr-batches", + type=float, + default=5000, + help="""Number of steps that affects how rapidly the learning rate + decreases. We suggest not to change this.""", + ) + + parser.add_argument( + "--lr-epochs", + type=float, + default=3.5, + help="""Number of epochs that affects how rapidly the learning rate decreases. + """, + ) + + parser.add_argument( + "--context-size", + type=int, + default=2, + help="The context size in the decoder. 1 means bigram; 2 means tri-gram", + ) + + parser.add_argument( + "--prune-range", + type=int, + default=5, + help="The prune range for rnnt loss, it means how many symbols(context)" + "we are using to compute the loss", + ) + + parser.add_argument( + "--lm-scale", + type=float, + default=0.25, + help="The scale to smooth the loss with lm " + "(output of prediction network) part.", + ) + + parser.add_argument( + "--am-scale", + type=float, + default=0.0, + help="The scale to smooth the loss with am (output of encoder network) part.", + ) + + parser.add_argument( + "--simple-loss-scale", + type=float, + default=0.5, + help="To get pruning ranges, we will calculate a simple version" + "loss(joiner is just addition), this simple loss also uses for" + "training (as a regularization item). We will scale the simple loss" + "with this parameter before adding to the final loss.", + ) + + parser.add_argument( + "--seed", + type=int, + default=42, + help="The seed for random generators intended for reproducibility", + ) + + parser.add_argument( + "--print-diagnostics", + type=str2bool, + default=False, + help="Accumulate stats on activations, print them and exit.", + ) + + parser.add_argument( + "--inf-check", + type=str2bool, + default=False, + help="Add hooks to check for infinite module outputs and gradients.", + ) + + parser.add_argument( + "--save-every-n", + type=int, + default=2000, + help="""Save checkpoint after processing this number of batches" + periodically. We save checkpoint to exp-dir/ whenever + params.batch_idx_train % save_every_n == 0. The checkpoint filename + has the form: f'exp-dir/checkpoint-{params.batch_idx_train}.pt' + Note: It also saves checkpoint to `exp-dir/epoch-xxx.pt` at the + end of each epoch where `xxx` is the epoch number counting from 0. + """, + ) + + parser.add_argument( + "--keep-last-k", + type=int, + default=30, + help="""Only keep this number of checkpoints on disk. + For instance, if it is 3, there are only 3 checkpoints + in the exp-dir with filenames `checkpoint-xxx.pt`. + It does not affect checkpoints with name `epoch-xxx.pt`. + """, + ) + + parser.add_argument( + "--average-period", + type=int, + default=200, + help="""Update the averaged model, namely `model_avg`, after processing + this number of batches. `model_avg` is a separate version of model, + in which each floating-point parameter is the average of all the + parameters from the start of training. Each time we take the average, + we do: `model_avg = model * (average_period / batch_idx_train) + + model_avg * ((batch_idx_train - average_period) / batch_idx_train)`. + """, + ) + + parser.add_argument( + "--use-fp16", + type=str2bool, + default=False, + help="Whether to use half precision training.", + ) + + parser.add_argument( + "--pad-feature", + type=int, + default=0, + help=""" + Number of frames to pad at the end. + """, + ) + + add_model_arguments(parser) + + return parser + + +def get_params() -> AttributeDict: + """Return a dict containing training parameters. + + All training related parameters that are not passed from the commandline + are saved in the variable `params`. + + Commandline options are merged into `params` after they are parsed, so + you can also access them via `params`. + + Explanation of options saved in `params`: + + - best_train_loss: Best training loss so far. It is used to select + the model that has the lowest training loss. It is + updated during the training. + + - best_valid_loss: Best validation loss so far. It is used to select + the model that has the lowest validation loss. It is + updated during the training. + + - best_train_epoch: It is the epoch that has the best training loss. + + - best_valid_epoch: It is the epoch that has the best validation loss. + + - batch_idx_train: Used to writing statistics to tensorboard. It + contains number of batches trained so far across + epochs. + + - log_interval: Print training loss if batch_idx % log_interval` is 0 + + - reset_interval: Reset statistics if batch_idx % reset_interval is 0 + + - valid_interval: Run validation if batch_idx % valid_interval is 0 + + - feature_dim: The model input dim. It has to match the one used + in computing features. + + - subsampling_factor: The subsampling factor for the model. + + - encoder_dim: Hidden dim for multi-head attention model. + + - num_decoder_layers: Number of decoder layer of transformer decoder. + + - warm_step: The warmup period that dictates the decay of the + scale on "simple" (un-pruned) loss. + """ + params = AttributeDict( + { + "best_train_loss": float("inf"), + "best_valid_loss": float("inf"), + "best_train_epoch": -1, + "best_valid_epoch": -1, + "batch_idx_train": 0, + "log_interval": 50, + "reset_interval": 200, + "valid_interval": 1000, # For the 100h subset, use 800 + # parameters for zipformer + "feature_dim": 80, + "subsampling_factor": 4, # not passed in, this is fixed. + "warm_step": 2000, + "env_info": get_env_info(), + } + ) + + return params + + +def get_encoder_model(params: AttributeDict) -> nn.Module: + # TODO: We can add an option to switch between Zipformer and Transformer + def to_int_tuple(s: str): + return tuple(map(int, s.split(","))) + + encoder = Zipformer( + num_features=params.feature_dim, + output_downsampling_factor=2, + zipformer_downsampling_factors=to_int_tuple( + params.zipformer_downsampling_factors + ), + encoder_dims=to_int_tuple(params.encoder_dims), + attention_dim=to_int_tuple(params.attention_dims), + encoder_unmasked_dims=to_int_tuple(params.encoder_unmasked_dims), + nhead=to_int_tuple(params.nhead), + feedforward_dim=to_int_tuple(params.feedforward_dims), + cnn_module_kernels=to_int_tuple(params.cnn_module_kernels), + num_encoder_layers=to_int_tuple(params.num_encoder_layers), + num_left_chunks=params.num_left_chunks, + short_chunk_size=params.short_chunk_size, + decode_chunk_size=params.decode_chunk_len // 2, + is_pnnx=True, + ) + return encoder + + +def get_decoder_model(params: AttributeDict) -> nn.Module: + decoder = Decoder( + vocab_size=params.vocab_size, + decoder_dim=params.decoder_dim, + blank_id=params.blank_id, + context_size=params.context_size, + ) + return decoder + + +def get_joiner_model(params: AttributeDict) -> nn.Module: + joiner = Joiner( + encoder_dim=int(params.encoder_dims.split(",")[-1]), + decoder_dim=params.decoder_dim, + joiner_dim=params.joiner_dim, + vocab_size=params.vocab_size, + ) + return joiner + + +def get_transducer_model(params: AttributeDict) -> nn.Module: + encoder = get_encoder_model(params) + decoder = get_decoder_model(params) + joiner = get_joiner_model(params) + + model = Transducer( + encoder=encoder, + decoder=decoder, + joiner=joiner, + encoder_dim=int(params.encoder_dims.split(",")[-1]), + decoder_dim=params.decoder_dim, + joiner_dim=params.joiner_dim, + vocab_size=params.vocab_size, + ) + return model + + +def load_checkpoint_if_available( + params: AttributeDict, + model: nn.Module, + model_avg: nn.Module = None, + optimizer: Optional[torch.optim.Optimizer] = None, + scheduler: Optional[LRSchedulerType] = None, +) -> Optional[Dict[str, Any]]: + """Load checkpoint from file. + + If params.start_batch is positive, it will load the checkpoint from + `params.exp_dir/checkpoint-{params.start_batch}.pt`. Otherwise, if + params.start_epoch is larger than 1, it will load the checkpoint from + `params.start_epoch - 1`. + + Apart from loading state dict for `model` and `optimizer` it also updates + `best_train_epoch`, `best_train_loss`, `best_valid_epoch`, + and `best_valid_loss` in `params`. + + Args: + params: + The return value of :func:`get_params`. + model: + The training model. + model_avg: + The stored model averaged from the start of training. + optimizer: + The optimizer that we are using. + scheduler: + The scheduler that we are using. + Returns: + Return a dict containing previously saved training info. + """ + if params.start_batch > 0: + filename = params.exp_dir / f"checkpoint-{params.start_batch}.pt" + elif params.start_epoch > 1: + filename = params.exp_dir / f"epoch-{params.start_epoch-1}.pt" + else: + return None + + assert filename.is_file(), f"{filename} does not exist!" + + saved_params = load_checkpoint( + filename, + model=model, + model_avg=model_avg, + optimizer=optimizer, + scheduler=scheduler, + ) + + keys = [ + "best_train_epoch", + "best_valid_epoch", + "batch_idx_train", + "best_train_loss", + "best_valid_loss", + ] + for k in keys: + params[k] = saved_params[k] + + if params.start_batch > 0: + if "cur_epoch" in saved_params: + params["start_epoch"] = saved_params["cur_epoch"] + + return saved_params + + +def save_checkpoint( + params: AttributeDict, + model: Union[nn.Module, DDP], + model_avg: Optional[nn.Module] = None, + optimizer: Optional[torch.optim.Optimizer] = None, + scheduler: Optional[LRSchedulerType] = None, + sampler: Optional[CutSampler] = None, + scaler: Optional[GradScaler] = None, + rank: int = 0, +) -> None: + """Save model, optimizer, scheduler and training stats to file. + + Args: + params: + It is returned by :func:`get_params`. + model: + The training model. + model_avg: + The stored model averaged from the start of training. + optimizer: + The optimizer used in the training. + sampler: + The sampler for the training dataset. + scaler: + The scaler used for mix precision training. + """ + if rank != 0: + return + filename = params.exp_dir / f"epoch-{params.cur_epoch}.pt" + save_checkpoint_impl( + filename=filename, + model=model, + model_avg=model_avg, + params=params, + optimizer=optimizer, + scheduler=scheduler, + sampler=sampler, + scaler=scaler, + rank=rank, + ) + + if params.best_train_epoch == params.cur_epoch: + best_train_filename = params.exp_dir / "best-train-loss.pt" + copyfile(src=filename, dst=best_train_filename) + + if params.best_valid_epoch == params.cur_epoch: + best_valid_filename = params.exp_dir / "best-valid-loss.pt" + copyfile(src=filename, dst=best_valid_filename) + + +def compute_loss( + params: AttributeDict, + model: Union[nn.Module, DDP], + sp: Tokenizer, + batch: dict, + is_training: bool, +) -> Tuple[Tensor, MetricsTracker]: + """ + Compute transducer loss given the model and its inputs. + + Args: + params: + Parameters for training. See :func:`get_params`. + model: + The model for training. It is an instance of Zipformer in our case. + batch: + A batch of data. See `lhotse.dataset.K2SpeechRecognitionDataset()` + for the content in it. + is_training: + True for training. False for validation. When it is True, this + function enables autograd during computation; when it is False, it + disables autograd. + warmup: a floating point value which increases throughout training; + values >= 1.0 are fully warmed up and have all modules present. + """ + device = model.device if isinstance(model, DDP) else next(model.parameters()).device + feature = batch["inputs"] + # at entry, feature is (N, T, C) + assert feature.ndim == 3 + feature = feature.to(device) + + supervisions = batch["supervisions"] + feature_lens = supervisions["num_frames"].to(device) + + if params.pad_feature: + feature_lens += params.pad_feature + feature = torch.nn.functional.pad( + feature, + pad=(0, 0, 0, params.pad_feature), + value=LOG_EPS, + ) + + batch_idx_train = params.batch_idx_train + warm_step = params.warm_step + + texts = batch["supervisions"]["text"] + y = sp.encode(texts, out_type=int) + y = k2.RaggedTensor(y).to(device) + + with torch.set_grad_enabled(is_training): + simple_loss, pruned_loss = model( + x=feature, + x_lens=feature_lens, + y=y, + prune_range=params.prune_range, + am_scale=params.am_scale, + lm_scale=params.lm_scale, + ) + + s = params.simple_loss_scale + # take down the scale on the simple loss from 1.0 at the start + # to params.simple_loss scale by warm_step. + simple_loss_scale = ( + s + if batch_idx_train >= warm_step + else 1.0 - (batch_idx_train / warm_step) * (1.0 - s) + ) + pruned_loss_scale = ( + 1.0 + if batch_idx_train >= warm_step + else 0.1 + 0.9 * (batch_idx_train / warm_step) + ) + + loss = simple_loss_scale * simple_loss + pruned_loss_scale * pruned_loss + + assert loss.requires_grad == is_training + + info = MetricsTracker() + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + info["frames"] = (feature_lens // params.subsampling_factor).sum().item() + + # Note: We use reduction=sum while computing the loss. + info["loss"] = loss.detach().cpu().item() + info["simple_loss"] = simple_loss.detach().cpu().item() + info["pruned_loss"] = pruned_loss.detach().cpu().item() + + return loss, info + + +def compute_validation_loss( + params: AttributeDict, + model: Union[nn.Module, DDP], + sp: Tokenizer, + valid_dl: torch.utils.data.DataLoader, + world_size: int = 1, +) -> MetricsTracker: + """Run the validation process.""" + model.eval() + + tot_loss = MetricsTracker() + + for batch_idx, batch in enumerate(valid_dl): + loss, loss_info = compute_loss( + params=params, + model=model, + sp=sp, + batch=batch, + is_training=False, + ) + assert loss.requires_grad is False + tot_loss = tot_loss + loss_info + + if world_size > 1: + tot_loss.reduce(loss.device) + + loss_value = tot_loss["loss"] / tot_loss["frames"] + if loss_value < params.best_valid_loss: + params.best_valid_epoch = params.cur_epoch + params.best_valid_loss = loss_value + + return tot_loss + + +def train_one_epoch( + params: AttributeDict, + model: Union[nn.Module, DDP], + optimizer: torch.optim.Optimizer, + scheduler: LRSchedulerType, + sp: Tokenizer, + train_dl: torch.utils.data.DataLoader, + valid_dl: torch.utils.data.DataLoader, + scaler: GradScaler, + model_avg: Optional[nn.Module] = None, + tb_writer: Optional[SummaryWriter] = None, + world_size: int = 1, + rank: int = 0, +) -> None: + """Train the model for one epoch. + + The training loss from the mean of all frames is saved in + `params.train_loss`. It runs the validation process every + `params.valid_interval` batches. + + Args: + params: + It is returned by :func:`get_params`. + model: + The model for training. + optimizer: + The optimizer we are using. + scheduler: + The learning rate scheduler, we call step() every step. + train_dl: + Dataloader for the training dataset. + valid_dl: + Dataloader for the validation dataset. + scaler: + The scaler used for mix precision training. + model_avg: + The stored model averaged from the start of training. + tb_writer: + Writer to write log messages to tensorboard. + world_size: + Number of nodes in DDP training. If it is 1, DDP is disabled. + rank: + The rank of the node in DDP training. If no DDP is used, it should + be set to 0. + """ + model.train() + + tot_loss = MetricsTracker() + + for batch_idx, batch in enumerate(train_dl): + params.batch_idx_train += 1 + batch_size = len(batch["supervisions"]["text"]) + + try: + with torch.cuda.amp.autocast(enabled=params.use_fp16): + loss, loss_info = compute_loss( + params=params, + model=model, + sp=sp, + batch=batch, + is_training=True, + ) + # summary stats + tot_loss = (tot_loss * (1 - 1 / params.reset_interval)) + loss_info + + # NOTE: We use reduction==sum and loss is computed over utterances + # in the batch and there is no normalization to it so far. + scaler.scale(loss).backward() + set_batch_count(model, params.batch_idx_train) + scheduler.step_batch(params.batch_idx_train) + + scaler.step(optimizer) + scaler.update() + optimizer.zero_grad() + except Exception as e: # noqa + logging.error(e, exc_info=True) + display_and_save_batch(batch, params=params, sp=sp) + raise e + + if params.print_diagnostics and batch_idx == 5: + return + + if ( + rank == 0 + and params.batch_idx_train > 0 + and params.batch_idx_train % params.average_period == 0 + ): + update_averaged_model( + params=params, + model_cur=model, + model_avg=model_avg, + ) + + if ( + params.batch_idx_train > 0 + and params.batch_idx_train % params.save_every_n == 0 + ): + save_checkpoint_with_global_batch_idx( + out_dir=params.exp_dir, + global_batch_idx=params.batch_idx_train, + model=model, + model_avg=model_avg, + params=params, + optimizer=optimizer, + scheduler=scheduler, + sampler=train_dl.sampler, + scaler=scaler, + rank=rank, + ) + remove_checkpoints( + out_dir=params.exp_dir, + topk=params.keep_last_k, + rank=rank, + ) + + if batch_idx % 100 == 0 and params.use_fp16: + # If the grad scale was less than 1, try increasing it. The _growth_interval + # of the grad scaler is configurable, but we can't configure it to have different + # behavior depending on the current grad scale. + cur_grad_scale = scaler._scale.item() + if cur_grad_scale < 1.0 or (cur_grad_scale < 8.0 and batch_idx % 400 == 0): + scaler.update(cur_grad_scale * 2.0) + if cur_grad_scale < 0.01: + logging.warning(f"Grad scale is small: {cur_grad_scale}") + if cur_grad_scale < 1.0e-05: + raise RuntimeError( + f"grad_scale is too small, exiting: {cur_grad_scale}" + ) + + if batch_idx % params.log_interval == 0: + cur_lr = scheduler.get_last_lr()[0] + cur_grad_scale = scaler._scale.item() if params.use_fp16 else 1.0 + + logging.info( + f"Epoch {params.cur_epoch}, " + f"batch {batch_idx}, loss[{loss_info}], " + f"tot_loss[{tot_loss}], batch size: {batch_size}, " + f"lr: {cur_lr:.2e}, " + + (f"grad_scale: {scaler._scale.item()}" if params.use_fp16 else "") + ) + + if tb_writer is not None: + tb_writer.add_scalar( + "train/learning_rate", cur_lr, params.batch_idx_train + ) + + loss_info.write_summary( + tb_writer, "train/current_", params.batch_idx_train + ) + tot_loss.write_summary(tb_writer, "train/tot_", params.batch_idx_train) + if params.use_fp16: + tb_writer.add_scalar( + "train/grad_scale", + cur_grad_scale, + params.batch_idx_train, + ) + + if batch_idx % params.valid_interval == 0 and not params.print_diagnostics: + logging.info("Computing validation loss") + valid_info = compute_validation_loss( + params=params, + model=model, + sp=sp, + valid_dl=valid_dl, + world_size=world_size, + ) + model.train() + log_mode = logging.info + log_mode(f"Epoch {params.cur_epoch}, validation: {valid_info}") + log_mode( + f"Maximum memory allocated so far is {torch.cuda.max_memory_allocated()//1000000}MB" + ) + if tb_writer is not None: + valid_info.write_summary( + tb_writer, "train/valid_", params.batch_idx_train + ) + + loss_value = tot_loss["loss"] / tot_loss["frames"] + params.train_loss = loss_value + if params.train_loss < params.best_train_loss: + params.best_train_epoch = params.cur_epoch + params.best_train_loss = params.train_loss + + +def run(rank, world_size, args): + """ + Args: + rank: + It is a value between 0 and `world_size-1`, which is + passed automatically by `mp.spawn()` in :func:`main`. + The node with rank 0 is responsible for saving checkpoint. + world_size: + Number of GPUs for DDP training. + args: + The return value of get_parser().parse_args() + """ + params = get_params() + params.update(vars(args)) + + fix_random_seed(params.seed) + if world_size > 1: + setup_dist(rank, world_size, master_port=params.master_port) + + setup_logger(f"{params.exp_dir}/log/log-train") + logging.info("Training started") + + if args.tensorboard and rank == 0: + tb_writer = SummaryWriter(log_dir=f"{params.exp_dir}/tensorboard") + else: + tb_writer = None + + device = torch.device("cpu") + if torch.cuda.is_available(): + device = torch.device("cuda", rank) + logging.info(f"Device: {device}") + + sp = Tokenizer.load(args.lang, args.lang_type) + + # is defined in local/prepare_lang_char.py + params.blank_id = sp.piece_to_id("") + params.vocab_size = sp.get_piece_size() + + logging.info(params) + + logging.info("About to create model") + model = get_transducer_model(params) + + num_param = sum([p.numel() for p in model.parameters()]) + logging.info(f"Number of model parameters: {num_param}") + + assert params.save_every_n >= params.average_period + model_avg: Optional[nn.Module] = None + if rank == 0: + # model_avg is only used with rank 0 + model_avg = copy.deepcopy(model).to(torch.float64) + + assert params.start_epoch > 0, params.start_epoch + checkpoints = load_checkpoint_if_available( + params=params, model=model, model_avg=model_avg + ) + + model.to(device) + if world_size > 1: + logging.info("Using DDP") + model = DDP(model, device_ids=[rank], find_unused_parameters=True) + + parameters_names = [] + parameters_names.append( + [name_param_pair[0] for name_param_pair in model.named_parameters()] + ) + optimizer = ScaledAdam( + model.parameters(), + lr=params.base_lr, + clipping_scale=2.0, + parameters_names=parameters_names, + ) + + scheduler = Eden(optimizer, params.lr_batches, params.lr_epochs) + + if checkpoints and "optimizer" in checkpoints: + logging.info("Loading optimizer state dict") + optimizer.load_state_dict(checkpoints["optimizer"]) + + if ( + checkpoints + and "scheduler" in checkpoints + and checkpoints["scheduler"] is not None + ): + logging.info("Loading scheduler state dict") + scheduler.load_state_dict(checkpoints["scheduler"]) + + if params.print_diagnostics: + opts = diagnostics.TensorDiagnosticOptions( + 512 + ) # allow 4 megabytes per sub-module + diagnostic = diagnostics.attach_diagnostics(model, opts) + + if params.inf_check: + register_inf_check_hooks(model) + + def remove_short_and_long_utt(c: Cut): + # Keep only utterances with duration between 1 second and 20 seconds + # + # Caution: There is a reason to select 20.0 here. Please see + # ../local/display_manifest_statistics.py + # + # You should use ../local/display_manifest_statistics.py to get + # an utterance duration distribution for your dataset to select + # the threshold + if c.duration < 0.3 or c.duration > 30.0: + logging.debug( + f"Exclude cut with ID {c.id} from training. Duration: {c.duration}" + ) + return False + + # In pruned RNN-T, we require that T >= S + # where T is the number of feature frames after subsampling + # and S is the number of tokens in the utterance + + # In ./zipformer.py, the conv module uses the following expression + # for subsampling + T = ((c.num_frames - 7) // 2 + 1) // 2 + tokens = sp.encode(c.supervisions[0].text, out_type=str) + + if T < len(tokens): + logging.info( + f"Exclude cut with ID {c.id} from training. " + f"Number of frames (before subsampling): {c.num_frames}. " + f"Number of frames (after subsampling): {T}. " + f"Text: {c.supervisions[0].text}. " + f"Tokens: {tokens}. " + f"Number of tokens: {len(tokens)}" + ) + return False + + return True + + reazonspeech_corpus = ReazonSpeechAsrDataModule(args) + train_cuts = reazonspeech_corpus.train_cuts() + + train_cuts = train_cuts.filter(remove_short_and_long_utt) + + if params.start_batch > 0 and checkpoints and "sampler" in checkpoints: + # We only load the sampler's state dict when it loads a checkpoint + # saved in the middle of an epoch + sampler_state_dict = checkpoints["sampler"] + else: + sampler_state_dict = None + + train_dl = reazonspeech_corpus.train_dataloaders( + train_cuts, sampler_state_dict=sampler_state_dict + ) + + valid_cuts = reazonspeech_corpus.valid_cuts() + valid_dl = reazonspeech_corpus.valid_dataloaders(valid_cuts) + + if params.start_batch <= 0 and not params.print_diagnostics: + scan_pessimistic_batches_for_oom( + model=model, + train_dl=train_dl, + optimizer=optimizer, + sp=sp, + params=params, + ) + + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + if checkpoints and "grad_scaler" in checkpoints: + logging.info("Loading grad scaler state dict") + scaler.load_state_dict(checkpoints["grad_scaler"]) + + for epoch in range(params.start_epoch, params.num_epochs + 1): + scheduler.step_epoch(epoch - 1) + fix_random_seed(params.seed + epoch - 1) + train_dl.sampler.set_epoch(epoch - 1) + + if tb_writer is not None: + tb_writer.add_scalar("train/epoch", epoch, params.batch_idx_train) + + params.cur_epoch = epoch + + train_one_epoch( + params=params, + model=model, + model_avg=model_avg, + optimizer=optimizer, + scheduler=scheduler, + sp=sp, + train_dl=train_dl, + valid_dl=valid_dl, + scaler=scaler, + tb_writer=tb_writer, + world_size=world_size, + rank=rank, + ) + + if params.print_diagnostics: + diagnostic.print_diagnostics() + break + + save_checkpoint( + params=params, + model=model, + model_avg=model_avg, + optimizer=optimizer, + scheduler=scheduler, + sampler=train_dl.sampler, + scaler=scaler, + rank=rank, + ) + + logging.info("Done!") + + if world_size > 1: + torch.distributed.barrier() + cleanup_dist() + + +def display_and_save_batch( + batch: dict, + params: AttributeDict, + sp: Tokenizer, +) -> None: + """Display the batch statistics and save the batch into disk. + + Args: + batch: + A batch of data. See `lhotse.dataset.K2SpeechRecognitionDataset()` + for the content in it. + params: + Parameters for training. See :func:`get_params`. + sp: + The BPE model. + """ + from lhotse.utils import uuid4 + + filename = f"{params.exp_dir}/batch-{uuid4()}.pt" + logging.info(f"Saving batch to {filename}") + torch.save(batch, filename) + + supervisions = batch["supervisions"] + features = batch["inputs"] + + logging.info(f"features shape: {features.shape}") + + y = sp.encode(supervisions["text"], out_type=int) + num_tokens = sum(len(i) for i in y) + logging.info(f"num tokens: {num_tokens}") + + +def scan_pessimistic_batches_for_oom( + model: Union[nn.Module, DDP], + train_dl: torch.utils.data.DataLoader, + optimizer: torch.optim.Optimizer, + sp: Tokenizer, + params: AttributeDict, +): + from lhotse.dataset import find_pessimistic_batches + + logging.info( + "Sanity check -- see if any of the batches in epoch 1 would cause OOM." + ) + batches, crit_values = find_pessimistic_batches(train_dl.sampler) + for criterion, cuts in batches.items(): + batch = train_dl.dataset[cuts] + try: + with torch.cuda.amp.autocast(enabled=params.use_fp16): + loss, _ = compute_loss( + params=params, + model=model, + sp=sp, + batch=batch, + is_training=True, + ) + loss.backward() + optimizer.zero_grad() + except Exception as e: + if "CUDA out of memory" in str(e): + logging.error( + "Your GPU ran out of memory with the current " + "max_duration setting. We recommend decreasing " + "max_duration and trying again.\n" + f"Failing criterion: {criterion} " + f"(={crit_values[criterion]}) ..." + ) + display_and_save_batch(batch, params=params, sp=sp) + raise + logging.info( + f"Maximum memory allocated so far is {torch.cuda.max_memory_allocated()//1000000}MB" + ) + + +def main(): + raise RuntimeError("Please don't use this file directly!") + parser = get_parser() + ReazonSpeechAsrDataModule.add_arguments(parser) + Tokenizer.add_arguments(parser) + args = parser.parse_args() + + world_size = args.world_size + assert world_size >= 1 + if world_size > 1: + mp.spawn(run, args=(world_size, args), nprocs=world_size, join=True) + else: + run(rank=0, world_size=1, args=args) + + +torch.set_num_threads(1) +torch.set_num_interop_threads(1) + +if __name__ == "__main__": + main() diff --git a/egs/multi_ja_en/ASR/zipformer/encoder_interface.py b/egs/multi_ja_en/ASR/zipformer/encoder_interface.py new file mode 120000 index 000000000..c2eaca671 --- /dev/null +++ b/egs/multi_ja_en/ASR/zipformer/encoder_interface.py @@ -0,0 +1 @@ +../../../librispeech/ASR/zipformer/encoder_interface.py \ No newline at end of file diff --git a/egs/multi_ja_en/ASR/zipformer/export-onnx.py b/egs/multi_ja_en/ASR/zipformer/export-onnx.py new file mode 120000 index 000000000..70a15683c --- /dev/null +++ b/egs/multi_ja_en/ASR/zipformer/export-onnx.py @@ -0,0 +1 @@ +../../../librispeech/ASR/zipformer/export-onnx.py \ No newline at end of file diff --git a/egs/multi_ja_en/ASR/zipformer/export.py b/egs/multi_ja_en/ASR/zipformer/export.py new file mode 120000 index 000000000..dfc1bec08 --- /dev/null +++ b/egs/multi_ja_en/ASR/zipformer/export.py @@ -0,0 +1 @@ +../../../librispeech/ASR/zipformer/export.py \ No newline at end of file diff --git a/egs/multi_ja_en/ASR/zipformer/generate_averaged_model.py b/egs/multi_ja_en/ASR/zipformer/generate_averaged_model.py new file mode 120000 index 000000000..5a015ee6c --- /dev/null +++ b/egs/multi_ja_en/ASR/zipformer/generate_averaged_model.py @@ -0,0 +1 @@ +../../../librispeech/ASR/zipformer/generate_averaged_model.py \ No newline at end of file diff --git a/egs/multi_ja_en/ASR/zipformer/joiner.py b/egs/multi_ja_en/ASR/zipformer/joiner.py new file mode 120000 index 000000000..5b8a36332 --- /dev/null +++ b/egs/multi_ja_en/ASR/zipformer/joiner.py @@ -0,0 +1 @@ +../../../librispeech/ASR/zipformer/joiner.py \ No newline at end of file diff --git a/egs/multi_ja_en/ASR/zipformer/model.py b/egs/multi_ja_en/ASR/zipformer/model.py new file mode 120000 index 000000000..cd7e07d72 --- /dev/null +++ b/egs/multi_ja_en/ASR/zipformer/model.py @@ -0,0 +1 @@ +../../../librispeech/ASR/zipformer/model.py \ No newline at end of file diff --git a/egs/multi_ja_en/ASR/zipformer/multi_dataset.py b/egs/multi_ja_en/ASR/zipformer/multi_dataset.py new file mode 100644 index 000000000..b0cdc1f6a --- /dev/null +++ b/egs/multi_ja_en/ASR/zipformer/multi_dataset.py @@ -0,0 +1,143 @@ +import argparse +import logging +from functools import lru_cache +from pathlib import Path +from typing import Dict + +from lhotse import CutSet, load_manifest_lazy + + +class MultiDataset: + def __init__(self, args: argparse.Namespace): + """ + Args: + manifest_dir: + It is expected to contain the following files: + - reazonspeech_cuts_train.jsonl.gz + - librispeech_cuts_train-clean-100.jsonl.gz + - librispeech_cuts_train-clean-360.jsonl.gz + - librispeech_cuts_train-other-500.jsonl.gz + """ + self.fbank_dir = Path(args.manifest_dir) + + def train_cuts(self) -> CutSet: + logging.info("About to get multidataset train cuts") + + logging.info("Loading Reazonspeech in lazy mode") + reazonspeech_cuts = load_manifest_lazy( + self.fbank_dir / "reazonspeech_cuts_train.jsonl.gz" + ) + + logging.info("Loading LibriSpeech in lazy mode") + train_clean_100_cuts = self.train_clean_100_cuts() + train_clean_360_cuts = self.train_clean_360_cuts() + train_other_500_cuts = self.train_other_500_cuts() + + return CutSet.mux( + reazonspeech_cuts, + train_clean_100_cuts, + train_clean_360_cuts, + train_other_500_cuts, + weights=[ + len(reazonspeech_cuts), + len(train_clean_100_cuts), + len(train_clean_360_cuts), + len(train_other_500_cuts), + ], + ) + + def dev_cuts(self) -> CutSet: + logging.info("About to get multidataset dev cuts") + + logging.info("Loading Reazonspeech DEV set in lazy mode") + reazonspeech_dev_cuts = load_manifest_lazy( + self.fbank_dir / "reazonspeech_cuts_dev.jsonl.gz" + ) + + logging.info("Loading LibriSpeech DEV set in lazy mode") + dev_clean_cuts = self.dev_clean_cuts() + dev_other_cuts = self.dev_other_cuts() + + return CutSet.mux( + reazonspeech_dev_cuts, + dev_clean_cuts, + dev_other_cuts, + weights=[ + len(reazonspeech_dev_cuts), + len(dev_clean_cuts), + len(dev_other_cuts), + ], + ) + + def test_cuts(self) -> Dict[str, CutSet]: + logging.info("About to get multidataset test cuts") + + logging.info("Loading Reazonspeech set in lazy mode") + reazonspeech_test_cuts = load_manifest_lazy( + self.fbank_dir / "reazonspeech_cuts_test.jsonl.gz" + ) + reazonspeech_dev_cuts = load_manifest_lazy( + self.fbank_dir / "reazonspeech_cuts_dev.jsonl.gz" + ) + + logging.info("Loading LibriSpeech set in lazy mode") + test_clean_cuts = self.test_clean_cuts() + test_other_cuts = self.test_other_cuts() + + test_cuts = { + "reazonspeech_test": reazonspeech_test_cuts, + "reazonspeech_dev": reazonspeech_dev_cuts, + "librispeech_test_clean": test_clean_cuts, + "librispeech_test_other": test_other_cuts, + } + + return test_cuts + + @lru_cache() + def train_clean_100_cuts(self) -> CutSet: + logging.info("About to get train-clean-100 cuts") + return load_manifest_lazy( + self.fbank_dir / "librispeech_cuts_train-clean-100.jsonl.gz" + ) + + @lru_cache() + def train_clean_360_cuts(self) -> CutSet: + logging.info("About to get train-clean-360 cuts") + return load_manifest_lazy( + self.fbank_dir / "librispeech_cuts_train-clean-360.jsonl.gz" + ) + + @lru_cache() + def train_other_500_cuts(self) -> CutSet: + logging.info("About to get train-other-500 cuts") + return load_manifest_lazy( + self.fbank_dir / "librispeech_cuts_train-other-500.jsonl.gz" + ) + + @lru_cache() + def dev_clean_cuts(self) -> CutSet: + logging.info("About to get dev-clean cuts") + return load_manifest_lazy( + self.fbank_dir / "librispeech_cuts_dev-clean.jsonl.gz" + ) + + @lru_cache() + def dev_other_cuts(self) -> CutSet: + logging.info("About to get dev-other cuts") + return load_manifest_lazy( + self.fbank_dir / "librispeech_cuts_dev-other.jsonl.gz" + ) + + @lru_cache() + def test_clean_cuts(self) -> CutSet: + logging.info("About to get test-clean cuts") + return load_manifest_lazy( + self.fbank_dir / "librispeech_cuts_test-clean.jsonl.gz" + ) + + @lru_cache() + def test_other_cuts(self) -> CutSet: + logging.info("About to get test-other cuts") + return load_manifest_lazy( + self.fbank_dir / "librispeech_cuts_test-other.jsonl.gz" + ) diff --git a/egs/multi_ja_en/ASR/zipformer/my_profile.py b/egs/multi_ja_en/ASR/zipformer/my_profile.py new file mode 120000 index 000000000..3a90b2628 --- /dev/null +++ b/egs/multi_ja_en/ASR/zipformer/my_profile.py @@ -0,0 +1 @@ +../../../librispeech/ASR/zipformer/my_profile.py \ No newline at end of file diff --git a/egs/multi_ja_en/ASR/zipformer/onnx_decode.py b/egs/multi_ja_en/ASR/zipformer/onnx_decode.py new file mode 120000 index 000000000..0573b88c5 --- /dev/null +++ b/egs/multi_ja_en/ASR/zipformer/onnx_decode.py @@ -0,0 +1 @@ +../../../librispeech/ASR/zipformer/onnx_decode.py \ No newline at end of file diff --git a/egs/multi_ja_en/ASR/zipformer/onnx_pretrained.py b/egs/multi_ja_en/ASR/zipformer/onnx_pretrained.py new file mode 120000 index 000000000..8f32f4ee7 --- /dev/null +++ b/egs/multi_ja_en/ASR/zipformer/onnx_pretrained.py @@ -0,0 +1 @@ +../../../librispeech/ASR/zipformer/onnx_pretrained.py \ No newline at end of file diff --git a/egs/multi_ja_en/ASR/zipformer/optim.py b/egs/multi_ja_en/ASR/zipformer/optim.py new file mode 120000 index 000000000..5eaa3cffd --- /dev/null +++ b/egs/multi_ja_en/ASR/zipformer/optim.py @@ -0,0 +1 @@ +../../../librispeech/ASR/zipformer/optim.py \ No newline at end of file diff --git a/egs/multi_ja_en/ASR/zipformer/pretrained.py b/egs/multi_ja_en/ASR/zipformer/pretrained.py new file mode 120000 index 000000000..0bd71dde4 --- /dev/null +++ b/egs/multi_ja_en/ASR/zipformer/pretrained.py @@ -0,0 +1 @@ +../../../librispeech/ASR/zipformer/pretrained.py \ No newline at end of file diff --git a/egs/multi_ja_en/ASR/zipformer/scaling.py b/egs/multi_ja_en/ASR/zipformer/scaling.py new file mode 120000 index 000000000..6f398f431 --- /dev/null +++ b/egs/multi_ja_en/ASR/zipformer/scaling.py @@ -0,0 +1 @@ +../../../librispeech/ASR/zipformer/scaling.py \ No newline at end of file diff --git a/egs/multi_ja_en/ASR/zipformer/scaling_converter.py b/egs/multi_ja_en/ASR/zipformer/scaling_converter.py new file mode 120000 index 000000000..b0ecee05e --- /dev/null +++ b/egs/multi_ja_en/ASR/zipformer/scaling_converter.py @@ -0,0 +1 @@ +../../../librispeech/ASR/zipformer/scaling_converter.py \ No newline at end of file diff --git a/egs/multi_ja_en/ASR/zipformer/streaming_beam_search.py b/egs/multi_ja_en/ASR/zipformer/streaming_beam_search.py new file mode 120000 index 000000000..b1ed54557 --- /dev/null +++ b/egs/multi_ja_en/ASR/zipformer/streaming_beam_search.py @@ -0,0 +1 @@ +../../../librispeech/ASR/zipformer/streaming_beam_search.py \ No newline at end of file diff --git a/egs/multi_ja_en/ASR/zipformer/streaming_decode.py b/egs/multi_ja_en/ASR/zipformer/streaming_decode.py new file mode 100755 index 000000000..935f86de1 --- /dev/null +++ b/egs/multi_ja_en/ASR/zipformer/streaming_decode.py @@ -0,0 +1,935 @@ +#!/usr/bin/env python3 +# Copyright 2022-2023 Xiaomi Corporation (Authors: Wei Kang, +# Fangjun Kuang, +# Zengwei Yao) +# See ../../../../LICENSE for clarification regarding multiple authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Usage: + +Monolingual: +./zipformer/streaming_decode.py \ + --epoch 28 \ + --avg 15 \ + --causal 1 \ + --chunk-size 32 \ + --left-context-frames 256 \ + --exp-dir ./zipformer/exp-large \ + --lang data/lang_char \ + --num-encoder-layers 2,2,4,5,4,2 \ + --feedforward-dim 512,768,1536,2048,1536,768 \ + --encoder-dim 192,256,512,768,512,256 \ + --encoder-unmasked-dim 192,192,256,320,256,192 + +Bilingual: +./zipformer/streaming_decode.py \ + --bilingual 1 \ + --epoch 28 \ + --avg 15 \ + --causal 1 \ + --chunk-size 32 \ + --left-context-frames 256 \ + --exp-dir ./zipformer/exp-large \ + --lang data/lang_char \ + --num-encoder-layers 2,2,4,5,4,2 \ + --feedforward-dim 512,768,1536,2048,1536,768 \ + --encoder-dim 192,256,512,768,512,256 \ + --encoder-unmasked-dim 192,192,256,320,256,192 \ + +""" + +import argparse +import logging +import math +import os +import pdb +import subprocess as sp +from pathlib import Path +from typing import Dict, List, Optional, Tuple + +import k2 +import numpy as np +import sentencepiece as spm +import torch +from asr_datamodule import ReazonSpeechAsrDataModule +from decode_stream import DecodeStream +from kaldifeat import Fbank, FbankOptions +from lhotse import CutSet +from lhotse.cut import Cut +from multi_dataset import MultiDataset +from streaming_beam_search import ( + fast_beam_search_one_best, + greedy_search, + modified_beam_search, +) +from tokenizer import Tokenizer +from torch import Tensor, nn +from torch.nn.utils.rnn import pad_sequence +from train import add_model_arguments, get_model, get_params + +from icefall.checkpoint import ( + average_checkpoints, + average_checkpoints_with_averaged_model, + find_checkpoints, + load_checkpoint, +) +from icefall.utils import ( + AttributeDict, + make_pad_mask, + setup_logger, + store_transcripts, + str2bool, + write_error_stats, +) + +LOG_EPS = math.log(1e-10) + + +def get_parser(): + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + + parser.add_argument( + "--bilingual", + type=str2bool, + default=False, + help="Whether the model is bilingual or not. 1 = bilingual.", + ) + + parser.add_argument( + "--epoch", + type=int, + default=28, + help="""It specifies the checkpoint to use for decoding. + Note: Epoch counts from 1. + You can specify --avg to use more checkpoints for model averaging.""", + ) + + parser.add_argument( + "--iter", + type=int, + default=0, + help="""If positive, --epoch is ignored and it + will use the checkpoint exp_dir/checkpoint-iter.pt. + You can specify --avg to use more checkpoints for model averaging. + """, + ) + + parser.add_argument( + "--avg", + type=int, + default=15, + help="Number of checkpoints to average. Automatically select " + "consecutive checkpoints before the checkpoint specified by " + "'--epoch' and '--iter'", + ) + + parser.add_argument( + "--use-averaged-model", + type=str2bool, + default=True, + help="Whether to load averaged model. Currently it only supports " + "using --epoch. If True, it would decode with the averaged model " + "over the epoch range from `epoch-avg` (excluded) to `epoch`." + "Actually only the models with epoch number of `epoch-avg` and " + "`epoch` are loaded for averaging. ", + ) + + parser.add_argument( + "--exp-dir", + type=str, + default="zipformer/exp", + help="The experiment dir", + ) + + parser.add_argument( + "--bpe-model", + type=str, + default="data/lang_bpe_500/bpe.model", + help="Path to the BPE model", + ) + + parser.add_argument( + "--lang-dir", + type=Path, + default="data/lang_char", + help="The lang dir containing word table and LG graph", + ) + + parser.add_argument( + "--decoding-method", + type=str, + default="greedy_search", + help="""Supported decoding methods are: + greedy_search + modified_beam_search + fast_beam_search + """, + ) + + parser.add_argument( + "--num_active_paths", + type=int, + default=4, + help="""An interger indicating how many candidates we will keep for each + frame. Used only when --decoding-method is modified_beam_search.""", + ) + + parser.add_argument( + "--beam", + type=float, + default=4, + help="""A floating point value to calculate the cutoff score during beam + search (i.e., `cutoff = max-score - beam`), which is the same as the + `beam` in Kaldi. + Used only when --decoding-method is fast_beam_search""", + ) + + parser.add_argument( + "--max-contexts", + type=int, + default=4, + help="""Used only when --decoding-method is + fast_beam_search""", + ) + + parser.add_argument( + "--max-states", + type=int, + default=32, + help="""Used only when --decoding-method is + fast_beam_search""", + ) + + parser.add_argument( + "--context-size", + type=int, + default=2, + help="The context size in the decoder. 1 means bigram; 2 means tri-gram", + ) + + parser.add_argument( + "--num-decode-streams", + type=int, + default=2000, + help="The number of streams that can be decoded parallel.", + ) + + add_model_arguments(parser) + + return parser + + +def get_init_states( + model: nn.Module, + batch_size: int = 1, + device: torch.device = torch.device("cpu"), +) -> List[torch.Tensor]: + """ + Returns a list of cached tensors of all encoder layers. For layer-i, states[i*6:(i+1)*6] + is (cached_key, cached_nonlin_attn, cached_val1, cached_val2, cached_conv1, cached_conv2). + states[-2] is the cached left padding for ConvNeXt module, + of shape (batch_size, num_channels, left_pad, num_freqs) + states[-1] is processed_lens of shape (batch,), which records the number + of processed frames (at 50hz frame rate, after encoder_embed) for each sample in batch. + """ + states = model.encoder.get_init_states(batch_size, device) + + embed_states = model.encoder_embed.get_init_states(batch_size, device) + states.append(embed_states) + + processed_lens = torch.zeros(batch_size, dtype=torch.int32, device=device) + states.append(processed_lens) + + return states + + +def stack_states(state_list: List[List[torch.Tensor]]) -> List[torch.Tensor]: + """Stack list of zipformer states that correspond to separate utterances + into a single emformer state, so that it can be used as an input for + zipformer when those utterances are formed into a batch. + + Args: + state_list: + Each element in state_list corresponding to the internal state + of the zipformer model for a single utterance. For element-n, + state_list[n] is a list of cached tensors of all encoder layers. For layer-i, + state_list[n][i*6:(i+1)*6] is (cached_key, cached_nonlin_attn, cached_val1, + cached_val2, cached_conv1, cached_conv2). + state_list[n][-2] is the cached left padding for ConvNeXt module, + of shape (batch_size, num_channels, left_pad, num_freqs) + state_list[n][-1] is processed_lens of shape (batch,), which records the number + of processed frames (at 50hz frame rate, after encoder_embed) for each sample in batch. + + Note: + It is the inverse of :func:`unstack_states`. + """ + batch_size = len(state_list) + assert (len(state_list[0]) - 2) % 6 == 0, len(state_list[0]) + tot_num_layers = (len(state_list[0]) - 2) // 6 + + batch_states = [] + for layer in range(tot_num_layers): + layer_offset = layer * 6 + # cached_key: (left_context_len, batch_size, key_dim) + cached_key = torch.cat( + [state_list[i][layer_offset] for i in range(batch_size)], dim=1 + ) + # cached_nonlin_attn: (num_heads, batch_size, left_context_len, head_dim) + cached_nonlin_attn = torch.cat( + [state_list[i][layer_offset + 1] for i in range(batch_size)], dim=1 + ) + # cached_val1: (left_context_len, batch_size, value_dim) + cached_val1 = torch.cat( + [state_list[i][layer_offset + 2] for i in range(batch_size)], dim=1 + ) + # cached_val2: (left_context_len, batch_size, value_dim) + cached_val2 = torch.cat( + [state_list[i][layer_offset + 3] for i in range(batch_size)], dim=1 + ) + # cached_conv1: (#batch, channels, left_pad) + cached_conv1 = torch.cat( + [state_list[i][layer_offset + 4] for i in range(batch_size)], dim=0 + ) + # cached_conv2: (#batch, channels, left_pad) + cached_conv2 = torch.cat( + [state_list[i][layer_offset + 5] for i in range(batch_size)], dim=0 + ) + batch_states += [ + cached_key, + cached_nonlin_attn, + cached_val1, + cached_val2, + cached_conv1, + cached_conv2, + ] + + cached_embed_left_pad = torch.cat( + [state_list[i][-2] for i in range(batch_size)], dim=0 + ) + batch_states.append(cached_embed_left_pad) + + processed_lens = torch.cat([state_list[i][-1] for i in range(batch_size)], dim=0) + batch_states.append(processed_lens) + + return batch_states + + +def unstack_states(batch_states: List[Tensor]) -> List[List[Tensor]]: + """Unstack the zipformer state corresponding to a batch of utterances + into a list of states, where the i-th entry is the state from the i-th + utterance in the batch. + + Note: + It is the inverse of :func:`stack_states`. + + Args: + batch_states: A list of cached tensors of all encoder layers. For layer-i, + states[i*6:(i+1)*6] is (cached_key, cached_nonlin_attn, cached_val1, cached_val2, + cached_conv1, cached_conv2). + state_list[-2] is the cached left padding for ConvNeXt module, + of shape (batch_size, num_channels, left_pad, num_freqs) + states[-1] is processed_lens of shape (batch,), which records the number + of processed frames (at 50hz frame rate, after encoder_embed) for each sample in batch. + + Returns: + state_list: A list of list. Each element in state_list corresponding to the internal state + of the zipformer model for a single utterance. + """ + assert (len(batch_states) - 2) % 6 == 0, len(batch_states) + tot_num_layers = (len(batch_states) - 2) // 6 + + processed_lens = batch_states[-1] + batch_size = processed_lens.shape[0] + + state_list = [[] for _ in range(batch_size)] + + for layer in range(tot_num_layers): + layer_offset = layer * 6 + # cached_key: (left_context_len, batch_size, key_dim) + cached_key_list = batch_states[layer_offset].chunk(chunks=batch_size, dim=1) + # cached_nonlin_attn: (num_heads, batch_size, left_context_len, head_dim) + cached_nonlin_attn_list = batch_states[layer_offset + 1].chunk( + chunks=batch_size, dim=1 + ) + # cached_val1: (left_context_len, batch_size, value_dim) + cached_val1_list = batch_states[layer_offset + 2].chunk( + chunks=batch_size, dim=1 + ) + # cached_val2: (left_context_len, batch_size, value_dim) + cached_val2_list = batch_states[layer_offset + 3].chunk( + chunks=batch_size, dim=1 + ) + # cached_conv1: (#batch, channels, left_pad) + cached_conv1_list = batch_states[layer_offset + 4].chunk( + chunks=batch_size, dim=0 + ) + # cached_conv2: (#batch, channels, left_pad) + cached_conv2_list = batch_states[layer_offset + 5].chunk( + chunks=batch_size, dim=0 + ) + for i in range(batch_size): + state_list[i] += [ + cached_key_list[i], + cached_nonlin_attn_list[i], + cached_val1_list[i], + cached_val2_list[i], + cached_conv1_list[i], + cached_conv2_list[i], + ] + + cached_embed_left_pad_list = batch_states[-2].chunk(chunks=batch_size, dim=0) + for i in range(batch_size): + state_list[i].append(cached_embed_left_pad_list[i]) + + processed_lens_list = batch_states[-1].chunk(chunks=batch_size, dim=0) + for i in range(batch_size): + state_list[i].append(processed_lens_list[i]) + + return state_list + + +def streaming_forward( + features: Tensor, + feature_lens: Tensor, + model: nn.Module, + states: List[Tensor], + chunk_size: int, + left_context_len: int, +) -> Tuple[Tensor, Tensor, List[Tensor]]: + """ + Returns encoder outputs, output lengths, and updated states. + """ + cached_embed_left_pad = states[-2] + (x, x_lens, new_cached_embed_left_pad,) = model.encoder_embed.streaming_forward( + x=features, + x_lens=feature_lens, + cached_left_pad=cached_embed_left_pad, + ) + assert x.size(1) == chunk_size, (x.size(1), chunk_size) + + src_key_padding_mask = make_pad_mask(x_lens) + + # processed_mask is used to mask out initial states + processed_mask = torch.arange(left_context_len, device=x.device).expand( + x.size(0), left_context_len + ) + processed_lens = states[-1] # (batch,) + # (batch, left_context_size) + processed_mask = (processed_lens.unsqueeze(1) <= processed_mask).flip(1) + # Update processed lengths + new_processed_lens = processed_lens + x_lens + + # (batch, left_context_size + chunk_size) + src_key_padding_mask = torch.cat([processed_mask, src_key_padding_mask], dim=1) + + x = x.permute(1, 0, 2) # (N, T, C) -> (T, N, C) + encoder_states = states[:-2] + ( + encoder_out, + encoder_out_lens, + new_encoder_states, + ) = model.encoder.streaming_forward( + x=x, + x_lens=x_lens, + states=encoder_states, + src_key_padding_mask=src_key_padding_mask, + ) + encoder_out = encoder_out.permute(1, 0, 2) # (T, N, C) ->(N, T, C) + + new_states = new_encoder_states + [ + new_cached_embed_left_pad, + new_processed_lens, + ] + return encoder_out, encoder_out_lens, new_states + + +def decode_one_chunk( + params: AttributeDict, + model: nn.Module, + decode_streams: List[DecodeStream], +) -> List[int]: + """Decode one chunk frames of features for each decode_streams and + return the indexes of finished streams in a List. + + Args: + params: + It's the return value of :func:`get_params`. + model: + The neural model. + decode_streams: + A List of DecodeStream, each belonging to a utterance. + Returns: + Return a List containing which DecodeStreams are finished. + """ + chunk_size = int(params.chunk_size) + left_context_len = int(params.left_context_frames) + + features = [] + feature_lens = [] + states = [] + processed_lens = [] # Used in fast-beam-search + + for stream in decode_streams: + feat, feat_len = stream.get_feature_frames(chunk_size * 2) + features.append(feat) + feature_lens.append(feat_len) + states.append(stream.states) + processed_lens.append(stream.done_frames) + + feature_lens = torch.tensor(feature_lens, device=model.device) + features = pad_sequence(features, batch_first=True, padding_value=LOG_EPS) + + # Make sure the length after encoder_embed is at least 1. + # The encoder_embed subsample features (T - 7) // 2 + # The ConvNeXt module needs (7 - 1) // 2 = 3 frames of right padding after subsampling + tail_length = chunk_size * 2 + 7 + 2 * 3 + if features.size(1) < tail_length: + pad_length = tail_length - features.size(1) + feature_lens += pad_length + features = torch.nn.functional.pad( + features, + (0, 0, 0, pad_length), + mode="constant", + value=LOG_EPS, + ) + + states = stack_states(states) + + encoder_out, encoder_out_lens, new_states = streaming_forward( + features=features, + feature_lens=feature_lens, + model=model, + states=states, + chunk_size=chunk_size, + left_context_len=left_context_len, + ) + + encoder_out = model.joiner.encoder_proj(encoder_out) + + if params.decoding_method == "greedy_search": + greedy_search(model=model, encoder_out=encoder_out, streams=decode_streams) + elif params.decoding_method == "fast_beam_search": + processed_lens = torch.tensor(processed_lens, device=model.device) + processed_lens = processed_lens + encoder_out_lens + fast_beam_search_one_best( + model=model, + encoder_out=encoder_out, + processed_lens=processed_lens, + streams=decode_streams, + beam=params.beam, + max_states=params.max_states, + max_contexts=params.max_contexts, + ) + elif params.decoding_method == "modified_beam_search": + modified_beam_search( + model=model, + streams=decode_streams, + encoder_out=encoder_out, + num_active_paths=params.num_active_paths, + ) + else: + raise ValueError(f"Unsupported decoding method: {params.decoding_method}") + + states = unstack_states(new_states) + + finished_streams = [] + for i in range(len(decode_streams)): + decode_streams[i].states = states[i] + decode_streams[i].done_frames += encoder_out_lens[i] + if decode_streams[i].done: + finished_streams.append(i) + # finished_streams.append(i) + + return finished_streams + + +def decode_dataset( + cuts: CutSet, + params: AttributeDict, + model: nn.Module, + sp: Tokenizer, + decoding_graph: Optional[k2.Fsa] = None, +) -> Dict[str, List[Tuple[List[str], List[str]]]]: + """Decode dataset. + + Args: + cuts: + Lhotse Cutset containing the dataset to decode. + params: + It is returned by :func:`get_params`. + model: + The neural model. + sp: + The BPE model. + decoding_graph: + The decoding graph. Can be either a `k2.trivial_graph` or HLG, Used + only when --decoding_method is fast_beam_search. + Returns: + Return a dict, whose key may be "greedy_search" if greedy search + is used, or it may be "beam_7" if beam size of 7 is used. + Its value is a list of tuples. Each tuple contains two elements: + The first is the reference transcript, and the second is the + predicted result. + """ + device = model.device + + opts = FbankOptions() + opts.device = device + opts.frame_opts.dither = 0 + opts.frame_opts.snip_edges = False + opts.frame_opts.samp_freq = 16000 + opts.mel_opts.num_bins = 80 + + log_interval = 100 + + decode_results = [] + # Contain decode streams currently running. + decode_streams = [] + for num, cut in enumerate(cuts): + # each utterance has a DecodeStream. + initial_states = get_init_states(model=model, batch_size=1, device=device) + decode_stream = DecodeStream( + params=params, + cut_id=cut.id, + initial_states=initial_states, + decoding_graph=decoding_graph, + device=device, + ) + + audio: np.ndarray = cut.load_audio() + # audio.shape: (1, num_samples) + assert len(audio.shape) == 2 + assert audio.shape[0] == 1, "Should be single channel" + assert audio.dtype == np.float32, audio.dtype + + # The trained model is using normalized samples + # - this is to avoid sending [-32k,+32k] signal in... + # - some lhotse AudioTransform classes can make the signal + # be out of range [-1, 1], hence the tolerance 10 + assert ( + np.abs(audio).max() <= 10 + ), "Should be normalized to [-1, 1], 10 for tolerance..." + + samples = torch.from_numpy(audio).squeeze(0) + + fbank = Fbank(opts) + feature = fbank(samples.to(device)) + decode_stream.set_features(feature, tail_pad_len=30) + decode_stream.ground_truth = cut.supervisions[0].text + + decode_streams.append(decode_stream) + + while len(decode_streams) >= params.num_decode_streams: + finished_streams = decode_one_chunk( + params=params, model=model, decode_streams=decode_streams + ) + for i in sorted(finished_streams, reverse=True): + decode_results.append( + ( + decode_streams[i].id, + decode_streams[i].ground_truth.split(), + sp.decode(decode_streams[i].decoding_result()).split(), + ) + ) + del decode_streams[i] + + if num % log_interval == 0: + logging.info(f"Cuts processed until now is {num}.") + + # decode final chunks of last sequences + while len(decode_streams): + finished_streams = decode_one_chunk( + params=params, model=model, decode_streams=decode_streams + ) + + if not finished_streams: + print("No finished streams, breaking the loop") + break + + for i in sorted(finished_streams, reverse=True): + try: + decode_results.append( + ( + decode_streams[i].id, + decode_streams[i].ground_truth.split(), + sp.decode(decode_streams[i].decoding_result()).split(), + ) + ) + del decode_streams[i] + except IndexError as e: + print(f"IndexError: {e}") + print(f"decode_streams length: {len(decode_streams)}") + print(f"finished_streams: {finished_streams}") + print(f"i: {i}") + continue + + if params.decoding_method == "greedy_search": + key = "greedy_search" + elif params.decoding_method == "fast_beam_search": + key = ( + f"beam_{params.beam}_" + f"max_contexts_{params.max_contexts}_" + f"max_states_{params.max_states}" + ) + elif params.decoding_method == "modified_beam_search": + key = f"num_active_paths_{params.num_active_paths}" + else: + raise ValueError(f"Unsupported decoding method: {params.decoding_method}") + torch.cuda.synchronize() + return {key: decode_results} + + +def save_results( + params: AttributeDict, + test_set_name: str, + results_dict: Dict[str, List[Tuple[List[str], List[str]]]], +): + test_set_wers = dict() + for key, results in results_dict.items(): + recog_path = ( + params.res_dir / f"recogs-{test_set_name}-{key}-{params.suffix}.txt" + ) + results = sorted(results) + store_transcripts(filename=recog_path, texts=results) + logging.info(f"The transcripts are stored in {recog_path}") + + # The following prints out WERs, per-word error statistics and aligned + # ref/hyp pairs. + errs_filename = ( + params.res_dir / f"errs-{test_set_name}-{key}-{params.suffix}.txt" + ) + with open(errs_filename, "w") as f: + wer = write_error_stats( + f, f"{test_set_name}-{key}", results, enable_log=True + ) + test_set_wers[key] = wer + + logging.info("Wrote detailed error stats to {}".format(errs_filename)) + + test_set_wers = sorted(test_set_wers.items(), key=lambda x: x[1]) + errs_info = ( + params.res_dir / f"wer-summary-{test_set_name}-{key}-{params.suffix}.txt" + ) + with open(errs_info, "w") as f: + print("settings\tWER", file=f) + for key, val in test_set_wers: + print("{}\t{}".format(key, val), file=f) + + s = "\nFor {}, WER of different settings are:\n".format(test_set_name) + note = "\tbest for {}".format(test_set_name) + for key, val in test_set_wers: + s += "{}\t{}{}\n".format(key, val, note) + note = "" + logging.info(s) + + +@torch.no_grad() +def main(): + parser = get_parser() + ReazonSpeechAsrDataModule.add_arguments(parser) + Tokenizer.add_arguments(parser) + args = parser.parse_args() + args.exp_dir = Path(args.exp_dir) + + params = get_params() + params.update(vars(args)) + + params.res_dir = params.exp_dir / "streaming" / params.decoding_method + + if params.iter > 0: + params.suffix = f"iter-{params.iter}-avg-{params.avg}" + else: + params.suffix = f"epoch-{params.epoch}-avg-{params.avg}" + + assert params.causal, params.causal + assert "," not in params.chunk_size, "chunk_size should be one value in decoding." + assert ( + "," not in params.left_context_frames + ), "left_context_frames should be one value in decoding." + params.suffix += f"-chunk-{params.chunk_size}" + params.suffix += f"-left-context-{params.left_context_frames}" + + # for fast_beam_search + if params.decoding_method == "fast_beam_search": + params.suffix += f"-beam-{params.beam}" + params.suffix += f"-max-contexts-{params.max_contexts}" + params.suffix += f"-max-states-{params.max_states}" + + if params.use_averaged_model: + params.suffix += "-use-averaged-model" + + setup_logger(f"{params.res_dir}/log-decode-{params.suffix}") + logging.info("Decoding started") + + device = torch.device("cpu") + if torch.cuda.is_available(): + device = torch.device("cuda", 0) + + logging.info(f"Device: {device}") + + if not params.bilingual: + sp = Tokenizer.load(params.lang, params.lang_type) + else: + sp = spm.SentencePieceProcessor() + sp.load(params.bpe_model) + + # and is defined in local/train_bpe_model.py + params.blank_id = sp.piece_to_id("") + params.unk_id = sp.piece_to_id("") + params.vocab_size = sp.get_piece_size() + + logging.info(params) + + logging.info("About to create model") + model = get_model(params) + + if not params.use_averaged_model: + if params.iter > 0: + filenames = find_checkpoints(params.exp_dir, iteration=-params.iter)[ + : params.avg + ] + if len(filenames) == 0: + raise ValueError( + f"No checkpoints found for" + f" --iter {params.iter}, --avg {params.avg}" + ) + elif len(filenames) < params.avg: + raise ValueError( + f"Not enough checkpoints ({len(filenames)}) found for" + f" --iter {params.iter}, --avg {params.avg}" + ) + logging.info(f"averaging {filenames}") + model.to(device) + model.load_state_dict(average_checkpoints(filenames, device=device)) + elif params.avg == 1: + load_checkpoint(f"{params.exp_dir}/epoch-{params.epoch}.pt", model) + else: + start = params.epoch - params.avg + 1 + filenames = [] + for i in range(start, params.epoch + 1): + if start >= 0: + filenames.append(f"{params.exp_dir}/epoch-{i}.pt") + logging.info(f"averaging {filenames}") + model.to(device) + model.load_state_dict(average_checkpoints(filenames, device=device)) + else: + if params.iter > 0: + filenames = find_checkpoints(params.exp_dir, iteration=-params.iter)[ + : params.avg + 1 + ] + if len(filenames) == 0: + raise ValueError( + f"No checkpoints found for" + f" --iter {params.iter}, --avg {params.avg}" + ) + elif len(filenames) < params.avg + 1: + raise ValueError( + f"Not enough checkpoints ({len(filenames)}) found for" + f" --iter {params.iter}, --avg {params.avg}" + ) + filename_start = filenames[-1] + filename_end = filenames[0] + logging.info( + "Calculating the averaged model over iteration checkpoints" + f" from {filename_start} (excluded) to {filename_end}" + ) + model.to(device) + model.load_state_dict( + average_checkpoints_with_averaged_model( + filename_start=filename_start, + filename_end=filename_end, + device=device, + ) + ) + else: + assert params.avg > 0, params.avg + start = params.epoch - params.avg + assert start >= 1, start + filename_start = f"{params.exp_dir}/epoch-{start}.pt" + filename_end = f"{params.exp_dir}/epoch-{params.epoch}.pt" + logging.info( + f"Calculating the averaged model over epoch range from " + f"{start} (excluded) to {params.epoch}" + ) + model.to(device) + model.load_state_dict( + average_checkpoints_with_averaged_model( + filename_start=filename_start, + filename_end=filename_end, + device=device, + ) + ) + + model.to(device) + model.eval() + model.device = device + + decoding_graph = None + if params.decoding_method == "fast_beam_search": + decoding_graph = k2.trivial_graph(params.vocab_size - 1, device=device) + + num_param = sum([p.numel() for p in model.parameters()]) + logging.info(f"Number of model parameters: {num_param}") + + # we need cut ids to display recognition results. + args.return_cuts = True + reazonspeech_corpus = ReazonSpeechAsrDataModule(args) + + if params.bilingual: + multi_dataset = MultiDataset(args) + + def remove_short_utt(c: Cut): + T = ((c.num_frames - 7) // 2 + 1) // 2 + if T <= 0: + logging.warning( + f"Excluding cut with ID: {c.id} from decoding, num_frames: {c.num_frames}" + ) + return T > 0 + + test_sets_cuts = multi_dataset.test_cuts() + test_sets = test_sets_cuts.keys() + test_cuts = [test_sets_cuts[k] for k in test_sets] + + valid_cuts = reazonspeech_corpus.valid_cuts() + test_cuts = reazonspeech_corpus.test_cuts() + + test_sets = ["valid", "test"] + test_cuts = [valid_cuts, test_cuts] + + for test_set, test_cut in zip(test_sets, test_cuts): + logging.info(f"Decoding {test_set}") + if params.bilingual: + test_cut = test_cut.filter(remove_short_utt) + results_dict = decode_dataset( + cuts=test_cut, + params=params, + model=model, + sp=sp, + decoding_graph=decoding_graph, + ) + + save_results( + params=params, + test_set_name=test_set, + results_dict=results_dict, + ) + + logging.info("Done!") + + +if __name__ == "__main__": + main() diff --git a/egs/multi_ja_en/ASR/zipformer/subsampling.py b/egs/multi_ja_en/ASR/zipformer/subsampling.py new file mode 120000 index 000000000..01ae9002c --- /dev/null +++ b/egs/multi_ja_en/ASR/zipformer/subsampling.py @@ -0,0 +1 @@ +../../../librispeech/ASR/zipformer/subsampling.py \ No newline at end of file diff --git a/egs/multi_ja_en/ASR/zipformer/test_scaling.py b/egs/multi_ja_en/ASR/zipformer/test_scaling.py new file mode 120000 index 000000000..715798436 --- /dev/null +++ b/egs/multi_ja_en/ASR/zipformer/test_scaling.py @@ -0,0 +1 @@ +../../../librispeech/ASR/zipformer/test_scaling.py \ No newline at end of file diff --git a/egs/multi_ja_en/ASR/zipformer/test_subsampling.py b/egs/multi_ja_en/ASR/zipformer/test_subsampling.py new file mode 120000 index 000000000..bf0ee3d11 --- /dev/null +++ b/egs/multi_ja_en/ASR/zipformer/test_subsampling.py @@ -0,0 +1 @@ +../../../librispeech/ASR/zipformer/test_subsampling.py \ No newline at end of file diff --git a/egs/multi_ja_en/ASR/zipformer/tokenizer.py b/egs/multi_ja_en/ASR/zipformer/tokenizer.py new file mode 120000 index 000000000..958c99e85 --- /dev/null +++ b/egs/multi_ja_en/ASR/zipformer/tokenizer.py @@ -0,0 +1 @@ +../local/utils/tokenizer.py \ No newline at end of file diff --git a/egs/multi_ja_en/ASR/zipformer/train.py b/egs/multi_ja_en/ASR/zipformer/train.py new file mode 100755 index 000000000..bfb037f50 --- /dev/null +++ b/egs/multi_ja_en/ASR/zipformer/train.py @@ -0,0 +1,1462 @@ +#!/usr/bin/env python3 +# Copyright 2021-2023 Xiaomi Corp. (authors: Fangjun Kuang, +# Wei Kang, +# Mingshuang Luo, +# Zengwei Yao, +# Daniel Povey) +# +# See ../../../../LICENSE for clarification regarding multiple authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Usage: + +export CUDA_VISIBLE_DEVICES="0,1,2,3" + +# For non-streaming model training: +./zipformer/train.py \ + --bilingual 1 \ + --world-size 4 \ + --num-epochs 30 \ + --start-epoch 1 \ + --use-fp16 1 \ + --exp-dir zipformer/exp \ + --max-duration 600 + +# For streaming model training: +./zipformer/train.py \ + --bilingual 1 \ + --world-size 4 \ + --num-epochs 30 \ + --start-epoch 1 \ + --use-fp16 1 \ + --exp-dir zipformer/exp \ + --causal 1 \ + --max-duration 600 + +It supports training with: + - transducer loss (default), with `--use-transducer True --use-ctc False` + - ctc loss (not recommended), with `--use-transducer False --use-ctc True` + - transducer loss & ctc loss, with `--use-transducer True --use-ctc True` +""" + +import argparse +import copy +import logging +import os +import re +import warnings +from pathlib import Path +from shutil import copyfile +from typing import Any, Dict, Optional, Tuple, Union + +import k2 +import optim +import sentencepiece as spm +import torch +import torch.multiprocessing as mp +import torch.nn as nn +from asr_datamodule import ReazonSpeechAsrDataModule +from decoder import Decoder +from joiner import Joiner +from lhotse.cut import Cut +from lhotse.dataset.sampling.base import CutSampler +from lhotse.utils import fix_random_seed +from model import AsrModel +from multi_dataset import MultiDataset +from optim import Eden, ScaledAdam +from scaling import ScheduledFloat +from subsampling import Conv2dSubsampling +from tokenizer import Tokenizer +from torch import Tensor +from torch.cuda.amp import GradScaler +from torch.nn.parallel import DistributedDataParallel as DDP +from torch.utils.tensorboard import SummaryWriter +from zipformer import Zipformer2 + +from icefall import byte_encode, diagnostics +from icefall.checkpoint import load_checkpoint, remove_checkpoints +from icefall.checkpoint import save_checkpoint as save_checkpoint_impl +from icefall.checkpoint import ( + save_checkpoint_with_global_batch_idx, + update_averaged_model, +) +from icefall.dist import cleanup_dist, setup_dist +from icefall.env import get_env_info +from icefall.err import raise_grad_scale_is_too_small_error +from icefall.hooks import register_inf_check_hooks +from icefall.utils import ( + AttributeDict, + MetricsTracker, + get_parameter_groups_with_lrs, + setup_logger, + str2bool, + tokenize_by_ja_char, +) + +LRSchedulerType = Union[torch.optim.lr_scheduler._LRScheduler, optim.LRScheduler] + + +def get_adjusted_batch_count(params: AttributeDict) -> float: + # returns the number of batches we would have used so far if we had used the reference + # duration. This is for purposes of set_batch_count(). + return ( + params.batch_idx_train + * (params.max_duration * params.world_size) + / params.ref_duration + ) + + +def set_batch_count(model: Union[nn.Module, DDP], batch_count: float) -> None: + if isinstance(model, DDP): + # get underlying nn.Module + model = model.module + for name, module in model.named_modules(): + if hasattr(module, "batch_count"): + module.batch_count = batch_count + if hasattr(module, "name"): + module.name = name + + +def add_model_arguments(parser: argparse.ArgumentParser): + parser.add_argument( + "--num-encoder-layers", + type=str, + default="2,2,3,4,3,2", + help="Number of zipformer encoder layers per stack, comma separated.", + ) + + parser.add_argument( + "--downsampling-factor", + type=str, + default="1,2,4,8,4,2", + help="Downsampling factor for each stack of encoder layers.", + ) + + parser.add_argument( + "--feedforward-dim", + type=str, + default="512,768,1024,1536,1024,768", + help="Feedforward dimension of the zipformer encoder layers, per stack, comma separated.", + ) + + parser.add_argument( + "--num-heads", + type=str, + default="4,4,4,8,4,4", + help="Number of attention heads in the zipformer encoder layers: a single int or comma-separated list.", + ) + + parser.add_argument( + "--encoder-dim", + type=str, + default="192,256,384,512,384,256", + help="Embedding dimension in encoder stacks: a single int or comma-separated list.", + ) + + parser.add_argument( + "--query-head-dim", + type=str, + default="32", + help="Query/key dimension per head in encoder stacks: a single int or comma-separated list.", + ) + + parser.add_argument( + "--value-head-dim", + type=str, + default="12", + help="Value dimension per head in encoder stacks: a single int or comma-separated list.", + ) + + parser.add_argument( + "--pos-head-dim", + type=str, + default="4", + help="Positional-encoding dimension per head in encoder stacks: a single int or comma-separated list.", + ) + + parser.add_argument( + "--pos-dim", + type=int, + default="48", + help="Positional-encoding embedding dimension", + ) + + parser.add_argument( + "--encoder-unmasked-dim", + type=str, + default="192,192,256,256,256,192", + help="Unmasked dimensions in the encoders, relates to augmentation during training. " + "A single int or comma-separated list. Must be <= each corresponding encoder_dim.", + ) + + parser.add_argument( + "--cnn-module-kernel", + type=str, + default="31,31,15,15,15,31", + help="Sizes of convolutional kernels in convolution modules in each encoder stack: " + "a single int or comma-separated list.", + ) + + parser.add_argument( + "--decoder-dim", + type=int, + default=512, + help="Embedding dimension in the decoder model.", + ) + + parser.add_argument( + "--joiner-dim", + type=int, + default=512, + help="""Dimension used in the joiner model. + Outputs from the encoder and decoder model are projected + to this dimension before adding. + """, + ) + + parser.add_argument( + "--causal", + type=str2bool, + default=False, + help="If True, use causal version of model.", + ) + + parser.add_argument( + "--chunk-size", + type=str, + default="16,32,64,-1", + help="Chunk sizes (at 50Hz frame rate) will be chosen randomly from this list during training. " + " Must be just -1 if --causal=False", + ) + + parser.add_argument( + "--left-context-frames", + type=str, + default="64,128,256,-1", + help="Maximum left-contexts for causal training, measured in frames which will " + "be converted to a number of chunks. If splitting into chunks, " + "chunk left-context frames will be chosen randomly from this list; else not relevant.", + ) + + parser.add_argument( + "--use-transducer", + type=str2bool, + default=True, + help="If True, use Transducer head.", + ) + + parser.add_argument( + "--use-ctc", + type=str2bool, + default=False, + help="If True, use CTC head.", + ) + + +def get_parser(): + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + + parser.add_argument( + "--bilingual", + type=str2bool, + default=False, + help="Whether the model is bilingual or not. 1 = bilingual.", + ) + + parser.add_argument( + "--world-size", + type=int, + default=1, + help="Number of GPUs for DDP training.", + ) + + parser.add_argument( + "--master-port", + type=int, + default=12354, + help="Master port to use for DDP training.", + ) + + parser.add_argument( + "--tensorboard", + type=str2bool, + default=True, + help="Should various information be logged in tensorboard.", + ) + + parser.add_argument( + "--num-epochs", + type=int, + default=30, + help="Number of epochs to train.", + ) + + parser.add_argument( + "--start-epoch", + type=int, + default=1, + help="""Resume training from this epoch. It should be positive. + If larger than 1, it will load checkpoint from + exp-dir/epoch-{start_epoch-1}.pt + """, + ) + + parser.add_argument( + "--start-batch", + type=int, + default=0, + help="""If positive, --start-epoch is ignored and + it loads the checkpoint from exp-dir/checkpoint-{start_batch}.pt + """, + ) + + parser.add_argument( + "--exp-dir", + type=str, + default="zipformer/exp", + help="""The experiment dir. + It specifies the directory where all training related + files, e.g., checkpoints, log, etc, are saved + """, + ) + + # changed - not used in monolingual streaming + parser.add_argument( + "--bpe-model", + type=str, + default="data/lang_bbpe_2000/bbpe.model", + help="Path to the BPE model", + ) + + parser.add_argument( + "--base-lr", type=float, default=0.015, help="The base learning rate." + ) + + parser.add_argument( + "--lr-batches", + type=float, + default=7500, + help="""Number of steps that affects how rapidly the learning rate + decreases. We suggest not to change this.""", + ) + + parser.add_argument( + "--lr-epochs", + type=float, + default=3.5, + help="""Number of epochs that affects how rapidly the learning rate decreases. + """, + ) + + parser.add_argument( + "--ref-duration", + type=float, + default=600, + help="Reference batch duration for purposes of adjusting batch counts for setting various " + "schedules inside the model", + ) + + parser.add_argument( + "--context-size", + type=int, + default=2, + help="The context size in the decoder. 1 means bigram; " "2 means tri-gram", + ) + + parser.add_argument( + "--prune-range", + type=int, + default=5, + help="The prune range for rnnt loss, it means how many symbols(context)" + "we are using to compute the loss", + ) + + parser.add_argument( + "--lm-scale", + type=float, + default=0.25, + help="The scale to smooth the loss with lm " + "(output of prediction network) part.", + ) + + parser.add_argument( + "--am-scale", + type=float, + default=0.0, + help="The scale to smooth the loss with am (output of encoder network)" "part.", + ) + + parser.add_argument( + "--simple-loss-scale", + type=float, + default=0.5, + help="To get pruning ranges, we will calculate a simple version" + "loss(joiner is just addition), this simple loss also uses for" + "training (as a regularization item). We will scale the simple loss" + "with this parameter before adding to the final loss.", + ) + + parser.add_argument( + "--ctc-loss-scale", + type=float, + default=0.2, + help="Scale for CTC loss.", + ) + + parser.add_argument( + "--seed", + type=int, + default=42, + help="The seed for random generators intended for reproducibility", + ) + + parser.add_argument( + "--print-diagnostics", + type=str2bool, + default=False, + help="Accumulate stats on activations, print them and exit.", + ) + + parser.add_argument( + "--inf-check", + type=str2bool, + default=False, + help="Add hooks to check for infinite module outputs and gradients.", + ) + + parser.add_argument( + "--save-every-n", + type=int, + default=4000, + help="""Save checkpoint after processing this number of batches" + periodically. We save checkpoint to exp-dir/ whenever + params.batch_idx_train % save_every_n == 0. The checkpoint filename + has the form: f'exp-dir/checkpoint-{params.batch_idx_train}.pt' + Note: It also saves checkpoint to `exp-dir/epoch-xxx.pt` at the + end of each epoch where `xxx` is the epoch number counting from 1. + """, + ) + + parser.add_argument( + "--keep-last-k", + type=int, + default=30, + help="""Only keep this number of checkpoints on disk. + For instance, if it is 3, there are only 3 checkpoints + in the exp-dir with filenames `checkpoint-xxx.pt`. + It does not affect checkpoints with name `epoch-xxx.pt`. + """, + ) + + parser.add_argument( + "--average-period", + type=int, + default=200, + help="""Update the averaged model, namely `model_avg`, after processing + this number of batches. `model_avg` is a separate version of model, + in which each floating-point parameter is the average of all the + parameters from the start of training. Each time we take the average, + we do: `model_avg = model * (average_period / batch_idx_train) + + model_avg * ((batch_idx_train - average_period) / batch_idx_train)`. + """, + ) + + parser.add_argument( + "--use-fp16", + type=str2bool, + default=False, + help="Whether to use half precision training.", + ) + + add_model_arguments(parser) + + return parser + + +def get_params() -> AttributeDict: + """Return a dict containing training parameters. + + All training related parameters that are not passed from the commandline + are saved in the variable `params`. + + Commandline options are merged into `params` after they are parsed, so + you can also access them via `params`. + + Explanation of options saved in `params`: + + - best_train_loss: Best training loss so far. It is used to select + the model that has the lowest training loss. It is + updated during the training. + + - best_valid_loss: Best validation loss so far. It is used to select + the model that has the lowest validation loss. It is + updated during the training. + + - best_train_epoch: It is the epoch that has the best training loss. + + - best_valid_epoch: It is the epoch that has the best validation loss. + + - batch_idx_train: Used to writing statistics to tensorboard. It + contains number of batches trained so far across + epochs. + + - log_interval: Print training loss if batch_idx % log_interval` is 0 + + - reset_interval: Reset statistics if batch_idx % reset_interval is 0 + + - valid_interval: Run validation if batch_idx % valid_interval is 0 + + - feature_dim: The model input dim. It has to match the one used + in computing features. + + - subsampling_factor: The subsampling factor for the model. + + - encoder_dim: Hidden dim for multi-head attention model. + + - num_decoder_layers: Number of decoder layer of transformer decoder. + + - warm_step: The warmup period that dictates the decay of the + scale on "simple" (un-pruned) loss. + """ + params = AttributeDict( + { + "best_train_loss": float("inf"), + "best_valid_loss": float("inf"), + "best_train_epoch": -1, + "best_valid_epoch": -1, + "batch_idx_train": 0, + "log_interval": 50, + "reset_interval": 200, + "valid_interval": 3000, # For the 100h subset, use 800 + # parameters for zipformer + "feature_dim": 80, + "subsampling_factor": 4, # not passed in, this is fixed. + "warm_step": 2000, + "env_info": get_env_info(), + } + ) + + return params + + +def _to_int_tuple(s: str): + return tuple(map(int, s.split(","))) + + +def get_encoder_embed(params: AttributeDict) -> nn.Module: + # encoder_embed converts the input of shape (N, T, num_features) + # to the shape (N, (T - 7) // 2, encoder_dims). + # That is, it does two things simultaneously: + # (1) subsampling: T -> (T - 7) // 2 + # (2) embedding: num_features -> encoder_dims + # In the normal configuration, we will downsample once more at the end + # by a factor of 2, and most of the encoder stacks will run at a lower + # sampling rate. + encoder_embed = Conv2dSubsampling( + in_channels=params.feature_dim, + out_channels=_to_int_tuple(params.encoder_dim)[0], + dropout=ScheduledFloat((0.0, 0.3), (20000.0, 0.1)), + ) + return encoder_embed + + +def get_encoder_model(params: AttributeDict) -> nn.Module: + encoder = Zipformer2( + output_downsampling_factor=2, + downsampling_factor=_to_int_tuple(params.downsampling_factor), + num_encoder_layers=_to_int_tuple(params.num_encoder_layers), + encoder_dim=_to_int_tuple(params.encoder_dim), + encoder_unmasked_dim=_to_int_tuple(params.encoder_unmasked_dim), + query_head_dim=_to_int_tuple(params.query_head_dim), + pos_head_dim=_to_int_tuple(params.pos_head_dim), + value_head_dim=_to_int_tuple(params.value_head_dim), + pos_dim=params.pos_dim, + num_heads=_to_int_tuple(params.num_heads), + feedforward_dim=_to_int_tuple(params.feedforward_dim), + cnn_module_kernel=_to_int_tuple(params.cnn_module_kernel), + dropout=ScheduledFloat((0.0, 0.3), (20000.0, 0.1)), + warmup_batches=4000.0, + causal=params.causal, + chunk_size=_to_int_tuple(params.chunk_size), + left_context_frames=_to_int_tuple(params.left_context_frames), + ) + return encoder + + +def get_decoder_model(params: AttributeDict) -> nn.Module: + decoder = Decoder( + vocab_size=params.vocab_size, + decoder_dim=params.decoder_dim, + blank_id=params.blank_id, + context_size=params.context_size, + ) + return decoder + + +def get_joiner_model(params: AttributeDict) -> nn.Module: + joiner = Joiner( + encoder_dim=max(_to_int_tuple(params.encoder_dim)), + decoder_dim=params.decoder_dim, + joiner_dim=params.joiner_dim, + vocab_size=params.vocab_size, + ) + return joiner + + +def get_model(params: AttributeDict) -> nn.Module: + assert params.use_transducer or params.use_ctc, ( + f"At least one of them should be True, " + f"but got params.use_transducer={params.use_transducer}, " + f"params.use_ctc={params.use_ctc}" + ) + + encoder_embed = get_encoder_embed(params) + encoder = get_encoder_model(params) + + if params.use_transducer: + decoder = get_decoder_model(params) + joiner = get_joiner_model(params) + else: + decoder = None + joiner = None + + model = AsrModel( + encoder_embed=encoder_embed, + encoder=encoder, + decoder=decoder, + joiner=joiner, + encoder_dim=max(_to_int_tuple(params.encoder_dim)), + decoder_dim=params.decoder_dim, + vocab_size=params.vocab_size, + use_transducer=params.use_transducer, + use_ctc=params.use_ctc, + ) + return model + + +def load_checkpoint_if_available( + params: AttributeDict, + model: nn.Module, + model_avg: nn.Module = None, + optimizer: Optional[torch.optim.Optimizer] = None, + scheduler: Optional[LRSchedulerType] = None, +) -> Optional[Dict[str, Any]]: + """Load checkpoint from file. + + If params.start_batch is positive, it will load the checkpoint from + `params.exp_dir/checkpoint-{params.start_batch}.pt`. Otherwise, if + params.start_epoch is larger than 1, it will load the checkpoint from + `params.start_epoch - 1`. + + Apart from loading state dict for `model` and `optimizer` it also updates + `best_train_epoch`, `best_train_loss`, `best_valid_epoch`, + and `best_valid_loss` in `params`. + + Args: + params: + The return value of :func:`get_params`. + model: + The training model. + model_avg: + The stored model averaged from the start of training. + optimizer: + The optimizer that we are using. + scheduler: + The scheduler that we are using. + Returns: + Return a dict containing previously saved training info. + """ + if params.start_batch > 0: + filename = params.exp_dir / f"checkpoint-{params.start_batch}.pt" + elif params.start_epoch > 1: + filename = params.exp_dir / f"epoch-{params.start_epoch-1}.pt" + else: + return None + + assert filename.is_file(), f"{filename} does not exist!" + + saved_params = load_checkpoint( + filename, + model=model, + model_avg=model_avg, + optimizer=optimizer, + scheduler=scheduler, + ) + + keys = [ + "best_train_epoch", + "best_valid_epoch", + "batch_idx_train", + "best_train_loss", + "best_valid_loss", + ] + for k in keys: + params[k] = saved_params[k] + + if params.start_batch > 0: + if "cur_epoch" in saved_params: + params["start_epoch"] = saved_params["cur_epoch"] + + return saved_params + + +def save_checkpoint( + params: AttributeDict, + model: Union[nn.Module, DDP], + model_avg: Optional[nn.Module] = None, + optimizer: Optional[torch.optim.Optimizer] = None, + scheduler: Optional[LRSchedulerType] = None, + sampler: Optional[CutSampler] = None, + scaler: Optional[GradScaler] = None, + rank: int = 0, +) -> None: + """Save model, optimizer, scheduler and training stats to file. + + Args: + params: + It is returned by :func:`get_params`. + model: + The training model. + model_avg: + The stored model averaged from the start of training. + optimizer: + The optimizer used in the training. + sampler: + The sampler for the training dataset. + scaler: + The scaler used for mix precision training. + """ + if rank != 0: + return + filename = params.exp_dir / f"epoch-{params.cur_epoch}.pt" + save_checkpoint_impl( + filename=filename, + model=model, + model_avg=model_avg, + params=params, + optimizer=optimizer, + scheduler=scheduler, + sampler=sampler, + scaler=scaler, + rank=rank, + ) + + if params.best_train_epoch == params.cur_epoch: + best_train_filename = params.exp_dir / "best-train-loss.pt" + copyfile(src=filename, dst=best_train_filename) + + if params.best_valid_epoch == params.cur_epoch: + best_valid_filename = params.exp_dir / "best-valid-loss.pt" + copyfile(src=filename, dst=best_valid_filename) + + +# fix implementation for sentencepiece_processor: spm.SentencePieceProcessor, stuff +def compute_loss( + params: AttributeDict, + model: Union[nn.Module, DDP], + tokenizer: Tokenizer, + sentencepiece_processor: spm.SentencePieceProcessor, + batch: dict, + is_training: bool, +) -> Tuple[Tensor, MetricsTracker]: + """ + Compute loss given the model and its inputs. + + Args: + params: + Parameters for training. See :func:`get_params`. + model: + The model for training. It is an instance of Zipformer in our case. + batch: + A batch of data. See `lhotse.dataset.K2SpeechRecognitionDataset()` + for the content in it. + is_training: + True for training. False for validation. When it is True, this + function enables autograd during computation; when it is False, it + disables autograd. + warmup: a floating point value which increases throughout training; + values >= 1.0 are fully warmed up and have all modules present. + """ + device = model.device if isinstance(model, DDP) else next(model.parameters()).device + feature = batch["inputs"] + # at entry, feature is (N, T, C) + assert feature.ndim == 3 + feature = feature.to(device) + + supervisions = batch["supervisions"] + feature_lens = supervisions["num_frames"].to(device) + + batch_idx_train = params.batch_idx_train + warm_step = params.warm_step + + texts = batch["supervisions"]["text"] + if not params.bilingual: + y = tokenizer.encode(texts, out_type=int) + else: + y = sentencepiece_processor.encode(texts, out_type=int) + y = k2.RaggedTensor(y) + + with torch.set_grad_enabled(is_training): + losses = model( + x=feature, + x_lens=feature_lens, + y=y, + prune_range=params.prune_range, + am_scale=params.am_scale, + lm_scale=params.lm_scale, + ) + simple_loss, pruned_loss, ctc_loss = losses[:3] + + loss = 0.0 + + if params.use_transducer: + s = params.simple_loss_scale + # take down the scale on the simple loss from 1.0 at the start + # to params.simple_loss scale by warm_step. + simple_loss_scale = ( + s + if batch_idx_train >= warm_step + else 1.0 - (batch_idx_train / warm_step) * (1.0 - s) + ) + pruned_loss_scale = ( + 1.0 + if batch_idx_train >= warm_step + else 0.1 + 0.9 * (batch_idx_train / warm_step) + ) + loss += simple_loss_scale * simple_loss + pruned_loss_scale * pruned_loss + + if params.use_ctc: + loss += params.ctc_loss_scale * ctc_loss + + assert loss.requires_grad == is_training + + info = MetricsTracker() + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + info["frames"] = (feature_lens // params.subsampling_factor).sum().item() + + # Note: We use reduction=sum while computing the loss. + info["loss"] = loss.detach().cpu().item() + if params.use_transducer: + info["simple_loss"] = simple_loss.detach().cpu().item() + info["pruned_loss"] = pruned_loss.detach().cpu().item() + if params.use_ctc: + info["ctc_loss"] = ctc_loss.detach().cpu().item() + + return loss, info + + +def compute_validation_loss( + params: AttributeDict, + model: Union[nn.Module, DDP], + tokenizer: Tokenizer, + sentencepiece_processor: spm.SentencePieceProcessor, + valid_dl: torch.utils.data.DataLoader, + world_size: int = 1, +) -> MetricsTracker: + """Run the validation process.""" + model.eval() + + tot_loss = MetricsTracker() + + for batch_idx, batch in enumerate(valid_dl): + loss, loss_info = compute_loss( + params=params, + model=model, + tokenizer=tokenizer, + sentencepiece_processor=sentencepiece_processor, + batch=batch, + is_training=False, + ) + assert loss.requires_grad is False + tot_loss = tot_loss + loss_info + + if world_size > 1: + tot_loss.reduce(loss.device) + + loss_value = tot_loss["loss"] / tot_loss["frames"] + if loss_value < params.best_valid_loss: + params.best_valid_epoch = params.cur_epoch + params.best_valid_loss = loss_value + + return tot_loss + + +def train_one_epoch( + params: AttributeDict, + model: Union[nn.Module, DDP], + optimizer: torch.optim.Optimizer, + scheduler: LRSchedulerType, + tokenizer: Tokenizer, + sentencepiece_processor: spm.SentencePieceProcessor, + train_dl: torch.utils.data.DataLoader, + valid_dl: torch.utils.data.DataLoader, + scaler: GradScaler, + model_avg: Optional[nn.Module] = None, + tb_writer: Optional[SummaryWriter] = None, + world_size: int = 1, + rank: int = 0, +) -> None: + """Train the model for one epoch. + + The training loss from the mean of all frames is saved in + `params.train_loss`. It runs the validation process every + `params.valid_interval` batches. + + Args: + params: + It is returned by :func:`get_params`. + model: + The model for training. + optimizer: + The optimizer we are using. + scheduler: + The learning rate scheduler, we call step() every step. + train_dl: + Dataloader for the training dataset. + valid_dl: + Dataloader for the validation dataset. + scaler: + The scaler used for mix precision training. + model_avg: + The stored model averaged from the start of training. + tb_writer: + Writer to write log messages to tensorboard. + world_size: + Number of nodes in DDP training. If it is 1, DDP is disabled. + rank: + The rank of the node in DDP training. If no DDP is used, it should + be set to 0. + """ + model.train() + + tot_loss = MetricsTracker() + + saved_bad_model = False + + def save_bad_model(suffix: str = ""): + save_checkpoint_impl( + filename=params.exp_dir / f"bad-model{suffix}-{rank}.pt", + model=model, + model_avg=model_avg, + params=params, + optimizer=optimizer, + scheduler=scheduler, + sampler=train_dl.sampler, + scaler=scaler, + rank=0, + ) + + for batch_idx, batch in enumerate(train_dl): + if batch_idx % 10 == 0: + set_batch_count(model, get_adjusted_batch_count(params)) + + params.batch_idx_train += 1 + batch_size = len(batch["supervisions"]["text"]) + + try: + with torch.cuda.amp.autocast(enabled=params.use_fp16): + loss, loss_info = compute_loss( + params=params, + model=model, + tokenizer=tokenizer, + sentencepiece_processor=sentencepiece_processor, + batch=batch, + is_training=True, + ) + # summary stats + tot_loss = (tot_loss * (1 - 1 / params.reset_interval)) + loss_info + + # NOTE: We use reduction==sum and loss is computed over utterances + # in the batch and there is no normalization to it so far. + scaler.scale(loss).backward() + scheduler.step_batch(params.batch_idx_train) + + scaler.step(optimizer) + scaler.update() + optimizer.zero_grad() + except: # noqa + save_bad_model() + display_and_save_batch( + batch, + params=params, + tokenizer=tokenizer, + sentencepiece_processor=sentencepiece_processor, + ) + raise + + if params.print_diagnostics and batch_idx == 5: + return + + if ( + rank == 0 + and params.batch_idx_train > 0 + and params.batch_idx_train % params.average_period == 0 + ): + update_averaged_model( + params=params, + model_cur=model, + model_avg=model_avg, + ) + + if ( + params.batch_idx_train > 0 + and params.batch_idx_train % params.save_every_n == 0 + ): + save_checkpoint_with_global_batch_idx( + out_dir=params.exp_dir, + global_batch_idx=params.batch_idx_train, + model=model, + model_avg=model_avg, + params=params, + optimizer=optimizer, + scheduler=scheduler, + sampler=train_dl.sampler, + scaler=scaler, + rank=rank, + ) + remove_checkpoints( + out_dir=params.exp_dir, + topk=params.keep_last_k, + rank=rank, + ) + + if batch_idx % 100 == 0 and params.use_fp16: + # If the grad scale was less than 1, try increasing it. The _growth_interval + # of the grad scaler is configurable, but we can't configure it to have different + # behavior depending on the current grad scale. + cur_grad_scale = scaler._scale.item() + + if cur_grad_scale < 8.0 or (cur_grad_scale < 32.0 and batch_idx % 400 == 0): + scaler.update(cur_grad_scale * 2.0) + if cur_grad_scale < 0.01: + if not saved_bad_model: + save_bad_model(suffix="-first-warning") + saved_bad_model = True + logging.warning(f"Grad scale is small: {cur_grad_scale}") + if cur_grad_scale < 1.0e-05: + save_bad_model() + raise_grad_scale_is_too_small_error(cur_grad_scale) + + if batch_idx % params.log_interval == 0: + cur_lr = max(scheduler.get_last_lr()) + cur_grad_scale = scaler._scale.item() if params.use_fp16 else 1.0 + + logging.info( + f"Epoch {params.cur_epoch}, " + f"batch {batch_idx}, loss[{loss_info}], " + f"tot_loss[{tot_loss}], batch size: {batch_size}, " + f"lr: {cur_lr:.2e}, " + + (f"grad_scale: {scaler._scale.item()}" if params.use_fp16 else "") + ) + + if tb_writer is not None: + tb_writer.add_scalar( + "train/learning_rate", cur_lr, params.batch_idx_train + ) + + loss_info.write_summary( + tb_writer, "train/current_", params.batch_idx_train + ) + tot_loss.write_summary(tb_writer, "train/tot_", params.batch_idx_train) + if params.use_fp16: + tb_writer.add_scalar( + "train/grad_scale", cur_grad_scale, params.batch_idx_train + ) + + if batch_idx % params.valid_interval == 0 and not params.print_diagnostics: + logging.info("Computing validation loss") + valid_info = compute_validation_loss( + params=params, + model=model, + tokenizer=tokenizer, + sentencepiece_processor=sentencepiece_processor, + valid_dl=valid_dl, + world_size=world_size, + ) + model.train() + logging.info(f"Epoch {params.cur_epoch}, validation: {valid_info}") + logging.info( + f"Maximum memory allocated so far is {torch.cuda.max_memory_allocated()//1000000}MB" + ) + if tb_writer is not None: + valid_info.write_summary( + tb_writer, "train/valid_", params.batch_idx_train + ) + + loss_value = tot_loss["loss"] / tot_loss["frames"] + params.train_loss = loss_value + if params.train_loss < params.best_train_loss: + params.best_train_epoch = params.cur_epoch + params.best_train_loss = params.train_loss + + +def run(rank, world_size, args): + """ + Args: + rank: + It is a value between 0 and `world_size-1`, which is + passed automatically by `mp.spawn()` in :func:`main`. + The node with rank 0 is responsible for saving checkpoint. + world_size: + Number of GPUs for DDP training. + args: + The return value of get_parser().parse_args() + """ + params = get_params() + params.update(vars(args)) + + fix_random_seed(params.seed) + if world_size > 1: + setup_dist(rank, world_size, params.master_port) + + setup_logger(f"{params.exp_dir}/log/log-train") + logging.info("Training started") + + if args.tensorboard and rank == 0: + tb_writer = SummaryWriter(log_dir=f"{params.exp_dir}/tensorboard") + else: + tb_writer = None + + device = torch.device("cpu") + if torch.cuda.is_available(): + device = torch.device("cuda", rank) + logging.info(f"Device: {device}") + + # Use lang_dir for further operations + # tokenizer = Tokenizer.load(args.lang, args.lang_type) + + # sentencepiece_processor = spm.SentencePieceProcessor() + # sentencepiece_processor.load(params.bpe_model) + tokenizer = None + sentencepiece_processor = None + + # is defined in local/prepare_lang_char.py + + if not params.bilingual: + tokenizer = Tokenizer.load(args.lang, args.lang_type) + params.blank_id = tokenizer.piece_to_id("") + params.vocab_size = tokenizer.get_piece_size() + else: + sentencepiece_processor = spm.SentencePieceProcessor() + sentencepiece_processor.load(params.bpe_model) + params.blank_id = sentencepiece_processor.piece_to_id("") + params.vocab_size = sentencepiece_processor.get_piece_size() + + if not params.use_transducer: + params.ctc_loss_scale = 1.0 + + logging.info(params) + + logging.info("About to create model") + model = get_model(params) + + num_param = sum([p.numel() for p in model.parameters()]) + logging.info(f"Number of model parameters: {num_param}") + + assert params.save_every_n >= params.average_period + model_avg: Optional[nn.Module] = None + if rank == 0: + # model_avg is only used with rank 0 + model_avg = copy.deepcopy(model).to(torch.float64) + + assert params.start_epoch > 0, params.start_epoch + checkpoints = load_checkpoint_if_available( + params=params, model=model, model_avg=model_avg + ) + + model.to(device) + if world_size > 1: + logging.info("Using DDP") + model = DDP(model, device_ids=[rank], find_unused_parameters=True) + + optimizer = ScaledAdam( + get_parameter_groups_with_lrs(model, lr=params.base_lr, include_names=True), + lr=params.base_lr, # should have no effect + clipping_scale=2.0, + ) + + scheduler = Eden(optimizer, params.lr_batches, params.lr_epochs) + + if checkpoints and "optimizer" in checkpoints: + logging.info("Loading optimizer state dict") + optimizer.load_state_dict(checkpoints["optimizer"]) + + if ( + checkpoints + and "scheduler" in checkpoints + and checkpoints["scheduler"] is not None + ): + logging.info("Loading scheduler state dict") + scheduler.load_state_dict(checkpoints["scheduler"]) + + if params.print_diagnostics: + opts = diagnostics.TensorDiagnosticOptions( + 512 + ) # allow 4 megabytes per sub-module + diagnostic = diagnostics.attach_diagnostics(model, opts) + + if params.inf_check: + register_inf_check_hooks(model) + + reazonspeech_corpus = ReazonSpeechAsrDataModule(args) + if params.bilingual: + multi_dataset = MultiDataset(args) + train_cuts = multi_dataset.train_cuts() + else: + train_cuts = reazonspeech_corpus.train_cuts() + + def remove_short_and_long_utt(c: Cut): + # Keep only utterances with duration between 1 second and 20 seconds + # + # Caution: There is a reason to select 20.0 here. Please see + # ../local/display_manifest_statistics.py + # + # You should use ../local/display_manifest_statistics.py to get + # an utterance duration distribution for your dataset to select + # the threshold + # if c.duration < 1.0 or c.duration > 30.0: + # logging.warning( + # f"Exclude cut with ID {c.id} from training. Duration: {c.duration}" + # ) + # return False + + # In pruned RNN-T, we require that T >= S + # where T is the number of feature frames after subsampling + # and S is the number of tokens in the utterance + + # In ./zipformer.py, the conv module uses the following expression + # for subsampling + T = ((c.num_samples - 7) // 2 + 1) // 2 + if not params.bilingual: + tokens = tokenizer.encode(c.supervisions[0].text, out_type=str) + else: + tokens = sentencepiece_processor.encode( + c.supervisions[0].text, out_type=str + ) + + if T < len(tokens): + logging.warning( + f"Exclude cut with ID {c.id} from training. " + f"Number of frames (before subsampling): {c.num_samples}. " + f"Number of frames (after subsampling): {T}. " + f"Text: {c.supervisions[0].text}. " + f"Tokens: {tokens}. " + f"Number of tokens: {len(tokens)}" + ) + return False + + return True + + def tokenize_and_encode_text(c: Cut): + # Text normalize for each sample + text = c.supervisions[0].text + text = byte_encode(tokenize_by_ja_char(text)) + c.supervisions[0].text = text + return c + + train_cuts = train_cuts.filter(remove_short_and_long_utt) + + if params.bilingual: + train_cuts = train_cuts.map(tokenize_and_encode_text) + + if params.start_batch > 0 and checkpoints and "sampler" in checkpoints: + # We only load the sampler's state dict when it loads a checkpoint + # saved in the middle of an epoch + sampler_state_dict = checkpoints["sampler"] + else: + sampler_state_dict = None + + train_dl = reazonspeech_corpus.train_dataloaders( + train_cuts, sampler_state_dict=sampler_state_dict + ) + + if params.bilingual: + valid_cuts = reazonspeech_corpus.valid_cuts() + else: + valid_cuts = multi_dataset.dev_cuts() + valid_dl = reazonspeech_corpus.valid_dataloaders(valid_cuts) + + if not params.print_diagnostics: + scan_pessimistic_batches_for_oom( + model=model, + train_dl=train_dl, + optimizer=optimizer, + tokenizer=tokenizer, + sentencepiece_processor=sentencepiece_processor, + params=params, + ) + + scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0) + if checkpoints and "grad_scaler" in checkpoints: + logging.info("Loading grad scaler state dict") + scaler.load_state_dict(checkpoints["grad_scaler"]) + + for epoch in range(params.start_epoch, params.num_epochs + 1): + scheduler.step_epoch(epoch - 1) + fix_random_seed(params.seed + epoch - 1) + train_dl.sampler.set_epoch(epoch - 1) + + if tb_writer is not None: + tb_writer.add_scalar("train/epoch", epoch, params.batch_idx_train) + + params.cur_epoch = epoch + + train_one_epoch( + params=params, + model=model, + model_avg=model_avg, + optimizer=optimizer, + scheduler=scheduler, + tokenizer=tokenizer, + sentencepiece_processor=sentencepiece_processor, + train_dl=train_dl, + valid_dl=valid_dl, + scaler=scaler, + tb_writer=tb_writer, + world_size=world_size, + rank=rank, + ) + + if params.print_diagnostics: + diagnostic.print_diagnostics() + break + + save_checkpoint( + params=params, + model=model, + model_avg=model_avg, + optimizer=optimizer, + scheduler=scheduler, + sampler=train_dl.sampler, + scaler=scaler, + rank=rank, + ) + + logging.info("Done!") + + if world_size > 1: + torch.distributed.barrier() + cleanup_dist() + + +def display_and_save_batch( + batch: dict, + params: AttributeDict, + tokenizer: Tokenizer, + sentencepiece_processor: spm.SentencePieceProcessor, +) -> None: + """Display the batch statistics and save the batch into disk. + + Args: + batch: + A batch of data. See `lhotse.dataset.K2SpeechRecognitionDataset()` + for the content in it. + params: + Parameters for training. See :func:`get_params`. + tokenizer: + The BPE Tokenizer model. + sentencepiece_processor: + The BPE SentencePieceProcessor model. + """ + from lhotse.utils import uuid4 + + filename = f"{params.exp_dir}/batch-{uuid4()}.pt" + logging.info(f"Saving batch to {filename}") + torch.save(batch, filename) + + supervisions = batch["supervisions"] + features = batch["inputs"] + + logging.info(f"features shape: {features.shape}") + + if params.bilingual: + y = sentencepiece_processor.encode(supervisions["text"], out_type=int) + else: + y = tokenizer.encode(supervisions["text"], out_type=int) + num_tokens = sum(len(i) for i in y) + logging.info(f"num tokens: {num_tokens}") + + +def scan_pessimistic_batches_for_oom( + model: Union[nn.Module, DDP], + train_dl: torch.utils.data.DataLoader, + optimizer: torch.optim.Optimizer, + tokenizer: Tokenizer, + sentencepiece_processor: spm.SentencePieceProcessor, + params: AttributeDict, +): + from lhotse.dataset import find_pessimistic_batches + + logging.info( + "Sanity check -- see if any of the batches in epoch 1 would cause OOM." + ) + batches, crit_values = find_pessimistic_batches(train_dl.sampler) + for criterion, cuts in batches.items(): + batch = train_dl.dataset[cuts] + try: + with torch.cuda.amp.autocast(enabled=params.use_fp16): + loss, _ = compute_loss( + params=params, + model=model, + tokenizer=tokenizer, + sentencepiece_processor=sentencepiece_processor, + batch=batch, + is_training=True, + ) + loss.backward() + optimizer.zero_grad() + except Exception as e: + if "CUDA out of memory" in str(e): + logging.error( + "Your GPU ran out of memory with the current " + "max_duration setting. We recommend decreasing " + "max_duration and trying again.\n" + f"Failing criterion: {criterion} " + f"(={crit_values[criterion]}) ..." + ) + display_and_save_batch( + batch, + params=params, + tokenizer=tokenizer, + sentencepiece_processor=sentencepiece_processor, + ) + raise + logging.info( + f"Maximum memory allocated so far is {torch.cuda.max_memory_allocated()//1000000}MB" + ) + + +def main(): + parser = get_parser() + ReazonSpeechAsrDataModule.add_arguments(parser) + Tokenizer.add_arguments(parser) + args = parser.parse_args() + args.exp_dir = Path(args.exp_dir) + + world_size = args.world_size + assert world_size >= 1 + if world_size > 1: + mp.spawn(run, args=(world_size, args), nprocs=world_size, join=True) + else: + run(rank=0, world_size=1, args=args) + + +torch.set_num_threads(1) +torch.set_num_interop_threads(1) + +if __name__ == "__main__": + main() diff --git a/egs/multi_ja_en/ASR/zipformer/zipformer.py b/egs/multi_ja_en/ASR/zipformer/zipformer.py new file mode 120000 index 000000000..23011dda7 --- /dev/null +++ b/egs/multi_ja_en/ASR/zipformer/zipformer.py @@ -0,0 +1 @@ +../../../librispeech/ASR/zipformer/zipformer.py \ No newline at end of file diff --git a/egs/reazonspeech/ASR/RESULTS.md b/egs/reazonspeech/ASR/RESULTS.md index c0b4fe54a..92610d75b 100644 --- a/egs/reazonspeech/ASR/RESULTS.md +++ b/egs/reazonspeech/ASR/RESULTS.md @@ -47,3 +47,41 @@ The decoding command is: --blank-penalty 0 ``` +#### Streaming + +We have not completed evaluation of our models yet and will add evaluation results here once it's completed. + +The training command is: +```shell +./zipformer/train.py \ + --world-size 8 \ + --num-epochs 40 \ + --start-epoch 1 \ + --use-fp16 1 \ + --exp-dir zipformer/exp-large \ + --causal 1 \ + --num-encoder-layers 2,2,4,5,4,2 \ + --feedforward-dim 512,768,1536,2048,1536,768 \ + --encoder-dim 192,256,512,768,512,256 \ + --encoder-unmasked-dim 192,192,256,320,256,192 \ + --lang data/lang_char \ + --max-duration 1600 +``` + +The decoding command is: + +```shell +./zipformer/streaming_decode.py \ + --epoch 28 \ + --avg 15 \ + --causal 1 \ + --chunk-size 32 \ + --left-context-frames 256 \ + --exp-dir ./zipformer/exp-large \ + --lang data/lang_char \ + --num-encoder-layers 2,2,4,5,4,2 \ + --feedforward-dim 512,768,1536,2048,1536,768 \ + --encoder-dim 192,256,512,768,512,256 \ + --encoder-unmasked-dim 192,192,256,320,256,192 +``` + diff --git a/egs/reazonspeech/ASR/local/utils/tokenizer.py b/egs/reazonspeech/ASR/local/utils/tokenizer.py index c9be72be1..ba71cff89 100644 --- a/egs/reazonspeech/ASR/local/utils/tokenizer.py +++ b/egs/reazonspeech/ASR/local/utils/tokenizer.py @@ -12,7 +12,6 @@ class Tokenizer: @staticmethod def add_arguments(parser: argparse.ArgumentParser): group = parser.add_argument_group(title="Lang related options") - group.add_argument("--lang", type=Path, help="Path to lang directory.") group.add_argument( diff --git a/egs/reazonspeech/ASR/zipformer/streaming_decode.py b/egs/reazonspeech/ASR/zipformer/streaming_decode.py index 4c18c7563..7e3199e09 100755 --- a/egs/reazonspeech/ASR/zipformer/streaming_decode.py +++ b/egs/reazonspeech/ASR/zipformer/streaming_decode.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 -# Copyright 2022 Xiaomi Corporation (Authors: Wei Kang, Fangjun Kuang) -# +# Copyright 2022-2023 Xiaomi Corporation (Authors: Wei Kang, +# Fangjun Kuang, +# Zengwei Yao) # See ../../../../LICENSE for clarification regarding multiple authors # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,28 +18,23 @@ """ Usage: -./pruned_transducer_stateless7_streaming/streaming_decode.py \ - --epoch 28 \ - --avg 15 \ - --decode-chunk-len 32 \ - --exp-dir ./pruned_transducer_stateless7_streaming/exp \ - --decoding_method greedy_search \ - --lang data/lang_char \ - --num-decode-streams 2000 +./zipformer/streaming_decode.py--epoch 28 --avg 15 --causal 1 --chunk-size 32 --left-context-frames 256 --exp-dir ./zipformer/exp-large --lang data/lang_char --num-encoder-layers 2,2,4,5,4,2 --feedforward-dim 512,768,1536,2048,1536,768 --encoder-dim 192,256,512,768,512,256 --encoder-unmasked-dim 192,192,256,320,256,192 + """ import argparse import logging import math +import os +import pdb +import subprocess as sp from pathlib import Path from typing import Dict, List, Optional, Tuple import k2 import numpy as np import torch -import torch.nn as nn from asr_datamodule import ReazonSpeechAsrDataModule -from decode import save_results from decode_stream import DecodeStream from kaldifeat import Fbank, FbankOptions from lhotse import CutSet @@ -48,9 +44,9 @@ from streaming_beam_search import ( modified_beam_search, ) from tokenizer import Tokenizer +from torch import Tensor, nn from torch.nn.utils.rnn import pad_sequence -from train import add_model_arguments, get_params, get_transducer_model -from zipformer import stack_states, unstack_states +from train import add_model_arguments, get_model, get_params from icefall.checkpoint import ( average_checkpoints, @@ -58,7 +54,14 @@ from icefall.checkpoint import ( find_checkpoints, load_checkpoint, ) -from icefall.utils import AttributeDict, setup_logger, str2bool +from icefall.utils import ( + AttributeDict, + make_pad_mask, + setup_logger, + store_transcripts, + str2bool, + write_error_stats, +) LOG_EPS = math.log(1e-10) @@ -73,7 +76,7 @@ def get_parser(): type=int, default=28, help="""It specifies the checkpoint to use for decoding. - Note: Epoch counts from 0. + Note: Epoch counts from 1. You can specify --avg to use more checkpoints for model averaging.""", ) @@ -87,12 +90,6 @@ def get_parser(): """, ) - parser.add_argument( - "--gpu", - type=int, - default=0, - ) - parser.add_argument( "--avg", type=int, @@ -116,7 +113,7 @@ def get_parser(): parser.add_argument( "--exp-dir", type=str, - default="pruned_transducer_stateless2/exp", + default="zipformer/exp", help="The experiment dir", ) @@ -127,6 +124,13 @@ def get_parser(): help="Path to the BPE model", ) + parser.add_argument( + "--lang-dir", + type=Path, + default="data/lang_char", + help="The lang dir containing word table and LG graph", + ) + parser.add_argument( "--decoding-method", type=str, @@ -138,14 +142,6 @@ def get_parser(): """, ) - parser.add_argument( - "--decoding-graph", - type=str, - default="", - help="""Used only when --decoding-method is - fast_beam_search""", - ) - parser.add_argument( "--num_active_paths", type=int, @@ -157,7 +153,7 @@ def get_parser(): parser.add_argument( "--beam", type=float, - default=4.0, + default=4, help="""A floating point value to calculate the cutoff score during beam search (i.e., `cutoff = max-score - beam`), which is the same as the `beam` in Kaldi. @@ -194,18 +190,235 @@ def get_parser(): help="The number of streams that can be decoded parallel.", ) - parser.add_argument( - "--res-dir", - type=Path, - default=None, - help="The path to save results.", - ) - add_model_arguments(parser) return parser +def get_init_states( + model: nn.Module, + batch_size: int = 1, + device: torch.device = torch.device("cpu"), +) -> List[torch.Tensor]: + """ + Returns a list of cached tensors of all encoder layers. For layer-i, states[i*6:(i+1)*6] + is (cached_key, cached_nonlin_attn, cached_val1, cached_val2, cached_conv1, cached_conv2). + states[-2] is the cached left padding for ConvNeXt module, + of shape (batch_size, num_channels, left_pad, num_freqs) + states[-1] is processed_lens of shape (batch,), which records the number + of processed frames (at 50hz frame rate, after encoder_embed) for each sample in batch. + """ + states = model.encoder.get_init_states(batch_size, device) + + embed_states = model.encoder_embed.get_init_states(batch_size, device) + states.append(embed_states) + + processed_lens = torch.zeros(batch_size, dtype=torch.int32, device=device) + states.append(processed_lens) + + return states + + +def stack_states(state_list: List[List[torch.Tensor]]) -> List[torch.Tensor]: + """Stack list of zipformer states that correspond to separate utterances + into a single emformer state, so that it can be used as an input for + zipformer when those utterances are formed into a batch. + + Args: + state_list: + Each element in state_list corresponding to the internal state + of the zipformer model for a single utterance. For element-n, + state_list[n] is a list of cached tensors of all encoder layers. For layer-i, + state_list[n][i*6:(i+1)*6] is (cached_key, cached_nonlin_attn, cached_val1, + cached_val2, cached_conv1, cached_conv2). + state_list[n][-2] is the cached left padding for ConvNeXt module, + of shape (batch_size, num_channels, left_pad, num_freqs) + state_list[n][-1] is processed_lens of shape (batch,), which records the number + of processed frames (at 50hz frame rate, after encoder_embed) for each sample in batch. + + Note: + It is the inverse of :func:`unstack_states`. + """ + batch_size = len(state_list) + assert (len(state_list[0]) - 2) % 6 == 0, len(state_list[0]) + tot_num_layers = (len(state_list[0]) - 2) // 6 + + batch_states = [] + for layer in range(tot_num_layers): + layer_offset = layer * 6 + # cached_key: (left_context_len, batch_size, key_dim) + cached_key = torch.cat( + [state_list[i][layer_offset] for i in range(batch_size)], dim=1 + ) + # cached_nonlin_attn: (num_heads, batch_size, left_context_len, head_dim) + cached_nonlin_attn = torch.cat( + [state_list[i][layer_offset + 1] for i in range(batch_size)], dim=1 + ) + # cached_val1: (left_context_len, batch_size, value_dim) + cached_val1 = torch.cat( + [state_list[i][layer_offset + 2] for i in range(batch_size)], dim=1 + ) + # cached_val2: (left_context_len, batch_size, value_dim) + cached_val2 = torch.cat( + [state_list[i][layer_offset + 3] for i in range(batch_size)], dim=1 + ) + # cached_conv1: (#batch, channels, left_pad) + cached_conv1 = torch.cat( + [state_list[i][layer_offset + 4] for i in range(batch_size)], dim=0 + ) + # cached_conv2: (#batch, channels, left_pad) + cached_conv2 = torch.cat( + [state_list[i][layer_offset + 5] for i in range(batch_size)], dim=0 + ) + batch_states += [ + cached_key, + cached_nonlin_attn, + cached_val1, + cached_val2, + cached_conv1, + cached_conv2, + ] + + cached_embed_left_pad = torch.cat( + [state_list[i][-2] for i in range(batch_size)], dim=0 + ) + batch_states.append(cached_embed_left_pad) + + processed_lens = torch.cat([state_list[i][-1] for i in range(batch_size)], dim=0) + batch_states.append(processed_lens) + + return batch_states + + +def unstack_states(batch_states: List[Tensor]) -> List[List[Tensor]]: + """Unstack the zipformer state corresponding to a batch of utterances + into a list of states, where the i-th entry is the state from the i-th + utterance in the batch. + + Note: + It is the inverse of :func:`stack_states`. + + Args: + batch_states: A list of cached tensors of all encoder layers. For layer-i, + states[i*6:(i+1)*6] is (cached_key, cached_nonlin_attn, cached_val1, cached_val2, + cached_conv1, cached_conv2). + state_list[-2] is the cached left padding for ConvNeXt module, + of shape (batch_size, num_channels, left_pad, num_freqs) + states[-1] is processed_lens of shape (batch,), which records the number + of processed frames (at 50hz frame rate, after encoder_embed) for each sample in batch. + + Returns: + state_list: A list of list. Each element in state_list corresponding to the internal state + of the zipformer model for a single utterance. + """ + assert (len(batch_states) - 2) % 6 == 0, len(batch_states) + tot_num_layers = (len(batch_states) - 2) // 6 + + processed_lens = batch_states[-1] + batch_size = processed_lens.shape[0] + + state_list = [[] for _ in range(batch_size)] + + for layer in range(tot_num_layers): + layer_offset = layer * 6 + # cached_key: (left_context_len, batch_size, key_dim) + cached_key_list = batch_states[layer_offset].chunk(chunks=batch_size, dim=1) + # cached_nonlin_attn: (num_heads, batch_size, left_context_len, head_dim) + cached_nonlin_attn_list = batch_states[layer_offset + 1].chunk( + chunks=batch_size, dim=1 + ) + # cached_val1: (left_context_len, batch_size, value_dim) + cached_val1_list = batch_states[layer_offset + 2].chunk( + chunks=batch_size, dim=1 + ) + # cached_val2: (left_context_len, batch_size, value_dim) + cached_val2_list = batch_states[layer_offset + 3].chunk( + chunks=batch_size, dim=1 + ) + # cached_conv1: (#batch, channels, left_pad) + cached_conv1_list = batch_states[layer_offset + 4].chunk( + chunks=batch_size, dim=0 + ) + # cached_conv2: (#batch, channels, left_pad) + cached_conv2_list = batch_states[layer_offset + 5].chunk( + chunks=batch_size, dim=0 + ) + for i in range(batch_size): + state_list[i] += [ + cached_key_list[i], + cached_nonlin_attn_list[i], + cached_val1_list[i], + cached_val2_list[i], + cached_conv1_list[i], + cached_conv2_list[i], + ] + + cached_embed_left_pad_list = batch_states[-2].chunk(chunks=batch_size, dim=0) + for i in range(batch_size): + state_list[i].append(cached_embed_left_pad_list[i]) + + processed_lens_list = batch_states[-1].chunk(chunks=batch_size, dim=0) + for i in range(batch_size): + state_list[i].append(processed_lens_list[i]) + + return state_list + + +def streaming_forward( + features: Tensor, + feature_lens: Tensor, + model: nn.Module, + states: List[Tensor], + chunk_size: int, + left_context_len: int, +) -> Tuple[Tensor, Tensor, List[Tensor]]: + """ + Returns encoder outputs, output lengths, and updated states. + """ + cached_embed_left_pad = states[-2] + (x, x_lens, new_cached_embed_left_pad,) = model.encoder_embed.streaming_forward( + x=features, + x_lens=feature_lens, + cached_left_pad=cached_embed_left_pad, + ) + assert x.size(1) == chunk_size, (x.size(1), chunk_size) + + src_key_padding_mask = make_pad_mask(x_lens) + + # processed_mask is used to mask out initial states + processed_mask = torch.arange(left_context_len, device=x.device).expand( + x.size(0), left_context_len + ) + processed_lens = states[-1] # (batch,) + # (batch, left_context_size) + processed_mask = (processed_lens.unsqueeze(1) <= processed_mask).flip(1) + # Update processed lengths + new_processed_lens = processed_lens + x_lens + + # (batch, left_context_size + chunk_size) + src_key_padding_mask = torch.cat([processed_mask, src_key_padding_mask], dim=1) + + x = x.permute(1, 0, 2) # (N, T, C) -> (T, N, C) + encoder_states = states[:-2] + ( + encoder_out, + encoder_out_lens, + new_encoder_states, + ) = model.encoder.streaming_forward( + x=x, + x_lens=x_lens, + states=encoder_states, + src_key_padding_mask=src_key_padding_mask, + ) + encoder_out = encoder_out.permute(1, 0, 2) # (T, N, C) ->(N, T, C) + + new_states = new_encoder_states + [ + new_cached_embed_left_pad, + new_processed_lens, + ] + return encoder_out, encoder_out_lens, new_states + + def decode_one_chunk( params: AttributeDict, model: nn.Module, @@ -224,27 +437,32 @@ def decode_one_chunk( Returns: Return a List containing which DecodeStreams are finished. """ - device = model.device + # pdb.set_trace() + # print(model) + # print(model.device) + # device = model.device + chunk_size = int(params.chunk_size) + left_context_len = int(params.left_context_frames) features = [] feature_lens = [] states = [] - processed_lens = [] + processed_lens = [] # Used in fast-beam-search for stream in decode_streams: - feat, feat_len = stream.get_feature_frames(params.decode_chunk_len) + feat, feat_len = stream.get_feature_frames(chunk_size * 2) features.append(feat) feature_lens.append(feat_len) states.append(stream.states) processed_lens.append(stream.done_frames) - feature_lens = torch.tensor(feature_lens, device=device) + feature_lens = torch.tensor(feature_lens, device=model.device) features = pad_sequence(features, batch_first=True, padding_value=LOG_EPS) - # We subsample features with ((x_len - 7) // 2 + 1) // 2 and the max downsampling - # factor in encoders is 8. - # After feature embedding (x_len - 7) // 2, we have (23 - 7) // 2 = 8. - tail_length = 23 + # Make sure the length after encoder_embed is at least 1. + # The encoder_embed subsample features (T - 7) // 2 + # The ConvNeXt module needs (7 - 1) // 2 = 3 frames of right padding after subsampling + tail_length = chunk_size * 2 + 7 + 2 * 3 if features.size(1) < tail_length: pad_length = tail_length - features.size(1) feature_lens += pad_length @@ -256,12 +474,14 @@ def decode_one_chunk( ) states = stack_states(states) - processed_lens = torch.tensor(processed_lens, device=device) - encoder_out, encoder_out_lens, new_states = model.encoder.streaming_forward( - x=features, - x_lens=feature_lens, + encoder_out, encoder_out_lens, new_states = streaming_forward( + features=features, + feature_lens=feature_lens, + model=model, states=states, + chunk_size=chunk_size, + left_context_len=left_context_len, ) encoder_out = model.joiner.encoder_proj(encoder_out) @@ -269,6 +489,7 @@ def decode_one_chunk( if params.decoding_method == "greedy_search": greedy_search(model=model, encoder_out=encoder_out, streams=decode_streams) elif params.decoding_method == "fast_beam_search": + processed_lens = torch.tensor(processed_lens, device=model.device) processed_lens = processed_lens + encoder_out_lens fast_beam_search_one_best( model=model, @@ -295,8 +516,9 @@ def decode_one_chunk( for i in range(len(decode_streams)): decode_streams[i].states = states[i] decode_streams[i].done_frames += encoder_out_lens[i] - if decode_streams[i].done: - finished_streams.append(i) + # if decode_streams[i].done: + # finished_streams.append(i) + finished_streams.append(i) return finished_streams @@ -305,7 +527,7 @@ def decode_dataset( cuts: CutSet, params: AttributeDict, model: nn.Module, - sp: Tokenizer, + tokenizer: Tokenizer, decoding_graph: Optional[k2.Fsa] = None, ) -> Dict[str, List[Tuple[List[str], List[str]]]]: """Decode dataset. @@ -317,7 +539,7 @@ def decode_dataset( It is returned by :func:`get_params`. model: The neural model. - sp: + tokenizer: The BPE model. decoding_graph: The decoding graph. Can be either a `k2.trivial_graph` or HLG, Used @@ -338,14 +560,14 @@ def decode_dataset( opts.frame_opts.samp_freq = 16000 opts.mel_opts.num_bins = 80 - log_interval = 50 + log_interval = 100 decode_results = [] # Contain decode streams currently running. decode_streams = [] for num, cut in enumerate(cuts): # each utterance has a DecodeStream. - initial_states = model.encoder.get_init_state(device=device) + initial_states = get_init_states(model=model, batch_size=1, device=device) decode_stream = DecodeStream( params=params, cut_id=cut.id, @@ -361,15 +583,19 @@ def decode_dataset( assert audio.dtype == np.float32, audio.dtype # The trained model is using normalized samples - assert audio.max() <= 1, "Should be normalized to [-1, 1])" + # - this is to avoid sending [-32k,+32k] signal in... + # - some lhotse AudioTransform classes can make the signal + # be out of range [-1, 1], hence the tolerance 10 + assert ( + np.abs(audio).max() <= 10 + ), "Should be normalized to [-1, 1], 10 for tolerance..." samples = torch.from_numpy(audio).squeeze(0) fbank = Fbank(opts) feature = fbank(samples.to(device)) - decode_stream.set_features(feature, tail_pad_len=params.decode_chunk_len) - decode_stream.ground_truth = cut.supervisions[0].custom[params.transcript_mode] - + decode_stream.set_features(feature, tail_pad_len=30) + decode_stream.ground_truth = cut.supervisions[0].text decode_streams.append(decode_stream) while len(decode_streams) >= params.num_decode_streams: @@ -380,8 +606,8 @@ def decode_dataset( decode_results.append( ( decode_streams[i].id, - sp.text2word(decode_streams[i].ground_truth), - sp.text2word(sp.decode(decode_streams[i].decoding_result())), + decode_streams[i].ground_truth.split(), + tokenizer.decode(decode_streams[i].decoding_result()).split(), ) ) del decode_streams[i] @@ -391,18 +617,37 @@ def decode_dataset( # decode final chunks of last sequences while len(decode_streams): + # print("INSIDE LEN DECODE STREAMS") + # pdb.set_trace() + # print(model.device) + # test_device = model.device + # print("done") finished_streams = decode_one_chunk( params=params, model=model, decode_streams=decode_streams ) + # print('INSIDE FOR LOOP ') + # print(finished_streams) + + if not finished_streams: + print("No finished streams, breaking the loop") + break + for i in sorted(finished_streams, reverse=True): - decode_results.append( - ( - decode_streams[i].id, - sp.text2word(decode_streams[i].ground_truth), - sp.text2word(sp.decode(decode_streams[i].decoding_result())), + try: + decode_results.append( + ( + decode_streams[i].id, + decode_streams[i].ground_truth.split(), + tokenizer.decode(decode_streams[i].decoding_result()).split(), + ) ) - ) - del decode_streams[i] + del decode_streams[i] + except IndexError as e: + print(f"IndexError: {e}") + print(f"decode_streams length: {len(decode_streams)}") + print(f"finished_streams: {finished_streams}") + print(f"i: {i}") + continue if params.decoding_method == "greedy_search": key = "greedy_search" @@ -416,9 +661,54 @@ def decode_dataset( key = f"num_active_paths_{params.num_active_paths}" else: raise ValueError(f"Unsupported decoding method: {params.decoding_method}") + torch.cuda.synchronize() return {key: decode_results} +def save_results( + params: AttributeDict, + test_set_name: str, + results_dict: Dict[str, List[Tuple[List[str], List[str]]]], +): + test_set_wers = dict() + for key, results in results_dict.items(): + recog_path = ( + params.res_dir / f"recogs-{test_set_name}-{key}-{params.suffix}.txt" + ) + results = sorted(results) + store_transcripts(filename=recog_path, texts=results) + logging.info(f"The transcripts are stored in {recog_path}") + + # The following prints out WERs, per-word error statistics and aligned + # ref/hyp pairs. + errs_filename = ( + params.res_dir / f"errs-{test_set_name}-{key}-{params.suffix}.txt" + ) + with open(errs_filename, "w") as f: + wer = write_error_stats( + f, f"{test_set_name}-{key}", results, enable_log=True + ) + test_set_wers[key] = wer + + logging.info("Wrote detailed error stats to {}".format(errs_filename)) + + test_set_wers = sorted(test_set_wers.items(), key=lambda x: x[1]) + errs_info = ( + params.res_dir / f"wer-summary-{test_set_name}-{key}-{params.suffix}.txt" + ) + with open(errs_info, "w") as f: + print("settings\tWER", file=f) + for key, val in test_set_wers: + print("{}\t{}".format(key, val), file=f) + + s = "\nFor {}, WER of different settings are:\n".format(test_set_name) + note = "\tbest for {}".format(test_set_name) + for key, val in test_set_wers: + s += "{}\t{}{}\n".format(key, val, note) + note = "" + logging.info(s) + + @torch.no_grad() def main(): parser = get_parser() @@ -430,16 +720,20 @@ def main(): params = get_params() params.update(vars(args)) - if not params.res_dir: - params.res_dir = params.exp_dir / "streaming" / params.decoding_method + params.res_dir = params.exp_dir / "streaming" / params.decoding_method if params.iter > 0: params.suffix = f"iter-{params.iter}-avg-{params.avg}" else: params.suffix = f"epoch-{params.epoch}-avg-{params.avg}" - # for streaming - params.suffix += f"-streaming-chunk-size-{params.decode_chunk_len}" + assert params.causal, params.causal + assert "," not in params.chunk_size, "chunk_size should be one value in decoding." + assert ( + "," not in params.left_context_frames + ), "left_context_frames should be one value in decoding." + params.suffix += f"-chunk-{params.chunk_size}" + params.suffix += f"-left-context-{params.left_context_frames}" # for fast_beam_search if params.decoding_method == "fast_beam_search": @@ -455,21 +749,21 @@ def main(): device = torch.device("cpu") if torch.cuda.is_available(): - device = torch.device("cuda", params.gpu) + device = torch.device("cuda", 0) logging.info(f"Device: {device}") - sp = Tokenizer.load(params.lang, params.lang_type) + sp_token = Tokenizer.load(params.lang, params.lang_type) - # and is defined in local/prepare_lang_char.py - params.blank_id = sp.piece_to_id("") - params.unk_id = sp.piece_to_id("") - params.vocab_size = sp.get_piece_size() + # and is defined in local/train_bpe_model.py + params.blank_id = sp_token.piece_to_id("") + params.unk_id = sp_token.piece_to_id("") + params.vocab_size = sp_token.get_piece_size() logging.info(params) logging.info("About to create model") - model = get_transducer_model(params) + model = get_model(params) if not params.use_averaged_model: if params.iter > 0: @@ -553,42 +847,51 @@ def main(): model.device = device decoding_graph = None - if params.decoding_graph: - decoding_graph = k2.Fsa.from_dict( - torch.load(params.decoding_graph, map_location=device) - ) - elif params.decoding_method == "fast_beam_search": + if params.decoding_method == "fast_beam_search": decoding_graph = k2.trivial_graph(params.vocab_size - 1, device=device) num_param = sum([p.numel() for p in model.parameters()]) logging.info(f"Number of model parameters: {num_param}") + # we need cut ids to display recognition results. args.return_cuts = True reazonspeech_corpus = ReazonSpeechAsrDataModule(args) - for subdir in ["valid"]: + valid_cuts = reazonspeech_corpus.valid_cuts() + test_cuts = reazonspeech_corpus.test_cuts() + + test_sets = ["valid", "test"] + test_cuts = [valid_cuts, test_cuts] + + for test_set, test_cut in zip(test_sets, test_cuts): results_dict = decode_dataset( - cuts=getattr(reazonspeech_corpus, f"{subdir}_cuts")(), + cuts=test_cut, params=params, model=model, - sp=sp, + tokenizer=sp_token, decoding_graph=decoding_graph, ) - tot_err = save_results( - params=params, test_set_name=subdir, results_dict=results_dict + save_results( + params=params, + test_set_name=test_set, + results_dict=results_dict, ) - with ( - params.res_dir - / ( - f"{subdir}-{params.decode_chunk_len}" - f"_{params.avg}_{params.epoch}.cer" - ) - ).open("w") as fout: - if len(tot_err) == 1: - fout.write(f"{tot_err[0][1]}") - else: - fout.write("\n".join(f"{k}\t{v}") for k, v in tot_err) + # valid_cuts = reazonspeech_corpus.valid_cuts() + + # for valid_cut in valid_cuts: + # results_dict = decode_dataset( + # cuts=valid_cut, + # params=params, + # model=model, + # sp=sp, + # decoding_graph=decoding_graph, + # ) + # save_results( + # params=params, + # test_set_name="valid", + # results_dict=results_dict, + # ) logging.info("Done!") diff --git a/icefall/__init__.py b/icefall/__init__.py index b1e4313e9..3077b8162 100644 --- a/icefall/__init__.py +++ b/icefall/__init__.py @@ -68,6 +68,7 @@ from .utils import ( str2bool, subsequent_chunk_mask, tokenize_by_CJK_char, + tokenize_by_ja_char, write_error_stats, ) diff --git a/icefall/utils.py b/icefall/utils.py index 41eebadd4..aab479e56 100644 --- a/icefall/utils.py +++ b/icefall/utils.py @@ -1758,6 +1758,30 @@ def tokenize_by_CJK_char(line: str) -> str: return " ".join([w.strip() for w in chars if w.strip()]) +def tokenize_by_ja_char(line: str) -> str: + """ + Tokenize a line of text with Japanese characters. + + Note: All non-Japanese characters will be upper case. + + Example: + input = "こんにちは世界は hello world の日本語" + output = "こ ん に ち は 世 界 は HELLO WORLD の 日 本 語" + + Args: + line: + The input text. + + Return: + A new string tokenized by Japanese characters. + """ + pattern = re.compile(r"([\u3040-\u309F\u30A0-\u30FF\u4E00-\u9FFF])") + chars = pattern.split(line.strip()) + return " ".join( + [w.strip().upper() if not pattern.match(w) else w for w in chars if w.strip()] + ) + + def display_and_save_batch( batch: dict, params: AttributeDict, From da597ad782d987220ecabff824f11f5fa46a8f09 Mon Sep 17 00:00:00 2001 From: Machiko Bailey <53164945+baileyeet@users.noreply.github.com> Date: Mon, 3 Feb 2025 20:04:25 -0500 Subject: [PATCH 31/59] Update RESULTS.md (#1873) --- egs/multi_ja_en/ASR/RESULTS.md | 1 - 1 file changed, 1 deletion(-) diff --git a/egs/multi_ja_en/ASR/RESULTS.md b/egs/multi_ja_en/ASR/RESULTS.md index c1bbb9ac5..0f6996013 100644 --- a/egs/multi_ja_en/ASR/RESULTS.md +++ b/egs/multi_ja_en/ASR/RESULTS.md @@ -40,7 +40,6 @@ Word Error Rates (WERs) listed below: | Zipformer WER (%) | dev | test | test-clean | test-other | | greedy_search | 5.9 | 4.07 | 3.46 | 8.35 | | modified_beam_search | 4.87 | 3.61 | 3.28 | 8.07 | -| fast_beam_search | 41.04 | 36.59 | 16.14 | 22.0 | Character Error Rates (CERs) for Japanese listed below: From 2ba665abca4d13a995a14d50a3db137a367c1892 Mon Sep 17 00:00:00 2001 From: Yuekai Zhang Date: Mon, 24 Feb 2025 13:58:47 +0800 Subject: [PATCH 32/59] Add F5-TTS with semantic token training results (#1880) * add cosy token * update inference code * add extract cosy token * update results * add requirements.txt * update readme --------- Co-authored-by: yuekaiz Co-authored-by: yuekaiz --- egs/wenetspeech4tts/TTS/README.md | 52 ++ egs/wenetspeech4tts/TTS/f5-tts/infer.py | 494 +++++++++++++++++- .../TTS/f5-tts/requirements.txt | 36 ++ .../TTS/f5-tts/speech_synthesis.py | 7 +- egs/wenetspeech4tts/TTS/f5-tts/train.py | 69 ++- .../TTS/f5-tts/tts_datamodule.py | 6 +- .../TTS/local/attach_speech_tokens.py | 108 ++++ egs/wenetspeech4tts/TTS/prepare.sh | 26 +- 8 files changed, 770 insertions(+), 28 deletions(-) create mode 100644 egs/wenetspeech4tts/TTS/f5-tts/requirements.txt create mode 100644 egs/wenetspeech4tts/TTS/local/attach_speech_tokens.py diff --git a/egs/wenetspeech4tts/TTS/README.md b/egs/wenetspeech4tts/TTS/README.md index d47687acd..8329ae948 100644 --- a/egs/wenetspeech4tts/TTS/README.md +++ b/egs/wenetspeech4tts/TTS/README.md @@ -1,3 +1,10 @@ +# Results +| Model | Seed-TTS test_zh CER | Comment | +|---------------------------------------|---------------------|--------| +| [vall-e](./valle) | 4.33% | ~150M | +| [f5-tts](./f5-tts) | 3.02% (16 steps) / 2.42% (32 steps) | F5-TTS-Small Config, ~155M | +| [f5-tts-semantic-token](./f5-tts) | 1.79% (16 steps) | Using pretrained cosyvoice2 semantic tokens as inputs rather than text tokens, ~155M | + # Introduction [**WenetSpeech4TTS**](https://huggingface.co/datasets/Wenetspeech4TTS/WenetSpeech4TTS) is a multi-domain **Mandarin** corpus derived from the open-sourced [WenetSpeech](https://arxiv.org/abs/2110.03370) dataset. @@ -131,6 +138,51 @@ accelerate launch f5-tts/infer.py --nfe 16 --model-path $model_path --manifest-f bash local/compute_wer.sh $output_dir $manifest ``` +# F5-TTS-Semantic-Token + +./f5-tts contains the code for training F5-TTS-Semantic-Token. We replaced the text tokens in F5-TTS with pretrained cosyvoice2 semantic tokens. During inference, we use the pretrained CosyVoice2 LLM to predict the semantic tokens for target audios. We observed that this approach leads to faster convergence and improved prosody modeling results. + +Generated samples and training logs of wenetspeech basic 7k hours data can be found [here](https://huggingface.co/yuekai/f5-tts-semantic-token-small-wenetspeech4tts-basic/tree/main). + +Preparation: + +``` +# extract cosyvoice2 semantic tokens +bash prepare.sh --stage 5 --stop_stage 7 +``` + +The training command is given below: + +``` +# docker: ghcr.io/swivid/f5-tts:main +# pip install k2==1.24.4.dev20241030+cuda12.4.torch2.4.0 -f https://k2-fsa.github.io/k2/cuda.html +# pip install kaldialign lhotse tensorboard bigvganinference sentencepiece + +world_size=8 +exp_dir=exp/f5-tts-semantic-token-small +python3 f5-tts/train.py --max-duration 700 --filter-min-duration 0.5 --filter-max-duration 20 \ + --num-buckets 6 --dtype "bfloat16" --save-every-n 5000 --valid-interval 10000 \ + --base-lr 1e-4 --warmup-steps 20000 --average-period 0 \ + --num-epochs 10 --start-epoch 1 --start-batch 0 \ + --num-decoder-layers 18 --nhead 12 --decoder-dim 768 \ + --exp-dir ${exp_dir} --world-size ${world_size} \ + --decay-steps 600000 --prefix wenetspeech4tts_cosy_token --use-cosyvoice-semantic-token True +``` + +To inference with Icefall Wenetspeech4TTS trained F5-Small-Semantic-Token, use: +``` +huggingface-cli login +huggingface-cli download --local-dir ${exp_dir} yuekai/f5-tts-semantic-token-small-wenetspeech4tts-basic +huggingface-cli download nvidia/bigvgan_v2_24khz_100band_256x --local-dir bigvgan_v2_24khz_100band_256x + +split=test_zh +model_path=f5-tts-small-wenetspeech4tts-basic/epoch-10-avg-5.pt + +accelerate launch f5-tts/infer.py --nfe 16 --model-path $model_path --split-name $split --output-dir $output_dir --decoder-dim 768 --nhead 12 --num-decoder-layers 18 --use-cosyvoice-semantic-token True +bash local/compute_wer.sh $output_dir $manifest +``` + # Credits - [VALL-E](https://github.com/lifeiteng/vall-e) - [F5-TTS](https://github.com/SWivid/F5-TTS) +- [CosyVoice](https://github.com/FunAudioLLM/CosyVoice) diff --git a/egs/wenetspeech4tts/TTS/f5-tts/infer.py b/egs/wenetspeech4tts/TTS/f5-tts/infer.py index 02e5f0f4d..6964a43be 100644 --- a/egs/wenetspeech4tts/TTS/f5-tts/infer.py +++ b/egs/wenetspeech4tts/TTS/f5-tts/infer.py @@ -11,7 +11,14 @@ python3 f5-tts/generate_averaged_model.py \ --epoch 56 \ --avg 14 --decoder-dim 768 --nhead 12 --num-decoder-layers 18 \ --exp-dir exp/f5_small + +# command for text token input accelerate launch f5-tts/infer.py --nfe 16 --model-path $model_path --manifest-file $manifest --output-dir $output_dir --decoder-dim 768 --nhead 12 --num-decoder-layers 18 + +# command for cosyvoice semantic token input +split=test_zh # seed_tts_eval test_zh +accelerate launch f5-tts/infer.py --nfe 16 --model-path $model_path --split-name $split --output-dir $output_dir --decoder-dim 768 --nhead 12 --num-decoder-layers 18 --use-cosyvoice-semantic-token True + bash local/compute_wer.sh $output_dir $manifest """ import argparse @@ -22,6 +29,7 @@ import random import time from pathlib import Path +import datasets import torch import torch.nn.functional as F import torchaudio @@ -36,10 +44,12 @@ from train import ( add_model_arguments, get_model, get_tokenizer, + interpolate_tokens, load_F5_TTS_pretrained_checkpoint, ) from icefall.checkpoint import load_checkpoint +from icefall.utils import str2bool def get_parser(): @@ -78,7 +88,7 @@ def get_parser(): parser.add_argument( "--manifest-file", type=str, - default="/path/seed_tts_eval/seedtts_testset/zh/meta.lst", + default=None, help="The manifest file in seed_tts_eval format", ) @@ -90,6 +100,29 @@ def get_parser(): ) parser.add_argument("-ss", "--swaysampling", default=-1, type=float) + + parser.add_argument( + "--interpolate-token", + type=str2bool, + default=True, + help="Interpolate semantic token to match mel frames for CosyVoice", + ) + + parser.add_argument( + "--use-cosyvoice-semantic-token", + type=str2bool, + default=False, + help="Whether to use cosyvoice semantic token to replace text token.", + ) + + parser.add_argument( + "--split-name", + type=str, + default="wenetspeech4tts", + choices=["wenetspeech4tts", "test_zh", "test_en", "test_hard"], + help="huggingface dataset split name", + ) + add_model_arguments(parser) return parser.parse_args() @@ -243,6 +276,392 @@ def get_inference_prompt( return prompts_all +def get_inference_prompt_cosy_voice_huggingface( + dataset, + speed=1.0, + tokenizer="pinyin", + polyphone=True, + target_sample_rate=24000, + n_fft=1024, + win_length=1024, + n_mel_channels=100, + hop_length=256, + mel_spec_type="bigvgan", + target_rms=0.1, + use_truth_duration=False, + infer_batch_size=1, + num_buckets=200, + min_secs=3, + max_secs=40, + interpolate_token=False, +): + prompts_all = [] + + min_tokens = min_secs * target_sample_rate // hop_length + max_tokens = max_secs * target_sample_rate // hop_length + + batch_accum = [0] * num_buckets + utts, ref_rms_list, ref_mels, ref_mel_lens, total_mel_lens, final_text_list = ( + [[] for _ in range(num_buckets)] for _ in range(6) + ) + + mel_spectrogram = MelSpec( + n_fft=n_fft, + hop_length=hop_length, + win_length=win_length, + n_mel_channels=n_mel_channels, + target_sample_rate=target_sample_rate, + mel_spec_type=mel_spec_type, + ) + + for i in range(len(dataset)): + utt = dataset[i]["id"] + ref_audio_org, ref_sr = ( + dataset[i]["prompt_audio"]["array"], + dataset[i]["prompt_audio"]["sampling_rate"], + ) + ref_audio_org = torch.from_numpy(ref_audio_org).unsqueeze(0).float() + audio_tokens = dataset[i]["target_audio_cosy2_tokens"] + prompt_audio_tokens = dataset[i]["prompt_audio_cosy2_tokens"] + + ref_rms = torch.sqrt(torch.mean(torch.square(ref_audio_org))) + if ref_rms < target_rms: + ref_audio_org = ref_audio_org * target_rms / ref_rms + + if ref_sr != target_sample_rate: + resampler = torchaudio.transforms.Resample(ref_sr, target_sample_rate) + ref_audio = resampler(ref_audio_org) + else: + ref_audio = ref_audio_org + input_tokens = prompt_audio_tokens + audio_tokens + + if interpolate_token: + input_tokens = interpolate_tokens(input_tokens) + text_list = input_tokens + + # Duration, mel frame length + ref_mel_len = ref_audio.shape[-1] // hop_length + + total_mel_len = len(input_tokens) + if not interpolate_token: + total_mel_len = int(total_mel_len / 4 * 15) + + # to mel spectrogram + ref_mel = mel_spectrogram(ref_audio) + ref_mel = ref_mel.squeeze(0) + + # deal with batch + assert infer_batch_size > 0, "infer_batch_size should be greater than 0." + if total_mel_len > max_tokens: + print( + f"Audio {utt} has duration {total_mel_len*hop_length//target_sample_rate}s out of range [{min_secs}, {max_secs}]." + ) + continue + assert ( + min_tokens <= total_mel_len <= max_tokens + ), f"Audio {utt} has duration {total_mel_len*hop_length//target_sample_rate}s out of range [{min_secs}, {max_secs}]." + bucket_i = math.floor( + (total_mel_len - min_tokens) / (max_tokens - min_tokens + 1) * num_buckets + ) + + utts[bucket_i].append(utt) + ref_rms_list[bucket_i].append(ref_rms) + ref_mels[bucket_i].append(ref_mel) + ref_mel_lens[bucket_i].append(ref_mel_len) + total_mel_lens[bucket_i].append(total_mel_len) + # final_text_list[bucket_i].extend(text_list) + final_text_list[bucket_i].append(text_list) + + batch_accum[bucket_i] += total_mel_len + + if batch_accum[bucket_i] >= infer_batch_size: + prompts_all.append( + ( + utts[bucket_i], + ref_rms_list[bucket_i], + padded_mel_batch(ref_mels[bucket_i]), + ref_mel_lens[bucket_i], + total_mel_lens[bucket_i], + final_text_list[bucket_i], + ) + ) + batch_accum[bucket_i] = 0 + ( + utts[bucket_i], + ref_rms_list[bucket_i], + ref_mels[bucket_i], + ref_mel_lens[bucket_i], + total_mel_lens[bucket_i], + final_text_list[bucket_i], + ) = ( + [], + [], + [], + [], + [], + [], + ) + + # add residual + for bucket_i, bucket_frames in enumerate(batch_accum): + if bucket_frames > 0: + prompts_all.append( + ( + utts[bucket_i], + ref_rms_list[bucket_i], + padded_mel_batch(ref_mels[bucket_i]), + ref_mel_lens[bucket_i], + total_mel_lens[bucket_i], + final_text_list[bucket_i], + ) + ) + # not only leave easy work for last workers + random.seed(666) + random.shuffle(prompts_all) + + return prompts_all + + +def inference_speech_token( + cosyvoice, + tts_text, + prompt_text, + prompt_speech_16k, + stream=False, + speed=1.0, + text_frontend=True, +): + tokens = [] + prompt_text = cosyvoice.frontend.text_normalize( + prompt_text, split=False, text_frontend=text_frontend + ) + for i in cosyvoice.frontend.text_normalize( + tts_text, split=True, text_frontend=text_frontend + ): + + tts_text_token, tts_text_token_len = cosyvoice.frontend._extract_text_token(i) + ( + prompt_text_token, + prompt_text_token_len, + ) = cosyvoice.frontend._extract_text_token(prompt_text) + speech_token, speech_token_len = cosyvoice.frontend._extract_speech_token( + prompt_speech_16k + ) + + for i in cosyvoice.model.llm.inference( + text=tts_text_token.to(cosyvoice.model.device), + text_len=torch.tensor([tts_text_token.shape[1]], dtype=torch.int32).to( + cosyvoice.model.device + ), + prompt_text=prompt_text_token.to(cosyvoice.model.device), + prompt_text_len=torch.tensor( + [prompt_text_token.shape[1]], dtype=torch.int32 + ).to(cosyvoice.model.device), + prompt_speech_token=speech_token.to(cosyvoice.model.device), + prompt_speech_token_len=torch.tensor( + [speech_token.shape[1]], dtype=torch.int32 + ).to(cosyvoice.model.device), + embedding=None, + ): + tokens.append(i) + return tokens, speech_token + + +def get_inference_prompt_cosy_voice( + metainfo, + speed=1.0, + tokenizer="pinyin", + polyphone=True, + target_sample_rate=24000, + n_fft=1024, + win_length=1024, + n_mel_channels=100, + hop_length=256, + mel_spec_type="bigvgan", + target_rms=0.1, + use_truth_duration=False, + infer_batch_size=1, + num_buckets=200, + min_secs=3, + max_secs=40, + interpolate_token=False, +): + + import sys + + # please change the path to the cosyvoice accordingly + sys.path.append("/workspace/CosyVoice/third_party/Matcha-TTS") + sys.path.append("/workspace/CosyVoice") + from cosyvoice.cli.cosyvoice import CosyVoice2 + + # please download the cosyvoice model first + cosyvoice = CosyVoice2( + "/workspace/CosyVoice2-0.5B", load_jit=False, load_trt=False, fp16=False + ) + + prompts_all = [] + + min_tokens = min_secs * target_sample_rate // hop_length + max_tokens = max_secs * target_sample_rate // hop_length + + batch_accum = [0] * num_buckets + utts, ref_rms_list, ref_mels, ref_mel_lens, total_mel_lens, final_text_list = ( + [[] for _ in range(num_buckets)] for _ in range(6) + ) + + mel_spectrogram = MelSpec( + n_fft=n_fft, + hop_length=hop_length, + win_length=win_length, + n_mel_channels=n_mel_channels, + target_sample_rate=target_sample_rate, + mel_spec_type=mel_spec_type, + ) + + for utt, prompt_text, prompt_wav, gt_text, gt_wav in tqdm( + metainfo, desc="Processing prompts..." + ): + # Audio + ref_audio_org, ref_sr = torchaudio.load(prompt_wav) + + # cosy voice + if ref_sr != 16000: + resampler = torchaudio.transforms.Resample(ref_sr, 16000) + ref_audio_16k = resampler(ref_audio_org) + else: + ref_audio_16k = ref_audio_org + audio_tokens, prompt_audio_tokens = inference_speech_token( + cosyvoice, gt_text, prompt_text, ref_audio_16k, stream=False + ) + + ref_rms = torch.sqrt(torch.mean(torch.square(ref_audio_org))) + if ref_rms < target_rms: + ref_audio_org = ref_audio_org * target_rms / ref_rms + assert ( + ref_audio_org.shape[-1] > 5000 + ), f"Empty prompt wav: {prompt_wav}, or torchaudio backend issue." + if ref_sr != target_sample_rate: + resampler = torchaudio.transforms.Resample(ref_sr, target_sample_rate) + ref_audio = resampler(ref_audio_org) + else: + ref_audio = ref_audio_org + + # Text + # if len(prompt_text[-1].encode("utf-8")) == 1: + # prompt_text = prompt_text + " " + # text = [prompt_text + gt_text] + # if tokenizer == "pinyin": + # text_list = convert_char_to_pinyin(text, polyphone=polyphone) + # else: + # text_list = text + + # concat two tensors: prompt audio tokens with audio tokens --> shape 1, prompt_audio_tokens + audio_tokens + # prompt_audio_tokens shape 1, prompt_audio_tokens + # audio_tokens shape 1, audio_tokens + prompt_audio_tokens = prompt_audio_tokens.squeeze().cpu().tolist() + input_tokens = prompt_audio_tokens + audio_tokens + + # convert it into a list + # input_tokens_list = input_tokens.squeeze().cpu().tolist() + if interpolate_token: + input_tokens = interpolate_tokens(input_tokens) + text_list = input_tokens + + # Duration, mel frame length + ref_mel_len = ref_audio.shape[-1] // hop_length + if use_truth_duration: + gt_audio, gt_sr = torchaudio.load(gt_wav) + if gt_sr != target_sample_rate: + resampler = torchaudio.transforms.Resample(gt_sr, target_sample_rate) + gt_audio = resampler(gt_audio) + total_mel_len = ref_mel_len + int(gt_audio.shape[-1] / hop_length / speed) + + # # test vocoder resynthesis + # ref_audio = gt_audio + else: + ref_text_len = len(prompt_text.encode("utf-8")) + gen_text_len = len(gt_text.encode("utf-8")) + total_mel_len_compute = ref_mel_len + int( + ref_mel_len / ref_text_len * gen_text_len / speed + ) + total_mel_len = len(input_tokens) + if not interpolate_token: + total_mel_len = int(total_mel_len / 4 * 15) + print( + f"total_mel_len_compute: {total_mel_len_compute}, total_mel_len: {total_mel_len}" + ) + + # to mel spectrogram + ref_mel = mel_spectrogram(ref_audio) + ref_mel = ref_mel.squeeze(0) + + # deal with batch + assert infer_batch_size > 0, "infer_batch_size should be greater than 0." + assert ( + min_tokens <= total_mel_len <= max_tokens + ), f"Audio {utt} has duration {total_mel_len*hop_length//target_sample_rate}s out of range [{min_secs}, {max_secs}]." + bucket_i = math.floor( + (total_mel_len - min_tokens) / (max_tokens - min_tokens + 1) * num_buckets + ) + + utts[bucket_i].append(utt) + ref_rms_list[bucket_i].append(ref_rms) + ref_mels[bucket_i].append(ref_mel) + ref_mel_lens[bucket_i].append(ref_mel_len) + total_mel_lens[bucket_i].append(total_mel_len) + # final_text_list[bucket_i].extend(text_list) + final_text_list[bucket_i].append(text_list) + + batch_accum[bucket_i] += total_mel_len + + if batch_accum[bucket_i] >= infer_batch_size: + prompts_all.append( + ( + utts[bucket_i], + ref_rms_list[bucket_i], + padded_mel_batch(ref_mels[bucket_i]), + ref_mel_lens[bucket_i], + total_mel_lens[bucket_i], + final_text_list[bucket_i], + ) + ) + batch_accum[bucket_i] = 0 + ( + utts[bucket_i], + ref_rms_list[bucket_i], + ref_mels[bucket_i], + ref_mel_lens[bucket_i], + total_mel_lens[bucket_i], + final_text_list[bucket_i], + ) = ( + [], + [], + [], + [], + [], + [], + ) + + # add residual + for bucket_i, bucket_frames in enumerate(batch_accum): + if bucket_frames > 0: + prompts_all.append( + ( + utts[bucket_i], + ref_rms_list[bucket_i], + padded_mel_batch(ref_mels[bucket_i]), + ref_mel_lens[bucket_i], + total_mel_lens[bucket_i], + final_text_list[bucket_i], + ) + ) + # not only leave easy work for last workers + random.seed(666) + random.shuffle(prompts_all) + + return prompts_all + + def padded_mel_batch(ref_mels): max_mel_length = torch.LongTensor([mel.shape[-1] for mel in ref_mels]).amax() padded_ref_mels = [] @@ -275,20 +694,55 @@ def main(): accelerator = Accelerator() device = f"cuda:{accelerator.process_index}" - - metainfo = get_seedtts_testset_metainfo(args.manifest_file) - prompts_all = get_inference_prompt( - metainfo, - speed=1.0, - tokenizer="pinyin", - target_sample_rate=24_000, - n_mel_channels=100, - hop_length=256, - mel_spec_type="bigvgan", - target_rms=0.1, - use_truth_duration=False, - infer_batch_size=1, - ) + if args.manifest_file: + metainfo = get_seedtts_testset_metainfo(args.manifest_file) + if not args.use_cosyvoice_semantic_token: + prompts_all = get_inference_prompt( + metainfo, + speed=1.0, + tokenizer="pinyin", + target_sample_rate=24_000, + n_mel_channels=100, + hop_length=256, + mel_spec_type="bigvgan", + target_rms=0.1, + use_truth_duration=False, + infer_batch_size=1, + ) + else: + prompts_all = get_inference_prompt_cosy_voice( + metainfo, + speed=1.0, + tokenizer="pinyin", + target_sample_rate=24_000, + n_mel_channels=100, + hop_length=256, + mel_spec_type="bigvgan", + target_rms=0.1, + use_truth_duration=False, + infer_batch_size=1, + interpolate_token=args.interpolate_token, + ) + else: + assert args.use_cosyvoice_semantic_token + dataset = datasets.load_dataset( + "yuekai/seed_tts_cosy2", + split=args.split_name, + trust_remote_code=True, + ) + prompts_all = get_inference_prompt_cosy_voice_huggingface( + dataset, + speed=1.0, + tokenizer="pinyin", + target_sample_rate=24_000, + n_mel_channels=100, + hop_length=256, + mel_spec_type="bigvgan", + target_rms=0.1, + use_truth_duration=False, + infer_batch_size=1, + interpolate_token=args.interpolate_token, + ) vocoder = BigVGANInference.from_pretrained( "./bigvgan_v2_24khz_100band_256x", use_cuda_kernel=False @@ -324,6 +778,16 @@ def main(): ref_mel_lens = torch.tensor(ref_mel_lens, dtype=torch.long).to(device) total_mel_lens = torch.tensor(total_mel_lens, dtype=torch.long).to(device) + if args.use_cosyvoice_semantic_token: + # concat final_text_list + max_len = max([len(tokens) for tokens in final_text_list]) + # pad tokens to the same length + for i, tokens in enumerate(final_text_list): + final_text_list[i] = torch.tensor( + tokens + [-1] * (max_len - len(tokens)), dtype=torch.long + ) + final_text_list = torch.stack(final_text_list).to(device) + # Inference with torch.inference_mode(): generated, _ = model.sample( diff --git a/egs/wenetspeech4tts/TTS/f5-tts/requirements.txt b/egs/wenetspeech4tts/TTS/f5-tts/requirements.txt new file mode 100644 index 000000000..63f1e237c --- /dev/null +++ b/egs/wenetspeech4tts/TTS/f5-tts/requirements.txt @@ -0,0 +1,36 @@ +# F5-TTS +accelerate>=0.33.0 +bitsandbytes>0.37.0 +cached_path +click +datasets +ema_pytorch>=0.5.2 +gradio>=3.45.2 +hydra-core>=1.3.0 +jieba +librosa +matplotlib +numpy<=1.26.4 +pydub +pypinyin +safetensors +soundfile +tomli +torch>=2.0.0 +torchaudio>=2.0.0 +torchdiffeq +tqdm>=4.65.0 +transformers +x_transformers>=1.31.14 + +# icefall +kaldialign +lhotse +tensorboard +bigvganinference +sentencepiece +sherpa-onnx +k2 + +# semantic experiment +s3tokenizer diff --git a/egs/wenetspeech4tts/TTS/f5-tts/speech_synthesis.py b/egs/wenetspeech4tts/TTS/f5-tts/speech_synthesis.py index 57f677fcb..7d42a00a5 100644 --- a/egs/wenetspeech4tts/TTS/f5-tts/speech_synthesis.py +++ b/egs/wenetspeech4tts/TTS/f5-tts/speech_synthesis.py @@ -82,9 +82,12 @@ class SpeechSynthesisDataset(torch.utils.data.Dataset): text = [cut.supervisions[0].text for cut in cuts] batch["text"] = text - if self.return_tokens: + if self.return_tokens and "speech_tokens" in cuts[0].supervisions[0].custom: # tokens = [cut.tokens for cut in cuts] - tokens = [cut.supervisions[0].custom["tokens"]["text"] for cut in cuts] + # tokens = [cut.supervisions[0].custom["tokens"]["text"] for cut in cuts] + tokens = [cut.supervisions[0].custom["speech_tokens"] for cut in cuts] + # change str into list + tokens = [list(map(int, token.split())) for token in tokens] batch["tokens"] = tokens if self.return_spk_ids: diff --git a/egs/wenetspeech4tts/TTS/f5-tts/train.py b/egs/wenetspeech4tts/TTS/f5-tts/train.py index 37dcf531e..5333b3f27 100755 --- a/egs/wenetspeech4tts/TTS/f5-tts/train.py +++ b/egs/wenetspeech4tts/TTS/f5-tts/train.py @@ -31,6 +31,16 @@ python3 f5-tts/train.py --max-duration 700 --filter-min-duration 0.5 --filter-ma --base-lr 7.5e-5 --warmup-steps 20000 --num-epochs 60 \ --num-decoder-layers 18 --nhead 12 --decoder-dim 768 \ --exp-dir ${exp_dir} --world-size ${world_size} + +# command for training with cosyvoice semantic token +exp_dir=exp/f5-tts-cosyvoice +python3 f5-tts/train.py --max-duration 700 --filter-min-duration 0.5 --filter-max-duration 20 \ + --num-buckets 6 --dtype "bfloat16" --save-every-n 5000 --valid-interval 10000 \ + --base-lr 1e-4 --warmup-steps 20000 --average-period 0 \ + --num-epochs 10 --start-epoch 1 --start-batch 0 \ + --num-decoder-layers 18 --nhead 12 --decoder-dim 768 \ + --exp-dir ${exp_dir} --world-size ${world_size} \ + --decay-steps 600000 --prefix wenetspeech4tts_cosy_token --use-cosyvoice-semantic-token True """ import argparse @@ -303,6 +313,13 @@ def get_parser(): help="perform OOM check on dataloader batches before starting training.", ) + parser.add_argument( + "--use-cosyvoice-semantic-token", + type=str2bool, + default=False, + help="Whether to use cosyvoice semantic token to replace text token.", + ) + add_model_arguments(parser) return parser @@ -378,7 +395,11 @@ def get_tokenizer(vocab_file_path: str): def get_model(params): - vocab_char_map, vocab_size = get_tokenizer(params.tokens) + if params.use_cosyvoice_semantic_token: + # https://www.modelscope.cn/models/iic/CosyVoice2-0.5B/file/view/master?fileName=cosyvoice.yaml&status=1#L36 + vocab_char_map, vocab_size = None, 6561 + else: + vocab_char_map, vocab_size = get_tokenizer(params.tokens) # bigvgan 100 dim features n_mel_channels = 100 n_fft = 1024 @@ -556,14 +577,44 @@ def save_checkpoint( copyfile(src=filename, dst=best_valid_filename) -def prepare_input(batch: dict, device: torch.device): - """Parse batch data""" - text_inputs = batch["text"] - # texts.extend(convert_char_to_pinyin([text], polyphone=true)) - text_inputs = convert_char_to_pinyin(text_inputs, polyphone=True) +def interpolate_tokens(cosy_tokens, pad_token=-1): + """Interpolate cosyvoice tokens to match bigvgan frames length""" + # cosyvoice, 25 tokens/sec + # bigvgan sample_rate/hop_length 24000/256 frames/sec + # For every 4 cosyvoice tokens, insert pad tokens to extend it to 15 tokens to match bigvgan frames length + # We choose 4,4,4,3 to match 15 frames + three, two = [pad_token] * 3, [pad_token] * 2 + return [ + x + for i, e in enumerate(cosy_tokens) + for x in ([e] + three if i % 4 < 3 else [e] + two) + ] + +def prepare_input( + batch: dict, device: torch.device, use_cosyvoice_semantic_token: bool +): + """Parse batch data""" mel_spec = batch["features"] mel_lengths = batch["features_lens"] + + if use_cosyvoice_semantic_token: + semantic_tokens = [] + for i in range(len(batch["tokens"])): + tokens = batch["tokens"][i] + tokens = interpolate_tokens(tokens) + semantic_tokens.append(tokens) + # pad to the same length, B,T, with pad value -1 + max_len = max([len(tokens) for tokens in semantic_tokens]) + text_inputs = torch.full( + (len(semantic_tokens), max_len), -1, dtype=torch.long + ).to(device) + for i, tokens in enumerate(semantic_tokens): + text_inputs[i, : len(tokens)] = torch.tensor(tokens, dtype=torch.long) + else: + text_inputs = batch["text"] + text_inputs = convert_char_to_pinyin(text_inputs, polyphone=True) + return text_inputs, mel_spec.to(device), mel_lengths.to(device) @@ -593,7 +644,11 @@ def compute_loss( values >= 1.0 are fully warmed up and have all modules present. """ device = model.device if isinstance(model, DDP) else next(model.parameters()).device - (text_inputs, mel_spec, mel_lengths) = prepare_input(batch, device=device) + (text_inputs, mel_spec, mel_lengths) = prepare_input( + batch, + device=device, + use_cosyvoice_semantic_token=params.use_cosyvoice_semantic_token, + ) # at entry, TextTokens is (N, P) with torch.set_grad_enabled(is_training): diff --git a/egs/wenetspeech4tts/TTS/f5-tts/tts_datamodule.py b/egs/wenetspeech4tts/TTS/f5-tts/tts_datamodule.py index 80ba17318..eab7588b7 100644 --- a/egs/wenetspeech4tts/TTS/f5-tts/tts_datamodule.py +++ b/egs/wenetspeech4tts/TTS/f5-tts/tts_datamodule.py @@ -174,7 +174,7 @@ class TtsDataModule: logging.info("About to create train dataset") train = SpeechSynthesisDataset( return_text=True, - return_tokens=False, + return_tokens=True, feature_input_strategy=eval(self.args.input_strategy)(), return_cuts=self.args.return_cuts, ) @@ -234,7 +234,7 @@ class TtsDataModule: else: validate = SpeechSynthesisDataset( return_text=True, - return_tokens=False, + return_tokens=True, feature_input_strategy=eval(self.args.input_strategy)(), return_cuts=self.args.return_cuts, ) @@ -265,7 +265,7 @@ class TtsDataModule: else: test = SpeechSynthesisDataset( return_text=True, - return_tokens=False, + return_tokens=True, feature_input_strategy=eval(self.args.input_strategy)(), return_cuts=self.args.return_cuts, ) diff --git a/egs/wenetspeech4tts/TTS/local/attach_speech_tokens.py b/egs/wenetspeech4tts/TTS/local/attach_speech_tokens.py new file mode 100644 index 000000000..9904901f0 --- /dev/null +++ b/egs/wenetspeech4tts/TTS/local/attach_speech_tokens.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +# Copyright 2025 author: Yuekai Zhang +# +# See ../../../../LICENSE for clarification regarding multiple authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import argparse +import gzip +import json +import logging + +import s3tokenizer +from lhotse import CutSet, load_manifest_lazy +from tqdm import tqdm + + +def get_parser(): + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + + parser.add_argument( + "--manifest-dir", + type=str, + default="data/fbank", + help="Directory to store the manifest files", + ) + + parser.add_argument( + "--jsonl-prefix", + type=str, + default="wenetspeech4tts_cuts_valid", + help="The training subset for wenetspeech.", + ) + + parser.add_argument( + "--tokens-path", + type=str, + default="./s3_tokens_valid/wenetspeech4tts_valid.json", + help="json file containing the speech tokens", + ) + + return parser + + +def get_speech_tokens(tokens_path): + id2tokens = {} + with open(tokens_path, "r") as fin: + for line in fin: + line = json.loads(line) + id2tokens[line["key"]] = " ".join(map(str, line["code"])) + return id2tokens + + +def attach_manifest(manifest, fixed_manifest_path, id2tokens): + with CutSet.open_writer(fixed_manifest_path) as manifest_writer: + fixed_item = 0 + for i, cut in enumerate(tqdm(manifest)): + cut_id = cut.supervisions[0].id + if cut_id in id2tokens: + code = id2tokens[cut_id] + cut.supervisions[0].custom = { + **cut.supervisions[0].custom, + **{"speech_tokens": code}, + } + else: + print(f"cut_id {cut_id} not in id2tokens") + fixed_item += 1 + manifest_writer.write(cut) + logging.info(f"Fixed {fixed_item} items in the manifest") + + +def main(): + formatter = "%(asctime)s %(levelname)s [%(filename)s:%(lineno)d] %(message)s" + logging.basicConfig(format=formatter, level=logging.INFO) + + parser = get_parser() + args = parser.parse_args() + logging.info(vars(args)) + + manifest_path = args.manifest_dir + "/" + f"{args.jsonl_prefix}.jsonl.gz" + attached_manifest_path = ( + args.manifest_dir + "/" + f"{args.jsonl_prefix}_attached_cosyvoice_v2.jsonl.gz" + ) + logging.info(f"Loading manifest from {manifest_path}") + cuts_manifest = load_manifest_lazy(manifest_path) + logging.info(f"Loading manifest from {manifest_path} done") + id2tokens = get_speech_tokens(args.tokens_path) + logging.info(f"Loaded id2tokens with {len(id2tokens)} entries") + + attach_manifest(cuts_manifest, attached_manifest_path, id2tokens) + logging.info( + f"Manifest with speech tokens attached is saved to {attached_manifest_path}" + ) + + +if __name__ == "__main__": + main() diff --git a/egs/wenetspeech4tts/TTS/prepare.sh b/egs/wenetspeech4tts/TTS/prepare.sh index cf86b3fa5..f1daa0e62 100755 --- a/egs/wenetspeech4tts/TTS/prepare.sh +++ b/egs/wenetspeech4tts/TTS/prepare.sh @@ -111,7 +111,7 @@ if [ $stage -le 5 ] && [ $stop_stage -ge 5 ]; then fi if [ $stage -le 6 ] && [ $stop_stage -ge 6 ]; then - log "Stage 7: Split the ${prefix} cuts into train, valid and test sets (used by ./f5-tts)" + log "Stage 6: Split the ${prefix} cuts into train, valid and test sets (used by ./f5-tts)" if [ ! -f data/fbank/${prefix}_cuts_${subset}.jsonl.gz ]; then echo "Combining ${prefix} cuts" pieces=$(find data/fbank/ -name "${prefix}_cuts_${subset}.*.jsonl.gz") @@ -139,3 +139,27 @@ if [ $stage -le 6 ] && [ $stop_stage -ge 6 ]; then touch data/fbank/.${prefix}_split.done fi fi + +if [ $stage -le 7 ] && [ $stop_stage -ge 7 ]; then + log "Stage 7: Extract cosyvoice2 FSQ token (used by ./f5-tts semantic token experiment)" + split_name=("valid" "test" "train") + for split in "${split_name[@]}"; do + echo "Processing $split" + wav_scp_file=wav_${split}.scp + output_dir="./cosy_v2_tokens_${split}" + oringinal_jsonl_file=data/fbank/${prefix}_cuts_${split}.jsonl.gz + mkdir -p $output_dir + zcat $oringinal_jsonl_file | jq -r '.recording.id + " " + .recording.sources[0].source' > $wav_scp_file + torchrun --nproc_per_node=8 --nnodes=1 \ + --rdzv_id=2024 --rdzv_backend="c10d" --rdzv_endpoint="localhost:0" \ + `which s3tokenizer` --wav_scp $wav_scp_file \ + --device "cuda" \ + --output_dir $output_dir \ + --batch_size 32 \ + --num_workers 4 \ + --model "speech_tokenizer_v2_25hz" # or "speech_tokenizer_v1_25hz + + cat $output_dir/* > $output_dir/${prefix}_${split}_cosy_v2_tokens.json + python3 local/attach_speech_tokens.py --jsonl-prefix ${prefix}_cuts_${split} --tokens-path $output_dir/${prefix}_${split}_cosy_v2_tokens.json --manifest-dir data/fbank + done +fi From db9fb8ad3113aaa54c81f71c3d64ea445fe3dd5d Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Thu, 27 Feb 2025 17:10:58 +0800 Subject: [PATCH 33/59] Add scripts to export streaming zipformer(v1) to RKNN (#1882) --- .../scripts/docker/generate_build_matrix.py | 26 +- .github/scripts/librispeech/ASR/run_rknn.sh | 200 +++++++++ .github/workflows/rknn.yml | 180 ++++++++ .../export-onnx-zh.py | 85 ++-- .../export_rknn.py | 261 +++++++++++ .../onnx_pretrained.py | 21 + .../test_rknn_on_cpu_simulator.py | 413 ++++++++++++++++++ 7 files changed, 1155 insertions(+), 31 deletions(-) create mode 100755 .github/scripts/librispeech/ASR/run_rknn.sh create mode 100644 .github/workflows/rknn.yml create mode 100755 egs/librispeech/ASR/pruned_transducer_stateless7_streaming/export_rknn.py create mode 100755 egs/librispeech/ASR/pruned_transducer_stateless7_streaming/test_rknn_on_cpu_simulator.py diff --git a/.github/scripts/docker/generate_build_matrix.py b/.github/scripts/docker/generate_build_matrix.py index a516a53c5..638e19498 100755 --- a/.github/scripts/docker/generate_build_matrix.py +++ b/.github/scripts/docker/generate_build_matrix.py @@ -10,7 +10,17 @@ def get_args(): parser = argparse.ArgumentParser() parser.add_argument( "--min-torch-version", - help="Minimu torch version", + help="torch version", + ) + + parser.add_argument( + "--torch-version", + help="torch version", + ) + + parser.add_argument( + "--python-version", + help="python version", ) return parser.parse_args() @@ -52,7 +62,7 @@ def get_torchaudio_version(torch_version): return torch_version -def get_matrix(min_torch_version): +def get_matrix(min_torch_version, specified_torch_version, specified_python_version): k2_version = "1.24.4.dev20241029" kaldifeat_version = "1.25.5.dev20241029" version = "20241218" @@ -71,6 +81,12 @@ def get_matrix(min_torch_version): torch_version += ["2.5.0"] torch_version += ["2.5.1"] + if specified_torch_version: + torch_version = [specified_torch_version] + + if specified_python_version: + python_version = [specified_python_version] + matrix = [] for p in python_version: for t in torch_version: @@ -115,7 +131,11 @@ def get_matrix(min_torch_version): def main(): args = get_args() - matrix = get_matrix(min_torch_version=args.min_torch_version) + matrix = get_matrix( + min_torch_version=args.min_torch_version, + specified_torch_version=args.torch_version, + specified_python_version=args.python_version, + ) print(json.dumps({"include": matrix})) diff --git a/.github/scripts/librispeech/ASR/run_rknn.sh b/.github/scripts/librispeech/ASR/run_rknn.sh new file mode 100755 index 000000000..304471724 --- /dev/null +++ b/.github/scripts/librispeech/ASR/run_rknn.sh @@ -0,0 +1,200 @@ +#!/usr/bin/env bash + +set -ex + +python3 -m pip install kaldi-native-fbank soundfile librosa + +log() { + # This function is from espnet + local fname=${BASH_SOURCE[1]##*/} + echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*" +} + +cd egs/librispeech/ASR + + +# https://huggingface.co/csukuangfj/k2fsa-zipformer-chinese-english-mixed +# sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20 +function export_bilingual_zh_en() { + d=exp_zh_en + + mkdir $d + pushd $d + + curl -SL -O https://huggingface.co/csukuangfj/k2fsa-zipformer-chinese-english-mixed/resolve/main/exp/pretrained.pt + mv pretrained.pt epoch-99.pt + + curl -SL -O https://huggingface.co/csukuangfj/k2fsa-zipformer-chinese-english-mixed/resolve/main/data/lang_char_bpe/tokens.txt + + curl -SL -O https://huggingface.co/csukuangfj/k2fsa-zipformer-chinese-english-mixed/resolve/main/test_wavs/0.wav + curl -SL -O https://huggingface.co/csukuangfj/k2fsa-zipformer-chinese-english-mixed/resolve/main/test_wavs/1.wav + curl -SL -O https://huggingface.co/csukuangfj/k2fsa-zipformer-chinese-english-mixed/resolve/main/test_wavs/2.wav + curl -SL -O https://huggingface.co/csukuangfj/k2fsa-zipformer-chinese-english-mixed/resolve/main/test_wavs/3.wav + curl -SL -O https://huggingface.co/csukuangfj/k2fsa-zipformer-chinese-english-mixed/resolve/main/test_wavs/4.wav + ls -lh + popd + + ./pruned_transducer_stateless7_streaming/export-onnx-zh.py \ + --dynamic-batch 0 \ + --enable-int8-quantization 0 \ + --tokens $d/tokens.txt \ + --use-averaged-model 0 \ + --epoch 99 \ + --avg 1 \ + --exp-dir $d/ \ + --decode-chunk-len 64 \ + --num-encoder-layers "2,4,3,2,4" \ + --feedforward-dims "1024,1024,1536,1536,1024" \ + --nhead "8,8,8,8,8" \ + --encoder-dims "384,384,384,384,384" \ + --attention-dims "192,192,192,192,192" \ + --encoder-unmasked-dims "256,256,256,256,256" \ + --zipformer-downsampling-factors "1,2,4,8,2" \ + --cnn-module-kernels "31,31,31,31,31" \ + --decoder-dim 512 \ + --joiner-dim 512 + + ls -lh $d/ + + ./pruned_transducer_stateless7_streaming/onnx_pretrained.py \ + --encoder-model-filename $d/encoder-epoch-99-avg-1.onnx \ + --decoder-model-filename $d/decoder-epoch-99-avg-1.onnx \ + --joiner-model-filename $d/joiner-epoch-99-avg-1.onnx \ + --tokens $d/tokens.txt \ + $d/0.wav + + ./pruned_transducer_stateless7_streaming/onnx_pretrained.py \ + --encoder-model-filename $d/encoder-epoch-99-avg-1.onnx \ + --decoder-model-filename $d/decoder-epoch-99-avg-1.onnx \ + --joiner-model-filename $d/joiner-epoch-99-avg-1.onnx \ + --tokens $d/tokens.txt \ + $d/1.wav + + mkdir -p /icefall/rknn-models + + for platform in rk3562 rk3566 rk3568 rk3576 rk3588; do + mkdir -p $platform + + ./pruned_transducer_stateless7_streaming/export_rknn.py \ + --in-encoder $d/encoder-epoch-99-avg-1.onnx \ + --in-decoder $d/decoder-epoch-99-avg-1.onnx \ + --in-joiner $d/joiner-epoch-99-avg-1.onnx \ + --out-encoder $platform/encoder.rknn \ + --out-decoder $platform/decoder.rknn \ + --out-joiner $platform/joiner.rknn \ + --target-platform $platform 2>/dev/null + + ls -lh $platform/ + + ./pruned_transducer_stateless7_streaming/test_rknn_on_cpu_simulator.py \ + --encoder $d/encoder-epoch-99-avg-1.onnx \ + --decoder $d/decoder-epoch-99-avg-1.onnx \ + --joiner $d/joiner-epoch-99-avg-1.onnx \ + --tokens $d/tokens.txt \ + --wav $d/0.wav + + cp $d/tokens.txt $platform + cp $d/*.wav $platform + + cp -av $platform /icefall/rknn-models + done + + ls -lh /icefall/rknn-models +} + +# https://huggingface.co/csukuangfj/k2fsa-zipformer-bilingual-zh-en-t +# sherpa-onnx-streaming-zipformer-small-bilingual-zh-en-2023-02-16 +function export_bilingual_zh_en_small() { + d=exp_zh_en_small + + mkdir $d + pushd $d + + curl -SL -O https://huggingface.co/csukuangfj/k2fsa-zipformer-bilingual-zh-en-t/resolve/main/exp/pretrained.pt + mv pretrained.pt epoch-99.pt + + curl -SL -O https://huggingface.co/csukuangfj/k2fsa-zipformer-bilingual-zh-en-t/resolve/main/data/lang_char_bpe/tokens.txt + curl -SL -O https://huggingface.co/csukuangfj/k2fsa-zipformer-bilingual-zh-en-t/resolve/main/test_wavs/0.wav + curl -SL -O https://huggingface.co/csukuangfj/k2fsa-zipformer-bilingual-zh-en-t/resolve/main/test_wavs/1.wav + curl -SL -O https://huggingface.co/csukuangfj/k2fsa-zipformer-bilingual-zh-en-t/resolve/main/test_wavs/2.wav + curl -SL -O https://huggingface.co/csukuangfj/k2fsa-zipformer-bilingual-zh-en-t/resolve/main/test_wavs/3.wav + curl -SL -O https://huggingface.co/csukuangfj/k2fsa-zipformer-bilingual-zh-en-t/resolve/main/test_wavs/4.wav + + ls -lh + + popd + + + ./pruned_transducer_stateless7_streaming/export-onnx-zh.py \ + --dynamic-batch 0 \ + --enable-int8-quantization 0 \ + --tokens $d/tokens.txt \ + --use-averaged-model 0 \ + --epoch 99 \ + --avg 1 \ + --exp-dir $d/ \ + --decode-chunk-len 64 \ + \ + --num-encoder-layers 2,2,2,2,2 \ + --feedforward-dims 768,768,768,768,768 \ + --nhead 4,4,4,4,4 \ + --encoder-dims 256,256,256,256,256 \ + --attention-dims 192,192,192,192,192 \ + --encoder-unmasked-dims 192,192,192,192,192 \ + \ + --zipformer-downsampling-factors "1,2,4,8,2" \ + --cnn-module-kernels "31,31,31,31,31" \ + --decoder-dim 512 \ + --joiner-dim 512 + + ls -lh $d/ + + ./pruned_transducer_stateless7_streaming/onnx_pretrained.py \ + --encoder-model-filename $d/encoder-epoch-99-avg-1.onnx \ + --decoder-model-filename $d/decoder-epoch-99-avg-1.onnx \ + --joiner-model-filename $d/joiner-epoch-99-avg-1.onnx \ + --tokens $d/tokens.txt \ + $d/0.wav + + ./pruned_transducer_stateless7_streaming/onnx_pretrained.py \ + --encoder-model-filename $d/encoder-epoch-99-avg-1.onnx \ + --decoder-model-filename $d/decoder-epoch-99-avg-1.onnx \ + --joiner-model-filename $d/joiner-epoch-99-avg-1.onnx \ + --tokens $d/tokens.txt \ + $d/1.wav + + mkdir -p /icefall/rknn-models-small + + for platform in rk3562 rk3566 rk3568 rk3576 rk3588; do + mkdir -p $platform + + ./pruned_transducer_stateless7_streaming/export_rknn.py \ + --in-encoder $d/encoder-epoch-99-avg-1.onnx \ + --in-decoder $d/decoder-epoch-99-avg-1.onnx \ + --in-joiner $d/joiner-epoch-99-avg-1.onnx \ + --out-encoder $platform/encoder.rknn \ + --out-decoder $platform/decoder.rknn \ + --out-joiner $platform/joiner.rknn \ + --target-platform $platform 2>/dev/null + + ls -lh $platform/ + + ./pruned_transducer_stateless7_streaming/test_rknn_on_cpu_simulator.py \ + --encoder $d/encoder-epoch-99-avg-1.onnx \ + --decoder $d/decoder-epoch-99-avg-1.onnx \ + --joiner $d/joiner-epoch-99-avg-1.onnx \ + --tokens $d/tokens.txt \ + --wav $d/0.wav + + cp $d/tokens.txt $platform + cp $d/*.wav $platform + + cp -av $platform /icefall/rknn-models-small + done + + ls -lh /icefall/rknn-models-small +} + +export_bilingual_zh_en_small + +export_bilingual_zh_en diff --git a/.github/workflows/rknn.yml b/.github/workflows/rknn.yml new file mode 100644 index 000000000..51aa4eb9b --- /dev/null +++ b/.github/workflows/rknn.yml @@ -0,0 +1,180 @@ +name: rknn + +on: + push: + branches: + - master + - ci-rknn-2 + + pull_request: + branches: + - master + + workflow_dispatch: + +concurrency: + group: rknn-${{ github.ref }} + cancel-in-progress: true + +jobs: + generate_build_matrix: + if: github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa' + # see https://github.com/pytorch/pytorch/pull/50633 + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Generating build matrix + id: set-matrix + run: | + # outputting for debugging purposes + python ./.github/scripts/docker/generate_build_matrix.py --torch-version=2.4.0 --python-version=3.10 + MATRIX=$(python ./.github/scripts/docker/generate_build_matrix.py --torch-version=2.4.0 --python-version=3.10) + echo "::set-output name=matrix::${MATRIX}" + rknn: + needs: generate_build_matrix + name: py${{ matrix.python-version }} torch${{ matrix.torch-version }} v${{ matrix.version }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + ${{ fromJson(needs.generate_build_matrix.outputs.matrix) }} + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup Python + if: false + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Export ONNX model + uses: addnab/docker-run-action@v3 + with: + image: ghcr.io/${{ github.repository_owner }}/icefall:cpu-py${{ matrix.python-version }}-torch${{ matrix.torch-version }}-v${{ matrix.version }} + options: | + --volume ${{ github.workspace }}/:/icefall + shell: bash + run: | + cat /etc/*release + lsb_release -a + uname -a + python3 --version + export PYTHONPATH=/icefall:$PYTHONPATH + cd /icefall + git config --global --add safe.directory /icefall + + python3 -m torch.utils.collect_env + python3 -m k2.version + pip list + + + # Install rknn + curl -SL -O https://huggingface.co/csukuangfj/rknn-toolkit2/resolve/main/rknn_toolkit2-2.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + pip install ./*.whl "numpy<=1.26.4" + pip list | grep rknn + echo "---" + pip list + echo "---" + + .github/scripts/librispeech/ASR/run_rknn.sh + + - name: Display rknn models + shell: bash + run: | + ls -lh + + ls -lh rknn-models/* + echo "----" + ls -lh rknn-models-small/* + + - name: Collect results (small) + shell: bash + run: | + for platform in rk3562 rk3566 rk3568 rk3576 rk3588; do + dst=sherpa-onnx-$platform-streaming-zipformer-small-bilingual-zh-en-2023-02-16 + mkdir $dst + mkdir $dst/test_wavs + src=rknn-models-small/$platform + + cp -v $src/*.rknn $dst/ + cp -v $src/tokens.txt $dst/ + cp -v $src/*.wav $dst/test_wavs/ + ls -lh $dst + tar cjfv $dst.tar.bz2 $dst + rm -rf $dst + done + + - name: Collect results + shell: bash + run: | + for platform in rk3562 rk3566 rk3568 rk3576 rk3588; do + dst=sherpa-onnx-$platform-streaming-zipformer-bilingual-zh-en-2023-02-20 + mkdir $dst + mkdir $dst/test_wavs + src=rknn-models/$platform + + cp -v $src/*.rknn $dst/ + cp -v $src/tokens.txt $dst/ + cp -v $src/*.wav $dst/test_wavs/ + ls -lh $dst + tar cjfv $dst.tar.bz2 $dst + rm -rf $dst + done + + - name: Display results + shell: bash + run: | + ls -lh *rk*.tar.bz2 + + - name: Release to GitHub + uses: svenstaro/upload-release-action@v2 + with: + file_glob: true + overwrite: true + file: sherpa-onnx-*.tar.bz2 + repo_name: k2-fsa/sherpa-onnx + repo_token: ${{ secrets.UPLOAD_GH_SHERPA_ONNX_TOKEN }} + tag: asr-models + + - name: Upload model to huggingface + if: github.event_name == 'push' + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + uses: nick-fields/retry@v3 + with: + max_attempts: 20 + timeout_seconds: 200 + shell: bash + command: | + git config --global user.email "csukuangfj@gmail.com" + git config --global user.name "Fangjun Kuang" + + rm -rf huggingface + export GIT_LFS_SKIP_SMUDGE=1 + + git clone https://huggingface.co/csukuangfj/sherpa-onnx-rknn-models huggingface + cd huggingface + + git fetch + git pull + git merge -m "merge remote" --ff origin main + dst=streaming-asr + mkdir -p $dst + rm -fv $dst/* + cp ../*rk*.tar.bz2 $dst/ + + ls -lh $dst + git add . + git status + git commit -m "update models" + git status + + git push https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-rknn-models main || true + rm -rf huggingface diff --git a/egs/librispeech/ASR/pruned_transducer_stateless7_streaming/export-onnx-zh.py b/egs/librispeech/ASR/pruned_transducer_stateless7_streaming/export-onnx-zh.py index 2de56837e..a4fbd93ba 100755 --- a/egs/librispeech/ASR/pruned_transducer_stateless7_streaming/export-onnx-zh.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless7_streaming/export-onnx-zh.py @@ -85,6 +85,20 @@ def get_parser(): formatter_class=argparse.ArgumentDefaultsHelpFormatter ) + parser.add_argument( + "--dynamic-batch", + type=int, + default=1, + help="1 to support dynamic batch size. 0 to support only batch size == 1", + ) + + parser.add_argument( + "--enable-int8-quantization", + type=int, + default=1, + help="1 to also export int8 onnx models.", + ) + parser.add_argument( "--epoch", type=int, @@ -257,6 +271,7 @@ def export_encoder_model_onnx( encoder_model: OnnxEncoder, encoder_filename: str, opset_version: int = 11, + dynamic_batch: bool = True, ) -> None: """ Onnx model inputs: @@ -274,6 +289,8 @@ def export_encoder_model_onnx( The filename to save the exported ONNX model. opset_version: The opset version to use. + dynamic_batch: + True to export a model supporting dynamic batch size """ encoder_model.encoder.__class__.forward = ( @@ -379,7 +396,9 @@ def export_encoder_model_onnx( "encoder_out": {0: "N"}, **inputs, **outputs, - }, + } + if dynamic_batch + else {}, ) add_meta_data(filename=encoder_filename, meta_data=meta_data) @@ -389,6 +408,7 @@ def export_decoder_model_onnx( decoder_model: nn.Module, decoder_filename: str, opset_version: int = 11, + dynamic_batch: bool = True, ) -> None: """Export the decoder model to ONNX format. @@ -412,7 +432,7 @@ def export_decoder_model_onnx( """ context_size = decoder_model.decoder.context_size vocab_size = decoder_model.decoder.vocab_size - y = torch.zeros(10, context_size, dtype=torch.int64) + y = torch.zeros(1, context_size, dtype=torch.int64) decoder_model = torch.jit.script(decoder_model) torch.onnx.export( decoder_model, @@ -425,7 +445,9 @@ def export_decoder_model_onnx( dynamic_axes={ "y": {0: "N"}, "decoder_out": {0: "N"}, - }, + } + if dynamic_batch + else {}, ) meta_data = { "context_size": str(context_size), @@ -438,6 +460,7 @@ def export_joiner_model_onnx( joiner_model: nn.Module, joiner_filename: str, opset_version: int = 11, + dynamic_batch: bool = True, ) -> None: """Export the joiner model to ONNX format. The exported joiner model has two inputs: @@ -452,8 +475,8 @@ def export_joiner_model_onnx( joiner_dim = joiner_model.output_linear.weight.shape[1] logging.info(f"joiner dim: {joiner_dim}") - projected_encoder_out = torch.rand(11, joiner_dim, dtype=torch.float32) - projected_decoder_out = torch.rand(11, joiner_dim, dtype=torch.float32) + projected_encoder_out = torch.rand(1, joiner_dim, dtype=torch.float32) + projected_decoder_out = torch.rand(1, joiner_dim, dtype=torch.float32) torch.onnx.export( joiner_model, @@ -470,7 +493,9 @@ def export_joiner_model_onnx( "encoder_out": {0: "N"}, "decoder_out": {0: "N"}, "logit": {0: "N"}, - }, + } + if dynamic_batch + else {}, ) meta_data = { "joiner_dim": str(joiner_dim), @@ -629,6 +654,7 @@ def main(): encoder, encoder_filename, opset_version=opset_version, + dynamic_batch=params.dynamic_batch == 1, ) logging.info(f"Exported encoder to {encoder_filename}") @@ -638,6 +664,7 @@ def main(): decoder, decoder_filename, opset_version=opset_version, + dynamic_batch=params.dynamic_batch == 1, ) logging.info(f"Exported decoder to {decoder_filename}") @@ -647,37 +674,39 @@ def main(): joiner, joiner_filename, opset_version=opset_version, + dynamic_batch=params.dynamic_batch == 1, ) logging.info(f"Exported joiner to {joiner_filename}") # Generate int8 quantization models # See https://onnxruntime.ai/docs/performance/model-optimizations/quantization.html#data-type-selection - logging.info("Generate int8 quantization models") + if params.enable_int8_quantization: + logging.info("Generate int8 quantization models") - encoder_filename_int8 = params.exp_dir / f"encoder-{suffix}.int8.onnx" - quantize_dynamic( - model_input=encoder_filename, - model_output=encoder_filename_int8, - op_types_to_quantize=["MatMul"], - weight_type=QuantType.QInt8, - ) + encoder_filename_int8 = params.exp_dir / f"encoder-{suffix}.int8.onnx" + quantize_dynamic( + model_input=encoder_filename, + model_output=encoder_filename_int8, + op_types_to_quantize=["MatMul"], + weight_type=QuantType.QInt8, + ) - decoder_filename_int8 = params.exp_dir / f"decoder-{suffix}.int8.onnx" - quantize_dynamic( - model_input=decoder_filename, - model_output=decoder_filename_int8, - op_types_to_quantize=["MatMul", "Gather"], - weight_type=QuantType.QInt8, - ) + decoder_filename_int8 = params.exp_dir / f"decoder-{suffix}.int8.onnx" + quantize_dynamic( + model_input=decoder_filename, + model_output=decoder_filename_int8, + op_types_to_quantize=["MatMul", "Gather"], + weight_type=QuantType.QInt8, + ) - joiner_filename_int8 = params.exp_dir / f"joiner-{suffix}.int8.onnx" - quantize_dynamic( - model_input=joiner_filename, - model_output=joiner_filename_int8, - op_types_to_quantize=["MatMul"], - weight_type=QuantType.QInt8, - ) + joiner_filename_int8 = params.exp_dir / f"joiner-{suffix}.int8.onnx" + quantize_dynamic( + model_input=joiner_filename, + model_output=joiner_filename_int8, + op_types_to_quantize=["MatMul"], + weight_type=QuantType.QInt8, + ) if __name__ == "__main__": diff --git a/egs/librispeech/ASR/pruned_transducer_stateless7_streaming/export_rknn.py b/egs/librispeech/ASR/pruned_transducer_stateless7_streaming/export_rknn.py new file mode 100755 index 000000000..cb872cca0 --- /dev/null +++ b/egs/librispeech/ASR/pruned_transducer_stateless7_streaming/export_rknn.py @@ -0,0 +1,261 @@ +#!/usr/bin/env python3 +# Copyright (c) 2025 Xiaomi Corporation (authors: Fangjun Kuang) + +import argparse +import logging +from pathlib import Path +from typing import List + +from rknn.api import RKNN + +logging.basicConfig(level=logging.WARNING) + +g_platforms = [ + # "rv1103", + # "rv1103b", + # "rv1106", + # "rk2118", + "rk3562", + "rk3566", + "rk3568", + "rk3576", + "rk3588", +] + + +def get_parser(): + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + + parser.add_argument( + "--target-platform", + type=str, + required=True, + help=f"Supported values are: {','.join(g_platforms)}", + ) + + parser.add_argument( + "--in-encoder", + type=str, + required=True, + help="Path to the encoder onnx model", + ) + + parser.add_argument( + "--in-decoder", + type=str, + required=True, + help="Path to the decoder onnx model", + ) + + parser.add_argument( + "--in-joiner", + type=str, + required=True, + help="Path to the joiner onnx model", + ) + + parser.add_argument( + "--out-encoder", + type=str, + required=True, + help="Path to the encoder rknn model", + ) + + parser.add_argument( + "--out-decoder", + type=str, + required=True, + help="Path to the decoder rknn model", + ) + + parser.add_argument( + "--out-joiner", + type=str, + required=True, + help="Path to the joiner rknn model", + ) + + return parser + + +def export_rknn(rknn, filename): + ret = rknn.export_rknn(filename) + if ret != 0: + exit("Export rknn model to {filename} failed!") + + +def init_model(filename: str, target_platform: str, custom_string=None): + rknn = RKNN(verbose=False) + + rknn.config(target_platform=target_platform, custom_string=custom_string) + if not Path(filename).is_file(): + exit(f"{filename} does not exist") + + ret = rknn.load_onnx(model=filename) + if ret != 0: + exit(f"Load model {filename} failed!") + + ret = rknn.build(do_quantization=False) + if ret != 0: + exit("Build model {filename} failed!") + + return rknn + + +class MetaData: + def __init__( + self, + model_type: str, + attention_dims: List[int], + encoder_dims: List[int], + T: int, + left_context_len: List[int], + decode_chunk_len: int, + cnn_module_kernels: List[int], + num_encoder_layers: List[int], + context_size: int, + ): + self.model_type = model_type + self.attention_dims = attention_dims + self.encoder_dims = encoder_dims + self.T = T + self.left_context_len = left_context_len + self.decode_chunk_len = decode_chunk_len + self.cnn_module_kernels = cnn_module_kernels + self.num_encoder_layers = num_encoder_layers + self.context_size = context_size + + def __str__(self) -> str: + return self.to_str() + + def to_str(self) -> str: + def to_s(ll): + return ",".join(list(map(str, ll))) + + s = f"model_type={self.model_type}" + s += ";attention_dims=" + to_s(self.attention_dims) + s += ";encoder_dims=" + to_s(self.encoder_dims) + s += ";T=" + str(self.T) + s += ";left_context_len=" + to_s(self.left_context_len) + s += ";decode_chunk_len=" + str(self.decode_chunk_len) + s += ";cnn_module_kernels=" + to_s(self.cnn_module_kernels) + s += ";num_encoder_layers=" + to_s(self.num_encoder_layers) + s += ";context_size=" + str(self.context_size) + + assert len(s) < 1024, (s, len(s)) + + return s + + +def get_meta_data(encoder: str, decoder: str): + import onnxruntime + + session_opts = onnxruntime.SessionOptions() + session_opts.inter_op_num_threads = 1 + session_opts.intra_op_num_threads = 1 + + m_encoder = onnxruntime.InferenceSession( + encoder, + sess_options=session_opts, + providers=["CPUExecutionProvider"], + ) + + m_decoder = onnxruntime.InferenceSession( + decoder, + sess_options=session_opts, + providers=["CPUExecutionProvider"], + ) + + encoder_meta = m_encoder.get_modelmeta().custom_metadata_map + print(encoder_meta) + + # {'attention_dims': '192,192,192,192,192', 'version': '1', + # 'model_type': 'zipformer', 'encoder_dims': '256,256,256,256,256', + # 'model_author': 'k2-fsa', 'T': '103', + # 'left_context_len': '192,96,48,24,96', + # 'decode_chunk_len': '96', + # 'cnn_module_kernels': '31,31,31,31,31', + # 'num_encoder_layers': '2,2,2,2,2'} + + def to_int_list(s): + return list(map(int, s.split(","))) + + decoder_meta = m_decoder.get_modelmeta().custom_metadata_map + print(decoder_meta) + + model_type = encoder_meta["model_type"] + attention_dims = to_int_list(encoder_meta["attention_dims"]) + encoder_dims = to_int_list(encoder_meta["encoder_dims"]) + T = int(encoder_meta["T"]) + left_context_len = to_int_list(encoder_meta["left_context_len"]) + decode_chunk_len = int(encoder_meta["decode_chunk_len"]) + cnn_module_kernels = to_int_list(encoder_meta["cnn_module_kernels"]) + num_encoder_layers = to_int_list(encoder_meta["num_encoder_layers"]) + context_size = int(decoder_meta["context_size"]) + + return MetaData( + model_type=model_type, + attention_dims=attention_dims, + encoder_dims=encoder_dims, + T=T, + left_context_len=left_context_len, + decode_chunk_len=decode_chunk_len, + cnn_module_kernels=cnn_module_kernels, + num_encoder_layers=num_encoder_layers, + context_size=context_size, + ) + + +class RKNNModel: + def __init__( + self, + encoder: str, + decoder: str, + joiner: str, + target_platform: str, + ): + self.meta = get_meta_data(encoder, decoder) + self.encoder = init_model( + encoder, + custom_string=self.meta.to_str(), + target_platform=target_platform, + ) + self.decoder = init_model(decoder, target_platform=target_platform) + self.joiner = init_model(joiner, target_platform=target_platform) + + def export_rknn(self, encoder, decoder, joiner): + export_rknn(self.encoder, encoder) + export_rknn(self.decoder, decoder) + export_rknn(self.joiner, joiner) + + def release(self): + self.encoder.release() + self.decoder.release() + self.joiner.release() + + +def main(): + args = get_parser().parse_args() + print(vars(args)) + + model = RKNNModel( + encoder=args.in_encoder, + decoder=args.in_decoder, + joiner=args.in_joiner, + target_platform=args.target_platform, + ) + print(model.meta) + + model.export_rknn( + encoder=args.out_encoder, + decoder=args.out_decoder, + joiner=args.out_joiner, + ) + + model.release() + + +if __name__ == "__main__": + main() diff --git a/egs/librispeech/ASR/pruned_transducer_stateless7_streaming/onnx_pretrained.py b/egs/librispeech/ASR/pruned_transducer_stateless7_streaming/onnx_pretrained.py index 298d1889b..e5e513671 100755 --- a/egs/librispeech/ASR/pruned_transducer_stateless7_streaming/onnx_pretrained.py +++ b/egs/librispeech/ASR/pruned_transducer_stateless7_streaming/onnx_pretrained.py @@ -132,10 +132,18 @@ class OnnxModel: sess_options=self.session_opts, providers=["CPUExecutionProvider"], ) + print("==========Encoder input==========") + for i in self.encoder.get_inputs(): + print(i) + print("==========Encoder output==========") + for i in self.encoder.get_outputs(): + print(i) + self.init_encoder_states() def init_encoder_states(self, batch_size: int = 1): encoder_meta = self.encoder.get_modelmeta().custom_metadata_map + print(encoder_meta) model_type = encoder_meta["model_type"] assert model_type == "zipformer", model_type @@ -232,6 +240,12 @@ class OnnxModel: sess_options=self.session_opts, providers=["CPUExecutionProvider"], ) + print("==========Decoder input==========") + for i in self.decoder.get_inputs(): + print(i) + print("==========Decoder output==========") + for i in self.decoder.get_outputs(): + print(i) decoder_meta = self.decoder.get_modelmeta().custom_metadata_map self.context_size = int(decoder_meta["context_size"]) @@ -247,6 +261,13 @@ class OnnxModel: providers=["CPUExecutionProvider"], ) + print("==========Joiner input==========") + for i in self.joiner.get_inputs(): + print(i) + print("==========Joiner output==========") + for i in self.joiner.get_outputs(): + print(i) + joiner_meta = self.joiner.get_modelmeta().custom_metadata_map self.joiner_dim = int(joiner_meta["joiner_dim"]) diff --git a/egs/librispeech/ASR/pruned_transducer_stateless7_streaming/test_rknn_on_cpu_simulator.py b/egs/librispeech/ASR/pruned_transducer_stateless7_streaming/test_rknn_on_cpu_simulator.py new file mode 100755 index 000000000..a543c6083 --- /dev/null +++ b/egs/librispeech/ASR/pruned_transducer_stateless7_streaming/test_rknn_on_cpu_simulator.py @@ -0,0 +1,413 @@ +#!/usr/bin/env python3 +# Copyright (c) 2025 Xiaomi Corporation (authors: Fangjun Kuang) + +import argparse +from pathlib import Path +from typing import List, Tuple + +import kaldi_native_fbank as knf +import numpy as np +import soundfile as sf +from rknn.api import RKNN + + +def get_parser(): + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + + parser.add_argument( + "--encoder", + type=str, + required=True, + help="Path to the encoder onnx model", + ) + + parser.add_argument( + "--decoder", + type=str, + required=True, + help="Path to the decoder onnx model", + ) + + parser.add_argument( + "--joiner", + type=str, + required=True, + help="Path to the joiner onnx model", + ) + + parser.add_argument( + "--tokens", + type=str, + required=True, + help="Path to the tokens.txt", + ) + + parser.add_argument( + "--wav", + type=str, + required=True, + help="Path to test wave", + ) + + return parser + + +def load_audio(filename: str) -> Tuple[np.ndarray, int]: + data, sample_rate = sf.read( + filename, + always_2d=True, + dtype="float32", + ) + data = data[:, 0] # use only the first channel + + samples = np.ascontiguousarray(data) + return samples, sample_rate + + +def compute_features(filename: str, dim: int = 80) -> np.ndarray: + """ + Args: + filename: + Path to an audio file. + Returns: + Return a 1-D float32 tensor of shape (1, 80, 3000) containing the features. + """ + wave, sample_rate = load_audio(filename) + if sample_rate != 16000: + import librosa + + wave = librosa.resample(wave, orig_sr=sample_rate, target_sr=16000) + sample_rate = 16000 + + features = [] + opts = knf.FbankOptions() + opts.frame_opts.dither = 0 + opts.mel_opts.num_bins = dim + opts.frame_opts.snip_edges = False + fbank = knf.OnlineFbank(opts) + + fbank.accept_waveform(16000, wave) + tail_paddings = np.zeros(int(0.5 * 16000), dtype=np.float32) + fbank.accept_waveform(16000, tail_paddings) + fbank.input_finished() + for i in range(fbank.num_frames_ready): + f = fbank.get_frame(i) + features.append(f) + + features = np.stack(features, axis=0) + + return features + + +def load_tokens(filename): + tokens = dict() + with open(filename, "r") as f: + for line in f: + t, i = line.split() + tokens[int(i)] = t + return tokens + + +def init_model(filename, target_platform="rk3588", custom_string=None): + rknn = RKNN(verbose=False) + + rknn.config(target_platform=target_platform, custom_string=custom_string) + if not Path(filename).is_file(): + exit(f"{filename} does not exist") + + ret = rknn.load_onnx(model=filename) + if ret != 0: + exit(f"Load model {filename} failed!") + + ret = rknn.build(do_quantization=False) + if ret != 0: + exit("Build model {filename} failed!") + + ret = rknn.init_runtime() + if ret != 0: + exit(f"Failed to init rknn runtime for {filename}") + return rknn + + +class MetaData: + def __init__( + self, + model_type: str, + attention_dims: List[int], + encoder_dims: List[int], + T: int, + left_context_len: List[int], + decode_chunk_len: int, + cnn_module_kernels: List[int], + num_encoder_layers: List[int], + ): + self.model_type = model_type + self.attention_dims = attention_dims + self.encoder_dims = encoder_dims + self.T = T + self.left_context_len = left_context_len + self.decode_chunk_len = decode_chunk_len + self.cnn_module_kernels = cnn_module_kernels + self.num_encoder_layers = num_encoder_layers + + def __str__(self) -> str: + return self.to_str() + + def to_str(self) -> str: + def to_s(ll): + return ",".join(list(map(str, ll))) + + s = f"model_type={self.model_type}" + s += ";attention_dims=" + to_s(self.attention_dims) + s += ";encoder_dims=" + to_s(self.encoder_dims) + s += ";T=" + str(self.T) + s += ";left_context_len=" + to_s(self.left_context_len) + s += ";decode_chunk_len=" + str(self.decode_chunk_len) + s += ";cnn_module_kernels=" + to_s(self.cnn_module_kernels) + s += ";num_encoder_layers=" + to_s(self.num_encoder_layers) + + assert len(s) < 1024, (s, len(s)) + + return s + + +def get_meta_data(encoder: str): + import onnxruntime + + session_opts = onnxruntime.SessionOptions() + session_opts.inter_op_num_threads = 1 + session_opts.intra_op_num_threads = 1 + + m = onnxruntime.InferenceSession( + encoder, + sess_options=session_opts, + providers=["CPUExecutionProvider"], + ) + + meta = m.get_modelmeta().custom_metadata_map + print(meta) + # {'attention_dims': '192,192,192,192,192', 'version': '1', + # 'model_type': 'zipformer', 'encoder_dims': '256,256,256,256,256', + # 'model_author': 'k2-fsa', 'T': '103', + # 'left_context_len': '192,96,48,24,96', + # 'decode_chunk_len': '96', + # 'cnn_module_kernels': '31,31,31,31,31', + # 'num_encoder_layers': '2,2,2,2,2'} + + def to_int_list(s): + return list(map(int, s.split(","))) + + model_type = meta["model_type"] + attention_dims = to_int_list(meta["attention_dims"]) + encoder_dims = to_int_list(meta["encoder_dims"]) + T = int(meta["T"]) + left_context_len = to_int_list(meta["left_context_len"]) + decode_chunk_len = int(meta["decode_chunk_len"]) + cnn_module_kernels = to_int_list(meta["cnn_module_kernels"]) + num_encoder_layers = to_int_list(meta["num_encoder_layers"]) + + return MetaData( + model_type=model_type, + attention_dims=attention_dims, + encoder_dims=encoder_dims, + T=T, + left_context_len=left_context_len, + decode_chunk_len=decode_chunk_len, + cnn_module_kernels=cnn_module_kernels, + num_encoder_layers=num_encoder_layers, + ) + + +class RKNNModel: + def __init__( + self, encoder: str, decoder: str, joiner: str, target_platform="rk3588" + ): + self.meta = get_meta_data(encoder) + self.encoder = init_model(encoder, custom_string=self.meta.to_str()) + self.decoder = init_model(decoder) + self.joiner = init_model(joiner) + + def release(self): + self.encoder.release() + self.decoder.release() + self.joiner.release() + + def get_init_states( + self, + ) -> List[np.ndarray]: + + cached_len = [] + cached_avg = [] + cached_key = [] + cached_val = [] + cached_val2 = [] + cached_conv1 = [] + cached_conv2 = [] + + num_encoder_layers = self.meta.num_encoder_layers + encoder_dims = self.meta.encoder_dims + left_context_len = self.meta.left_context_len + attention_dims = self.meta.attention_dims + cnn_module_kernels = self.meta.cnn_module_kernels + + num_encoders = len(num_encoder_layers) + N = 1 + + for i in range(num_encoders): + cached_len.append(np.zeros((num_encoder_layers[i], N), dtype=np.int64)) + cached_avg.append( + np.zeros((num_encoder_layers[i], N, encoder_dims[i]), dtype=np.float32) + ) + cached_key.append( + np.zeros( + (num_encoder_layers[i], left_context_len[i], N, attention_dims[i]), + dtype=np.float32, + ) + ) + + cached_val.append( + np.zeros( + ( + num_encoder_layers[i], + left_context_len[i], + N, + attention_dims[i] // 2, + ), + dtype=np.float32, + ) + ) + cached_val2.append( + np.zeros( + ( + num_encoder_layers[i], + left_context_len[i], + N, + attention_dims[i] // 2, + ), + dtype=np.float32, + ) + ) + cached_conv1.append( + np.zeros( + ( + num_encoder_layers[i], + N, + encoder_dims[i], + cnn_module_kernels[i] - 1, + ), + dtype=np.float32, + ) + ) + cached_conv2.append( + np.zeros( + ( + num_encoder_layers[i], + N, + encoder_dims[i], + cnn_module_kernels[i] - 1, + ), + dtype=np.float32, + ) + ) + + ans = ( + cached_len + + cached_avg + + cached_key + + cached_val + + cached_val2 + + cached_conv1 + + cached_conv2 + ) + # for i, s in enumerate(ans): + # if s.ndim == 4: + # ans[i] = np.transpose(s, (0, 2, 3, 1)) + return ans + + def run_encoder(self, x: np.ndarray, states: List[np.ndarray]): + """ + Args: + x: (T, C), np.float32 + states: A list of states + """ + x = np.expand_dims(x, axis=0) + + out = self.encoder.inference(inputs=[x] + states, data_format="nchw") + # out[0], encoder_out, shape (1, 24, 512) + return out[0], out[1:] + + def run_decoder(self, x: np.ndarray): + """ + Args: + x: (1, context_size), np.int64 + Returns: + Return decoder_out, (1, C), np.float32 + """ + return self.decoder.inference(inputs=[x])[0] + + def run_joiner(self, encoder_out: np.ndarray, decoder_out: np.ndarray): + """ + Args: + encoder_out: (1, encoder_out_dim), np.float32 + decoder_out: (1, decoder_out_dim), np.float32 + Returns: + joiner_out: (1, vocab_size), np.float32 + """ + return self.joiner.inference(inputs=[encoder_out, decoder_out])[0] + + +def main(): + args = get_parser().parse_args() + print(vars(args)) + + id2token = load_tokens(args.tokens) + features = compute_features(args.wav) + model = RKNNModel( + encoder=args.encoder, + decoder=args.decoder, + joiner=args.joiner, + ) + print(model.meta) + + states = model.get_init_states() + + segment = model.meta.T + offset = model.meta.decode_chunk_len + + context_size = 2 + hyp = [0] * context_size + decoder_input = np.array([hyp], dtype=np.int64) + decoder_out = model.run_decoder(decoder_input) + + i = 0 + while True: + if i + segment > features.shape[0]: + break + x = features[i : i + segment] + i += offset + encoder_out, states = model.run_encoder(x, states) + encoder_out = encoder_out.squeeze(0) # (1, T, C) -> (T, C) + + num_frames = encoder_out.shape[0] + for k in range(num_frames): + joiner_out = model.run_joiner(encoder_out[k : k + 1], decoder_out) + joiner_out = joiner_out.squeeze(0) + max_token_id = joiner_out.argmax() + + # assume 0 is the blank id + if max_token_id != 0: + hyp.append(max_token_id) + decoder_input = np.array([hyp[-context_size:]], dtype=np.int64) + decoder_out = model.run_decoder(decoder_input) + print(hyp) + final_hyp = hyp[context_size:] + print(final_hyp) + text = "".join([id2token[i] for i in final_hyp]) + text = text.replace("▁", " ") + print(text) + + +if __name__ == "__main__": + main() From 86bd16d496ecdd7d0d487d1949a137df624e5147 Mon Sep 17 00:00:00 2001 From: Wei Kang Date: Wed, 2 Apr 2025 22:10:06 +0800 Subject: [PATCH 34/59] [KWS]Remove graph compiler (#1905) --- egs/wenetspeech/KWS/run.sh | 6 +-- egs/wenetspeech/KWS/zipformer/decode.py | 1 - egs/wenetspeech/KWS/zipformer/finetune.py | 39 +++++--------- egs/wenetspeech/KWS/zipformer/train.py | 65 ++++++++++------------- 4 files changed, 43 insertions(+), 68 deletions(-) diff --git a/egs/wenetspeech/KWS/run.sh b/egs/wenetspeech/KWS/run.sh index 8472b8531..0af7c1595 100755 --- a/egs/wenetspeech/KWS/run.sh +++ b/egs/wenetspeech/KWS/run.sh @@ -108,7 +108,7 @@ if [ $stage -le 2 ] && [ $stop_stage -ge 2 ]; then fi if [ $stage -le 3 ] && [ $stop_stage -ge 3 ]; then - log "Stage 2: Finetune the model" + log "Stage 3: Finetune the model" # The following configuration of lr schedule should work well # You may also tune the following parameters to adjust learning rate schedule @@ -143,7 +143,7 @@ if [ $stage -le 3 ] && [ $stop_stage -ge 3 ]; then fi if [ $stage -le 4 ] && [ $stop_stage -ge 4 ]; then - log "Stage 1: Decode the finetuned model." + log "Stage 4: Decode the finetuned model." export CUDA_VISIBLE_DEVICES="0" for t in small large; do python ./zipformer/decode.py \ @@ -170,7 +170,7 @@ if [ $stage -le 4 ] && [ $stop_stage -ge 4 ]; then fi if [ $stage -le 5 ] && [ $stop_stage -ge 5 ]; then - log "Stage 2: Export the finetuned model." + log "Stage 5: Export the finetuned model." python ./zipformer/export.py \ --epoch 10 \ diff --git a/egs/wenetspeech/KWS/zipformer/decode.py b/egs/wenetspeech/KWS/zipformer/decode.py index 340a41231..a628c7e58 100755 --- a/egs/wenetspeech/KWS/zipformer/decode.py +++ b/egs/wenetspeech/KWS/zipformer/decode.py @@ -35,7 +35,6 @@ from lhotse.cut import Cut from train import add_model_arguments, get_model, get_params from icefall import ContextGraph -from icefall.char_graph_compiler import CharCtcTrainingGraphCompiler from icefall.checkpoint import ( average_checkpoints, average_checkpoints_with_averaged_model, diff --git a/egs/wenetspeech/KWS/zipformer/finetune.py b/egs/wenetspeech/KWS/zipformer/finetune.py index d19172b38..cd437da4c 100755 --- a/egs/wenetspeech/KWS/zipformer/finetune.py +++ b/egs/wenetspeech/KWS/zipformer/finetune.py @@ -90,6 +90,7 @@ from train import ( add_training_arguments, compute_validation_loss, display_and_save_batch, + encode_text, get_adjusted_batch_count, get_model, get_params, @@ -100,7 +101,6 @@ from train import ( ) from icefall import diagnostics -from icefall.char_graph_compiler import CharCtcTrainingGraphCompiler from icefall.checkpoint import remove_checkpoints from icefall.checkpoint import save_checkpoint as save_checkpoint_impl from icefall.checkpoint import ( @@ -110,11 +110,11 @@ from icefall.checkpoint import ( from icefall.dist import cleanup_dist, setup_dist from icefall.err import raise_grad_scale_is_too_small_error from icefall.hooks import register_inf_check_hooks -from icefall.lexicon import Lexicon from icefall.utils import ( AttributeDict, MetricsTracker, get_parameter_groups_with_lrs, + num_tokens, setup_logger, str2bool, text_to_pinyin, @@ -254,7 +254,6 @@ def load_model_params( def compute_loss( params: AttributeDict, model: Union[nn.Module, DDP], - graph_compiler: CharCtcTrainingGraphCompiler, batch: dict, is_training: bool, ) -> Tuple[Tensor, MetricsTracker]: @@ -289,7 +288,7 @@ def compute_loss( warm_step = params.warm_step texts = batch["supervisions"]["text"] - y = graph_compiler.texts_to_ids(texts, sep="/") + y = [c.supervisions[0].tokens for c in supervisions["cut"]] y = k2.RaggedTensor(y) with torch.set_grad_enabled(is_training): @@ -347,7 +346,6 @@ def train_one_epoch( model: Union[nn.Module, DDP], optimizer: torch.optim.Optimizer, scheduler: LRSchedulerType, - graph_compiler: CharCtcTrainingGraphCompiler, train_dl: torch.utils.data.DataLoader, valid_dl: torch.utils.data.DataLoader, scaler: GradScaler, @@ -418,7 +416,6 @@ def train_one_epoch( loss, loss_info = compute_loss( params=params, model=model, - graph_compiler=graph_compiler, batch=batch, is_training=True, ) @@ -436,7 +433,7 @@ def train_one_epoch( optimizer.zero_grad() except: # noqa save_bad_model() - display_and_save_batch(batch, params=params, graph_compiler=graph_compiler) + display_and_save_batch(batch, params=params) raise if params.print_diagnostics and batch_idx == 5: @@ -523,7 +520,6 @@ def train_one_epoch( valid_info = compute_validation_loss( params=params, model=model, - graph_compiler=graph_compiler, valid_dl=valid_dl, world_size=world_size, ) @@ -576,14 +572,10 @@ def run(rank, world_size, args): device = torch.device("cuda", rank) logging.info(f"Device: {device}") - lexicon = Lexicon(params.lang_dir) - graph_compiler = CharCtcTrainingGraphCompiler( - lexicon=lexicon, - device=device, - ) + token_table = k2.SymbolTable.from_file(params.lang_dir / "tokens.txt") - params.blank_id = lexicon.token_table[""] - params.vocab_size = max(lexicon.tokens) + 1 + params.blank_id = token_table[""] + params.vocab_size = num_tokens(token_table) + 1 if not params.use_transducer: params.ctc_loss_scale = 1.0 @@ -666,17 +658,10 @@ def run(rank, world_size, args): else: train_cuts = wenetspeech.nihaowenwen_train_cuts() - def encode_text(c: Cut): - # Text normalize for each sample - text = c.supervisions[0].text - text = "/".join( - text_to_pinyin(text, mode=params.pinyin_type, errors=params.pinyin_errors) - ) - c.supervisions[0].text = text - return c + _encode_text = partial(encode_text, token_table=token_table, params=params) train_cuts = train_cuts.filter(remove_short_utt) - train_cuts = train_cuts.map(encode_text) + train_cuts = train_cuts.map(_encode_text) if params.start_batch > 0 and checkpoints and "sampler" in checkpoints: # We only load the sampler's state dict when it loads a checkpoint @@ -691,7 +676,7 @@ def run(rank, world_size, args): valid_cuts = wenetspeech.nihaowenwen_dev_cuts() valid_cuts = valid_cuts.filter(remove_short_utt) - valid_cuts = valid_cuts.map(encode_text) + valid_cuts = valid_cuts.map(_encode_text) valid_dl = wenetspeech.valid_dataloaders(valid_cuts) if not params.print_diagnostics and params.scan_for_oom_batches: @@ -699,7 +684,6 @@ def run(rank, world_size, args): model=model, train_dl=train_dl, optimizer=optimizer, - graph_compiler=graph_compiler, params=params, ) @@ -724,7 +708,6 @@ def run(rank, world_size, args): model_avg=model_avg, optimizer=optimizer, scheduler=scheduler, - graph_compiler=graph_compiler, train_dl=train_dl, valid_dl=valid_dl, scaler=scaler, @@ -760,6 +743,8 @@ def main(): WenetSpeechAsrDataModule.add_arguments(parser) args = parser.parse_args() args.exp_dir = Path(args.exp_dir) + args.lang_dir = Path(args.lang_dir) + args.return_cuts = True world_size = args.world_size assert world_size >= 1 diff --git a/egs/wenetspeech/KWS/zipformer/train.py b/egs/wenetspeech/KWS/zipformer/train.py index 40960c2ae..5d9d8de36 100755 --- a/egs/wenetspeech/KWS/zipformer/train.py +++ b/egs/wenetspeech/KWS/zipformer/train.py @@ -53,6 +53,7 @@ import argparse import copy import logging import warnings +from functools import partial from pathlib import Path from shutil import copyfile from typing import Any, Dict, Optional, Tuple, Union @@ -79,7 +80,6 @@ from torch.utils.tensorboard import SummaryWriter from zipformer import Zipformer2 from icefall import diagnostics -from icefall.char_graph_compiler import CharCtcTrainingGraphCompiler from icefall.checkpoint import load_checkpoint, remove_checkpoints from icefall.checkpoint import save_checkpoint as save_checkpoint_impl from icefall.checkpoint import ( @@ -90,11 +90,11 @@ from icefall.dist import cleanup_dist, setup_dist from icefall.env import get_env_info from icefall.err import raise_grad_scale_is_too_small_error from icefall.hooks import register_inf_check_hooks -from icefall.lexicon import Lexicon from icefall.utils import ( AttributeDict, MetricsTracker, get_parameter_groups_with_lrs, + num_tokens, setup_logger, str2bool, text_to_pinyin, @@ -776,7 +776,6 @@ def save_checkpoint( def compute_loss( params: AttributeDict, model: Union[nn.Module, DDP], - graph_compiler: CharCtcTrainingGraphCompiler, batch: dict, is_training: bool, ) -> Tuple[Tensor, MetricsTracker]: @@ -811,7 +810,7 @@ def compute_loss( warm_step = params.warm_step texts = batch["supervisions"]["text"] - y = graph_compiler.texts_to_ids(texts, sep="/") + y = [c.supervisions[0].tokens for c in supervisions["cut"]] y = k2.RaggedTensor(y).to(device) with torch.set_grad_enabled(is_training): @@ -859,7 +858,6 @@ def compute_loss( def compute_validation_loss( params: AttributeDict, model: Union[nn.Module, DDP], - graph_compiler: CharCtcTrainingGraphCompiler, valid_dl: torch.utils.data.DataLoader, world_size: int = 1, ) -> MetricsTracker: @@ -872,7 +870,6 @@ def compute_validation_loss( loss, loss_info = compute_loss( params=params, model=model, - graph_compiler=graph_compiler, batch=batch, is_training=False, ) @@ -895,7 +892,6 @@ def train_one_epoch( model: Union[nn.Module, DDP], optimizer: torch.optim.Optimizer, scheduler: LRSchedulerType, - graph_compiler: CharCtcTrainingGraphCompiler, train_dl: torch.utils.data.DataLoader, valid_dl: torch.utils.data.DataLoader, scaler: GradScaler, @@ -971,7 +967,6 @@ def train_one_epoch( loss, loss_info = compute_loss( params=params, model=model, - graph_compiler=graph_compiler, batch=batch, is_training=True, ) @@ -988,7 +983,7 @@ def train_one_epoch( optimizer.zero_grad() except: # noqa save_bad_model() - display_and_save_batch(batch, params=params, graph_compiler=graph_compiler) + display_and_save_batch(batch, params=params) raise if params.print_diagnostics and batch_idx == 5: @@ -1077,7 +1072,6 @@ def train_one_epoch( valid_info = compute_validation_loss( params=params, model=model, - graph_compiler=graph_compiler, valid_dl=valid_dl, world_size=world_size, ) @@ -1098,6 +1092,20 @@ def train_one_epoch( params.best_train_loss = params.train_loss +def encode_text(c: Cut, token_table: k2.SymbolTable, params: AttributeDict): + text = c.supervisions[0].text + tokens = text_to_pinyin(text, mode=params.pinyin_type, errors=params.pinyin_errors) + ids = [] + for t in tokens: + if t in token_table: + ids.append(token_table[t]) + else: + logging.warning(f"Text : {text} has OOV token : {t} , encode to ") + ids.append(token_table[""]) + c.supervisions[0].tokens = ids + return c + + def run(rank, world_size, args): """ Args: @@ -1130,14 +1138,10 @@ def run(rank, world_size, args): device = torch.device("cuda", rank) logging.info(f"Device: {device}") - lexicon = Lexicon(params.lang_dir) - graph_compiler = CharCtcTrainingGraphCompiler( - lexicon=lexicon, - device=device, - ) + token_table = k2.SymbolTable.from_file(params.lang_dir / "tokens.txt") - params.blank_id = lexicon.token_table[""] - params.vocab_size = max(lexicon.tokens) + 1 + params.blank_id = token_table[""] + params.vocab_size = num_tokens(token_table) + 1 if not params.use_transducer: params.ctc_loss_scale = 1.0 @@ -1216,17 +1220,10 @@ def run(rank, world_size, args): return True - def encode_text(c: Cut): - # Text normalize for each sample - text = c.supervisions[0].text - text = "/".join( - text_to_pinyin(text, mode=params.pinyin_type, errors=params.pinyin_errors) - ) - c.supervisions[0].text = text - return c + _encode_text = partial(encode_text, token_table=token_table, params=params) train_cuts = train_cuts.filter(remove_short_and_long_utt) - train_cuts = train_cuts.map(encode_text) + train_cuts = train_cuts.map(_encode_text) if params.start_batch > 0 and checkpoints and "sampler" in checkpoints: # We only load the sampler's state dict when it loads a checkpoint @@ -1240,7 +1237,7 @@ def run(rank, world_size, args): ) valid_cuts = wenetspeech.valid_cuts() - valid_cuts = valid_cuts.map(encode_text) + valid_cuts = valid_cuts.map(_encode_text) valid_dl = wenetspeech.valid_dataloaders(valid_cuts) if not params.print_diagnostics and params.scan_for_oom_batches: @@ -1248,7 +1245,6 @@ def run(rank, world_size, args): model=model, train_dl=train_dl, optimizer=optimizer, - graph_compiler=graph_compiler, params=params, ) @@ -1273,7 +1269,6 @@ def run(rank, world_size, args): model_avg=model_avg, optimizer=optimizer, scheduler=scheduler, - graph_compiler=graph_compiler, train_dl=train_dl, valid_dl=valid_dl, scaler=scaler, @@ -1307,7 +1302,6 @@ def run(rank, world_size, args): def display_and_save_batch( batch: dict, params: AttributeDict, - graph_compiler: CharCtcTrainingGraphCompiler, ) -> None: """Display the batch statistics and save the batch into disk. @@ -1317,8 +1311,6 @@ def display_and_save_batch( for the content in it. params: Parameters for training. See :func:`get_params`. - graph_compiler: - The compiler to encode texts to ids. """ from lhotse.utils import uuid4 @@ -1332,8 +1324,8 @@ def display_and_save_batch( logging.info(f"features shape: {features.shape}") texts = supervisions["text"] - y = graph_compiler.texts_to_ids(texts) - num_tokens = sum(len(i) for i in y) + tokens = [c.supervisions[0].tokens for c in supervisions["cut"]] + num_tokens = sum(len(i) for i in tokens) logging.info(f"num tokens: {num_tokens}") @@ -1341,7 +1333,6 @@ def scan_pessimistic_batches_for_oom( model: Union[nn.Module, DDP], train_dl: torch.utils.data.DataLoader, optimizer: torch.optim.Optimizer, - graph_compiler: CharCtcTrainingGraphCompiler, params: AttributeDict, ): from lhotse.dataset import find_pessimistic_batches @@ -1357,7 +1348,6 @@ def scan_pessimistic_batches_for_oom( loss, _ = compute_loss( params=params, model=model, - graph_compiler=graph_compiler, batch=batch, is_training=True, ) @@ -1372,7 +1362,7 @@ def scan_pessimistic_batches_for_oom( f"Failing criterion: {criterion} " f"(={crit_values[criterion]}) ..." ) - display_and_save_batch(batch, params=params, graph_compiler=graph_compiler) + display_and_save_batch(batch, params=params) raise logging.info( f"Maximum memory allocated so far is {torch.cuda.max_memory_allocated()//1000000}MB" @@ -1385,6 +1375,7 @@ def main(): args = parser.parse_args() args.lang_dir = Path(args.lang_dir) args.exp_dir = Path(args.exp_dir) + args.return_cuts = True world_size = args.world_size assert world_size >= 1 From 171cf8c9fe41666c7d70ba58c0481ec6675d8941 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Wed, 9 Apr 2025 11:52:37 +0800 Subject: [PATCH 35/59] Avoid redundant computation in PiecewiseLinear. (#1915) --- egs/librispeech/ASR/zipformer/scaling.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/egs/librispeech/ASR/zipformer/scaling.py b/egs/librispeech/ASR/zipformer/scaling.py index d345c2931..6d6281903 100644 --- a/egs/librispeech/ASR/zipformer/scaling.py +++ b/egs/librispeech/ASR/zipformer/scaling.py @@ -160,8 +160,10 @@ class PiecewiseLinear(object): extra_x_vals.append(extra_x_val) if len(extra_x_vals) > 0: x_vals = sorted(set(x_vals + extra_x_vals)) - y_vals1 = [self(x) for x in x_vals] - y_vals2 = [p(x) for x in x_vals] + + y_vals1 = [self(x) for x in x_vals] + y_vals2 = [p(x) for x in x_vals] + return ( PiecewiseLinear(*zip(x_vals, y_vals1)), PiecewiseLinear(*zip(x_vals, y_vals2)), From 300a821f58abdb9975a3743caf1a78953e97711b Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Thu, 10 Apr 2025 10:30:37 +0800 Subject: [PATCH 36/59] Fix aishell training (#1916) --- egs/aishell/ASR/zipformer/train.py | 5 ++--- egs/aishell/ASR/zipformer/train_bbpe.py | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/egs/aishell/ASR/zipformer/train.py b/egs/aishell/ASR/zipformer/train.py index cd253c597..dddfe52fa 100755 --- a/egs/aishell/ASR/zipformer/train.py +++ b/egs/aishell/ASR/zipformer/train.py @@ -1343,8 +1343,7 @@ def main(): run(rank=0, world_size=1, args=args) -torch.set_num_threads(1) -torch.set_num_interop_threads(1) - if __name__ == "__main__": + torch.set_num_threads(1) + torch.set_num_interop_threads(1) main() diff --git a/egs/aishell/ASR/zipformer/train_bbpe.py b/egs/aishell/ASR/zipformer/train_bbpe.py index 46a5506db..dbc262c5c 100755 --- a/egs/aishell/ASR/zipformer/train_bbpe.py +++ b/egs/aishell/ASR/zipformer/train_bbpe.py @@ -935,8 +935,7 @@ def main(): run(rank=0, world_size=1, args=args) -torch.set_num_threads(1) -torch.set_num_interop_threads(1) - if __name__ == "__main__": + torch.set_num_threads(1) + torch.set_num_interop_threads(1) main() From 64c53640857d0b9c3fd63070c2f741d374051ce9 Mon Sep 17 00:00:00 2001 From: math345 Date: Thu, 10 Apr 2025 11:37:28 +0800 Subject: [PATCH 37/59] Fix bug: When resuming training from a checkpoint, model_avg was not assigned, resulting in a None error. (#1914) --- egs/wenetspeech/KWS/zipformer/finetune.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/egs/wenetspeech/KWS/zipformer/finetune.py b/egs/wenetspeech/KWS/zipformer/finetune.py index cd437da4c..249209352 100755 --- a/egs/wenetspeech/KWS/zipformer/finetune.py +++ b/egs/wenetspeech/KWS/zipformer/finetune.py @@ -593,6 +593,9 @@ def run(rank, world_size, args): if params.continue_finetune: assert params.start_epoch > 0, params.start_epoch + if rank == 0: + # model_avg is only used with rank 0 + model_avg = copy.deepcopy(model).to(torch.float64) checkpoints = load_checkpoint_if_available( params=params, model=model, model_avg=model_avg ) From 5ec95e5482867de02eece6615ff51662c7ed9b37 Mon Sep 17 00:00:00 2001 From: Yifan Yang <64255737+yfyeung@users.noreply.github.com> Date: Wed, 23 Apr 2025 16:18:38 +0800 Subject: [PATCH 38/59] Fix SpeechLLM recipe (#1926) --- egs/speech_llm/ASR_LLM/RESULTS.md | 8 ++++---- egs/speech_llm/ASR_LLM/prepare.sh | 13 ++++++++----- egs/speech_llm/ASR_LLM/shared | 1 + 3 files changed, 13 insertions(+), 9 deletions(-) mode change 100644 => 100755 egs/speech_llm/ASR_LLM/prepare.sh create mode 120000 egs/speech_llm/ASR_LLM/shared diff --git a/egs/speech_llm/ASR_LLM/RESULTS.md b/egs/speech_llm/ASR_LLM/RESULTS.md index 830c70397..01c48a82e 100644 --- a/egs/speech_llm/ASR_LLM/RESULTS.md +++ b/egs/speech_llm/ASR_LLM/RESULTS.md @@ -42,8 +42,8 @@ huggingface-cli download --local-dir models/whisper yuekai/icefall_asr_aishel # For multi-hans fine-tuned whisper model # huggingface-cli download --local-dir models/whisper yuekai/icefall_asr_multi-hans-zh_whisper v1.1/whisper-large-v2-multi-hans-zh-epoch-3-avg-10.pt -# huggingface-clie download --local-dir models/qwen Qwen/Qwen2-7B-Instruct -huggingface-clie download --local-dir models/qwen Qwen/Qwen2-1.5B-Instruct +# huggingface-cli download --local-dir models/qwen Qwen/Qwen2-7B-Instruct +huggingface-cli download --local-dir models/qwen Qwen/Qwen2-1.5B-Instruct # First, we only train the projector and freeze other modules. torchrun --nproc_per_node 8 ./whisper_llm_zh/train.py \ @@ -57,7 +57,7 @@ torchrun --nproc_per_node 8 ./whisper_llm_zh/train.py \ --use-flash-attn True \ --use-lora False --unfreeze-llm False -# Then we jointly train the projector and LLM LoRA modules. +# Then, we jointly train the projector and LLM LoRA modules. torchrun --nproc_per_node 8 ./whisper_llm_zh/train.py \ --max-duration 200 \ --exp-dir ./whisper_llm_zh/exp_test \ @@ -81,7 +81,7 @@ huggingface-cli download --local-dir models/whisper yuekai/icefall_asr_aishel # For multi-hans fine-tuned whisper model # huggingface-cli download --local-dir models/whisper yuekai/icefall_asr_multi-hans-zh_whisper v1.1/whisper-large-v2-multi-hans-zh-epoch-3-avg-10.pt -huggingface-clie download --local-dir models/qwen Qwen/Qwen2-7B-Instruct +huggingface-cli download --local-dir models/qwen Qwen/Qwen2-7B-Instruct mkdir -p whisper_llm_zh/exp_aishell_whisper_qwen2_1.5B ln -s models/checkpoint/epoch-10-avg-5.pt whisper_llm_zh/exp_aishell_whisper_qwen2_1.5B/epoch-999.pt diff --git a/egs/speech_llm/ASR_LLM/prepare.sh b/egs/speech_llm/ASR_LLM/prepare.sh old mode 100644 new mode 100755 index 6f5ed5448..8ca3c1c36 --- a/egs/speech_llm/ASR_LLM/prepare.sh +++ b/egs/speech_llm/ASR_LLM/prepare.sh @@ -7,6 +7,9 @@ set -eou pipefail stage=0 stop_stage=0 + +. shared/parse_options.sh || exit 1 + # All files generated by this script are saved in "data". # You can safely remove "data" and rerun this script to regenerate it. mkdir -p data @@ -23,7 +26,7 @@ if [ $stage -le 0 ] && [ $stop_stage -ge 0 ]; then # pip install huggingface_hub['cli'] # for aishell 1 - huggingface-cli download --local-dir data yuekai/aishell_whisper_fbank_lhotse + huggingface-cli download --repo-type dataset --local-dir data yuekai/aishell_whisper_fbank_lhotse fi @@ -31,9 +34,9 @@ if [ $stage -le 1 ] && [ $stop_stage -ge 1 ]; then log "stage 1: Download whisper-large-v2 multi-hans-zh fbank feature from huggingface" # for multi-hans-zh - huggingface-cli download --local-dir data/fbank yuekai/wenetspeech_whisper_fbank_lhotse - huggingface-cli download --local-dir data/fbank yuekai/multi_hans_zh_whisper_fbank_lhotse - huggingface-cli download --local-dir data/fbank yuekai/alimeeting_aishell4_training_whisper_fbank_lhotse + huggingface-cli download --repo-type dataset --local-dir data/fbank yuekai/wenetspeech_whisper_fbank_lhotse + huggingface-cli download --repo-type dataset --local-dir data/fbank yuekai/multi_hans_zh_whisper_fbank_lhotse + huggingface-cli download --repo-type dataset --local-dir data/fbank yuekai/alimeeting_aishell4_training_whisper_fbank_lhotse fi if [ $stage -le 2 ] && [ $stop_stage -ge 2 ]; then @@ -41,6 +44,6 @@ if [ $stage -le 2 ] && [ $stop_stage -ge 2 ]; then # for speechio test sets mkdir data_speechio - huggingface-cli download --local-dir data_speechio yuekai/icefall_asr_speechio + huggingface-cli download --repo-type model --local-dir data_speechio yuekai/icefall_asr_speechio mv data_speechio/fbank/* data/fbank fi diff --git a/egs/speech_llm/ASR_LLM/shared b/egs/speech_llm/ASR_LLM/shared new file mode 120000 index 000000000..4cbd91a7e --- /dev/null +++ b/egs/speech_llm/ASR_LLM/shared @@ -0,0 +1 @@ +../../../icefall/shared \ No newline at end of file From cc2e64a6aa11c82fe3925c224e9bf1536edfb5e4 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Thu, 24 Apr 2025 17:04:46 +0800 Subject: [PATCH 39/59] Fix convert_texts_into_ids() in the tedlium3 recipe. (#1929) --- egs/tedlium3/ASR/pruned_transducer_stateless/train.py | 2 +- egs/tedlium3/ASR/transducer_stateless/train.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/egs/tedlium3/ASR/pruned_transducer_stateless/train.py b/egs/tedlium3/ASR/pruned_transducer_stateless/train.py index 170f37767..2455f3630 100755 --- a/egs/tedlium3/ASR/pruned_transducer_stateless/train.py +++ b/egs/tedlium3/ASR/pruned_transducer_stateless/train.py @@ -422,7 +422,7 @@ def compute_loss( texts = batch["supervisions"]["text"] unk_id = params.unk_id - y = convert_texts_into_ids(texts, unk_id, sp=sp) + y = convert_texts_into_ids(texts, sp=sp) y = k2.RaggedTensor(y).to(device) with torch.set_grad_enabled(is_training): diff --git a/egs/tedlium3/ASR/transducer_stateless/train.py b/egs/tedlium3/ASR/transducer_stateless/train.py index 6fed32e81..c6fa34e70 100755 --- a/egs/tedlium3/ASR/transducer_stateless/train.py +++ b/egs/tedlium3/ASR/transducer_stateless/train.py @@ -397,7 +397,7 @@ def compute_loss( texts = batch["supervisions"]["text"] unk_id = params.unk_id - y = convert_texts_into_ids(texts, unk_id, sp=sp) + y = convert_texts_into_ids(texts, sp=sp) y = k2.RaggedTensor(y).to(device) with torch.set_grad_enabled(is_training): From cd7caf12df92ac21f67a4027cf80b940ee15bef6 Mon Sep 17 00:00:00 2001 From: Yifan Yang <64255737+yfyeung@users.noreply.github.com> Date: Wed, 30 Apr 2025 11:41:00 +0800 Subject: [PATCH 40/59] Fix speech_llm recipe (#1936) * fix training/decoding scripts, cleanup unused code, and ensure compliance with style checks --------- Co-authored-by: Your Name Co-authored-by: Fangjun Kuang --- egs/speech_llm/ASR_LLM/RESULTS.md | 15 ++- .../ASR_LLM/whisper_llm_zh/decode.py | 104 ++++++------------ .../ASR_LLM/whisper_llm_zh/train.py | 97 ++++------------ 3 files changed, 60 insertions(+), 156 deletions(-) diff --git a/egs/speech_llm/ASR_LLM/RESULTS.md b/egs/speech_llm/ASR_LLM/RESULTS.md index 01c48a82e..42dce80c5 100644 --- a/egs/speech_llm/ASR_LLM/RESULTS.md +++ b/egs/speech_llm/ASR_LLM/RESULTS.md @@ -55,7 +55,8 @@ torchrun --nproc_per_node 8 ./whisper_llm_zh/train.py \ --deepspeed \ --deepspeed_config ./whisper_llm_zh/ds_config_zero1.json \ --use-flash-attn True \ - --use-lora False --unfreeze-llm False + --use-lora False \ + --unfreeze-llm False # Then, we jointly train the projector and LLM LoRA modules. torchrun --nproc_per_node 8 ./whisper_llm_zh/train.py \ @@ -67,7 +68,8 @@ torchrun --nproc_per_node 8 ./whisper_llm_zh/train.py \ --deepspeed \ --deepspeed_config ./whisper_llm_zh/ds_config_zero1.json \ --use-flash-attn True \ - --use-lora True --unfreeze-llm True + --use-lora True \ + --unfreeze-llm True \ --pretrained-model-path ./whisper_llm_zh/exp_test/epoch-3.pt ``` @@ -77,11 +79,11 @@ mkdir -p models/whisper models/qwen models/checkpoint huggingface-cli download --local-dir models/checkpoint yuekai/icefall_asr_aishell_whisper_qwen2_1.5B # For aishell fine-tuned whisper model -huggingface-cli download --local-dir models/whisper yuekai/icefall_asr_aishell_whisper exp_large_v2/whisper-large-v2-aishell1-epoch-10-avg-6.pt +huggingface-cli download --local-dir models/whisper yuekai/icefall_asr_aishell_whisper exp_large_v2/whisper-large-v2-aishell1-epoch-10-avg-6.pt # For multi-hans fine-tuned whisper model -# huggingface-cli download --local-dir models/whisper yuekai/icefall_asr_multi-hans-zh_whisper v1.1/whisper-large-v2-multi-hans-zh-epoch-3-avg-10.pt +# huggingface-cli download --local-dir models/whisper yuekai/icefall_asr_multi-hans-zh_whisper v1.1/whisper-large-v2-multi-hans-zh-epoch-3-avg-10.pt -huggingface-cli download --local-dir models/qwen Qwen/Qwen2-7B-Instruct +huggingface-cli download --local-dir models/qwen Qwen/Qwen2-7B-Instruct mkdir -p whisper_llm_zh/exp_aishell_whisper_qwen2_1.5B ln -s models/checkpoint/epoch-10-avg-5.pt whisper_llm_zh/exp_aishell_whisper_qwen2_1.5B/epoch-999.pt @@ -94,5 +96,6 @@ python3 ./whisper_llm_zh/decode.py \ --epoch 999 --avg 1 \ --manifest-dir data/fbank \ --use-flash-attn True \ - --use-lora True --dataset aishell + --use-lora True \ + --dataset aishell ``` diff --git a/egs/speech_llm/ASR_LLM/whisper_llm_zh/decode.py b/egs/speech_llm/ASR_LLM/whisper_llm_zh/decode.py index 882ce4fbf..3036b471e 100755 --- a/egs/speech_llm/ASR_LLM/whisper_llm_zh/decode.py +++ b/egs/speech_llm/ASR_LLM/whisper_llm_zh/decode.py @@ -66,7 +66,7 @@ from train import DEFAULT_SPEECH_TOKEN from transformers import AutoModelForCausalLM, AutoTokenizer from whisper_encoder_forward_monkey_patch import replace_whisper_encoder_forward -from icefall.checkpoint import average_checkpoints_with_averaged_model, load_checkpoint +from icefall.checkpoint import load_checkpoint from icefall.env import get_env_info from icefall.utils import ( AttributeDict, @@ -357,43 +357,6 @@ def decode_dataset( Returns: Return a dict, whose key may be "beam-search". """ - - def normalize_text_alimeeting(text: str, normalize: str = "m2met") -> str: - """ - Text normalization similar to M2MeT challenge baseline. - See: https://github.com/yufan-aslp/AliMeeting/blob/main/asr/local/text_normalize.pl - """ - if normalize == "none": - return text - elif normalize == "m2met": - import re - - text = text.replace(" ", "") - text = text.replace("", "") - text = text.replace("<%>", "") - text = text.replace("<->", "") - text = text.replace("<$>", "") - text = text.replace("<#>", "") - text = text.replace("<_>", "") - text = text.replace("", "") - text = text.replace("`", "") - text = text.replace("&", "") - text = text.replace(",", "") - if re.search("[a-zA-Z]", text): - text = text.upper() - text = text.replace("Ａ", "A") - text = text.replace("ａ", "A") - text = text.replace("ｂ", "B") - text = text.replace("ｃ", "C") - text = text.replace("ｋ", "K") - text = text.replace("ｔ", "T") - text = text.replace("，", "") - text = text.replace("丶", "") - text = text.replace("。", "") - text = text.replace("、", "") - text = text.replace("？", "") - return text - results = [] num_cuts = 0 @@ -406,6 +369,7 @@ def decode_dataset( results = defaultdict(list) for batch_idx, batch in enumerate(dl): texts = batch["supervisions"]["text"] + texts = [list("".join(text.split())) for text in texts] cut_ids = [cut.id for cut in batch["supervisions"]["cut"]] hyps_dict = decode_one_batch( @@ -418,12 +382,8 @@ def decode_dataset( for lm_scale, hyps in hyps_dict.items(): this_batch = [] assert len(hyps) == len(texts) - for cut_id, hyp_words, ref_text in zip(cut_ids, hyps, texts): - ref_text = normalize_text_alimeeting(ref_text) - ref_words = ref_text.split() - print(f"ref: {ref_text}") - print(f"hyp: {''.join(hyp_words)}") - this_batch.append((cut_id, ref_words, hyp_words)) + for cut_id, hyp_text, ref_text in zip(cut_ids, hyps, texts): + this_batch.append((cut_id, ref_text, hyp_text)) results[lm_scale].extend(this_batch) @@ -439,40 +399,38 @@ def decode_dataset( def save_results( params: AttributeDict, test_set_name: str, - results_dict: Dict[str, List[Tuple[str, List[str], List[str]]]], + results_dict: Dict[str, List[Tuple[List[int], List[int]]]], ): - - enable_log = True test_set_wers = dict() for key, results in results_dict.items(): recog_path = ( - params.exp_dir / f"recogs-{test_set_name}-{key}-{params.suffix}.txt" + params.res_dir / f"recogs-{test_set_name}-{key}-{params.suffix}.txt" ) results = sorted(results) - store_transcripts(filename=recog_path, texts=results) - if enable_log: - logging.info(f"The transcripts are stored in {recog_path}") + store_transcripts(filename=recog_path, texts=results, char_level=True) + logging.info(f"The transcripts are stored in {recog_path}") - # The following prints out WERs, per-word error statistics and aligned + # The following prints out CERs, per-word error statistics and aligned # ref/hyp pairs. errs_filename = ( - params.exp_dir / f"errs-{test_set_name}-{key}-{params.suffix}.txt" + params.res_dir / f"errs-{test_set_name}-{key}-{params.suffix}.txt" ) - # we compute CER for aishell dataset. - results_char = [] - for res in results: - results_char.append((res[0], list("".join(res[1])), list("".join(res[2])))) with open(errs_filename, "w") as f: wer = write_error_stats( - f, f"{test_set_name}-{key}", results_char, enable_log=enable_log + f, + f"{test_set_name}-{key}", + results, + enable_log=True, + compute_CER=True, ) test_set_wers[key] = wer - if enable_log: - logging.info("Wrote detailed error stats to {}".format(errs_filename)) + logging.info("Wrote detailed error stats to {}".format(errs_filename)) test_set_wers = sorted(test_set_wers.items(), key=lambda x: x[1]) - errs_info = params.exp_dir / f"cer-summary-{test_set_name}-{params.suffix}.txt" + errs_info = ( + params.res_dir / f"wer-summary-{test_set_name}-{key}-{params.suffix}.txt" + ) with open(errs_info, "w") as f: print("settings\tCER", file=f) for key, val in test_set_wers: @@ -495,9 +453,13 @@ def main(): params = get_params() params.update(vars(args)) + + params.res_dir = params.exp_dir / f"{params.method}" + params.suffix = f"epoch-{params.epoch}-avg-{params.avg}" setup_logger( - f"{params.exp_dir}/log-{params.method}-beam{params.beam_size}/log-decode-{params.suffix}" + params.res_dir + / f"log-decode-{params.method}-beam{params.beam_size}-{params.suffix}" ) logging.info("Decoding started") @@ -574,23 +536,20 @@ def main(): if params.avg > 1: start = params.epoch - params.avg + 1 assert start >= 1, start - checkpoint = torch.load( - f"{params.exp_dir}/epoch-{params.epoch}.pt", map_location="cpu" - ) - assert "model" not in checkpoint # deepspeed converted checkpoint only contains model state_dict filenames = [ - f"{params.exp_dir}/epoch-{epoch}.pt" + f"{params.exp_dir}/epoch-{epoch}/pytorch_model.bin" for epoch in range(start, params.epoch + 1) ] avg_checkpoint = average_checkpoints(filenames) model.load_state_dict(avg_checkpoint, strict=False) - filename = f"{params.exp_dir}/epoch-{params.epoch}-avg-{params.avg}.pt" - torch.save(avg_checkpoint, filename) + # filename = f"{params.exp_dir}/epoch-{params.epoch}-avg-{params.avg}.pt" + # torch.save(avg_checkpoint, filename) else: checkpoint = torch.load( - f"{params.exp_dir}/epoch-{params.epoch}.pt", map_location="cpu" + f"{params.exp_dir}/epoch-{params.epoch}/pytorch_model.bin", + map_location="cpu", ) model.load_state_dict(checkpoint, strict=False) @@ -643,8 +602,7 @@ def main(): logging.info("Done!") -torch.set_num_threads(1) -torch.set_num_interop_threads(1) - if __name__ == "__main__": + torch.set_num_threads(1) + torch.set_num_interop_threads(1) main() diff --git a/egs/speech_llm/ASR_LLM/whisper_llm_zh/train.py b/egs/speech_llm/ASR_LLM/whisper_llm_zh/train.py index 5f224c984..7947a60a5 100755 --- a/egs/speech_llm/ASR_LLM/whisper_llm_zh/train.py +++ b/egs/speech_llm/ASR_LLM/whisper_llm_zh/train.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 # Copyright 2023 Xiaomi Corp. (authors: Xiaoyu Yang) # 2024 Yuekai Zhang +# 2025 Yifan Yang # # See ../../../../LICENSE for clarification regarding multiple authors # @@ -42,47 +43,32 @@ torchrun --nproc_per_node 8 ./whisper_llm_zh/train.py \ """ import argparse -import copy import logging import os -import random import warnings from pathlib import Path -from shutil import copyfile -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Dict, Optional, Tuple import deepspeed -import k2 import torch -import torch.multiprocessing as mp import torch.nn as nn import transformers import whisper from asr_datamodule import AsrDataModule from deepspeed.utils.zero_to_fp32 import convert_zero_checkpoint_to_fp32_state_dict -from label_smoothing import LabelSmoothingLoss -from lhotse import CutSet, load_manifest from lhotse.cut import Cut -from lhotse.dataset.sampling.base import CutSampler from lhotse.utils import fix_random_seed from model import IGNORE_TOKEN_ID, SPEECH_LLM, EncoderProjector from multi_dataset import MultiDataset -from peft import LoraConfig, get_peft_model, prepare_model_for_kbit_training +from peft import LoraConfig, get_peft_model from torch import Tensor from torch.utils.tensorboard import SummaryWriter from transformers import AutoModelForCausalLM, AutoTokenizer from whisper_encoder_forward_monkey_patch import replace_whisper_encoder_forward -from icefall import diagnostics from icefall.dist import get_rank, get_world_size from icefall.env import get_env_info -from icefall.utils import ( - AttributeDict, - MetricsTracker, - filter_uneven_sized_batch, - setup_logger, - str2bool, -) +from icefall.utils import AttributeDict, MetricsTracker, setup_logger, str2bool DEFAULT_SPEECH_TOKEN = "" @@ -286,13 +272,6 @@ def compute_loss( Returns: Return a tuple of two elements. The first element is the loss tensor. """ - # For the uneven-sized batch, the total duration after padding would possibly - # cause OOM. Hence, for each batch, which is sorted descendingly by length, - # we simply drop the last few shortest samples, so that the retained total frames - # (after padding) would not exceed `allowed_max_frames`: - # `allowed_max_frames = int(max_frames * (1.0 + allowed_excess_duration_ratio))`, - # where `max_frames = max_duration * 1000 // frame_shift_ms`. - # We set allowed_excess_duration_ratio=0.1. def preprocess( messages, @@ -347,46 +326,6 @@ def compute_loss( return input_ids, attention_mask, target_ids - def normalize_text_alimeeting(text: str, normalize: str = "m2met") -> str: - """ - Text normalization similar to M2MeT challenge baseline. - See: https://github.com/yufan-aslp/AliMeeting/blob/main/asr/local/text_normalize.pl - """ - if normalize == "none": - return text - elif normalize == "m2met": - import re - - text = text.replace(" ", "") - text = text.replace("", "") - text = text.replace("<%>", "") - text = text.replace("<->", "") - text = text.replace("<$>", "") - text = text.replace("<#>", "") - text = text.replace("<_>", "") - text = text.replace("", "") - text = text.replace("`", "") - text = text.replace("&", "") - text = text.replace(",", "") - if re.search("[a-zA-Z]", text): - text = text.upper() - text = text.replace("Ａ", "A") - text = text.replace("ａ", "A") - text = text.replace("ｂ", "B") - text = text.replace("ｃ", "C") - text = text.replace("ｋ", "K") - text = text.replace("ｔ", "T") - text = text.replace("，", "") - text = text.replace("丶", "") - text = text.replace("。", "") - text = text.replace("、", "") - text = text.replace("？", "") - return text - - max_frames = params.max_duration * 1000 // params.frame_shift_ms - allowed_max_frames = int(max_frames * (1.0 + params.allowed_excess_duration_ratio)) - batch = filter_uneven_sized_batch(batch, allowed_max_frames) - device = next(model.parameters()).device feature = batch["inputs"] @@ -397,11 +336,10 @@ def compute_loss( batch_idx_train = params.batch_idx_train supervisions = batch["supervisions"] texts = batch["supervisions"]["text"] - # remove spaces in texts - texts = [normalize_text_alimeeting(text) for text in texts] messages = [] for i, text in enumerate(texts): + text = text.replace(" ", "") message = [ {"role": "user", "content": f"{DEFAULT_SPEECH_TOKEN}请转写音频为文字"}, {"role": "assistant", "content": text}, @@ -516,14 +454,17 @@ def train_one_epoch( The rank of the node in DDP training. If no DDP is used, it should be set to 0. """ - model.encoder_projector.train() + model.train() + model.encoder.eval() + if not params.unfreeze_llm: + model.llm.eval() tot_loss = MetricsTracker() for batch_idx, batch in enumerate(train_dl): params.batch_idx_train += 1 batch_size = len(batch["supervisions"]["text"]) - if batch_idx % params.valid_interval == 0 and not params.print_diagnostics: + if batch_idx % params.valid_interval == 0: logging.info("Computing validation loss") valid_info = compute_validation_loss( params=params, @@ -533,6 +474,9 @@ def train_one_epoch( world_size=world_size, ) model.train() + model.encoder.eval() + if not params.unfreeze_llm: + model.llm.eval() logging.info(f"Epoch {params.cur_epoch}, validation: {valid_info}") logging.info( f"Maximum memory allocated so far is {torch.cuda.max_memory_allocated()//1000000}MB" @@ -648,7 +592,6 @@ def run(rank, world_size, args): speech_encoder_dim = whisper_model.dims.n_audio_state for name, param in speech_encoder.named_parameters(): param.requires_grad = False - speech_encoder.eval() tokenizer = AutoTokenizer.from_pretrained(params.llm_path_or_name) if params.use_flash_attn: @@ -671,7 +614,6 @@ def run(rank, world_size, args): if not params.unfreeze_llm: for name, param in llm.named_parameters(): param.requires_grad = False - llm.eval() else: if params.use_lora: lora_config = LoraConfig( @@ -728,7 +670,7 @@ def run(rank, world_size, args): logging.info(f"Device: {device}") model.to(device) - assert params.deepspeed and world_size > 1 + assert params.deepspeed logging.info("Using DeepSpeed") model, optimizer, _, scheduler = deepspeed.initialize( args=params, model=model, model_parameters=model.parameters() @@ -764,7 +706,7 @@ def run(rank, world_size, args): if params.sampler_state_dict_path: sampler_state_dict = torch.load(params.sampler_state_dict_path) sampler_state_dict["max_duration"] = params.max_duration - # TODO: load sampler state dict + train_dl = data_module.train_dataloaders( train_cuts, sampler_state_dict=sampler_state_dict ) @@ -806,15 +748,15 @@ def run(rank, world_size, args): model.save_checkpoint( save_dir=params.exp_dir, - tag=f"epoch-{params.cur_epoch}", + tag=f"zero-epoch-{params.cur_epoch}", client_state={}, exclude_frozen_parameters=True, ) if rank == 0: convert_zero_checkpoint_to_fp32_state_dict( params.exp_dir, - f"{params.exp_dir}/epoch-{params.cur_epoch}.pt", - tag=f"epoch-{params.cur_epoch}", + f"{params.exp_dir}/epoch-{params.cur_epoch}", + tag=f"zero-epoch-{params.cur_epoch}", exclude_frozen_parameters=True, ) # save sampler state dict into checkpoint @@ -824,7 +766,7 @@ def run(rank, world_size, args): f"{params.exp_dir}/epoch-{params.cur_epoch}-sampler.pt", ) - os.system(f"rm -rf {params.exp_dir}/epoch-{params.cur_epoch}") + os.system(f"rm -rf {params.exp_dir}/zero-epoch-{params.cur_epoch}") logging.info("Done!") @@ -865,6 +807,7 @@ def main(): torch.set_num_threads(1) torch.set_num_interop_threads(1) + warnings.filterwarnings("ignore", category=FutureWarning) run(rank=rank, world_size=world_size, args=args) From 4627969ccd151caf335c0fb6a2bfbe8bf609e04f Mon Sep 17 00:00:00 2001 From: Yifan Yang <64255737+yfyeung@users.noreply.github.com> Date: Mon, 12 May 2025 14:19:53 +0800 Subject: [PATCH 41/59] fix bug: undefined name 'partial' (#1941) --- egs/wenetspeech/KWS/zipformer/finetune.py | 1 + 1 file changed, 1 insertion(+) diff --git a/egs/wenetspeech/KWS/zipformer/finetune.py b/egs/wenetspeech/KWS/zipformer/finetune.py index 249209352..b1abfd79e 100755 --- a/egs/wenetspeech/KWS/zipformer/finetune.py +++ b/egs/wenetspeech/KWS/zipformer/finetune.py @@ -69,6 +69,7 @@ import argparse import copy import logging import warnings +from functools import partial from pathlib import Path from typing import List, Optional, Tuple, Union From e79833aad278f09792deceab5962b09ae4f56378 Mon Sep 17 00:00:00 2001 From: Yifan Yang <64255737+yfyeung@users.noreply.github.com> Date: Mon, 12 May 2025 19:28:48 +0800 Subject: [PATCH 42/59] ensure SwooshL/SwooshR output dtype matches input dtype (#1940) --- egs/librispeech/ASR/zipformer/scaling.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/egs/librispeech/ASR/zipformer/scaling.py b/egs/librispeech/ASR/zipformer/scaling.py index 6d6281903..11375385e 100644 --- a/egs/librispeech/ASR/zipformer/scaling.py +++ b/egs/librispeech/ASR/zipformer/scaling.py @@ -1403,9 +1403,9 @@ class SwooshL(torch.nn.Module): zero = torch.tensor(0.0, dtype=x.dtype, device=x.device) return logaddexp(zero, x - 4.0) - 0.08 * x - 0.035 if not x.requires_grad: - return k2.swoosh_l_forward(x) + return k2.swoosh_l_forward(x).to(x.dtype) else: - return k2.swoosh_l(x) + return k2.swoosh_l(x).to(x.dtype) # return SwooshLFunction.apply(x) @@ -1477,9 +1477,9 @@ class SwooshR(torch.nn.Module): zero = torch.tensor(0.0, dtype=x.dtype, device=x.device) return logaddexp(zero, x - 1.0) - 0.08 * x - 0.313261687 if not x.requires_grad: - return k2.swoosh_r_forward(x) + return k2.swoosh_r_forward(x).to(x.dtype) else: - return k2.swoosh_r(x) + return k2.swoosh_r(x).to(x.dtype) # return SwooshRFunction.apply(x) From fd8f8780fac68d959df0bf521f1b76aac35f88fd Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Wed, 21 May 2025 12:04:57 +0800 Subject: [PATCH 43/59] Fix logging torch.dtype. (#1947) --- icefall/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/icefall/utils.py b/icefall/utils.py index aab479e56..ffb926566 100644 --- a/icefall/utils.py +++ b/icefall/utils.py @@ -186,7 +186,7 @@ class AttributeDict(dict): tmp = {} for k, v in self.items(): # PosixPath is ont JSON serializable - if isinstance(v, pathlib.Path) or isinstance(v, torch.device): + if isinstance(v, (pathlib.Path, torch.device, torch.dtype)): v = str(v) tmp[k] = v return json.dumps(tmp, indent=indent, sort_keys=True) From 30e7ea4b5a0449bfac4e7fa5eef9a23238230013 Mon Sep 17 00:00:00 2001 From: Tianxiang Zhao <162714929+Redemption-ZTX@users.noreply.github.com> Date: Thu, 22 May 2025 12:05:01 +0800 Subject: [PATCH 44/59] Fix a bug in finetune.py --use-mux (#1949) --- egs/librispeech/ASR/zipformer/finetune.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/egs/librispeech/ASR/zipformer/finetune.py b/egs/librispeech/ASR/zipformer/finetune.py index 2ff631914..2c869a57a 100755 --- a/egs/librispeech/ASR/zipformer/finetune.py +++ b/egs/librispeech/ASR/zipformer/finetune.py @@ -140,8 +140,8 @@ def add_finetune_arguments(parser: argparse.ArgumentParser): type=str2bool, default=False, help=""" - Whether to adapt. If true, we will mix 5% of the new data - with 95% of the original data to fine-tune. This is useful + Whether to adapt. If true, we will mix 5%% of the new data + with 95%% of the original data to fine-tune. This is useful if you want to maintain the performance on the original domain """, ) @@ -1134,7 +1134,7 @@ def train_one_epoch( f"Epoch {params.cur_epoch}, " f"batch {batch_idx}, loss[{loss_info}], " f"tot_loss[{tot_loss}], batch size: {batch_size}, " - f"lr: {cur_lr:.2e}, " + f"lr: {cur_lr: .2e}, " + (f"grad_scale: {scaler._scale.item()}" if params.use_fp16 else "") ) From 021e1a88469f5ca8299daa4000d05a5dd89b8623 Mon Sep 17 00:00:00 2001 From: Mahsa Yarmohammadi <35142904+mahsa7823@users.noreply.github.com> Date: Thu, 22 May 2025 10:06:35 -0400 Subject: [PATCH 45/59] Add acknowledgment to README (#1950) --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 0e550ffb1..498f7e3b4 100644 --- a/README.md +++ b/README.md @@ -383,3 +383,7 @@ Please see: [![Open In Colab](https://colab.research.google.com/assets/colab-bad [vctk]: egs/vctk/TTS [ljspeech]: egs/ljspeech/TTS [libritts_tts]: egs/libritts/TTS + +## Acknowledgements + +Some contributors to this project were supported by Xiaomi Corporation. Others were supported by National Science Foundation CCRI award 2120435. This is not an exhaustive list of sources of support. From ffb7d0563526f42123d2c65c284646a5c375b74f Mon Sep 17 00:00:00 2001 From: Zengwei Yao Date: Tue, 27 May 2025 12:09:59 +0800 Subject: [PATCH 46/59] refactor branch exchange in cr-ctc (#1954) --- egs/librispeech/ASR/zipformer/model.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/egs/librispeech/ASR/zipformer/model.py b/egs/librispeech/ASR/zipformer/model.py index c7dbe1e0a..f2791e51f 100644 --- a/egs/librispeech/ASR/zipformer/model.py +++ b/egs/librispeech/ASR/zipformer/model.py @@ -210,10 +210,10 @@ class AsrModel(nn.Module): ) # Compute consistency regularization loss - exchanged_targets = ctc_output.detach().chunk(2, dim=0) - exchanged_targets = torch.cat( - [exchanged_targets[1], exchanged_targets[0]], dim=0 - ) # exchange: [x1, x2] -> [x2, x1] + batch_size = ctc_output.shape[0] + assert batch_size % 2 == 0, batch_size + # exchange: [x1, x2] -> [x2, x1] + exchanged_targets = torch.roll(ctc_output.detach(), batch_size // 2, dims=0) cr_loss = nn.functional.kl_div( input=ctc_output, target=exchanged_targets, From 06539d2b9d5b4945ba9ce6edaf301eda13748b6b Mon Sep 17 00:00:00 2001 From: Wei Kang Date: Tue, 17 Jun 2025 20:17:12 +0800 Subject: [PATCH 47/59] Add Zipvoice (#1964) * Add ZipVoice - a flow-matching based zero-shot TTS model. --- egs/zipvoice/README.md | 411 ++++ egs/zipvoice/assets/prompt-en.wav | Bin 0 -> 675496 bytes egs/zipvoice/local/compute_fbank.py | 288 +++ egs/zipvoice/local/evaluate_sim.py | 508 +++++ egs/zipvoice/local/evaluate_utmos.py | 294 +++ egs/zipvoice/local/evaluate_wer_hubert.py | 172 ++ egs/zipvoice/local/evaluate_wer_seedtts.py | 181 ++ egs/zipvoice/local/feature.py | 1 + .../local/prepare_token_file_emilia.py | 90 + .../local/prepare_token_file_libritts.py | 31 + egs/zipvoice/local/preprocess_emilia.py | 156 ++ egs/zipvoice/local/tokenizer.py | 1 + egs/zipvoice/local/validate_manifest.py | 70 + egs/zipvoice/scripts/evaluate.sh | 102 + egs/zipvoice/scripts/prepare_emilia.sh | 126 ++ egs/zipvoice/scripts/prepare_libritts.sh | 97 + egs/zipvoice/zipvoice/checkpoint.py | 142 ++ egs/zipvoice/zipvoice/feature.py | 135 ++ .../zipvoice/generate_averaged_model.py | 209 ++ egs/zipvoice/zipvoice/infer.py | 586 +++++ egs/zipvoice/zipvoice/model.py | 578 +++++ egs/zipvoice/zipvoice/optim.py | 1256 +++++++++++ egs/zipvoice/zipvoice/scaling.py | 1910 +++++++++++++++++ egs/zipvoice/zipvoice/solver.py | 277 +++ egs/zipvoice/zipvoice/tokenizer.py | 570 +++++ egs/zipvoice/zipvoice/train_distill.py | 1043 +++++++++ egs/zipvoice/zipvoice/train_flow.py | 1108 ++++++++++ egs/zipvoice/zipvoice/tts_datamodule.py | 456 ++++ egs/zipvoice/zipvoice/utils.py | 219 ++ egs/zipvoice/zipvoice/zipformer.py | 1648 ++++++++++++++ egs/zipvoice/zipvoice/zipvoice_infer.py | 645 ++++++ 31 files changed, 13310 insertions(+) create mode 100644 egs/zipvoice/README.md create mode 100644 egs/zipvoice/assets/prompt-en.wav create mode 100644 egs/zipvoice/local/compute_fbank.py create mode 100644 egs/zipvoice/local/evaluate_sim.py create mode 100644 egs/zipvoice/local/evaluate_utmos.py create mode 100644 egs/zipvoice/local/evaluate_wer_hubert.py create mode 100644 egs/zipvoice/local/evaluate_wer_seedtts.py create mode 120000 egs/zipvoice/local/feature.py create mode 100644 egs/zipvoice/local/prepare_token_file_emilia.py create mode 100644 egs/zipvoice/local/prepare_token_file_libritts.py create mode 100644 egs/zipvoice/local/preprocess_emilia.py create mode 120000 egs/zipvoice/local/tokenizer.py create mode 100644 egs/zipvoice/local/validate_manifest.py create mode 100644 egs/zipvoice/scripts/evaluate.sh create mode 100755 egs/zipvoice/scripts/prepare_emilia.sh create mode 100755 egs/zipvoice/scripts/prepare_libritts.sh create mode 100644 egs/zipvoice/zipvoice/checkpoint.py create mode 100644 egs/zipvoice/zipvoice/feature.py create mode 100644 egs/zipvoice/zipvoice/generate_averaged_model.py create mode 100644 egs/zipvoice/zipvoice/infer.py create mode 100644 egs/zipvoice/zipvoice/model.py create mode 100644 egs/zipvoice/zipvoice/optim.py create mode 100644 egs/zipvoice/zipvoice/scaling.py create mode 100644 egs/zipvoice/zipvoice/solver.py create mode 100644 egs/zipvoice/zipvoice/tokenizer.py create mode 100644 egs/zipvoice/zipvoice/train_distill.py create mode 100644 egs/zipvoice/zipvoice/train_flow.py create mode 100644 egs/zipvoice/zipvoice/tts_datamodule.py create mode 100644 egs/zipvoice/zipvoice/utils.py create mode 100644 egs/zipvoice/zipvoice/zipformer.py create mode 100644 egs/zipvoice/zipvoice/zipvoice_infer.py diff --git a/egs/zipvoice/README.md b/egs/zipvoice/README.md new file mode 100644 index 000000000..4bca60301 --- /dev/null +++ b/egs/zipvoice/README.md @@ -0,0 +1,411 @@ +## ZipVoice: Fast and High-Quality Zero-Shot Text-to-Speech with Flow Matching + + +[![arXiv](https://img.shields.io/badge/arXiv-Paper-COLOR.svg)](http://arxiv.org/abs/2506.13053) +[![demo](https://img.shields.io/badge/GitHub-Demo%20page-orange.svg)](https://zipvoice.github.io/) + + +## Overview +ZipVoice is a high-quality zero-shot TTS model with a small model size and fast inference speed. +#### Key features: + +- Small and fast: only 123M parameters. + +- High-quality: state-of-the-art voice cloning performance in speaker similarity, intelligibility, and naturalness. + +- Multi-lingual: support Chinese and English. + + +## News +**2025/06/16**: 🔥 ZipVoice is released. + + +## Installation + +* Clone icefall repository and change to zipvoice directory: + +```bash +git clone https://github.com/k2-fsa/icefall.git +cd icefall/egs/zipvoice +``` + +* Create a Python virtual environment (optional but recommended): + +```bash +python3 -m venv venv +source venv/bin/activate +``` + +* Install the required packages: + +```bash +# Install pytorch and k2. +# If you want to use different versions, please refer to https://k2-fsa.org/get-started/k2/ for details. +# For users in China mainland, please refer to https://k2-fsa.org/zh-CN/get-started/k2/ + +pip install torch==2.5.1 torchaudio==2.5.1 --index-url https://download.pytorch.org/whl/cu121 +pip install k2==1.24.4.dev20250208+cuda12.1.torch2.5.1 -f https://k2-fsa.github.io/k2/cuda.html + +# Install other dependencies. +pip install piper_phonemize -f https://k2-fsa.github.io/icefall/piper_phonemize.html +pip install -r requirements.txt +``` + +## Usage + +To generate speech with our pre-trained ZipVoice or ZipVoice-Distill models, use the following commands (Required models will be downloaded from HuggingFace): + +### 1. Inference of a single sentence: +```bash +python3 zipvoice/zipvoice_infer.py \ + --model-name "zipvoice_distill" \ + --prompt-wav prompt.wav \ + --prompt-text "I am the transcription of the prompt wav." \ + --text "I am the text to be synthesized." \ + --res-wav-path result.wav + +# Example with a pre-defined prompt wav and text +python3 zipvoice/zipvoice_infer.py \ + --model-name "zipvoice_distill" \ + --prompt-wav assets/prompt-en.wav \ + --prompt-text "Some call me nature, others call me mother nature. I've been here for over four point five billion years, twenty two thousand five hundred times longer than you." \ + --text "Welcome to use our tts model, have fun!" \ + --res-wav-path result.wav +``` + +### 2. Inference of a list of sentences: +```bash +python3 zipvoice/zipvoice_infer.py \ + --model-name "zipvoice_distill" \ + --test-list test.tsv \ + --res-dir results/test +``` + +- `--model-name` can be `zipvoice` or `zipvoice_distill`, which are models before and after distillation, respectively. +- Each line of `test.tsv` is in the format of `{wav_name}\t{prompt_transcription}\t{prompt_wav}\t{text}`. + + +> **Note:** If you having trouble connecting to HuggingFace, try: +```bash +export HF_ENDPOINT=https://hf-mirror.com +``` + +## Training Your Own Model + +The following steps show how to train a model from scratch on Emilia and LibriTTS datasets, respectively. + +### 0. Install dependencies for training + +```bash +pip install -r ../../requirements.txt +``` + +### 1. Data Preparation + +#### 1.1. Prepare the Emilia dataset + +```bash +bash scripts/prepare_emilia.sh +``` + +See [scripts/prepare_emilia.sh](scripts/prepare_emilia.sh) for step by step instructions. + +#### 1.2 Prepare the LibriTTS dataset + +```bash +bash scripts/prepare_libritts.sh +``` + +See [scripts/prepare_libritts.sh](scripts/prepare_libritts.sh) for step by step instructions. + +### 2. Training + +#### 2.1 Traininig on Emilia + +
+Expand to view training steps + +##### 2.1.1 Train the ZipVoice model + +- Training: + +```bash +export PYTHONPATH=../../:$PYTHONPATH +python3 zipvoice/train_flow.py \ + --world-size 8 \ + --use-fp16 1 \ + --dataset emilia \ + --max-duration 500 \ + --lr-hours 30000 \ + --lr-batches 7500 \ + --token-file "data/tokens_emilia.txt" \ + --manifest-dir "data/fbank" \ + --num-epochs 11 \ + --exp-dir zipvoice/exp_zipvoice +``` + +- Average the checkpoints to produce the final model: + +```bash +export PYTHONPATH=../../:$PYTHONPATH +python3 zipvoice/generate_averaged_model.py \ + --epoch 11 \ + --avg 4 \ + --distill 0 \ + --token-file data/tokens_emilia.txt \ + --dataset "emilia" \ + --exp-dir ./zipvoice/exp_zipvoice +# The generated model is zipvoice/exp_zipvoice/epoch-11-avg-4.pt +``` + +##### 2.1.2. Train the ZipVoice-Distill model (Optional) + +- The first-stage distillation: + +```bash +export PYTHONPATH=../../:$PYTHONPATH +python3 zipvoice/train_distill.py \ + --world-size 8 \ + --use-fp16 1 \ + --tensorboard 1 \ + --dataset "emilia" \ + --base-lr 0.0005 \ + --max-duration 500 \ + --token-file "data/tokens_emilia.txt" \ + --manifest-dir "data/fbank" \ + --teacher-model zipvoice/exp_zipvoice/epoch-11-avg-4.pt \ + --num-updates 60000 \ + --distill-stage "first" \ + --exp-dir zipvoice/exp_zipvoice_distill_1stage +``` + +- Average checkpoints for the second-stage initialization: + +```bash +export PYTHONPATH=../../:$PYTHONPATH +python3 zipvoice/generate_averaged_model.py \ + --iter 60000 \ + --avg 7 \ + --distill 1 \ + --token-file data/tokens_emilia.txt \ + --dataset "emilia" \ + --exp-dir ./zipvoice/exp_zipvoice_distill_1stage +# The generated model is zipvoice/exp_zipvoice_distill_1stage/iter-60000-avg-7.pt +``` + +- The second-stage distillation: + +```bash +export PYTHONPATH=../../:$PYTHONPATH +python3 zipvoice/train_distill.py \ + --world-size 8 \ + --use-fp16 1 \ + --tensorboard 1 \ + --dataset "emilia" \ + --base-lr 0.0001 \ + --max-duration 200 \ + --token-file "data/tokens_emilia.txt" \ + --manifest-dir "data/fbank" \ + --teacher-model zipvoice/exp_zipvoice_distill_1stage/iter-60000-avg-7.pt \ + --num-updates 2000 \ + --distill-stage "second" \ + --exp-dir zipvoice/exp_zipvoice_distill_new +``` +
+ + +#### 2.2 Traininig on LibriTTS + +
+Expand to view training steps + +##### 2.2.1 Train the ZipVoice model + +- Training: + +```bash +export PYTHONPATH=../../:$PYTHONPATH +python3 zipvoice/train_flow.py \ + --world-size 8 \ + --use-fp16 1 \ + --dataset libritts \ + --max-duration 250 \ + --lr-epochs 10 \ + --lr-batches 7500 \ + --token-file "data/tokens_libritts.txt" \ + --manifest-dir "data/fbank" \ + --num-epochs 60 \ + --exp-dir zipvoice/exp_zipvoice_libritts +``` + +- Average the checkpoints to produce the final model: + +```bash +export PYTHONPATH=../../:$PYTHONPATH +python3 zipvoice/generate_averaged_model.py \ + --epoch 60 \ + --avg 10 \ + --distill 0 \ + --token-file data/tokens_libritts.txt \ + --dataset "libritts" \ + --exp-dir ./zipvoice/exp_zipvoice_libritts +# The generated model is zipvoice/exp_zipvoice_libritts/epoch-60-avg-10.pt +``` + +##### 2.1.2 Train the ZipVoice-Distill model (Optional) + +- The first-stage distillation: + +```bash +export PYTHONPATH=../../:$PYTHONPATH +python3 zipvoice/train_distill.py \ + --world-size 8 \ + --use-fp16 1 \ + --tensorboard 1 \ + --dataset "libritts" \ + --base-lr 0.001 \ + --max-duration 250 \ + --token-file "data/tokens_libritts.txt" \ + --manifest-dir "data/fbank" \ + --teacher-model zipvoice/exp_zipvoice_libritts/epoch-60-avg-10.pt \ + --num-epochs 6 \ + --distill-stage "first" \ + --exp-dir zipvoice/exp_zipvoice_distill_1stage_libritts +``` + +- Average checkpoints for the second-stage initialization: + +```bash +export PYTHONPATH=../../:$PYTHONPATH +python3 ./zipvoice/generate_averaged_model.py \ + --epoch 6 \ + --avg 3 \ + --distill 1 \ + --token-file data/tokens_libritts.txt \ + --dataset "libritts" \ + --exp-dir ./zipvoice/exp_zipvoice_distill_1stage_libritts +# The generated model is zipvoice/exp_zipvoice_distill_1stage_libritts/epoch-6-avg-3.pt +``` + +- The second-stage distillation: + +```bash +export PYTHONPATH=../../:$PYTHONPATH +python3 zipvoice/train_distill.py \ + --world-size 8 \ + --use-fp16 1 \ + --tensorboard 1 \ + --dataset "libritts" \ + --base-lr 0.001 \ + --max-duration 250 \ + --token-file "data/tokens_libritts.txt" \ + --manifest-dir "data/fbank" \ + --teacher-model zipvoice/exp_zipvoice_distill_1stage_libritts/epoch-6-avg-3.pt \ + --num-epochs 6 \ + --distill-stage "second" \ + --exp-dir zipvoice/exp_zipvoice_distill_libritts +``` + +- Average checkpoints to produce the final model: + +```bash +export PYTHONPATH=../../:$PYTHONPATH +python3 ./zipvoice/generate_averaged_model.py \ + --epoch 6 \ + --avg 3 \ + --distill 1 \ + --token-file data/tokens_libritts.txt \ + --dataset "libritts" \ + --exp-dir ./zipvoice/exp_zipvoice_distill_libritts +# The generated model is ./zipvoice/exp_zipvoice_distill_libritts/epoch-6-avg-3.pt +``` +
+ + +### 3. Inference with the trained model + +#### 3.1 Inference with the model trained on Emilia +
+Expand to view inference commands. + +##### 3.1.1 ZipVoice model before distill: +```bash +export PYTHONPATH=../../:$PYTHONPATH +python3 zipvoice/infer.py \ + --checkpoint zipvoice/exp_zipvoice/epoch-11-avg-4.pt \ + --distill 0 \ + --token-file "data/tokens_emilia.txt" \ + --test-list test.tsv \ + --res-dir results/test \ + --num-step 16 \ + --guidance-scale 1 +``` + +##### 3.1.2 ZipVoice-Distill model before distill: +```bash +export PYTHONPATH=../../:$PYTHONPATH +python3 zipvoice/infer.py \ + --checkpoint zipvoice/exp_zipvoice_distill/checkpoint-2000.pt \ + --distill 1 \ + --token-file "data/tokens_emilia.txt" \ + --test-list test.tsv \ + --res-dir results/test_distill \ + --num-step 8 \ + --guidance-scale 3 +``` +
+ + +#### 3.2 Inference with the model trained on LibriTTS + +
+Expand to view inference commands. + +##### 3.2.1 ZipVoice model before distill: +```bash +export PYTHONPATH=../../:$PYTHONPATH +python3 zipvoice/infer.py \ + --checkpoint zipvoice/exp_zipvoice_libritts/epoch-60-avg-10.pt \ + --distill 0 \ + --token-file "data/tokens_libritts.txt" \ + --test-list test.tsv \ + --res-dir results/test_libritts \ + --num-step 8 \ + --guidance-scale 1 \ + --target-rms 1.0 \ + --t-shift 0.7 +``` + +##### 3.2.2 ZipVoice-Distill model before distill + +```bash +export PYTHONPATH=../../:$PYTHONPATH +python3 zipvoice/infer.py \ + --checkpoint zipvoice/exp_zipvoice_distill/epoch-6-avg-3.pt \ + --distill 1 \ + --token-file "data/tokens_libritts.txt" \ + --test-list test.tsv \ + --res-dir results/test_distill_libritts \ + --num-step 4 \ + --guidance-scale 3 \ + --target-rms 1.0 \ + --t-shift 0.7 +``` +
+ +### 4. Evaluation on benchmarks + +See [local/evaluate.sh](local/evaluate.sh) for details of objective metrics evaluation +on three test sets, i.e., LibriSpeech-PC test-clean, Seed-TTS test-en and Seed-TTS test-zh. + + +## Citation + +```bibtex +@article{zhu-2025-zipvoice, + title={ZipVoice: Fast and High-Quality Zero-Shot Text-to-Speech with Flow Matching}, + author={Han Zhu and Wei Kang and Zengwei Yao and Liyong Guo and Fangjun Kuang and Zhaoqing Li and Weiji Zhuang and Long Lin and Daniel Povey} + journal={arXiv preprint arXiv:2506.13053}, + year={2025}, +} +``` diff --git a/egs/zipvoice/assets/prompt-en.wav b/egs/zipvoice/assets/prompt-en.wav new file mode 100644 index 0000000000000000000000000000000000000000..b7047ce9b4cfb61b49db59aa99bc4600316218ee GIT binary patch literal 675496 zcmXtB1)LSt7oK_j_HD4c>@FLyba#W~pOlmkP)ZOKBvz49k&sYON>GqSkd*GOrMs7I z*v_}FXXgKXcaZbDckaZSxcA%>-#K^b{$bm;Yq|tUy<7Hb^TnvKC5uRsB!f=&m!zpj zB*`EZk%kN!J7^NFiIl>mXsL`;PD+$2Nfo5>QZ3Xa(XJ`g0DT}e05z4GNv)-hQg>;v zG(Z|C^#cu%21&!DuccAa1Zg~I3g~BPwlq&#AT5!WNh_s)q}9?UX)|cIv|l`)1i8ejC7@&A6K?6-pl5pZiX19d*O`FNjfoXf!V8EBE-y+6WhvTV0#Cx8`4!V7DuH+(oty_#_AtwJ;qI^Wf;Td z(jsY=^ee{nXK4mnGo&9uQ!(P-VZH~N7`$SLimmQo|ZJJrBN)ugK6 zv0_rZ6e^h|JIJrOG^duQz0zK4ueIk|igsJOp`FrBXvej~+5v5gwoY5Gt<+X&%e3D? zbG0d;soD?PByGC(otC7H(Z14#fck2^wI13>D4n&|S}U!E*7Q9!M!SjD7<;UnqtpRa z)vBY^)T(GTwECz?HF2Cr|EueZU)0ClT510SH5c2q+K1vfpK1fNUfQSHU~MSgVv;r% zG*i6WbnO@IH*LPQ1hfKgzEWEY+Nf>NwrRVxgW6GTAL`@UX_Rx?HI&^&3x^G?gv3eaB#nn6-EvlNaQ?ANTA)r#iRB*`WP<5;K^g&r;riUmbs zR)uZ!zSy62Ck@tV+rCJd~%|=rZ|ozf=Y_tR6wgVC;_A& z<8vzGHas-@=Gz2&$I_PdPBP>_?oqm)@ii3J%ZL* z@l^8K1H9cke905>4bSklnc~~>z)9>02YNxq{0v5qu_sHT*AvC4louMICb*(8^g$b` z9ZDC>_O57k2YoE{mU>E`pdJA2FbEQ11ZcRF1R4$*F$UTq2~y*GNR1hgCv%}I=0aC2 zg0@%-S}Lsqt(X3mHcDGX*$Ta}N7{jM5ao!VW70`zk~7dG2O+giK%1PE&Y|Z18A!8> zpsOf1Anz^-EkliS9<5`z?hFdIx5X7_@cUz+6OevKarcwB*9ly6410(0#C>@FE^K#6 z+o8iYiMLuMw*TP$HbPqd4Wi7X)LexxSb{>S$*=hXEgo5f?OGv8=Shq4y?=lfiSzo^ zzv9?jNLcF4U(qwaqF;VR|I89w_7Qt(I?6Qk-H*_+--E_T5nA7fZ=V-H}p`@Zt6UUz8*khEJC{NY9Xg?E2?qT~_ zO;I1H_i*$cwzsf-q+VAqt2eQK2m6oki|e@NF}80|KSJT1QgLVAKU2+Dv+yP^w6j#V z>J@D_-Yge|#J3NKTGB!^hgJj>qJ?TU)J3$S_`(P+9^adwmB4o=pj5$rIjti4p**O% zRtY+*I{Ky_bXgPhR148}?X+&#c7b;4gkEe79n}YWy`ULCMNbdHmi^ra+X3kBVcM6_ zTO+m6ppjxb3Ho%rHW8yU8Txe!>RHgSGoj0VLe0^eBkJFTUi(w~3tE|a`7ey`8tAuG z81XI8a_cbeZ0!Rb5JZi4O31Wx+6BnF>)I{owI`69)OVCYFSRsCpm##9x`hn#Ym~_f zWU~StX9CwbKvtAcq2nSUm8jhkg}y5(IF|Z{dZz|t6m?oHY#YI5ZH)Q@Y}aR+wk4u*Y+(Ip6r?53tuB`+Tpac(bOErPQ|6r?rG`t%R>BDYR*M=)QQ& z!g$DHe!l|Q>=QaD8~ybX^ccPP7<3E$dm6ob0=>QqcE(PO!8(i!=iV%g+a!$ac-TuL zG3tZC6`jBdol$=XPH7F>rUk5ULo69McMyx;mTOI#>4@Y!~&3ikO7 zALcoas;U+9-l|zJUwFidZMYUC=0_ZA&Wp03szPcs#k^>&HG&lBfG_L~t|#{o5SH2q z^ujpM58!t4_;kV5GljkOJGgm~=&@zs=M}KqHi4hFYg^Hy)Uv0*$A=-|4xl~`8Ar`Z z-Fr^x@YB%d*P+?(g2>;G1!q6U_~@hb8vOkd((*NUoulW545bwHL*ptaZo%g&YMZc) z9k4shLJvEzPrD{oZ~|?+Vvx<{gq>6gd{YauyDqq=K6tAUc&QdRi~QXhM6T=rj_e8g z1T+9zVGwAj&=Z59DMkogF&z7!Lt78S-bmLX$%+zUFy(+lraNkA&oc_Wt<5Uwxw0rnDl8V zqii^8#ns{14+hctq#eb*F!;yV6prQ%ZXOZ9%0>_fOT^kbP~4EbO6`5vjv{A175QMh5mB|P<6Sf+)!>Tw?k_wdL-FyaO6Jd zDULppUt;?L^&9yaw(n4$%Bk4C1ZB!;at?@lFK|8u=kB7u`~F^c@TBv2-bMKw-r|6K zSU!Yz<#(@=m!mIMpk566U7jz`Mc++B-~A|$l82)|*{37rf#~DDpiUTp_VWMaW*CiH zay7ZUTm_{pC|V#ML^X(I6ap8>0Algry{}Je_XOw&c!PHBCSjj0hkZ%Aa|XC*Ja}q2 zIBTfTeIJASS_^&l0VE7P!E%s9QQ&0CB^ef>9sZ~r5-c0i%!N7~lqPihEm(#0b*@7; zUVzs^zvYO~Pgg%$pSD7eQHwQ&?5FSE5Or;ohR|#Eg`ZE4zlNZyu%c>$=zu`VD-g2lDaNiQpbh=eYOR)$va?$q`1q~R7 zQVc{bSQbUM8{_c1IQ+6S?pqp9;MelI^c;Qgq~^v-QNUtD`7Xb*4n~$>;n6+ zH~M#gunoUNe~;0=5q2SU<}a`&I4-lb#n6^3p)VI<)c(RKu7}p#1j}O^tp8mY-JQ^# z+o3P1Dd~xxfQ_PiyVRU#p-+?H1z!MN5gc+E+(HkTzVUUG+t8!LK<;6Wmf}5}rT2PQ z)by+G!jruvJn9>$dDmp|kGTYO;Dk-K(C(-a)kYqkU4;d(?l`E$T+l7InM2 z4STEA_3BRSZN}b4bq#2}x=j5`T_NhFsF%Jc9{mgFSK>FzQ2&kXO3+`}TOoeA3`hP3 zty4GPUVo#k2W`OqMm%k!c=o?|+D37H3+i2Xvz;h=)B~WCc;jQJlkq(l)bsFyF2N6a zghFrVneei*Ut`}vUz}twm9RO$0 zIz9^`HxhYE2FIKOuiX-S_5}9YYk|7Gg^fmEBp0&G4a>|2T*wY>u4=)Mc#*ITh*c4- zB2qjq7PgOW3t1(4dM4Ocw0?=HrNI+?hmr;l z=N;$?=Ke#>@C#z5Uxt*ROyHJu4Wr00bP98*b=e3i#MU{xsG=trbn+eOJEC*-0u*LUJ z$C+H9BmAa3aFGPWM8%id@%6L^iAfS!3&Xwxt!Nw}&Jv3rBl;4HUMr2hE002-@B{R3 zebK}9v87kpQScf)%#P@Hjz=3YHeH2{OT33(XM5PZ-C*<6TkRrvvLo!qo`N?yjvY~G z5qA`4=?k|*y9Km8ePsHq-LT(P@NpNkyP|vyZXiz#fbBU9Gzw)L{DD!T{t-3pc3S(4 z5-<*=V_HPHR)cqm?fnbx-45GhpRhwt3#5$}3GI?I0;SVqLRS%=xPy4bBN1YcGh(cR$Mv|%Q`oVLzdXk_562k8VSI#FJ%t7I2=!~kLm2sB6ogN@4y^E! zu!o5X>Jcg;il;?fx?FTzI6%A>XKRcSGK76WHqPm`R2^(-7cr_ri;BNtT@SycjnqU) z_$Ih>E3{e&l&=$N#!)&$*6a4$r+A+Mz@&yj`wqpMl7_+BBQ}*JXdHC*6qKKY1wzSB zxxY#vPpg1*tV5giTJ!*sDQ+2c*@-^c3XQ)PbP&3KANq{CfGFh=^c>6H_ws=F6lDOB z^?fMB-Y)6YdKOUMBa#*;Ax z&UwaU^r#FY7>r-gt}X+~Srs~kr2*!CP0W1aRK$rIKn_sD(1Rt;)CKZ*AULTH<|DoK z&%tY>gl9ey9`e`FGpxUbPfU+|0%#Iw3W#1Jr9Gp0^t=B<{g#CHYDzgL{y-;YbX=0}TjI%$BE4j~m@=3E$_-o#S-o-re3u4&xQTZP9 z)5qcc#$Y=E9tWjJlJIGhFkcus_*}@Ff%wWnX!jN0`!V`}8n!cfgHnp<9z7G{*u+8T z)7KT6xhnRW;^=>|Bz;TG#28cpYN9E`Qc9!mDQ~$Y`bO_OUW^HG7RI)CEv2#^J)`cY zAH=i79ZQPqV__W-L(naR1RQ0H2p;2Rl;0l9z;Ij7uLbo*AZ7T_C_uI-xpUEPhsq?4Bnp4;+K^~n_8J~L9C-V z-Z&In@?o&h(vhf{6F?6y9NTalcL2p82FW;p0e+tuwI8~Tcn6~bta+RfUFtjfgbK7C zEhwkZ=xLY*1>&d&8lUIsJ+c^Ze=D9vbR+_A6N@eTG9KT@UaX3DWUPv_Bz_z8e2v=x(G@8A8taNm&ge<8sj3}z*W>|ebAzgB(BmAbBc1BQkvK= zukVgRnXMzm`aOsaQzLf7FL|6Amb^&4O5S9wr6aUxYaAi++Y;}?ncM~~qI5)k>*IZi z+Ev47^DAoLE6U*rqn+e8@)gG@0e!+Krrt+Hn213UKU9TYVDDx^H!vQV23siwHtjXk zw2@LEpDttn3Xap>rENp6D;ZLc67Vo&-)_iAN=Hh`t&o^2fqkwM8ieRE=}*Yo-yv;( z!sRq=>H^jSa8H)6P)5Qd7>4o<3N^%d_ym*T9n3`in|Qz3;vJ_8 zk998EeE&Z|jK}{Df8j5oA=U_Af${lG;_G+8W7W$(NO#(ldK^`^5^qEP(?UEatVnvQ zv=+~bp4R==Td)x6QQQ@J;FYiqX%l8a6F5QHVg}@a|D9+N^<^xLwjyKFw1^dqDy>A$ zx=`>vQQrh?i^D=BPjKc^j!=Hk_n>TGW(B!OmjZP$A8LWa$bEXWkDT{`m=nz4p$2J( z(oI-{Jq6F}m@v^`Mr?+3Gd{`f93kZwqNcoG z44ZE;ycotpSHnYDFS3Ak!DiYF+-Wy#sDDA5MNO-UWxXgHv1JUHzRoJtYfB)N}E~h0rF0hSN+w`ria|nA8girzkbT66nTw%IL=##>_K6Ww{`oMn%rCG> z?~u6>Aw5BaJrBnW(lhK6Gc!r_8S>ye6o42jaH6JPK~E@6)al?XJyYf}`0F+Ho?(7J zgw4SjPLJgpX8A45@XNxAVLbMtuwm%^UIbs$OWKQ>&%6`zH6;LN{x0zT7Qy-3F#nlx zLw|a)m?!kFIeYYZ{uAccM9lN?nA2n6lP6&Y>oyzna2R9cd>#bKg z2Z+(gJ|Z?rKcCUcwm_hmo5L)f`oO7le2wuOqF{-r;{|TOcux%QmI#3%IDoh?0vv+9 zAjE@gKo}g@7R5FO$X6V$j0b)ehY~N4Fvd}d^!?sr9t7%TBI z{q7g=1diyzZmCks6WLB_r_R%iZi5P;+ijo+}0&GEqh&Nk9^;!W3f+p z{w0X9XkA;3!PCcKk8d$f^!a%4Zj(?`*UZ7&QvWOfR}zEd_IL0seI9x~%YXr`g)hKb z9}i+cR$5uGN+$ezCkkNzS0nDJ=NyDULOfGC|=bjhs3;*p?xRUxPC!WuFO zD~5BPHW95O+C8+B>?o{l!sDmsPiu;Mto2w>6zm-$Bb4;<*e0SxLdMVWBzQ9c&xmp5+lMse)2T zJh2R(Q${W>C(5Pp4snQb6_-n5uQ=L~c*98Ki|QGre0L3BkdN$3J#RBzd~K@0A?^WL zxF@nJnPGVk*ur)6%{lbaS@agOl=h?l_M>$Ky-3;t`WM@+LIZ9T_})f=<82f=?;mJQ z;(A0DmWc5n%0TT$Opo5wA{1r~&4BhJaWt7%H3>TKJLteM(Bd5D5g6|Qpn(Fn=?&ZW zV_2+xz!hC!qY^u63SFq@CJ?`?CTxk)s4ELwgPst*RBmIzw~^p;`Ua{N3ht&}V^#tS zqoXo(Adw5^%w@i(G|VZYdoMAw-eQJ57dq_;j#19(`5(OEotSmhjI?K&7n%;;$o4z5 zGI0fe!3;Yhg+xTs#2xiBL~faP$NBsaM9IonJR|b=u;pCWkweDmb)=AzfG99^=Vj0p z$ZOikJf>qTjOsHk!QV3$@fgp3j`|6n^&C$m5jo~JG9!c7G2>sa@h!R7X2QyIy|?%H zWwZkS%O<3aKv|;K%*kfYQMNFaOnZSnOw`$d9%m*e{W318tig@~dZSA{~}$dCsP z{EA*1r4UhTT^bRCB+AOW(*tA1BeNiBuM~)H*iEH?VE?lXit)lJ0xUxkePD@;z zmWmT!sb@bvK+ov?OPxf7ly)(DkvI!`QO}(CFAhXxg-4D;BQldhw@D5{Mia%NKSD2L zJJ75RppBqSIKuW?Xf2{V%sW~J*}njVS;4r?ORo(H`U9@X_-FK9Sj<1zBs3AO^WXqZjIxCNsZv?V&j zqG=4Yi;+i0BAWsas|U-5aYaTOtHLHpgjG@$)=4O=m7*fUCLFB@*f`XxtEeGtI zAmqPTVAX`8T~ydQ^852P}lbE#}&7VZ&a;Y~-AK0*mOC zm~ot0|Do#SM_SwDV`^PureR`k5Z7T|Gj$cqK89YkU77yY7- zAr*T=Thd=3zQ*+p%=zjIV(ttvs{b-t$GrE9MhQP@jPO1vZ|Oaa0**y*idi)y;D0bv zgOZrP*6%RZjxth|gzNYVK83RU+xKOp@W#gDX`^t@Z*g4`jxcJ& zLVs-v5ICZJ)CR=#i1YmlVrD6$KC|F2@oH)cqJzvGnS`1i9Pdm0!7LwQhumj=DUtmt zDC2Q_6l#7OeJp+_iEl{1i)WcXM52EAl4}=$TA|(X;=J19M~P<8sSq zGJ?TK#&8rOjLeZD@}p}>ddPjm_ zjz`b27pGzj*vG_Or(r~x*UR`DJy^D7=Ox(EYUG-qCHR^J_^O}ro#ZoqHFLI-(B@d`k&pqH zXT;?gb0Uh%c$1FW5xpb^Rs(aGSS77t#>(hz+o0v>Su;V2684yH$($8>%6b_C`A45h*EL9?O4uCsp7zGwY*-kkc_jn^1F5l2cQV zm}AL&X(wX!InWWzlx7AqbD5d@%y|A?vAu)V9nd3WKVL%r!Zl}vLC-O(g`Rp8N?DZB zsF@MNcvB;2m3Gi8orI^^8@(|^_@hJ>*>gXkuO|tQoL=AxXcIm2e3S5c^;Hq%fV1!I zK|E#+@t$XPwY+D#yKT}0hY zJn?hfcPO6z3FOZh$R@_%869eX7E5~(3t_ZK&ma68bBcM7)N-vb-xx(|i6^&$_G^h_ zO)&$BiZ?^MH@tz~ViximJ;u)*(iS+bXO0px?};n9&Z95hnrQb2xI)i%V1_Ai^?^9Y zyy*ehXTE^$VYJ76n88Rr+Zx;UcrxXz9#dwvwSFzvkuVFJXBn#|T2Ehv8kw2bUB$h- z;4IPi_Tns8H|W?rpU7-&=B&}TAcgDhFA`6-$+Y%IZ-a$}Pqz7;TBgQF`k1>WgEa=NLO; zG>8~(1^k-4MtR4`4r2g}_;9@?N2?Luj*)r3JAHXtC+rDopTVf9jhG2K3~h2DtqncX zgq8s#d(4&9qw%c&%QsmdV%0=X7YdJZI<|8lx4ABbD{zR{E<>B%CNof|kBI9MbEW6W z^(oX}v@2%guK%qHWE^oCxLA)1&cM@kO~~9`+BDSkv=OMAnRz@EZ%_Tn*+e_*a}yAChs22N`8iHs!JtXGday1CXy}3SP1SB8* z6{2W*6h!x1l7OEPqtjP)426%vcuH@0Cw)-UYhkOWu*kVagqc>|gjD5PN#_4^+YU!r zK15s32Vgb;vjtm21~(Ntrv-c%J(rRhjs#S57kDkn1iR!#kj^ z;=Nkn?u_~qZKa>k3vWt^J`C?itdQTQXM7R+Bko7cm0ldP^cZtu6lWE@2F9w0u@d3i zDf*Zzk941sk$T;me}>U{C{R^qbhuIbWC;LYs(lZ4qoIt`eh_MvR9w zZ8tsUu^z3>XzMGcX^pLi6~?o)#JCcTw%*qF79p*}|8h`>*D=3FUqQ$GZ$?Xq5b1Fe zX2tdd?CQBWE9kPCB+r~fFrSP0 z7AG*L^lVUO5fi21nNtGOX6A+-z1WK_ZMMCz(HO}fX247kuHaZ8i19woIj-)YUph#=_mlfH5{%v`R9W6Xq^&5W7!7QY#Q>$>0^adsV( z;Pbx1y_uam5i){E`B=O$r3S51;vkeKT!Xv{^%~f*j8JeD@)pdPt=KXK%sIo}<+?1Q zTwJZn45ND(7s^87Z^WtSX>&~|B`srM#H7gs)L@L4am@)eCnJlrBsfQUgS)7o=)KYB zqL1_)xR4ykRU%yZK@Qb39#?~}{{~0Xr{UiVV0Jn^j$PpKf6?Yj%-ujxiKOZt4c9er zW{`G>nL}TOGm3w6fb~k0Rk%NMvpDzY1M7N>IWYQspkJ!5Wam7lk3uVp9-eN|5I^q< zeZ%;kUW-IXix;|#?nJd~kHPp07f# zU@ik!xbW+k`@*j$*D$KV7%IIxYHvyo#!B?HengciKZx*9Q!|E|C)!!y$pRFvSI&m~ zAjV8YnQP!uAm_RAnwpxc;vS+rfK+*iV{dWh8Lrd$oAEPZ#MJ224~)0H1jj!EG18{v zOpLp7op(C+S#$Nh8#Uv(lw`Co82cfPN~Bd^|HCf{7U-;=$43m85-|!kNi^mSSJu+Q zU`7x#a=8AU>)ko}v;f&>v>sXWxSk2bo~E_Po}^Dw2fx+(ROcJ^I7f*&WLyc)UM_~` zarG{1W|-1eV%8J)8CTM6CS7_HMWzI2uk&m8j~P15Sa3sH=L*Y+@|r!7iC!i4%ba{- z?CfF2q?xZz1elV9GUW=6u%?wnpPP0PBchCnGCs+*Ap213d;Gfy$073%<2*feW~Z;Ns)4LYX}bQj-c0N!aB_KA;=hVEe=&R6(iY9IFAeDoW$Y`7wXBeMY-nIp{{ zZe}Mi7DH}fB#7D3oJ+J57!M_nCW0tAD}kRWMe9RO>E3D^$gI}TEL?f3M=}S#x4xL` zs3UG%6-Y0KImf!6!Yq1b=B|c@pjEXMdV*FJb%>5^QKv9M$G<7Scpd*b%{j#G^uJGY z32{wk2Vcc8M)XdCj*4q{i@O{XanOs1_wlc@@UOD)FS}$RGg*&{GJly7QRW=q#y$94 z{?!25YDChQ39}z>upQcpI2~hri}3EW5;x)8QT*mSp2_$jt-u{Pausi)=PxtLcmr>D z8oF^4enDNwEE(EwzxN&qT0UbWKQYNHmugA~Qi3 z3;jubIYD@C^w)?DPsMu@!JLZTAA!4kj-q?u)cJ#jF6BLlj_Q9ch`NXhs^}HRE6x*t$4=A zXq!fg`(z0@bsN%$IRJ-Y{cMNypsXNvM@g|l5WR}sxPm!dv_C1UE(m$X*dcSe7@6S8 z0p`x^Lt#ciGOnQHdxG;XaTRsYbD>3;yRGL6Fowp+A7gNgnCg*IW>iK(TS)j~YC2|s z(KqFHdoVj_fn~jywb|l~Zi6w_q2~io-ZS=33xn1=ksJ03@eXDeawSFs^hyn!O@uaQ zMkS-V%rz~IR%OgK{@sTb(CS<-&{-fYgTXcA3gS@Yg$bBbwAJX(5IJC87xS@5%g|zB zKGv_`TdrmyLclC7+FC?|`Kwjn%Jty$P2h6fVx!l?zwU7e^XY{6TO7PV~>rA&Q$z{Yrbi|2B2qQM+F#c^3dR{*Z8;=MGbJV6nHgJ{kB(#~w z#`VUuLWxuVw>FvCZM06u;LG{9GzNgPiT8a1$-{LK%rvI0MLUXC7XMBb|Aq~7Q;GC& z6-P1b(bgfpQxe43IY*1g50M}qWzGieAC6=+xPWn9>RHCOx$Z?@2glqU%FnIfN)oeJ z_%{`ff{%%5a`g-KEhQ`Qhh+FbyFeR+3|Oo48mj21nuuW-#bxN9$g3w(xs>Q-U^%sQQg zZ=sf7gRkVbo)Eg_0cIOx>k;TzT2L&so2xjibh+s z5=OQ>uBMeny~G(9izEE2NX!`Q2x&vfLu<9A@cS8cqaWP{^S2lZ|2C2nQe!1##x0Bw zZO~n~5Ay|hM~B3}59HG9csi{`u4}4=Iko_NXqIMT{1iwZ{@slA(48rGD>t%;BBfjS z&RNj8lQBMx(VrtRx_+%Hc(9tZ30|&AdJF$<0BnF(_~l!T$ixTW^W~U1i?J;!?M3ew z#V8)ss8e6z8IvH18i6ZM!&Ydexv)M^(*DL;!3yd;wGV8qJdE-;=<$)ba*vt-n_)Gu z`}O!M4C#3C81zX2zM~J;i1!4-mIuCTAZ2PFVtIFKr-r zaXxw|MH?rb#c0itepe?emDP*d32BXXOC6{bS08JiWBoY(){3%PJuMY61WTca>$g#k zYR{$0hV6zlt%TZ19WJdf43<}E*VT7{3F;|%qQN20R~H9nseR<0Mj*Uc5n5cmB!3G| zZ!hgvugUeWg0!w$U!7tMK&n~Q5Ur%VQhFkf#oH~B9NJ8^iQ#}e9jndb&^LYLPo%xd zVQr0J5$5@8jNWBrP1Tg)XZSm4zZt(Uw342yr33o|ADGXW7R#TiA-)3Tlrhyb+i=>y z+E-p`ZVWS*Gt5_hRgmFtZfx8meWNY{=QqV#{da*~hB2Il$VZqfSJb*%PZB!-Wu8~!!Xk*L$)$c$@7mhJ~Vz~C?)^kw;P_Tm*wy5kFm0}igHy-x0=+^ z23qLD4L+^4DMH&V)i92d9nu&?w91=4v{X0Pd<1Nj1YCq8*iwnLn~*8j=-{|E@gF zcv^Ls&r6#Pp;Eg*q+zO3({$VVOl_*TrJ_vcEWYZWYm81tqr*yu=plgDrtX6*DYn`v1%3d1jc58G1NHDxZN}c zUz(ydH2kJkkVlv*82(UOgHwv(iJjEnv_r}t>K0Q+Pn0sl+{;kYIM?4$EvAkz{9~$R zm@ZZEO~+U!tACi)z?8sC?W#G^G*@YFYN(|c>IM{lN$HT=W8Eqj^$%1A`+rlPSz6gD zE4O^}mEv-iX}@8hx3c%Bf4N*iEvH>JchQQwB(1t3+i$gwHJ3Hi_bmvVl;fqN_J*3* zGf3Lu{ZkoZYKbp-tHf!+-lw+V(yYMO^4-F@wo1lde0Pms7tU5E%U?@Q!}m%b&wkI| z;Qh8bS{vW^f@|9L@U2!gf2GS{8e^|(pCc9b_R6nl`6cvuV19nQahsNF>tgDcSKB|= zUpGV!Tkm(e9%g>77PECSKa?kCr}+kkRFKOVF1iOgTiP@28>|NlZf0E#>TCPcGRssW zdqma{>z6UBY%P_nf~$qSr4x?1n%C}pvYx6KgllJjSKy4!c#+K1=nr{^?=wJ^z;Z|`gy=%7 zxUrRIl5vmaTji~*U7)-9t{gBlbVewT)Ut9LLxT3NQaq4m2`E1(elz|ihkD<+2Y+2+ zzU8iEaiFsIf)}xI%-}}0Y$e1$)0?braV%9PxsyzDA)gj1gA7y658N01ZM_wZSELnk z9ot@CoBR!GWm9AM3ppWB7t-q|HP-y0{Ij>dwpltP&y}{g%c;qRVd?}mBG3^$d)+qD z`@8p{yg?aZ@L6LDFMB&%N1Mtk!IGkEbe9QwYl#R{aFzD-vU=@z<$B%@{@a0LYFHpy zULUg2muu{(=a8~+YRGb*`3|eoFTRHk8nIKjPUvl3yiCzajq9qzWkN4z|>RS=B;estITvq8Gkdr zmgXYIbD8^hV}|EP)BcENfpWgdsaxgIMc27vWZCntXHU>eZIvlaxse_p{2*v!_Jgq3 zo}cox;LC+p|F`Cv(plpY<2h-syPapKHA1WJs$iWbkM@0S2zQ6cdu=ld4`^l0Rn_z6 zvV{rCNzXpx*Gyn0mXuOyjV-Y>N`a;U#e zAUR}8-b%+gIg_`*{o=w4WclH$qEIc{3u7^Xh7eUW+1_t@A_ zOAPK8xa%$JdS!6LMi*Yo_SjR^8pi2zhIg^GuCI}?n6bWX6&vzJ$$F8z=DCn12-T6^O>mnQd z7u{2HmjsFwt06V=-wd>M?{aLmPd8TcEXwI`{V4dEx+HK!?PxYhL*1Q&&johm^eMJF zP_po_vd{mot)yipQc*LN?&=SbTY_I^F3GPO^s6~a`Q5(9H>5BiHwro-+a!-XT^i&s z>b81fg3d%gcTe#w&Z(y!j6LqC6_}A3<1PrQT{Oh(NR3lE8~-$Hi5TMe@a0Hns^wSn zndq#*5!bi=Qt}Z~Z^KgMmOH~JLfphJ7>e( z>Jc4cTc}&|zt630eGwjRco697neSeyT0&Nb!us$`3S13bF}Acv880~-7mhN^Aql2+ zzB{J6${W)I>ns1_kYxp@+*i$;r8X9q_fxH10Dp_YR>^)C&lwt$V$5@GlxnEuwQn4M zx<|V=SVu}@t+jj;Tm^;+=FV!Y@{fIsZ>FnTSUvBLg}()FHzg^TvmV*^*@^`+nCB&jW*9g*K)Z#w9OFLF5fb1zTD8;SZZk9 zul6;$)I8@GN)5|W^I_{HPhaJ}(#TcTK0c~qPJDV=*k0QJe?Qfuwe%lV8fk|O*QI5K zPu-=pJ&rwr9p;h#jed)9pz_qNsymgJ-bANQJ@1%ko8>mj)4UgRmxl*JZaQ7c70>t5 zhM<1tgYL1;0h%hcwS8%B;fe9BP$koEYBR-YgYDv+WT>ppm6ppFwLMalyC!&bv&C(3 z<+qZSntpb74cewI&T$zBTirvsDXk_-b4`=AeXhOU&6Z~J z$CiK0JM--(buY>OpP|10vaxIE&-Sx9<+2q+ z=OUHOM!!eSF9;}$95=mb?!UY_rXQs)N+(CR{Akz7kmQhNfw0`(zCNL2VQ-%`4DeU5 zPVimN2 zaJT8XXQ;AG9pZIq_ib%m2`0u-y;5(FOG&lNj{T(YN%p-!njy(H%F|LBDA#i*8!DJL zY8kfI&QHC0=6~g5_Q%?wKn3fH+|2es`9Y6Wo^!sUe<_{>mt?HEfw`^V~BYQ_|c$)eW*s9%$O+E$}ZggoMruePUi? zNzQ+p=P`Y3TOCk*1N@f!T0zT;Dz0jUN0kHWlkjCBi_%88&zpUQN>N3vuQD3Af47&H z4=BO5wSf-`TiD&fWqoQ!w!dM_#Gt}JiZRVFA-BH0U&MN+(U+*4R3haQj%WT7zQy+W zhIpSLI?uHoPqwMEbk<k-b&~T9H!`*t>_{}rFHDc4$G-e5 zT`e-S2>b?jTh~EXrd5qS%SYoGbr_I03~Yi5L7P0iaS-3*;+=oh$Zzu;F z)8f#scB8vWUL|vl;K%S0F20+hwJO)&wZ;F^an;@3ay}$7zpi(J)HBBu)z%-Ge=>Yp z{zC6*>o`N(Fr#C0zO3YWmz#%$-F>q<^j>&G>L{~fsFbgUueG(+x){H4=i8h6UOLYi z>sk94E2@utW&G2mo*{O}Y;CP;tZP`{)8K?JWrbH9ctR@>LZ=8tZ<$&W35`IKe%hiKhlTR zg*ii9N2MCU-^WeOdH(jC?Rzy$o^G$CB;~})uJBCdl>5lL{c?k-ONAdReM0xkM)$}f z^$P9T#Y0wV(`|X$6<>45Z!WiVCs4tAJTyM*U+sbG*By(xU2B#J@)7ncN==SAQw>~ddJo`{#a^!`l4{iO+oPYdR!HeMi z%J&hsjXN`=*u5EMVG|QT_1?j5eR{StuT2Sd%A7vQo98LdNNp`rhDjAOi{aDZ=sL))` zU&(aF)2P^`LVwQun1%AW94W}EEzHUezvQh~&^@rm;3#+47n?TS^r<&a8D{g?ft$OR zx%(k2V1XkisBK{_?gKF>CXal)3qzg;WTS&Mofi=?*r}^Bl5V z3HB94xjSOvN~PGw(J^@y3~^>>#>wixCq`Viq=&@(>z%F6D{&#e=4;t@$~;SoHXX>@ z6Id7#Z`tMV95|BQzlc({ea_M++3pI_hsv*U?|kq>!Ls-p#VY5HF3ieGvc*I=g2!mS z=g1{5Ua82PSDQSSA_@vE*2DG!%P+1$zN6YKcm1L#%qwzL-y2gIQ|G`5=?Ci~OQdg+ zHae`0ud?N_t60I6V(|_|+m?MVxVVWzyOz$rD4ek>4MPAcK5g`e2H*6I=pJhknrbeua+LyZw-_|quMBtN% z+`I)D`yzV>!ZL%xx+|%Lwd4!7L~ClET<|8OLGc?(xu;*4E0$TASK>)2IVNmJP^5Kl z;f{iX(!@k_Y?ZfLAN`r#uX0|3HT@G`r_8$E>E$m6<-FdUTRU`};ict8aHF*0?`9Qg zRW#07_1!w;8cV>h~M2xjBwJ&@%DsPnBx?H5?`_#h1 zD(-K~N~04cHd4IVce9`Eij_miSRC|x1Cc1vy9t9YuR0Cj_e-(i?Csb%WJhC7JQZv`Nr2zrpfYsUhV@&vKql&&c~ME+>9e zcK^b6>8|`Su@58Gx+i5WEIbr_$^3(5gRx&qmiD71NvrMnDBGC7uc*1m=6s9mV8+lA z^NU93U34a;8zP!Gp7;-1pX5$;N0x2tpOBqszMr-~qEl#%w6khPpp$PzQE%9_w2xfF zLwA@$O_Nk##%}B6uwJjLD_4WZ8uJV5yR1RW>=*p@-2WNVgR9vuIQC`KeEEyj7*?Ph z3z_Cyj`{aX(2$%{^5girSrK`Mqyp=PusadoW#73zHhfV+AS9yT)f=;XJ!n%%x-r6C zkZqH@1S{qnQnlRg4OPR(8~YS2%*wTX81vY0OMaD8y*^m{6uM3Z>2~xkDli{yz357%ab)${`XW4hmgO!_7CqqJD zLt&31&BN@jaoNTE`$MGAhSnwp<#S5;M_c}}ZPkuhzR#SU^H-6nvCGtA?`}vHZNo#~ z`2Kx|1)ed5QL{7Vd6KjP#&^Y+xl__cDSt~p774Tb;C-f5wnP*p8&1buEtryjTiRre zi5?utNlVV{;M|_DA+A<#-^{|?b8;QyIzw{Ai@a-@$JMZqdPQb97y4`amfQYksplUM z{DCh)`pkMDKgzt$@{RvOXf5@N!cq#-$1{iEb;Q<(Rt@+$O_bTDwmwAA)aI$rRtdA4@VQ$&8~{mT$y z=&neH)t+Rfv8kf#)&b4mCxqcD1z}lRaHgtbOFQW}E+rE3Poq^fRmq#48G3sR!Z7)>V?E zV6~P1h~%@5x7Ja56jpGKGJGj_^lvk4@;~)_XjyI_X*;BJ@Tb~7u+DYVvo3JYa;u(` z>d(QKLT9_D=d4lQ8p_!^n`Zi!dMbJP`SvQQYGZA3;h8`~vsoHxtSqI;e*m++Bd1F5 z%t30={GHBPz6>cS_=L$&*uAik@pRZw>qTQx?@-UjUgYN5?*$fnn*k%rF~!@uTOO-@ zr5Ey9(-h+^ztjJ~H!H9n$eK$juaxmV^It^thNIdVx`qUoA)&6wvuA)m0G zRPOrrd&(+Jt&wsmHQ87`FhV+J`36|(A?Yv0sup@j`3&+bQ){J{7>@&sx2fHfpMOnEE@AM6TAYCa=_zq@Kz{L7UQ+D7%5vQrz6^}}z~4r((g!Mwt7((qE-rW`k1GBh_fHKiE(NU3rS z*nUYyvtf^+h9${7-E`PcS;A@!B^iIn<7!}r8Y{o`O$#I&GvrNLarLtHyPBlD1`6(0 zZwG!&B|=4xp}uaAbq8sQ(~2A`0Kjc13BJO zNoel6VW8SL3qFtu=EVU9D!Bl$|W%emd|yRIZxzGs0y z+PlHq%D*IVIB0ZZBulhkQz?$tZ49iSIOwG+}f?7EqSmy`r2)=0F zXCLMm9I`YdE%ZoOAo$0Sj-i)A--Nad{Veo)Xna^gk)R@$u$Zv;8`IXgENA@3W|5kwT-Y= zLYmSI+XHJ+OE+_Wc@5$?-4MOHr926Y@U`^z@OE)GE}WZhb}n@eaIVe2lUqNxe|C+G z<7q!;OiSCF-a37BTHANi-u(MIiMrP8Ly7K_TcKQt6yFAp8xQ|$#ce2(y@gnqEGZZwfVx=kFyKn z)L%na)y`}4RqMMw>wLDYm;K`feM7p%|L=OE0aaI&nO^$Ovd+qzsyDA?X!y2C)u!_r z+^n2X`hKw~kZI*SSJQUgoqlcV`K*&~&fGcw^72P_gI<))Eb40RAB>grHKp&(RqY;Y zgP@k7+rp~EOfB8C>iS0iYj5iD$pFunM@P&V`OlZv2lzhT*YVeuJsQlbGP`7YOl*@wkWM=TBID_HFQD9 z3Gia2kn-U{k>|pfgiQ`>7JfSNXmnES)Hrk8ftce(^CP+y>4a3e(WV}T+S)Mx*Pf+? zXL9RiIWsac)@6Q?H9LDnwkan&$CT4MD>ySFZAEHaTDy0jzASh;;mM4YvbUGqx^`#U zom+Q5PDy$&;r@)2lsk)V$k&QqGM?{z?yEDOpWJ(*{Sn8(#Qp2{&p&Wu|2GHs9%^@} z-SN>p3h|Y)%l}fcdR$@A!Qq!e zjyPh1zOv3RO){)Byf)6YeTDx*pj^VxvZpHzs~leG`|=A*^+}izS3T}&%(}>iVI70p z2F2MY+wTM|3H~xHy=cw2WyQZs=o=dsT_ti=#DU1fs7{fegm()4($USj#274BSH}eA z1x^OO^e^|?-17_4bJyg|%h{h>kaw@(YC&jz)ttlWGvA(lwe{t~=eB2W9vdFtd{pxO z&o_cE|9a`srDK=MUmksB$K|o-PoA5R+$niX^7->8&e<;@V(I$yRJ)H10}rH@iR{jvWyeI|d@x7Fu${w;kqD#frRzkOD8 z?u)|Vfe5)aqGqN2iT+c*euec5hU6W{`X_bCvl@5CpWk#M<;3{&*RCs%$ESM>KQkOD za1T<+ngDeLZNx9i>MbbrvDIkyhpY;mp3<-GHMUl??0&SmeVk(UNtnswHgeDQRx z6TOdpboAGwn-9)D@bFZx=TZL9ur+0F*11^!Zlk<5(LGc8?Cx3XqaJO3s4=wEpT+(% z8w)mPn{t22o|!$>86U`Vw{jiSiW(=XtE8%?^-^E|XvEQGznFY`=k4@+tM7ZCHOV-U z<8=NYXN9edx0Ibz-O}i{CSzOdYt^dR^}5c=$t604|7D$TD61Jf-n^`wOBu&flQJIV z{o>stkFcE$zEC7N@}KD0#fHW%DPAgZV4|mFpw!h;3lrmu{~bRpep^CN;tcK{YO%+0zW5sP=VHGp_8_KLv4Ut@lqI@RRBlA;aAVkZTfAX`|4m{4 zg5111`T4oGveMGi(yVFE((h&eTrkO9!~c!a4tWD()nAozu=S$c<(+vswv79)Cp;VX zuw%;acSG;pxcB=0$&@P(zQg_sNQ7mN9zAaMc-rGFDYiSet{JavxbR7Gxl>1vH#|CL z{|9@McNg8!a{J7q%Wg$xuQQj8uT{}rwSKL?Kd8{g(VuEein?U5CO-)SEg)KCBDU8%SA-!)I2koNJr zuy-4?W_v!jd>(bD^siNJSKm-$RgDAHzOS^tT**>h;$B1?bSyG`raWqwW9fir-t}MDp!VvJV@dMinn}x^n+oN|Uket9E?XW^1$AHOH3e7`4IP zU+wGsKKDvaV$Rv@Qh8Mh7I_Tv7{|h*1L9AXIFix*SDKK4oj(Bo%SW@h=Qu8Xbs8*)-%v%1c^(w6>Gd^KX^r(nKAu-k)#$Lw#hW^Hv z)(&BZqSE4Dm1Z{%Kre`p~?? z{KkC9oNS(NnP|OiUGaaC&H}uNrU}4tcXzMQQrz7gio5#`cJ?BjJ;jGfk(HZsW?sWN|d&xTz z8h%at@$Nf2YHY;xFDE|?eTRHo@apDA!%uErNnNg`2xDR#69;#=-s{%@^uU<|hxKj> zHYz6xl(fAlynCvn*8H@s)F`urSuVJ=I5{JfpCq}fyr4*x%oYF3Z>09YB_Pf@mKbI? z$+aUYC8Zb3r`8FY&Y3SGh7cDpGK73hOz@(vQ+oZ^=SB~2Cr0oIO`JG})17{u_#Hz< z8DRdS1{sc3(r$4+N_P4YJ5oEBbaI9C4*u=8L@`ACiT#|k0i$V8bw)y8W~*l?awF~y zcd%%t+^Wj=dl0ZEphBZks--6dT+S(0XU0fsGx;`Y5U~eg2>vAQUtB0oOwbZk zgps%)%oOAy?+y25XMp2^RSRk4$G|cNW5bL380};2Hm#{Hw9Z}oPwoEd?8<`5$CddN zhKhsbOG+4pkMsKHW#x3rS&`i*OOYO&ib)=pxc+zFc*-wXTzK@O$e-cZ4_DufdcWga z*92qsmWuh!YFjoYnKR$-S(gJndA-tlF6h$Mp@W7iF)~k*zF}T^@y==1TFVRb3ftE9 z2ZYg#p8}bbEL$cUD|bl)gj&`Snh1Z#d)t1|5YaHahERRB=DD`ANn#Fn^+eT?CA@Z7 zuRvtisGdD~6?QM{va937fT`+SSwBeI)|DPc9!We+T1?Y3fAIIou4`0*dxP*De+6v~ zuxf@Y^`dL+Sjs1CX8SF7h+E}8;!g20P&n*sf}E;jjNk+cc8V?1Fxd%Ny=0RpO3=b@ z<@Mlg;f`bDnQLh8C{rmt$>}69={JD{J=W2fLgW+gKzE|^s{KD}s>#`Uv-zSY2rDTpOw4m<=VaAq#bpXJM1LNqUQ5nN zc=&r_0ww-@+~JrR5h)*o->>>OG~(;8iy3jHyX!|8i`qT3V9C{h$(?h$WQLRkJr3xu z{3Ytm-A(I;TY{i?G|sD5q&>rpLw>*$76DKbwM>Bs{?8nRRb$#RCKA>QgOVpxKdNKp#p{jicy8n z@&@Ky&C+MAOdF9>np~8S`m6fKsn~TfpQFxyTk-k+`vYOOzTN-XCu3j9l=@$7o!ghv zxH4lvM#$vOPeR7{ol#FzvPF|wC&-B?g8P;&*j8wtZhi0QgX~6lNsZww;cpSOi04bk zNg4&~S=T9>uxxLmeW|Icd8)2&{W0w}-ATR5dINg<7SdYoZ|S0dkDZQ#9O<3IL-uxz z3|!-XNQ0GM6yD|bVn?!{@y?2l%jC-Oeu5xR$Er@9x~vE(34E&=CGW-GL0f_Ehpcf% zTHjeW*bdv)IOy*Ao^c2qb|y(g6Emt=XSq>aBlj&w#^$hYvUajNvR5+K(tA+{PzF)X zQnpdD$x}&5coDWQ`UENy`MrIeyTo2#9&TLM+P`^rLwfD1YERXssza6baz@$JlJ_OW zC594SsiI_B@tMMw{Em5DbLVDX%6$B1LF%+5!S4=n>X?)WZuo%@N8VGz27K-wx#YJZ zdsg|w#&pLBLQft;E$O80zN?#|(~Mw@KVBX!y26oBPN7eG_4aPo_m)&AP!X_l%5m0D zP9%SbI9w7e2^HPo3F!af$%q(7qG4V0^@f#oel>IIDjLvwuKA-2j~PL|%NMD7b$A@o zvCH!=|8mzF}h>uG(DDuj1dbgC+Rl z;Nty7rG={sI~Bex_+Fqa{F#3^`%%W%KSt<9ef@PHZgjLDLLH9$Q1wdu@}Cd5Z@Reo znYpFi>Iaz%Fsr$Pl$jwPdnNQ%b}bEg-SNC;nmk@Si<3gUfng!Ao)Sl#Yakj$xJoVH z4i^j(CQC-j-%H67oai)LK-q&Xb61;tH|rXvXs1^vRT=9N8;&%;G7_D4F&x@8!6Ma! zfcy?GgR498Lk@Nt)%jVc){fgdZ15YWxuUu1U)&+DKlx7BG7SY31%O*EB+?# z3w9rti=BZ%VsKa-t^hX?7lP@6-0ywjX1ea#<1N|7^{w}s`ZqkRX{mTywxZ-=(dmL$ zxx5@@)|L!*#*05SX;c2ZOxvBZAaT|2nx9EOlA}jQMt;fuuuh4%P|HYjw(*Z)OIzR8E6q{5liILqX~l$6Q&C_cBJW9dL&mm0VX2MD z6-l2G3ldiR9{Ow3kI&JXD9_iEpO=S4y!{>a_~YuxiSbeCeTzQT*o>DEY2*ODR-w~` zX?FV^@waMDsTL_$%IgIixFeZ3`gh88>P+T7ZmrNF{UZG(rOQt$mdS6*RMMk-EhB|A z1s{v_y5~6QwoxV+a59cD|7*oLJ9(YxpM*m+Ckw;Z3-d%}k{Efg>b2%hfFh_cIJTp@ z^Xsl3yUTka`!@Dn-|u=~WbaYk?VYED6a`*YpOg;cuVw_2;&4v%4pcMpJ%ZG3bdPc0 zbD#6bVa0)u*%IxQd6k|v*HYJJXRF<7>uUYi{M@wDh%!8EE!7w4_rtu$sm;xe8M@)R zehpS_uy#ltuhvq%yXv2cYo(uycNTptSf3Y?BhDgbd`Q!zHYaP71}DCXpBI-FeIzpO z>&5WvUq*g&M^*hwOWZ_4I!j1&p1n0USF|EL<;kNFOLVXvPLS>=4i~wo^%$>D}78U+z)e9p7yZ zwCX=PJn);Wye65?Kf>xmUq`VJGYAjx6L21E3eJRc;9lWca18u@+yI;rGaOAvO+~Es zu5gQ7zZ{$G|Jc@B7g%PSZ=2p3|1orGo7H+p|Dk1F^OdGojjg&6-I0bX_3pa;wc?tv zs`3g#`QlP{byWgP>!HkriG%|A;dp{2+ zh~&TJYX!AJy#OifBoc|n2s;XUh~A2~NiWJbD?`-b8lj(A6R(-8X;tsg4E0y|U(r0# zO!xN-nC?&apW%PX@3UsIMxZgOU#bks=MZ~`$)Cy0(yP+Rk~`vYk~WD}@<9?Qc`xlR z>nZCYTPn4QhltBX^F`kU6y9X^69$Z=ks|RFtO+^B`@|Vw|7fW+?r5Xx4>SpNKee^B zzp7tUy2^9QIAuLbV~YP2vJ1ZE1?OgGPsz&3IFr8N&oUUD-J6U_8lJE!e#g(}KW4{% zi@6YEjNK7GD#e_>FsDP&^m0lSvu3z9s6o{5v|*2KNn;=A{e?ArtlzJlQlHqsY0B2m zGP_h#-Oy`{zc9g7@Cn=sOvK5ue$EpbBQ{`0EXXQ7=HpM!nLsg_1<8KP+ z5cE&*_F(_upFubn(fSz}6f`189*7S3FMt`C@4wb>v)>rMR`nRQNkvunS3OgVP^8PQ zOEltJ!X*An?ooCO>lNz_Gmar)+@;6TchX1GU(#vxqqHWP4B9*`WjJ{NNkG_&eTD9Z z>WVn%xduIr$+kllw`qg1ux&=`mlkC6&BllZLVbSS@mf>0zVdnbg3^|vNd*nL1=)F- z8`Jfv-;?(M#$?3LE3t;}KO%oeEQ^}^Bm1{JWnlWjoVdb|CH(SXRrs3Rn&ow4wd{t) zy4|{?jj6h9U0TzPmP2hv%*D1I9v)^t$v{W3!+2jsHzl8Bd5T?%q4IcHvn&DpcbR0n zWVfhF@K$hN_&}T?O_i0%jmiP){^}p9i>fr`9feNbB0nxaBu|0Sl&P>kMX^$zCHo>@ zu3#yKDtD+3sh%t6DKg}rWO~UX@f=Z-03(_-)6O(GEX$EG%RUF=pQvdZ0xLC&@jG!e_ct<*6KHv zQRM+;<4O(|#^t@w>5@Gm6OsNa^-{7VaboTZQXnn-#rP zkx<)irC8~woUI(EP|Jz3V95_rlW3=;Om&%j>{)SYSC{L`A|DVX$a~h%Oo=d?2Xf z59eEW(Y!^xFWlvvuIx}&3&TN+qVP#3Tr9c(QSK4B9@$S?i_KKiY{M)VYffuQ*Jal4 z)K052)_AKN6+O$*rDuvp7ktT$&7x;qOk0}rG?DQ8$WKe`g=kmglW$dDtY7v=4*TJb zUz{XOTc344uTz1sP+U?~iY;fB@2Egmf2$r+v!Uigjk7Mj!QM=Ud5z1R35d;v`*byj z#(yimE&orsL~T;{REMhWDeX#w^1EV~f++7UohFHw?2(^WMX58?votw=-~B)MZPQrQ z_f;Xve1%mpN+Ffsmx<-C1i@9mTQ>G*?j=vpO7IWx(P~_OJ z@^I%Tb9i&q+PISV$B7Tp8nfzi1bNW~uZm6;e=J^G(xv=k#ezy^)%!|o)zrFkx?TFS zFjM!o=M+{;dB8fx9VA>K>!$pwI;|O}iB|Pj4O4}y?8+9!F-0us|2b)(;*c8Y-{2SP zhYC;zHV3r(HEYgj+EpmkV%1R9MCCzPPB;*@xAHJqqBK)_R3?H*YKpQ}@kG%t5Fn)2izZ){v{eReUJxT^d>RJU=`qDYNX)>y$r<;@^^= ztk|yK`$o{dbpIUlxgdh~gBJfZ;X=xc%!r)2+?IT0QMcldVsFvN(z)d?%4b*PR~)H) zQH|85H72!U&DHkH2qJMVeJ;C>S1P_EAEz{`D%5vW*Of`ifvPHHpn@Zp%T`Iwi(Qh= z3by)yS`B(q<2T2DoS#&asGg%*tH3GtDo!dAK@*trV7XqZk_g0)L@6Q-l-wpwk@l5t zmY$O;qyr@j#Lc3SqDR8#!f@d&;cC5SPSB-;#~a&B^mGEGs%*ysLO$X>obyik}rD zD?e54td6KX-0-PoqA|~!;Qo!7Lb=8G#$GEROFPT+6gN~;RH+IRj6f$T1&SlGwbBak z0*A!m(m43vSk-J*vzq4@r#YgoRnb)r`BIrp%96d68DukMouw|xX30uXnjl%=4}K#| zbXi;}9w}ZRju97&YK2{eWIkVD;~>k+trhK-y;f{j4TqQ8ff5ksU zS)wTMa!E(YWyv(jO|eWIE?f@d_@DUu1Xl!8`RjR~d4c?$JQb%mE1bEX*@JnMaT?m= z-()}XGtzNF2=)t7=EMb

MRkt|Puv)a1VJj{$UKQ;8!M>O4Re5(6ZFRdd~XH;aCKQCKba;ora z-q0LJ=8^P1sZSG?@q2zOkFJZ1h!`4qG`j!Ky}z#~Hl=Vfw`5Pw#^r?NAwU{uiZ!Jx z%KKKlsa#*xw|ZG^aKq3Rnh|g93A0wp2@B{kEDPtQ&?-U8{N&XNqq0V+P*tg1%A*ja zJyZnCr%T7k&M6OQR`_W(+x#s4#(-4;@BRMOELOi!w#u)|#z|Ww2{InE)ltyOw#i$e zl^rcHNE9-WtXR5N`d#u{JWQ+vjD%RQUa&+sPdHgn$p1(1O284k;T~d#ure55=@i;d zN*!q_@c^MGel6w;g6yet7_1x2T};(Rhhb;y#pY9up}M{G*K5sH7b~`wT`aj!v>hTe zLUv@v+COVk9wq$z*(VnN{ZYj4Z*!yManATJ2@{fbq**d=Wv$L8=Hl~f3u(n$OA^3m z^sj8GqEViS|BT9hOd=!+OQ(!CxZYElrS}f%Y*|xl6TQovy~I&MGD; zs0xbwtL&a4Np1A|<#*8U7>xYC3ZMo&@{?)qfxpEmKFY#nT@;U1E7d7Vl|l?YRV=Rn z>v$=hBikV3%F-ns@elEQ(F`G8I7n!RElPj`-?2zoEZELF#I9x1nHYwK_MGe?7C{Ql z8@R{l-|fBKy&bEqSn~&Co*}Yrn|^oGdR;(+yxv%Qvg%@aMCs;|z@pLl^KJ*2F};7$5_%^Frlh4$&N`4Cl-nzBbHSyeb;YHnC(9KTepTkG zku~SFvl>aQIi~+?^)3#&frMgS=6vOi6b+T0ko8nJ6&U3=WxDE!`noDkAyX_=$Q765 z-xQ(hO223R82=;wn;=pb1y=c!{OIbHN|}Nu7s>q;5VNZ0sOBkeDNOQBa+a*4G*LQK zRwn%>O_IzMmx?|KD+Qkgiv?>1-2`J`v~)UuBmXOzW+n2rB}yN@Mc z$jB?6aZa^ujG1aoZ#&!ir)5af)`n?XT-}bE1yu*i14_xoj)L&K@@!&e-#_IkYm@r? z9uPM?`gp|QFR|hDh{zbp&wqbSgor)!PuC1~*2U~Oxg+uw1-pu@C3{Lgmk+KSQ^l(7 zRUguv(nd5twuO3LVne7Gm@_yf{Jr8S((AG$xm7+&u}x`G*;PTxX!#v^wtT((rTm^! zrTOTm@;j+H;}`A!PeA{GhkifQEY(NFefb~RJ~>w@RNYi%DwiwJ^8K%T`N8x31Z*T{4M|0M(_JV)tMwvxEL?q%zV+9yI`WNDy z=Z|B7^`MDlIMzB*zrT5LV_p5mx-T_@tHUZsl>3z&Dpcp2a}H;HOixWkCQnS*`70^* zV$`FrgTf=iLn39dhPYS1ekHh5`u$m&UY&6>>vqog+=$%Sg{%@p>6o`e3pQl3_os8N3N)$bIK7(i$mOc0;O@ z^p#8yj~D$EP7#(0o(R?nmI_FMS^P5)W6k28W+yR@)9jQ(pI{HZukG&%Ci z*Ub^jVmxsNf35qSki6#4i1a=gjI3SRp}Fk5CHdB(OC{B%*ULXubgtIdzHh+jNye#` zOh*QyiBLr?V-Pq8_`^hLVyk3~bewdLY@k8~{n)wk9O}(lmb<<%An$@pQ@*60yUddR}|;uLiu7DQno-gMV2KEkzSHq6Mq-+ zM7M=Agh=5r!F0hEfk05jd&&LC*}*Pm1~dMpy{06RMi7(mGTeL&4#jQ%;3C+U!fa%t zp{(_2%dDn7y2JJ7>!NBJs@xS>W&2AUg|z(8oadR`^hv4u#BK2pe;oRr5P^wg#q9XG zK0YL2Tyjxbd3vYJq^w)nlHB`wqYFI6simjOdRK&2yr~*l_oF_oX>DtUvC3xjXt0yX z0rXX@BV0ehcF}&A?ZB6emI~$XpkE&#pCmPk^ThR%MY2lyBqdViRGt7|5vX~mS);k3 z&QVQM?N*&p`)N|t!_~p+6Y6-?a7B@{vvdx~Q7w&;5F}yZsp3M>7E!oxG>n`C2+N?= z9Lw*E2ulkM~DybO*koTIOZYpns&MVoB2YG;yXe zvnmsneKE(JM=8uGnp@Vr;$%gyDrg34Yqe8#DJ?yXb1cssciUIt9*_soB+L(-0R9x= zV(4K8i>FD9GK6xbqE7lm>=5OOS4)K8FF6W>0-+qDybSH(F69W-0@V%WRAsHIw_k!^ zUrkT7Reeb_Sv^?sMEYF1MYcwEN_JbiM6v*$8I+4jVzj7AI8Ag#G)%Nb_*Sr8u#i86 z`vl_Zw~P!Jv71ckO}b6!LYM~rU?i$%`&7?rSCxIOwS#$>aaUWges)uNeO0YrO-xl? zMc1-x#f62uf@!(qvL2=HPn(!BFtPBL_s7~F#Q4^v`zgVxoBlk^!shPCY0i0=cdKAZ z@#_+DxuWWF)#vKQn&{f6_4vlvW`9Gm`KetG^GSOVlPM?YG1cUj5xF&W4%gKnLZ=~5M ztH2u$B@D)|#A+ZH!g}ux_ZX+xj<7bFvJ8J(kF?xtTvR``c1ZPZ=vT~uF`VjxcX@+y zDl&Vg>rxLV&q}!Z>-x`uzc-|K(suqar?1YQnAa<>LtbWn&!QcrBg)@ZR8)_t?Nc{I z`$W4%H>dfq{(~VL%5U^cM(-x*$XjS1nQJ*{oIDOeqm@)J%v^2^PauBJMa2Q{KEkYZR z{SXVionbbs-j-z9ZklPB(rRcHG}dapHJ_`FS6nWG=YT~63p9DcoTyCwpZTf1l4}wi z@!#SPCpuCmrQ6fXGnKh3@>k`z<(CxxD7jcJtQ=PTz4mk6XYJJb?hWCMl$MLFM~$JD zv5pDe%jh6Tu=JX$V1~0*Tn2xkFjq7fW&<9OWJpTI?ZVCi0bkD>&X3_w0v~=6o`aqe z76{3rRM9laEor1IOdhM~tfVXP%4v!ja;dyXnhQO!I^lRxgs4?`N;pZ_8zLgP@VsCN zNZ^*>7(|HYc~`g{IBfPk=0o}!8jp%0FCzAW=8JWb3HeH47=_uzNsAAoV-jr>9UL;Qt;{z5MFpf`$7N_xs}%Wd+( z5JeG`CWwyZvffgyc(Z5C!)yFH2 zm2D}J746AioEx5fGcz##ed>qgg-NTDN>T=;kIj6Um7Y_aFE7d|3Ml!v%vCYFI-;hq zZcM`xovbmfF|CQH&u-mhIBH^BuR6YXW};-c%{xOoxy$2UCDnTND_ny7l{r^!lf5rbj1bnQ;__RG*J3cazj=GDyowyF4g}s33 zgswv_L&STRdT4HmE89_PORzjKH5h1Zb6Pev7Stc8lh-_jktbJ4e$kzRWqE>}Pd$8 zh8dWzE?%*Bd zjpQkK54rt$YThXhnd4@$SreFj=tk;g%1Kfz;W4fQ){OoaB|^T1>^K@I`)8-o-ePsY zQ;-6~lh!#cm5p%?OSN5UmDL3mb!E9FWkq!biFxeYLD}~+Q_?^F`S2$;{bJ^>tjpQI z^KuJc7vC?L1M{@!RX?hAX_Fg0=GfSKI4N8=uNQv+|CT@}+AnSc zxQ$9OLi$9KCT)n3t7H#}&Z z+!C+f+&acE#CXBv5A&b!wjK_va{{c`)*)$_cw82-1I13cL)*^uW6$E$aewfdp?2eV z$9W=NBX<|~D`zfe6Q>{d8t)K4lD|OEQ*=jsRXk4oRU9Va!Mt0J|BP1; zGoMFuCa|kngV~?h0*;lP!*;Mofb4FujjXM#8LW-WA&hbKL)1^?YGNL~9Xk***gS;{ zJcZs@o}cbC*Cu#wL9#Vjj+pmAYyVttYOZJuYZ$C8uc@p$ThX&jQZlv(T{tEGL+-kq z>Dm6-*Rv1iY|8DGzoTeu>B_R4^50c?wUFSXf!ox#Wq#|Lw#SCqrs?JfmeO9nYA- z_`sONxJf@i-$U<9A4J za*b9=U(B4zDrCvnUD?ao*V%X2gW0QCUsz@A>zsMqQ9LwXF4!S_BeII-h{B*X_ybQU zwu;UPQw2=^S#B|BEnCJSFkizejZiv?o=6L&1<_X0`oPwS7D}a4L#gEyDrFgYABhWb z?GjuNwhf(&It}Z3hP1zhH5jq(eXdTh3S*1CuWg;>t*MvsQCo$c-6ClE-mp%)xb}K= zX{D}QTK2SLcCoSWVZpKduX*S52>Ia!A;tGgN0pzh5LAz?v)5NQ9%^~jI?uQW9_y~Q z*Ezbmwzyl|eD7Syomhl;k94A%ez5$j0bD9zOF^vjHe%)`v%%)zWbEG8&y;c19lZVBHBX~NrrB>qO;UhZ+uDtN;5l)0YK z0Y-Hqso$uVs57Z|DMjQ8@?El<6hT@^swVP?a|w6xM<5@BGe(|2*}aZ!@Iz9 z!QJkf?ONs(fq##+&bQFb8O9lg@2xaFuUXu9qoGz?Rohm*rYg8Hr+iFVO3933P0^*o zb%jR@_ZICfIbFuB_*03kd0BV5VRqA}mPu_)@Z}3_x%RitQ1?1dFKGLbvM~bbCDg9eVblUjJy}VfPYNKV65WJd z1TB6#J_*Oe9mUSapuwwIAR`;(#`T7RPao|vIwPISonel<_Ea0qrm{*cDD!XQCPRDc zdA+51ZBqk0U)ibU)P1k%RsC;eWBJXpb*0BkJ{R{c-cY=*WLnwCino=1)x|Y5?JS+9 zd5HdETaMA+GQjr3o&l?zE_yzBquM(mPoSu16PgX{X!jAKi9w{}q|xM+lu9a@{)`?C z8<~M-3}yr|PcsF~DGWN}EhCGW!eX$;vRm0LoTuC;+#%723&J@fhZIVRCY>ZlP%cuZ($3J%)5!E|bS5K?QNdWwh@)51lNn*m z0jx=^1uQ4a!w%=1;m|mx?4=xk?h5W^t{-;+r;J_1Zewp^Z)MpTXX!+`fc~5Ii+YQ4 zi5yQ_PD&x(BJ9Vv;r8P`VH+_$Fw@YZP#b`T`3N%ko%bH~q`0@Z2f6EA9;ee`v3G{D z2iuN8*(aHInobyB86LK6Z`J6VnkmiVrd-{@23dV;9jUHMZHJm~RqHB8zzo6lmGM<& z)wY_sb$#m}>ozo3>Ra2)#@(=1t=ciiJ=VLkJsJ@XxvZvRHe)toMqmfyPT|iI1`)pz z?Zh#pPUL}EW=xY&D~e(F1&dgZU9^zmBzrwVcIc zA7gK34`csg-C;doJ!M^Dkyr_gY4oeK<+OCFKg=%ML>^DtL9`JB1Twx4?l86%Bg5=M zC!t!AF32Gn2~;AxC(~U6_RVyCadvgycU-q$v5{;$tVT=l5AG1zNsgf?U>(wS z+A^Ahnm{#DZ_wt^S&Xra8oHd}V$6nQ&kq>Ij15c|Gnf?wzMvP_#cAeirikgsFw)qd z>-E$g)EN{s`8!ccd`GCnN8kcM*BdcB%udL9B1X9&J6fm|Da8)OR3#agNG*~ z!>fK*(W`|N(*Bu{*pF|K4)nM~GNhzdZWIqazT1D+di>IE0^@F=<6X|#88|aJam+4Cx1m-)& zc*aCVHlq)-50lCK%-F#A%uq8oFu6=G9Z!El+f7?U`$_#q2_s)7RS_2xj}boOf5Qa1 z@z^DpaI_3&TZ}~>fL!R8A-m%p&l5MD)@SAt%s(E|KS$cRZpFG5{8$_hK?IOR>Fi4Y(Efv-oRJ>s^So#FeCGlAPR) ze4TuNvYA>!{Z4&N{X-3=eV~cxo5A|0(Yw-r(}y#3^y_p7{Xd4D@drx1mobVlkfDWH zQwtc8bS1Rp*I*syM`{iwnf!|MgE)a$Ovu64zZK{Y2RtXwYRhzv_;y8dW4SE6w-od{noY@${uLz;wWLMp7M^mh4z&; ziPjrDKr48`IUtP^nvDkQ>cJys)7F4*Y@=0xeUyQgL{M|VUwo&ul1oVl(oEtP0-Z1g ze-2lOZO3%R>_?}fs*x>_`8W;oqfhpJ^{j(+%op9g+=pGMPLcDCF>cIhECal54Q?*}HQt0@N6-`Y z6DcG=QXla6{mEa+!IX`Za7rd+FZB`aH*FYLFp4IkHBoWk1FlflQU8IMFw5_& zce3|~N9_rBhq_&^S+0{#w6oZe0JSf49JHHl+h8{2YipauYbKkAn;sg67{3C@wtw5w z)`xnuesasr=GvwXpaJ_Exs6S_TwR^6b7NFvM$>?nkXD}IuJM@pt2N9%)tTlx40ABA zfYuYxy)b&rNGu+gfeXM7#*e}u#ghqF2mz2^!bOykra{@WNe@2prjCS2^daRPk2Ppdzm?_V6Edf8Tai%!tI-2b3 z>`ePd+az0})nVyj*<~&=O)-@kryK7=>94k#TRXQN)jL~`wCJ1vg?8_C)1oG9le2M5 z)59iX)9vQemV2#XhP9?z^DgT)dl2+kc6e;wp@{v+O{in&?U+o=8Y~Al0k;SD2zMIS z1h$`s??_lpcnGUQ>IpdFNa7dbZW4mLfqb6)gj@$!;0GmrOg;;x+Yj3c@;I^#R!S4e zXz~e&5B8J(kV-&qyFqh^qypkDqJhv4#@`&cZ8$t`1z5X3<}ms^Z8O^$yYcKb$qzHNluYo3RG9r7vvmdBCM`-L5AUO?eDxp0KNRyv%pjAUg{>e zqg-QM@y_Ya1knFeFz0oxtq$6~7)uvRw0Wu7Y}#TXo9-Gr8Vd|t492z>ZIjx{TA#LV zXdT@;u+^`1YOAgFW!p4EpmDEhjHSfd&;H5L*OlOAcvrVaAYLQCqB7AhFpIHoV0QLa zD7_XZhL!Dm@t^Tl{C2_@0+!f^xQ3`F_9ne2jUsO*UnRdITgkIvol7@LcS8n-NAm{ zhPK_b4!8cX%&}lC&&&hO38on)v+=%hurb4M$*>!u0exFmTS*%N_RbjO#%kktQ@**I z^{A~Lw12rP->rn+%pAm0F`wG@**XC{;8Dv+i{1RhyxQEs+-Q1W>Sb~mql|Nn zMB^93Y(qB#-M}ymGmJOvGdK;0jgw6R$h*+lI?;B{p65WgM!45_Zh9}YFGR?ZiC_mT z^jowXvjXz~sC}JxGvv*_=#hB7xktN`T$5ZG&V^2`W4D9n zxNjeAH`(slX4oX)3y)goSb5f5%MHtXi_lVLerG-oF;Sx_!gSCy*)-ndXOe(iCYvss zYD}T#Tyv+Z~Q`avMD2ogdNp#%7~QG^kMp@fcv?u7A#*@X3k z4d6Xb5grkq65fFwJq79QA`AivRN^1vXX5$L1`Nm5V-I8L*y9)yWRUYm$DkIV>X5sT zB4jeq1L%-3eSNzC#xo{Ep74Dhtmmwo?LO|3xV}0^ISU>896cP3_KT2VLkfQJu5FKP zg>AS^U~^crtZzXsldXYPCaj{1gVDJYmfe;emhG1FmW!5`Ac2n7f330Bj<%<^f%XLZ zSVx^>zmw#8;tFs-a(DN<_w?|-@$!KvQv;dLA0pZjGm*cLgHRtpCMTc=yae+TqkwfC zdDzaleYhkX89xNS9{&zsj5p))uk0iq7L%aH_YyV}4im1!_K@&|a2X_W48EBG z@*+X*j4SxjcrETfTo5iDy9Ffi5Yr9f-vJO0%mPU)Lz*FTJPQ#IS=p5k4=(hQyzf1s z9=H27tP60tUO{xg0BNjrQlam^4%SVi+wa;p+owZ>=(RQ4e%T(|4uZe*$z`Bz2=tHr zY+Y>Kps%R13E@4^Hq^GtcFmSxYX>=8w3kCh%?A#V^QhD9-0ZTrcDozgD?M4B{@!O^ z8AzkKeG8%$F%OxF3`D&_^+i8}9zp`9KdesGK&$D%jl$o;x8S=G#t{}mxvvqP6JEpi zp6~!leHO}ol5mi)o3IVEV+5faArLgjiq8bO9l-a&H{%}Q20<*i3G2Yzzyx7p&@0fb zs3VYzB^Eg!EaMGifA@#ABG-UN;MZR6y#`rfJf26M!Jb;kLNLN@|0@lo>$mfobFs6J zljN*+#5is{jySeB7CUBu-}J@KYIrdmW_z8z(w<{avnShs+T-E9#7+R|b#cse?039% zWINg&L!7&vADjeOkMhwabZ>WOxQBYKdTKqxy{Cb#8w@p4+CB(q8a6~IG7`y0Z9`?D zB z{BNKI%=LDJHZu`4_pE0Vv>6JI!JX#5>)zmAUG24rhro z3AQljW9Jj+E!f_{XSDM_XEl6dbW&Z3&RH&wYk+HntAlH~>nV7kT34Bi;GX0@1ZPfk zPw|ZKeDpMU#L!yjd4_rW!xjkn-hKSY&p{}P;F)a^#cRC@f!F9OMq$F6Qbx8?|ttYFAw@qFFZ#)i#-7z zxrgPkyBQE&_4KUqZ1LO#X?^uX!z;pb#j_vew#qXVdR$UybxPb3pjF%58{LN?^UfQ$ z!`&0klH&1r`glXVyS-sRTW)~-_tbWNI|(Sx<=%MjSMNLT2X7q6Km;U@O+Zda0(#m= zU{gFtG$DEb8Q~G~D>4C@1NKvhOh#Tu{tIzWD6$WfX)L@CLrw$V=R2YhNkt7p?M59# zJwbg1S)4&FgLU-^R0A>&c^)|diA5Fwm*pf7n0zD?HuRIFz{OYsgjgSY=p$(5S@0Y4 zVeCMFXaWAk6W~^Kg6J|I2ovYw?stR6yaX;Z6X+h-ff`W;v^6s@ILd&RaRGP_L!eY< zDARrKzmVZ#zIUS+)KTzLXDHazIDiAkOczP zr9K3itpVvIBV&=LkW+#DuoTD;i;?4zK}aR&ixSS34CV3Bv3@|SeG+&d`G_d^>jr#J zL>9r`ebBrWz%)q)yEp^1kr=4Ck6;5ZHk{)baBe=fL(T^vaM+>7eUz91P~Vr}3_n5B`@&uLNLAf|L*ygl zErI@QJrI4U$VMowKN1IZ@&Ph{F~Dw7P@?O|O<)NxkT;;zd%()?Lv0<0lI?@<7J+O^ zK!*3=e1}1%DT6?Ou&6@<89Ok|1{S379)84nlz$EI)n3EuycfXt2?Ja6F>WH^>PQ%$ zKLwgH9`61CaLvMjQ?nN=Gz#>h3Rq-jLZZ%xaz>90Rilm$G~_=0?CgCe%>(n%|rO+9c(Y)EYpE<fzqsj zvwemCc4_yOKY8Sj3m@s-{&UX^#4_XNZ?*P!$lz%!or9`Nq;jsfu zBC6YYz}r~}ny&}$4+{E+Gr;5PK+a^K*%*LP!uL)x2TynD8=XY%`je**3Lw1huS;_=PZQZ9fK>JfrxoJkPG8L>R2en6wuj!z#|5M zw+sjG;0L<#xjg{Lf$za`v)h+}6!wBulcAjcAcq>zR4ee9qQJxWxMy?zw(LGy)Hq;? z{XmR>zQH;;>mevzcjzJf29F#9rM>_f;sMW{33qV{w6Z%KeH(~Q!+{N^MpS@@V*$}< zE!d`yw&`OSib122p=>^GCI;;E19*`saF!bI_Xpv~RUii+<@z<~)?+v>4*X|0+*=4( z^hS`Y7|x4Fe20?j;2e>_iCPU;i-wlhN5_*ud)yg5x5N7su=F`lD*@mYXZ+<|0SW;q zOVi=MOQ95E(1X)Z`#t{N*#)3lErYV;z?C{eYt;g6txpO*o*)}EU<$}n0kVGpcjE%e z)F5cF(!pXzg9MX+mc@lOSq!yf1C4y z?xWz>KY~8IfU}yRmSaFuZ$X*Tq5OMbOlCVsbw9}A9+b5jq6Iy)9dkh|is8z7cmW#; zNKLok_p8A&_k$%Qfb4WY0$U4b^>Hbsh^=7jUxBan5K3VO?HKkq=12o(mASn?)aN3Q zM-li0B*F6e*GU)y;F+%+eBI}UWD8d_*A zl-I|EPlBkY7Owgmj_&|Hs1Y!t{uA_L1883<5ZPM5mTEy7>)_YbaKDXUnX{mNpTL#k zfT>{shLN;BOz{8Uxz05M)vh5_kcvqtD+*flPI9B_FTW1a*)HXODvOa-k-CWY+*7 zd-aF<9}P83292Ex`ac`~J_z->8U7v(HoF6?_AF@5T6hnInC&pMYd(VCVK~Z1&N~E} zxeqM=0z`E_BJ)$w>sWZd_&3H#fOg?FaQl4UCxd>xfbB9^iH|<{5q|AE!*1{=8{l;d z&U6m0unqPP!S^TO96mDhbFjYm(6UBDM41aw#3!%>9}Dw2oZ%7t>J7BJK8D~0xU!E$ z=_5*R{LA)R341Hyu2=uv_Q9F2!*M5pFL(ptBMe5toxOuIMf{CTiXj%Lge&-%p2@%# z%z=2t_xCTj@*B9)eYl^?@H!18{}*(89cYA)Q@IRmCKROPqbd5hmP_EO^TB`kNN^*; zw&s9bL*bZN@Z?}N$aElROmDEnL10ls;Bz=w*#C*peL-*hz%qRdOELHX8qnfU&_4R; z!PdXt)<-q2fja&_Pd6X-wV($+Myl`A0JUuTtFtz+BUpL_`yTMkLh!~yut}c>^HI3F zfW~$K#$$hoQ+(7^AFXgG-05_u8s4Ql!rY?nd0K1!pHP4AQS71%ot z`$ypYG}PDsIo%uJ_ZwjIk>VGDpO^^S3@GzRkij^(#y}{suiUtIoyX3w*T{^ zS%3F^TvYMjJudv$*Xv@zIVk_5=_|n6*q*oJ?nVf~-L+WV-G%nnduvp0-QC^Yy;Nu^ zP$(@@oZ?P!w**2UuK!7Y-}mgZo6YVhKBWRZCILNW=6yGFkG4UNYy%Bj z1G=UPv|s^fn><)bU^PGQEO_!?!JdDHmU;pHlbM(K7I+`$L6e^a_C7N&@;1=pdtrM3 z)(x<(1)pmzcwFmXyA-zn!26|O#TSGBv}E zhw$xJ;MglT@&Giv`TRY&_AQ*d4B!12uDA%VPvN@ju$noIU%+bqwQKNt2EOwQ{L)eQ zjh(RUhu_}>>;HI>x4{4X2kvJz{MU_eujad5_4lsVz>yX3|ChpfGympt*qa0I=fX3H zfNR#m8UfzzEMVar58nP%*fXCqGcNy+XWIO1*1~UX`FqDl;YpnT>qqT?e`4m!JPdPY z<~w)*_ZAJU^9t_8?5n;7d3q00WR|FSXuTI;H-K*> z%UlCeGaGu01ne~n{6-4&bQV}j+TR{%K4t~7Z3pk{KnkouC$0nSITkd{Y|u54pjH0` z4KxPy#YE8F>p^2o0q(3jXs}`0txRgPQG6 zBxpsmJ~eB{J%888zjqc1_F*pEXMW(8wM zCpf}|u@4JaO@%N|!iRHAIKzV3BQAVe2%qBozpFWLo&#eE5_|&(t`x#mOnBw|y}}+o z#f4)W_>?_-iVxr50IcY4z=j?S<8dpH!VxgaSY#RvaySgUHS_y_z^k7CQfnUXtpGXP z0Qa={ujjrSG{gSCvUd{X(X2ttTeJ)5>Iy`l=ZSlgnORzBJ#R__ed4%Bs zJ;UsGxIo{r0zV`idf9a7cjg{67xv67(NjP!Xa4nJCcv@T@NOaW)P>;9m`8~VVOa{j zc;R2GZ?^o);Qb0%7Q@w(;riLoC(ZqO8uVYY$3GqRC&JNL@M=Cf>u+y2`ydnHSItr| z295;&^-|2fu{Zo1Z|L7lcduF@q3jI0=`nuV(ItY^g3FPnv_-;Nh`y@tw!Mwy( zk_6;^WFa+_b{De0Rx+5}MZjxfNqhv^IyZo|?t^BzZi*pNlWo|IT7qgrx8c$F?dWDa zp8)u9*iqO|z=3KHW9Zk$I76p_tG@)1Ii7a9LZsZQS)v>%$Eq>fslaaa9r=p%lR!mt zDeH)IVjtxlG_@1CXVV%*7vAc0`rtjE8Xbasp{ccTw zB3qBfN_mN`A-jt=pNZM$YCuI$>5eGV2m!czT$SQ%H zGLY3jS6Vi>XmF{Lr5>j6(N^fo_1>s7tOH>##0rZcim(d33Fm_82Y%ij-9GIO(^~Bj zm0B0Cny9XV5z{sF0L~U`LB``|leaS$k)o-~>F(%n7#V&w3J>HeED#rgpOvW##d64c=M=)+P_AzfT{fS$NmIMZBt*JpbSzo1^+y7F% zQC~br(;YIN)}xF9q$|V#&l0dyN2(=tKO=*2hh(6?;*6nVSt2@;Jc0NWzZ^RTW(X`1 zO7#JGqsB`m9Gb3Y150i^x&a$T{+D!v@R72Nc$=ui%|nG5Lkvu$&6PLZ+1ZsK`-C^j48O*lrzL%it{X$E=- znTKvS7>p!Uj>1FnpIkj;H{d%oUNx=1N0vAkJ@94V$>4Tn=%9zJSE`V?0sG;6)O8Gn za1G*cg%Gu$&zQ*UgscaZX%$j$yrZESTMS!`AJI<40ZJO}CUZJ-8avkV5_dH>OUUG| zqR*k^6Q7_?=w}$B)H8J)#ecG0>Td&F=`CsG0Bh)B{}~lucUFHI^$0T^TY?>namK%- zuA}>se?nGBDE17t6Sc$m-XMXv14{eccmUz5m&2UMFC+);gp-o56KY6b0OjfexfClv zpF&L3ry2GuUdxd~`2LU5z2bL+ma-et?W&)qAJ`W}5%~whf!RuBlCM&&scY~!Trpy# zZtehn&`ExyC$)EC|M8(SL$R7`Mx(JAg9Wb6^TgfcNu+heu{0YzAEFd~nn782U$*qE z>}5|xZ*%WqvA5)&^xDuN;0@iVe~X$%T#o-u_M#qyIBz!c9A-aepT1giUU6~gq%>Y! zB-z$~b70z_z3R7Sj4l>gid&D{fe#`!5bhJQu~H-%>8{IHzL8Jr%aE)QoBG~L9Q)k* zS4cU^d}WQgLc7XHM3tMUkdNbp3POx9rt2WMqMojquBwNOj}X;p)m6=R%|gR0y~KDA zy$0EVOTxC`ID|7e3tTvM5JN@AATB~~@Q}_|-wJGk+jK5^Pp}g+frG*o&PjmH8D)3_ ztec~CX~vhDZCX#=M!8ba0yv1HH9ne$sxiht#%VfBlq=E}Sq<6cXVDt)?0=&#pk5%p zq9y`o>MF3nLeoCPVT>Ir9)U%sny#TjQB|0y=xl>|c4E*lV89^y4THeHI}zr}zacdc z)0vFOH%34N_d7%|mKevI8jv?ldSJa;j*c-tLCgkzz61IS<6=ay@x8u9uQ#~C>?ZT? zSiBQ`5ODzg0^5z;08Gs{44%OF3ZpW3GCwra)Qhy^fx`l?f3IJR_@S@V%8gCvYgi?g zihl!$X!o&Cf#r5SssgDoM(dXtE*SB8x?z*i+b~=ogxHNLM|zDDq{(Q)D}09X!oD$Yg}pP+)v*vOslU zCt=nj(~z@Kr(cjTvtoQtB1sl~{e6k!RSXxuMF?$c;6~8_3-_25uVomIzcEMDRPXFR-`K zqmYTf*18CN6fHsSGYIuM{Zw5$aEiMcD>aGG3by)*$U`V5x*cKEM?)_zL&u}TFeSh~ za~C+%JJI)%Nr34$7rdTp$Xn=H7&R7+J&5T<#le4Cjy?p>p&p_Nu7-Co&+KWMj<~P4 z)h*Ccw3iGc5i^0`gaGrQK`66VIsqbHIHOd8S>}12+c02XPtr5etAdeKOn^4loT50M?O7ABOm5 zI;X2Ncpx{TAE4t<{NFKybZHyRwvi`Q| z7eqo5!E19ir5Pp}OOU|E37)w-Is|tZ-$rO5SQ4%i-s8KlkvKI0LrlW0K=+%fAe#Hu zaL}MNeAI7-Osw0$zx~Q|6X}Um8BZ8&^-?V#V&SI^Ox-EvPslFw(X7+i!nca`Ds8xK zvwo>z2KcQQ<8K{b>!Dq#o2R8{GBi!f>qDR9O7$j9w9;RVQ7sreGc;8-Me|M<19LDg zy3bHq^h;0CC+e&9zX0>+oI)sniArjC}u_B1fqj3Yc-}pdsE^Q4XoS6glRz=K2CWfx2u&4yeKH??v z5ahp#Npp#0Vg+Oh=aSw-&Sy8}DK(w!0C|DyshM;E>nkgi9m1`Hn$!&z{d}Xv6mA%2 zC9j(IjkkrvVmZ@40-Ed|@&QUQWf4>oASk2B3n?nvPN)l5$*6~FueFSw^d+7~IdgFpLs^{tUw z_m}qF>Yv?@>)$I;h_68CwnXeAnJy;x#`He#+1cgZ#pv$uF6bQDDeMweZEtL=Xnx&bRrjlwSjQDLSD&wzR=ZWMD$$ic zs~g?`ZenlWpnT9rfkC{+Zzf%5EVbs^b8NraA9FqKQsnsCX`<^q=O!B`+pD%dHm|Hs zTRQO$aW3<&a@o8s{6foMzCE7;HQ?zMbx>bA!2e`P{nLdLa z$Z-IKrO84Y`|pl%uD`vu0#`+^pN0Q>U$$SJuaB?DyTjwEYpRo@qZX?3xx$lHbFA9< z1HAXV(@>w~!BcZ@ajeKRQBONd*+hK_S= zz&>^;%Z|00{fD`LzMQt4wu79D+kg$hL?X@eBKC&WpyA$X#;T`57FLe(iPBT`Rw+^T zst>9^CSub>*4mgo>xd zr~c&TrDgY}VSkJBwPmYIZkHkJq)kyx>0R$Ndx^k9!Sb?nvmLOLI?nMD1>pim`9=E9 z_BiFV8xV|63hON5ISzCw$%~RgNhCifs4-j^75>3q1glg@lHwdO2GeuhSK}k>Z?wNL zRadL2P{b;psjGFFhEC&AlOJ#fq?7ltm4FL(oF8P{=rGf9hdbTJ*DKI@jD3KehdtWP z)uzTKUND?_nPN#PC#FG--DW^?I6^{DhS4{(t${CSpQW$W2H{+Lw#zNIR<~dmj?-qR z5l%@?$K1O;oLyTS0J+V^+4{K6b-MyPCmVNx#^SIAg?E{E#v+(IiKV0~poR%eccj{q z`NV&4fiO~vgREB?VI_`*(L$EjEzDHB7H^H4fLV(YnZ_fsAq#_n>@tyzj=K9Aw$?%G zq}i@=Q%o9ak$K3z4V;oX_K|v3o$l=sZ9Ch3wpKT%H2$g|FZxkEP2^E$TZa&p)eNh> zU(->YBwAKeS{YqAqGCdsW7(9l;?jSM<$o^c(sBo~>@)f^Ze{kS?M$}*rAxdU9~hsK zxH~JRbYrEa&Zm2<*sf2YTSJ{K*y21UKsKx~Xk*aCpqoLhq5QD%LFfJExCRJ!us%|f zu}P*^S^_Zt{E|ND6Nx4LJ-y-WRUJafZxh( z>9Fe=FJ34MR_pcUNG)DZS-}2fdB7&p{)ThC8{a+7)7q!d^SaX(>*+iSbC4n?7UOh? zM)k!3UqEvH)Ah7_y?A$S%Hg>xfto^Y+-@z=1HQ6Og?1^%Ia4b*iu zz{q&Aej;#dKS%rHya{*7yXoIqEN(q-j>S*zS#~#5#)ze>fbP7G)QvlimLj5z^`Hr_ z8vfDU)|9DY6{q9^`ShV0*(reBTHCi@tm>K5^|sx)&9C)h^XewcrbCVL2626O{l+?G zU0H1tAjEAFsjFXCib}T^yA+=;=*&ap?Z^qpc$W4oZANP9uNgl-CcQ}f`u$h@rnr6I z7JYg3G5B*~mZoV{t9Qp&%^_oeavFy28a}RS=JbU#maLv1Ir+i_(v0v0^JXud)-q1$ z{SSW)O@&J{-qLOv7~MIhmD1!_Kf2*`b7XZxQB<+DD6}J15-*7n*LFIK*cI;^w~G_{ zE=d0ySlS!d#_0egE=8|t1P(`6aJ?<-Y=rKP{sqHKA*I9Tk9j{jWYmJu`6EaBJ+)rR zVKL_t^{Bn3yNd9xjGD3{-$G$=V~KNBXH9(dv+7;-^tPsMe_6CH7%jkSNMc$ON5GHZ ze-@Nk%@r)MN(C;fyU6=hBXbt109_Ph0WVMm%Tov1`Xl?^ zh+}%5ccD7MThp3vG%K4W%?(Z88&1_d6QOG4Rc84*U4}04E(ib)g`mt~8EZ1~(_&Mi zQ@WDB{}lcV__g8Jvm|*UH&OY$6%yk17e==nkUWqHwDXX+vE8glkG8R8bFMA?xM1tt zaWmIU7tW5Je`(Ia$vz_*Jam??XcsUkI(PY_p1F-*s@WC!rLT$)6ng);k~J)Aeg2WE z)6LDTl$OAz_jRvo$G3j&d!-Po?kKQBCuHt}G!@^Z!4;EZ>3nX3rP^BO5bK8V@e1Gt z&Ki~$$_eEUOY#5a@zXY)bAbAWa0cb2JvDHn-KTm(!KZ9)_JVA3jx=XkPIAtTKQD^i zt0y#WpBwV8%=R6if*j74>bQZbp^(gV|@;c+q@Un8h?|L5iE&{!3y;u4! z_u{$kcj|Eb>d0_B;xOON*J=l+j)o>#<2aZIl-hL15UzW!IjkO`3Dkuf_97xN?RX#Z z40;5+0~nsqTA&2yt+oqyT92}B5`^;dm`M6AY8b_d%phgr$74{)e@xenBBL#$AF_9% zfkn_t|5f`~vtBJ%hALEp@zN_2PVbSfE$yu>fz5Lom36FIq$ss|SJnQ?PZdkb%Sr{M zn@VY=oD#d@w}qPuW*1l%_~pCir{%lk56cV4ZOsYGeVa2O8<1Lu(>aenjucvXRZ=4n}{nL!d zSuL|VrxlE!6n@&L%6^K)SL!khS$AL{rc2c_xsg|&CHhcxyZl4xin6WMt<8^nM18~h zsr^VPdhoYuAK=pG;4*NVaVDtki6d+zkEP9EPUIpjHw*9EaO{6M-F3U_Ip8(LC(N(h zpB?bkcY()N$7HK(oV&CY#1-fbx+od1E3M&8)x*-efC>^{h$`Ape7&@=;wGm}NY1rZTYln2v#S`hXm#Xu#Lc+v=r%+mpyUVX6Lo2Tfsm`(t@KdD z4B0@tz3pEkA@@?ccXZl!j0LQp{6=cwRZf>ny)DFB}s;W0t@~X4d%c@UQ_rv<9 z%BAW`Rddym>Z3KwYW7us0Gx=O<*c%C(2D;QEzDn(lbIQsH?x#ozoDZ^5+@t32+*Cx zjA0LR80}99zdQ2VsFG2pF(b#ln?RZ1F=l?)ZohVya^X+T2yy~)zUorn-p*C6nx^{= zPiiHV?12|u8=Gd=Y%l+} zq_AjC(bJ-D#T_L_%2!m^*ROB;-Me7WRqKheCN(hM@vUv9JBXeCai@9L`=ta91yu!k z1^o>0^|$exzyuznb1PermW_Us~5+n$Ww6*tJx3Z(OWpjg%D6X=za#w9qvqk5`p1Zx*dUJbk^eJUl zsNC?T-Yw6m*w)HwEsxEP zq+TJ15VJ8c=)DMzu~}E4ou!M?w&^!Q7THx)J7ypDAN;>WDrE|dz?jbTfpmo_fVo=5 zSjRX{`;X>BT~77_M)qge0@O%Ev$0A~*Y5%ElnGdy$f2|Wi+)#0YHw!G{;ts-nzr!P zj%Hoc*rv3GL-iI7CH0l{BkK3nxz1F5gn@iUPGzb zQL|jc6~$M7t6pBYx7@qDt;D(Ldp;=-nWO$wRvKBexhbFn)pJ+;Y`{*7$4_Dp+PrW% z;{MR(CCsh~t)&gLy0AJ* z6JU>bUFbR1`=KYl*G`%}WUHPE6|$F2+mTFE9O@Q267v~bftyR{CO!i%UQAoSIKb>= zZDU(;_HacOXn}`d58s3Lg~g?pQ#O$#_?4K;rcWBnp(%ZuZes_%?NYO;adcx%!{>%A zjo+HF?W4NCNPJ{>mEndpn0(S$#u)Bo!F1a_PKQ0*{GJE#LyNnqKlJe^R1^+C(Qy9kDy+)9%W`2-?uTr3Z2k`@qrfyr|j z?Ib;h;m2%etYFX>ar6cBdYXxPkWxlkMNGt>!>&S`kna#8;~)Je?G@nYnjpU`3zLTS z@g()VwcYiduR89v+qT8DjBY7tnb~rwWfkB9IyC7T>>KAao^CkOSl4hITJdbHbL}b7 zN6~&!W3`~ht)`@^ppsdcTbWeJsPHHcEDJ0x0w1RD&&q{e!=@SwBA@tl*b^~i!F77MdfjCIIMM2O{ zGw-oiaJ9T^d^z94a+BblFv;e;t=eXX5M%k2yNTV)yhA@sIZWt-SPowGaL_?I3;3(@ z05N7oo49pm+t_wV2OtWFU-f4V2C7#WS70njv*|XRa7&VHoAWKNA^-8gUSZ3^iNnW@ z*fdf%(sLwncwWf)z#YC(o||2lJ3h9(YW3P8kG-7ngF2l2j-UtD5_hN)3c$UCYGXH; zf4oOrNqS38q|Sx$VFKeWBLo(E29+VDZ=(MIT$G1oe^LX%4SxkNy#|r4h~dU7x_0#` zT2SF zWpQ$OYiU90m{K*cc=#1Q&42i(I4>dBBll&_iR?|8+tat_oGO)x?zcsX8~f}B6Ev?; zDU3+lShr@d3jYR#hFFc{jBTDMn>c6e;E38`4BrT^Y?mS1Pl9abHUbJ`jR-Ux(mKdH z`ZtPQx{O^vB%`FI114!EU{mZ7bNeR^H7bgg7ohrTy}C$qOD6@s?MWzCtP(#&TucpO zayecWM+7UZXWJgI!#VDCx$A1@I^OxGJ=SJ~)fY>xg_6C77D>2;u-3ehGh}6bNb&cs zbL~|vu}%F=6I<@JwztQ0QF|*S)dTaDT{?FJ67v`zK{`yo$@?l?@37Ft-EEXdvDX*h zsX@Asico6UjgX?*QHH;XT>*q7IZ~)jBF2SqqZgk=h(J}w7PvFYE4De<;v=c#pO|@ zJ|*Xhlm+t(ZsaHB)#T34*^(8V8Jlr2UHki4nq_KAa&}5VW`E)0s?|+JoyU3~5B#HP zLN8;dx_O-X*(*>5MiBv&YEplVVrUo5|>Sfe_k`K*gI_yO;H50(ma zP;&wAnaqD~)oQfVdi*86H@y2j7P-!G3U{anB-wujk9qUhqnHorUV!;Hg|dp& zL>x&>BiyHN6^sG@-UXIYLq z>ruw8-=%3cQwDyH`#Jr`3YZHCjH7`#kf!0fnH>c&~_<}gBJa7dUkZWv^%zQ zIwo|E?^1M?cmLCSQF33paVS)2QXSQl=y0YxC;c@ro8ESUMnuw$8X>A}=3lzeh8xr}n1 z5(UTtEl^iJ1~4`jkY|%-0It0rr-b_J9XLm<8eN5&0_`&!qPwvWEsxOc(Oyu$R{oYp_^^32WY`KgDKBa<8x znt(%g>eu;SIG+!Fd=|AJYV?=$3E==U_^|S1ZD4D;I8qfu_-&!Hf9hfFQ|)^@_~oe2 z6C$Qgo4R@eYjkqBG5By$lYgKWVEfx05}XBAk9szn{)P~V+6PrVN~AOT9HK_6mgh;+ zBwdmveJ+x@-FMo$o2(jJ8V>>6#GiI#*ZiKsGw1lr`T>v0h$12I-_Y|A>+}~@9fNUzbn;OeJFrJKQDzA<;Bx6$ zz~f^pFv<+Y6^Lk{l@rze+6-Nh;W#jDEyV63U88+rNqK)P-wQkKgwAW-nO^qZn|!YO zWO$GCdE`^)HQKY)t;p%SeWGo;wUuzIpxfd;H;1iZ5}20E+l&u@TyukJ3k-@=$w|aU z0w3RosYNHCG|(PbOdW<~JxYI4cSaYd3)5+}hqdQ5`!ri2ZXd6nt?pIw6tzR?gWF{E zfld8RlKP&suB;Afdvpt{X??x4Cb{x@Iis|*Q1fSO4mD#=>f~STKZpr?;$mV(N7sFD zjT-Z=`>p)btGF#ck7R@t^p*ZsbGyxFAX1-9yw1MJA1U;-le;+kEe@F!-W8S2&LtG#g9$9aF1t>ck84Ixha7=5fK7KvuhBTD3gj~f z*UJd9?E|=hZ0Yua4}(p>9eCa_7ny~bN${dl*l8ADtrBb!ZHMek4pz=LU8CHXZe75q zamtD46afg~GaN5DECzmsHFkIHfRDs>hs|qYwACd+C*Q~7IA;RWm3E74A~@qwST<_B zDNX-UyIeC-BZY_xS3OJhPPtEcO_`@`Rc=$Vl}7%PXMH~UuMK(_)azg3jq|W{FLrzEddkVhcB91rjZP4uR3^TDtMX2NXXowK zw3g$ohg-c`$G5aLm$sPNEqlK9eUfDi#VIgqt-#7bgs1;n{494aruLdc)MU&LF=dCMANbOPVy7g8F|7ZZgHHXSe&=sUDY znhBbP>RAvuEHU6rIe-=tgX+g@!e1vkk^cZ6ULj={^*BvJe+^iNvsr95p7Wl&$6~zY z0|7%AVZGGmr!CFC%ih}Imi3DT_6pY zx=Be=WdAn_PrR)ALuYLJtk%fpJ&ot;b_3g{eZ|(&5hd@6EDF-|cIIx+{*ZYzWB>1} zG*SB7O#7VH{Mn@i<@l=8b&s2yJF3LS*X z2-YEVk2MLZ!>Xrhh9+1WtJP>I8V|r2n53PrU8V_9FIVL$<;rualbTK1o7$1OC3=#P zg|I=_WBy<(agl&Cc!#tYB8=Ho7G#>G00*fLc@ya@(SuNhC*l9Wbzo(H*U^k2V%K3B zG3T%cv3l%%KxJU!DsU1&B@p5MK$Sfm_WhSmUf?NqFkWd0i?|+;$6a8`~}>0tOM{Vbi=4A z1!}760i|lO{-Lg0yAQC6S~c$hZ(y-zhGv+Cs6lATA%jH*2n?^3O^N_Th#WD59Hhvy z0htijcSYSiaNFcbKLqCYIA(!%m!KAyV|oNZ}0$bRi3N(QvRguHOvDaE@hSN zE%&P$A(~ZpqF&wjxMf#caL3B7t36ABCGL~-gKWppQ~4W3k#d%Lmu99;Z9pKNA`8%$ zaD{|QQUf58Y9OKR6cfj?V(ZvzIeDCD&Rx)12U(X{i&<{0Al5cO=u2hG*x~RLHgcA8 z-f>QH(3}Kz1p6_olu2XK8D4<3>P{t5h-5b8i_Ie*AwDDK5o?IikimBckWTIZ*2y?x zC*)8d2~PM)ID70@^kq~6@+o2)WKmo&jxuTu&mlu)iQ$!@$xvZX8vLOW-C$e=+5N8| z4>uk%ZI?g}xiwS;--mplK$v~_7q=W=LQoSWBu~n})Fre~Ko?A5xC5$Y4YQjW%9;ad z#8SW+tc7=2)*t2p=2qqurY)1j9AKPc;2{cT59sI3R4!Pr;bdPDgNP^e;eJELU_4}W zZ9%SwT#-e_sRjpqzV~8m-s9}+{MxC=k?82>&}3(0%d);CP+GKe z(VPt|K64v=1@&JtiF6uT=p-qV^n|pPBp`hx`V;RHBzO$I15hR|q8VVr^8jrv$?)4y z1Q;ti$RjI+tgc0nDb#5k4>|Y?peD}`>Z0u-i|#Vyrd)wcZCj{EIgFT!j6|(Mufx2@ zmg279!wJ)f=OHI^JD?uZ0l}0G&wLG)2*_|_=sNm6MmvMce8xP@ie+EpAh|p)hD+m$ zIKMexIP*C%Y(Cqb#bu6QY@}bKrBZp+X@J(hj3gw!2X0R!HXeNmH4j;C;ut6DhiliV zb;?LZ>QII(OZvF)h8WkoyF0Zr4BF^tOLkL4Lr~pOQFS$aTTF5Fx60Qfximz=2BTeGsRxM5^-XlqPcdHatp zmRJwAC0#j3chs;18HrUB$kaIcR8}JA8+Qj!z!wSx!hdb@?C&`A*;U#lIox&1clzk; z;hO8F@p$IB+XL%<$h8pIj`lc~*r(Y#*-RF41=lUYxd_fqwwhJV9L7kZRZxTA8NH`|;qBVuLidL57R**&5`q?nTzuK1HF~762J5X{^8a!02wAEiUdzCEsEt?=EiyFOX}?@ny4F51BievlwDpB49T@C$orn{2a(a zyN@YB8&Kx~tt1xE85*IU(GM^i#-nbbzN2QM=%_Bhr`QVljU?1I)CLp|ML~t2FlZ~x zMN9|gDRwOGE3O9j1u*y8@n-ZSCZHot0NkY0germ$v60wA8X~7s4pI@2XC89zbu z?E+n=WPX8G+Q&S?EMW>*{wyL3%YvjVmMhC1a7LS%2bn0QJp)HqQdN*Cy#u&kNQ6JQ z<&cHqgNg!gRABt6UkCZ4mms(Enj&`yKe%+@LjNnt>)y!je>!giI&?y_ym3*(O<;#Y zRxhc14U*Jae5$Cm0GU5NFEZz7*4hle-$zojlT&_uNOt_K$$XI0muvH9abaIkbS!J=#6T1Lrx+tJ7<^SBvK=;CJ2Z`mfVMhX%WYwj*s;3)2BD zoM71qafD*dTFzInQ@2@fnU)M6dL-4Cf+0^No+7-%t-w4-A4J^*Ykd}KrbJK=Qv%t* z%}{?l2I@dw117W*GP#nu9-^r(Qkr3c6;$dBBOu0_AVyu!9)*WyTcXZ$ey za{MVk1l@se#(x1+v_(V%@eyec*$?m_OQ^SL*8uzU4Pzs-6lAv{5tj zA@e;m32YV-Bx({Mw$Ff83T1v~80cL3Fq$isNy#LQBsSoW;ON*S^nTPPC zu!LU%)cQ@p`L|{0ob0*ua^DN_=N?fPr&G|ryfwd>+cY0|>SIKfHP@<`mH(DsDg9QQ zTlgnmmdDOzWnakDrN{kVmG&)#pKSQ~`KLCSoi5CpnlqTYs$g;P*WynlQ_AWqY(#GL zsOHADoR0l4lfF_CAgz|2P(0OqHoS(a4FO>IkEHKoCb6@*t1V|(y%HwbzOi?5>;kT$ z9@k{IIqq9LXL!YW-S!rF8@v~LFY-F+>FROaE!!o|X@&#FZlO&l@O*e#Z31rNhZduG zDDFFU5=+cn$`mqA(;idjQ9hG;VNBVG4ZyUa&Y&6q{i6!1x%r3(Q0wDjG8#WX&D(3J zzpF6fp!O%$I1H+DP*4}W2CB)DAnVi@O~72hoWi;S2Gv)PrSX8Ub%kIgTp>;;eJAyS zRq~{~q;LVRmIMf`ULZjySc9xRY%M#0^OS?<(t$^yozug?a{D-aoCZ!MhYL8>Vcdya zJ8my$Kc|wt0dR}mST~qwVRmmZtp#u&r;w%tIyD<#i+za+MvIZ>Ag40UFkasc@vnT9 zm-4i{a!nE4HN2ih{~&1RwY#^%8;e5#a|0M@<;wD z%cbPp$P#4cq+k2ZNmHk!B#-{J^5@>4!&83!{+?mXa>~7(zp&_d(dLq0W#=p1YX|FV zn(ua$bjifm`%|RKL4OzlZP(vHjK*CfsVI*avsuyX9PT9xu^`Vn$~M-)1NdF{xOuvZ z-0M6rUJ>5)-cx+m`sVoU@N4yn_V)C8?Ll{!yYxDzJ6&-|wL52{60(H;f)u{Q!oX|c z#w;0eScM%Q{eMqq+1!)*yj2@=Q zV4m(J?J#{bqlK}RX~{yfo7f4QiQEeAao#T87T#yxA0EYm2E1ni;8eZD+s_N)@pyf} ztgx6H%oTIY*y0n}Cs|v-t6MByd2Q-ui<;J6EL^H?|EoiZ?rSKfUG=U^)+R- zymOEz3z9DCyDqNl84hUaCm_P`r#ZE;qQ0TFqh?C=oywf@-cmtHMA3u1zU;EBml<=@ zgHnF~bWP$Xu1sJgG{if{6@Byn<{qz0bWQo5zArN(>sOXb-tL0AMe?#mHI?--Ep_es z?#=!G$+oB_=&g-Mkm=}&xMb1{S|9tmMG1eVum#x96C7{4C_Hv}kMa5J``$O%Z-3y5 zpqD|Vfnx&deWSg8d!F)g^xo$+=)TQ0z-g*ouQl4LjNi@s%I;@Og`PQqk_=2gOYs?) zB`72^1Hphy*fZ$U$bH6Y{UgASUZ_6}e)m@J$6p%08k_)Q!44S<6=kQ9PJnk^hc)4b z<3+d*Ts!_OaVe#gmQFX&UFlP3Rn!0&E#&~CLndnkYYpI7qd=2+fXx}pddYmvv|%N% z#H;|;QNVbnG3{9x&I0a2o{T%4Qv_pU1#2BAo;!v2oM*|~2y*ow_Z~Ngo5;Py4F_Lv z8hC@dIB51IW*~z}FQgfO@1T=%g2ce%F{Oa*e+{tjs0NrDR(Z?!%jnW}Nl|ZFw@X)7 z`-N7A7F{E?evZf+_`3cn>n_<)5?ExB@0B+uds_xEeN0+S>Z8<%pAXA|{?LBXfywuwiI#K(hsmHpDN3r*4Pp)X4a;wf)GRmiv*cBlPCgdOZ!^o@3CFb;Hs?znT1u2fnQ9l!Y z)%*%hX8%I|xSwDXKjm9nj4&o3=49NwOk?>#mAL%LANQO$SsvN8RTXVN#OE|D^jgAG zqBBYa)_wy@L~j+obH3!^=Gp7<-QD1H%*or?+vAvjeo)`A;eikQw)!6O-5M~=zt#sE z(BYTibKO15o^HEUz_dzokUPC};8`;{Mbs&{4M-j080rUZI_WF<3sHcM*B7X!$)Cwv z6wcaR$nSs^z7+k!RI4jeU)9P@v(Q(tVq7l%A3_ptC2j_O4?drmOYx_&sT_))bdCI$ z+)X7iAG69hQQQ<(k{O1A(nFIo`CK21&%oy6&AG?eOKYSoqzPGtoL~#Ig$ejTG|W*j zs{O=lVjH=)`3061GWLcdG^G<%;~iIWd9kVSBJZV{Rm9(Q~LD#l=$xR z`Qc9t3iN;I6YSpQGS4~3@xJ3#2ZE5vnMOZP^`Nn-OQ=VIOCSyBifq%hDoze2$;F!Y zhFU-=Xg9cOgbMe;SlJGRrADTwBKM#+AZ4bxhGNi-v-I(%O!PwBQ+zzW4-h$@Vz=Xu z0k-Kq0);e(yqFR|v!G9<52K6dXTi%11_bOov_}jBi_MMZqyw+babRFgqV1udU_NJu zLstGSei^Tw6U2VWdd5D^3$eT@@Bz*?B(&0?Kx28_k{}=n7h6|YzqVdu{a%=0b=k_z zYKdU6G2{=LA$$5k)xJ}r4K*rpPj5EH_tykj|A7tyL7X3H+ zq9l`h30+^?hqWGQA~iVH_0&XHH&?k;SywJ8yu!v@}R)@cFPezHrESA^fwkYB_1gq{fW^|te|@_!uI z80sE!FT^`!Z@^Ojw?6ZHA9)YBraIrVKP6l$sN}a;wpjXu#yUxRjGu~%GPvtfb?K&R z%v-D;A=H0Xy&fu*MGRh3)M$nb;ZX5G)9W>Rlp9qOGz+vR3@OI#h>_@KsPDOryacGE z>rn#iJ?t$^1m2SrNETDJQuk1XL50v<;zi0e+HZ!0$zp}E{eWHj9Ag~caQ7-118>( zq09kQf1M<`rwA$)cDDZ4Ol(@w@V%}}bhm1-Y<|g5;giBw1)uYJaxhuQjAQ93X)(za zKb*dQiI4woonVpB8B_Jy<@j4HcBhRH{L0yr7pHMFBk*#a?45fXjhuoZr=~XMug^sJPKUkcQN2ykbTJ2 za8_7U@VJm0AuGdwg`EgK;dj7ev}=vScDqARHxTP^(;>mCgWEwx)=rtP9CT>X>9e24F;%2>rg(DBNU0s2z9QOqR-aLjr)+#Q7aLx2DHJb zUt}srT0+Fbfjk%LqL+~#5@Ns)v?F*ywb4=5b51Jz2BQT~^G{J9fY-4G=4iV(4cvU- zgBpS?7H5VJYdiRXE#L*F@JRe8{4|RWo;RNh%n)1iuPS zTeS%()^aOXL5;;3-W9N3;T&6T6-ez~<}TVesC?K95fd`{H}XD0Xo}GLYswVNp^Smi z5LcYpo7A1&dA03+%eiK9W<2>6=`LKCA^}*{4seV7->z(Jeaop*Z5z`pOHy_ zeoXmpADi>-X-w_c1JSdiroaClbtF-O$y!s&Bun zyw^{1WvCc&$Uf=w@Drg3e~ZUmuM+=pLHR*x0h@gBerNm-1=WQ=3ELI? zIP_H5m*ES-uMHdFJI(F7^9v^r=dG^yT>71T9NR1dSl*OX_+7Xv;IbHrUqNgkoJ7~@ zqg6-c14EvQrz#v&jUES#(+bT!4pSL*yeuYj5;H z^RO52R|&~5*PeP5yyRvK`%12ryv2YryX zo3@QsMNeSg;)>u|KV#9@A7IS&g7b@$$|YGC`NsuSmP7n!ymj0t-UW+Z{#?r?zz5Y2 z44&gGrg6uEk8&HvVpBQ2>>(DGX$RFqY~nUR3Z4zKO=YM&M6eO3t5scTHugdmboN--EYG0#VNazfJY(Wh!UJ3e=XB^j5qd;TD4?v~qm!?B&t!gYv8Pf8ZD3{l$~y72vlt*eX;OGIn^> zaDMptkUs$@JWHGoIodcUy485i@UnJ$WjB@QL{B2qf&b(=;UiH-wxxsp{8K`@%u?nDc`9nG#-^sav{{<1_-mLa{3pBFJn3V8Ss|m(>_w_Nu}f^ zG%@QjPsw#=-(}>|dCX0$MZnjylM}}~#s6b@#M0kl0%sKa14qI00RF~W@F~139{~4u zAUBa+&nAG^c!sx#=fxdhks19IB)OD0jkt&y0oAr&umh;orfa&DYKcN4pC<1c+$GyJ zpy*?YFLW(#qqev-)i*3_Xsvq%S$?|8ma>b*cKPRXGP0brYO|6vBhqiAy!?4ODJ5Y> z+{u_TUrWC$m1C>cH+<+g-aAX2 zD*aEs54nzUo3G>d(aO*>aTP3y@PKowTe9aIZ<9}e{~Mpno?AR5Ui|^5!{-i<4=)-n z51ShLVA#$8ljjWAx2`@OYrJ`W3%oO3Np^+YEev0JH*E&}B25h2Z`50)2WW+E7estI zA%E$FF&Gel0Gr8x&}>p%8G^vHimz?ZZ#AwoOw%W7Gl0Qli8@vNe;l0!SQ|?hhIg}Z zcfq~56ev*FTX%PNy>;p?)aBN0-Cbx)sZ&Y|lp@95-9lV<_doZ4_DM(rB$FN4nKR${ z-pRJ9#7N3c+7sG%>ff+C?2kHPcW5&i`&qZS+xT%%9|Ku3=1|ro&JEaG>?Z`pqA*uNS%Ub5aHw#y_?+Z~lYF5Avis(v-_Cfo827Twkvx``9RnbO7yVj za7<+4hj1FPrMS{=#izlp#sF|N(SrxwReY``(RkIMfZXo|cAW4d=i-AQGTLYv3*3>} z)_A)NnTd5!pHM-^jO0K@cs*7{f5h6s`OJIHvx0``1`B8R=U(9n1xCSU$a$2AFAHz; z-Cz!S1E`l)$wBF2SSg)_)p3{jgZR4GU6L*B2@K$4s?iE>nUB;NsF-SnRk2^WLuF9; zIryj?6$fN@ftGkv{#LnArBp`9cS-@B0BpFiz_yIzJ>|uNe)BEs8*rS56IX2Qcph1A zk2Bl#k=n7EQI{cFdvF0NS?9BqScf=LaWm~k4z7)_jupnfQHcMP}jgGeir}*FtT?{ ze@pK(J@3b)_dMA%qGxURxIpi~?V%qcjz;$BzAIp@*B<9Hs&>T?g+NY~QM$AMiDRis}obIB~Xv!*Nr(IyVV+=AiSVHU~hyl*T2iW!i8)%qywDpYzw-ylB zC?7BpwGT!|4IbO5lGxFlp4^lC6GAaWV*83d3&I5{{OkND zf?lx8u~~dt@#pi75a%D1R+$k`0D2?zAFQ<>atdHCnku zzEZXlW{g_dI{7TcIYlna*Bd1M;y9T9z2eW{*8mL&&U0 zyZiOd_^Ow@7uTNEKVJ3d;=|(S%=b0#9O5rN9C2^^>*DYIGOalt*;T)@|M-+i>fGu_ zG%4Ev%tXv(W%2dAXxeY&F=LPTrNcZI5BD8De7^(1pF=VNH~9tldIY=nSOu91&w4HD zS=Q67*M?sGqmx7T2kHXV-5EVrL>7j=^dIik%_YyI+scU{=oZRd za8sxNk8B?0BdnPhGd8gP;s|-ic=Ne4KKMLc8*}@Xx58!`(6X5(s0!QH` z(MCzSv|47BPLR}#^Tih=Z=`LqzZAK!Z|bGwC`T);@}siRGP*ol@l_S>FjkeTpePLT zo-k_0${xt}$(6ubPLms?Y2us0FxcgH0|uRvH-mG8HHQ&T{f^4OiD8W`!kTDSn06Xw z>$1A$c34|dn}#-O>-*M8YOYsCl^2vG75&M-{AWE& z?0VxB@B2#ea_0->vt>{IdAvG4FY&><0dMV(Pdy&@rsP{p=8_!moTE9Rh2ty5_3Tw6R;QPf6g9h?{-i^c~YD zwr6I~sNS1;mqo{QpBX$kI4k^NWPRl1up0sSJ|8_#xIJ+itSkV=;5q39;7fm$iusN7 zARwd`V__68*pteK+?pVI7$wI(6e6G#EOTtb@reW%ABnRdf;+~3-ZI{_!@SS>n|O)t zrbw|PDCFZ{cfb=x2#mJj^w+F?+$i1!U^OsO+dB-9hCN=CRDH)aNhXrM@P= zn!wcodVit+OO?v=GcJa;qjs4<05Q7(TY|0yTs zz6!N`n*2LN;IG53$#?My(Q@!_n7}tck3I}!Vqf}JAQSt6!fHIe&sGcZghs;?-5ZU( z%eV7h+t`+F4Z}Yst%!g1&tIh<;d< zSob#NH4Ai{*bBw;f)`$|u{X!xTueCm!SW^kt0;;5zAUXGb6f7b0&7X{%9vWT$-P45I#R5Jd%vO zAEl2bqt->Oiu%}Nc;tfc9o^rDQA39YuL)EIEbwdgx#ztPb|n`;mi=R=JB~wCmlVG8 z=b)WgA$}~}E7-zAxNm^O4eE0`M%zh^#6F;JfnR!**oiN+4+4)5v#k_jPqTr^kzoz7 zEwKG-YqU+W@5Ma`9w7v0mZ9J&+h?WV*37U9wxKG&})OVjczE%=d)RDg|*N}~7Ey<`#)BMa%DNR0<^yYKb$HMnh-_3n%kEgzR z{KDsX%X6>zHLvm4F>k#SM|__C)%b;#%=$_EI+@d%8(+ApY;)!LI;`=2!|0a29b+{4 z`t61oTOmYt>ga=+RBndY59Y7QjvALb&u+e5kmqwIhzRc6{dRbCghS-99uuMmM&F29 z8|5Aq)niV?l<<|^b)k4jpWw8>H39E@?{$-R>+*8;Jn#0-<(o^n(?f^Vst>S-M8J;Z zI`MPiWBzmA4UUvOfOVX4kUpLkhh3%&MQ=hD?*Tj<=i+7bWF=e)ci#{i1h^-yV5A|J8*T zC!T$I`r*Zf_{#VfuS?(BKTy7geChXf;P>mPKQjaWNb;YQFe*G#{FfUQB7LY9Q~3ipcq9qAA?EBaxKW6y&z z(J{tom+0v|A|nri!^WqumXLy=uYq&@FZz!5jrE!1vB-f=o}`fS`>X1-{h#& zMO-1=!l&~p*>6EL@h_dErej60+jWgxj#t}WS=Rx>s2$ipYV$21BGs7AnIK|s?q^{F z38)#^Mi*^EZKG`%5MY{dE%7&b4ta{gP>Sux#!=I0o9N3J513C`HS7zV1>81X55aJu z5A5nUi)$qpq>g0tM<~mc^HqOT?^QolGKawqn;jN7>~#3afskGLAJ&(=~UcLVS?l z_N!Ul5L4H`=1SF-idUuki?sRjyuE+=WbggmmHsU?=x48#@^25n9{>FAW9WyoiDe0H zZ(UyZes%Ii(lhJRQ_oMn8vEMi&BBD#585y7U)f1xzxPc$0i$7FfvhyK@^@{Yrqbr% zmS632HRB9J%sp(M&^SgIi)5GbPKvK8?yBOPqTFfTJN?oFslmd~k729BTOyl#+=wcU zJ{7aA=aZh*F)LzTM{kT8-yWTKJy4T+Y9&5M~%-8_kEho^k-0MDD>K*v?v?S!P+>ED2^Oa5e}u z{V|3Z2O9SqZN|SqJz8i+tP*Q)YofKqhT?O9S2iBJ)ouW_vk;?$;xicNv;$c$*cg}K z3i#s$98nUm*5jo&WCe0}rCE7JbLbwD_se=q--suPo(u9|UYZXh;~+*~+D6DR7a(Co zkiEZkj(NJ#OJA$GrQX>wwT;%YuTfl|T(hvsv7)r}Uh##(mH8ucSL95{>i&EFubXL< z)c79(DPNNpLG6wI()wxS$By@V6IZ^=OxXRl|C^ZC&97$1Uwu=VP?EUi!-db&lUM$D z@iQ^4En{`g(1N2S#T7oayBb~F&ULzVxoa;NHdu1(IpiK{KXw_9Bb*|3ltrjEJKb=} zaUbXXm*1kG1tD)j$8=v4E{wd_V^ow!v?aPF=4a2qp7faQ(R*NwR7VU8?+hCqIyd-X z(2aoAzBhahcu(}|a*uS4a~|z<)?vQVUH(j(A$cbb5>@h_g15mamJ6u=1HonSAreWh zhW+zS%PPp$oo$$>cLuM;qq+v55bo2`wPhN+CQ@tCp3|+@to^9(zBQ63y%(Wk$a5WR@cogKF*j^vBLosri_#y zl4gn-qJ4r?*nQi`ehv|^Hfk5lKnB4`@z|DcaWn@SNu9Hn)z#cl-gdR+R@3%|@H%bv znab$$fKqm`v~WUxLhkTC58=x8;CEC8`b(D1N?Vvp{=D`x=I6m5wJCE_ZhW^VFG+6w z_WhgBx93S)lM=p8|9a)i*UxdEi#{Lz`rzB_@995s(>#CA&R+N@Cr@9vru2N};@a3o zqP4v9o0e)^Y3{PRB{xccpdCkjanWpy5E2 z3*2YOW@|=i zR%We8hgW~EL0(F) z8qa?`H6E8dJUrI9e{xH5O>s$dUg_lEcu(b_Op^DIos%Sj!aSb$nWJGLOfD$(CsHmV zKZtyLwKdPIHu}Kmc~g_1W^~R0-R@P;gc|Cn)ori2RaIHByqsG$zT|#UOkrcbKF%XV#UhC0U_aXqGv1a@O;#TX2ib8j!Utt49_!>ssdE%(Jj7vm%q1 zIp}x5@1l&`86m$n{a%$R&-VSp&F@_pT+Aq2UqMzKscovi(d^kiOYH$H@m6z}EeX#e z`=F00&uCQUSoRlg9Dlv2T!KrBR^mZS%Swj#-)%$cG*NR-!?~E ziY&#}A2y494sn_!kb#t?STePpHlLBle9xN7v2myII|NlimUyLPzmz8TP>7V%R6QLQ zLa!`!I^;}odF!&xHOqCB+f}y`H&1s4jHI1z=iN5BMZ5iRJ>bf6J>gR6%yAy-wBGR& zM7?Gx9TkPLH_~XyJ&~vI5VEWUSQ(wZqy^@@U{{ZL1f6N`md4?R_4z0Rt zj@q{~v;B12?pCiBQ&U}ITEpl1i*<>$e`+387giCKSmlz6H|4zY9c8kzj?%c&&{ApX zkCN;XYl*s~XX%2{6Qz_gL+PNhxn;J}s50-eWo7%zRAsx$_LTK0>t9w{dZ=_z>Bo{C zB?BQdc5Cs*;+@53i^r9CmtH9gu2@vrxB6=Bf%?EkujYxZ-W>^Qqo!CNXG*Y?+V0~* z;3sFI7qCk-KZXO$?&Q2@{48kuMPe7}9vP-cS0*^foR&MUbqR6pfZU;Ew^{C6-Rs>6 z_ec-9hrjy?w^y$BAunW;(*eg_4pFMVAab!+rjXtie--V4*&UD1;?3p$&FRgK0JqCx z28ZDW>)zv3K6N{G8#rmZ(81tDcnQ4DQ6vXcaf`vDeh>JDuL76#p5W7b40QC@F*j-v zbtmlteGOw7^8jlh%xd;?qd|9E#7`Hr2+xSlh)08xqf5G579_tcXDMDOdMGoLF{(ML zyQ(^v+i)E;DvpBylpqGxSJeyE29-?pMcG@~08V0Rc`t~1RY7F=gyg3=NKmc|UaALQ>{q8vuUkwW4u9%AR&zFD@JH=DwYeGL(MotC4uciGf~)hjxG zbhvlSYAHcCuW$hldX{kq@PhqpXY4QW zTJm3%j|I_E=tr3d+l%vtYv2V7b_hw4MY2Ga0P_ryD$JqHVT|T=aZ|cLfDNNgv5u3EDb7M8D!7 z&(4qek>SkP4Xo_%z@z2RUQu^Y2U9DseW32%1T4jKSR$+kmx1$no$Ysj473$j6-+98}SdIWmzF!3sJnRuAwhNMa2CtW4ICykfB zmHsPTDLo9D{1RvjP4M$8s3XTn+agIlI{R zSZ|qk8QbU^X#0TUdYQ5TGBuZyTZx7Ediy4D^VkNSM**gxMlS6q;MMWf`>MxQf2|r)b+|II!VBv9ukr!q z?&UMedsPgs?5sRcwWwNAW2v##j;;4?tZ(vYEpG3j-lq9WN9cbWt)@83DqA$Zk-Ut? zV=iX{bwI1W54ka( zOb1pbYdE`-y^(`MukPedhMbEc{u)7*V6gCz@CxXWTOryl5e*hSfqD-T4;GJs+kZOo z;m`&ai8qVSh%bq6i)V-{MF&JfL^|O`;cy{S_&_j1z!%))3;4@;CEWRtrz_&jWxr$< z0=IS?#4;w(W)Q9X zWwaKzuv<; zs;RBXsdlR_tFlx-sJ&UcwAQuuTFtMTX7I~eSvjoyZrQf7$>mGSZKVfc%(s^GE1g-I zR-!EFSA4C+r>v!XNrk4OPgTEa=enRKym?nUrE8Stpz)abf^9q=YhOw-vHOez+(3v{ zDkWcKCl#9%p$<qoh3f&~0^2>_BDwe}F#c)_LO_dFnbyMUj|EPpc-<*y) z>QyOGi9a11KQ>r#X`m33PjOM_DqZl$HF{6j5Cfs z2^0~NFfq~>H(K_XUKm)AlNi@^yW?l;_$FaPRNb1|u9^?k_iJ)$_kgp>#0p*Up#rBe zP06vMy=4z7kP2C;Pig;>Nk!sftduC_m2xYD+k3(w1j*rE47iMMtnMW{ zVNZTr-J4qf4liSX?G906kD`s^^yB|jO5L(u_xfl8F9!b%;f4+iBkml%? zO&v`2+0OOqzMAE_>xLFfguMruPTj^D&kGk!60Mhr<^QOZ(ffsWmv^<- zX)mMaP0t&isC%00d*{oJ4WJO(CU=mm7U9Cn!iD_j+}Dsrmk5~yo!nTimVJ?v$NA2a z3YH2AMQVwUa*<=5;{a#W`IO6M_jV79=R+?HvWjH>p9As(TmxeRy9cEQsRQBzeEnLw zt@OR>)9e%Bo#q+gkq%dy->T*EGRYKCtne!TGmpcqW-g^a!`#qVa<~11wZ$?G*vAT!*0JLfiP(sV!|y zPn+Ev&(s?m`qVYmrq`XX)>hMN4p;sxla#Nku$Hn)E>vWc%r3rO_NVCY!l?XLIijrO z+*L(>1t%b*kzM?#=zHGs+#gvtekW#*$&UY1lvkGXPtMC6b0S&;a< z=&1CZVvuWt@3PSC(Q$p}^tl(+5s?)!pvRpk*BDW6bDuqZH}o19wYGZzsEK3!Fkik; zp1a5?P5DQ@P{xpEiXsJxyg=?fRxOQ*e!yqq3y>1(F8V{78?_gD7a0yUth27N{UmMJ zH+m>70gXpy5xeaV!A*UnjbWD%*Wj7mJkan__fA^`QTL1Lo9g7&aV=Auvg$|Gf2_%? zFR9&Kb+A%ce7&e7KeC{_#I0;_bz)_G)w-I`#V7uh72MCCm3O9UUH!R=3k`vF_sain z++DT0qNB;N928FYx2hxgk}q~Ia%9Fb=DSC&sf2FXE?3jW2(n1*{h^WoTvE} zMd>0Z1<&?Z5Jn>m6!&Eezd$;~?t|`zzagaswKKyaZ48J*+IT$qDu7bieH$?DpOz z(oyKx%_Ya9+|MIO6|}?uu#b}m*Tu!rMX>|sN%v&#@^Go0uL3Q$ght2Apw(Vv{b>#xqsZPpA$uG%HNi5<)!fNgf)^%D2#ei_pv6R`A zeUyu6B9elxL<-4sL?Cbim_QUhOY|eI5}kNEP(vNbjYMC2q3sRqH4lfV%T(({+YIY2 zOS8eDE4DqpIjMPa`{p*ZJ-`s4X>03j)>c0(KTzgb*{kMu)8OXX#_*=vdS+!<30~Y$ zC2J09+X*AXfcm}V(bb2_PE}UdoPY?_%dUmmhxPqxs>==)y(o<%Jbvb@c!#diuJ&ugQlqGE?(1 z#^>(NJ)ZOa_uiZ}IS&dLTE5_b{DA9N!A2vmIi=xzTR#0rH)X`Y!TJ%i`+G+&b^O6R zNt?h*baf1S*T-|%#sO!$wMb?#Uz*)4e~2~MJytH4&su|xMTAyar)SfMR(aPiHAlzP zJa5~syHjroP}rAc$ zoH_VGe*P=FSv^ar8(W2V2{PM>1z)Dud=FEihOSvC^D)SB%W)zI8 zL@ST~5tQpn?Z5VC+)k-aJyH@<(Ng`p^+JP7jd$h9@_QYNC@-jCFoUH~DyXHlpA3a7 z-EDP1@5u77_UO{EBzM$ng6j^qHR0^&cfE}Lz6S_A1D$t@G=d=~OlUyOJ=Dfsa zA>GWUJBMq(>s2O`eHSst_R5%}ziQ02O+dC1-%%wik8@67;pOvUWv^66oo2Y7^0a!g zA-9d@De)3{5BI(5e?H`7a1WoAo)=uZ97Z}^P^^*9Q9TFsx0~!H-;G&@=He7!PX7)0 zYqcZ?R_r38lIBh~VSjN?(u3Gj*cLiPyhJ|HF~(iz8y)b~i`MOlyV5<+d%o|}?&l-( zqWq(+5m!Ub__TXDdMZ7Zcsq0(>mShVtq0*ePf{&-z#hoD&+fyV#nLmXslO>sWS7;? zq&6~)<4wt?^`;3Hm3bieE7Y1-f!o%6vj`l!Kbw7lRQv(eQ50w~<*A)RdXrXs9g<@| z20ZE=WR#t4V_5@@ZTfCqJGvfsK5e14r?f9>`_rZEJ~qs%zftj}F|BQV6Bk;RPg_bWrDIZE zamSCg%}ryC#rnsc*SoHEG&N>5Uu}KdIad?X>e2Q}bEKuSmeUpkWB0%=SM4X=(a!Q3 zv|890+c>=0Nux9qrbf*VZCLx@+F1=l>iF$p)>LB{kxM^hb2O%@<3RsX-1SOFB?7p) zjP7(lM52Arx~}a>U0vHbP&YIIE26h46FeOI*3;WBw2tlEYN|J{wRPgV&1_@3d8qNH zVW4rDu^VL_K+A;U0iqMaslt)M-=Y%fO2sZGse7?!ziw;1(W?f$`Kh(oRXv}hM+Df=rc7kEAInNOK( z=uGMk@{|3N^|LwN=59M-TL=n)=L7{`XYWOX+MVpVq!x9@UQyX}xU8^uFsCt7SmQtu zFqy4pZD1{9rEx+yC%J(vJ2e~}iOwS;ay9W2KVbK@4KbE$b)9?K;#!Bb{AgL)DrmdV z)~$JKgQ;?I`GU%SE4S7@X-sHtY34RBsQXmgtuDJNvYFL!yF;&i)7eey+4ZXBNM~w? zx-&-iuX&kusp+e(Lffc4*J{)@Y2EZ)Mz^lx#u@6QmaYz-u2}1?6KjInow{o3SuL@x zlbbhaX6X|_0~ZNC@3Qu8&6u{qz%tibJ!}@dNq4{9M=jKsSq=8?;9OjX496%$m*Iw` z%M@xiFf&-&+3Pu8R6q4heJ@}-7)&dyJ8c}> zHfx4uq+ybNxnYT+$$Z&z&|jm2>9EZL-#5h77ra7z&kHQJ85+6?Vl^H-C|l!6VlW7;Uo zQcaU~l!2l-U|ni$HTfet>uSRq<38PG<4_Z@8PMH82#iC8)`7-gjiGCW)oe?*jR8-C zDTIf8rt!Xhx@L_g%sd`QOQ(@%w!Vg&x+O4wUaQktM-UOjKa^_g3Vn(`xbvs_u(`o- z5xqgvp%$c{VX48>;G}_IKeY`#&QGP?hxyw^>pa}U)<{fcu4M1y4;LKch}eC|*+d#P zllC|JI^R_|RusWL%2+}ZpmxwTmU9~+>-aeT61ABaOnQR9O9I80{+4x#+5-`yeG#T9 z&;E^^N2FtNbUQ7E`pnu5|7ss?Iz;)3QK)lxDU=pUCz)m%0_>+KR)fuZ&}fwP++bQXT|XgBI%+^L5L8W1?-b{R^>*%0vc}pYZSIz2-hp-)*2PbHs*Y zf1|BPGae0DBDwe=$`a}cRvpACy}3hKD;UdJ3s{%98sQvCk}Mk{9EU)cF-1}+IwSi8b=33*;qs^)2J*EqWA^IIKuDsK3)n3sL)^E{YFrL)s>(g|rbO#J&+R3KVX0G{+ zZMh}a`pOz+S&KijJ+u!bKG_?AwR#6zM2$o1VN?L88uSOma-Lw1&{BxUCm{^H(oY$uqSQd$4pRL}qN}p;>GMv&5(@TwQ zph?|;p-m!VL3t#% z*4RIgHnbAx9=X8Yx{hYswvwShDOg0YP_{B^=ohJHsCN-P5eMu_$ik&Esqu{2^j`D; z+Fic~6X8#9qPl7+*052vj~=EA;&J0*;MkM@jyk>N;n!Oo&nk)yz@90c!5 zqfDd|)YqsPSp@vK#kL#vc=Qz6i_)E%j~*neZAYzNL2YQp1!yeQhuRO=U4iy-hV4L- zI|=^fi>VaaXv!Y@WEfBP!QNT{nuhTh7wAvWX=FOgZVwT+NI5l}*+Mr{mr`yL2|z{q zLHJU80ttu$WT|Q_in1GFqK=@Ab3m&oC^Z$CNRfiV$(iB`MBmBySRxgfPkyEPQ=T9v zDF=yE;*f0#o(fK@muVE5ijqmS6Ibz0)SYNXzG6!;@ZhGTV1Hx#@mO>eI7!4)FCpWw ze)M`QAI3!s^0#dgF&FhGa=^7P0-cR1iNDB|ww>mg_7+Py{s?yf9qcVTV(V+|Vd9(e zj0xuJ_U%?7tkYV}eXWO0o|ZGv4%oH>K-8q*=j{%}DZ(3BK}>)%T7>ZE{`4=5FAP`4 zT9yZM7rTv{#yQXLEj$geM2^q{{3cSx7sNv(hrz$8Uc6QEN~VQUfZF{}^4p)0{4HH8A7=Z{fp6zTl5 zd%CW4eb#tr7wF>*xrRFiEs%IZ)VsRU3{#AkEWVaJBVjyZ$T9UWueRuHW?MP_5KjiZ zw%Q&^2=OAs3!Q@nP-2iQ^c*o2l!#?$6g3|5hC_%P{263Wc%svh27-nIeZ}q$KNs8P z;zz9~?IHG4mH=z1snc@TwhrubYs_ZzJ#%+Mv}L`WYWoSTqPHd9)BrtTDiG-!ZAYwD z+YrdQI0QM?X3J@t-8zkw5Z~}2_Ti>WmT}fqJwXX3wpHN-(H|L3AA$Wx2(PoQ#bc0BxD&RLvI5JePDL2hX7Eo8qd9>Jb2s5m zjHcU|!`K`~5b^|6;uyIc9LxGLSI}M~o3UM#)0E-ZM4$!ZD38Ws{X;uMO(v({c0_{| zV2`PnVEuXoRPycUQt~%=Zmq@oQnhF-`Wvnd6Di@~3QeNF$>o$5h;%u?PFxbYgHlBO zP8Fgwv=lhEJ>i-@mu9C^AMIkN@oz53yy~71G-tfrsH? z=r3wI^&agcO4x%TFDA#Hg*yThF$B2^#J6jR8Q6N>@Y+k@i?&kSv3=wO`yG<7&$ahM zyT~7uACwHTgs|F{;%s6ddXh4R>;-G9W5C^9PC10FLVu7WY>UmMmW6l`-VF1oZFV>O zob|S4G!SWj;iDnHbh_QqK8rXABtzKpfmXz_55$kc*}X+>gC5obV|_Yk*?&;_Q5qp@ z^C2n3*w}UIAmCnLl(p0f+Bs}BB^K67Ol&kGk+YXGkv<;shd&Tg&@c2Nx{fxHb{A`+ zZl>+RzF-bOGaN^8LeC*%iH+oYh|!2q1c=$izf=(1zFCoW?7CaqE!*5zK&@s~S`Os#x$Wmet@|4iq ze^HW2jr|o;jvoY8S_e5FqGFkd78yb;1bRX`l26g$W6_^TjE!oWY9)w5$eo#nUL!E@ z&sanpM2|sq;S|0B@}J_#Fz`5h#DB|*HI_T0zI-6T?LHz{peEIXa7j;L~jAvo zH-n>mA(2HqA|4ZsL<(GcE)l~BJ18a#a4FFZ)X4_?Iv$TdhPwZZC&2aWJ#Z5D;Fs}# z;MoiO8U8OGi?0J}XBj?$7zO`f!P9Vrm_eK)GKgB@74eejf>tyb=;Zy$iSRQP9PCYG ze;`ylBjv#7u7?`z0j+H$=o|Wjw+sq8XjiaHs0X@bJedyM^c31ErFTOWGH_xAZ`x;N_Pyn8Zi+aFvRQNELX!(04@Cky!Q!Uvfm;f zf+N)v@Izb*toDuMe)0i%nmh!L55jQ;&d3iqUcz_23H? z%yBC8wLti!KA`X)`G1=DTx1!PaVL~54%$;3tX{4F%li@X9In0ZAq%bqsYjZDQA9zx zs05-@L(uW)WbiXviY`Xip}WvH^afn-;^FoLeTcqB-vR;d7y1pH&OXC4(6gfn@cLyq zfH?|(zXx58PC$pl`wRhnNM-KxBkF2?+aw}|7hxC z!G&lRyvu)-^yzTF7+DClFdvQu@OUwtiIq@iu~0)h;1jn%J*|ghHQctrE4z^c@bkaN zd!Y6f!{07}{IfamzyH1C1Sr=C_>{r$X#=1<{{L@{TsY%4xP~@CZI(i-{7SxpdOHs_ zcLM5eGr0!Z=L~2;RLyS}go#$Nzt`30{qb zGX3{IN1#;up}%f|-|U0uC*U{=zuOMCICyjc943#zvjy-M2jR2sLXSI20_Pn1#(!(` z|GvuyIJ5iVb`j2T3Y7T+e4krzWWZ>E!Wt+SzF7s7yMpxoe_wNi*W92l^npIu7w!Y0 zwcLdss)CvviIkED5D9u4>N^CV2HFD$sU>F# zqZ4|;ap<`(z;W(hL;(Gb0;Spje_@7CzV)9I6STYMhzdQ1CO}JNqCKH(!{BiO%v2vh z-wsFKg74XB7{6ViqzT9i$iG{QgrO&3)c=K`WFR;Qy#qJZ1auDafIJHCTtIS=&5+07 zK^7qu$Qkqz@)2t5B|(tS;0pDYbc6MHD|~YXIh!&Yq9D)VOn9Q#NE;B)$H3Vh3gd1f z0+A8)D!!fAhCCskQqIs;0!8~RdBA?yo`D_&+A)sJz#557HfO7|bu&5~R%W%de<;oN z7?ZE{Hz~p%(OVfqsUo||zTMW{!p2JIgK0Zy=g1p`!T!{I5xGjsf=Kir>;mphL{I{V zpX7dQ9Xg+!Y3HDQsiD|&SU+vBd?6kH1!)n*2y@J7NH6>gWj*5~DYus)4Z!HaVD0B( z#lZFcHt366k>0id>mbBHyNs@(>|>W&5UmHa`@KjXP{Gpd=S@XsXG6Yu4L_PUn{t{p z-+sU2lwmkyA&g{eIVMU6cFcZ3_t0JoSBffznso~tHrw!X;9J8I%|`QV$C+oz1C}$k zQ??>%J7ocWi&2UGV_t@cVLpF^l7Q=pYV0Ncir7LKLQtu{s8sVY`#RJWc})LCi6J*Z z)V;>i4;LW|Xb+K0y9}kH3QTQxBsOB*t&?m6z-c-hEjLzK3XM;UcIs(fHa}5Rrz{kV zC$Ag$#?#1c<|0t*%w$$u3)NG#zs%K?<)SFwG`mf=2mCg!;=7rT_~X$}<|n3eO$p|4 zTntvjHs%WAfCV>*4a}~ymLjRIa4eF+p07Pu@DK^$&M*y-^Bm8alDa;TpRFAAb<}}Z zOL{Xeu;a~7EiWkpX*{y8or)c@6v`9Nn z)6W)5?ad!3dCpu)WNQz1Jf61?~^h7j$Jz+I87_XblZ9>iwMwe!?DUNAl^`UZT zEwm|yeN9J=-5DdW4~)x_owO19F}iH+T5TWXoL~`uJ&&Y5($3VonjUNS+D{9s#Cz!& z#nCR+Z0XXQ2T&KWk8%8XiB!3srQL<2=tL}t&A>)8huS-v&Qmc(4&K|q;|!(7w?|lt z1q-++{($k0?%#e%lTR7WVF|i(7jok4Q`?+1gOO(H6~=p7qCJB!;@3>hlol4(TBecG z-wDnjHI^UXWk<0rX5JFrBt}q&Q~v7U+sE>AsdURe>;Uzab}aD;`%U~yt;JG-pZ$wc zOd_Nd8)6RcnoVIze&Uxb(-lkXjg66HHvNiDWF(yrb3+>I@$DQ%r^Y;4zLR!_x(b_C zH=^r2_aZsoo+BE^?W@0pU#IP~RhvH;45&>qh&u^eCfH$qUzUP(Fmmem*|td5GlpV= zgj~MR(A+*sb3i|dx{#L!u1{Z4y0w=f(r9X3sCjR@f(>>)shDc}q-*JjuN$YPi6R{T zm2Q>HM6b2Z1&RELwnXB*e4yhUMJg-H>eVT0Jk@!MTqjy2ea|y;V(eKp;~Hbl1CX8E z2vLpPUGUlL*Pd6mwC1&bFmJq4jKO|gG- zjnEL_=e}9)AxzbtDqYl&qtECFMO&R`JG6_oGF0d)lT&tb8N^Fj1E_H#Nocd;zCNN| zUvZ$~ra@^er?`f_4=zGa!z?j2PxNcDZnj{w)7+5n&Tnk<+g8=iDxcW#fwfcR=`u{Q z52IFHE`NY6V3cWcq{W^d=G#sGbVgf8v)@U0Tq#kcGk1C!T}X+j(e*2LuX$jrV@YO* zNSf;kJY`Xvc}Ama?Tz|>EGvz7MWg$09an0mc5dkiZ~LIRtzKsj@_6gA)Yd@ca@w_# z#bNE~%R3-uiHxo`|?Tl@BwWZOLU zb8aSY8tZu%U7KTT(Ucfhal7$isZQck^rWt#$ay6Go1ZlK?gZ6i7u4dNt1 z*RrrVoA@os;f=z^;w#vm^a*8iTTqq6p-8le^HSdZ+!c^C#vcshuIJp^QAaQ)tzB88an!Eeasit?2hr&@3JY1ScGnP zZ4q=Wu*|@oGkd`vD5BU!J}8)}wR&BZ{@G1L>noO6tA(I;GK}nQWqGh zypz^B>QdSZ)id%+-jM2gry7?9dQa|{fN0ClU*(-A7!i5~;R{zNJNGWdhc-;2G%qjS z+}fX&$c+*DxO^AwZSZc`3k=P*RaLb=93KR(wLh?JgI43#aR)0QW5I#sG_$w3J7)ub z9qox$+L7I{+HjoHON>zJC<7V&+XmMMaGI2b)|SpX=0NdFve@cIe@U!siS3f|jp9wF zOHE$v4%hV*6c6RDwt5yPR1dI)(UN#?XDSBFXO52@F1068k$NWjZeygoap70-+PhQ{X-}TOh+bx{V71}S9 z!%_z4yg8q?PW4!8slRR$AP4DVDeq}Z_{ZrjMxJ3Kqlg}?MNO9}8vWX?wY075TFF=C zZ>y|17@x(fZ{1NK!;~I3_$xWLJf6v??g%l{xRo?sk7cq|Cvvm#m zA@rDkBz_6?(k@8;k!wKT3af?hf_C(f3AR@3&VAn5YTBv}VI%IdSgUD+6bSQF9F5WA1| zc^vNv^`grcfw^6*_pv1b;p2i~CfQ$9qENZ)7sa;o3b*NN89Dm3EPqLp=oSaj#hc$y zbf#Cfx!lnlPx~o!6XM^ITEAGo+f(7vZcG#~lm}X-sdds*6pywDBf`P)qcwk8I-N%Z za;;z0Qxw&llKk~`r7pANpG>v*URWc#bxmpi%?agC!xl4B7%jGEx*<4=-kp47$T3IK zeh__-FqO(-4(F-;bW?2SPD`gnf?DyX>_3v}lv1pXYhevB=hv(;X0j8FgUCo}CB2w2 zM10oJqk>e=lgD!2>wWnd4vzFl<9zEz+6BYBmg(kb>@F`|^-z3UKelDEDZ@01%%#Ur zD|q(>2RoM6-LaX`Uu?D{P%xA>M3bX_%Q`L|p`NYX$zQFFYv0IcOCQm9aO7g7@nYs~ zb0-f+2Pv)unt9pUWL;bFw94^7cp1%bb8L4}34;)unOV;+|J5~$<}NtEE(Jg7JHl1` z1nXC0qp?Qkqbah`khSKybd`K4ZzW?ncfS2XTabEOM_uPoG*>>+O{tP$2h=;;7HVFZ z`kB<`1BN+WlU0Tuc8?scE6t+$^YhBj9qk)Lbg#u;a+kA=D8sOp9cnS`d|cuVltk0# z+uOSSN7GrrX;nRa{KgYoEG!+1G)jXY0!m2=(nv^2BOTHWf`oJ<4YHI-3X;+wNJ>gc zFR;rl8&BPMzjOEhe(vYq=ec$6$(i5GnVEAbag)(6bl*3|Jtb;<)}EPf`DXe4eSbPB zA$p;A-+NCWTjI|NwX!@dQ0RkG+20uLUtLM5XSGuEC}zUG*DFImVw;*-TjIIkeXrIC z1fCvEjm};p=9O_fu-9ESI3#6jrd~M{^z*Jp!L83Xzikx!Aj{>#g(JtkyZ(NH+tj-G zzsR*U|47e7t^B)T_Xqsj`Tfcq-xR!-HzCW;%zN~WcLSFWr@Yek`(t9hEBPu%MPHwk zFJAohY>hb)JJn*5qq1yI-|(tkAS`u2dZx%;?(1nsBYWkDi(2Jvmt6eukMGKQ#zw4; zx}rT&`+3`#gUl-WKws;W!*(Ih5ZyMOco#V^NA zx;3(O{!7_bX!(`9+B>6Vp!J&;ug<(VmHH@FD_ki&uRD8G?o7*rtKZ~Kn)trUyE6%1 zB{|2O$Q)|Jh+bj!(9f2LveI|HjPnh0b#~2W{`QVm%ByL)g12*&%v3P_U1EG-V#M;~ zNza-_ZOpna?Z2?qh0bel5=xrWQx+xkb??lbE1M^^cEUGlH{Z54Zs)j|)9+35Y!2TP zihY;&MYq>Kg<{RB!BnGEM7QieX1VE^8K4@VEN9^n~%pBc_Z!JBC>w5ECE@)e0LZu;Jzj^FeoI@Qz;y4DlwrghZN z(9Gmd@oXuH-u!Y}?nifCEDgKDEcKDRZ=)Xt%i0sQ)o;c>OiMkiJqkr-+mN$s@M=nX zKfz!_Xadmq^OkZ=R|9PHW z5ue7te_8s!>}d^iwafl-uA2q_44K9(U)ECYMy>M}39tRZU(vIZ+9j_`sc)TjXAhK$ z9+3Z#UOL#-x*z}7%OAt{XZ|GeglCK1#G3c^wf%SYqTXLas&e0ROZiI6=Z{JG%#2FE zVl>LML7Qq<$ag%}Wlw(lAZc6R$9E-^2ia2flAb)#6~lfBf24kqpeHK9otYvtx6&SX zn`B82e;$14&lJ#73)^|3>V)ku9{T&(Wdc!YpGL3GHUdwMl46vR}z4o%zgXhNTutc*|)a|UT0<|?OvugkB+x329dUkIE z_2cxJVdb;m@nrLMP(M!j)OR4_dc+(v*3M6awKQJ^e~wT|bw!rm5yKO!1uA6y(7V|x z?7E?s3*AY&|9X_Cc#g_eHno>$lh2cO-5L|I-`?vU9KGoI)mO!`2Qu|cy6GMhzAzZ> zsi=IPTqKY=d~0}Nt6fsAv|n=O&(=4%)#zt*^^MbOMYVAc_0G!L(|taE=(}uIjBzn- zk1tbNvK`3#MbYfJ21e}kKY6k9zb3DTzJF#^&N8e}vz%q3@<)tF>h<*P>n(3jCmqo% zV3Af`PyzlHE8~iK6ANwHtu`GW_-ATH9$A5qP%d}8YVij%l z&d+f?>Plkvm$lPcrk+jR$K2Bo?xEp-W}2#3#cJnC>WCzF@*cHJ&hN90jM?w)_-@0U zMlU8>-|DeBpBAm1sX^#oN__mj7uK6Yfd%TYuuc){UFWmz%DKDy=T1|;?k zyp5cdseYCj(Ivx98khZg^2QfMeB~m>dS1s2$#&g0?)BdR%{A`*_~bk0is0*%SL)Ue zCgqRQm%f|wAo0b=X~X@I?wmPBrs=Wiu{;*!Aj zq%YoY$ZJ%T+oaI<( ztF~9^mDD`&F4K3OWbZS#|6RF+*1_H0g^_P##%2#wcD_Hs8rNsjN~1A&Qj~-_HV?ELn zQg)=O{(6De%G&VB*_!D7kYazAd_KNW%BP-Ry~`ua?1i)RcDM1&R!Ss2!dGc}(poKN zcn+dl7V<>vnL_>5nGu1su_?DbNuiS|6U+*ps`g@IpteXmMabkL;eTn})dJ3%Mp6D! z!DY6hSMW57yr2iv;bGTIUwRiKPhfA_d}FS%A-sL2CvIQn#W^ls7V*-Ck@xjkB6_lDr%H!IUyd5(v*(C@jvj`}>Tpt{mKOljmF z?Ef^)`|fe*Klf{IrEC>4uVQ8?RT-AHB4vGg+&c_q^?4Efa&*eKJ6HAa*~-Si&4lLh z%aVTe4NwCAYMk;_($@FCCDjY0Mg_B8icIqQl!ECU@c_A$@~2fa zJlWkzuNm21J(IE}Wk}lGv~s~?p^BmF`q+pWSzVcmC?R!>nkD_$#L-x7)=x`RwQTdV zPsUWRJo-1d8EUmJP2u}$#OEcabdb=z6XSGn~{+Sxa z%*gj!)@AxW^Hp-ggkJ)^Jhk0ZiM77oHH5jbpjIz%Htj~}gmp*#({)7sUi(l2z~6$6*1dwVk=xI-A?*8tx@Xuw0-6}))hSN zen^D;(t2IxJL6bjn6FD(dg}0ClAT%G>%JO3%so%t?b6g2p_9g3bWQPr<8~9*4`JU& zj0peCGr@h5;3K+r-nGGfSAXZq>rPa2yQX+dG?Odb=U9X2dHCtbo!(`{zTo)VTja$ID?u>S6$uHTvAEP!^sJ98>6v2A{*b#(Xj^bdRNT}LdI`MSkg9!JzC z`ZmuE_byKxZ)pRDPMVTLbp#G%2(~@+bw#QzA zeL<(-X@AE+T>6Rha=~ANC5*P{viq8QjK2bb)X4YWChbiwpLRBI&TNRqRCcR~`Agto z+Od=`Q!1yL>7znh%p9zv-NyhY`_uh>d<%SGp{JqcjM;}uvTfQCcAAkZxFm2t6m3nh z4=Yu)c(mwImd%=W4+GtUON>s|9J{k!SDmC>vfiuXm1<@IW2*60V2(dN_+jvA=$hG2 zIi^+7`?`0j%j~7rrr^;)Dq2t%`uP9U-Fg#!g_={Xp}MgRx*GZ^*ua=z7uD|SiF#L0 z70&|ZvY(m#LLL0NuZYj!my0 zeGq%&U(71O+rIUFHM9fS_NuF>H-~qHdox}HVXA6g4-{Z#{)*LJ$$}M%J3Q!J;oa&v z$a=WL?IdP+dzo99LAY+u(<>2g?1C#?cZ@aqE zM&Eio`a5lrwVyg!5U623R#f#1&kS$ju+P0A-FAO#4>KN_-vb6;`w|K`rANtW9K1s=Df2>?%xiUub2hpW!wOG7ouNX|2y9=KTo! ziqXVaY|Ia}uzLi*XU(%hp-0wMuF2Pi(EftB(joRZe=PKqb76C5%x%HqtzLoobg6utdS}Q`vvXL z5%U}7wHnxK%*R+etO<t5wkxe}+IPN4Vt+lc3)vHyzy8=~`#fl|%ckW|uIWqm zwZWV0QzPEaX_=w1Wqdv0wt(^6gK!Ujbm ziAOXFuN{8Z`@OrTyP>B`So4VQGnv^A=02DAdH$~X9^?sR>lA$>Y6U*aXWaulH^ZAp zY>oao=5Ce}Sq^5(A0DBvH{SVE)61vrNIT?j6WnA>wC>s!)c`Z=?alCDxc}euNxt2_ zpNKCQQlngkyGwY(@JZpTBg#aa$9L~g#K(~P3%!fo)ogVWRpJYAD-RQB= zhgc(|NHo6kQB#N?(K37+YclP1Z+GQ%S9eu+RmDoDozmBA96IT@(i^8uPTPZ>+Wqt) z=_h<=(hH@xP2c^#*!$;+RTGCKmP)viygp@tPsbuT*6wa(GiC zXfWB|7%SCszC-?1!Hz~fYp(LE=GVV=t#s{#s>a!4&G^uh;E7-v?A@wZ9hIEgHWs&; z&zb;{%0c5OD~MiC??B9}i@^$5Q8mLF#H(#lc3Z8?$WW)iXkR=3HGfi|fe|o|Fry!X zb?`HFuu{@`$~<15;DF#QW42kx#@a{k;?C+i?E2K*1K-iF?B>>Rqh7FWD4+4t_y&uf ziTW2zaOMv%{FnJcbR{ms%18a z&7c}OXKV~swEtms#HDsItD(8WSYm7s{bTG4&ca_PHZ;PnXEtG$dtJy-+L&w2C#-)0bB#%?I8oU88|%S5$}YVRu@^UKnXE_l-oQh%yFWgCUvPGs zFZ9?~#z-aRUQ2D9y+ci9w66<2O+W54Qf{YykiI8vQ*a$$`~vD0=6#6N}!5Ve}?YWlSRm?NgJ0>i5c;4_RA~x3Xyk-Tp9@tO~z_ue%ouT=(PI^B* zQupY8Vp+6Dd0{_R`q|CY3hF5}Qs1DS#SX8EwneQ1ZbAX9j9?!9xi()Z?)p@Vhr?~q zim>`p9(;^SC`+v;*s?sd9N@5v<1XaR8RpXKx_i0)R@=EpE4}qUT_x1+p8mSu{fj43Pxq|AO81ub zpK@DW$x321vAjEH{YwnVcKnfgQOJ^hNOP2}3lBeUJeJ~hXd9EY-g zl{tGy{2%on?YOIetCT)hYik{5HT#kHDqKjKlk_qvEvZh*+_Vq< zt%C##wTVKtC2lX`BwYSbrFqH*NBOc<)TVN<%udsESK96gCn|x z_X5LJ^p;&V_usDN?s@J- zo*z96iCMGQTQaPL_q=DXdy31{E~|qS0~@XPreU5k^IH?J@_KBoweDDdS(~i+)+bnW z{chUCu)SvOXML+3s-e1A$!{Pra2C@q+IrGFqdea^YXl$g!UBMqh?j(R?q{d{wUZhST*=7UQ+{Ex2aXIPOuC}v8DfM`as_( z-><&1fmwl}p{)kBiY;S%iR$N z)%I2Qt@Wq*cZQ0Xc`cjhhf%8Ej;B^ug!TpFgB^ntf_`JCk=q`v{--?98xn)!XSEl# zv)9-eYHRE>zqMl3joJ#Wjr*4Vt@e?Y%j{;@fmP|}15bT}LLO|(-nlNhntLv~E7JqV zn;SzbiKh729}(DSCfn2XMc62R7dF{*ofTTM+g?k>2Kx$rWO~n1u z7De@~x#lQZs|9P4{h$Q(LHf69DeXTa+^QYw>6;(w9~f`+v6jMXd#c5tMANaPS;Jbi zrH#C3I-`ums;REjmxtAMM}*yQHP?r^I$C4YZCJ;RQBAu#u~iGXBfa6?h3-B2W92C8 zC-w+#v|fiQTiA1%**ve5o7x4}R%@dEUucby-NG-zTxk&7Q|+Y|a+meoc6r=x_Z6(8 zFChhVwZFuI@C&spwKCHEz;zsI(?ETuFSP5LdF`vlw^lBDg0)#I>B9elHEIjFH@JH# zH}uiUD(wANSzl-y)Un!d)*&C~Vlhf0a~{&>*}as_b}w_PdfMEpt-{i7m-}OTo8HU1 zXO^}1`>&ccgIA1+AhZkme`;@i4Qsgfa>c>-&M70Qy`f4W{0V+{)z-3yg?pE}PKDRe znlb`&8;4!%jjFa!``P%|+rv)OTY84t37!OS-&b~)z$|;d{~YfxHGkJu*jMzwSTk{f zy35XDEoOzuZ$gEw3dY~c2CNibdQCmXJ1{Ju-3VLDT3-LSx|`o?zbQfMmaCU~L2v3_ zt?!{nOwm^8m035c0c|)){aDx3_gZCrlKNa54@U^sO4wJG{c0oor1l4^{r#ldN+~@| zzoy>B-$GTV!NJ?Bf4WoXBYnKZv7|1m4QDKTW&ULy2qhY4LbJ@KW`EnS=1>}X&S{sdE@n4N@s7)P-gW+=xy3q^YZ)nz?Q%b-+DVNG#$&ot~TrE zVg-HF7>!);o4=h=$zLA2rkTCfQPu&yzfwzg={uBF`ar9pM#KfXy%uAkV^!J{+vU7f zOzR2_T59z(+8bGo%V;Hz7{3_j?e->Ez+O+ZiGhsfKZE00ZDnP8r-1FB9Xw(tn#-`Q zoTHc3Poia7tmeb|upRQ|0;Rn*U)xIUmhtq^M~78pCDOj$U-e7gHu?;APSZ%E@g;4Bs?#uRd zEm{53h|wDxAL50v&;rM+*1vvY)&V^L?QHAe?Ff?7OpeFhY8}Vr^QjhpAr4$d`#(GOv8xq$dr2lPvZ?3cA z(l3M-_|B!DGjjMlf@QiIy_5lFwDH9J&|fTgATZ0fE|e=IwCQK1C9A%+QETIMHaSq& z*VSJ*^=#m|KYQqqn$_;>E(%Jx;eM=CG!uAksSY&Kb@&_Mk?eWe@*Q%Hu-OJ50*ti}DHn;BtwuDX@Z9}!J_B&wN>k;*hPC3G73y14zB zxtwUFdqN|vL5y+i^+NqY(pNxD>+sFkYGgAjhZfmej97cGf}enzgiZFh>Ug68J|9Cv zxs@`;NNcjP&0MP;L;G@D{ho2^H(uaJde48=pU+>xUq4Wg=)MQcT;@~k%ul167$2%* z4EKK=yyeT``yudS+RV_lfM&$m0b`!l5MPEo>K|sL+RB`6&oGM^jjZdz{^;ly80VQ| zOn}CFiKsV@eFFB^cX;ZGvD#Q=)en7TM}hKtXqlA@dLiQBTSR5+fT!6*tor{@rrFh% zfcm%c5!C;k>ki1JtHHMI4vI(AKWH)T7zw12GmYW|WwkM;uGO+F zdnWwpf-8sqO3UN^UVE*5qSe7ZT2}*RGwmUIkL+HTQZekftF)d!a-m%+qC9$+so|H+ zA3Zg-{))%jS^31>kVqr{dNlVi*AJe%+CZJi|MpttHU2tJjq6zczp$dzIgE`m%48@? zbr4n)<8LJGH$+vQZxkoe{6M?E*;x6}7-r5j>iAoS#u0NXhdcit zRr$!ihQJir+T`yVdKb7Em}86#1d)%w3FZx*#WJ%_=rgl79+|7`qxhXX zGo}Yyg(?J2`Dd`l1%C<+FoxQxRw)&_LmDqjfBe%`~H6plD#7Z@({ppipQi{c;Ji=q&dN^w>qc$MCa?RELF5n`7`k z&w~6pPwT6#b0vF^x`W=Mcvf!Hd#mTIzP4c$VJ(x}fl7F}H8E007U!n|THEJNCp0*N|U3PEp8rKfbpJ5f$ey&_rztE579shN(^QUS7@8_Os z;hu9l`D&gC3 zNWG%9QD@i%&BLLA!BhU$svmf=dp{sH|8>2p)&)(QbBKxJg^(skIhAtoe3DI~EIYgOyR1 z;OW!KzGS?`qvluI@k{lv>$zvJr$3Jh*EXR(rZ4?Ri}rc>6cwXQR8(0Jp$sTI+LsMF(QOpcN9mgO1do|7iM|>S<-WsQVFfJ_H>2a~NY-4rsO5H*$GiE4o|m@! z6WO&OKESJu^P#CmN3)4FLRDCYIRV~UMO|W5HqKJoTcKg*IW)|v_5j6H*W-87NAHC{ z|4>rUp(C+oD{}yPi2-=n&cL_o3p<~dq83N<@z@@w1kDYUds^^C@cUp|AU3pwuXVPn z+Rf00jm5j^Fs=E4T@n4`O}r}aB0pEhC#910*r;O0niavX&5WA%1o-+n&u^Xt&n3Nu zvd>(Lr){zH-RaE(Q;j=z7qy$nIOuj-u~xHcbPwJNwhfMF4U4|^b2TSxj2zQysNdLg z%wwUY%!7Rwa+~+DTd1Yvz+)qay?{9ezj4kOVMZET%~w`oWhZE*oZeoWfN#h)JO_Ij zz~ll=Ao#L@R^UzUn=7gzKKZh3Flh;cMBN)!17xTASmof(b^LMcW6eh0>zq3zV(saL?A*Wt3WjhUhD z<_GZFG03_%)Qj}|vRYHt8cNo)>bvj+i?%ixbMdyD#5~bf?TYIMDDgA3E#4~^(8RU1 zehn=SQudy`-b~>cPgut zq$-20zgb;tv2hda%mCv8Jf{cv?2D!;Jpm@GDk3<(}G7 zn}xi4%{s?ya&KDgY$%J_4^8t8#_4Rd*I#0*@iC=7%z6_s#s+5CwkS{VnR}@o!8_@) zeTiKDVU9(wvxM(_s(hh-N*;>Y7tL^UB=q1KG;@VI0eN7rc2u3JEXLQhDsRLY2aPg# zPG4lE>WXp>-=(kc*G~i^>;WUahqH_&a>N4Wug)2_jQ(axs}Z`f1J+IJyj=q?P+c8E zTaL0H;dgeOs}t=g@Yf-DPH4^G` z*V<~nHijD|jem@u_{f;VzS*v9VU#B0y_I4W1$D2%_pB%WI&o-zC!t^7pzO26%4nK48NJ&N_^W=!DD&XkrYZm6 zY3WhBsIh7${IGH>o$(53j_>O+X!}&9AbcZ=^?_b8&P}T`b#hj%PW(TC& zrbgN-)6{=x9r4EOt!7o%;6c`a70+{l8s;(c@Ls8nCO}gc+N+7288A!PpP`f41eYG= z^5K=9Me7eXtj^5A6nl`pntCqhcxP@;c;2pM{k^7&zP|dG)0F5`dW0u3H8Tbj>}#>4^GURPS3-kIn# zDcS%nKlNx>f0~=kL*{0*uCtUm>K*8KQN~yZf4O${pLhX(gXeRCHNh@`7PyW``XH6D zlw~tB7DI@S(F9-1F#Jac!G|W}F?Jho;1c-XMl0uOy;gP-Xmh2t!b)Yn;X_uN?5N7< zj#OXKZvtS4Q;d$GjE__J{asX}h|92t`?sM{f8rNd7H!T*VsX?^|G{7OIUc$yd}|Q> zY?AT`-}{gnyM_jF5FYkBtjX4NyhLl_4LOt=UaY3mHvcdm^DR9sCw0_Jd4!&HGk(Xn zmG9L@Y8P#k){!2bLfsFe%w6yyybBJP&I*`i$k$)WNk&h!GKu~cZSS>WDQg@fu?Zft z&;z^_hf(sm#5+Dr&v{8)gJE!@m$ZU-80Un#9aWlxKSAK=+uLW#GFvv*}A#4+MO(Bh!`nTcoNVIj>XIJH|q2_IjKejhy~E0X|&J@wHdzq!}0ihiLdT@#_CX> z`yHOp1i#SYq`sT?=ioOfw4fv?IJc6I6e{9_xdGfTfoK||X^o0%F6uKMC76T{>KSN@ zLT(SE`$(of`cP+;m7Kg)i{B-*<49&%=cr%d1)X1QOb>pg9O1jmsHMYrLyIT=J^Djk zYH7UMo?5J{-lhIkMstLH9*UD-^`x%y!;e=o?j93Eq9N2i#vaFv$~C;%n_I>4CLN*t z&E5RcAMn3!ir@bl{FTqz-@-4aStUU%ORcxWpm@v35gzy{vsoY5SGZo8wVs<<_wi+& z23KuQi(bGlwI$=|DgL&7m=!gU)klGZPD1yt5D(y}b%w||26IKL@yji3p8!jEt=Z;W zB>&GyXS2-S;vqXpNyHyH29C9rxu1U4P*Pb*J3YmJ`5fNNx4`_Ztt#Mw+;FFsT%f1)P93HAm|@8V!8|Mx^BLKIw19TlxeM04(^@ zF3ul8U4DnNZNh(gBD{ha+)(ktymx|ppJojAMRR$~8pGFj5u;$Mc@^D8AD(>#bsq&k zj8f9+@!ybsBWsZP06u!yNQ9!SHS19ieej6Bi-&Yqc;-*mF_Q=y=#91`OYJhhf_ueL zf+lQTsmb+rQSeP`D;l1+#{APPN1Hsyg6K3BOquZ%K8u&L0sgsf_BG3zZu2wh(gm0N z6RBeg7ECL+zXI-6p7P$O*0$1`ONmkND=l(@NDA+X@>!quTLd-lN{zp@;)#fnmzq05 zUyr3ENzmCn^zN#pP{+>gaGbVI{oVj0?WZS;SGv^XMP||N;#*skturlA9NryAkNlnS zQ~|9133P3%wUGYsJ$>Od^7(fBoJ(R2lYsB{Kg5XW2ZyiEm^=tADF^5K63^%t)Ky`4 zlz3oA64zoKe(_#p547RrYB5m^O5mT}jy~H5@8=rOfa~!874*Np%`4qBM9wH3fW<(=5Q0{9)t0+vm=&zxNTX-)@&7*#ZC-+_`*Gn{c zUs0EBpyGE~MQSpf;}f+LoHr8B?ighi+P;&_FODG=!~{lQD*bIE{^Vt#75#`5(2Vat z!-A)q?V}G%q=}y>s|jjKx5q30!G!`)z*6{P?`Av|2HXBbNp<@u^8!a1Hx2NDFAY6! zM?{WEAo(weCRQFwyBodJF_5ZlwzmrK#g3$tjW`NlA$R_b7H=IjKb!HYTc6SLO(@%s z(2u4>Xc$b~g(B)n#@tEx>Sko^3iPIPaM7hu%T1txZt$K`)WHR6FsIUq+`QucAuERG zmxB8;LlK7Ip703O=D=wtI+y|(1l$@yNHKg=76qt;qF|bQ`F*raeBruzFZJ% zmGY_&%D91X(||jpp+>gS9XVhY5lpa)0|VCKJK;nKX~Fw%;C6kXoP*&nxzHLMrnTpA z{RimlR9bc@$YlY(^26B%&>mZfII)vEO~`dlIYstT=Mo)rm|{25#7nUbcX{52)z`O8AWXOQ5F0b2}4l zU<7B!>9Y^u21np_a~PrDk(Ua5!D1{vp}xn0p(R2{e(EqxVOD_jXOX+bJlTtuD*}g! zXH*vBYZJj2envK20G}>wK~*Y4E;pL z^G-)%_EW|()K+2o&s8YP5nBBa>(Ti6QeDbcm74gJ++>C~q!4q$f&!|PsUrFGF~%Q=;pw4YUIPKx$N7duVqThq4)`C^8K$%AIt$}>^IP_~YR-z@jI}cRcLM@71+>2U> zvpbQaFUZkBcuF6}P6c@MR`n(r`ETrJN~w?Oqg9=nDMKA~bz1i>u^mQ$H^ze>S}=>1 zm(bx@oq7K4Bu zsVvu|J>Hk@3i0+bXytlzK2xD&WkG%ypf6j< z*8rrE&TQl9v%m1pRmyt-+O&vmIAbp#{p>#WTU^b;DL#ZfeKb$s~l*lE3(BC_1aLPDv`52JdpTxJx zL@Hl0l1=iwn^fk5*Pd7|-mOn9_5o*=gCl=I{WO994kQm188_KUU&iM(mxv_IYVfSSJT(nOG@PxDoz>n2?HCGgtOK(80W6r0 zFH|FqrLEOn{;E%_^kYOO6&LEWgME|@C&%J_d_Y+UoqBOPm#wglv z8rSyI+VRj#3rYS2+-@6G_*bs3Ah)&1QIHz8s5dVqD8!e)B-K^CDLtYX^-_-7>_*SH z2aimIQe=kn6oC$RU}UytYY5#e0X0rX@8~6>3tAESOj=sN59;xY2~n5bMOpwHhi!X&lZNG2noeq-Q6y~t%Akh>pRexF>41XY@pJ2Ixe zhGX|(MD=3pz;oq6awVWV62a*+o+}AQ7F;lh*iakbibvry3*hAa$x}l{r3N=hCe^cW zlxNg#4x;pwgfcaQhIMnssvt{=m(>;fowAIGpO7#MP!@^CwHlsv8@cv($~7N4RT1u9 zn4G?*)+&&rt<=araNH2OrUd04{C61jRh81@gI^Wo`R>q^yRIUc&43`QYCVfmG5_PP{=Un*d9vQjW3NOr(2

ZmyB=7EzmblSQD-|Yg{FHY*mXs?Cz#ufB#k**~= zSrlz~10LOnQg&wa4TsXKfcIaeeoUVDjJ!4Ey94N9-O#IapjS4aHHy&p^Hb{Lw0dQ( zN~|u4vo;exASGT#={u8G1sQn_y0C$a+)b1xoHwh{&gIEd8SaPCqJ@!gLzLhcy=oW8 z<0)9MCGD~ZKJq)b;Sfe&NJbb- zM}N3SIp_2AJl;D`T~*_`mQJtxk$yOt9M7bF8Zh?Cfh{{w&Pwp1k@SVx^v(g)oWzW) zOMc5y+N{cJTIB-ziL>PO47Gk7Jx4q_cY}GhKnF{~nVLcuCH~fEo|NcaPiQ}h4|kIq ze?tF}$Z-dqGK_^@%%a2(81u*%weUWm#hQ9cbT1AHv3&Jc&s*yY=alC5Lf)ahL3ANXd|J6vbJw4<% zIC~dH(P~EJZdz_FU#-Jabsd@`^yh2(OL=5%p-my=ud1|26W(pgy)N{lo}5o|5VmNK zyKtu`rD)+$m!6bnC8P8nMWY=ECb~r|H+@Y1Td20)m4j)QA|27yf{M>{C>752ZOfT`qmJKvs38#SRtdeRT1r+4AJ9W^CmU=VrfOwX5? zglnnk7Sw1-+NUXPFp+j$O^S;cH9}DeKxuN*YZ~(W8a6?Z{b{4dyfcyWU&z-Mez#Iz z5;0OF4>y`?p;kV|w;vjKjFGg7zP5(ixl{5s-ubd310XUV`351Ym37dPg45_d3Gt(>`PEgF3?d$Mr3aXhkOG+Du*8R zJt!!}DugtnGvl8EDte3*qQn0M*;D~fHen1e=1USwuN7ypycGj>@SwB#7+#-=-|Th) z-pB=t&_SBz_)aGW(^UgSR0Kt!jpxgq;D1%X2lbJNKH}MYoR#3c;!unFXg^Am!oQ%^ ze?eB)Kz9YeDH5sh6P_)^GZLHdW415hq~*Xn(P$qMz<6GA9zh-ofh;~o16PG-gWx`c zaup+I6)ABE@Kz6U(U;$9e7_2*45D_{Krz-kcy>7TmodL{obi5(5g8B7*#>3$3Cc4G zL^g}MUPt{8r;eLbqn}e7ACtp*)X!OmQ=EjWj3%|+?8ivKWRzqDsa)b4yP;GMKs++$ zn?Pd=LP;fZq)4F^=vRa3hl4;P-5D*Sf2agEs73#5>0qF#+#ADtLI?UW4y32;K~_KL zP?k0H?Z4=gR~QfP7>N%VH)p^DarFH$q&u76IKD6vs@IhF2k~z9a!}oV^Zx4v!Hqz4> z2i2SeW5@_x%(oVR#`-Y6K81fZ<6A=eIzUf5Q2Ius-hizHlqecX90Sh^r(W+2A*vnD=)|96YcjUJ< ztzD13GTfv#U&;-K$;sE;@R}I*XpVWP8PP2G;rIEu>*Z=@&U5lCF;g5)l9@c@fPbc< zNhrYc(bQxl{jCD2e@LosW;T+kv$xO~!5H_bp*z&%BkJxLwReH-phI&Kxpoz-v>Wsw zu{iHCCSpMJDygQE#tp9Sp|0Rab zFx2iizbC;Ihq!Z|6m~-i<49|v6Twp=kM8Dsf`26TsKgyTLb=vEB{;#^KHfeKjyO$T zpHLsq$hAUknbbupWjV_e^4=Lzxx(`oNoylCdK&4^VV_3{e&RR@`KJ$bOAtzLc9|I% z;-Jk{_rI`DjdXw(oW(R9$x4BqVr-RuC(5=p-S6sIPX zyFO=CpqII!njb^;zCh-z4`r(gy(`OB82VTaD%X>Ie2X0WId8V$T}i1w-}%rnlz)?WcPiH;Htblg3LlU!4{`Q>j^rRw zW2e*87SejloIWv)C#4Q0tv-~mKl$v-wzW)qyDBisZOBgSJLmd(Vq`Eef>P+ z<|^YW6|~@E%syg_CDJ1w&_gvwwgsP&c~KW5UMOBpMtXj>SolW~#(zO5iTnyp5Z!n& zaAA39K}qOOIcQBiXpO|3Z3wLrgfH^9@STQminh=snPK=28YS9Qk^3dyaAWv{;9ZGq z+>$3H%5htVd$fbHH$|eU3Ym-;zh9Kg9mb(YB3%h~4hejHfv7pT8{536%{=25JqJfSNiL4_ z^}Re9$G)HQKggTdnBRlrK87E@b8HsggE8O11>d2=P6f}t!TKQ;uKWgDgLiCC;gO=% zKgGLe_`N_XJIKWjQr$})_LBDReCsgR_j7FvSN`Da2)R2&Zq7J)l=36Yk&V2Mu z;bD>RE9t8WnvM{?Iu%(WnT<&%Ho1~SKM&`=26qdGUlpbg%gjeU(#huZ@51o05~MA& z9)i}T`>PpYGgE<#7&pJF`kE?FxA5 zs?%E1i+^|8Rr-*ir>)?swWK4xc@g7bK1gdOUy)o)W}iroq~8uFC(^c(w~^#VbSbmQ z!Ag+JY9yky>|6f#yTP$s+UlH1$}8#bGSX(TFJiRJA{|M8JZT9!k+|@E*kvw8;*)ph zC>k>9E3yeLl`-6w9wJCZG(Mt_mYC)p>08oQS~|U@CH35b=OkjfP=Q9!iMs4EcOqJa ziu4iDU>B#Aq|KzaNIZ3kjh%;97K)XRR?fpFdM?r9NObkeyjzmqCGS_Ie@RLexmwQY zcjdTKnmZ+UrW|)Ga9)hwAR4ti^t@~wb8^h*^v0}CFUiT%qPG!hSe=xrf_Y0gc`3qm z(Xfd|Mr8e*PM>u-cv5JtNbJ&YAEQaR1@=4(Znyv*IL&q#x-G~<;^uGWxQ*j#N0+}E z4kUa@c&l(%*+iz8=+OQN@Pcu0gbAE~4^I(3ISBqDT=82t$58l)aFI!Ho1r{80$w63 zB8>Xq`%_3sc;9sH%>;)m123(B`-yb38XUC7`M1&0lSs}sJLl{8749h88aBZ*8@aZQ zcQ*6ZYNr%&9OHOiER8n9PbJ>}0k-2{sk06qiiaQF08NR+bPVLAfK_AAK3xY>9R>@= zQ(uwb<;&dNj%0fSlqlLgLGG8?9#flMG;@j6r^I@{0eAC(GgEl;7Uy@smho^uLHu{P za+!0HiZi}@k@Fj1wq$tYOa5I4pIzWL1ApEi5Bq78gS3vIQfZ?-lt8q{+njAPwY7-0 z5cIR2Hkt(@o9OWSpBzrUls1!i^3%8`SP#z}wkezmJ{rY8X*r2KKb97n%DoBv6HGFk zYZ6<2l*8?Zaz$E9aD=2cgLazE_LGAc=934BK);B5tRim`-Cpvyi{G8>Td28=c34Vl zNuOHA?_%DbNgA_AZ6Pf#$ZRI*Ed5_^lRmf5>1p$vug&DkqBW41{89#aQ}QBxXA--h zJIU7!=WG^tmHEhwuM$W4%D z^Z((x`i@4T3K+H;*i`06OMzrdpcN?L{I7`2Ujn^AF=nBPvx^)rdV)eA+v4oS_$OOF z=YCm^#dxL0>`X}n zG0XgntRWz+CA(l$8I41@C&*2(V;{C5T?+qUb44V-c&^0r z%wtB^1CAoQ-DR}Ck6lJ(3ja*Txs2NovUY})D)=)iG*)aGxKe zbRM5VX{$oJtHBopeO7}*H)Rvsq?YiO7VwnLaH0VmWoAZrOfPuQaQM}9w#jhbA2^DJ zUAWn5c%Y!U`S3N-H*A2T#<6Wh|FGRTij~eLo>>APTdHi}3bglyDH+Q2xuz)p&Tkpx6N*Gr?%G@<%snqAOcdYD}=1 zaMebj_3G?`&IFIurWPA>wIO`58s}2m8S_~+oO-WH3smBtSb}`!@J>1Vlqco45?9MP zdpY=`Xgi9*6AL@igtVn>qVdcI-xQu&3_O?5;j?04QG|44Zd$Z9f&@j!RfPAe@?Z3) zB5%kHR&{b#i@aBIG#yfk&)BQ-PGz2}#&xm1l6g8wMR>b#`bwP1f9XSVloeRyS_SSE z=8ei+7c`M^Ea;$4;oULxM$y$rfhk2pk_0ajN$M$@iD%B3huG z+{iPJoqzWk3y(N^!ufrky~LQg%vcnze;)3Cp3!raF)g|e(Gr|zJH!8rJbBA`<0_+8 z^i=ZZO@~WK3K>-3sxyWk^Tl}Qn=i>{lJm7^oD1(uWRuygAY(86f8$R|5@eV83DE^* z1u=&^WfklkNr}Uq(I_}b+D-6I3EEKbw2V;EU1wNK<>5IQw>h|$fyYF0ti)E)X)&=m zYCx-rZmJ$fL15Bv#3DgPv-C6RFVZTq7LLrOigvRyUzGL|8c>=uX=Ukcbx220mmELk zUIp5_98U?hmp&ssrkInM_EEUbW@7 zg+rl4GL{i1welJNWMo!%#%LX9oC;nR&24Aso2=U-mMcAxI7T}oevBiX$~sMg@`rLJ zBV8;>x;w~w1Xz7Mlx`B6NGL)9e}sxGf)37so-F1la}qMwAhN6+w>tP=sMTIawv{Wp z9SKn+-A!z3p?nLViEH>La<@njTOArElIV7}wa}^69EExXhCVQ zY-}<7k3kNMg1=~NVqxibHWRe>5+ojCBqTch-v|18>zut{jLOIrd$I)fCv3@#Cb6@A z#K@CzEmEhfFZP(LVrBD)(Jk}Qw;awY{PnJLE@NJ}<_(Tw2_YVX^4@ixyvo=<&KSSw zjDA77dq8Qc90V=+dKDN<@SNaDSxIP~gY5*jtpd5t16v7}6eK3PCBYLtz)*tsdw`>w zgIj8`eFEkw@63;sWi~@>H41=yVnICxm~AQG{N~2S^aEx@GU2(FgI^X(X0{|R|HGKc zabYc!jvbBK8VJRDrb?sv;ah@=X@|+oyr;d&C6Rths|6R5_TzT$Xf5`7c z?#MG@_k5r8SETWr^B1J_nzOeY-*bG+m%MycW@bFj%1{bcHVS#NxE@QsqM5JBhXqU) zbeOp)lguBL2D{0OQ$@;L-9d2WIj;gf|pXe ztJzk7?RSEe1WSpwShUV!#dnc5l-3oL^wyCS-XejxkmtM(#f)|+WDNZmSx!;Ae@7TL@MUL%x=WE^-9xih{-*FT< zThL9$?27!$7%?&j(}DNeLMytGavzSZNudLq&>PWYh&D$wZ$e4r+mcpW_MV)Hd@lDo z@GCf>DPL|+z9c55Y$D4G(ii~Z7!1Og$Os+7=$!=)8O^w!g;qh<9GmLU7@;q-8T|_w z|AM`y^E;p8Y>soGJ7OIm=u7lY^3(#T#eB}Da+Dd{xty;D9qosv$of*EITJ*cp(ptZ zUC2R4vnKRrFK2t$Pdb_u(WVF;5*>-qF!?X2YYX`6Hz<{)BYKqG&bEOkL@#*cf9Yq? zzwLZc5TM{R$>lO~Bsfv@Afh{&PD#Z|P;87Q^KUBuWVO68P78_dOe`P7f<wBcmusf>HF+MlE#SEZz)N0R%} zc0cmfAIXL2m@;tmNY16)lFNyFTUMJBZX{eubWNkl*GTq3)JZossgn-$t#(dt``Q^H zGS^jyIuY$ldA1K7UK-2Tih)au=FjhF_{1VjLvD~w=I;b&395Pk^%Ytx$V#*Vf}ccI z7afY&4$It0927PV9mGy&6K#c1(nZkG`Os6*d&sJ8vaXxV$4ui~<|ahPG7-6K1e?sP z$To#5laTX9a6MzaIFX`dHMk76exZaB8AI&EYw@%a+zz7 zg{tS^$t+O#Shnn(#qw@m=zKBf-v_)U`$rtbUQ^`tj~#hmr1;WoA2|}d{FZWd@%xZx zq;+} zghpgE|D^xSr-nsWwuyeXgL3=e!V(N47&RJvBNn}aGh>mEiokD5fO|x*Dbmqr?1G8}S<9~=@Qibj@n;U&wF1(o$7M!Q=Sqlu?)T@XXW^Q0gg3z zO01Z~CQ@dAYH+WJgN?J1`$)bZl0aSPTVqGgtjFkY#y^oYMG7lMs<95|tV}M2XNol6 z9=_D#f6s_4DO%kyaGh9?so3LRcIwN|{}+zja|=7pK2WGdJaZp}7DFxMCiQN7DI9Cp zowUw6>R-@xadIab-6(S3h%e@3W?ImljFrRCkw;LE6VMW|lRLo|N>kf{C&kMDS6X#H zZ7f=u?XNrUQ+1PBmRlZb`yP# z^q5^dwVw24c3Cty$9YQbzozGi-QhvLx1Dmxx{>=ycR$-n-hM*AN#aTARU%zY-MFVQsGQu+bA4IRwNZ2h5VN*GD{#9Cc>qJ zYsk!n{AO?V_5;5&IW9uRSb&VNnqR>WGOsd;G-Pf?{>$8;Jhy=JjJXbx zc8DXzE_h`;oKGx$N5Gqgb456(@KV{Pa$W9b*s%1YUkTdk23``4lFU-#P7lvmwN{=Lt}k;R?MbBzsbpX}k!~{hy&MJYb>yxf2RReuCY(~F zAF<&R)F!f!NJJSJv8TgT`+znEkYC}u8CXGhvS6!4?D8vfID)k1Q(t16CYa`Du4Kq* zqJI)S*9LZ(qZBVQnH5^YnIIsUZxVkanP1()nONU#0Uzz=DB3G&1DQ+Bz;QqFEwPmR z|2e3M4vG920aaPo{{+&>FFj%jdx@YQO1eL zdtyf(!a7GRX1v@}dCtqG@Q&y%Eyhi%^ZaYZkjx$=b1oxHB+0j=knvB(p=e;9ab5mp ztduM?K;|Q!GFrv9HG?O-`CIYffI0j&|4M5N7o z&he(hWA1Y8mh*&YrS9>RPyspKh3d#AnkCURW$2GjLr?axod7i*;V5)PXw5Em!B1if zBlu|zG;f=OGFS}?jIjoKDziO;Vg$R$oPx-$tN9g-vXZ+QXhq~J(PgjUzRW6ZCZ&w` zH}ZtcJIn%~%yD??1n|`~XD&F_iR1JE%}n7pqZ0I8yKO^@UE03*M1<3xc9o z@Q*bBoHiFsEA1~cmW#>xN{72isrNaxDl-yd^(Y+jveQ%V(97>Qef2&gMaF>G<_Q;* zQIyDNlCdUyNyfEsB;nq&%RH^<9A*AKL;EZm>l#qt>d+FIJ+24cm3d{+yU9P1!aGBc z+c~sV=A~s;T4ZdYuEMiK0vAbFtXDGR>;}HwQTH zWyscIC(w<1G9MvaPAoNt^DFY2*is7r8OKqiq@mDmnNydm!}urCwfK69v5Iw3 zMrtyjF8Vs5+@d=c>Mj2>J5)Og8g1cwA5NRgta%7}Ec3pCkCXq`BZR8TT$Ip9p_ihM zd*;wpp_HPrk=d;?j8_@a$DNUWkWndf(1M^tZRlV0S~RTb(dc2e)!gTFOcIER0rj>KX=bN20MstFkIP#I5Yehyi2@WdjbAQxGV-;*CYR&k*j2&cf3oIbM z8+>YT+GtNe;b>_d(>KqzN2>xO+O#f@NeiT)1rE2q&ZpJ-fV_d&f%+SyuAj~dZ?3og z6@-L=yaUE#XPzC;1H6&vh~{LtRMatE5WZStHNuKP8^Z75we%gd9>2b*66@{by9(;- zqRq|FJIRHV|4?zn7!b%Ql8~bCUz^uVK z#yaD@z&7UT*3`3Q4R*76xA-oMp9c$?@5Qu4STfkq?No#9_3WAUP_1K4!{x@uh=-H{ zBdL$oa^h>|86!^~&^j=_KHA=2Q}Db&dX@}etRT71S`uKK=0J9zz4b1D+#(-%PrzA` z87yOIAdNt54QmpX6zoafFRp=za{dR85sXSS=RwxgNISgY19Vk*OF(|H(1vM%U1c~} zMi#OL{v}rmIype^BXSD)_Ou3rWWFJ}19~`BIz9LBbamTx6f-jghOQLf23DjA)CxGm7|J%-fOKnsz~FL8un}5 zam6MVXv;DAdj8UR_gniGDSkMreg4yR{T=lxvlai0=jN=gYLB+pTUs%4JBAq3mv*F+I$jju_9BTsFpXlPKqKT-BGFqFcT1ZhT zrKLpGRas$!;D%s+ZB)N-fnc%UY1RCr(q|tLb~jwlW{mZ8UXV5mql)~CFqB|QgOYwD zN&mYv0Q3R0b67H9FKZ>KjW^!dTUcbYhuC<=V{VfrNDG9eX}rYXWfXkPv_uuuuE1nP ziW(UT=0@uRKET@wQyLF|(Np>M;8Nou=ga0hg)=$yR?F~&WCyqb8s^jb63|bPDq}T8 zKb;^g5C#Z(C1}b-eKq)Km}4zp(c9KMDF1U-OEW~r<6}kg4A=h+x7vzFtQH0KD=ivV ziszoIM!KVA!vjY$!l7(sN%R`}){Jfk@6xa^4No9db;2^XIjuDE8Jz^bxzYddt#b@$ zA{r)mU^UeqYfUQ+gO?+a*c`=rG#Z$oRjqHGb|1}#EFW+IKsMQm9OsP4cDAB_)Yl#$ zp4O0Z_SS=ut5ef&Ad1zKm?W0vru!GFeohLm_h zE&Nc`I8O_CFh)<1XGZI4rl4ZR$2P`#zWDjDFpvc3?BGQpB_p-Ja@pzm;U&a(2s(v) z!J9Kq765D@_J9$DtpPIhv~>=!CS(m)ghU_`A1esih1LQ31lD9M1Z<&)b!K!D&?v3| z+X$OyoORu!tU2?}!92nunXGq=0rnA*ORq^vz&SS9-W1D+1mg-h0ZR0Rri+D0!Kw#6 zf@if@h!f1{UxY1zGp&>t4+g@TTiDcMYd!cCNEQWFf?*&m7ti2p4L@o;-@>z&>O5ds zaJt|sd~F@ENc)&0VRC_q5f1>P;alwu7wmhDLB-%!FVb2NE?5(@B$<$BQ#5{4wGGMz zf{6c(C&;%+Bp$k*sZ+jdSn#|zXmETAAy7aHpl3d_GFjmP<1M09zQVjcMr3T`fu}PL`9s~!h2Fv^_6~iNdGdjz% z>Oa*w*u}H81g?Sy3lA8XKBmj^hs*i8g)qSXq)gK`JXc@=u#(|^exP-%{Za*Q6HHUE zo*&fG!UM)mqQwR4fGxXF(iN^Mo~spF&(#{1D%Wf{kJxL(YGFmKSE~%ZvP}cLMi3lO z7=yYHnT4fAEr8Sz*~K>6aU#8F$?+P%wgx>rs6F?p?6TG$w&wR*$AjA+v`{(nK7hi3 z!I_9b7$G2jXv)~4SZN@IcFBV{NdxTEh_-~J+kzXyG{Sa+X9U9vJ~{1RC9TJ5G2A4q z7}`TxJS-zDB=mRK1iTq&G{oGPLd(=n?+rSuX_N8Azk$gnr;bnzQ3~K|7 zkk}_824#XHw@Es zwB|-idl+sZWurAkmtf5(6*P-s))@2*PbggAN2T{YWX&6PFj}Bt2tOj7Bc1m%h+K0r5kBqbF*3=kS$xSzjUyC-Q{g5d-NyCadtQz90~C z_=kKiK_TrR?BG4-3%PBp5tBwV4g=B{$=gkBso_!Jhe77vBaH>=bla*%CxU?iS7of^ zkn!f=Q35T)Zef0^=HOJoNFf&NGfNK4S4$7q7oP}>xSu6;ev#}TvJ5uKW-EG%oB>3X ztyfv6b?Y@xOxM5Splr7yt#&JJ=a9;A4M$ZDDz<00;&@m(WW`<)AGSedyLHqCotLw0 z*1xRQnTS^VM{~rA5zl6PGI(f^aj;%s=9vL$3BSd}BEz)9BSSjvbw09DcS>X8v}BCx@wX2pptJ$*1amZhty};ppR%Q)H!49TJj@+ ze4=53fWRF>yFmjYI*5IZKM>6FNv#K|H2Ul~&4aTN;rxQmX0)Q0wASb{=saU|9)qhI zrVX|>5*S_Owibi#^_aB;5{nLlR`Y!OZ_#-w1wO>`mjBU`*x< zP(*M^?2@0=TCxv*i%r%Sg(L*kG)tUqo7N!bcIZpPFT{6dJU^g}_>S--rK^p;V97C- z@H4@oF&<|8R5xYg;L8GiED_d&KMcl=L1w^V`AWfOZg1zyCDw+hCZr!S5ziRwkrSMS zbAa7P^~E#CHhgH{x$HwMfWd*7C!UY>MBX#tt-fJhCB8AkqoyrE)*R4%Q_q`J@Qzb& zV6gb8Kx)x*;Mt({!0I7h#c*xl+k7DDfi!vDk|8Fd7#<~&8DlL8KwCRV?c)6^1FiUz z{uJ7#>MIeNC}PAm1?@j8oJ-A$}w8;)*YuR!(l0#%a*KMKrUrK zbST55OJj%4rlojgU$@!9$}3rH)mX8jlYNRxE>d=}aGYIRV%iqaSlv9tIh1#jxq$X4 zR{a($Zn;QxU0@ySSFCfkmKc+N=%SuPhVts1w{igy@p@WMYpWz9QO-<>szwIFKv18z zmNmCT(#-Ha=II@PqsZHURHFR{4|!UxJ(33N7G5kiIZ>rx=Wv%==`F*r1v(DKKwb{8 zaqwsKdiXK$zsYfNL2ob`uJOKOd%!KE{iemCoi-@p50({$RfIi6TpU>#@Y)i^hy653 zU!(ey(l-m64jmCq@}XPbIEWz{-Q8+S?$JPd&|CaA+KP4-v#yKgdZ}Gd&pXyyvKzo% zyIWIOcc6{kZ?!$NAmCwWfbiPx(-~>Uj6O*O+hFV8AJJ8e)<4gXw)eQcTjjXcwjOFZ z$;$AK);=QXGEnX59{s}}wI27XXC zLh_l+5|cGg+#!Q${n77@M8K{x**A#Sq!5!z4h&+5v7<~T4W!X%Sr5ofqzrgCnJ}=N zMpD|$lSr3++8w|*h?6sO@x#a+1XrZFOix=b(;82 zmJH_;EFGRHF`rd$u}X>ctfeLOvTYUCvs}aCkisiv4i@Qc6uG6v6m72Wj7(9)krG4D zQS%*iMGWL^>a8o~`@B!rPmU3E11!FQ)_w9_Qwziw7~Uun7N#)SMP{h>DaNYhnelGT z)>EFZr^EAxJql9to!$lBAbb$mQ>*lTfRqp;O`C((czcQt`Im;Z))*@r^ur8nA@dAA zjh{8o(t2r2VBDO8$WXW`t2773LYoHq%~oW)(LG^Ib2Yr5yt#kq7?$Cwm^f&z44yR_ z0N!+>x#8Tf7LPjj!2gpE=x1%^fBur*N**6@nr&)VKxXzzgWakzeJLE*vO#T*OY=LY zR+xN_a6O6Cp@q)Z6rKpIDwqH;AK`-IvxFH=3@Z9AJWG6U>WF3qPa`J~`ZYKc(fQODwaK>&LS=kJ25$pptR%(@}Mg4Jh>eh=aK+9-VD70^jHKk>xgHajXNBiLbTHa7PXNt%rS5ONrf6(nVF&aqV3rYgNc~Xb*_JMkZf}@!_&E$elvp{u z==j-*pG0>(K4$sx;AD8|i(`t+maxmZ6Ep4OE&)6KpAsu#NP=#wyJ% z?XZ*bYPZtRNy{y@jje4pG`Egpe}?utYbQ&??5_DX8ahb(y~7%Jm+on_O#Z8Zw9IDG zENiLM(UsQHbym<|^vq=4O{%4JR?wX#tB$?8OWQ47ks78h&=!kSGh~WKKfI!+N~T7h zGUc@N!-LY|cM2uhssXG7zW{txya433BMy79&>S$1k8Wv&u%cjF!H_bnV>lT`Q^a2a zY7f^7i~t-0yaGNLQCWE6;Y)#M;F$+mzzYJ_KtvY%f=a-+!V3on>sd|Va1o~oXN;H} zSj_lH;OH_Ya>vB)!1Q8oSkdHWxgEDd)FjMhgWtf$#S=lSCg{&EmOcy%7vzb8kA$o+ z$gJlESg3pdD zEJT4D?8W4vGTJ$wKjsXB16BuFRAD+*mfjBY2ObdKSA%5ZKcUZ?Bq28F^&s)dnukRd zwLB`sSYUe=O5=x91`DQ8n8OXtBP+-l0x#EOK!9D34S;+BeSpWcUu~uFC?G3{+eY@1 zb-~CBT2~|pY+8KFNC(<|w7X@RqBWuk!!Tf(EH7vRXr;t{f>EFWqwnHfA!7)d*F=lm zNhWZt`KP3XgHDgJu<1e4sP2=tk2cjy`brP!Euao?5bjVxo2ns=$i&s8TFirr@}`&7 zTxmsFCdkX~kk{QQ&OxEP_d$8?b5-CMz&IeLljuz%H!q86a6vqS3syO)<@1XBJS)Bc zYtN`0uy_S94UTH@a%bKIsnNDdt<)qHRS%?(9Z2i+2>wnV4Mz(=uzw)6Txs68#hHw*CR;2LDTqaPOC`YbVzBs;-zk`EfBg zB5sX?Vpt@IQA>T;P02wk&YvZv)BL(jyvnF0PG(3awENAXAV>M(&t6(N`p4h^~Rj1#|0t%lCl?gt_S&9GPo$Zl;vDrD!3@HX;1qot)w zy3;d_dKT6+LlPb=C`0RSO7drFtzBAyL;6rzX*|i&hDup;6{I<&TG(8orm50@>S(Es zHP1E*_#L`ZP3a-kER+k~1MLH>i$0aAX#Wb9PL(SCBgs0yOWH_^^)HE<8cnOR_DFz zX@fk5WAUNou_2b~9qA$_AB^#*;A=5i6<~i1wX}1xtr;#UOkcbpZS=149>eQ~Zx6ee zuaBr1vS{E50O<#*$D>C~3|wYVd;B6-Ec6Tu3_F5>><45^fG2v`@>CGN09 zz>LOvBa^D}Qe4s0#QR><{#b)}HIAteOJ}CoZ?rI{)z*-EA0Izil8$JZ94o|VU}^5r z)YzNkUfF+ZUt$rkfQXL4`;T;I9eIxaQ#-cBS|+FFMQz7YBDdHM?QvXE{Dv||dh3qAWN9UECcSugS#kC`+ zg2!SVIy7b*TOyJt0gZze6E9*(6^FLONh>fte=z5kmVl;!8D!>7z6j0}*6|K&OkXS4 zB%`t|Ey5=bAepysXnTnYOh5jo&?@K?msPS=kk^IQUQrzbA21;K7SO(CjLa*MSHQAGRW3Vh+T2?AnX*1cVon)Ifw=CBt*7mxVW!prSsquc`6X~q^&i`+_ zwzurs<~q8*WCZ>Y{2*0s{RdWSO||)0(~Y#gvIe9I*MT&kpM(mM7^#-Dz%EBxAW!g} zuvEdCavV~GBk(I#);P(MX$D2`N|v})5;fQ*C%hU5EbBL<`^R@;{3t$6v4;Jcf)&^- zxdNWSeqPC)M9HRdmVDxzoDa;PlnMw&lH?Dy$#SA~1@V@2Vl7yQ%ldy%hnQq;$g;Zg zC5?0SG_L9iUD9(prKg4TJ|s!LOV4k+#ZVxdE!L*VtMRQRbr{3nBfivRr6AVV@GoEj zU_s-HF_8^KN1)ljPB*w}mdFd^voToKNvoYE^PL&M7We4=C#(2I>8T)lcy@@}BdU)%VgSKL`C8030}=-c zN^1`03E~G9NK1b^_nxuxR%s6kQHjPsfQAg#hqZ@P0P_dq10@6x1oc~G{X0{G!g3{s zN5Iv zxLbHy#Jz)gQZwcZXz#?UoBu@Oq0z%KKuWTVElgY~(iiDTt73fgFv3ipK_lUb?=!Kj z#5%#qhfhsRM?}$HFsfa8#^l~Y-$(n$swcuDS%oFEZcqTUdJ_YL&P^s?Pz0vrghI~- zi>zm9-msL3;h{Gp^cusFt7C1WMT0GjHDHc0XF^-&3}!7h0oDWjV{!@;QG=EbmktYs z>*G4%&hefSp8{8eIARmIf=$C4OO!p#2=1D<*F<{3?#x#k45!3I?_acVS6GK8Cl&ED zW)40CNH&P~IcY+0JkXZVdGJx9x$vGJ*IKv|=vU+d;5nG6VZIC^f_Z8@J#-}_2RIKH zhe19vtYa^09eNnK?G2I+er~=u{G{OUFgr?w55o_H)qxe~R56*w%UREpm_F)+FWA%z zUVr#B#9qTKBrY4f56zM~gi*r!5({^RheAxZOPmsOZr&E+vSE(FQ8ZcGh>ZgYBJ&qG z5Zeqh6BGyzi^uwZcqdq#oCB7N$&MFi^~^;dD<-l#N41`DtDfmC2}>3o&#-v@(mV{5 zE!I~=JAxO5mVs7+FWn$LL>DfV*0)e=VW-d=2sv|2w4vDw!vfzAtpj{ryg>L0;Q`T3 z;9bFgfDgjp_wX+8N8s7P4{iK6?8jRCG;1|B(cH98cxO0*HqJyZmFy0E2bH^|R@#o^TfY(V+ z4d}oi4CIM58FBHMzo-F?n0&E#7vZ`P!HmDrFg4MO`&<6FfzpERw|sLjaC%tCMsML? zCa(*AJGeP`&bw;ey;_24(_0!e9&~s*AZA^)%rP7fDuTDKujS=?Nau&Si3jr$s|=II zMSlciq^WaVWMBi2BX*iB{U)yt>>7O9vxJsR(HMRWnSjv63~GY(KmyR}89lOvQ@6MsSooJLdmoFRp2Bl2%Ref=2_6-69HZ~R8OhgHpVk&=z@LMrSfHb? zYc5;53FW+sK^9I(t2}631y=*dhy44yEicgqUGrXDIZP6~1)$#a2>^$aY++c?t1U|! z{}O$zuT(t{JFq}#H#soq+hxAA8v6etG7Y{cT8?2&k`>EDViS8qgpJ8ph}YmTJqaRd zh}?t!-%E&hI~9|k$K)*pRWey5L9LCCPRd=l98dtf<*&>NsVU}Zy?zf$OCA4 zf#4f?@z!8i&7gRX=IRaM@yhp;!|WaEMVk_XUZS%YC& zB0<@6rpHGqj4EUXWG`$$Y(X*v`ee5dqlP`ll;|%Cwv12bz#`-y%wUf6DyFNH{>N?Q z^#g?lv#cbWl{`=Q`025U=(#S!D)GOYd{bm$9%6luu&_spc>;TTl-h^qE&G{l2=K&V zio+FWj*MR9w>JE8xEr+6_*+-%t0STSy$MABxMU+T6<;z|QI?*E$wUCkL5w!B800ZV zIw6;kS|;`w#Ib>%DhMKDG9jgEO8#3IM|Ct1sbsvRM25B3*9(uC{x9&3!cK*;L{GNF z<`b8L&xu%JvIxL&B0m9sCAhaHyAFBW$k9fIIKFT6a6GEyzai5>`>3ITH-Vy8TWu9$t_OLbRmEvE0r8{*6hFwkX8q96b zH%sn5nAfyTtJPA$`ol-{t-jCs8fIH)Cebj@Ek{Uq+)vpQY6YL8e8aU1_N+C7Q@& z&o%LE$OpV?M2gTaX;0lJ{CP5L(I$}*1B97KKVlKd0l`~Gyff_;*mNiDPc$4@ph2<0 z%;|-JC8G<|iV;@_f{wmS9fBB`d=YhYwdmLM(*QOOmPpJA5rOFXWSpYaLl-uTP-=-9 z3hspC`I2?HB=t~mr2YN7!52Fj8)%375SF+xq_kuVQEk$c;Y!UQ1o|Jha z=%Pdk<6j_aKv`WMxIS0P{TS~6_5k*UiRmW*K%I969p%z;>eDeOm+TKG4x7f*MY-$yA61j6P}DF=#+0dj?taOyrPZ1TltF zkBxFtPYryT_@kTpn#}tG``oE_h^zr7tCPvvjOU1mD(pJqM?e`mTkkns6#BLy!uAQZ z4(Leashp@b6#qPti|?r55g|Su&pT0xXy09}DULJvP7OTVGNAD`Bs7hRWTgLagoH84)1aRw;@+ku%Q`eEM zA=~mqT-CL5Ww~0qsjD}f4BGtTx=;K+_$GF$;GKY3uvz0BR$C3*fL?2;pUqZlZTJaz zEyynoW7}{U@Geoarhei5d@DI+%14q~pXo{9ry}n?`3HzYp>;>nO;+1ZG>OU6HcoPK zqzagm$)Z8#pWbTC;2qGyo1EiV{U8Fg^L6yCSJgMlS7_q6X%n#6VVdFBC{&wr(JJu7 z@Dq^R546be!07RW+%OcABNPrL(P>ySL{P!PCmtO=1umpvLk^TagMEU3zMZrjIFI;e z@zdY#$D)zeR*|Mg4o&*9z?)0Ym~}1f$Z#allQ}Yv_ z+gN93OBWUTh%=cmTk9&wok`YAGDnjo6a9;8gGGSfn5*odGvQ0-jCdd@#5b5Ihfdn7 zsda24&9&Bl!Og58jguZZ8tVFCV8TSeFKxI8)iqy5YwBrAZczH@pkI!9x&r$5fZy3v zcVl|`AddoD`4{pjG`03&FIb&yqaLWq`YPy=)Cn92n5A%~$yIuXHQ!NMvAOb2mIVmE zx}yddswTFMtf26+`)awb#aM-*eUD`qJ}8e2HsNh%H6C;r-NbAVD@ZO;qBw}%GrgF= z!bW?&t-*3u*retkO%$QYw2H+^HUnZ4iS#gih?uvDEMG*c@Lmz)i0)5p#|zSb;h3{! zv~1?FnvRrJihVlH(vyeiEri?sxRyrfT8HS`h$h5N=x*J6537DmPblO}ckb7@rB#4bk7_H`yYe37(TJ@~-UGF^soq+cq7AKjyH-nBc6i zO-`yO*t7b_%IKjH{kN|^@l3NRQ0~jcJ2>2)YnCN|g7#i|RmI$e4bLjTxXgm{E#!8KuaWQMRX5CaAn< zdsXAtY!hu$RHoTJvVCIvRO1hA@7q4NePx@cVS$EkZ3{KOQ1jnfJ{rh-b(f@Q!qjZIjYZ#`e zn_;?M;%**LxzCFDxlg_&qJz5X4&hKX)4kTwxPd%L)l>sS*OZk{2DW3osUEM5-uV0<}Xp>W_qMv1h<`sZ6D>i{u4=%qwkz-q!@ z3PMSiH=;sdJehn4aQ49vU_PPE8df$D?DTU64hoAKzmDPc!TE!C|F^Xc>~@QV7}MkM zUTM0<3yc2*Bp)3YYXyD;1&hV>_=0~0jssKf6T}`L40aBdzy%d-PV#nTTXh3&h+T=V zfx0Bd8ry;>f0IuIEP}QJtACfj{15y@wzdCZ85rr}4R9J<>Lrye#BmGaOAUApCs1o#98-cbg&T&77iu`ii4+ag@vKgt1s~lOSQB^bCkjgTkA2pzZ7ff^H%=cn?5&*#n=S ziSRXfRzdhdqd4>LdZUQ;z}HCOjpB;%2jCYlZ!FOwrk6mr<4-i+MIwiYNg}H#ahv3A z#o_{MHdDh%A^!#vXt4@sB=!I;*6hg`j|IK>aTIX}#JBM+5OKg19t)T@IRI{d4d}pp zGvs9DDmWv44ZH%}KmTH`75oyv09ipy)C;-mP4o--At=-nQW1ZJ$;gVQ(nPly#LoCz z*yAr*vP7S**S7^97cLNa2Y6cy8bI$Yd>^z&NKslKJXXekL=Q3W9ZYsS`X4i2h~fXy zBP3`59z=SRBzq8fHt3fK)C0@kOP=8jPOIIzwfpfD@Z7 z28EQp{+w_|m=U$8ZD zUYo0;W$2^*u)<7^s~%RX0jrVTk-(wAI_b#`1l08B2J;asj@(pmO-!CD6W>o30kQ;u zg_=A`WK{wk9WUFEJ}^uURr;P_=?z(mSo35%pa%_d%+1o&WTZpR(gz&5h(Je|-SUTk ziV}5tNNY|>M(>cWvsnC#%|brUNhV*_@jG;`4`l@}kwr_62%;4B#+9|5k`>+3wmq(< z%_noQ6x*>XrqYU7sO?mB<2dy!mXqS zb(Ritzsmo#&kKqL?knwOgzX*M7t&KU+1`+r^S#!ux6P2&vq75D&(ecFvMrasG)Lp9 zTDwqs(4V$HY-?0@>zJb&@3397ZL=M+UC@z7w6xZCNyAZ<^|mXve{K718McGATw8%H z$7Z+Z+KO!sjUD!Utqa($Yn|VguWbc7Pl4ugZ6RBU)`n~m?Zwz`57@%CBx^s;m zS4-qE?=ZO}>8T6^5j4^CS%ppp76`}1E4&x~9-eS#+zrWWa`vB6i?mX6*JMZZiW{k= zvuZ1A#C<4TX{Fk@i^AAWsSRDH){f{qBGLMHPK5c0tOSOu&bkckeZSAUQ zBiTdcwFMTTPc~MjEU=5R-F^`V{)gy_=sH<(OSQa2cHD~SchN7Rv!e5&Z>xM9eIxoo zbXs(}%BNa4Bl=Zze)Q|;uiA5!Y`#_E(67;X*618-HTHztsFUn*S!cP}i|UbIUbdrhnU@Gykps*luz8 zuqAiPiiC;3UtB&c(hZh1`jcgguF+Nepz@v0^pno9RA>KMXPc#~_%gakWxiFu(LO(^ za5ulGn7dl3e_W~gmDYc5&_81P8s^}CvRgOne|PKu;0yj~oqd^(U8=%a7wQgYX*^HE z$I(wzd#^_)sZ5T(5`8QBvdX0BRI7GB(s;5}yYFhB_q26-+17d!t>V-J^X~dPN6DAB;Y(^SvA$9vvKgR7XD=9icV<)42y}+k;wuQfvB0 zN9*Wu(Q3NVu~C2IMs#;%Wwf;tgq(^Vj4af-y6gIzXpcdv;kMRWQb{#eOZ!jLet+rs zajN-ub*5FihFPk$$5d;d=x%@2{r;f&NviE%HU3&lQ?<;~S*yBRq35zl*ufimhHvPI zEjs5iJ&BdV2z=4g^ ze}O~L+wz%a>#WE1AA7Y&3+u}x>r!uF1Z{NuJ^E5Q2y-DX)eL=`gRC{6M$ZT-|4KT{ zulkl(3CVd|+lZyUM_+4KeYGR4BWCHlenvLP6tN}ds0ElW9c+fa_l5e_rwjdgTjRGZ z&1$h)g45D+{IUb`qzRER4y%o5S6US;Jfi`@@~k24Cti4YdyR)kCvT+o$QP>@mmYg3 zZbU3IZf;CAe=JR{ly4XJD1)iez%_Sr_5*sJknjUL!yB7OR?P5Q*s0}ns zQG0p6u;G4cDaT8*n5|ZEvSjT_N%c%g?1Msw=WF|3tv{sWUejD}$=46H9=v#`Wbt`P zS$Oc9)iTFRhT0@U7i;f{YLov^i=3@}cf=*zR>Y;+4(Zr~I^wEY=+lzT2PE6`Eqr>P z?Ci}t+aXD5a)4YFb^#Jkwuv>m_6-`=Y46?Iwp;(SPi_8e$!>bN*d;`oKEgiH_!HgJ zM?#$G+l>ChR_lz1^k3_=W}e!8Sg^0^sEJwv<6bQ^Wuj!oYtlGY>KHtr3v}=OWZ92V z-A$2|^@^~T7c~5)W64zbwCZG-JP@;0Lo2oQGpp9h>8VWDai?{Ub98mnRa1+#_p8!v z)(G`(rz0MaMvL#av#xoO&M{H-JVv!SOm#F-TGwXDplaH~rhlC!y^2hU&uYsUUD?;V zujRU<3A)FL(g4ZZ^pMW^lCB5*2-N*+-5I$Z$Lha{oE)XS|ED*GKH9MdN9sH?b#?D+ zuP61y-_^N@8EmT>yx-dY4ej4wwegO&eW;pvQ+58BrMC^y_!a#pS)01*T)m{@P1Y4G z(wOjTSsGgf=F*MAJrxhAL<$Z5rY&(?JjIXPK(Mh`ILL#4-=clBT2YaPDg zrJDYxXN%|i9}DSPq8j8`@6=ORClAl}s;}+R`mPG+%GLXHO7F~G@mNmDPXa3iJ>k0L zrzvZBSYRxti2qwt`dd|DU`ZB-8APaw*1%jgXwd-aB4eb%jS%8B%F68m1~yDNV#o*c zxVAkjP3C2xQLkBLn((PtHB6JPH$`*LXxl_By)R5`fwaHxY(EOw0_*zJnp-Gr>n&+Y zpGZSoVtZHH7fU1jPTJF-+VZvb`$`(q*V3Qn=!gXxPt$qd68`n6G)1(jxhgX(r0jjI zn`3E`bER>;sdWpq^p*9mU}K>ld-Hp?3Wgp9XU1pouk7y);s(tJ;FmT~*xas;Wac;BcqONdr5GA*^RXTs594m_X?rdWXY(i^6;xl?gS{!|L+$(u=kF+wf#Nch| zW%*mkFV$M;JAI@z6*^x=Yp+G?3C*To&A3>$72j}8>>Oe#O2p0~r}SAxOAxb3v_?SF zLSgkpY-Cx8{Z+}a>+O~8ZE0kroT8m6GT4JyIh)z@t+YFgFmUIWI7(Sc3N(sdcvpHNPeRhmSnH1z1!&w zy`3EVlS?kCbGDtc=2kXg@F$CWp+$(%W-Q@#< ze+ZxOQ9Y$DI$I~n{JV8LYafz)C!YuzQ5cgC68;vRg@JnJBcu;JrKJ(lCq`NC#AwNS z@~w>3K%XsSFC%j3RZYoaW_mh(MpO1Arz1VVjMEv$-trci%wS`6Hu7MS@ezh4N77f! zvwF|Q=&YkPkj=zoJAYo+ME@}GtT^K{+S44t)v)}m^pnxLhY>o8SSHiM{p-4d=k-qW zW>3@}lNfuNIr!)5iK5K#|zeqG;WS%WWW-JIATal6z9-mthu zv6_;@XT_&~UzqQK7;?LUTC2Zfi>2j#8`}_<5gQYCId;EVGnb@tG)B+r^npQdoa{-g zYkwidRVA&<2=S>TNlyzZxaXGe@2xfgVQm(_a4$DJ19Tc|Dj zrJ?4i2At}7f^34Or$q80UyzLq`=>}XR%q3r-@?3M*u$U)0nd`|OP%49fGbj{d3um= z>JG5BOLP@C)#eU}YatD}c3c@**?+4oJ|N9_RSb4ged$i6rAgc?jpaG%E4?gh&% zt>q!A=?Aoq{#F{RwuoR(RGn2556EkI_{!-x;)YvE`|G9u#@mCZr;YxxlCXQ&Rn7FQ z=zE9WRzURHNkSo|K>CQ?YOMWQX@A&caM{Y}U(C@Bw8i9KAX^K4ynxpek3$Yj81*3Z zw0QSGg*-SiKzEZ|wN5B##>x=#=|dUTkq`$f{y$HX}4X7o4w zf%L!bB^b!?)oh4i>3x{m`o_BUyP=y2Vn0WQ0xyRFc%%Ow4`N}IbX zq~>es*1M&| z-8mx-FQho$fY#;89`I{gEV~1X;hePnQAR|48#(pt-ra*0~lJdzR{ed?pJlZ5Dnb z_QkszUXYeff6NnAi!bRt0u3Fc0c3fk^;W@(q3`Mc={bNx6NyEyPh{i)we7980=6G| z4o?)kwwum@Cy}iGFb0_e4W@s?R(jJ~TJ!V(Nw4cI^xl)j-1MPCL_V4jm=-;>l+l+4 zx0&2Qc=aem`P<{@aq^nJ$|8NO`V&5}BhsR_ zs_lgFMqj_gEfLj341>whv|iek$)Wa@TIrwEev|DD=`vr^XR=y#_z}}2*+y#3my*8s zOID4O#3NVcNJ;LN(hRywwsn#Q^^jyXeChHkNd89B4Ct}8y0nPqmaav=v~X$Qw!mm1 zj|S|4O2PvwNq49!4JA=?91*8|JX#`Vm+WI;Z)w^FZ>62)>9f7MG&i&o`ly9hL+PQb zAVc7Tx;m3jZJ2Zc1|mM`u?@*jyyp~2NcxUPHX=n|(w!pb7VEyrr%etr^5l_O45Weh zk-t@^#D&2VB$~x!ncc1}QFxTZxhgg;wj#J3((hMB<=3<2$~lHabZ%jm#_YYVY!-LDLgZR1(hg4HmEVFi|a0 zqT1R-?dj7o#F!HcPVc!e0f~)*D?uMbH#JWL;C@Sj;$ucy(hmhuTX3-HH-h#(NoTyN zYo{N-9jXbue;aiLtz?^gZma5;;ds^+b~bWlxNbTN9huI5;zxUydRrw7^SU6oC z&IPVgt}BiUwx)4iqCG-;B5Q?MO^FmnH$-NIXG_DqWE<(|?E1}JHhy;Ee+iXKpGc`t z?xV70Q~yZaU+#f&(^KrF_b0sXZQ^j){)pW0uPORB??&ElH@n=NdUIax*xci}uA=FM zdrHO!CWl%@uEk2(s>I&6O_vQ{LEp~2f|MVcltl}O)Xwil92yw$tM4o!N)_*B0VFEQdaaa9!c>oY942ita3E z;rlndKJZlJvTdNFTEaZ9-TOo7X^FkO3*w(}-E>TfT?lpwedaq=(xf=0q>=xS|Cy2` zTYe}Um+y+%)OWkf?do8w>zHU8Y~K*hEItry8K_m9Ye{C3I~J)K^~U`d>K?2ZXjrne=yFkBaC5YryPEw`?>9+>2}_bjB#uovm^!~y z|D>6o`(l^ETT1NtdkTt+&gbqbEMG9NpkBCo^Zpu3OvZ0u>2uuDyHvb%cX zx%h5nCX~wX_DcRN{{47wQWxh{=bGr~SX^*~$`1dT;Eq74NK)i6N!+*GRqP+x2gFae z|Ki->9ueCT9bqpU`@H0XXzjSOA%E=MNPp*#LI8@LFS^G&PkA?a){@MQ7eoPY9PEVkuN@U1QWuDC+jmABeABluVT%O&6Xzbd$|=#RW=Ij`UBopmsK z_>EVyR^DusbN|(T`F-*#-^?jmU%brsWy$%X-9=C3H@#6Md)$=^8S&{8&qPjdJn21h z^KizQ2QJRLwzptT*yX*u?7-?v8ok)2Z~MMoj&@$u^VjYjI}L4D(&WQhg_Q~ z(jO!~?Vf7$#QusNwtwZG?LOz3;g}oyHFDk8q~x#S5&0Q6p2==@wdb{$uCK|B6fcSV z;W|~iQsqDDwrU>V_O;Fxd#vb{cK4CHXWsQg_totOv~V}9R=anVMdb^VS9_*7*2R8_ zd>L&X8z3*&ee!iQP-K{l595zYncx|L5<2c37KstU+e&od4O}_DQZl*{;nhcHVgB ztM`81>)URJ+FWmbvUy39mmANm_i@_RayODA@#B+DCv}d`N_e333rRn?$HqDp^}FG{ za^Iyor%R69aq#Be?{DBwr2HgjB zDt*U=CjILDSo1`c=H(if`rGql^w+?hK~LzZz<&Qye;t3RV7C8!(f*ui*D|s<-kh1& zPj1HxH=fO#?EfqFNs|o?GL__3JZk{xARQ zlHB0wSUK0--aYX@C9Y2Fom7ytCZR!m1?OYY8~*i$L-X^pn_OFzIqh=6r9KyWUL17# z$CKZm=yhuCu@T3rA5YvrW8as%2JYGX@25M*?RakM^&OixE!)~*bLB1Dw|sSIX3_Yf zLPv4g1Ie|@|5!V#OX9s%27Gq^lYNT2uWZ$-X|>iZT2E?lzE(=PA&Gt6?$~fgDE4-+ zF#4};SLjelQEr#aikHI|>YrV5vd5`i$DTNnwQtgY#rwOTI+iuNWVF4SXGhXo<@!}` z)?{hx!Cjm^le%4MKcelgEkA8Iyk6znZPV^3pIxT3Qa%?&^Fs$h*8&d*j`?;Jugs5T zcggfzIC19N^Yt#($jHCE{`>>!xmO1k$fKK>oqDc%_XeGs=ePc^W9Kegy1v*owc`iv zZnUi2&x`)b?o_WSIuosFH}#|dc){^Ebg|9r_G zzAe6S1()--<+RSuy|OQT{KXOH!pHAEJbzz{-J$JoZW_3$`MPRr-ukQjsxSVmvZ~TQ zlg}KuJSpq-ScT{U*FPys8a&?qP={JQ@;a99aG%vduW#m3{?d;k1XAT_RzPtIJ-v2h(F>7DVqt`B-xb{WP zu%ecsPhBr1w@H1YV)v>OYkpLHNwu{#KCbwD1$WgqtMskfp=x2;&}xU%#;fF138a-y zTT?N$%w6$8d(G(Mp&tIY!Y>Qf;4yVjY?NI5Pim#--n^vRB z;>zDu*i~tCrPS0|O2eeKrQ4;PDRVS+Z25;$FO+Ye`gw(h<Q zc`!LpTzsOiRAHHdp9@D6l;oYyo1E9R@VmkqMgQcN$|<=q@_L_Z3$uDKyz2-;&e~uW7pc`Rw;Tia60f` z^Iv1it>g-g;Z>ZK`*p{^)vT>MX1CT%8YVu1rfSzdhkg=L@!tA#ZV9&Xg-T zm%E-Xedfg3Cg--EsdN6POaEQ&neoe&(b?y2ZBecH!%fl~71f_ycW=#it5vA-S%s=)Pm~TN6~-U) zboadHZtrU7Z0S(1bdfKD6MajHTNRxx_#v-C?$DdI8&hu%%3YSfxoBp|c>jNa&A|%c z_L0W&=T?dyj?|Al5Uv%v6zCiLEpRq)GuO-V%?kV~{zhTbO z>qS>wSNdk$aWV74z2{O-k2-nw_$x<0JF@A>&~s-mmregG`^#%}bH2~dD{dPe6=~-f zV*k|9+5NG-p}nfBO8nZy=_%dHy;f#I>V_(1(!Qwvd(8>8a%=xo=eydMstu|9Ou2hf z3X|_ix*Y$r=a60gv*?&`Y9ujYi~JeMjf@F3@@M&)75`N5ec=lQf95U7dn5Oi+;zFj z^JW+9D-HxFMRMb!&JnJ7Pip+fiFHerB=0Xfw@l+Q|CH&PGCSq#vZc$xbBz@6->=y1mE6_Oqo?H!&S`ZX}gm+wpSulLUhEb%|< z_xM&7zh9hHc(tf{@#x}HC3{L-zVf~g{O9~Lf*XU^LY>0jgf@h~4Sf=PDR^J7T(D7S zL|~hLiT`-Xf4&VR)k@0xa!cm;&PWIOt>~-5`Gv0(9L*n6u&mHm_(`)<8=M<@Kh!k5HT-_a6|NJR7`Ylb9%~SNE%I#KDf`2&YwpGIKPRRqh9txOOPrhd zQNor)XQ|7H50tv6RKuj76TeBQmarf`7JoV3A73xgmoOqBF5zGAQ=SFxkn=l7p*=_b zsxO4MOp^7PEr!Ls=-SA*Xl;4Mx5QqPC$gn9$n)|w4Y5CEUueJRnC~3ydfokkr>gh7 zcS3xR_^;z<#Lv_D+s22y?|AokQao;VAJ^lKZ^R3_D4YCsJ+n^X>Y+-(O#jpV?|qTt zjN+=r?-o5&bgpo2VX*LK;g!Niie4(}R_re6Sd#3U>09h?8@LpBJvch#3C|Dz5&kAp zEBaZ)skd@fWKLvS&A_=RvR^jGNJV3pwKfmi(#{T}}u-`~ETzFxk*z9S`BC5gT? z-^0H3z9j!#f4RWNfscX%L%qY3B157>ly6c>B?}n zccwdX?RVQV)qle|p)LL6-jcojUTkah!`KFSL#D)LMDG=DAZCb#8tf8!{z2^PxNoE< zD*Ik6C+?&W^&W9Q$bS-!c8^qy+!rYfRu6Rwo%cTw*yX=pdjFM@_lln>-d5C9@5#f3 zR|?V!%E?vPJ^$_eb@|@>#Dbmq?t*g#iwdR{#uq(VRJQof;vpr!mQ456@K5&N5hxv; z7mVnAY#&~s?{8S-aHO?(8k6O*3FzH!Cojc(+3*d7N*#IrQVj>h*&SQ1|= z!I3aCzD~l__&V`l#Lx4->Rsgh*c0)bcaL+|b$53;Toas6J7+k?Im$ZjwBKRhCx2kF zSoF2^rWFaPX&$>F2FKUpoZKZw%8!whXoJY|$oR;m@LziB+2Ib6+2MoX4`stn4X+3# zhkp!BkluSS*emo>@U39CU_l@!@KxaDK+Aw5aMZurKifaV-`gMZmGy7;O_9bi$v53M z$M=%&N8iW33%>cjbYEM4OaF5J8h;lZcQ`OG_xc`;g1c+JJw z1mSxFY=yQis-4k75d4nW&K1tSu3@eVF1Nd{dzI$J~DOk zW0r*$hxUYmp{n7w;g59x3&Jts525hj$di$$BRe7|BBvvFi32oQEU>fD>hcfG6av#q z_+@%r4RH{Y>|N|x_J)q(j?#K-evzHq+jYW~;_l^M=Pu>x>RIGj>&f=?_73vC%HiWdNaJ~-gVvs-fz4EyuDRRPValF!Oosy_jm5WdSiZZ4RqOE%k<6^I=*vEc3ifv zuzz3=Dv@MQ4b;A6qM!KT61!N$Sb!4|;>f`e2S%Y#1zYlo_a>W6+*+maoc5S|%cr#G%; zWNIWFi4za&Y&1n*?^=0c7Rx^kmXvQxv%g|LWxvDmf@8a*sdKnaeQJw@(D^G#@efJG_DR-KCg!?Pk&#wDj9b75a z^B>|&aQ@(U#WC8k+#ayMrndB$y}50*xR`U}N(rqVCci}gxF7WGPFLGfE4D{0si}cg?@^kq8@Y3+0@PW|G@P*KvP=oO3P?ym0(8Hl?!HuETdP}|wIYTu zdWZUl{s>JAEtS=f9=fP;wQz;-D7CUv!^Nt}?C^rfz{r9~MDM@|eV@mosX`@T;&;^3 zzbHh1p17uaY|q=jvHxoC?)cQP!ja=>>|E`f?ex0F>k0nlTHyNHRn8sf?(A;w?&w~q zFQ$WgpS##qS9S24z95%-n(KF0E7v4fqU&3y&6Vn`=Pd7h!EwK1isO0vBzr6SZ(@#; zM{}i6#ErrSv&5R)6P>3P?S;tu;j7_#;T@q1A#bQx=vZ)Y@MQ43+Jf4_2LkT~1_$m8 zd=@D2-x+uyP&4poU_#)#@WvKigT*-sB?kyoU^Cvb*0)k>zw6GSL<3+Th=)1J0Ek_aW-=fajbT{ z-E*j!@(9>QIK>g4*GdN6zOXkkCDK(b-@%ASTtK&^*fHgETqPu3xh&=1sUzPp*4}b(D-oG@T;NfAzS!dXsVv%On;@=IN!!#nQ(I8 z!*GSbv`~lOr)u>^2EPrZ2ipdlN|PuFz89LS7WCOre56C<)o@$A=|4#8m@j?;S(BH_ z6Iv|aR5jZ%F=-a*i>&YX!TzMJucv|MW5=7$4;&-hz2mESwz!(XurcvPw=-#m z`{l$Nr5w&-Zs)Plp5|6JRQE` zyTkX4-tcdV!^MpQeTsV&)hrrZR3N)1u3&s_c5d~Y#y3jkpSjWZ=7HRIZ@!hYEa#4# z@i%+l=y>y=o6ek4H?Lpcne#^O-kci+SBfeXzFyMX-^Tw(=-yDX@W$vs=_o^OFFEdZ zzvW)({WJbV((lO~k{>I(yj-ht^~!gWRamp)ybAZEo=bhO;)>Mism&|gm3pxJ|5BHh zA6tG~xrJrgmdQ{4xb)KG*GnHv>Y4OI!ea@|5@Oz0WS~f-Tv; zMtt}_whn4pD#U%EC-7qAxA5`Mqk$g&UrVa{(n^ z*?HMZ3py2lUg9r&t*Df~`#q7aafdw@%kQY}X_V3E=lY?ByBno7e5FC(hSeJGuiLTC zy)}ncPpsOn+K_s48#Qlyu#|EKoT*s0a#8BRGRsOYDYdHn z=_-e-PEMOpd3;)xv^8meR12h)Pdi+3Px-8rZ#^T_(Zx_j7<_FTZC@LW1WJ`$&3ES3 zyT1I|*{nvg9vvqiIWqg;%ZDB~a@Ub(4?lHq#)0E|=Wd_BIeAlH{gJhk{{Cf+_m5U9 ze)=VU^|bS|@@n3^lt19cd&L9t?=6gv8!tA`h+ubDlavD~HOuy^u%p=ny>~uT;o+P< zuXg*f`^rw9J3eVUruDbYuQfT+a(K)6jmFjcw*JBvZCf{N(zf>W@(unA%*>hy82XRzl^&w zYNxNbeC*;sXFfZ*`tYKIZyxM?=*{CRkMBD@>dd7Rj+28AyuJV1-7R;l-{If1dh_yi ztJl2ur+?+YM?2(?&Yc%GKfKY`_U7lAgKZz%>%_J7cME>(c` zi!M(;|KQ~jS>>-JT>s?ChnHuZU4Crc@ioU+9sl^)&--@nxMN%6{l`u{b!kgZXTN&! zat%#9p7??1DaSllzod*x`86BWN~yE2*63zu+t%$lw{P;jgZjnpTz_Xy$B)`BYEEtOo5x$0Z+fL}LG5>Iw5#@Tg=?j2c!ox|mMkxD<=-gCFWOu*+qa`|e%@Wz zh62GI6vSQZ<=G;{@x@5P1pZ}4<|K)ACu`{n-!I|P(fuQfF zV4DB^K%it&!NuHiH+El}pE>)|rc=Ki*}nIO{~p>^W9!sSxqr|9{nb^4dxo9(;G8RS z{J~}W-rV%&9`A+X(~0TBFR#h??dn5CAIE+v^=;+WH7+)<+p>T27n)vg_Gk0>#!oeA z(bCnapj&*O+Wp__S-#7F&f_}O>|VJ`&9+}Rc)HTel&pkVp32@&+(pi1&JK?9v3;S% zMH{b8yR`OfZoKJ&zDN5`Ld;`D3hPGugrwpL00$L039cKpg6*;8{bmb8sd zaQ5_;OZHZ1R=q~;R`usMyw-4Si-s+xH?PogWz$*B4z{@5wnpckUGh7B(fQ6hdNps; z)LY+Pr(uneX^{%u%8oDHpj4Nn>fZm{=NwN)D~7t2yjD;(ugSH`857g@U-~{{+tqg0 z3$CBb?sNT<>%ZK5G`DsBNBP_HhZN=%PGUzW!jgSU*=fKhbf;Y zA1!?@`H7T2%IsHr_+ROriHG99bh}(DVk1NOCGB&czxm*mv01w^GA>lPlzDc@shuaD zKGN^tls&0CzTDhtY-`%_0 z{`cXYmk%93e&bBzvvtnSIJ@lp;*9Cpzvcd&yC~=RoRXZSg}(;mAN0OkIyH4cr506N z)HqP%jT$qmPpCGw+L`Ka)?QP;Ws|#`ec5De(?=UkX?(Tuz(x}qd|9__&C6AOuAEol zkIFTwOsX=eYMsg%sV|q!N;=?~>Rb>vEmAddDfCXHNn~HRQN$Tq8#paHf4%>b|3*ox zlDxvhh4$jvz7D~Ck$!O%?B$)crDOf%_}kuF(-Y41o{8}fCOuWUWAckBwNj>}%uQL9 zGA+4$^4?OZl0pfI#}ex&txJj~?M!?(F~_^veai8>?a}Dz(B)v~zTtF^LxS&J@D%y{qO?lZg3H8{ET*y%%E4nDoV%i$_VbI&|~ zI`{0H^x*k_(i^7_xYQ^8$mKyr!^+ko>hN!!$XaqZqlhq<;Haybg4J5)*IC-RraQKE*nglpPW$ox1>smjlJ_+ zxwc7>=Yrv)>iLs$j$|*)j>~SHy(hcb^(p_y(p7**adpx0nbo+F1PM-Yw<5*eTU?73 zcei2%3dOa!6e;dj+$kO;NRR*t@$Bxn{Abetd*6=RKDqC`d(XWm<92%ajDwjAGhb#p zv&v>&&sqnnqXoWV;t2JM@qsYt$=pqWa#nJ=+`qdIdro_=1Sf~?h!`I=CnhBJL2T#P z6S2Kw2NyUNJtJ~%#I(>Ip-V#ekg>t1gR+BIZ>GDl`+(y!_a|GKxdp6IduyVx->7PQ zhRu4QzQ)+94>U$&wsPF43$#KHOT?^k5I0CT>>TV(_XLOjiIGM9$X_GNMNWyH5}g^7 zSYU9R5+8`~Qt)M=Jq3yQGjOL{r&Z{?!WSJ^D@lB!FI;`jcR{_A8t>u@Bsi#s~ zf9{r2^UI$1=ROa5fBSRk)TUp`Ww*|Fm3=q&bZ$YhvbM(flN`uz=f`ueU89BF&Jvz2 z?m(z7I5BE>fz0>~3AKvdF4irfZQ*Tk9}2XL369Q->QX?8njLc}CM0@TR9r-jP*>1( z$3yNPCKu=@*%}ChM7nldF)+(cRb=f4rL}TbE-BuZLc}@Za`~ftO22L0rP>NX&I#^4 z-pSq!uNssbEQSQa_eBnZt>xE3a}wSroG%tpyizf#=%*q#i(D_v6*^ZaCcaku-2%&E zrbS0a&ItF1TnSp{uH|I-3QTFTn%Pso4eMtmIa=&3CI$#GN7^Zlm0wA}i@ybq`S<%a z_@4TY1TIKtm8F`~>;+$!ru1>@Ayb9v$sXr^f$)h73Q4GJ7DzZV#dA0tV1j@+~#Sckj7~Xq{8wx-w8auU~XpKD`?>20w=^pu>8J~ zR>=3Hw$MR|%1Jq0`CF-_K2X-Hm-TMyQq0vNjN9f|D;enM{m@-JhQC%9qKt9Qm|}I% z7V9(g=jvtHHTtS+l~U4b`KlNhxFfy{1pGxrr&ur`Lic-tjl}i7L;vbglv2?Ufy>&$K4mV@A6XYX0FD^fv`xCGRD};$IUw8;c#aC8cW|SK zGGvsMO}19YSZ_>S7KrA`8nc3aSv_JVTTK*+I7)Ss&vC18Ldc_J- z8g4llgObg4!rjx~tbwrFY)m%y5g*B|UqlfU`6Xo@K8is82%<)e3 z%I@3lc+Y#!NYvG@K{Z0+LX(0`SQ5CPt3fY==DLr%kGUPrJv~k^i^N9D5EUqwior&>c83uW1t=bR8KQK!GfA(Y%`vidRc)`$&{``A(5~N?KS)>Q zx@vQ|t5#QOp%qrgYME+^@?0Ay6_;8;4?jFGU27;`Rf}rdAvHT|YvA8EL0zTYRiniV zN?G-REa_v-EPO|V@u%5OKBQSXBc@2aI$7+cyhRNzqzy$V*%iuRUzXfV+UTpK3ep>~ zDCYIeq!m(cvxp+$>XxccdM4+nTa;M2jxt+25~!zU8@tqzW?N$=Ia#Bi{co;Zv2Lqf z^-FT3`CL1poFyKctE_fz@-$i>ZQt&FT1hqcQxsf|&}DC?w$Qhh1g-z4x!+~V7y z#7QmGLFfxM>f5ArjfY=hB6X1cms`ZQaaC~@agKBEbLF^r*KBVSZ<)}k5!0hD$5x5! zQt)bA=h*WxJ7Ste)QMOU`8w=a#2-=1qo&7XMK6dBjZ6wH6LQTn%$dW#5SnmH*bZ09EkP2E4b_BbmCHFB5mXhBQ7J%S=I#z_mV z9^5>%Nl3kr@*yQc+6F)I&JW(>S?n(3Y3q0)jN#LvkxPbOPA#je(MDgby;85s-zoK_ zb$(AEDsOJi^PC4+=dymznvgXOV}jB-g>zTs&d*!ttKbXrpY;z49PrnbWG>}zfbq?_TBV*#fx$&WsiEu zJgCmnlJxDk!>#lddr9rpD_T#Z5%7Ki@)9(_GmY=mlFBaWf;`WktG$&AsKu>t^_|g0 znI&b*XZ)FYmBk-(M*4CBb$w;U>tY+ZegHNn@s_{5xY++Cu)x0tbEb+upSa2YRO~I4 zkzXh^q_O&R=^y2gss<)2)#Ytsi2PjMF5XsH94mpk{#uVzXKQ8E@37rXd9GiO%d1z_ zAX(GODL2%&T1O=k+O`n&zJ5k;qj^or_yK;g)8MCD)og5S)k|1|jUIZgbx-?jMv;?@ zZq#g8lNwUvsb8Q)Ezf`AcMG3gRUKp9th=dj#vLhicJzZ4Pjf$W7Z0MsxG*6iHvD<` zyU?iM?ZMaFQSKYAsm|fiju1ZB?LFrifD<)K&U2tc&X%j7#^pf}RBL2o*y{hI_-DVI#vUg*FV| z8P+i(J8Dgo5qls$AwIs)+xVlg{+P;Ne#8 z#)MoSWT6=vaUJ{7D*8YE_O6mZ4hn}gQ(8^mg z%?o-YX=9I5$&%)M>lJZBFAiM$XR`-Agm$ro9ltn|ojbfGJjJ}(-rDXxt_RLt!WLn! zsNIp)c76cwrnJ=wnO2R6V|fMsehSCikvY0&5vX4cU! zsW`F@P`=;8Yx)lSf@Tvox5_rEI<)kC=_jzw#8M^5-H4QNk@$`rMxCdAWoENq*`54; zz7}utmHFMAmw&}O9MgsJp2wb4&)pCzbWG?kp*uq_gcJ<^Ie3J(sdtmt6O`$l7W6W- zRmfk#y~9IrW@s?u8RTy4cqfeHw=n5k57@}QXEk~S^9!|LMhX3m`d)1&FAz72eqXMyl+?uk zD6m6GlE$l_)NHwwmM9IAwn%G%ke%Rr2RZu1SH?fr{}lG^UlbN|lg^mePS%30RoX(m ziBVEHqpFVO!N(O z^PiRKDx4N;erNrFxE>z}2X&SRhZlY#{vnID11?5uBYle5%H0=U2^}3{9S+AoLNI@p zd%+flZst7`3{C$dK9N7eABIdiDZJ&s7s7ay3R8jE0lU~Ah9MV-TD^39M?z(^%u|w zn{odn_5rMdci?gmOAn(s@)dk2KEXb_1ooj$^jUTalK|W51FpaDhMRz2O>P@^o~g+` zq(3oh@x=@9eQ$(3M_GQH^NG+vXeo?g&vFNt<@9NK64epX@fiJ{8P24##pyoGYI+3h zZp#skqlC2<+K6f9K!Y{P10i!Ah=d{Vmi!DW^>clNeo*VK?obBE#ne!={Wh>01se?z z+aO@{*G+A*R!5nxwv&^jG3rsZy0%~Y*GMtOLjIH1DC3iUR2vED7Nw0*e^*EA4PfDJ zVSduD=|}ZW>LG-Eov)nKY)jQOt)%){J*@RozEe&0iPBcNryQ5Fq%TT+*t^Ck@1+Da z33IMd+I{tmx?FFMdBkLGkovFOQ{gc({y}Yw@nDXz%X)0QGv^Wej792Ry_h-*bCpJr zd>xfG>U({TSsL+MR$5^|Po33HYK@hmQXM&2j8jUfoc6|y1-ASHS&sNiuWreDveI9# zu1%6J>P~Hyw#AsGze4PVZ~8I<@lEwM#3SPo>bYWc)$G_+KBEUblaEsU=nN`}Sx&Ve zhtWe||G!HWAZt+#nbmA3F~ZIdN93*?!v^eyO>1wG*=R~+!x$FZZTVn8_#xNn`5~f_TQ#LE^OXwoll(o z9jT6j&Q^{u!VF=(km&eJ&;_rnqw|Ghr}MGngz&F$g}Z|utq1C(NyXq@i%E&@$25cm z_!RXoJboU)lO)~fWflew>K|i(p`o7f<_n{xz5@2yx7s_sF;Fwd^ubzl)SqKoX}u9_ z!L5OtQH&prx;ISwqvb0!An-?@_S8{$Pw(leAeF z6-@^|ozy?;e;O;~rb=~vnAB2jXWY^3=xIZVW@J}bBzIcT+9KO*Bp^9gGOF zfmxOqMv3rLNHnV$RWu3pd$yLSZABcBCT36K1IZ&sOhYPFAE&jn!i;3Kula*n)mVpo zEdq2<3Puid5$j@=6=j??u2|)vNjhoA@G&=_h9(i^Ad3;N(VRdGGF$2efh;d?tRZ$l zJ8+1~q+h@q_!~Pv7gN1rJET)Z#lklQ_VzweS zViX=m7KWc$Tkwn+E`nUUZd3tw>^`x``eJM~W&m|@4R*<1Mi9_ioe*gx77<4Lz>~2= z7ucug8jbXFMkhqT*+Mjh7yANY4D3+6<%U$nMYYfEEebA(;)DV}#8)7{oH&a=w>$z9zu#IxF4H@IO) zosi?uVzdle@A=^NxQ00HLJRhUZwY)`6MiLkm^;ZfWzNu5$!Dg+Xr?7XH-1JQq3~+5 z@)-TV5_OmQQ8}orSBA(hq@MB#X@)#mIiYsYx9g?RbB)#SU^MdoFVN7hVMCM;90w~spIh@!UWe-U5QrFvhowYh?s#gw7uIk(Z< z$?A^V^j?PK-xw8%XJjkQN8TWIS=YG=v_dsRu=pcvkdRD2r0-d?%w@!nMny7@-A9Bo z6NC_^JLZG)fU<}rPr|S6z14>zhz4dIwhA+yisWw63y1==LIU~2WRqRtciPl?L7cE? z>J8J9xyQB2Bmu>J9gd?7`%45$q{oxwjaL%!c{~qdBRX z8k<2OayVujOJM)0M(wvA5>JeFHel2X_H}*lBKo4EP7u zq(7CBYJ2sKw!k`S?t(AjKgMt%hx(yBep8N02Yn9lh|(6mcRPTA*=f8te}L9?n7F{- zP8p<^RaQ3{BAzR(Z=4L>DY%Y+828IXT1TqviV#D;a>e4;na~2BR>0rdb zie`!dFBD}Y$qA}YJtgnAHc&8VX)czI`95ijwN!SLP=FdvAJvEXvem6}S?L&am-+-F;Wx#qcwwlOTV5Sa+_gtY?Mifae!aWp^WIc}D_#Gq3RBLUBHwE6cp4 zUBFfCrH4Z=Hi+%RTp}yd@g!o5QEM@0_?g~C6~HKN4td@xM;aK_7orCk$H{eUO|v5T z71)v^B>Z&ABlH8KiauW%Yc+@O^&)BlJaH*;O&}8bKa zNA))#?0z!G$zyyol?CEc?N73s9$|$VTm3%-GJUQ5D+3pFFilWJjSG^c*B3j>lYuoE z0-7a1`nXBC>$0yxXZcJ|B?nUpRv+V|93^hi&r5AYMKQH6);(h;(20F>mWpBU1MP|U zYrC}UK$y5wrOX6ix_0<~4BXe_0vohRL@g;;JfzN7qLlK`=3bR+Ym4PHahmUEb*kRo z7oz`SoRU4pdDZLtlvl)vH!doa5v@J{nuO3%(^>Rd>ei~kD zvje20Dm#rc*1u#2%cuMic%Ju@_(Hp6DO5#zr7=nA?@#bAmMU6{=-c#qs*9eX+*U`y zVjwZwh0>0Vj?-Ke`wM?csORYCZs_ghZ5H$}XdBS?Qds}c=3$@1I)tx@Tott{dUy1~ z=$ny`LpB9Z38^1$MHY&QiD4s}hdgrAu2b&4Ao0E#oF2NvJKCAS2M|M(b&hd%cWw*L zb>Fexn|2hmhuj$N4d+a24iH((sGq4Bj&W>rrJ^u{tYnm7vYp+CAIVwnKLY=1Z>$#D zJuaS0V4kAXUs{c|KK`ym4(DR42vvm-T4;`CRJNRED7%b(Zw->Snp>os`h51a@<4fN zeE=f8kdPCo?yCiKhh&{Iw5(J_KI)va&!C7gnW<{1yesD$_08O$RmJF|bV$Fc2)b@u zk`9OmrK4$4%45YXu7tg?TcC;mhROwEebeP4$>GbZ|uh!I9SyATNFv3ZAb%0atPDKOqD!76!WMNQwI*bG3#oD z_0r@w=_~M@H^fF(Y1*)ol~LpjZMpwPHV)q5d+D?CCL^HVCC}48(BtVypk<#!Kk-kSDBj&*vz| z@7^HyJogGuQ|}@Al%sa&496n(!Kie1N9Xt9JU5jc5A4KU?{7hegUfl#5H}r>5sL)T zR>pdQ*&|l1@MCN_rM}BSWvGok z3+N|GW3~~M!jGgY^AsB_Zh}swyJ|Qq3aU1nm_*H1>Qdi3ezIo4`xVg!6-q$4n$H|1 zw0Pj1gXE@K3+|2eSf5Ir2(U)6z(f59wwqdn?kvVgzgQddrt57fc%MJvdW`N#YUK#kJ)wMdQPb7G78-vBEaS;w|SHEuH zD&W|fktA#>i46&neBw9q^U7>mDK@W|RQUBRQFgm1|mKob8MOR6N&M&n5RV9JNz9 z%*3z_j1otl&E4`DnJHinw4ab#)0nA@yMndlr&M zxHzE=b(1gQSnYYj%rto6JmF@_x$g+SQV+DC4{*Fy5ir z8p=iUC0m-jL?3aak;~L**g6}ct&9`)kWc8_n#2FAFp>%-zGL607wDnnRcQiU!q`iH z4%({B$o*ox*AEcO9Q6#+ddg<_A4wI--ueh?F<(H9mTy73e9^dw=(*J~&(!rWb%i`! zD6TaU-!hebyX7_P&*nz48?{mW!9QOsX^v6vBdX;jWdPa3zc$d?+-^N1d#S<39)FlR zk2$ID!uV!?K-atJh|Xe;^hHt0X0|egyr3mpli=}x*X*jNY44~nN~OGPBGM?wE~4kj zi^S`u0&U-MXxvtk)dHn`pQ#_ol}aB%BR3$%K&Jk|I7OYuthu{cMD@$7*)+Bdc!XL7 zIvD5p)?!I=ma8*tWxr60Kg}w_l{72yKrPF2jdRvveFBl>NG8jhos83BC6sNFK9_jJ z7PD?>%Ylt;Xa0uS&nxo}>7|*&)gU|2JIDZL?*X0X{-n=S{j5g%6|IADUpOh0BNT=u zFKL8%NKaxDo#UxkqNm>5yg^54b9sMIeX6sXLdID~*bMIh%xB62dB2f=sh)wq$VKzG z6Y+^O-CcocBc-s21VVLUJCX&tJfp3&mfFBKq}K7};oCExi&9SN8_faaYT+W#86(~0 zt&jeHm`?m_;FmLCc}dadlgH@F+IU(PmZ>>v5w5N>iKy<-Wer<%~06j;Z zE3Q=;eguJWEt6~bcd43Ll{-q`A*aJu-$~`@QrsMsqb3pu0}N3^ zE1(u|bf7;NaYT#2Iin^uM|UH3!DLwM!;I6)Owuj%AP-o})HQOPp3J8j{f*&#ta?O# zX@*nNG5<|wOVKK~l%A3OI`;@SnX1Y@aJ*FZ`d<6j%T>9hj$jKm9(`{5Nj=S)!)Ci4 zlYY6E$!KT&o9Wg3IOYPeS3au#MlS&3UvV|!R|BEuB0C1EFsAEo193!oS-%#TL6mbq zhD*(v?Yby#XS2w1a$Tl{^Psd#FG%;$3bB)E7O2+lYCL(6oaW+OGMw`%@Ed9u2-TQu)uapSmzW&_Vvu5H-UP%2 z4;-`k;#4Vfer{3uDu2rJCb*S;#-EPywTBe)2U@hikt9d9qCZ zTV0T|(>OsLQNB2Lk`J|3RCVctHX1ROINGV?7|FC-UxXNg_4GDEKWKg?^D_b!^}pHE z#&-4-6R&QUT4@Tm%k03sW7on5XrNJxa}jgINzMw)qO6)M^p!?2)~$XeK9ZkBfxY91 zBwG4LC>NO7!gPAJ(L#x1>T@Y_9qKiCTY6{>pnA)*jU4)(Ba5u;E3Di#t2(!ZrfKD1 zg{W^fAr7*2%vMY%x}=!L4Tsn08}~u$8!?5sFIH0H&;sJ2QMzd@6QU9Eq>%dyztL(Z zwj(Pcf=Bg`y25?4n$_FiRr1m zQ$QX>Xq$WQ~Sj)2BG9Kly(pkRD%=aF`m^+y~nbn)= z#NAW=Fn!8CYJ@r)cCgX8Ly7h7-sUIiJ>4Gu(QoO4{39(?4kD5jAK%d&u6z-CGgqk{ z!1nZYb)wf8&xl4Q7pSK%;ydcO{9I;#V3zNjxzlyaSjL<4TCJ9eMdE9@9U$;U-K5qKZa~#bfT#KyI6n=wsufUiNstF^vXQt zpyZVI1}HX1Ze1W2 zym=LZl3j#!TN|I-SNR%LUg_YPM1KCdkQr=5TRA~j&BO9GYiUj=wNGSIK1B=2v$AJv zpXhqbu;3A9y=xD@z*FVR-~dlVdwKaR3~oN$E#9Od_U$_*r#5kO$|)o7P}9{ zY(Q*)H}pk+VY4Z>m!!PjmG?KG zx1zbvqxj9yK}r*95i=}Xfj)!ak9kgJi{xA0owLL_o@8|mCf87iGVS#kFMV@dE(>4D=<1BX~@CP5|4$8>2TCr=t(%k5TIeAR>(r_|cBY!R3 z*Zph-WLI*nLk4QkG8@VsS2N)lamKUEte;s={+U@ETEyL-y|0F=*Rt#QtA%ai zI`F?xp420BpU9zFRdb)Ghp(PE)0@f?p>uML%!*`+p*fxkzq$@mKS}E|{x$`7w$dW# zFw18iK)keJJWsc`S~-6GnyU76^VWcO{^=8Ib(9bD!S9 z^<&(CtiH-${Or_od`;IvCmHU^?DzT#d_4~c$&vd+%&n|)8MEXxbAM>B=yc!5%!AR- zQhd1`t5+9do=+2-29FS#kSPVrq^CSIqF02B{92}-SFH6YDcT`t3Vi(Kw}XgGy8UOS z(jg)_b#HWqx13@WXs6x@FKhn%{%TMOWf@T;q6Hc6?2wa`LFA1u(5Tp_Z^hLF@xqtX zf)yfiGb-o@v%Y^JaV{YW9x%ypI}@oB8NoXh66~cCGv@ zJw+;Pc6e7s=oq=)VeqXSw_dGy%@++y(16gqmfkw`B-t(gOibk0@_`~|z1)+!EA5=;Rn|H&Dsq-Z8RpmK?Dm2W^WL~$vdfcGIVmXVOL=Eb z+`)`Lm3h?Zj5<-b^3Ldy--&NaNaZ{^Srx=nQBLoNv_--cScWs*_w}9Dk2&-7?T$RN ze|T4`(7P?f`S3F7KZK1Fu6(H*9+atiHwyE=ZH{rh>Y_6VzXeR;XVW`(9c`o*IWab(@n&fL*C3yB}Ix+#TX8u1bG5Wgr_^yj!& zmW;}5nzr2WAjd(BD>;_xpA%$MNiOI1#C8|vvmbN9Gq2`k@S_VJaQOnI-}^Fh*`wYP z#TWZGXOE_|v(@ z3M2?^K5%B&&>qR_3*0WY^~D5!x$EiIx=hc=@?yBuKfI7{L3)?)nRIn6;GG0LUIptG z*)Kd%JYdx{r{s-y{$~CJ6+}4GO{uNz474in#k~3|FOuLpWtA&_GH=7%x}l{4we)Yr zN;3gpovi6HA5?{{C`7y4_Ur5u-N)L2(mN(|Sd;Mnv%AvNjfw@-w{bUn`>L5FjL zfn44uZ4+*X)y+G{hdTavAH<}G4kt@TzYCoE_5+m$`L?FO{k)@J$AuS{-h9)GKH*np z%w7MRPNiuhg{;Ob9b zH=@Y3c~Vq{zkC)^bZGXVK!&4@{*@`H-A?!OIb1KZA=fEuzgp9K{`24{G4hYBr+Ia< z4twvgz3I11MQu7_$z9U3!(NAc$?Qh7Qqt4?;get=W@6f<9ZY%`)zfpua70vBH>4&7 zM!6EHAaDE7_n+o}{>HB~r?StAmh*T0xE4BrlCFE|y|`ua*0(){8)lyRo2$EPfd7EF z#~hjaR+t0iOf}cz&%fnOGHSVMMZabGekm$F60-G$WGd5|T%;~x_xWShKU^E=a=Jhq zR~=bV{JP)+VnMnfHzBj7J_#H=mt%j*a!TL1hrU@3o8KNKnsAE^i0gSp9gD0Hfm5Cg z>Ya}fu9$PO(t$pjpye^O^?u?^=k8p}91t~KuArOl!fB#E&Oy3XQlqkCfr_j^-S8H4 z%j}eIwKdrP11loSkRR!HM$^3RfzCPgBKJfV%=(U>M6k(oiBHbvX8oWf=55x3to>$f zW3B60e7d$cyP~g)&XNbTEA-zH#d6a!8wP*NI?kO6jmY?(S`SEj7Hq1#`SvFGJE>*f1fe9+z&}b@?^x{f zX>H5{K|iY{0<+xdUyHf2B7VvJpbZhn5%al%bUnmeZ6oKYqZ})(NcJbIuXC9>J3T?D zz{Dt=U=pkekxXJP*^eNAkJ0@evRSt}Y`J^KaXR($#PIW{M_Q|5Z&jvc8{>o4?< zi10lW4}@RJkun4BSeDlMK0)cg;CjS|z%K?Q-RyN8#<$-;q;%X}P=N zqM7=}MYg7~QyMAm!CW?1?r-gKl!ivY5$Xm$wUm3aHbaUD*=GieJGtU|OCimSS5|Ts z*&#TRY9(iKnbIz?le4;&PG$*30*9H#RzqVT^V$Emw;fw6^A;J!y;PGBTN<&<={9B* z{RX}}r$QR&J2Er8bNnANje2(VmwIG^25H?-z~u_2r3B)x?-Ol(gtLG+QT?v=(wd5!a!oEm7yxAlH>UI2om zU0zXXacFB;9P6{+K(tum)Nfx07G~$t#?9 zP0vYlB{-S~ZqtR8%oL zWhqQ(Xaa2#$1@)ymh&`fWoV1s1a-L4SSlPkn+Y~*n&~-lw1*xaM?$^_f8&UMfynGb>BZ(kGyulk`UOv}>sHG4B*&L$@$xhvH?W4Y|jZf?^M5 zUc_N0L3(36S3l?dX{r2NvNDax5B~Ma5qd5CCs&T_BFz^EYn@obr8CDBhq65L2as0!%@MRNz~Ez2H*#8UDgXb4|CsZesPC- z+FIi(=SU@P8vV5qu;{jRKX$!hE?M>btAPG|thWf;!1guMjEvmH(oD<}rm+vKDkyK= zA40@%xeUI}tPYQ*M$|NZCE_NGhgI^he~|f*9!NHD-DirM$x^y>ULVDbVViQ(oTtV1 z;xEu+jT6QOk5MJ_Dm-a2fC#uP1d*eS<6;jhn0~-MaIR)+QrolJ1%9M_WVq{h;VGgb z8RS~)IrW3Hz8Ochv_8XIr>)kJ`JJPU3e-VUmJDZ6Vz1()IP)`ml^UW)X{6=Vmyx5P z_jeO2Ez9AEzOq0|W6RQ`p>Yc_&uTfmLM+tFv9pPidQ+;O*;JV)$ox^n;$XDG#sp$J zdxia8H_ZO@Qh$~;mKjTQcSjoWvQtY}zOrkbKaj;u%{r%CuyznMog^$FmYBz2qpU+V zV$Sd?@Q)8D;B2A0k0BH3DkNvrHo!Ij-x+YgV86Z9jgqZa+H$op*N7R)g0V!KM>Uhb znG1v@E1Le1I-|`XGnqJG6?&P6m2j#!UyvEhr)j;Vg+NBE;~zU062aumz%27Nzncwr z)h5lS>!Zo44J`Cw|27E*hXDM5Q|waWHQ7>#vMi~T)|ptY4`+Z*Ff6j0SxlT|9f9A{ zV%{R_D5gGI<*kA2a(V;z#q>kMWb60o$--{(90fzSR9*W_h@=Em2WR}DoDMuu2lF2| z4||$0?-|b{>jWlgpVIt#){Qb*d_WcIghlA^Ljkuo_`{SY8_3VHC!8y=vQa7GQt@niNJwWqBA$a(0fKSR=t1^9pI%U+-PpA*zr_#n+ zKrLrvqrG-Yovk+}uT#)dn%z~GMClEg+g5X6K`lgWeyW2>j3W52#C`3HDnrgxpc~M8 ziOXiX-W~{+uEb@s4AGr>WZhD?7{~P0dTClDo0uuqY9miAZ)(Ij>PPk{(MtOpo-13( zzo;(oC5y**)Q6?wH*okAr@9ak@R#@<5yj6?E$Q0u#BKzhJ--QK8bey)=2-Kd_P1G% zc?s%@e+{qNQ9ok+4m?J4(q(McYne~1{d739ooHs|0!cpG+DElw+fl=f`mja^h)7^{ z$1<_jUGxj+MA7yZbmOFlsN?7%1Y!OL@k?Xz6Bm(Kl3v^kH|9S zpyf42B+Ok*H>xY~4!M(O?g1{QAAOB{Z0rDfFP*4O=K*t76%oEi69dUoSYD-6YX$fQ zE}AXDn>C2|i|UGqJGsQK=0G4-%32iS;nYB+^UvUCSd3^qnZyS2Pel24lh;v?5->8j zLRO&G;Osx(t{HBn;tBg8frb-Miq?kyXJF**xUsD;B(Lo9<0N0ZaA1SDj;foEVvfF zM=Y^2#7@M}?}F=hKq{;Pmrw?B&jk*N?O>rmWPEs8jxflf3pkTreTOKR-3pDjshAy22ZU8Km|8Jl>RfYI6ViRHx7IeWr=g(>aiINk07r8 zE96pJb!Qx}h^SdMmvjfzxSlxE5MNym_aH7S_J6`t8scmIhq@Q_rV3I#wn_X;6Y=O0aBm6ZiUh8NFW_6SIb|v!61mO7VY9?X! z@9^S2n`<%#|G5^D{iS8FZF9ogtTMH+UooFwq)dK3*TMg7g;rrR%h$&Lx0$DHuA4%* zqbjzmV_6UP*~e`@whGv{8EkAmYnzk04N9R6^3!IRZUy{l57dVN$kX1~w;4{lAir%6 zno*FNUGSC;IMW~d1Chs_QN|6xOk>~G8T<9{?C()p9dX4VltLFgu}yx7birLcu~#qu zi|XKtKDeR>O4~yk@O=$&t_i-Z4$8hTO3G#gE{$9$iQKTc#0z7q z2A)?3ch|wcDTy4knK0sz;&HvrWD$b=55;yY?ubVIli&&>uw?Q}i$`9QC?A{S$j)^U zE$@9kze*D1-AlBGw^)7wcZSW8^cBfoz6Y0w&G7LQ$!4KSLTk18KHgyeCghEsY=(@} zXs0KT&OriP$zR$GA}1izj^V5=6<5N4{SWw}uSV3tjqu}H2Q=#+!1&IDAI40uuuMVx z&YzL`f(fmAKBrkfFt7B+zRhsg7Sf^tSXLT?^VDWntA)O!0k*0jRfnu823Z>iPL)td zEDwAOdHiTg0f(Jw+H7zJ`0G;h`QvOZx{qMCOUC*oBD5ufNzRU=@Eq|OpZ%YC?m9T< z?qmBQmJjmN4KTspz>(YkXXpI~e7x7eR=UdIm8ppcg7uLPHUrF$`MK5$OPjB4aDGnOY-~2y+BD?tWGrVxA}$37%!d4* z&3d*AlJ+PVUk;(94x;2Ppv(@U#QtL{JBR)Ah(~)5j4ltshw>2R{uZ?$3AMuJC9_#$ zZN`TM+I zaV%PLJZgI^B!XS9OXt`Bs;K!D^ILUwENy0bTUIo{x;k2f%{Fgy%-eiK&Cu3b;@1%^ z)aIYEndaN#NZWiF*%@uLEoAP0yzxI_%a+wWaL(p*wA<`Y`4XWUwtD04&N$u~+XErT z?IZnhwViC~ZS&6e!O~``>VmC-INC4&$WPesmM;?qVBePYHg|ou{Oj7``mXu!vak9H z$L%w1AS>G9`#WIW5EA4k+}APxsQsVp|5gtzx=OxGv)Q}syr=;wP!73M0`j3M_W$GQ zvUw;gV%xr^Ouj5FiQG)Ub^?-}w}ta1z0Gw~C_jhek<*b#K}a@Vr5Dn{E+rB&LWA74 zlg-AYL0-uDd_6v_C2Rq5ievvV&>85L?W2Cw0Gqid2g#Nr8TjSkjEY)d;(EI;pzv(F zKG;X#gb9gcODMZW#N^ilyN|KC=Hl>;#j%dfmuX>Gm%+E&tYzh}E)Cgk_gcmC_lsk{ z9P+`Im*tQuBH4YOooxOyo6D*pYD^X6n!VKs*V?SK&5_st^#*n=X@X-7^X0ZJ*PG+l z5@ljv)e6_yHN>tfU2x1U&7LSFn`LVh%6K?>Zaa;@ZvvLX!Q5u|B14fT&#f2D`&%*XR@SrWm zZvmcS@6SbAhUnflf9?uw&BJ}R&#k?jg*)e9dlG&#@Ps9}ViiWKOK{yXtnK^kH!a21 z()|0E;@S3D`?nm|uE!m#k=7%v`Tw51-k5*&8eFpiTT8IE@3-%@M`!;%%|7)DUvT z=5?)uc2ynqtSD+#nfz1%HK=rct+YpkA^CNe#i+^V<0H|h+A`0s;bwlTv1_~CLekI< zY_`U?V4t!Xsgkh$1u`@#zei2}{~3z7g!b_WBXS;*y}zJ+oy1e`VKmGjTI*A^urv6j zKn`p9{8||}^Ah(yKnwGuKek()-5Qgz{RLNj!&uIi)b=kEM?T@rHuu(-{P)@N+wR?M zHp#a*{wja@23zlO&3l~titS8%xqYAgvqu9dxGE99=eYh2)^;E8Ccj6ppX0-wpRl(1 zZ9n1tHs9iZj9m8rwb|U>V{M&gCEdk8|@({{9gh`v>VJ(w+S0oX1oC!BcHs zGkbj<o;KXD}1k61*Mpp4e*!o32{+iarGQ3vdr_!dvd0xRf$ zrD~69vQUrTpdO@P{WiaarQu%t3Y*o`fvx}gN_#Y9&vsaRsXg2A;@K?jH!&YF^ZQG? zr?lU0%MhC*+U6#;SxS@f=Ksc6b}wo(zS<11Hvi!RY`sMLzKPm<19kb>|Hp3jD9vWz zv&U;UFs|E)vA6B(yc4bCAd=lq?D5=59N&v=JNoz9{0P+Bv3~$Z_h1IU7Gv_+7=72o zNRtOYbSlQF8HiP#fbsWiw55;8r3jQ`OUTM@=z)eo_S-dR6P8O*Bf91HvhgS}o86Vd znuPVwjzQHL&)A9Umt*|f6;G~? z8AD4%9p8>z%z^&G9ywJ&U9-RM6=cUcjFboFa~99W7wpZ?*L7e{YzE#15wnDMm?NZ_ z9CD=vuCv*Rry!TwR#QTDZk|EU%~g%Vtba`@DYPeSc(hBEqem5;;c!*Jir;2MkI8q48a zK6T0Ua7N!AJ}+(q?Crb4Mz{l?{-jS))A!qMxWaSjPc_iK>CgjxO12HqGIv6m=EEK@ zpgu%kJ2w6sxwinuqCgC3FKBHJfM+)nuJHs~JrQu756$<2es6_#@*F&wWl*oax}O7O zt^}3I6QK2;16r?_z^3w`)!cxq&xLxw4eiDkKZ*jhv;o@f6u^LHD8V*pEd!u_7DE|7 zK*S@jr;%iNpPk~Fb?>%VHI$<(tlPr2KE~P*YWvDKJ+yU_80_b@S*s~ ze^@>8-x+<;4Il362sPxh-AITk5#ULC0TcPg0MbJu_xC*tB!@wO0cs&l~634^?$g`rw8iuU6o+ReHQqCo{bsogKumv0^3pn zTmBa*^o{jCOYg&QKKzystOO$& z|3M2(110HZurJTy3>V?|=kVEwKz_sLGO*mq@csl|@8H;Eu$zgn#itbh5xx}wPDzIE zsc@BII6nU$YVz6r|M}-7(8_#z;<@ljg*`vO5x(!fd#Q)<#TUKFhxYgu&YuAN;Tl}~ zHpBoP!1vei`V4=*5#t8@;_La>;Quu!R*7$PMk2jcM8Vc&_c4`f3@D-4HTvI(riZ`i7WxmgXo z5&?m&f}Z7z_YHzGjRyO23b5I0PM@ z_K)HH6O`;U_+zKwsy;vM6rA}OSeqNLbo|vI!SA#|I#bA9Z%0HpcJ%OI_ z5p)Y2Tpm#fblXjDou?L9IdL%Bo^10etDlCa@*Y~=C8$H6 zm7R!4hEdoFs%+i7!+_;;IGleSD3!OtoKtsTqge}<_6(qc2{4MrK^$==Aeu!m?q;^d zLhZbS_UThNe+9jVj<^Wu2Ms8T1<2wY*cdC!42^@md`jc%fWO59V<#T!VIYj9&jBy+ zfidP(TQpc@pPq67eAfVah==jM6s&D3^yv?9rzk`n+@%`sJs0f4Zm<^_U_Z7&S$2W_ zNq`!91y-{FYAX@8KZRem|HC&M06T1eeGb6BM*&@20}Sy5kkc-}9UEaEpFa0!h>QB7 z$MOGg&nmzHQ{d`8^fDQ?$HO-tQt~NfuK^qp55LZb_mP0W;oEFD^ElXh0(_baWjO$)KLY=cz_Zu_FW;GW{yXXv+|gaYC%#y?&r9`r zvOc`E5B53*`y{~cNzh_^b1?yc2ox}~)xql^>~$8dehWT*g!mhz<+4L*9W*T2N(kFIt+Ra z3+&pYf5>VF{P#tt`a;`^hHC_ao%TR4AOKSGd3C;d6Q55w9opwS*xnsJw+A~F2Cc(4 z;xnPFK9uB}tMG@O<%`3efWEgJ`uPC3>oB;gZ;a5uT;f33L-Fs7qoL1Fggade?;GLF zL;hWv0HvfszeK?3(-BJK564deyL1fh=_cGwFZebNYCH}`gL;@*Z2o_rf&yE7_3Xnn zC*ZjDP%l26dl8h$25r{{PbwMOxEWqFsHM?hd)LA>2f?R-&?82{5dnZXFz_6kpzZpi zIlh@l-#9c8a8GwQe=Ojk@$ktX;qwtQV0=phYtaPbCLYQm18XINdkuo890hgdLrD>E zZ9P~BE!=xE_?tct*jFAODw0AeebGxF`ty0qBd?^KmB`y+HCLlY264E^E z5*mXR$d2YdW;S!)Q3o-fQKk^?V@g5YD@$K)DoW|?AeCJUc%xYsc;}%jD66Y|; zAK+%Ll#y~)C1xhl#93)GY>lw^#RtT zR@=(#Zj;*D-#E|s445dk!#v3WTpn&FA&ZzyEToJi{iN`L*Ab8T3rzer&~6I=)||)A zd%zp_-F@FP#BFmAcjtS*fqKw*)H3u6lo#~{Hx@S&^j{LN+X+LliKHOXL10XFpz^S8 zR4?>+Bw#{_b02etdXqhD#8Bi3)LF~}WFvB4+ew(5><0SD z!-1E5zAFdvDOO=#wS{9|;WYT8jNya=vJe-J2txBA@1n(_v(C2USv-zk_F=I0{h8Z{ z>W-X-i^m=Vw!3mrz?PZ2TJp4$)sec&=5+lM)hYc%(*(;g&)l|L%pJ@pY&PyaFdmjc zZs94{0&6X>=Bf-E&8fN)b1&NjR|%;dzV^r|tc_;i5xoc+s5j&EfT;=GfLb+q4_bwt0JxImnzmWbxzb_9FM)_Ab!Ucf(|JK$7ahe^laz!K+;xP)`NfiN#AtPT zgyEfjx|w2KW}60@N*mh(kqq=Y%y(p~cYyllSmbJWrb|$pa5Z>3 z#sK`-5>$WmSnn~!6t5N5N)HC+PLaFH@y7ZP7>x;by`wj9Q^mUW0S99|%7e|oy(dOe z@~FS4kBL@bdi;aH0>@$`tia57$lUU_d~_prHm(z?BlQGzF+GM-4cTjVkjLBpc-zAY z;%&CRwg&SqYqJgONOVcsyokf-40IrN4X!_aCoUD`k1TL+au(Y()@im+me1xMmRK7H zwAH#HW&`i#34(_>4KyUn$z4gQgv;2;$Q7Ve6%P6U-RyTQv+TvTK*u7F$GZ{n1B=C= zP4ORID+Tt*ZisJ~ndp(Au5}onf#(6MMhTXH z79sk#&Gr0p8$DSrq2r5F=$P-61D4K3?!buAp^!zGjnbpSQ3ue?NCoIJJcRhh6;u&q zvS1L^o`bOR{XOa{@G0q#cTvNjjw-R&@KjtO?j0zF{zR9f%An9`fj1Aorqeo%de!7M`(ZHbT@9yXx?^XfVMtjeKHYV7c7r@Z?40RWU zgKV&!pe=S86^1YYuRw3_eOUA91r5L==W)kZR~Lw9)gaQqBbkk4qde#kTtCR8J%#uH zdX%AU-$HYX!6!bXYOMVFVv>81RkkJp%J-gER$n(to zyG;VBDNms8k0Fm?|BzM0YvgKTXPgqP#+KmZ_y|~!-xKw%P2^P}@TghHPZ%7!1ySPN z093hwmGFnuizvkR#?1k>%ha}gp0B_F z&Tw?G%maZAb*H;rQb5#krPvMxeT( zGQ0!4BivNa6nBert0&b-^;CFw0^{I2ca7KII~yW$uMuldEuI2E?8kv)Q4edfbAaio zALPXGfq%IN9UBcN{AfVyER@S(ThP6NYD5xyr$PUey)lXegi@V&^nz}j6; zSwmydI+Fzy6s?AG8}w-N zQw6jaM=-f(6?QN`u|nES}5WD8RqxU}Tu+-eA8AoVB0|4S9S7 z4;u0wcOgbXeya;uIu))A=SSCZXQq9XZLKxHtTL}L##=tx0uqzR(FNLTIM7#lWYQUW8rSNtYpnX4#*v1}`lV{UZiuOuG2VoO{X5zo8WTXDY8I&5 z-39*kaLa7NemxTQ9k08mtyBjpSt@*MscN3#7${obRaa=8I+h+*0vWm*Mi^@K19k6I zOEhzIf7IdXSuJyw81-`1MwLueqng~(ttq25Ub9QHSJPj6QT3x`QgdFz(T1{yfX21; zepRQ+J5)54Z7-8ler@`uzir%S>xVdiT}E!C3wS7rNRAH34m=pRPccB6C3z=4&d>CF z!6{{Yp<8KU#yEN;c@n9V{FOGDexH%U9?T{%^XLa?OMruW7PXu@j?zN>3aqwsu`*l; zKAG5uJcnFMO=Kw873?UEf*&QQ6&w*s#Xm&Ff^Gr}KaabW%iuNg-uOND+s?l(Kng>I z-NfG{k0f!D8Inx@Ibw_8DBr=;vz2h2JnCr*lafT84H{O{saL6oXg7gdZ7O#yw}3mC zH=ftT$z$=^JK3LEw;3a8vnWZ#FyeZAXIubSuvfrRqtD8v>sROY3b8?yeY8ZdEMih3w83E?N!?TJ}Ht^ggxjbgMxQ} zqs?{!Rw&%^M0;1bi=1?KsP_gk9TS4B0$th^WY#~THF{o|S2L2vv%C4=b2Yy`facWxUf_}+6iIR*hc;alfG zPgt7+aT{5IT7w;g9|ejzeW`CiSz-^BLW`omV1{wtawuFjZ-bwb|5VUl)KB7+PM58f zZ}B7G887%E@S&+?_gq4Q4k}`2Bq0|?(?9WE45c!k}N_C z)->LH+Va-89C~&?(`Mr!^Dzs?w9cGvd1xY9s?2*$rKY9EABNM=H}iF)jJ=HQ^>4H_ zTCR?%Nl~BCo`&c4M!i#gR`psrPKDRmRM(Z;w6`=fb?>ytG&KDm4O%l&yG%1xeOHsF z{iKdn{ZhSG?{7WQYH7XHECI!U$Bnlddo{$@{jO`N9bKJKX{xL*!BQvSRNSN`WXFm zy-JTT=9&VnXYC_g_q0RIo<&NctMOnEI7r#=jY+&aW``mY!{=J z)|GmUj3c=~K{*xw5KqKQ@ZAVX!f9eZQZgxlyoJ)9>Y~1-`O!zx`_hH!y4{m8Q)6zyxA zDCcfa%8qqRax??OZoDJh*$wohhl9rW1y`Kw3h?iyxI#R?UHe>{U7uhLOc$ra&H<*_ zzV^kov$p9rj4jz5W5!w*8=n~8nzk8kg8o*Jk!8?phZ^=8Pv~`ej$xfaZxkBx^d93a z;~~Rz;|tReQynPmRhae|$C|E~!_38|!Nz3pa`O#g`bIt4*xT??cSZY6+h2Q0Jz3pd zov1vl+|ydulH0PY`Ej$M`E6tW#siHz8{ag}YEU=qYD#N*+_G9F)5V!STf4Y+Ajnt~ zkxBoL9q%_%X!B11SFMlYl6zIc&Y!U zziv2dtT&}woYqErvGbK%(N>PgKow$U;7;M22^LV#7)U`;)zp02WBLn5IP)iy!CJ)n z%G$%~$a=#hFtLn8S|{oT@)=?rz7wpinuOfgmg;E_xK!fY>{#O1=pcfgcQ5x3&q&Zu z;iFO5eB2{KAn78xg_2CYL}SqX=%I8p{X6XzO-7qU&8I|CE|U4c#IS;Ng!GwI0i5G4 zq$qL^@?kQKGLUi_*npQ%dQmdS%gCL`S4lF`O;Cp66K;T(?K7MedU+0Tx#~foy&P4J zszEg)dmy8cKF(GaVkua+^WM$gm!9<=wr4A-gr9XIL7{akAntmX64rekb?@`s@tg+r zp=I8gu=4GKm)3^zMgZsdI^d&@0grpP>$_9x(%Q{-qW!pSoNXtlSjAeFnv+ajOiiX~ zrZ6+gd>qu*Zkj)vpIJIvl-6;!YOB)5w_micwJ)*Xv*+4&*<7HF*58_7onWI{YE2r` zD$@pIw&A&Q-c#8Y zjYOaqVxD5>;4c!c5mQM`WHxXzU!#T56X^s-FUAhWOvW(AT|f}MX&tBpIP?s$A0Znj zz+Q)#3QrqQkA*jc|g?ylCEceW-%yLT~>vk&*)^Xm0 zSi>Tl+WMb$gY5u#-fOLwEPU$*&_qkO#98*3)6LJ!$!4=D%hbcX&!jgVF(Qp}<4j|T z!LI)g+6hs=S$9kOO1nxMs=cgHt1qg*!)v(OsG6u+rV3DM&|Q#AQ~2V#YD#c;pv|VP1on{N}a>;C=52&-)8_eCI*AMhP*&)2{Ad zKkm4fxMSQ8L0Kgh7;2BWW&vY(ihF@8)0ybp?_@i#I{vfc?L7N3+jeV`Wh%4*ilxy^ z18X5MZ8se>Wf(`9CYV^})#h;XaoaxX`ay=<`kuN9tyGt(@-iaeQ;<%)2s#g6#I=3L>WfcG7oab^6vTF z1+?xboFz!`JH%bfu3`3MU}p2P}`lOhGGAY7`oM1~m#54f^rBP`6O&ppJSR-GuHAO7vZ@Ok65%1AZtW2b5df z#5<&3%o- z9h7-)*)m}C7-D^5$ukeO+yu?8#imupBPNxplVO4Wn0}4XYJ9B!sTUe<8e>d$y<30Z zaL>*Ym15E7>aM8ak1>p2Q)rak=PdR2E_* z6W9-AoEAm9>lgfc=iWjlGop zm_3Yr6EMX-CXYFbF^yhC1Jy3-cFJCeFN`OSBcuT?WZ=4jI*tMQF9H1%6n9?06RAT5 zqR*k3m|V;c?0ej8{At2P;wQM$R?2;dH$0?wW?W}Vv&!{5aP;)GZMteq4AC*p2c#W)Djvn|~l?)?r3<1c8KguBqt<@f9%+xWU$-BF_$d}J!FKy_PzN|=8*NLp>MRTk(JV4uGa3!W`s=z9uRXkRIQ(OqV_Ck-F?+Y0>XEzPpkRrQL2l|Fl9*V z+~!e0@X@F7b3<^$mAXMSKP!h-peqPv=l+iPOD^tTSe1{@y`K}6#Y|J=^euW^-oK%x zwa(DbKG~Ct9z@RN{Pu4S>=h<&Kd9rC&T(A^N5AgyI($WNH>tmH3+EJLC2bc0;^WRT zK$AlaX`tUK)9=&EO{tbGwqdUS+LoenF?(=yf*+YhGc!)H?{P=^-SHd4t!Mjj_Hw>( zr}Bn#leiPWr*Gul^qb3{&*%H)@uu=Fawl@rIE&aznYUzRBL z&draSZ#CsLMmNl?AJI@-Kdhm-E~j=^{f0VEow-h3ORry4H?eL;?SksS+L*fJ+Kn}P zt7uhmRV9^UEALjYD?%y~%Ac3gON@V?l;oFeERHK!To7HL%o~|EIp;}s_k2s)vBn?T znbs!HCF}|khW?(nE?`sSnyA>A`CT~C{b2Fhh|W#z=LZdz=JP5jr5HENB_?<>tTf$G z^%T`*w7SYgN z5IH__X@|2B5#ggk_XWQTe6DDcBK_wGesk?CJ;V-nk@rK?Di8QYy`ZHP4bj93Tm^n3 zaWgrA+LO_aMdozk#&FHt)qc(3@oo}S^1u5va`Bu^EIRWry$sY(q6l70CsZV2yw~G; z=@8k@nY$P(bxiGR^-EQea%(HOrL_s)bfs}ZV_n0chS3d+8tybag?L_*bO%IyxHib4Z8mk&6H(qMA)pP53wXHQDYvx9SI# z$(4I52<3Cj+$G+>$iKUbrx$(AUz%5tw=b_W_gwa!%=KAsv$|w5GFmdf6k4jI)l1Ch z?H9c9H~|&QiI=u(|E04dYI~Q}(cYNWuGeD*N4K`yEgvN^vDIV)J`OtsO>6sOtuRQ8 zUfo6AOydPZl#yY51ndb5JWJf=pmWl}ImmX2Sqkf}&wBG9etO3vMbI$6NFLe< z<|}q7_pflTd{@xyP;ppQ$gq&LL4^vMqC^@YuJH?CJ*CJ94^am^GaNrG0j6?&vi6U* zUdJ;mGM+JhHFmae?H66I+qR)Iw5_pULw?=ZT3oHFrbErDYFpKls*K8rs*ja7 zE8A6Jt9MoARwPwKRPw6`)sw1w)K=CN*Pg5cDvrjfEvK3XK)T;oWy;MfjjX+;yl*&aim}N& zXq1d}kG)n9BPtXe;@^k&4dN>Qablw1V^I5z0KN0G3_e@Kp2}i zmP?lLmI(7*qgFpa_g&Mbaw_AM9b4u!&ulu-*tzjfW7nqSrqrg6O~|GVjhT()rl!V* zhRF?g>MeD`x@&bo_0#Jo*59b3)aTTZ>tpL%>I3S<^?&R5wdk7Es{E=8m2b)`N`NMDZld*IRNf_{Ptz@5bUW2b<{Ya#@I#`81cEaEN#mKZ|R;^XkE!DoI7b2|p? zHoO>=67PY|S8vKQ$^z&w)wJ%kE7W(?09ZUQhW3!wj^+%gZ8u)-U{B;j-61c(-%6bgjz1$ly>g8PC7L9Fn;@I7o9C3*{U0}bMC{#NlC zu~C#PvI^NkobaWfnXmSv_$}i7vL^my>d?}D5l;3t-s!&N`KWr(GSp`QIl2cTRXL2 zn!7ixZCuijULRH;TGyl2T4SkMUDLH@R*hdxzv=~5qpD1meJej!R#mR4jIW$f*;I+I znpPQCwYYj=^_%K~>Oa-o>bX^&tGZMMR-7p>DjQas`IqtcLDAKM9r^#|{m6ZjlbU@o z8o<(&4;YFXJbGeVu45Q5qM_9eC8f+EaEuC{@&oG5!MmMkkZK4DXXcev=?*)BZ4u9 zF@#ynWU<<@-*VRRg86LW9?>=NP5-5m57HO13-ScTOT{<^M$s($F0GP;OQuV9OU(Y` z{a=WcqT`|yqCKLIqGh7@!a+i*pw90HZz6X9hsl1&JiutB87Rle>qtBz7PL{{;)pmo z=p6Pz-$b^8n%gnYOHgd{bB?lavL=}`jedsfy6M{48kRamnb5klC91hg)3V0X4JYas z)S+rmRQpw}t~gx&vW!$#Qi?2flw_2Y{~h`FRq^!Vx5Xce4-~H}{!pYTOe`E**i!Jm zKvp0v_ysuWLcTL^dft>gT%Lb!yWCE>=W>WSyRtP|#aZ60r#brzFn=Ec>2-L`l-h%} z!|R^buWzhro}}8MEj7}tpRFhC(V)P()!n)6DS8=xEcp!m56oEw`d#sxAy_Ml7oYb( zCc#6kn52hgUg>SwN!eoAT-h4g7Fn1~DYMCEC@K}J6+dJnq$vMA!i)TKJUxfaPG{a` ze5W-~E|FpgzrZg(iX8&IRDgYm>5RDy9?(wYOUSVL0oj$`JtN#@u5G|76YolLjdHI7 zt+y+nfOrIv40$ZOF&?Y}-=BIFU!e};%?=g<4+KpK?nI4 zw7f>?Q`tCqx_q=^lj5mD9#E+0rRbyRt%y?GP;>!R<5a4Jy*_e)i$qe(l#8a=hr2{2pd_|wX%Q3!17sT%SzXjy#G5HaL|k5 zX~kcQ@I|)@0}BTh%+Dvlv->ZvF?VckXLzOLY{@yFb1`Rm&X}BD&?9$cFUz)Pjmjcq z4a@qKY0N0fWas7-hLmor=v(cqolzfI|E&IbKXbO=G%6zTL^It9qMq{nkCt|m^^|{8qzA|XjEa?k*8&d(WCd`7-UST|!UU}f!UP8g z9}21nd>^nw?v@-Crwh_~aeyPwGM>|8=qc2hWIbUqz6e_hF$X5Z7G7Xn=u8v=w6hRi zipL)~`7|&yHPA8GvB)vQ(baL+@zr_SwF%UCjsug517$V0LafX9DkwpGdG@$PhpFS4*;`#j^48mGUyVQ{GW=TA@;G3+NxXH1K)gs-TLX zp~1(3lY`yCCBa*RvB8RIlkJ zQYoPxC&L~9p1J{uTizxW?et?bYhk5vmET@0)|0ZZ~wSzgvr| zxmq=|(pCPuOkcX7R8p0d0J3fYU<8ZVd{=Ql_`kSQJGWo<`iX>xXY8O#MMPr zOKbkC6E^l}-K6ycWw6Cg$fWV)fI?#p>OQWOe3Z%NtrjGUwu{{&j9BFVPU0uKD(@P& zJBS;!Jg6z?Yfw$l)8JqAF{4Tb@tTiB}5Z^6BSrUmqp@0TQq*n)n%yKEs-L>JJS zD6M2Cl@Hg zdT%#@F56;vZ_uZF?&EW4dw?8`R$+SJe-YP^Q$bU+f+1s%;7;`83&Mq)MLWeTf2IF5 zNhj%DX&0GOHd9VhI25J;a?sJB?!khP%^~g(X=rQ6K<}3Xf{5;HbK_UXv9}9-E-8{-`QZFX1iP0jwti}MHpVt|sq0j8q3S^8l8TMxm&^W?QcH0q6~(%uF+~pxZx-Ck z&&jLEEze2IPRU7Q&#&9xU}+wxrW$5hitTvNrjfRtLHeQRVUvkV z={C**eyXTYGEbT(X^^O;;qsA+eSrnRx{&738}Q6thO~tQhph|qhMfbiFg_x_oj9Bs zIxy(IVymrhUdV;{nq{Q<-T!WW~+0c+ER4U96*Q>GnU4_mHjC;-2eS@1-D~A%j3SRYF)u zs-lqSS&Z|n|2V^VEq=QMTA@dT@GtP6EqNnBN$*SRWP=pA0AqkRFe&JC@ad4wp=&~| zp{K)chXsW#4P6;BK6pXUe}U5i0u^LARTd~kOKym}LN3KU{zJcOJQQ~*`#aM?uc2m; zONhmI5B4I)gX)Jo(012jacy(Hz^g=>OTx|VlMh{pZ(Ikooc#Z_Y~ zgDXCj-6|bl68BeCJf%3b$WzEFTu{&}e{Wt!?#P_+*>keaWcJAP&p4evCp|5Vl-`=Q zK5cDUV`|^DBWVv($E5O8XZ?}?nVzEhU6^t(tt9Jkp1Y7-dZU6<^{7f+GqHYhV_M4* z4cbJs|8+@W#efzm!zeNBaMwxq8MAnUgjfCf@{RK4vgb0ZjG(XvWCiaHW49Bu=XZzz z?>!|<5w3^lhlrTmp<`rHhv4>;!aoJy3gF1AB&+-%isMCx_!@36j)CE(OeM`AHV}5< zWjH$a0eTGT975=Aa2*CchAMlM{RPC+{4G)N1g9C#K(@w1V+-Ie1I*p3%v!V1YO)5{ zQynZYfFGy~nJWh&FI`(mQuoE~e9`m_xDh9xM8F8=N$B!JuEgKX%RSbT49I zyHgRF?RT`l7p@K-5|k0JRN<6%7hmN~WNstPKt+0soqVfH-%~S1)l-$A4K@rgUNo-K zMX6u6ls2WeEK)vEK2^pjT`e@_T5XY$V&CIggggbrA5US1XBKH0p$ub0RiHLtf{Ekk zXqKA&h|`ai$9%xL$4TP7=drn7#&MV{c9Qm!Xp}*;?ev$-G|n|%Pkw@54Cg7M1w4e_ z)K`o(>{xCYcOG{zX92gPV819%B9xU#jnc8wR>=d2pHw1UC#{jM2uKW~g(^e4hH8SY z22{w7OD9QZOMgkG`=<*B`1R#BvG1`zv1Tym(<3PziCrP5@|CyVdBhHBxE6~!+=4Yf z0Z&@5I^8m&8QC-uA_NQTAte+~vpOALD)|e&>BJPFnVf z_3rc=TSC~25wCvzrqRoiJ~ zm(VWHA`5$@53K4F*?nfG5ADl4@H&i&{?Tn?+9F{D#9Y1 z3{-<~mdCBpE&nM?H3q%Hx>H}!65CqXaIg8SF2nfC_R#*w@z*ic(bFr2EJL=Jf!u@3 zpdS**MIn9*I0Kn%YAoXc?~Wi;I#QY;>?W8cK*ICVvG1^UGh!gJTg{2%EoH@%4r9Xs z&n_lbQM=QeFz5dZR|EPIy$KIU&!~wsJ7Wg(IlC)+C2JPTz*xfBFSshr3HTYfH^`zG zAU`O(t%wO;5V|A0Is8WW&hQCgze0`%M+c<_Py$%;T=@h=i{y`RH1{v$ClKj_>5bIE zhIV8t`k+QE+1Z+TGFR< zciDuBm*p8{b4q{x75+U~Of4c6a*NIt%*-##i_SflwJ-f(`ux-lsd4ESQ|N!X|1SBa zND_ROB(DCj{QcNZTi(uk*X6A^;lYa`&j-Dj^W$C}xw%Zm*9X@P`FlC{aw)30$nlgv zP`E6ltlOearbvB=Eubj?A9y_aNmtC^fT<1R9YY>>l}8VYo*c2YGcNX2ryY?$+HH|( z#D&r*;Taa5af`f!^pp~VKkFQB+M{{gxLu8~B{+G`ced-=XElz>cU3>CM2#0(I~o?) zd%3@Qv+Ntq2}YB)gFeY7#R8i@UnzLa=tMmUc{mkx9Xk*_(4&%GGL?U(^r)E5_2W(E zX9*JhXGr?@3o^_7e3d-*yd;xwl{ur?x<05B`-$ZVImYF(@ zL?$;TLkMULE2xm)sQ6*`OZ3Q`IV72L>)%$<=Pn^T$dG5d1PT0keGGp79>_It>m zvfnkyx&4O1iU#Gm9nP7W0`t`Jg=v2SP_SG>}y-T!NTYhIIzbW0W zI*WQEiR3rPzjbJlBKckEE7>?neAwW@gy9*Z$+I_(miHLcVRpNU_UGG8irL?HSij=~ zZ}jGOm>xV^F&kqjr=hGMCmuhl7NeVLYl;TL_wl`;wQ4Sz>A@2 z!Pf&i%btkR1&MwNZZvlf4+rx)ncVC2XwqHWQQ#mw;+48folhKh?6H>jdY*>VI;^E@ z)A$B)9lP>=`I^!mC9!`e7Tqi8ls7!DEmxC03_P=mSyOWpawD?;q=)}`^1CTH{*NYY z+@B@s_N-p%U4Hw0BPHJYIQZj?&sm?+KNo%-@KOCHA))KDQO_wU$E*63s!KK(kH~EI z>)r3B!Xr(C+d^4m8DGVZf&{D{xVz{AieCJ^fG-B>`^*9SUy<-5KV-i^cjACgbp5` zQ^1@^B{0#n+SdeGTeVcbNZ%yvX?DpB^8DVKjDW86Mer*47_ZL28 z`L9R6di>nw7UqJAm7!=&OcaPfGjHs;6tl;7F091E~WFZOZ5P@z_`$~R*y<`~L ziq-5_oz^tz%C!{32h${7c1wEmEQsws)C@CR(`yXBEK_Xdmc>?E+az2a-U-n}6LubH zFpa_-!7UNa3jhJ0koy5H{|pJu|AyeYG(YrQWORr55phAE{NIWf`3+~!<8Bc?mlEW^ zBp%*lW)<}kZWeAJp_X)tu4H~EPsnR!k7JfGPSU&3{!-^s2UE1fLAW!>GEb|s*q&kA zVwIXE7~(aI)|d5;KrTx#; zr19^BZ(hBNdMkYI|2h9t%RBn(+-DmfOujqg;nSb3WykWSq^iH!zb1UV@_xb}TUDbK zjZHvLA`B)=-BYze+A#=@$QZMJRQ$la?yLIrY=1(UDachMb-vgC{Y2Mn@07lS9WfUb z6ZywPT|%FC!woz*z}VwTmrmjDx!*8n5hUCl=1u>%^24GORuK7vZAic zI7%fog>jzSf#1R#%1R_WN428NG@U2)Pa-!eJsEG#UztjkrrFSpDdE?AT?Cu*TRrTT2bs zRVx|?)CJbetJVQ#&n+R9bSu1=9hZJHh4uSvvMiTf=FJmwClt%3cB1*Re&+EwOLmb$@@AB?R zQFFsvg-0lxh!ymS`~d-1gW|+DnD2-e-2$CUi!jZwymf}P&2w+F&d?8SJ=CzZrehtw zIYX0SoNH<|9yYe=_N!pzHS92>99@7Lxi^x5bK|P93QQKEoc4e@hFi_);a4p@DJt`i zl)jUe5 zcfv_rI+lv-inrtYg0iI<^87X-yS6QNI~^GNSIDv{Gd(jTso%HsX)0-0R2K|L+*w*# z`ucBXe*c_XnYfIS)DwR?{i#ivoqYX=_S=S}4xitDFudCb=&k?TudlLS@Dq?J$8*R0 z?33jG@n#|~k@s~$-n81VmI10JU9KhGdQBs04mFKLpXL7zzuYCY-CB94z>~5l$u!xI zc4wmOy=24gjyDX-={%`JLf}LB>hLXHr2TgGvv*(H@Pwh+nC;c()JPjRuszl`{-9Yme z`x)mH_iN8D#3S@Y;9G9N?WJ(&T_AeNh1C-Qf)%2H{;>8#dPrI%Tc9XZFaw7N{#B?0 zh6Zg8TCSKOEt8P^UqX-iD2@=V;&0*K$$~k z+hNsO9%K~O+9q0W8vE)Nt4OUwT9TSNHoULBQr)hyqU28DiTt`;W^QQqu*{5%P8p8W zk3Ua*Kln8y=|SSO#9@hxJ|?|8`FiKuc18Hhh6K#&^+I&`IbH z*mKy!*i>{N%7`pNmLW?~E;JSsgdURmT0gW( zo8#)vR(G$8uUJ_+wNR12D*q#7k=)GGW<;dZQ%v7KeeLm`^<&|;_(ai1_6PR+V@Vg% z#-<~FUr8?gIW_s_&;H59d50@*H!M<~HPl%KTOImOs%+y;ubBBtzCI)M4p*;@u_|SE7m&uXagFlEPxqP-jFwP$- zLrWF>6-+l_fVarH#=6_S5;FIV&c3#-hA_3cX?wk>)?3%HrLlFa^0IPUtG#({)8R&J z^HtR$z0>3mIkjB-0QWCXx_fEceGG-frSzhD=qDM2m=`%&g3A!*7fSa?K1e1>hRC$? z3fU1^pdu{jVNjOhnS?D8!#pNg{8T(tv|4Z#p2kErl6jcMqQ+4IV4YGbJ&JmecmXHH z&_N$SiD6?ef>jE~a?rOCh_-{?Gi_AlbJS%}JD7z!i+BL}#>0Rc1BKX+%s@^;=6Mgf zN4Wkvp4l(kO07#RNOPERy3VDV+w!?7uxWI|g&I@E<-}fbP5_O+9C0_s3?*ro7>x?6rD^h>{M15ubnD)azwd3FZb-LEcnh%Dx<^Xf4 zW}~XYv>w@?2XVD_Q9&o9;i4mgE24Ljejy7ZFLoO|Xy}liz3z363f~q`5iqdbtC;jY z&fe4>dC`X>#s~Zo3V0>FPyP=Sqk{_N8GHm|4*r$*f3|Zt(Br&k%PJI{i)wzKU8TzvTqQXW3GTP%IXpc>_2(ob7&9ur_5cKZ*03 zVW(6PX~eB08g(!29`zP^9z=i2+&v))?smAlyUW4hhs$9<+}+&{4GuvAgg}UUh`VPko3)w!-pT#H zXP(*JnH}w}uBz^?dSA8$nYBg0uiyfe#F4npoWYvMT+Wa{M=7kNImHyyJgzyrX|Lg; zmR0XsBdB_U>Hm4K?S}lk+_bDU89!5flKUnW{8|wIjJa>~z1UtGUjiro~qEj^}qN$o)`+b|9+Jz4s=hJD)mwuugdeJ{203O(tqc01to z)Vavb$M1P7O}n<;XY>i_Ij>#!kWRib?;`&rkwsliJp#M4#?9F;<&RS6wYSdG6Bg~QWd)^uqouAN`=N4373DD^AqS0Kp`%-xtJ$Z$+sm(n+Rc_Jr%Z``+U zSH48YM1B7Lsp+FB_H^oy3|;cX9}~avf8370oiesSQ8QV)T3^;wYr1YK)*H2pOcS_I zUG@Y;ge~(sbTBvl8UWNRP>bpkYodMVv|VekSzC6ER~KFm^sDl?HpHbU+#6T zKhFUXIG;GPfoD-n-UsJUIeR+t@kcQS5^2Vu#>CN*jF_ycg zkw%s|oN#i;c1Nm$@4s$+)O=MZ$6YQ{ycB^RVQ(X6wuugp2s-V3#$E45dJPRa5$+xt z9Cp}q?)sB9$>%s}#09F&QVa&*@e@wKpJ6dlTQvne@+E~`y z+T?EXM)uYc%+t3v^|aiz^2s>+a#j%M1FI9!$XLQ&#qTV-AVw)mGmx1)N5T`&MAq4M zhkSW|Wd~)gj42%_$`F1Qagl4bMSN0REY1|$h39y4I27BPJAgNfw}-oibC`XE^^BQL ztRr?4XNiYIG2zX8LX-lRpLfvSph*AG-q$_&?(uGqJ)iit4LIuG)<@&6 zasH}~SG`d3RE5fUs`kqB4i%#Q0y8(7y@qw3*+iI-#j(LIBLk2h)zx+wSd|^Abn=d^ zjcqr1&HjpciaU?LglFdFaqGC#;XQU0jFjw@KT!rNddWnR*}@dTZ=qGvT7FTTwIB|W2>Wu6=C01&mpe6g zKu&R1dglGCk-z8WAI*PVkYBK{s1GtYXH;a@{I5Z-o7c3>GzyIGJ*;N(A|c{sio;}E zT!sT$mBWdBy}{Ju4Pj5pyxBzMOp!JVPP8 zGXh7^2a#5^4>>ZYgr9_k!f_&-IMTsccHH5uWVa{?yL-ADlwKwp>dh)eW^|44_iCiH|rVeLF-ZLK5MmwZCQvI(M0oS z%Xw=Q@*gag&z5tRwit!(Ycaq#yK5a`9e^y#hUQr(S99WrIaSCyE*Gk)!>@DtEw6E}d;o8ENg;j+J9WLBb)T?-9 zabvNjWK!A7a`(#S>Vx$k8g4aaXgeAHXx?QyY+hmG+HJ%b&N$vX{w={rNLG$GTDm}P zR!me5R!w#cRj*X5)K?s99oMR(oVvIqIp1@7ifu|YP_<9FP?@2y$@AsT@{SHxi4qZ& z8$_3oNt`cuD{L!zE^IGw;lJRu!x+F#K@i5d2~mk~ji|j?FB&X5h7yiIhVu`HP14hd z5f~!gC^>{{XvHGQ6^TmvL^4w>7flf=g@=XRL~=yOCsjbY$FRP#p0ir9 z?h*?ad5GV0v-hVulhbVNZGYI@!RV#4*vu=<{mmWBT2o)s)#g}W609}#2l|Gm@m3Sr zG|(7keAYCysROdjx;0%heAFA!bHAhet$U(#)~U1uHQL5UjpG`ZH*BjnBA4)c-JH7K zwSU)?R)2!eA5fiDHK;1La!^%|s=ZZy)n!#sj)G#0J60Hte}zK~22j}rL+aYrN06&HzHORh_fN;Zloi&BLa&dP1d z^{QX0e^m<{y&Q+Co+)1{4Dwa-Me;QHEX4@LW%)kYZRs`45-gJJMP_wpj6y0UtHe)4 zYlIA8XW@3?0pZ`mFyS#lJ%2UwZlibsysOYf@f?3>nbwJw4C|=9qwIPa9<1oSDhO|aPiD!td zqAQ|hh$LDmY!VC={NT^w^Z6fnTX{B&qJ(izB8T+_Tg{#hzakQd40DK2Kr%Uq{Hm8! zTk1U-N|xE8Z0l^{DBC$!Uu(K$4Or&BnD?4Tn60M6CO1=B^P%Q0&2iw}OljKD^k>u9 zrp`b!*lrkRa5QN2TD=MR%6(AY705EpL;kWxA7pSf2nHv}0X4E}~DJ>SsH zFvhR}(vJu^prd#qlW|>BGRA7>8q8;s{uBg8hTNl2gpt%3a7C${#K0Cftf_?}4J>q72bm zwCwScC(xQ3rH`cohq(?19lkqI4gs>UvdzdL{D>^mOR_uIB4k#~bPadVN#{tlIAevx zSMosoLF6ymh5T@q@T*`2`i7VIZ2mtODcQ?)<;HT>V1();yEXeFi@_SodY1R39IFdwe9^|X1|YOIf~Q<2B@(6Yob9d=D@NjC2Snnt#1m+22vfJub$ ztoY{VDD7n^Z@n?YSa0-%=9q_a=>bMgRkK^OPqU)g1^*Qp>x^cj!I*D+Z`^L2iV~cK z=&AGB%aNwPP^OzrdrVJFOtZx7VRkpO%&lQ_=b$Ae z11Id1g=;-xHCgA|Ds3xBDR8=@>`NH_#7|-#uuKvl zYpzvon`qmER#s&TBR4>sg;6&t3$+M1#@&EYTfi7dTqm-L1wbZ~gT3)6E0-k&!{cW5 zWp*P*KhJQ=Ibv=kcPMua_bOL{So_nwN*=`%@jYlzj&+5z8- ztV67=tWqp;tijd+*1lNV;{UCNwJh#%odxa(NbXfW_gVodSr>h_jSubAZ4ur z_S8$uZ!GEfzuaQ6@T`H>fzV3ZfS-2PT4Z$uf9N!5fU~v>wtSlmm^|b0RL95$vM;p@ zPx}mfRzf`cPCTA7Ff^|-5-^6n2K}f4qKN>G88aK%zvo$(Sl?I%^jbHvbJ$Ltp`1>f z9-M`oi<}gW9e6~Zh?4FD?%N~i17F~t;~wWmL*{>TQ@L-sZ@7uvY;FbC!^jWu`(0HK(6b^E@W+BIkU={G0ZE>b<6=wAu|y@y2ap){0)@SxjV7$^?S*@_xsMpj+YC1I;>nvahFsKYNj{HQXk~*@HtOZVPI+;x-lL_Q= z@({TRHFK9N1QAgNTEKjAD)|@oUL_AB(|jRn>n?K5W63A@wT`@q>*&ckd_E=5k@2L7 zkokibz!z*?9a}~j?*ie|2&7IIVA%12 zg+b$=o(C>l6~hTVw^_s<;uettHr@`*Ud*{@vjt3NR%g}_$n*}@an=de2OyN`SuX4l zWJwNUk7EyH`ysDh#O}&o%w7wWg+c59VA(|jXRiz`z?aQt|6*NaEn|&k%>d@YCDwh` zKGr~%jFrbc$6U#r$NZDIgt?G8hACjaL@kE_->eo(GlK#q^#N#N8rR+(*r_$ZD1HL$ z+Pgsey92b^0YI3`03(DH?3~)U5 z;fN2^6CAab`V+_=6R~e0b&k4Cokfi;#5pul-x2(~1a&tS|F6S;@3C(;j`zWch@O<; zzpG$idqm9xD|s*#fumO8jOVy>92JA(qi}}@*!qdP*MToV1ZkcMwBD^i<=qcd)f}ME zh6DfgAK+tuVWfj2Af^SQ+lyF9942l6abN+l0;2~fi8n+RkPw1^bl8nKlsOhigTqi# z3xS<_ig}T_1Hb-a9$@~cTsqQtt#7>=0nx(pb_oN8^8tB-tqWoxmd=3j_q3 znc)?1fVX0+9eUP|V0&=KCkZ~eQk1}Rl=mH!=T(&D18~&EV10lxK2Kf8z6U6`o6tx{ zA*p+jb25t>1{>QMSgXCT%to0H0m4Qk6@Zd$kMdnY{f)BSg>@audpeM#rvSI*U+A?w z@PW{rdD*zY9e`W)a% zwgEowcp$>G0)j0A*pZIFxSfMhnipWCp&4ra0*)`uZgl{--k(rMzrjqj5AaaA~##niZi7tlfWnNE%@3(kycVVC`mLl;R?u z_Ykm^e*sOGM%+IHtmjC(6l@zKfZDtn*oLlX!2n93oWZZ5gESRDzP~|=pQ9{~VD$6| zwS!uTQfmWCI*1yH677q;b2Vil)sR?U=z_kGQ8%gq`X&#$!%q4Gt3U&9JQmy8WEN?J z7GXe-xKiF!7|sx46e@t~1_YGuRBv3lCw})qJ#9pNEd)|#TkM&KHgyWMx)1eu7A-9k zzD6^cR2nH6O1&4fU>GpEy8w-P2P7s6sMv>qMtvI6pAW2jntzQ0-4lc|8VJPVb-g zFm{{^By*Y-Aq|=?9=rqD;2lT=O2cV9!FeEamjD4d50Y>kt#K11Iu4~g416LK_=81| z<}m2-8t9Hp$TMLt$6L|uxQbGs9*k5)3r}MP_69)nv5yE4aDwIP2_5SXmS%6F1I8fT2@k9x*xQYmjDE!+{1y;==may6*;#~yP+(CL1V-!A z!DW($Yvtog6z)-sYu3Z_`3BAL2@C{ZaL-S;`!}Gq-^bI?d~p3B{r_=^^gvyAgjS(3 zuqn)S(RkGPko#PW*5%+crlo%V1`7E~us)0e>iQh?Z*F318Ok9LSnnO6X9l8G%mQjU z&1pk(PSWf^0a$2Gn;u|K34`Rhqs6p_>`lR$%P_u8=ReYXpYx%E*W;UJwuy!We8%zt zcc2-J-U9Qw1X3hz!OQLnEN>Dx;WcQjfv`T~A)}MPOVAGz8v?mi15?`%7~nL&BF(Kx zbDPjiEi{KT&B<~IUJG3&$?yzM0R4OZ4^lSOh}vu*lBYtJX?BZvJZA%Boze1i%|PImLL%I8B+W@B2Uz5dZ2#SV^1aYDa}#B!+q6gm2&u-^>``@euWzN;57ei4?H;swX8*5dt;&Xpf8T~ zfbUN;!q5zZGzZH*_^aSQ$2ZOBbpa*!5)wpf+9JqUCOnfU*vV+fR|@u4VJU@Ne`+~~ zX0@<0l#oy-!iq>98@B0vxeOg-S3rp1O<1v~46^zHd<3_ku`gkHjO)F_9-2Fc=9Hj$ z1kXd~pMhpSigg!s`%={9X5c~XYtc-baU{)xaRA5Cd}xbMhcp`_&E8A%&$LD@(d=4c zNQVPtM+?pcx}KQWOJk>-P---XOAT}pjh@dz8_=R0f1u=_03l^1IAljM3c&;~gv1)4 zfkrTBR+4Dw&V4A=L~t_nXG8&wJrC&Zp(w{@oNYzjb^=~}8%QaQFwbbgUZ=StXbJYl z_ee<9Mp(G{uuC+z+f+!}Dm?Lq7L@o?kmxPYv*UrNKL_tT6NvRwp^dwuRG*TUS^lIm zs|TZ$60+8kMk3C>0q>mz4wc6!eH|ra&bKcB^92LB9X;`ui=iRh>}q16JrMP?#@+xd zt&jFe;A5E4A`1b~+~Gi0J`B$7Q+SUjb`Rikc11~4pj_(FMhU#Bj;PFk=n z-Nl_YGA3b0Dul5JtOIxP-UspDIJ*ln9!J|-F)S2)t#Od+4n3nfHW&0l|U|lC5c3d4T22!hqdmFS|0=XTL+n(h|<`J(#S%&m?43W z7);^-q>IL;s)hb5h9qhsRsQgvLcxF`gVyu_P9_VVF5tJH4wT?q@Z#4NZx_v?7l5lup|2QddEThQ z5zxpZacmnLMSHLoT%G0urnz4!+^YyW?K7S`3HmG#67mCY@D4JQg#DR#9-51t{*~LZ z?<3yxH8ki=SYet=Y&k|O*1)DrhulrTG7k2rJ8ZcZu#O1$0o5205W67e3tlc_zd^_MhAHf-0F;{yQSNI11w-CO@d-&RqaF3g~BJF!e;rcOn0-CQ)17>GF zo{51mfktfC!$)C(%{dJHs9~_3G=u#zSjr=?zo)?uLfgp8Et=^Y>;ugJk`L?g8Kpv( z6>TqQzA2jN;{(bk278LIrv&SNe4Y&`cbZ|4<|5T%ZysvW`m-AJesYFmPNW2X+Are zljo)xBi{LG}h+kK6Jk1P6KkpNqN%tdtoO^e8@`LKbqNwF~hkG;`K3Y{y|q z#5c`nNb`c`;Z5qHb!k>70jx5O8BJR_eT$x_In`*6-~N!-xoAIoVA=M=g3#QrC%_pJ z)zUU!z?wb6uXnIPAF^!gxX*OG$Ve>VPi^cL2n)?^ltp7VR8h3dF2BIrC z?ilRaF4%L=2R_2iPGFcZ;#KgG=i!)TXpxKH4^BjzoP_pB^N!EJ-j(q5HsS1l z(Ep&Xy97OOntx{$T5Jz^i>=T*3`D#2g74^qUmeiv>VkjT;qyOs_dYnP2evx2Z1=|J zaGX7&Ma`+a@!N;?r#k%Z*kGvmJ_YcmcdBJF%-3_|k3w{gDvn|I;v$gACRcJm` z9`?}OdNdz9&1Cx@J9{W(kY>!HrIMD@NIYXui|0eLsnN`u|FLh=EXqS*&uD(KzwmT4 z6En?9_D{=mt;W9jcng|2b`IJu-FK%Mw`hNfW>Wu;OM5%oGa`C$&IX)I|2l@2LtE4H zXhXL!DsU5R_yJnr|MxPlV4ME$W{cN(5;m89SDGhnBCg&Kt%&xBX`VSX-i`@v$%GxF zSDK5QgHM{7o-Pa8!>1RTx0T+e-`^K?5r`6t#G7|P3D9hX?OWd24<$u&4JmMCn&Y0< z!gSAA-=c?U{?=Ty&T42b+7mTme?FvvW*Vlst!R$nSR4_9b{Yp6iiYGog#5k5-gl6~ z7g%rL_bZ%5GZ7a;JJP+1XzZmKUgNRf3RYPjo<9fIdfOsvG;6I1T9WSd6+)B7U@tg9 zph@MBtWZdn8=jPAOQhLxY5vLHu=lj~>C~cUT=2e;&`13tP1{ftM^Umg!}rk^O?MNT z>=SfbCj6l9sDYEH+s&xw{iuQe%Ae-oruocg!>?b9e-Ggrhwx4NyW?=x5iQa|%jn;b zff+5mI+{b6_I-w0y`0k)-?jHWEGCukGo99*$1GpPe6k|!&B2tb~|zOY8=0% zMNa9S=6oDUk0(vSug&=XJf!UcEXF&$dm0#)e&hQ!Y|Ud_<1wCsX2d;PD@)Em+l zgtJb-lD)QK*P_0emn~Ol>{lSXt66aLwOWr?hf`wL-J_d zb_n)k7BuyA=wAAB67)3fr_F%JHVW$yY)yo&rTJ+4V5ER&Z^S5K4wa6z9wRq0jLkU1 zBKTm`+zFqwJ|ByFFM_R^jWb7JdpNE=0M{Cgt1f9-Ct!acj5!3hjI&3;tMYB(#^qvM znuVw%9>%Iw821UlXio_KQ{dVX_=&QX|78DnM_<^W-WUrS2g|Y^_HZw}u|xRWhZ4Am zR`{PS__w7ken9*Ah&2zkmSzq$wk%BeyR^4UGoDwVohPA$VqtSXpw-f>bul<6uEl$z zImP4fe?0z8YVmJk@c9-0|G*hEuX+;Br~Tj`C|4Telx82zfll}aPw52g>Sov(dNl1R zuKf`8b`(#%6;FE;+cf9sThz#LjJ#}v1wI7Ja~s=t@g7HE;f~-~+UD=a*z7qRu^txx z&z5(-iu2E7rAKM$apnx@sb**e19S>KS|NZWb%C{@xnNg8R<~g}0I9wU=}t$#tN>ad z7OjDn?r5}yVrYai^oQti+aJ&kKha8ZvDCKop_=et4Swg~y!@7Sk%a#;;Zc3V^00@U@=Hbe7s02ZdsBK?K$>@(9w}>v#-!Oj zX{K_THJbJ|`{6FraK{zU61!Sj{1rSM{lwq#UJVRByhjI&`FW#9$%ikg#HR#fBYgZa z!>6o88PwzLb6Vb<_9COu-cO-i_dxs5^8ukKbt&|DGyJ%0JRQx^eH86+68xBs@T6pr z?+WyAo>M0^b=c zvsurM6ID_P7;t^a$m~!~0Kz7JP}P zH!~d3Z=Q;olat7Ks3zo0e`a6IFm1wY$S&YHp9My2A~UrG3%;6Jfmy_C@GfQIGYNRY zznCe^DCQ;RcIFh!qNthG#5?rZSEEm^CK@0M+o1)$@&3_}@Nsx&IcBbJVBV(#Ojazp z8#CAb;7rN{F8p@eXw0ozfdKr`dLNN-JFI_O$6+Qpz)D*5mU=|`R%3Rw0H0N0c`|^h z#~m}PL#<0N&-?{v@etv)0BF+m+~_3CizZ@TZxMQRY-pjUc;m(J_(gcKETS!9mkOD^ zfMZRvrho~^%AUq~!QpWib02cKyiu58&fqaHx4alL$~XBh`QMP|naWS*|HSX>h=uxt z?+B)aqrB0Gg}BH4gImH`45akG*?QJLEDGgjL|Ilr*S|pY*%3tW%qPDgiu0J2XPshs zYW6g5F=?A8fkkSV@q5#BU|H`pxEP}Kv-M8;7~OPTkgimFOS?%s81YX{nsm)$&0Q?l zG&eQBG%W21?O|;-&>!=4GxWv!U52(zzncyjCjuS6!1UaF*s{|)!`7edL%}a+1QB7( zaftOC%h}ESmzT#k3hIPfQN7qE@o^9VTl}5;h~iIRvfflhIzDi0t&UZ9bo!rDqLavZ zzVmtKM_8_6+2-8a+2(Y@DFP9GPU_8$A5`T^mQtzU0loExbOz!|PYWXiPk8>^vur2U z9ztuMLZu)N<0WFi4jOfa>H0)%Z_SMcR{a9-vUpVOt}vFZE3GO%T-3F&CjU#`#@`cj z`{s1c?vphtb9u(ObXNNPwA!>!>G>Ipvr@80YjS7v)J{h8{x6kQ|>*=r-$!G zKaKyHKwS_oWMJsbuqmxRgcnE5iJTs(j8sJWMcO0QM(hbc*XnB6hS1=U6G52)LH?h8 zKlns>Pw{&1(bv7q^?*yH^A&ZDDns!_7AMUSe-*Ca8#!xOe-Y2^#bl~=ySZ0$c+(Pn zsg`Q|Qop*kbM=tQk>#yQ%ZpAF?9O|a`!)MyW<>gvlr>39628Z~{p=8z6Z<08^nL4( z!1x2d^a+EL!jf}Rt^nD>nlV0WYfjzoBl#YM;-cS0AteV&FO`Q^{i=CVAF35MiOt(= zsrI9+Fy04YPidh1yYh$QD<{f@ay#tN*>jfXch6~FbG+C4O!eL3_tbw)pf)%>>`*JW z@Xg_CA~r=XYVFwOYwHfJS42LJI2k@Z?0ax%ptFCx?|GjY-d{YYd0ck?6Uc&L?%Ukf zxUO|6b{^r}-|2+oJ>_xvHHR9>CD9lDe9m}go!yiAVGFUgH*=cBKrkMxyREs>aJ`PI zIa~cl)vSu9(gDSl1&w(@zk_ogWxA%nO36>!nb;#CHh#>{i$A{oEcmrP@kY|H-qc^$W;dmp4YvLaE9*~Q zlh7bpD%+$eQ5HD{JEyqpcJ1J*cD?Hw?bh96vga_b!QQKUy8BHF5CmNf)`d(DyWMI^ z_{fO=Ma+!oA2B=pK&we%R$v894(#qf*O%c_;q}!s+M}n3#(foV0XeP8CP zte)BHb35b>$PXzPSWs5*u5fG7`jYIjafr?RRCi2s+>q8h(ON)7GlO_+(RwK%&r%es z+Blh9Zn(v{J#f3?Ho$$Y$6ua9y}Z1cK7)MU`1KC>6gWJ%Dr8~UrdIpI??$j9mqzx9 zoDfme>O$C<(80lq!0&!>K0@!Ep0V!T-S)fYxD0YpA}4sbQ-5`u;}6H#j*C?<6-Q)i zq)Ws%giZW9?r`>2LTHa5i>>>B4msWU$k12cT{l>p)0k0ztF}jtu4-B3%JQs|u|>fJ z1M=9peSw9og-g;-|?|ZuOMYyhfZMC`6yV)yT&dc zj@q-y9=2(gHm0#gRg=`f*5A=4G`_6o*TvWDsg9`}Q=U+=x@c*^l)PU#t+S4&TT-qk zElX&KFNpvBi=F756p$R9VoS|TQ)Vp9JfC$qyFMoi5$_!f&H(XAj5i-xw4m6pw0-%4 zO81(->q|62hO5ontUv6|>>s>%Avj$ewkY;EE_MFlQslA?IUAv_Ot;-`&)l5cpS!>I z2=jXA-PKp-PXvAll7`$2b#8U4Rb=?l@QLB^t)7K#3AF~t2JZCl<9ot8-SaB&%&xf# zT+g_;x$JO0>{OzDtNx(3s^F_av*@@|SDaVtZCLH*c^y_4TOXA@~TT;8! z6=|XA^D;hVdS;)=S^oPTaC*^)qVGj1#jQ%UWzx!r)vWsb#!B5iW4xu8 z-Np>&9u`E4{bcKu%hVapF)kji+g-(Ob#9UFZtnBkGu^Lwtnr-g)!VzD&k4UD0keWm z1z!&d4%3B2x0)U95I(Y%LzqXXHaIzOg8vtvJ6>Hq54n$YTj<)uWj=V?t~y0J?LwZC z-7(rx>3B~0P7Y2t=}B<{m^p>qL}sr2E7{++*D@IyHdTgAa3X(cW;8yp=hh|Ge6Ai_ zwWz{cdboH|p)`MW?wPE_^p2@7l1?P7|K*v$PTZ5|kTfW{b?U&h3F(oUwOO*9$lToD zT4?c&g?|?=E>sk4FZx;ZzF1nivus{PQ1$$}iH#$5u}yExgQ>$zSMCbI9k4yIm3`C| z&JnahM zlSQVVN!^!Rn&_XDmGozFP)fVh^0et0k*Md3InBBLd0X=z721k67VC?XiUUg?mJp>5 zWu~%-3R~rxng#V8H9vHXO;^k*q?#GaVG07peI5Q({C2zrET3xEP@tW5alh?e;hyT= z?C#>R%_Ge7isw8p2e9tF@J|a!3mhFR3^^MT9r8~|+YlM*IXZylFY~?QwcX>9+i#bh z&V!wLst+KW{=6zj*%wIj?G@b>Y4RPi<5FOti>?U7d>`&*mX^_zinKkk*iECGmo|kM zLiAjn8v5);!?F6!b%$yX)-b9=Dh1{2(x75>p*+7L_j7hbW=VRxG|yCd>WkF1X&2L} zjQd#~a;D^}^P2O=7d|b@F78?SS6PSh(sHMYy%plhwUq~|o>W)XY_D6_&`BGw-(Y-Z zuD3Z7BF-f~Pi&ON$d9Nzoibei^7!f5&ij>jH{aiWR{wYYU;Qrl&h**hGurnmte7&; z4!>o3$ePfnq1!`Wgq#Z&26Yec^jqt_*dxwurR!+t`;NC19cAaF9O-Rv{+*Q!kz|RL z5}kOSxJr0||C8GZyr(tH7{+um(d^Qk0tT)q{Xy;g#x+QY-chTsc~z58olw=PQVVo$ zMbU!%7r7qU>oXXs|0YO(Z2fBcIO^THw_o0Vh#vi6Pt3m9&hhP2TjhF|T&n%h%ewi?Xn~D z!n=hT0>}G(_MGqHqKJV+Dm=oW6{kofWD(_gOSL16_IlIzfxZB4-Yq(SRgjQ=>)rS-YzX@#Zy}+qd zn!p{(xJ6C_muh?4Bt*PEqTaGn`D3N)6~k5ERC?uR#h<7eiY>?dGTw*LBcTmOgDJ_miYe=PnsJfU}np)^D@qPf}5 z5FSxEc&!h)8@eu79(>NPz@xLf-q~9oBIpIafmYTFh@5|6D-D?znPR@2>KE*_4*BVcYK2TBJ0sb^uEc0T znf0>u7v)DRW2FnK!~qVa%Gr*))jVfMmoZMkYG>tmuxM&{yV&{Q$6Cai$1Y`m6~;Pd zs5kf!!6D&Y+YM@es`IXn+-`v#)!qJXXN4jBjch3+!<=O7thrn#s9RL|u&hr>Uu5MktsttdS7+3bK)TA;`~rU-r5mOZ z)rzW|DxMcr=b5sur5;TD_H%DsXl$Qve|{;7S@yZ}$7`{!$(J%~a(kCPu8wLrV421F zDLEs5qTo3!mYsB7=v=67r+VUG;-zpJcu_pQAcpsboyIQZXNdBomGV)_NL7?-koti7 zx1(0&qiio{NoA6Yz)jPE`QsaZhAh(Qy@$YmTfq5{`j8o|M|BA6G`#(t@Ug)|eGOjQ zy?nfEZduMTO0{CYqK9LRssM*YsqO*DsKq;BC(omG+j14ZhTe$rZ&FnWJON- z|H?zE##Sl8>M5_k&={eW>N|l?*1d60t*9oaVrciU8#(&t)#rtHF6d|%>ie#BU@ye;Y`n+tjXa@fh zR|uY(FPu4i4mg=6i#|$PIRwidDcd^5s&!8N9Iq&LDGD7XJES@smrfMg_(|L_5m#~0 zd3;cp0QbQ8p*LD7+og5L>2kV#&vqkQRR)d;+8A)wZ>-mF4+odlV1YlQe6OJ77bIgH z{*v1HYdLn}g?$3Wq~=+8Mpr}M#*DffHTSAsRY`0As86i%t2I^+tj%mJ(hS$J^bVTn zniz1GUN6=ZJ?`>DXTBCy$ksO0BXqhhO4rd=6(kKg3~^Q;=U)z^dS*!RJHOt>fCCgCuY3c&W{X z5M99Q01y8jzK^^eJRZZ72vkyP-jd)%Pd zF5mWioA6Z{V}JkRtspuu=68JIFC_S;Z^;~2aH+glJAphayu|Iv4i)~EY?o6`Ctagd zOMtl|6|a&^0KVQKagC@(+*iy18*e*@pVBDVe8=fdla)Ul)RN=E1;PvB9I-<1g}apf z1v%uu6jXz!z)rER=If`Z-z3=C-HukbE*sd6+s_EVQC zQxxZwf2+PIraA-&FL0`uS&To3rHlu**3DI#Pqkxe&ooTb4bk1#j0Wc*(a>6J)u-ye z7;bC(HU6v(s(xI_D1Ti%u3&%OsGN-S(A1ZS2NQcGsN>y!{Q7qD^U3I#H}_tV(Q9LK zzj?*yBv<|Vo^H&)S7opFvhC%V$%!l(GR(>x7rN)Veso^%yv(_$lIL9E*{DD)dK0 z`;M)9?CpN&k4t^BIxcRrx79y^_Mk(-xxUvtez?RsXF5J|+V9L%&qBuMBuA^fkawFm zk~54L!l<`RGhfgjY@A)MZ&;(7X>ig#&^9*?ZxCsG_2c!yy3NQfd($|gF09(CW?I#} zieJEZ`lnEmJ128znlPnrieuu}xYn_|KhKM{z3TC%+b@; zN>)=k`P+h#OU`rVBGGsC3FkpBD_rxO{N#a-eVtuhm$*OjNLTAsPEK2u>58K&cZ_UJ zQ8J~C0=2-4r{FjN$3KhnjXje1A2peg!3yH7RC=nz+~oe+z(W!G)^j`m+jG<(zj`pb zq;=TQ`eLipkZl24|K8r8-B!76c3$Z8-sy)+Tjy8m0Of6mE22Z3Pl!d{NZquZZoZ@Y zsp$->^V%@QIKz;j->a={T-h)I2t!kVMEk7nT-~MG=IWPVnEbo;k6Oo?Yn4>V?Bas_ zufJRU-kfy@Ue?OEYcVI^4~X9U=Hu)2pE`XW_vv`N>#v@_USyuj`&BHeJJIZDxNV%t zoQipc2JsBpL+KrvyLz|7K=~P$P^V$euRVnBeVuN)`MbYTr2=g|z%fX%NA^w}i*d@= ztcwJP6U?4Te4+;0bHGs$!j1zk@e_xwN`GYs_u2qK;2&Y8w%m3p?ay_%({@6{frzuA z;X%59_kJh5Z@AxalejN-PH}qYlCIt;7fV^9MD7#LVq%+3V?Ax&X`F{#+4;sGpsOtf zI(HG6`YMnk_s+mJZZHl8Gut}DRKs$8M}uE;2jhgM#fG2So0>cInYB?hNfpX+Uh&KP zgxpV=Yg4Z#i<2(Ki+-;9arkFU;_jsDX&Vjj`d)=);RyGE>%5J9#iZsPOsM+*2^{AnBsryakNlLnsk&ftbw&+`L&PH+A!ZU*Nn zs~bnfJ<9IPO=bTCLP#r&w~r-WQESM4wqT39`E~O?V`x*Mu9JbOZKoNnacuarzO>$4 z|GGA>{&fABI(z;7I`8_a4P?XG#=c-EbI{xYBkl(MB7?$kP+zAzp`UEX)J-xB*YDIk z)+==`+8o_`WN_NG=XHy8pR{xINA-gYPnu>Kry~x8__M}i~5KSeR3JaJFSW=Rq-98O8I9TYON!)%!a_HT%+SQaS%QyvAqs&QE8upi9x zQL?AVORiQNS9VnutLjx791l4Ta-8FM5?SDGjy)aS9IqmGbD6S@VwIdH`|2=2dPcHL z+(9%-_?7oJH;+AswT!ufn9nG}>|+dOJnq=eBfIs8`G7gu6lgvP^nRB4ttlNjd;dfJ znF86`8%%`hg=x9zM{`26%9z^Jz3G_#nEslsosK{+y;{2+T)^uzyBhB`^lzxDKi`ni z@U|hi@j+vlwoaR>y{KEM-)xxP^t5TD@!#gVrU~X(W`?E4!bWy|pp9cwgR!(N<&XKa zHFhOt%IIj;@0cBN2Ob8GxeVDnM_3N*)ofSJcFtkWVeT^CF8*f03xN(C%k9M8$n(vV z9FhK(x`Ai;yu&Y)UYKmILt98@qU5DyhqxIQStDF1Tq~eC_J#?fc~5xvI4tr1BvHb8 z@mpR5XDB;@=}3&W^QgIGrLC5nKz6ZtfS2MKM)hsx49jkdi%DW(0Tnc>`H6ut6a#M| zTE9gr#C(8SJ5SfBS=%^L^P};rMxh>>+In731*=Zos1hQYFq*B2fTh9`JH|H3e$zG2v zN=J52_P;C^>oTh~dp&X+m#}uSs@PnPiX92-`wUr+;E$tcQ#jAKe@hgXycGw9bax7rhNCnFFkkc64PP(~$YCfj44aWrPX z%Mm+Rh#ZqKQ~~(~*=p6~0dgboGHPv&Hh1K1t|Oa(ppi_DqEzm{JfmJCRvStY98TK2#}l1?KKVtc753?ZHY#epEN+Dlml3W|k9) z#1p~`{G8hf6K2Z{H+hM3%@SmE|@8`>ljX$1rK1< zQG*z#DIdg3-k?J4I$b4HL@&aZACzC5|1X+b%W+%)JIwMEBlWiVyO7+%P)@18*Yo>JyX8p9* ztH|4x+lC-F`V?mW6l4eT9_p%q+=T3%X0+&1>ON+PV-Q_E5wnr|Avb^HT_nUK%xVuJ zIwQMl7jimS%nd|7ApxWDdt{^bWlm()U@q?`v!2<=+zPha`K(`C;L>JRW`u43P2 ziIM%hmz59x+x@IG)&ykwxglRq$hySzWrZRSZ~-%euo6y82j*>{#-$??c?)8kn;8*^ zb`B@LV}_YSoMDVa8J|VX&P3$OK+JTPVz&PgX8MaTOFR*CrkNBM+3InKP#uSu*Z!C{ zor2uzT5>XV9hpi>*pk_pV`L$EwTfgSmnNMYN3rZOim)FujEMI`oEWa-^w97j16Gu{y~j3dMbNK!3Y&>h6NFo|aS6+F#m`%K1t#4%_Q2eA-w zBF7QexCu{g#M>;izr}mGVlNw#&qE}@4rFlg!(ebS228+CEcUPldNY^3@jbD@zOZGS*rBo+2BDvu06zF91? z&d!mi$b*#KHV-`N#~|0K$Z}0Yd!ge1YRM*ODp#@>bp*5NBsGidM?Rwr;8%9XTur0x z3i5h46w%dvnPY6*Q75fz zB(pa(*I3q7dpy+#pY`?<;t+Ix86qo|qdrG4{vup}!nd2Lz`G<9|Jdg-+hIQ>ACUzs zh#Y9HbHshdLu68JXRJXs<$lI{Vk5DIp~oEj0Yc1VF!K>tf~*^8oWId7yczxMCd5k| zvCjd1%YJ)51~TH1lkztegLs8k_5+NGi0nWX3u=_lSc3X?gU(|SOCZN-umwXI9#kl5 z;i$bGqYTkIJ&;YIvQGv&nk%&Y2kI;2rrLfA<+vHRN1JVGVmzZG6^?Py0aP(#EAnFL zws)DKvkgJ6)^VxMo&U#FG1%J&3-HETVw9mPoQMV)tkGFh}Fxd%!%K1}304 zWHuxF-XC&n=EGQ`9k zyzM|_(@U_OXYWIJA@U@cScsXZzQS&{Bbj(QBXf@6 zAj^l7ZSTVhVC}ctf4RFb(p<=CuumpZ@mzM>S9>>RKT>5sV_j;yMhKA=J&kBa`&nhLBPA4_ z<@(+-oDso1Wz$hEY#%~n*~Zvy9cg>V+Dc8gOtg1lj%55|hjXUd7cxtE_n0bZl_%s1 zVhQ7mHNjj1b|R)_HCm)U+EAnEh*<^c%D0`e9JQ@BA)=m$v#O~{_Onzk`x+|2T7?!o z${J%yunlKyuwS>;*rQ=-^O$192syJ}Grgd_jzTllAh)L{+URb01w2^WclIO9H^g{S zMXe)?>=%e$_TR|8{DS-$ijjjn=U&K-X$SpFGA+zp_P;<3sp6dGn!);#j*-4V&Leg* zvdafy1oH`SXg6@`IK|ve+z=pG9pJoV_u)nYF^k1<;-2Pi;jZMJ!dSZt`c?$b0(=D% z=L3)#PO{6`e{e?eym)Vb9}vZz$d~hW@;IFCY&Y%-NdA3R8sP@?fxb+CVi6;p*@TGD zkHlE$YzsA=JVG4?#=>AyYE#;rsou7ZmUq_vK&+VT9Fx8Io2Ag2WX(1o z23E*Q;2QR7vg+pPoPg8ROMh4Y+Hl?=H5Ox`Pc8eZe_#><+enl8F^+I8A9 zngQDXX{ObVss5`jts$~@YxV1z=ouYN_*4?jbXnfan!~V|JhIHXRW*;Ha z*%cU5i%_IH=Bs{sGhH08t zwO6E%v&RRwOKuySB(us{m9{hrK~sO6!hK$ zWIA6%G}B??HM>92h4La-F#MUv;a}}Q3uiK|i23}6%x5lQGSAw+n}e;-$sN|TW^U8M zrYB}s%P`}Grrpg8n-}W7H=JnPt0`|NYM9!%tNucrA9}hS>tt}CuPSW{lN zqW(kmy6PKM-z)QLwrIk&D*Za0k9Kt9lE%}Gjn$Hx&JCLyY_&7%Hfa9RK5H0OzoVw8 zCc5@TRZ``%>bo^7%MX?;uliE@tzbgMosvt%g7VzFfAh!XFVAiKosjOFyRoo5U69ow zw@=2$wD8p61ZhHD!lIvJ<6?f8f3=A-f9;o~%DY|it>&Aqpy_slm#(_WyK$Gv%f8b3 z7w5kCm-w>0v-3GOKbKdoOrOubef>2-aY3Cu7kP+0nCgdKb^d0*jX{>+&4JAxW$uJ4 zS8-Nx!(p1_hNzzZKT!!VJ!f+}v-UFka}zjQftT8km1*m0``cw@|G zb)0NsD8mOj{VzoRu7ban4kRU}<&WmDrgUwP_M~P>Lu38A7KZMK`ac^0*i&b(>s5cf z-ccBe3h7`>sU4Ze6X+l%8 zerj`-p}TIR?tH`h`W3aJ+S^sx<&ib>YaKAB#njH%-2yB4hMFFQ`%3bPQ+@}P=M~52 zZz-KvEH6A*bgtw{u@~^~b{Bsw@-B=k8B_Q?Kf8EPc6jFH+?iP`(_iG?&cBwaOP`t< zn|3#8Wb&TG^~qO~PsV@Cc%Qu_r>yO5hx+^UdireD8 zxcjo$0*kx57TMxn+}+*XU0z&E(R$+Abm-wKqs)(*hHNH-`NPPe($bOaKVj&xWYaK zU-FjZ*7?=#$KG#kzE1l#>s$KI%XyUwk{v0|2CkmY8Tt7Iv-21Inwak^m{eHF73TWG zJC5t-CiyztSXTqCv+J$f!Tsl+A6XBK^XoB1I#<7?s9Qkllj55DyI)cSqp%#oiye(wAAK-Pm_Q}PS5 z_vP*_I*~i?`?jB9-#>gm@%h#_A?MML`oHYi*MFpZTbKD1`6#8{Sl%stQ~F)2k8vLh zJ{Hevmz$gGvX_zc{7`|D8i*ZyaRMhUB(~@}8r}xa3*#dX#6LtSenrZzq>;&4rP4~s zDF;$Yl$>0~T5>MZ>O>@@gp@j3LSO1wiSdc=V|z!|3Y}y9WS(nIGc*BpYb%n4m+OD9 z<>{RysjUFrX%>h><#ByqM3181v40wR=o=d14abnWvB(-66c=(ctV#Hmu;`e3F^^;C zBt#^JCk;(niUfhZ#W$8{UNS0WR>`L+-BMPkHZQ%VOs(==D_p7Yv`l1b;}k=&Yw^8e zKg9Ni%5i97(_*e-*NVvrhvLGczlP#Su$D0YW7%!~VeJ}p0Q|=`A;p6?1U9jySQ1Sc zpe!XDwxNa?W-JF%V6t&8DqHLH>AK4LO6)DB2VF!A*D~cKdA4K|o1og^1wo0HLd6(g zQ|D*d308{oM!pDU(fR2=ZS@5 ziq_?{LH5hEpWAW=e;x3p*VpN9+P|;+@!qS9cT?WSeXNifkX19cPSH9SCuXQtiX{hQ z{kK%kmm{@tWPqu1Kw9|SV#bt5Wj0s5RlZln+GRGTq!;%k^iL?6v?Vd2RHt%d%Fe2E zqVkjSj&e6k)k^Zlb&O~nT-w&#oMF43&T*Mpt}--YrwGvY{ey||!+MTyzPrX;aRTZ{xm=F3L_66hAj-X72R-SB3xC_c)_nZ5^t;X3?d5eSY6ubI!>e zSMJEXl7+s)l8$N4CXP{#gl~i^0J$xpK8ts?s}jGDE9r{H z%(uXM$G6?r%1ij#yP{nuJwF{sigbn53v&yr6x_=HpW}Ve)WWK$cQ-0rTEG=0xpuoc zd5=kL)CE|{og=SMSgjMoKu}hx{}|F(63O!$!=A;|h%v^eC!C8N7jKXH8FeSd7)8Wx zjh~aSJ^4klRN`sLnZ>o_v|=f7WOQLz{ZJyLPf&}HtD%#^)`q_iyA@U^BrLG8ZLYDQ z;U6}S+QCevhS4>3IqVVeTtDi3`fSvB7t)W3R;U35Bhhm?J)MzQP4`TH$?&&PHnui> zGIcWdwam7hve}XHIVgBQXuB{u>~%!<$N^F1q7BiFq6b7D!cPU!O=Hi;>__r>!>A46 zJ0s%4+o7MnCwOJ>ufS=xcjh0)9)`25ij3=2^qF0xjGXqdR5ZPpDn@Q4LXku|17x(P z>Tc~8@h`5Y{i){E2e=Yez+ciFmC#buQgRTrgZu`^Q6qJmJW4t(?BV|80>BQuhkl9R z{pOAH-E|iOy?Yt@2Q7=9*@K-+3j$DMA7514*{-Ot=z!~z{X1qlBa157N9MoDKU>J< z?OtimqjB)qz9P~cnuL|q= z2Ydh+hZ(*L?#e#VJIB{lSTB^8`zWW?4#aBAyf-No6}$L~w}RJq9$dX2Mj~E3#U*p8m z#hNBENsEfrPpp?PB>rstri52zy*%~7U}`ir_EwieTk_K`W%0;)E84#QBx?n(J571zVdbY*mAJ~OkJ59~K| z4hA5Nv7WBBuBq<5{+PjIsARfe8epDgwwmvmmxJbPvly(=R*Nk!zz}#Qurqo!NrCCM z2%E(=)H=dC*~SDs3h)HF11|>E2$~!q+iF@k&s6@AJmP>`LD?H=Nq_*(kzdRKYAdRcUO%lj^P z-@>60?d#LOjIT|3+--D%K)tanUw_zIsC>@S#- z-?d;>!H!%K`R5t28^@T zvAi`cG7dIYG43{uHM)#e(-l)yQ!mpsW0LWep@e}nRMIEu4UAc4x5@7J;XYrST@V3qvGrsbrifyHN@W6m)x1&=w| zGSbq}ddj-SI^6mkRM7xSp{b`S-gE;T(ZR;X=tb_tY&{cObQo08riLQ@I(;P4oHnyv zS)R!NdGH20J~u%Vy$)9EXpmk0&=#nN6;^30*Mv7|uMjLO=HGC~xQ1M1kXHNqH1BZl zT~C(#nk(LQ#QDxK#nHsU+an#?iC8?U(JB?8of0>^A$OqNhdeixw7rF5-)( z+OzFO$6*KMdqY6UKvy z8G)`&UwFS3p${1g-_&DSkk_Ezyl#M(W@khkoh)v=5!j^^21rxkfa7^He zfOL#p#jTN+G3Gza|A0RoXzXsds=uP^rE?_eraem@ROokyVJCgPqh=p~Mz>QQ!bF1k8qG8W5!kd9vG(6(H*M_F3m9F1A6nZsKQrLN8^~iF84;Y@Qq}aFG-|yLL4E+iS31x z7*QATA)sQ%bGx{H++yDg-0p!AJkt9MGb{q!<%{0RzTLh8Zx0{CmG|-9`o2=W1HSqo zzmMW}aE-wsF3DfxAM=a(_QEUBmrIND#FluS2Iwir;niottGO8zzsl$v+rf-E41eV) zbdr9l{k4jqHra4iwMU;g0`uFF;QytoX&}i?*7~X}2)G`3rTiS@|6Hj6g#Vdfzl!2L z@N;7%3f%N)p(C&H>!5k)EKUPYH%l5P&48NtuJlqaEq{_a%loBj=yGn9e<^j;1!|Jk z39ryd?Wzn_<|u2FEpU-uRb%074Trxq89v^YM0fQzh&Bh|6RiTDXbo)~`ar4LPo;vo zURfuns}tdOn5F(t*{$}{Z0aIZoU3R*;nppIPydwqU7JLPf;7|tb0;@h6+ZUs)K@(B zOfrQmNojbcp%_m_5T#M`R?z`(sg8j|d#=K$PB4EHl^s%P`H{3<{6iWh9ubgez}Ls@ zriCzAj71-Jx$-Zn^agago1xcOnmS7NVQ1;G^e2sr&9Rn=mg&~+wkNjbwo5iozJN%ZE=4e-NgHS;EVO1&2)x{UgK| zyc5X}8G(ZXN(M{{D6oC9jj(;R=2(-hiutj*hxw;zo=I)6c=dI#gAlFtDgDGxm| z+<+Iln06h;?f-uhy@{Bj7O&2DZVPt z`HuS5dktQPw>fy{U40g?xX<}sc^$rRUqf#tZ&%Mh7(HjZCwiWHk9kk~*5O>b@3VU; z??CT-?^JID7~How2JH0Jd>XjY{|Og_8lZje1bezYekzZ-MX)?euC3Hl4nPgSNp5kR z^rv`HG)woye7Qb6&ZV?DP(#er<|t*=W5~<+fX}xSdUL1o8B`*BfRE6SsDbym8+jT> z)g?INO2cpcKjI4JT2ILJR3@|uO^7fM=*OTRya&{`Gw@~~(VYI?EP5@|aTe>;dCFMz zIl8f1w0wBBRS+B@wl&qMCPMo(~aIu)ucn|8W;t0;3&@l|9CA~gSJy`=y%9_iJ?bO zI&hiu(AT_0E+Yn#2e4l2L3f~6Aqip@#=!2>17bM29rWaCbT#N-ezM2e&+H_O+wa&` zpjxNvZH9w}Z^jPh8|Ea-KFfG(D^%=0TbjZBamu>Ec0Ax(;M>4PL41%i$QoQR$PlzD zND7=0I3{pfAQyNka9Ch?(2PKPz@dPCHp*tU-Z#HAw=};sJu!VTUNNpP4mNz&oAe>N zQ|u<>6Q*H%Uy=i$y4ek8$~4d*9)WebNm(SLiz-i-W=Mm;yZ#F!u_z`>i$PQ$h@;95 zj(8Ry3(ET&_>*IW%NS3WV?F;+cqD8Ux(l;~455y=55DfZ(rWn>MzuZi1$mi#8V>MS zxvw0BeIKGM#3~NiZ%{wf(K6I~%6pKz?xPRZ2A%Z+s0prNZ(JsCV{GaR?ZRsIU;7AZjuiOq1^_xBUUl)Lg3C{g+;&Cn~atj<;%$_DwRR0|m_^~4flF=>uCMi?%P z6iNy~{7bm1zxWP%CwiBG=Kt0G#dXl##52)--d)kX#LalRyF}+}mlN8mS*~Pvf+x#$ z)ioGt3zVmWyM=qNXRRj#j~#&dz+F@Tmihv~pA2_w)B2sEs;8+cg#}gvYq*-iJ=%1(+L$ag7^O*ri0Mw?d zpi|TUl|zKK$pmHCT&OB~Xw$(d*#O$bYG|LnLBa8aJWnmBPNTbMVdgUT>DQR|oS@f1 zGt&Zkx?1Qwwuf`RKK&=%nre-edloWETWny-APLDyQK9ogl0w!8+d?jetj3IgMA&~}YeVjZ z%nnWu8WOxIcuC0U;B7(hE`Z{{$+XS*#n{C3pQ(bWozZOQs{715h8rmwRQcxAcW9xvKHnk z#kfg)b6(=CTtn|}kJpppy@tHAP_dJAQyME(K>pWKezed__#_(TR#1Q3RNp~!QUha8 zSLzJe0k2&RDzZ65Y3d@C2{wVuJY@D#S1>yZC;vq!Hj;iym!~Jt^_UtsLc)=f@eyjf z@nAk|(W)!$Fqa6`PT*QUi3}jlDy^il;wR3)pYWaaRPdehrSKu5CIrjj;!?h=Z-Kk1 ztCKUsX>)yZZAMm3th1aw3@d^Q_T}yy&ZDjaj?(sbMW+heI~#ecNROmoP$mn-h49LE z^NbLyDp~X`gF`2n3aS23aHLAZB~WXjC2EE3t`1iJSzQ(55QC&|X-FYmV8S<12GJQh zz4-&yKsyZ6=(EaTdd{^bEIFFwtJ;8hYB+XaX(H)t0 z##*)omN!8wgBOQ%4f+*0JaB|f4*ncs3@77v#I=oE6+ba*XvE9Vzk@!890`6Ax-6^^ z{pwvI@VW;*4;*TnV(xG1t|#<1<7s^@qXP*btIx0hRppCB!A*Honkc;xjbdXyM9AV2xZS=?&ok#n`)YW58stALIBoyryyG&tuQ|%RBViI1!!*(_)SspF(phDc)JZ%KUCM0sHO^du!Hk}4&orgd zF&Dl~<};0yVELo6O4~_YWBXginFKbFsw;KpCUAi+gOUqHQZ^GwcO}m%smj4O5YUG)|cgMc7wP#tLAb zxP}=+ZPlN}zHZ12RriTy`4sM)Z-Tr~d`& zRhC}MZeTN+afV>_qZ+JC5PwOLGR_%Oe_b8w3*A#F$9?wA@D3CYY9KGlpR}^F?5*XE z_l|XM6*}{GC8TZ&5;xm@)#Y{WEF9teA>H&;Rc}c{g}dBccRkN#?gu|ksH!~I24i-c z%#@{H>k3UB%q=Y&Ew9bbt$S<>18*bcv0c=%@bIXk5qCp#f`5f(B4KlTcr$3z_lC?3 zS`u)~*2{4Yv6gxGzT^AzI+<4|Ksv%%N{sbuHTxmaj;qzg1wG;oGc(@W4` zy6dVLitDcH?(3=$a>~mkl z=swa_4ZW)>1?LK5?bD0Wk;JqaBYH1aDPKbm!3D_45~1$Wo+`aS23jM%1Q~HO*&7Pe zKf$M|jv0OeQ2{HwOramuAtM*mp8KE(W zR0G2^eGaQ*zapdhnOX^Rk(F$L>2lB@b1Q2%Ltm;EQIYQ5hSjjWCUAt2fI4f&Bx z)FYxIR#GOC8+4;BtIQQlJ;7VpDV~ytNX4lPdZ_NYp(pc{-N96%eC!M4I1Oa}B5E;t zat!85he(|+*G!vUm?!Dll4G?NTwU#+^32f8{Mr1}nx@;RM#<@(46e9RsQqBa>yO)t znIh=LM3P)uSSJ~k9?Ug1&|HIc(gWC0T1iqSo>7y?CG=8g63=MwX*U@`7wXm{7e!)H zwFBaO?T$2Bv9mLDS9C@CLm*#|d8a0$Qtm@%^5<;Wwo$ zPRUXK;&`M;{m&bYIplh|1hkCRq&B`XsA+9+l@Ln_adNJ5L7uD)k@xWXFmiU1`YAQk z=FBKMfqp>Vq~@W@Q`wYeS!d~IS!g(99A+tQx^3+p&?mT2$nKz-!TW5Nt@);QrZtu+ z))BTvfu#cDZ0C*hb@|L}W-fad&Z$TGPx=dn26_?I#JbFX(Ca5KSJ-$RsUL_m$|nq^ zJBX~9)yOdVscVO17#A{pTCjES=;v%lYAunb{wXIwB{dKly)w#Qa$j*b+z3n1vwP?K z*SXuZ-m7`{dv5CR=={;9bKE+qz9`ffs|MN}6h_giKB>Y5Wa~t1{o8jH;JWm9YUDBq#5vwR>`8yo!CG>xqg5X;KS6ea@gOn+x zsLP_G^_STEjWscs$z@DyiX)on>eCdn7DS>JR0LJQuh>)> zW-Bp6iKQRPR&A+#Q3%pnKuf+<8>UVqKdKeAYHBY@RVAs1vR}JS6u|3}M*qNU=L*pU zE4yvb#XhCh(a*tM%v9S-pE4;t{n6y)EEg$kxz|kkLkhj@25HyA2(~{wnq=rF&{( z$u}|k!?(eiNd#67$&Id-xUP7YARcu!~DqqA_oX^wVwaPKq9_UPUpK+aV-*XxpWf!21JpuIP;GSJNR~@ zLHXIQyE<8~9smNkK?((^wx^{ZEIUa!tw!yoE z`>EvdJ=NYki>$Z-T%LSTdx*Jyb*3>nS^HaFj4OC^vKBFfEzqrF#?V7aTDyXI!!6~E zW>E)_1<+XbQP#?@go9EYaW}s}UL^)fsY(^xW0Rh7_kVy@~jPeoWnDy3(JS zznCBjehayxTppQB9hJ72{cNXPNJq-jR*}b)?dm?P-i_patXDs?4;UBYPZ~c4WqEJ7 z0LEjr=+zBi53{#ej;cW}*Fx3pnC0{!FHtM#6grNaN8Ld^z&}H)q)^HoDEj}>`Vn{3 zx2Uf-5GzZ44)oXGF_$mWGSxh(KGHXrDF7xxk=lxK zfVDRZwb@Zn>Hn)%0K@Mi#;H-#8*#d{3#+(osNCFvW}&|L(Kmu`BD|2^A=N~ujfUFd z1;+EW;wn_Q21#F0%S#iBD}PFHq5*TKr{WsvEdL(Fy6!@{aDd+{WtshziXPdK|bdAjg(t9c9h5{cEjii8i+}Jk{ke^XXw!1Ns!|A?={gKCKQzWj0oO zKy(B*FkRUrzfv|LA7Cq4g{e!IK~7bDsv%hk#K%(9a#~=X=oq;4;u)UogIuoLYIW2a zo6J=cZ3ekdoljP#w-Ni{rE)9bq(>{M)}$&D<&euhhW^AHqG?hg2Z2F( zn2I20ur3|NF43REvD%l&0JE`ywud}NRYE_oHCvhq1mn>u|1DMmL$xN>xuvKFbfGp{ zt}aXzLd0z89Mpho;2ud+Ey{ZFJ{*yYgwN7lWddluCs2(EQ{se10xcevZ=qW966tOg z)Du{X$MESwEvXC=s%L5^@Tjw5C_JVWe2K_de90F|d3;NL0)Na)dmnp_x%0hOy;ISN ztL#4HXyu4={OA1J^G$3-N0?Hriawk8kGHx;c&93>^>ah|M3;+i9a}Z5qiu_!h`FRo zv_20HPBfLQU#eH~r07=L@IAeNv{9fK<4T)C6qv9Hk|C4ULo-H5OHjr}$i!Yl&1g`fime3)zi4 zif-O>I*Bo|Q{lHPWL!u|3`WYYLK*1R^bbbVT{Fa+p6VO1Bbj^PrdB88smAPDmM|EN zUgkA>jnpZFw2g9_cAdIQA3&mc0ll2ED`S)a=w=*NpTiR}0M))Vs7h@||5k;gv=Zuh zPsqRM9E>M>(VhB)eqIUGJ}V$IupKftpTkXcj~JyTpoSZRF(14^svgGu={QTv&@0J> zRAVNXK2JX&C~6Kypf>QPHGumxlF4TF;YyK#${R){Wgt15UW=>7FlsVgOPe5#Qm#pV z$xgWl759Fmlk5Zc+#0EcI1D|Qw%TzuTN^?xS7ShM7@(Y#FNv#A)0nPaAZ9BRX(KA4 zw%kR@P>u2e$$^=_VFlgJU0~8(BP!}Wy{@HK%q}#%cR1A;1%wOdqxj%$^;xf4r z#z{dLDN^EG{*vzkcM%D!mS&c!R?X5F zDwXZ#G~|+wH}^&osfu&vnR$Wvtnry4SDyo==38)tUhCbuh5ALz8Jb6T`0i?}HdLES!FL@(l-HUeO*%`e ziLT)ioM}Ule5pw*a7-7IW(t*{d>SOAO4sCu(m45>Xu|mMQ8LUpHw znh6i{YpFQ$o^NYIsnzsabg!FfrI11PQJo0qRUs9IYZ(py+Gg#Ok_9I)1y$H4RQ^U0 zvbsoVCLfd6NwVBk$%KRIH1gb9DKYXmIYr8roH&A-qbFMomB=#i*O~D(O8!@FChvxW z^)!x(1o&G1R0Gvp=wEe3hicew_VWU)3;zRQc%POFujUcO4nO89_(NZl6UoimW3?pC z@eQcCe}!xJGFB1-ysTZ&sSPAG9ILa4{`e#V;5*w0FJV)4o8s>Q-$X5@8Y;Q9$u^jK z=ExRwgsb7q$|NZw9)!;Ma4w7R?KOeY?E?|4MXJ^C9EWi3KSk<-9y6Wo==JW<_Th+~ zjZt9`b&OaCg6k-G4m@@(QMcLwciVN+3i@;~Txy@R7Gx254{q5V$T#{4Z*NZ}NHbDf z>E}#OW(Kia{h`vHo`OA3HmUST29Ez z)aq0-jCO6As$?+sMj$fvI*_sS5!8S~h#^WPWxxCd?;-$4BMZmbXXurBNLVFntBFBm zSuGv=;gI}Rxvu(Mvt?0z7^7~1>UAVilKY}3w-&zN6~qprHYvfe*pW)62)M|PLOs@k z&I7^tHch~n-ke>@et;I@K2sHyo$ri47xj)#*7ejWY)Q5v8_do}N~*%9>&~$n+)f#c zhhf=I^j-D=n~jm^C0T_$h%U78QrapO<*}^EsqjzjLZ5Um&Ym9lD}O%hE_l}q;1UkRX!r&Ga}5r$B=uip z5T>CkJP&i|BY2&ANT3XY3eul!V+LV86}_F7a7uRj&1J=v( z!cP451HCgHG%Eohg!8!n2OjBvlou{>0ac)mMerC$MQ{SeOP)#%y# zGX(sp7`w5>Yp~7hkqod7ePutN-Om;O|JiPKxIM$LmP+{jIy!7&9FnTqg5KQ|p6bSA zGPb}B-*x~VQ3IKTEy#w*DkRDOq5r)fBM+{**oGo(*F*FdCxR&OC%TRP9FThWER#W6 zS8xVBf-n3Cj>uVH)lbB2COYF2K^bU?5u+4X{N-`X_Jb393GSZ`hxIV@11CcNvK?bB zJgj&Ie|pXL-~alv*4W<(d*K+{Rrv68vA*(W9;u+x^T^}#XIlCh?r-qr&$fC4H}*F? z!k@hN4bSxM_v5pF=WY12@*KF{X2TQiC-=L-brvKlk1LxNQ7=?(fGI zzyqFzXTF2;%un+_hok5SI+?q0m79fkHw^D>?vDZcJqRl=J@^nLJ^_E4 zlmA_WKvz=&_x-}(y}{o(vCSUba`6m#_-p^RX5&5v?Cf>;>2zaPxnLFPq8GnG!|PNj*JEa zw%`Am`*SP&xjp_A9{(s|z;`{)BlGWU9{*VZ;{yIZ8Se<(0Q~h?YPtR!JBpb^Hkb9H zP5f7Wt~H=1Gmq%HHKR|8GEB#b-So15W%nl_NXnU2PaxbtI z^zdv)k=wvXp|!u^YWJyb`15z;+UZf-Xmc?V&jVi~g}^)o`?ov348$WSAO8jQV3^uL zsjn3hv&l622z6I$2`74pGFY2NzM#%wJ-3Z`q0Ca7V@3CpXijy&r}Yp{@UQZA^&~XB zQRFd@5+)#Zz$@2NIunuVAY}(UhD}h5S*oz|StWret2S2pU`^EqSKkRzp41D3i2~Gt z=Mo$!6wjdtyGN#LKjcQ>Wt7K^@B!Ic87!x%@1YsoSI2}z_sg{mL`=JUnonD zmfBADP7(}odh}yz*`jLJvu}k|XmgYrqrMy+kXFnNQ=xUR#q&-5muO>H# zJjhfbkJ1MAlhzo$`)d3;=^V9QSCgItV$BxRV6MwXqKIioG?w3M7G;t+P?t+PxVvDa zlozfLVaCr)8RVe9hFhexvIu{1LATT}O>5~}EHor-b@)#v5trpQ zYC|THcnJOhL9N7ZV<-(C_G^Z_hM_9-wQNVe3qOTiDaFa};G{WA&eHW}V#JMnu8=Fv zlK*4on3n7NGYh4QSnKC;JNQw^gm0}oX$VC{H=nFX9OZs`+S9}I3G5SHDf$2r%m3{f zD%K|l(`PL$&5KBIf|T~qA(BKG^fjWcka#Jb)D9xIW1JF2q_bBQMX6wVL${Hpk=L|N zUbj@koWblOErwlmTWJ_?R#s^LNR_D%EX95`&7s1D@!BDxhccZSXL!L3h3Y1joT7vg zS9K$qZweR)j8|@;b=SY9O~_nT#dDY!4pOJGf9T8TGsz<55XOc5@;kDxPSI+^6L?P3 zVGE8KuA$P?U*ADq>^kUhoiu|rT+xKzqKKj2*LyXUCw?V-?u3O3X_r5KN%r|kj zl`BSTEFE0ce&EM^cUk!vl@&QYI@5Hf(!HA((SbuFkYoe24xElKnaXrQOWvqFY? zM)R>vncF^xH&&a=b#*;7lnB{RR#K1KB8_LjKkMu|WS>s;3|@dW(QHdIy)7^Q%V*-g z;f8)}%z}W^x=He(FO_~Camqr4z^d`-(Psif>DooNyxklzuV<5EW7|pwhArkcR?i)x zoHIWy`uN>oSRQI5lMGVeC~`+ZwO`d88Nvce4egRzJw8p1%1-!TviHCmxv9RV?IL~4 zy(e&3WPsb|@{%Q>0sUa9Vj2oAvl*1qb;1`cBjm z#%bI_^#WZhl0W6Rs7Y0&-TxVu(1L-(s2!Z+= z=B-Q-_UTh$pyvaQn+V~%_JD0d)mF+|2e40^r-dt6Vf=$G-ZR4~)X1W#Bhqu_1d$~U z<0N9SX&t}TH`F{;H3*NP!mQ189#?afDijvoK+7InQe9wQ&F)#*{b4QAX#N zqSgFnYC4?GSEyL>gFJHuRPRVX%6>T-(KhfE|s8^0T_GJC!1&#lr7_77o| zsdm^K;%0?f*La=wIuW@@}#qew~4KXHsqD9roSQKZY}MIc^ua zybI+Nw!Q9@T*5M(PzD)E}7PGE! ztA3;7J2Q$bmG?s1#3XRDs9olFp6)*{sW+K;VUMLua91&`sExjqIlv)#YBIaLn>}^J zm+aFJJF(IIKQf#c4PEjC%NoX!KY`w_?WHa%2krkePf%-VPmCGfLi9&>iVJ*Sq%z^H z$jpMhM$EC9cjju`TV)8hxTrNotrhxI=7;oDIN(ZG|5mC?Mf7y-tMEbEN!8VV;tshg z&|{Rr?k1)Z=0x8TYL{uXID}teSmQ08b0VNWv#4OLX|?4W+nTCj6v*E8jr{+VeXcb^ zb<JOpjxRC`XY% z7%78eXKknu&s2{bsj9m9ro$sHOmXN8y!#8b*0mTLb9kN zpI|I$e3hg4I+F#)R zj%wtWNJwUuqrIbid*lGBCh=TBRYTsZcUj9Tm&A_B$fC~nH(+BdQBwj#!txalKay&} zJ@G80cqZL@PTkCM;LT>yJ+PK+Z(5`8MAN_wIcd6c~*udULHjE(Jxj{kz4o% z_Th2_NQ_Ml5zr_cCI&FSsA#2uS{h!$D5V~}UJvLmrqW^%zffO?B783IT(C#XJSWBL z6RA;>o3$`KJw@KDrdfpSeQ8vvBB`e3K2=OB!x~AW(%UmYyA-fNx5}JsDXp5sLhUo( z$h(x8u3yP7m7UfR!u}^F3bkF9^QeW*^M}ddDsfMx_7u#fqKHf46p{bldqGb z4Ha8kgNRG{+bKg(IpvYJs+vTHYH76IbcU_2t3~gJr#KCMg(}2EGFLyE zSt1^%Ttp?`RYqt154y5nbaT(Yg~JUo0lgW@_{x%{j^u*qhU6sQcczAQ9cu9f;5v;X zKF~v$b3zQfdI-%$JEje*jIIiOZ%W-$NB7nFAJ$P6{F zH$2rUS%0Zx+?8ZrvirXBok&6`r7kergE&^0Ii zrM4OwwYFyjbw_JmaFjTrTkH*H>l$6ENmnPZD|=5#B_FsC7p!qVBoBdf)WuqpsUtll zD;fkf($|}g(8u!Isu*tpxU)@jdRn|Qss^`)$}lU z8c1cG)lXhQTmU{m6Y(3nNZH8WWkh{bCPLRie@tn{$?^eys}N1SQX5gDb#vj%i#Fm2 zq0T8@;jN>gcLunyv{uDB!h|X*)j`VPb4Y?pWoD_V(n@cHkftv7?vn@UPGg)bB&L$Q zkS#Y+xA`i`Z8*ud7#^?T+8p$=Gni^v*$u@kXt?(vHwIn@Gf|#q*|NqOR4Te>O?_5w zqcn~W=8D0?G*F%o1;rB7*89*|yjd!#{~tM!AEEZv&buyn2WgpP9cGl}wb89SS%rqXn0ts*-Nin&qdQ|6UyDtn1YL{?uM`C3 zgBG!JjZyz4SgNNk+xWrQ(KO$bY*^3i);`N#VmGxx34mJmICYIEMW>Tfsf~sThO1C8 zB(Mw6;g~}mP%ncxHk?k^S2VOU?zB`kFQ#sAZM462{iza!Pk)HE6WtA*?hrFyH-~(r zw4(mdPIHTet4f-81ux6(L_1Xpoxhz-xVlNGHbgx@zO5u0R9uI48!F*)G@LK zX1xk^5{j0Jgi|`oos)XXG13h+8d~Sk%3ajuT8k}&WlBq88~O=Crf5S_>-J)#X%Z=q^5P3=N{f=erpXrgvR zJ!!N$6F#-)66QqwN_n9aE#WNI0O*f6kp(D>k&A;z%Rq0a+H z2EGb%*e+N%1$GPiWHSe51pgaUKKNU(Ct#WNv-y!}k$xAuTR+-J!Ec>lZeq&Rx!EWt z4gIT|pzNJP9rukk7rhin+cYS|Wdc-NFp=aexQu92S89uO#1-OV;Vb`+A0lLeaHo^5 zC_^=a@=cs3{p3%>Z8w9rz%@`_OqF8LwSNyD=ojUPdRZP0MsqwExZ{)+s6cGcioxx1 z3hdhFV5%m7j#&aVrv>=*a!?a136k#{!l#CTY1sn$hhy>%IYN3MT|fuuEqYAdlr7Rq zVLF)YMO;U|yHH8oA|}Zh=;sXsCwDU(X^eIil()xlaO%{i;2bswq2(3W1}B*F@VC&^ zEpjRRpWn#V)Omc;IZzGtX1?HjFHcn@#-duj1+~gZ+=8KgOM{zls3xGFBdS5*mfnRrJ$?AM;E{j_lM1ThkRx)6}27Lglq zyaYj$F$y)~sZ=OEo*f9T;{~XR)5-ptOHNU0N}I)att4@S+{1iEe|#tiylaR`sIF1! zNA$@YiWzj}t)vy&%PQytH>Jv<`ZgNsr}|_HIhGni#-Xco4LzgUWGSiz*7~>LS~d~` z$i*O(wnRVpg?dU^u4PbX(VJZcXPh2hxPz#|fg^${+gQ+~N1>y98m#EE%5n9vd=Rs! zo#=US#5;8>Ji@o2i~laq)e^B9q?q2Qs3tO(2`fDm4owR&K}=A}!>#57GvX<=pZ*55 z(i5-GtuF>DKA&z)#*^zEr9>I16Zdr`IA~qUZmcV>*1T34YjG0NW=bW#)MF_ z_04d#x3gh{i}9lKT1XvK=TUW0Y3@cR!>f6k?nvE*YSl#zVdvnzpCc=vDwQeLhPvu6 z=_^xCYRxXArpf`vCgM|i2yt9kVc5y90(-cz!Wyng)9HVRJ5mF}AT5C7dN}b2?8-5^ z#*9mMmi&{#yOnA6A0b`KmiiF~sM^E|<2-UUyOUWhj;AQ^e)w=_xyC5*+)3F?bd>8; zxypIQW5o#Z9h9W+)q_x(q|jeT4t(?w@-nkg`wKmLSqvc#$@kUAbeuY$Mh-Q( zgRZPT)LF>)^ms~@r_k&8KzLV&Nd2jD@)4>$$q+Z_MAAW@ME9RZcPzK3fdQ4+;saBvz%tBJa9q_O>(<8NL zaKx8r3SE}yM;6g$fRBEd{1f}vPAyR`-(Viin@#&<6 z&I5c$IR7~{jNeAJ*RGQmCIy~@Z@eI^^;~kLc{e(I{4cLlE``Ly82Y?4fMEHG{5G$( z=$2ETn`AfV?=0Np4!39V-<uioHo4p1PpK#A2CYLO>063gK1ih!fO!Z6^EDSl1#3 zbhDK*|39M61m3G@{r}(ToJLVZMYoi4ON9JHL=+8FB8o^-i6T?n)Kx^3TcS)+E~1DC zDaue35_Lsa(Jd7bWhiyd`A+}$d#!r?&wlNF_ID3!uf6tK&-i(s)i|>^cjxZ!cJwE3 zrPsE$+cq$R%o~69+jGm_YS~WokSS&V*z@gQ|JrrVUx)9lD1Rc~Bhfl@S!uP(H&h)| z{iqsi54`xG`)j6ZK78QM2cA`9YDSmQ<~LS=Eyd_?YZcOAh?A##;wmQmsyvEkuj_n7RX+ zxdQrGiTO6JP2C8tWLlvU(ZVe&o+$sYtmWSNWnbxs;o&HsNDUnm@#n^f^~+53C??aBQ$iYR~~Wp9=DW3Gc*+36LRlZW7x@?XgmQKftk zxT5O$tvv0p^rF;4-lrMp{&&*VOYbS2U-Ayt-)pLF$Ntl%O7BdvQjhc#>He83l22wv zr@l=0On-~trnHzYEX`J}s9!!}@7H^~5VR~Kg)hnaW;DKzxR&X`~6?{?`cKO z+L;w+W`EAlz)LwlITOFu{6u%8=dVEs&4vT~9~AM+^iHsM7i0!ynCq2n&5MesCLVxx zzXXz}ys$j?BzTXB;9);QT9}*t4Ak}k75^?@QgJi-!kF^c%C=OruNahToWDMwBbVfx z#5f!Vc4T|@;QVP|nEQZ7C}mEPwV6Z6ru4VWyi9jy?5>}hgT{Gj@)~lsx5B?Oku2L4 z%G~ufaRz1NOEr`+V=(vO6ly%GD@-m6!sy?kDiw?{dw- z`9j$UX7S#dTbsHf`Af;B0 zWA>x6zhm|Pd{^hczAD?W=d9f7+_LPO#bj=2_WJz6`-G&!rMtEDbyw9smDiOXle(tZ8BoC=%$xCh;VH719K)O&r&X;|b!SP{ zbZH@9ej*XI2kvUVyMOsN6_+M=kl*-}DnD0hTQV@Sr#L&C$~^_X`xpEr-y)0rReBQT zZmxDu)q|^aD9L7yNSsbi>0Q}-i+2{rCQFFIKD^R$;_(-yK`j<}Mt-lu({JU;vpJlUq>*jjxU77uX_P;58VB+D@2THHZd{$*_^3%+X$h#pm6^2@@1$*J*C@p6#a2Z9}5TJcnJRmB>ltOXSplMSUFqoN$1c^!7uCKc_A zyDF|NFBIpO?EuerYN2K6_sMRVUyx*u#Xh#C_*`~%@z`u3+a&d8*&&%WiS@amRW7P{ zEOT{gK>2{u&1HAt4X=`0lpa!eFkOJ-1fzhsn3dYa#tjW=Q@HGu2lRX)qwucH$4_Ea&Y3~^84~XR-9S!M&Y80SMjj5 zPA*R$gnhhveiicW#bv8=$AE19h51KTB!;C|CqKkW)+*UM^JM;Z=If|V)atAGd+2pv zrk)@Md`0FGC_!EFKKDr6l%Ii*eMO=JcH}&0xdo~E=mr-RpFqmZ<{E;M+FWSIxVeJ3 zkcEY+sqM&^{fhqsC0ey`cJWA}PrhS5j<*vVv*}b%Y$2OVpG#F#YFPSSx_;^6;;h7* z`Tteqa<`SgmA|=oc;Z1ajCZJfNVT2hMH-O$2;|z<)Luabk(FIIUY z(SYoNx$?8~7iXuV=TFU7$sdvVGkJb`OQosFL8Y71Zx+rdS;aSLpJ-3+*WTH$z^_d$ zPDrK_-AcL_zAil(`Z%U!?%qiyz01z6m|nc^kN1nU_kK{er8uelgya_R5wk%zPflEr zx+ig9u4eI&?CSDg3ysPi#&&mh{(q&dQ!izjRz5fNdddCi-G!3mi^vQm`FG0eRJ>cb zD0@$G2=RT5Du0BJv`T6H6moO%w~9vaw3=lDa}B|ImZ#b>_s$EYKV&MUvgy}zBlE*^ zt@l2kU0AlfdWz0D-B71Qd(N_c_a<2ZT2W`2Z8e)St|dV>_+^`vB_+) zZT6=`{fbg_^h?XfCqBtSc@qDJm92~nO(!QBC;J!s()Y?ME-22YZRRKL%XdwW!^{6x z>0BZ}8)m8}PR|^d9}D+vk{z2oGe5I53-X`LFpd**@9JlasRt@s{=SlQVDUZ@??mvtm#3rF@s%iK*|h zOOS0UihG%x;%PFo%wh(uX33r)9=9{IQMJTqaq1&m#)HmMT^t( z@f#kN8p)h5=N7NX?6a7tr`91QHP3tmZuq6tz+|_?8Ohbm zP;+Q%G_jx`kWr;->gU3-P~v9!7073QV?KrpiB*`MNo2ZZKFHMLD-KI^!M-rJFpaDc zvxqUBnHrXvhF1Jhx&y7y7W>b@;>3J{5mEpz{Z49l`a5hZ^-_~Sf%XF1G8XLQRb*!Q zy)X@&*WZiJLbs1j>>>O^4bcwIN zjAHc)ZjWUQWD|dalqeaWb(~k0eGFu46VNQfnpt z4$g8Y^C&YrRIW$v#lq8IqP9aXUVuK7f=m7?*QmHAH=D?Tn~E#bW0Hp>t&9LWR7xhB z+Y1k1>sSFjI;eO*sO!hlOVMPnDH+VHV~;|2j!rfLW%o$(mc;SIwtSDpdIW7-!YnT> ziw6|mB)`xlpy+NYbWbb-O@23HwW5&AHD<1tl4MhII^9NO<0a`C;CbhP-kqKrlvqIq zo3qnhOS_VBXJmRQNTu=E{Kn>cGK%U5zE4U!) z?@t|*`T+@ZX8Od^cgT@>W_mo|?7Mug!Xvpk*-qG0Pb>UTXpy)S?d^v2FCae7PJU5% zv`{N|c zj_FQ`7YZNbz5+$qykbqZC-b=cm3)dgfw3jOWbP_iP||~TsZ%J)Rmz@?Ub&=VUans; zk@`H-r{pE(NclX|H*;@lP2z&W*ldUFw^*9$5ykQrmbT8!x;VDvR>0s1x}GZLHi!b&;jXE`c$KQlR{vvVuXsrb2KY_6F9PvX7Q z3+d}iI*_&J*7VCrIn8rTh>(8^TXNl8zrxSO#eBCdP|IIa&!tvisl64u&g1z9a<$3K z_4n*j_}=k}`LtNIbf5IP^buHyl8J}%MKZS>S8)is>rmng{#ERqyao$U<&rxxk3h>C z6&vPPXSb6Jwr6fe{yieI@~IOu2bYY^{6PLFDOhhT)GI1}AkuYVc2Vw*{M^DSX1gK6 zG&u}}=uk#LLuPnsk$t7&AK8|<&iTG*-EXlpM)uy3sD-_HBC`?g#2S1{ZYnzLB%(Mk zWu&E2Cy`CEB$Z3{C5ClSz6sfsI%aF+J|v$&7o^O!#LT2X-Cxbv7+rXQsE4!i75Njf z#a|4bwLAFK^`K|xLZ5blshtX@dl|j(?&2lMd8y50$T}@G4aBw`~?u@E3SQ%CLqXhhE$x_n+MQT$=f`p3g796S5S)Z{1A4^nz5D zXtcAL7j^)a?gzn=e!$-Cg{oM>mxtU~_kuY;9F*;7?8`k8b&8$xOR^VMoLT;E`K`3e ze{+u(z9jO#Q|fp!wqyNFHpb(2HM20zBey}*?3(N?`OC4&A~ly=O# z&&XMf*J@(fbTU4lU9mh@uedeQF1;%A9^-#gdJc0MrShLt9AExhS!*&Xbg8Hy+J8G* zcvWU|szc7~7n!wqFgy~Qbhak*47B3yE-###*oIYa9$CESrY=e@C$8!}WRt1+vzh;J z2vHnA5z}=8^Qw&oDLaTIwAMw8d_zTC0cFNj4dykGn+aabx(AG*-mpIU-K}sf}cW+-$(#fk?L7kQ^>cKLM6_8A$hL#g@FwR-$(L z6&4nDW4XA4ne{#gwL6CRgB{?&N8l4V2r1-LqAe0&?)xWpqmlj-Z1Y+$;6?@9#*u1d z?COV=>&C=Nq7GgNkuRg6{%w<|@x~*Q8?ookYcMW}U#(rkbQP#E!mF@^i^4rPr2TSvtFGO-lo*v~cP9(eIhq-Op&e;VOxr+a0-$2g9Q7Jv;%zSwRb9mJtGSKM2d|@e3Px_25CbN5IM%14C zO^lKu#5dH3V%^B|N)ikC27e=m+%<*25ouS4sJ5krlfgUx!8q$dzM0eLNzZ{GuT8CX z7o690B%|gtu6PrgSP3h`6fC2q%sW|RB)te;zYTKvd-SYd!PZx%hc+ZT>@fPwWa5q< z;VaEy?`3qe3n}|RdfRMbCKeU*%yBw`xS>vDXxK^a-}>bIeU%>7IJupBh(8DIyca3v zT5^)^!Gd)|>Txm)UZ1)obtt`R317TV@YJ=qgy&=wz3s|{p`-OT!N@~My~ z`7E@a+KiEA)aoHhPoxe>RiQ7g;BC8e<~LgOdm_ae(2^4=GeOpe3u&|SoJ*6;qOW3e z`afd1HW9P+6){^!5(D}Mb^ANL{<4&LMHfTM>LxcZ#y;TtW)t<8RkS0e4dZ#Q@tn;p z>v{8gXxVznze4OU-9j~3K4RvsKa_9+ZIy7(`YTPQ^s2%zV2oE)+DHs z(I7oRft^E4QgixuHc+^l^tImf!KZn~e4agv`C0qm!LCIQOrYUZAvVPLj$Y&l9!@C} zDSIBJKTnz0BRTdYlBEgP)P=J&C%U8y*WE>3-=;2~Q?r%u*_qrmkSEs%YxgCwGpl*> z5!`nr+{1{wWxVm*P|bVLxw{h^a{}=hhf-#JkV9vMn3&t4tIu-f%ZdA-TIUdXl7!P4 zjkKDmq?PP#VO$dCz_J&Lv4%LFm*9vGqmzygtI=}Cj4k30BkW!;K1v?TJK>vsSVxjS zb`HnCqh`jr>>{>j5t`R%BJDbpWwR-xyp);#4q@+Xj@?N~vnbVt21@^iKx+qNC8Xng#Wq}g>mg^Ga?`~Q=9=`Z@;!G=X#X7WrS3nd!PAuw7w8)>Z)$Jiq zNiF<@M{?CE+|d9kXwKJ#M2C(hmf()U6I}ZxrPU&{YZvO%hSD}s#wA3X9)@+gK5?bJ zh;n_I(rWOuQ@G+pB3HjJ%c@f*b6oFG5jZ9o=+pYYthMsASF@b87RYmyA0 zz4=}=RL-`;Vjx+o})+q2+yg>n5oN_m~;C1N65S`ONeLtBE;A2 z=9)T0%$>}rk1?mU>FwJ?B;0rSp7#(fdLX^^SVnU#o>syr`;lce_pc4HZd({5@-O2( zcT?*EoU8`d)+914h35AwZ?lzUcNmQaGLoC~ETi3evvdVl(vq?(GaK)=5L>vKvAY3j zEJ-OfsoSBHV?^pIYX2q?p5}+1$krJ7-faOR@dY%DrIfXf z8u$)}Q9Q)2j6ic zcYMn8wYoh+?oQAX(5MN#;|uf#qbolO(T1-)aqZ<%{!)Y<#hhPM%@6%O!fPXK{t2~wkL#b}>~s7le*x`byBIUPh4=iDmLWF+ zb$xWMsY)$SNe3|Ac2R3{QE3tKQ|PGQbK-`kZAE_rFbfZ?Mj%w6|DSa{NQC zSP||Q^R<-vf53Hfxb9tQw~+IT*)9q-`GDFiWVIf1Ssq8)cqA+j&_*+88{6p-q=_fP z(MfC{qTTLd8ISyOAHVKn?>^e^pDbf(`N5(6M}c_sD_u?d_d#A6$alMiFEEU~EBQ8; zBJK3${2;!W@w8|0jZY$qyEWUR={4@R3O%GPXHMYA5%f0uo3znL@j6gc3Z z%QK9nmZEUFEti&9R;1)c`b(`mw-MXa9=J;$soO<^|XYm675|bM|uO_E4so z0?f7`-p?9+6H401IeDW;e8NU*zLvL$a-N*#Ginhd>DPwZuMRYP3A9gYz8KoNI&g@0 zpogzPC12y%XVm>mj<00-1lnq>{W6y2;a;QE&8aYptJXm zWc`KY{6mr}RfyWFFIYUOX9l;%zGXah=CP=YkDJycl2Pu6dNAC>Fc+QRw za=F+0q_Hdya$bHko@ESo-cMbp^A^6*bm}z`4)hFfJ(KsI#s62}6R+^!)|k)xFJfzJ zyhF>$`IphU%Q?OfZnA{^Pift+`BIzd1x)`!pEQrfUcR+`q67)4IwK(FlrXo$fhnkjIOg7Z)Y*`A}?sixpw@H z*&odM(1zA6iO;}9#4^*bc3&$KcGJ}YA`g*90Eh&9Oe%g0aY80 z);J37Y62AUfk6Al!!z!HDmrTkbEZNor(k=SfEG9#4)8SV>rmMj*?Wc~&%j$2LapD1 z=E^_bhVH%(z5X1kEoXR(;|rnAE(zn(a5!BIa2)m&M(k#cUw8uOuU(lbn(btwRzLilIF={`ecTS@RPY-=` zDr4;_ek*&u#J*C{GxX)@>^sYYBlHDE;%LUfgGf+g!#Enrh?v5!35o97j2&fOGr24b^!Yu0NzuN5Hc6SjU|j9b67mthKH*yPi7cbEB~acqP`+gxUqkyy z*VlyQ8}>fs_u73F`b$P)l-QNM-=vm~UbXE-+`XKZTug1}b7TQ~%c!OKMqH$O^LS_9 z^9kB*EN$=<@2ZUYC~r55|H{zM&;oP$HJ4*vIfIsqa`j{Uf0QFqlP91sw$@CJOyg;< zP=hzQ$0yIF1i^#8GyBq2wzwx-aO5{4f!$m8h(G=jJX62~DS^=$+W+^Wxpi~*oCIg3!d^XGR zEPO46)|BF|im>PXu4eHu1CXVsun7IZzE>t$Qs`mI;JbLPbWmzo#??u#vkulGQpYT3 zJZj#by&SbxW$gb*y*I)iSJ4uic)Lwu`!(NIjwm1A$i5aFIhmS|y`qNq~P!I?ae$RrI*e>7rvzZ_a%Q zgzybLSG`VN?HAY@zQs1`zBA~zDc+?353seX&}Q<6JnxtaTtTUG~si4i0-4bGBQ!`a-Dw zAeO;_n%@E?Q%6ukkb0d*d0n}-2PG)04u$p);*MS{ms4^d%IFekoO0`h;f|jCzZ{Oy zgQXjF??4H18TrVS++jWXQ>Uv#nbzlO7Bdf-iN=gGW|;9=*TQWgpScCft45->q9pzh z>r`m+1A*^IpUtxMEOcDDAE~tz{Q+q9XsGnqK)1a@n*R)ECUez&yytM^k$C=K{@)g) z_Ph8$gmnVvC$T(29hA+R(4H;H`jz7;r$VDGr4z^dQj?CMK0|4d61Yb<{x{&Q$FPj! zF7qgvKc@$@{WMx`6nFKeP5RQxYJ*4cEzjZj^|V!Yo+IaPfjq3d-IODJIa>wU_HdqS zcBm#u$?cG~>u^ka<)(cS<=2b{a~oxY6uBL0x0zlTC6SGc|Ig^9QfHUpfYL$M4EK|Z?~x$Fj{Gjdl33F|hbtx@4zAM$Pu z;|jBD^~0ul6}Cx_4dLFt93RG>S?I3d%s|eVm+Jzq1*yc9mvMXmr4OfkrNC=gJTibY z{W*3WclrMc%D#!KlnKYe&F&7*{|7a^n?*Z?JnY|$O}W+#Mz=PLSNJuJ|MS?B`Y)gz z-e=5e8*!_3T@#L|Lu_KyujSl#TqVVk zhkeTLFFCU*(1i8S12aTz;~J^3RAGIfU1r5v&l$A>vq9Dj`I$aU^klK#5(bDGi zJ1L_KdL@PSO7pRmar_VNmU^Xlx}3qYOe}*edmw9ddTw=kY&Cw>My_c_k3NAOeSFXX zm3TbTjL~#F>k0Jw^XSiApc~!7(lN-e%BNDkww#gLT+T=^qm_&Fq&@eT2kkuWQGWH9 zG_5OJ?I3M zXLo4B-)Vted^>e9Gnh%=xAK+L<=iqE1Y7x(c1v|ZwNx{Y`Q>c;-CXar8(3D;FLtr7 zKB->#JNJ2|Ur$|jC%?Uq*fRRWkNn>jzKl;vz)k&9KDC0QySYv)S6TQvau;)N0a*(i z2K*jh)Xa4Ds06%Kew^i~Ur(EBhQ8!G`mTu}1yrU4bJkTy=dKg>s?aKa?^>aK{N~kZ zF|80@VGpRnGpln=Y(cqtbxO9r_Msg-`wyP7gGG77+St>#La%Jg)o@QGI<;S|n#y+@ z7#F_1+}(3)7*|UIMbpl!797W+9vJ0H`Skl?yM%FTMz}9Hx(qI@^eoqwk2|NLqgR`@ zSA9e|%7V(89#^mT3MDFKh-gJ!&e1Q!FDb=m>_^SpaWD7RR`BJ%uWueVzwOUq6wC7+ zi~hG&QnZlv@LIH)o-wTmKwcHL2ckO~?E>fl@f$wKtjRt~=Wv;hqj*ytfNwD@9qVvXizn+sIbZ zgW$WH&9FJ!Q~BSH-;JSK+PSpl%8ievt)yNj@L!72l55SZ7+bA**pq_9{EJ$}Z0qKn zcTrN)a_7FSuO(Q0{aDJkrJJ*-UD$7ULik1}P!sEBySSQD&#aH?<;nocg4U-Ui*lYh z_>}+D+NFnT?U5oXXI+g%bsLh~Ft($RMD7T>`dH+m(Lqn2fSx`bxoTRFpdLrAna=Ve za*k-ox#-s)Vo&`vSX|Z2SK|@>FV@+$WE|Z|bnWVpHFXP~Z?mT6zY{i0ATTXQvwsA8hk-Y!&0anH)yF1}$A5hUxmFM0Os$ah^x*LSKq6wQ zgtAiLREj*Kz_Od?ZD;!%>t0rmCX>6V)hcTH0aoL$f-Pact@vGR4BDzc#!4V6Vh;8I z?FU*3UJma*i+7)bJU1c8jM_9t(^_`~*?4%6iiaSnn*H!fWY|mLOL7{m`fd4gj>i_E zx0of;F|C!dI?I7jzvxB&11c7MHu8Dt+&55<Y)q#ULtO^IzVzxyI<4M4K$ z!KCLc_xf{wDAJgDfqNj6^+eJd9Hg#(NMM%)N$x7P7jR|(*YqZ{>iKwU`*7By zeZmnvp%-wb59JLAp(|H+<9~Z<=KZ}x8D040(O!J< ztJsQC=oS8Zzcp~NgkF4!ew27N``!3;5y!joq)WmT?ZFS|fA7E(mEe1FjbFS&Xwg$? zXWR4wYSWHmmKnduDcp4&OEZ?^_(E+d=WOmFPK+z#x6re1Ic-9loyhlT!XslGQ7;b`clI-Z;)r4#!7@G4>CR-piuaq3wGF`=c12<`K4RJ){qF zZa7zK37IUBACbmPu7#>Dhw{c_y zcZ?24uj8F=Wz1?jzm2yY%KzT9gSox^vX{^zeYs-@t>72$O1s6cW_w;nn_Lmvs3*U= zac%r{6Ezvi6Qv161kp>Z(J*Q`ni}6qjcms|L+$mlnltm?y!)epMomRu)N-z+YZjDD z3s=nLtL0@rGFX(mK1H5e4fXpb$bITI>w>JSRHzg!R&poOlAaxHC3@i1E$TpN<(_23 zfZn!%H=21i`clt=CwAG#4?DpJy9JJT9<;g#>s3%~J)>&${n@*c|2LzjN4vvl_9g{8 z@EB|X_ptvMobVxdg1)}#SPdS6H%#W}6f6l^6XYJR!zHy#&xf15&E9-?%m=~Z{00_k z?b-{&v3KD{Yv7#g+3F)*#@UbA6Jxaj?zsidC3STO1vLXkXs|0SDnY6u3I0R zsW>sWC%}7;0nc|NcQoUC1Nd-rxN$4iqd3=;>+6NPYVp4z%Q2kyY(vg9=U8=~T?LGu zyk4$-0Bf!Al*-htHg%{%ZL9D%8)Uca&D7xUfqJ6raL)JiY29`4Us zSM8IMzT)|7II9171@--w8hd0l*XXBq??cXe>??Z5_q_i)-upkCUCG<(!(PX`|A!LS zvcHmYeQPDK^K{Tcv)#>k@44^kVcS@g-?s5< zD}8J)*=GLW8=14m@3Dd7|6?KUimSGAt@7VGYP5+XKT*Q>Jn6qY`9Ewu>eIdQ2Ug|6 zP2rC9)JVDPQ)>8MYV#7;Lj z$gwvvYF+L?Lemp9h>?6XW4$ZmyDy{M?AN{76H6i5!ffYSJ(a4}9rS-)$f_i*7e)=B zCCf?B6D4=`DrJ6isw>kSfh2b@65YYrWz}@p)2QnA>xDYO~82+JUS_F?+7kx92aqK02%fX(4 zo5%%*!3}PMS`UUwOQ+?0*Fo+3!3UJilwr<>U&=*KhtIZ!{&(a^XZG50q!s^WJk`mVNtSGwo(FCmTe!(XiND@sO?$1@~wSG?W`At_tDns zuRi=&QXtZlw~La5{&(?QT31Jgem0i%gae>eS8JF)&prJZMj z9a$~rHT0SJp@*BV--sQ(g8v_7rsYKqNdLK`#L=>jMJvl@tSK9V)ypxZ?Mz#d)+4QF z+u8n(e4uSE$q3xVagn;(=k#6eT*DuWd)$uVs*j^%1B znJ}uoPutlZo)Mp-ja9ist7m~H_;f9)TKKfv>FxSGJTH3vv%%W6m$O>)Fc#rPa5gw?|CCrCi~>JEF<-0x^FS zz9RkJW1znxoutJ2PKIM+xP|?HhAW2R)sRZx&N>ds%KL^>!j0id&sy3Y!9p<_IzJ|q zF@iG2QAYf2S@-Q*hF5+Cvg7*aF2;zdyAk)`ut-LcOn8XXn;o4Bz&LdNH49CcJzQaOUN16 zAiIREs61_w9?`Ps{4!oCQm-TDqMfo8XFEYL)TR8dZBai>clyUU;kfqH3!x$QBGIbrr=o%45)QB+Bhd>!_3?u(W-pUztF6XU#vz3DO z4$`J=pf}1^>p)AML3=x<^xrF8w&ZI?E58`dD#(_8)#yE`68O`84~_b-KAq_K5kV+Y zP_M$qu;_iV=edYF5b-Gf!BH2zn<7G7MN^Aj5Od?m5U;Lh!?V#JBA(sRrIx-Jz1>l` zDCqF-u@&ng&gYq+?>_~vp90Sp6BF_6aVzRYRNVxu8DjE8+uhCjAjigWOpAwjyoe_k zZ#0hkwSnlr)edrV*lKUkhIuo`hhZ1FIqYda84+v{w}#`xuzhGK}I|ZdAJes@KMN*hapMUM3StD zIe5vev3R152M6GQnvj2a`{Acs7E!g_2C9R;9QcXddqt+lL*{=N3@~C92 zyx5PH)z+y*tW8iU_yW!}NAO!4ozkqB)QR-T`$MnQ3Mek?8T2mET(jx*bJ4nHGa|$WDZR=~jQ5bAsJ*?( zv3COhaLkB05<#FeE3)`gR%g44d{5+(K7b$DMoCXuPb#m>nJpg3SXa-ihE!J_&szg% zgFb0})_Ty4JWw+=mueg6a(C!)ry$8mYoxrcm$I+M+l#Hy0==Na`YSGkB1^OT@~a2u z^_KO3#(S?4zK|NUOYcA@@a1>|<3)YbRZ>-D|0f%`dcl{j`iIg$VAi&SzJ+oa^& zk+WS{^+#%36j9rqeWmU0w4d=7wzF;QV#_;s+NB)18oqaZ;EjW6brH1skCYolI1PfE zC{-#Si$!t{E@wORm~jJ2+BXKCc_Vw;I-(@194%66808J66y;PgMm|ZTmT@D>?0tBq ziwFuOe6d*rIqFxxlxwt)##!FFaYVi)zt`vL%wFdN9&S(4v#M`ZjFym77ekxMHTx5Y4gIplacm{jQ5=Nki_?;j4{d}x+#<3drp^TvI?PK=l zBNx1n!oi>6;jY9uz)UEVQ zs#hretE1e(S#^TDf_9+HV%h#u&Jqb1wGZ1)oU^z&(R5E!$0vFFnE{jie6YS5u``=D z_N}yMub|bUb#)o`yBH0mm0P>#3haJy&P^kAw8q21;g#uNEnpK91DTmeKx5y`{ zwKl%JQ1Bf=;u6L4U68+=ZBX4nRF7D%^_-6gA1xVL{*?dKRg4f=!=7^keguCI$@>K| zn|47xB61GpGchjWO+>ATcu_V}3sQa~-xj<@WQq|SZpEA$jpDIbjyjTi>Q(X~kuvI2 zVq}b46#b@zXvsb+YF=7w-h}(jfw#?t|LK7?3eY%?f5R<}?|7L1#&S$$J03YrZc5y&vXVJQtSfNaHhDpJP=zpMxYahh<*ab52yz&Fa-6$d%QSv%E%y`pgm^}K zlDdJtVhPuN#QFEwdw*Y#6fy4=UK3X__V)1a!Z^@E<=D`=CEieu9ubM!*o<|NtBY+C zEp})?1*z}U-?tbm*@)5T9Gk87jl~9xO6~A*Zc{Nc>axloYOwO?sJ$9LDK@5E;5a=P z?Y#q!k|(R7>hasp4fn^e+z!o%97a7M_z+^fuJcD945DM+&!;-*O>+g>obpqY~fXDi@;= zag6 zZ%mH$kDPCMcq92h~DC23{Qkpi_Qfaa|Tdi9Wbj7 z(ZZq0`Wp3Da7okVfEv@b116=>IE4R79;fZKV225 zntESf_QiJhrZf@YK1prdQp6KU3Hxw|*dL!gkkwM9o7eDPZBL4-?spl_ypAip(sJb> zQrD;rig!^{jGC%jaVTy*uD?UCiQGmVRL_ZYT8*+jtGcLO9c!hxM-8$!&ld}$=Vw1Q z#y9uh;svGlIoP_t!#ylSe$qT@eGXHfssSXwBuGS)4U)Kk}2P|Gq*Ao{`%a$Mo2~ zCB}F9wq_Hs2aAx=52U|x9IHLrc&8XiX$j8u{u-qjIk|v6wd*)9vazc291*y3nXlnK zM(rApt5r*^yt7M3l(b%yEdH4S-WL5fm4k1@IAt|LBPm6{sxfN|ZNnHhMl#M(tj=t_ zjF@qGL^tS$c5%HUSE8Q`;=k7a>yf~t{v{n!CsV>!#+G}?FUCT_9t*a6C4A+1rFQA- zv(U(Apuffvj^&C+psRW)9*5K1!)o;8B&hOiDCvt(L$A{&dKcS=_${Z=God%bvZr#j zvb;KiJ{IS0R^xNNW^1CBB&t&#Qg1}m4fO{M<(;&{8r!3GXzY!P`hToZ}sI^ zx-*WO>ELR9YeUvbaUp9TN{F*p8yDmfzs6AZv@#o6AGKzEG||7IHat4Cidyj)-u-4; zN4;2$Ku?J6p{GWFjXJqyDHj-zySGEoW>g?cEpeyUtiB zwQ2R`h(Kx<_=u8=vx$q-J(eTJ<~O2OI&-)<`|5Zb^(-30BQ9R4$;=c#vxw%8Q5ez6 zt2f1LDq4As3^y87zk<3&j4<%Nom``LbO-;9{&B3xvto3BV`ew@7X6o%d9KL+7>Tgo zR}*mymZyHB569RL?KpZ=^>G%66R^I{M=rM894%%b&?D3Y3ygUQ8Zzqi5{Y!)AMtT2 zqt4N19}!XQ8S&SS$Cg2o(=u-BX<<^vJ03~USd5eCKc~{R+H=g8(T=BhR!^t48xhlO zXx%nkuSKW@*J&Sf4)KOOsTnn@LtU+*7ht>u&N)}RHal~IG!L~sjdxWZ(eL8e zaZKr5)A!Pb_tx4LZA*?iEpkdO=JPm@-X)z9wW1wHOOw$uJvpO&&+|R#p;~FAVvZ+! ztM(S9E?_OUakg-_sH3~ z2`N;`dm?o*ZtxsN+_ki3P3rmq<4k$}k76^9c4L;09mTF;q;};y*I=y^5iGVR$Pnk&R zNmS@%+^y85{atIlQkJrm&(@M3eFtP|4{xu;rOakYgSkqtj1p#l+H)A+W*BX96Z?8^ zV#WuhQjzE}zR;}_lO7{u)T0MU{a87}JWg|hd|{>vBX!L?G@n%&$?QIQewPKC%GdNG z{kldA8>OdR-oCeyUKo)j(qOR{E=JhtgVj!LB%EG2f4 z@pe*Sr5ttdUwZjZZb02q_B-SknTc`xC`mw4&(@NC=YSfES^7)V`I_UFJ>J>xvx^%MC7|6 z+^^kWDn}=9$4riE;~k6yIFO^m!to*O4drSP##6cbN$Mn0B;ra&P+zUyK5-~D6ti&) zC68n8pFDpQ&k*e~n&bYuh4pqyFoMUo&~_2CHr>aYXg4>f;)HOI``U`fBT29H)DgPI_B6hf-q*y(>AM5P=CAYvhYi+ zh|z;u)bz!h_rql?QePJDnb@xiWWQBd7tDudhNJ4drP4-2XhMtu-UeN&&YMS{TwPkH zerN&niD-S3PBi3bW7=5zNJJ%w_~{U26KR3AaA|<;+APo~Whd!MeU3C?&(=Mh(y9d+ zsS@&`Sh|Q#P@)v2AgWCn!2KVgq|!_CIC$K=6J~r{HzE+ zk#X*eBJvwVWWzUHXP%`^w66Gth+&YPuV>LdBVs3Nj{01Uu>O)`=5_K~v*j7b?U^-v zE5D$42y-CKd|su)f;YW1c%!=7CS8+ik)<4v(@i#OE@65~dc0=gj? zoyu{~hy>8)A{W-lX-! zhH33K6U}tCb3=QY??qp|{t(B$y&(GP9rerkPG7P$epoBSVzx{97BM51xpuUdf5Cqd zt9~iHF=hgB9NTxyOyg+(BJ{x3;kOxC*3erW%Rh$kV1L$LE{ex!;q{!|5c;y$hzr}F zgHU}hW-xOr_Do+>#FfXpcXGA)T*QyXTrNibi*MEYUNii!!K>xd<;ZWkSp{J|H&N>{njTbHgQU@C2Qk-9R4*e*fO*`>v7f3x){ISS40MEMNcpq zz$h1eek+MPFtX(*qCIvKy%OU=YZK8~jm78`|C`&nkjN#f6De|Jh+H{>*wE^ntxUG$ z6l<2agfb!$cCeI%{TL0CeER}-~hOoGc-tjn;qiyM8Gw>FcAJoO&9m_B$jBtF7%t?QoAA~YN>s$X6E zvo;F7*f)o3wOp%VnGZ_uIde({Nz4o_5j&?PJIX>vDa0HG?a*u09+le6a-c0G>aum9 z!qR>1)#4z`)@M}r8E9czZselUcJY2jI%vPq?&AI+wzu#sEimnPdT;8gwrEC4W2Jgf zf8_!56%G&5Twm@rSBWxOE3T=*ulihlHTySHFJ`Lboz&9x&lssLm(beLkv9-cdNOx5 zV0$>{K*N z`o^3)l_&WIA`2o8Q45V|SHFfbl;~Sgd@iGrg|wmRX&=N_i&)t1{J)gDB5I*MEfg(Z z5l0!VSf|n+9r%vs*Nn2}pwNb9w7HnpGsmM+kk7rHr)bd=)2!qvg3x}V^y|IKZpM~P zQ^uL)dY6Kt0ZfJ&ab@nY0HQF%5-b6HFf^4_j10~Wnw2M8tmz3#i zC{oO?^--X#=2g(D8@Zw!Ox?>|fyS{IvHUPpTF-_NmH&XYiwl%*4SQ{aEbUl@rR5p+J?2$`esAgA#%8)tG5 zzgkgR3;s7{F}k%0KIJ3ft~I%0Z-{8g6VVbQSoAWB6O0*~6JYvw5f8Jau&MAJ5ioy- zsFy!Ewwt(@JolD!bQ4iC|Kt2d_I@QQrh@pFOnCM|Y>kGo>;!WL{6L&djHfAtoZ(5z zuONQL2paw5HTZAXV0n1X_I=MYvPXQQxJJ>6HEE+Gd7oo=hcs7bXthI-6^`TmjUv*! z60w1&b7jr&HhR!o(4yUF`@!&RzpUR|pU6${bMXN8!lfUCBbyUazHBteT=pi@6O>Vg zz|HUFmmUza%<0K6N3VVh^QxI=O-aeTlzP$R_6y*+T9HK5iGPhbs+H&F@f73dU!WI8 z{a*=i6Mb3iotaa9hL7tNnFpsB(so$DlMRa3@9f=t$OY0=+Xg5AI8QBqKtaPGP*^DvTHoV82H(H*f|5Mr87&kSkn8(zp z-T$AjW*7HqWj15mHolC{78ww6?FpV2V|vYn7OkyXIka$y7l?V|cA@8(eP6AnKHoxI zjusa4qnMYrGWS|9pR2V-d%C%d%mb}npd@Cjaf~wfl)9TGj#TzMoe?c(A!)k~YwW3-!`1GiR#HBYa#l#gq_ z*6L!$wytmwJxfXj;u0dw?+v$6Ham%HxM#|`8pNnMnDbmWQh=g*OZ z{zP^J#U6vzt86~=f7*g*SJtoG%I?bZyBK^eTi|;ev z=ewLSLx)_|tfSgI#0W;;+S4EoL_CQx+b~)mpU%Qi_tESWT@;T0hK&XMS!oRT*R2mv5MNgOOE9{rm7N+cF^qt6QbuO{U(ZXUjY<)g0Y1hhFb#_2IjVsb3vNzax zq-WZx%opZdB#ZIX$Bg*$!uOzN+Ad-&v6cp9s~6ylV#Q_zuf5rc=fQA0i-P0nBn?e&;*!xaczxWg#N&1|$M)0cK#{pQl-i{&1uMv-}w`ASP}U$IOQz z-pyInZbHJ-4skM4*s1)FQL*|ex*}_sd&M}~D8rqK^mPQwkw|FukX&jZ$s~xDs9>hH z8b~m;LhQyt{5Sq$FOt%?%-{57VIA|BtVOa?0$Wx1gjrBNVFr^0`{potgt<3fVm9}; zSYKn7j+dE3BZ_F}(Np>v^qp*z=I$~Oa6UKUW(H^Un&;eFO|{3oQw-7H3IR}i7S zk$YAaK4AWp|M1S9{f)CpTE;v&W@kADd9QM4yGpd2v$fPhdaTJ_D||e9sji^iyVA-^ z<+a!vGu{HpS4?DYzNfi`&39t_&Tu60KJB>;Hq!Va`BB*)P1E#(sDEp;~@V+izZO5pJ#`;0n^bWzK_v7Fa)#R&L{ zxlZqc)(wXS>A@We4U5!K?N&d;V{l0kde3t8{us5}I456Hmo&ERkw8!X$$2pr>dF0i zf;?0F$v=YyK)8MJ>te&=m)x*_VB#i zU<7Y0V%|0}HdubF-O6~w8AuhP{EZbf2bieGh>aBeDC%59zcB+M)U^t2K#KVm2}|qH z1~OV`>2hw0?^(>zlEeyC4{6ES!45()Vm3#lC1*%GjJzpkU};L0mD7^vFt6cx$upCu z@Jku$Xr!%HWLP;G$;wPDCnZ}WeVxVDV@Fcvk(_DD*89EZ7_K>maylf><~mDijm*}D zS;Ed@KW20}66x&(<|T~h+i*p5mZMpllFy|%>rw1A4*wf-+~4)7yE$8w^OW*@>QVeZ znpz&q*7K!EgIVl&)E7g{b~#SRTRV z;kVJlHkvhJKt)B33B7L=XJV#dy+$$4+UVt&d00QAUPQll%qP6x24k<%`XYMAs3*@} zOuuPQzZ6}fL=GZH-(`lZ%D zo%BY`fd)Q>?c^zVbo2~(&wK2P{hJC;kJ;L_fj`O-vsJsygI|jcoXw(lT%UrmYu>BH z+DHN^q}i|aP>J#Wj_2!-inUuqjrEO-2Qsg>*r6D~B~^@B!_AVgKRdw@v~omgN1RU! zTG*DX%PMY0e4e_I@mpq;tP|`^H9`qymsO`Sjyz8+nBMK`oEMp6eumiYMvz4ml3zwI zxF{XJON=rzi-Ov&byj1QBk1M%BD|}9JL4XV!+4)}*QaANy8fjYGpkR)d7=D18#t=p zN&P=&YcR6Iuc9YPZ;)O>qi)4&?Ff3Z)-+??;%ru;2h4+RgsCY zP2YTs^^5ZhDSO*uVvR%xM0C~R;g{Aq^;NxaTKk<7MRa}49$Jn5XSP@K28$dsUPZmu zqvFBjg1%jhQjc%#m)A39{#8+zG4jOB`t~$EDRbymZe8t3F+yJ-nI16b2N5MI0z+$! zo|h5t}LXHU1pK%>cIMX8B)r`CUFWBi?hGUi$?RzmBpMEk? zD`xCF6I-KM1v?{QngeD(R@$f*V>k3niB@qPjGTJ|H8&!81oEmm=Wb)suVs8k{H4@w zo<=h@npfrKeR9r7jyg-pXxhPRVxAunEi;gPoHM`}YbAB9U$(Mn-vzAB8!(gWA}-kY z)EB~C&d{L6OZok|u#GbnIP-wl7&j87|Jm$4!&MRa>#QZ_NswY_bBpsMhz%Cc{yx1y zJi8uiQNz)m66Yy%h6TOnQV;Vb#CgwFg%PDyN?NAv%xnirSo&b~V3{4kkz^KZ$Gjuj z{3hlO7Hwlz38{d+*xoBmHLFXEg^zg=%z3PT*a&zvC~a&B?vL^F`pw1fdo<$r^^!@S zOBg+M!bsGM5@!<^xh(Gn&Nuw~6$E#gi(Z6;^)l2)d!F}rh|QAz5obgPBSm>c(O3o%6c z`)#rPSxuwu;BbyhhwAXY(sAjO(Gzk_TQ|}+>6RWV+gTbV6%-F8GE={VszoV2$E|Z@ zS$5=j${Q~G=jn>)^xZY49FHoO=Em z#{4^*m{uLXp5xl_nxW>7cE^5&@2yoj%E*49 z*sq##hu=k{xBb@~0I}ct-##Dbj*R5qlzuSkU0iviKkCv$fWJYo6oB76l>qenW z`=PYOtBlNYEJob56wDqVy>*OwXUwH8tyCs=oTNBkgKg|Pj*Duvv;06Q$XVlTd8H`P zWTG+E2$Z4pfI7CM)3&`cJ2s^yBK@&mQjmB|Dj8!JDK+LUaNI~i zj5)Ivl?9DCHsiXpCrHtw7bhaMT62|OH`ZR=M2|@1U{V`BBI+LUI5iBTWVXU{^@QmY zi@9Mo2VSe?P?=TCQC#KOadsU0wGlXarNp$wd`s#O^BF55Y|Z2<_g5xx)HqVqA;iQv zpOZ5ZdAAyiV@gRS#wr`vW2weED{aR4tDOH-^owXbtu9(`j9AcbW`2k$Z!4*qMatM_ zWq4%_Wp?pcG0s_CMNP@AF+=9t5ZM(qEai3Y-Jf;J5}Z-Mny7^-gNP0@SI!b@t+z4G zld_06jZw0ZDi~FzX602PiPWKt^HV}HbDbGMl!?tjrR7P~SjMJ7al#Glg zF(cmoSxrS3MT?fWWu<_cp>IXBh2tyw{-d^G-jV1@G_FOSDL=7yI|BAE`sU;V=AvuD zc{5B&S>)96TH~;e4`;+3NoR~5G9ywedbcIXdySMa|CH27l#J4st-A`=0j>@S&?FSpJOn_bgC~&8`R;Wq-FF+q$yI`gZU+YQJ0K9$T%O9QkK7! z4x=olH9#8Sl~NQnP~}$d6{Dj5D^-?<`i6Q*V#Z=MJZY=(9@-1FY8kH_bA@Qt&?;z* zhFKo;XBhc48LOQbENyquE9M-mqOZ)BW&Q>uZOzo+JbGqNSd3k3A-{|Yc3yS!)q2Ev z;G+l5oUrC+@Q(da${%8>F@r?CD4$635pW~B^aE`h2jqf$@g&8qsQ6~{o z#w5SWFPA5&wU*5B_$!TQ6U*{Ic)Mr;GDn^sn7&wgjJ%4oRK?89#;}N9xr!DNFRXl_ zAI)t@oMY7)nauwxe^dSt`x@o!B=lD<@_qZDN9{Wv zF`JvxyjqCA>~F3yM~*os^$MxCORtnG#0Ho{FJf9_CJ3p!ctmM)L@7!mrN`27^+v5n zb6H*rROjF5EZU(}OD-xxbC|}t4V*2*_+y-g`4NWwH;@T7E$-^$i3>?W=2&Dk6Gt?p<_E!fvZ}z zaks9{6JSh@dbp?&^P`J56=f1*jg55f$JuMp$R`B5gPAHO;$w(8%FL=DT2y^mdMw3# zg;t5%*`bZzq!rbM^`&X$(Ze8zHSb5{HS75OGhfEsMq-@(Zc+jf6CM%qAfm*1?UYYN zHkjGT{%jZIc9RRFUU!ZwbkEeeuv1#18>^M5)rVXSw)t})qJuuW9S*R zeYN97Gk(c`RKUay@{jw?#ndz@1;M*pwmjJ0T}D5u<4~KLt9BhFCM`S8Ea~h4#xW^TM3lL@M%;^I*{`g$5%cGYH8!8Vb0FHL zB5tgg8qoUChW1hsInibiBZs2is&=8ou7(+Do3gK&aGd!@E*dpzWpVWnJ^pg^C;`a1 z&5|u|Q)iI_t6M55NQp%h#2L861gMw9sL!)FBQ{kYuik0Ik+T&kCzz8*-Ot!0|EqOd z5A!p4Mma-Vjdf9KS8I^!%Egsu+*jt;vY|HMh%~#J_9Nv?XP$~PquHB`O)&1(jHQld zu}_Y~D2-~bH=aO=a({fT@LTqE9*cHG8oiE5REN5nES@ReI-G+{znmU{Bk%?drQ3RbrjuyrTY( z*-*6t$W_&zVq{$8*~Z1mwWYScp&VY#S-I1Erp}-&;zeYN@wR#q_0;wWl1PjMH!@bN zmbuuBf)Y1kB$aZRXd>r`P(Bko^lXrp^ee`UVfqiv?_w^}50JW+vO2qf@|Lp=h%`5| z|2lk$-?QDY&zrVApy$8E+xRW$s2}h<{*NQ;*v5SQBG@-^#90!)4Sq;7@hi`ng=;ls znuAMO?!O!}!~f@8>&yWg*jvV)vjO~<+N{HK=d23O6A))(@JY&rpHgq)p1FCP`BS+#-Y>!@dbXd$zpIxwX1h^p7UylI z2D6=wW7Pv3WoWOnq)9=_*55c3>DycYM)+Y0eV$=a?TQ&S4*;g_VZ1@F_7z`$d#n z#IVOmEcNr!=%W&~T}sJ`XYS$`4{oV)ReU?;ic8T5;6^ z)GOs%dh(>mT9CCztBJ_l8nQ$ilXh>Br7p(&M!TH0tH#hjEyb3pRW!pdd7v7M+)|61 z^ij=3-9;;E^qod-sohB%q?}8Cq1;7jNU2u&DtfjPL{n;K(vPA{rIkiMg=gK5^N4C+ zlXrM`%qk(OILfdQ+otVJ>B=_JrW3OzDI@BEQbu$JX7kkOz1Ev$E}X~kR+-Jld}p)x z!ZY~NMvYBixs#qF&gf?P%81}&jAKHH-wZqthVfx`Vf}E9BN1$7=b1=v@v1o6ts^VW zc&<(61!`jypt7_QaI|{aSEBb_A99>!TCA}-_>2w{Qz!Z;%E^vxdw~&L_5tVfG|o$2 z_d61)`k@v*Ex~>_ZQI%rlxw0lKs!R5S*{rrNQ=n-BkC)lqsW@J$K8G6?!gJRxVt+n zvMh_cyZhp9i!CmTJAptTkdOqT#N9J4J(|BN^S$RkeP*V+r@im3Tk_OX=*ve2EA)Th zH}bF`W)u;|$aIBVo@h&Bl{y^LFjoZetcaBGFok+>vB(pT^^QHYRWO4WaiSi~56VE~ zj79GRGD{$S6f;T?8@m}~;zn5WJ4ohH&HK*=J030dg|NJ^`P`TvGz0KEh>S$u7p(4z zJQj#g_2eI6Hq>~y{)r%65kZP$aVPi}bEJmCdJn1+>ri7%VJO@sa)4k)Qh%7R+BH^p z@zi?345ndl6!dKOfFrhoUyMOub`t8H$O9V;AM{D#JTksw-9CI`h}1+q6r)ENtH6vR zjESHZ0sT;zLxn!882A^x?>GV=HZ}(O6pjd(QICEujKp9BM+$uj?SHiG(IQ0q4pHie z_QX7aJZLHCeMP$p5g6!cMNc7Gn;v@rTlZ~f-^jV>Y4@i<;{6RD@6;S8_f(cax!(z7J=Pw<>LbVrjrgM>jX)l(@x36@b8Mg0{~9q0u{yE7MhVRP;^*;b_2Yx#gghVctGs&oOHa^AHe6 zhlq7#O+ci)rxH8rY!_j|Z`8UlHx!oz0- z=D;r^wvqJ)F@lH+!}|89{iEdR0x|&W86b)uB>_gs5Z{SU9_>nuxue(52PCp50){bY zPaGIyKsX{}rV!STM*A17VD$cCyayv(I8tLQ+!OgiuQA%BC?PyJF?X|7h7tm?3TPps z&jLqO^lM-&1ieC@3>+LcJoRHVuof*^^kiU`4#xf|a70{-`~H8#4;~e_z#(BmK6Y@efsf?MFd zp$zo2T(r;esi0>7quT$ACZit#V_vAappAz5v8M;1F9JQTXp5qLgIFot6Q3e-fumo- z6J1Ad0D1ua7oR|X1jg_^7y(@SzgouVg}~^h$CrVT3+z?6KlUQ@V&J?7G2nT7czw8x zu}6=th4DumhcUK*C;;?GcsMGM=?j-IKLXcd{KIo}kADQ`F?Qj>7+|CUkA)T~MiB5B z;M2k=A=;_L(K}kgqWFy zUNwvdqWnXv8hsHM6~}lNO2H%;d2l59+^hkk?uEWk^n>02$%3HwAq(X_@=GH7BO<#zm9)@HJG0r=_+K4a zv@+12jG2X=+OxPdGUp;=(B$S(194%955sx`h`z#F1bBw2u!PnP+BnE`fXu^a0j&Y8 zZ%?zHhiEh83PLVjw6D+}IN5xUs7s=^&4aH(?Gyc=7`;P20Mz!;i{|mppw|c4*-$%1 zo*?Xt*as2oi6cFF7I92K4?eyP)EnMjaUayOU9Ol+B`;^v4QZ4tAdFbm#OYY3UeQTie~4a#5K8nd!I zoQRlRh1pq{2erT1{`PR2BX1&lO+2|zSl0=;YS5a;3QQQ~LKHAYzVOV*Z0I^JnTg2vJekx9g-Sc4a zJhltQco9{Cw#x0!x2GpTis)v{n#1 ziMce0JH(!db}?3K!?n2ecKBy~v#oTjV5p#NW!klM4 zYF~H_2VgyBW?&6C^fGMtzd6*&aPPmHw>k`K@rbCqdFIwO`zmlhJQqp|wBoUj+N_;J=`Ed)TtE1}*v@Fs}rC3Ft3CB<63RS;m1!p=Gm18wwgIqCYX48}Xlhpqu(Y z_KCcC5;W)dV5T!>XL~ZPv0fl%YqQ|Pgms<_Xa-~fAv?GPKB)c1z&Wsz^Just^pW5* zJlbq+Vm#vmNCNcHB2PQY1hn=s`h$obkH-hKL5%cRU@fAKF;gD778TIqI)H95wz&^t zKS6I5W~?FaKWa=^dFU&6^)$_#6|}voS5HP1UZ#;B&?^Fcs9F8!RJe~X(d=H0=gZ8jz0BnPd9eg-eNb?i` zbNM{(ufb@CQes=P{kR>Dj8&qs z{_-|BAI5XfH;>ilVeU3am=(}p(T=(Ya_4xn?70kUH^C8*x%WEEV}uIp-(bEy`pOUm zgP359?V*K*aVN}2MDG{+^wG{rgn!X1g6zHM1wo$}a`@uNi8x>rjF^Z=K|Uz-@?%s4 z8Er7;g8!iggZc}4l<-M=kiM8(f!+}h;t4G&)W|Tl@4=fOstU(a)H0D(6l3`4A3=l# zGNB;vERK*KwoOm|wFf(Z`9)Y08na_DhZn1uVEx$f@Qqw3m<2i<+6($d=ELYP4B96m zGy56~Z1~ zO=)iA1X~419}h>F0g`hJyvroWO>a$l4y|S}9E}Z322Vh)4TE-&0B_O_+S7D+3#UNS zL0{c4(j#bPsH^n^T_TF~4kU00Na#ekG8X9HNC)5= zI>Q-BaBu#ka@h7bw3}pT_mQyeXON|fp}o-H>8VLyVGdDWV%X0O+aNj$5Yvcp zc(iPg>kH%Wh^RwWbj)5suQH?!zN1pN5Xr8n3#9%g+H?<%s-_k#Y? z3Ur#z&Hg&{(qWCZ&fsB-g!yRDcRGT0gVjjFV0|3uCs>mXHJ?^6j|?;3u;=;1?|V~#!4uyK+D0}bauBG*2>^NCd_-ZA1C~Zfb)e4yf`e#<;8i_kGPOG z%>~WK8`ikp9!3~BEMYzko;4EkfTQ7q6^7e`b`}EH*B$1O5w}11`R0QZ!3>^3@HAJ# zglum!;fbOCc>pBPUXU|5VY~(zTF}-+Bo4lf|8WQ*iyKA;kW245yvGk91u^;(Ov;7+ zjTl$V3qdraXAJRVe&~Q*7$fKyZNsr#_7`lpcF=@fjUw+eYX z=fZl7X=1z*?J49Ez?zbt2q$thVRkL{HMD||dlQd}QOQ4`PkB7>$i;=M0nO^~vM@OF@Cpb$Jz z$fj5hy$|^f5MN&oN6m%a+6eEk0^TLE7i=7#7E56-6q{zcpsYITU@x5JaN!WNA%hv*07NXN*I z29AeEtAg!475@N4>z5uQi5h4X}-54vw$SFoy5mSwr5KmMD?+fcdpfpE~I04>1R`f+Q8)~D7 zX1ob)7nv&1ONCJZ^yd7pS_Z~Xj)KfS3{o5!Ej&4gxQx+ttTl?d@?o$>J@u=$fu#3% zv{p6Gq2$`tJhuhrFlLUvF0^3Rz!GLFZh;A{VUG=sS~tekJ(bfpz?K-hMotTqc$fj` z$$~uAY*!;U7FKk^Yr!nQL!kE{68tR4#6ut(v1SI=Qa=TEh0+n@ohTnY*(ix{|M;xX zPlZ@Jd}5fz;=!b2ObL4$_7Cg}*oW}RZ{tF0%m6FcBndNmF0QP&dNeT?Ty0 zYB*jelAMe(^Az0KFwhDgz!j%~L|g`T>N?PvP5}4)-=G_r;PVYE)Hz^_UI8hIsO7sL ztM@iXEbfC{nn7ZdzW|2D3Cu~~A^RO|-U`^`2iUil!DdG;BMSK?Y(c=kCg7}WYR;V3 z!?%?D6}XtJun!s7DIH+b`vTV`ubC%64wgDHF9pIH6Z}^McVGpij*zSc|9~%8+L2&$ zcW9paH0S=e10O&zSocD(?495-5P@~hXl5Sp2di8OM+<@c{V-q-@PTu%fo~JPCGg9K ze*~_kyi?;BCNb=7RSD88`LK zzKTW?8Lk}JJX!D$113HkO$ckuu+Gu^EVS^9G~nIH2cJdb|9Ln-_J{i72VB@Qx6FlmRvvqMOp;-5@sUOY?U;_zw_kg$Q@77md}jJe&iG@P5&YfYll? z0)-4wh)PG)Jbn;wgf+B~!_-ql@*=d1!|?8rqX2!FsQ;qB68)FRpn)1YdN7em%EM5; z9L8siH6i~GqSTSy2%~Nohep&8;>~(PyTfdFj1*ud0J1D&l%Qp^Wf2VHG)6fvK7?L> z)Og9zTTlnaTruQA!wFG)82hnXq=#OT#R-@@8#9_f#0Vb2JM zV>XUyh^)i70FGoHY%(Gd^spx)2|ZW^#444;`B1aMc!dXNiTs?HTZQOK%yaknNGU>JwTDz>P3Vf_@31&@&-vNJ&K1LfsJ}%B?3oXZ|f3E_^94^OtfJ>}AgN zjt2`mIOXeEw;l+~`i&fm;wMyYvY<6Lcjt)Jdc z|5KZ1uCWcXS)FWPKw3x6cMc`pY|11q5(U-=wm9o3(=SVJ>sE{2Qfb>xIzS1fq_YUt za`xXW7xgixoH7UeZdYuLO;bz*%#X|i>em|RjavQRroS3fEOzrNa|g#K%R9G~wwmO_ zU&o%yc*ES_2xHBo4kT}-?{sXYOmi%Cb#S$?30+l|2G?%aH`gY&zz%){N0e){Ym6n% zUEw_MPGU@T$I%>AgZm_BuEDy*P)ZvsZtU1g? z(JbaVelGhXs}=vb>mF5XXBY-K)9RAUr}gzUaYS+1Bs0abPG8UPwbgL`)2gnWIYiUf{+nZ+`5J}QG>|Kk%o8U2#L7O( zeHAa+?FB+b`=ux0ju8?x*|$$(3pQm#kscvR`vgvf{a0XdJiDT4W}O zbNW8S3(FpxuPfa=(cQ~kNjV~jWXFj%@QazL{59^kObix*w(uGvLl(X{C$F+f{XGr;WB9?U(9HvT_95JHbbhei|MGY z(ecqiZpxw_rjFs2a~}$Ci289lau!glNsZ1$`l|ZFr9_dwxW2$r5?Y*6Hd#|(?A3IL z`a4r2V#(+FZ4cQKrVDH(Kf!l0(nw1kBKuwQ1^s%>iP|_#zxq^7H)54j!O3E;7W5aj zVsGaTWt?Jnp!K!?Vcyo*sF~2?*t@DrZcf`ZY~+-gG=O>#Ra!r0%e(yCuuvu-+i1x~8}n(|a*B%pEkI`wv^3@xD&0 zlj|<)&YNt+1ZR8q5Jm^d9OZjOqe|yv_WE7gm6ysL$sE`eL*(hsXjJu=G+w3#qKGtw zwvC%B2#^o-R?Cxx?^yM;JxxWXa?P8HMMaMb^o2=P_Zl-StI5j+LUpKb`=H-LlOnvr z9tDjH2=-!&w^7+{qoq$n|C&kFe)Wy@mIj99fi>40O6;-~J6%^z%QF@MZ zirUpV$MUCXw0>~?mWnrJ8!PtKhnQBnThWIwV%ZVgjbdN<0$H3$!P-Fk!}X87myJnm zCX$WI3`_L|MuK?cxB)uSD{|YWleYbY)4qagr?!UiissmD_R;Rv&a(8d%N?)mw=Fiq zQ9~c&E%OrdF-wSTsr66C33oN=U-HhTfs}vT-EEugddqF=5p%gA!g|EC*%3k+L-XRi zqL|#T%#?>|j!EGg|WW?4EQ6`GzIFUQ=_o z;etL+(^wy`X<5C${7U(&#%`{Uq$qYF`!nkqYd38Nt!>j#_Z4TIMP%2xZZ>^%eIuoE zH%Mj5iQdt{lOwK&M?~tvKZo53nC$bxFUWVlpFp`xykGoUv_bACEs^h4Z&%yAC;3xD zXdwYX#{%92u>Ex2&%9nK(&Y=qs|3GskF&mWx^RAQPl^^v{1n6G5el>FsG_Y}2SmiP zq_+jJf+CKCHHp=ozK!}8G&?%2Gi?#Wz}m{)ED{M@3sSh#*(cb$8JB3cNIjg9))$6I z<4!|@&R3__S&dJvZo7ZeOEQNxlRTKT0=zYT~P%XgKhsgJ^A?I(>KjRYSJJ!l+?s z88!9>(?b2VN_zS8lHuhc6|IU(OT5bJ$_AEhEAy%~8Qa@OyBE;raUywI(kjOc>rTCQ z?bhPZVsVlC*ZA}esq<4S(vq_d=6tA1(DE&VU5TV#+)`F=${_b62Vjs~3foYEWol(= z&}%Y?{z4l8k1-U&VK+0Ci zy#4*uzOVcSc)#%(?tfYROmR^>L-<*|QIf9eukNBw@?RM|IWi~Kx8>PZ*666{voU=l z`-S%PH~CypRmr^NBZRq}b)4mVmMBixSFDw`mi{H%sq**D_35TQuAZX4pjxP0Ba?{! z;Ot>%vE$f#IpcYLyjYfi70kQA?a%Jd?#J1}oJ|`|-9f6iMVVHcqRm0p-|Wfm$z&;L zxH^Z%Ofu%1{v@I;BQ39uLo{zI+m&~!3ag9J*c$)SrKu1Y?yqGT zwRv-j#f3WyHWiY~+Lzc%(yMKa`;2XgTec_W)p}{;=sHK;2t!wEhUKy~*EZ61jz~2x zB)T?zVw7;}}Zx60|ZOgH_Zd!g;AlbL6`&7|Y>{_I4? z1=bZdn?qx7W$%&flD3glN{dBG-Z;j;3_tEF_AQQ1cuZ*Leq(j#aJdKR>sbHrCrL|G z9aSpdAHF=_+x{N{%%OdwM#PS5IWT%tL_^R+)ebR(znuSzw~RN6?_?ci>e)2@IN@W- z9G@k@1tD9*q*1RU76b^?`SPEV7|=ClFw)7F9Sbc~^9JL2v)Q`dQPQ-6+LL8vHnQ%r z?=TlKey6@9FLmeIDy&=0TD@2!uaB%dRy(zBcH?HvQC*%+W=JULJ$srtS0MrD4*;mXN%L#vr(>r1W` ztt-^z`NIIK9c@JflU>`}F^Db8AF^>((i1=`k`R}l~0V;n=s2bft5 zB`1|Vp4W?$MfYcQVtwLH;;vzgrtfTGx#j{51HB#X zD=mX1<*P+qq<)HGgq;g4^Y5*mE1JWvU=QRlIWt*K)@9}; zP6j_txLY<$wZi+W&skr)f1{69xk!41e~@#5sijM34ic>?+o84nu%C0Ra&C71?gs0p zslV%%J=rF+{j|Oz=Gi;hrx91or%nEb0R7y?cXgNR#@DrP)N2?TjwY=U-a+Hb2Ce># zWrHEqaN6|Uc(;B**{dpB^_KGLva_Y7B?EG3KW}6Y$}(qH{`yw}H}Lv$cCa6_>$o<~5Wc^tC$EwpFAkB8RSZ&YQJ+&>6?sX< z2%GrRL{*|MiocWR+Ezv#DZD?52IHFEmwYz*=iC0PAlAPkf#qSHd7qSY{ zGOM$KvtMSO`l(Nwn|tY3-4FH8OR0+FLtoRy9?>GGr&qi*UU*vEUnUXU<@J*kNcl2_yheFXHAFeuo9g#Nc|c%d-k`BauN@OX z&pU2gVZY~gl44y61mD&`yd=u)FC2ecAL@g3LAuOFb;GXu5%oRmzckkC#~ViJ4glaw z(OA72F9j<(4(1d_F2toiJ3rc>rBZ(8a8*HvW?v~{*Bjg|VA#)rmZ#t+5~ z#`Xr9c9wRl{+ITo{;Z*!<#*61ezTs``c_UU-ClGg`{ehzX|?GczxGYK{QXo~+V=zB z*QZ@hO-(9D+W%$um#i-b5-XC2WwgxxReHm?fznbmAwbshX`HB&rIWtJxj;f4&pSxs z*iR8_%=-=d&3Yn(FuTrDJF-9X-f<5wx6xAVeRXZ?_SgPZ`M%^!<##R5^rx$c^<0!F z?X7y@8|&BJ`=rlsuSEGi!BC3028QgB%b z;J-u7VWP0pp&3CR{I~nP_a;=oD{sjsDrc$EWInP#g{yc~94cobXEdEkT}@urG}E=i z-kXTBOapm2(bU2C&TvSdq1&(b(eKbHbRBdf^g#xSKEiNN$2VjekC@%ox0a5kM>>J_ zSXG~@LB&sUH)O?TKK=D6Gb}qhD>*YaZAHr0FZVwA#qWKQ_Ld$0_>((TSs34Ni2PF; z9&|C}anPdh^+6;3*2|9Yw{dQ;cCm-@o(rc6+bEZKPY)Op^ve6Lyq2?rY&RaS`ck+* zcY4m+yfI~Ks?Rhsjb|NwsBP(|=xRET){6Y3X^DFt`3Cs>Yw3YPANhWz+xKmFM091? z-SEYJPh{7*f6y;6!^m`3R?}kuiiOj2=-t>Av_(xP$UOFW#wDtYVHCuQ$8i6WWvG^@ zo7ArYoS~}A`@?{n4_>O8s%~FWAY5yEvZe|hL^~=MVU+XcC%bQ z#AQpcb%xbs?QVW)c!{CMoCY{-C>}Pmour?Of zB-XdCT~RuxD5>!I?MLM=PV?Zny0DPqO2;kb+)DMc!L8-X`O2rNuBuS^Dw$Z8FG-PV2|5829XR}WizhnMJ@0+qyVxAz0$!y9e?i0Smb1R$JbY+9$l7uSlr|P5VBJHP~?v(?Pjj_$>~_s1wV|D7w4y>>+11lAmhiG<>RLAxH!Rkb8h_dQ({{7FvA?qjwm-X^ zd5t}YpD4^xZ1)NEnc|!6zbIgn{|^72{!9J%-Vc?NWIOoxI1T&&{%297PomFg@3TIE z$_0`}DP0yL-6EPLnk*}jQ6)O5kGw*@MSb3@N?9v=B`)Geb02exI1_m51T#fr#8jb) z6-3QrnjIF~L@?J*{6`6l8?Gh+UZUTDT~_3?8H4OxFF0GmWgA*S7MVFcufsB77-Y| zIr3V_7O#u4(~@jq2N~NxE%H#yWidNKIsTpGV}t{xH{@B0?OxBlyLfd`P7@1+IlRfd z8p&p5sGKD_%r2!hku{_i)St{S!AfaAS)4eF-HjaLim>O|zB+n1x43>eQ!RFVvgT1u z8?eawRc)%AU0#tNk$vFjkF-(g^%;wPhG&k;>hbeP`qh+%FRMNuN@|y^O?mo#aLSn^ zNkaX*rnhh7`RT9g>f8euZ+K0tQO=9j9~PN)yt|lFuKEyU2u=un7S_+d(sx1N@u2Jw zBIJ3%8&zLnJo6ykkJg5HQg~EaAgX2$Ya)z`8@|>bY$(yJF|IQ&wyq|kjo%vMG)!Y_ z=LK3a)0;DvwTk+M(uo?%JSG0%vm#_s_~o#5!H@lJsn;v()$>CY?Fu_Q?074#B6fM` zci+)oe+3*4JrUKvb$HDCz&ly@l@~;2Ycv}0k?poFAlDXOHpPID$sb{k8 z=c#kzvWqf$rR05{{pm}>%?}?wdL?QTmc0*roAK(ekB4$|8^<^vk*-tLyN2tx){^VK zYp2`W@&f$EL~e*}8IusPH&E;C>)p%$V{lxUDRfWJZ~ip(W66E7R4!MoQydTtVdlFm zrXISH8ilT(SwQ@9+;)Xqg6lOE>~cZX;)Y<8(b3I))K%*$b@?@&q%GryN!}{wtG6j* z6X(grQ5l)^uMJRGnXPsHj&- z*K$$SrRuwtlS)q&Df1>}cFCk>iZT~v88Y_%(0tEIdic5Zr|J*FkE9Pf-VKZY{C34B z|D1xV1N9l2LX*PONi!OVM0RUa%`d3El+yxpg9D?V#B>V3A0i0c9%K(K4V@5~67@WM zSx8XeeE%1L7Xm`PS1ASw&oEz-+d4;Be;JR1$A$u#b0=(5jo}RoD_)dFm#wQf-Eho! zz&@A~M4i!e$NAj;*}0GWk{-`*lzx_5yaIv>LN|ueqas^RX+vuh+0kAxd|J_)f^mfh3i{-2$UjuLFR%5lJK4W}6#S@9ZIwFkhdZ@j z>cf-`U#EQ8`+@w<@#f}h%j>HjcVvvL(P^L5A8PniyQ*YJ;h55jnwG}%l&O+2-a~zt z2GoZ52W|4N^&1g*DEv}nZd6`WSk#}f@v&oKHpgy`eGzdp=#fgrC+K@=UzsBIAl3%n zWd2TilsnAiu3A~Xy5d9aHe;B%(zeyP$$Z(^T|Z7EGiNx0NIK>S&RdRE5G#40IHuj9%L{Sd|G>f>ZMews_QH|yuuuCGh3E35ghs;Km5QKv#l?mt-(+5L0I{A&Me zM)rRhV}5K+bEj-g<|iLXUXr>hbvEeV+Y^s}?Eazpt@cg#w^<)@fAlDm)>Kt>uRK#W zwD?56cYg0ORs9BHIFl|KEE^!NSN*G=s_dkyRCf=!8gd|Xd&J+7gQBKHr^K9Vak1s< z7C$4TLHAW_L>%E+@g?aZNsX8(O%VUbIYH7}GPNxlhwFZrsMgEYTVUNCu;rL~>RRja zjB~8Tj>qohWD_$+d_i_V`j;}$E7<>B=!o#5u(ZfcEk?BVX|t?#>$r+8`c5ewxp9+P zwQMyh_DJOX2uWyR=*Wl@K|_7|tGY>E3QlqYXt9(J%U69q@5mfjT2$jxcCn&cMUVW3oT_X=@q~s-;u!6zu(zm<=z?;FbhkKPK0?_T z5Err}SQ}xDSQ_y>`f?08c4;epi|^5b(8Jyi)em{R?3E%>PEm@y!qxADy=hUliKZ08 zHNy;RJKMjGagZlrvKx$!hSK_vx-erQF@RLgqDty{nH-5Aga1n0#Y-IUJ@{-y_wX%I zBO+&q8e^+k|JSaf!^iexxK^9MMwF9)^P!$ehDE zMT;jNaz67kGrdciHf?I!DL4!jRq8>!uj_4nAGA6jiqt@!y<*oX+m>+&G&KBa@9$&2#(cGW@lHIINdCMp@p)pwr|N`O2}|DjzZmdr-z!pL{MWrd9_6mc%+KAP zKQ(()&XL@e(S+!U*8J*YL&*4=(t zduE%A82`vS5ko@*f=`6K2zwYhF+4Xo!e@hY0B;_11#>RzICBcSGkXlvLCbP4ux$dH zv6pF&jqCKYuW8yt4WW!8ucIAe6fjqF7x8WjI*V6{+bCYD_j?uivqK()Eeu~9`Z>fM zq6jVwvW89%dl0UUS{=SBR1)-??>f+g3*}v9-z9y-J%mpv6NmAd(j zfpvdYDJ#B|WEPecjLzfceb4!x`R~uRX~yLEuRh<#fBi3M*w@xc-xC*n4F0etKJIPU zs~InDy&C&b@?~Puu++G;(V06kOViC6@6y};9Gt(YO530^lsdPPtC&BTLmB@G7V}?- zi@i?yRs{J#Pu>;NKely@HP+P16|0Z!6#Fc zPYL6B9XO|0_vlB-RSuJ7rGc%RtbN;e}yLlb3uOleFb?(x-wCtoT`Pw!B{UD(qE5{P52ipP!{{{n7GA*Nk1MtG~5Q+44Ow zQ&BjjX0f5b+QRjTyoC(xo3wX~4(xVfyCTN7OE3}CJ>*We6PlTH*15puZOt<(_5XskWNiFd z_o!+~`TU}o{NuUPa<=7`<@C>r`RScLKUMRco;KwBm!!tTxg=<_Y?2i-ba0^y#2iliutnPl9i%tp_zYz ze}?;sbBN=|h@|q}yQ~K+-cx@RS1e<-+A zVk&pmF4K9LyVzDbwa#4fK5A!bH1DWjl|UzxDjU^wKe^wuz_^gIV0jof;(lmsSa`T7 zEHgYMVpGJ8@atjKVW~mIez*J?KE0G1R7(`GvNS1AOadgzH`ed0znJYQJ;`cH2g)## z-CaVv!N{SFWEQdF>5C|Zv^I==bP6MZ`%WkoL`m$@_mV(aytJLXQeFh3;(cX+db8TD z;(4|A>aPw^5AbrS7+?>)mi{J*mbQ`Z65IGUc^x@d=qo4}$qnv-_M6T;$G_H%w${XK z-Prn`HKf`*)!CJjs{cyuC96w*D|Hq0{UyuWn7uvwYVM)zm6;tgU;HqneMpN<4N7yT zE=b<}h4wWzbv%sjs-J0~369GhS)5qxEclu~vN)ikx~zZodtJEsZ|f)9pH91bE+7E^ zrNy&5vU9m-g#Bf7^?YTO7u9c^&wJl?ffIv$gH8p;2c8f57`QR`WoS-taga}t%>S70 zM&EzDb}Ic8cjVpVG17&Szs0u%tGPQlN7+-E>C8IT7UmkNKW#2!0@zVo*kjpcoO2u@ zX9TC1&E#(9$MH$R7;z8rRoOg6v8+hGQ?W?VRh6XlQ~j&Ds~n`7s=BNy@*3(rQ7u;0 z%m0>dm$jBW6+ILj5G)a7^Tx3c)0R=UQD>4Zu1NBB=RNCcyO3yWpy}FZiYgCRvg_X0 zm)8%hORl+CHoG{t^j-jJLc`@d(V5k z^1L!Yc~xOlK9aqX)+%Unwm3y}Qmv>jqzJDI zck`13F``?XQS7;#?eu7JA7_$#B;ZPylm2qwwALAeto0_L(WXDxu)cPV(V}TnKS_JJ z{(j}msbC5>ej6{l-I z*0e4S%1_JMpE<9fUnNmCscdLzMqW%IQKlD=x^pdH>}zDBxwF*{{|= z-+x5F)R6We-2%e`s(^Chf=`5ZrB{--S@oAvo zn_!P1Q80_=&p*WtlSGS z=q2^z>Jlnh6|t4>;uRIsEAACNE4^3|TQa`vXjM_+ieDRkJ^rQ3yIPQ3;3<P*8BH zJig*Zd2Yq&ic{t7%CpOcR0Y;X)SuC8(e~B-WBcOTBY_Bt`zQ+ zyp`NlZt?Bud%<_8e_OvYpEkZ{eV_S-`FMNNd{%qC_BrY=@jLJRP)$(}QTob<$x@_~ zMZ*QJgl>K}UPpckuZl8MUf+XGmK{h`{oGTb1d?j8gT+Z((*vcKv9K>qF zNML?pZ{@t=tYi+Sw4|&hl{(9tzd?-LhTLBjrwrSR?SYsc*_^dJ#*&< zRo!jvER97|uj|rypgyqfUc)(EucR!cfD}$Z3?CIVmx9V z zAVT0J%-{#{tlWE?v+T*Nb&Nd>9rHXRiXOpw%)ZN5Mmx#SFz>^7J%`neK98cMw`Q>E z6PSOn&8$Nl6)%a|o7u#Z^Xpkz>_)+UE|+^w@>YCTutBO8C-MSCX7N$(Bk`_sOXX-;2qgnPa9 ziqq!$Xo|HBai6dRS~Bf!i?@Yu?{8gao=A`&DzU`2z+A5%Oaz<88}<Z@SiU$6S3W^1jh@_Eg&y8N1p^&c81)!nS_URzYVucoZlU8k*SUpu<)K>hKC@7nA7 zC)#@kyXCI=KMNS34wrSHEyFq6xx>jP52Cur$<#E4hskSw}fm&VTHqoNe4! zTn4v{bDZbPFXZL&R|yh$&$%6Vb=){^KkjbMEA}ULE<2s=$GO9)ba0KW%Up+ZQ+^oj+{Fwkyu=b{}G*Ey-4C-)>!D zIcR@xH(CF%1v#ErzL*c$ds;0fw>1_ptyar^BEmkz_Ln7vC?&>Qvk8uUwB?<77qQJU z*xJh$YfZAqi9^=0R=edDppm~=7MQ!43oTtNf0;#=9i|x59n&DwVpE)nZK^lYO*VbB zUTq{BG};ULUWPW>nc9E!U3Hx_T3Yv!~jp*b5!=To;^)PNSO*navwWhbc;`4>gjuik?pA zG5nafnOB)~RtJ`XWnm^W8u+p9m~zqvv#oGu!pnTvh&zAfKlkiehX7qc58Mk z_As`S-JY|MqvOc9cR5)cC3i7U_K>-ioM`Uv+`qU-xmUUVyg@t*cRcSSe-HmBuZVw8 z@PdDyA1&|_bQDMg_51+A0>NGWb$%+pgg=>I1zUvkuX9`RcJo5GwVXI^IvbEVoOITo ztN?Zji^dwy`o%oKsHBS^z9VDaW2k6LsTs6Rv{{sF%6ck?`X9wXUQZ^-qX0ACp1cI` z`8>#sDThp;U+x&-(8zT60Pc29(>wQT_fV(}D}rp)vyd~o&E3~cgFIF$FmA*_&gXNe z8Jh~*h@0NJySvvw^exY2aHYGpyVkpgxqMwI&MwXz$DfW^2gmWyuC-}xpKL>H(YB|A zns{PeZ{1+^vmUi5E&1lF<^g89d7tT`X)9RYJxyaw4~=t-SB*!E(~PZ*gN=iXWri$> zgP$-QGW=l(H&p3$`Xqgh{*XRIze4{(zghnq%)inP)EDU}`a0b+-5uR~-6vh9PNMIr zKc{bPxN4voZyR@;BFt;dP3Cu&`PMA!G(u*3X`5}QI;tE`ol9M9+$6|Jdqi4Grc+*1 zj#9hQ;%UR_rSuVuXN;*#BXblh5yqq(c00~#j+DETo68mOX7FzFUckuZ!;j*3=1=1< z@mle+xm&q@+}E5*9D=P~7aY8qu5r4z+Meo5X+9tGST2FPr< z0@*8rAnO5hDOT5v-T2uI%^hmYgdbF?SI+8~Wj1 zUJl6fIs6BF8^4!e8)$XD!b$LE%74;Ae70nT?5p5Hl6rB}a6Wtem6=jP` zL{?4j8w}3cVtT+hfg_x)EmvU6ghZeccV2 zTltXH6bd=Kv!JKHgE!sIO}O4bj~(cex-tO8GsoG^$#d2LYX6jDrDL=s%we);+aKEx z+2`BG+XvcP09KV_FS8}v9^1~@cGwo$#@Ytjy4iwl3Y&!}BXWuN#53Y4@gGcAh*QKq zVmmRPm_dvudJsK{wnPlkl8A)Zw2BZCT!I1n0zHn{R zY|Cw%Y};%{Z0Btc;F{mr(rr1mN}C1VJH;-u``QEHo;unG!QCygue0y5pS3@;r`gNx zBxvQG9b+A<9LF5j9G@Na4j*T4=Mv{>7$po&f5>3jC@@o=xrF67-HsHW;L@v>n@AMp2B{?=7HV%52uO~ z2_wTpZUfhwH;lK1_kdT*WAHnIL^%qwBnM;(T_A$dKnbIQQV=W%ginCL7bcm21=690 z{~fkF&)>qI2zyES2HscRIo?X%0A3_$)%o1V+`qXqxGlLZ&J)fWK<8-L&)AFEvFv)* zBi1TbE0&RYpE;E&g|@Sr(V9_8KSS?Bchc_D#?olC|ELS75^5IZ0%a1#o0363N*)3> z-dD(g8wB}iNsy~L1~>t#pyK!@sA=x&wz<{f{>^tmp;2m@Unc{1w+KskScu$|WEY3VjMp@DYxo%li|5RZrl#2w;a;w*6(TEsqR7pq~~KrA7a5gUoEU{9Zg_VIwY z1ugP3OzEJ1R1yY)WaGfGqM+Rlv;AdTXFFo$!!EK%z;%qaFR^ci*6`4t zX}8)Xj!urrj-`&XjyG@*Zb!JYuXBO(l=CC_gp_cvD_mDxnJ%*{1ZuhOgbMNHkZ~0W zj1YSvM>Y*|@`50<>`!0>cmX+|4#?1K0d=6KkXMn9kS~+pkiU`(p~X1J6bh3frKl;v zlt4-VJncY=A7o{Pz?>fioU8B=Q}`57^EZ#8q{v`fKT0&EJ*69^7iAP>Hf06n0KBOu zlw695BB8d0-oKc73R+m`MOp$alV+o->5=pS^ilMM^sV&$^nd90 z=nv>G>8bQAz!eqIE9mvjla{Wf>*=+yTnztwroRVyc!z!h_E}B;gFcepjoymxMJLni zX=${(w9~XLvfRdd#JxtyHdTVF3L~JT^NaGPa%Fls^V=-`PbOhcCojcq8%zeha&OOUL+}+6??)Gvk+!VJKV#N8b zRM&gg6W3kWO=xkKT>rX`x{kO`y3V-H!RIP0UxK%C&vhNpKGIQiEE*2wrjF$9LUAKuFkG5uGX$Fm)ym05l+3c{{NA5 z7GQ0xO&iW;B?(XjH?(XhJ-Gv%fD8=11F6;S+?>~E`Ko3yZeP!O6 z=edV!pitVI4yIeut!O{mO1*{1e=OA#u3^+PY6Eqgx=3B1&Qd$!qdrs!g-~`9Be|3} ztgb!aZ+cOksA$TMQo&`QBB`O&TKJ0B)N|?%eAO=a4trp1J))9fbcyLEbZ0t_9uGgw z0C+o=UPd2>pZ^bCP1|S!VlF4VWie{FFD+r`Udrret}{28yYS}|<~{R`DP>#?0)&qU zh}(Js)2ETJ)<{fT1pEGOaGM_kpZqC!uaV7h6y9(2q5prk=*RH3k*D6+Tl)VgbdCJD z|L4LrlHnT3Z(HH*e)#&vPO^=p>{sANZzLzCfS0t9ji!P)v-SUYN{yYquYfZ*1KiA9 z@DaC%Xk;039Jz$NK+=#>#EMAaWFE&J52N@f`x^TJ#9=?#MeGJP2MvTcrY9PQ4n-%R zv(Xjk8i;HTqsP%>=y~)OdKbNiK0xEqSMYiZuczok_`93vW%Mj);kUtO&qt@iSB``E zFc9sIMxkD)0acMqVP%;Pn#5c{2QVY4EiL zNFjWW5_n;V4K4@Fcnq;1M)=K*@No_NpK`bln;wz3iE6;jOA-ECqL2YaMi&`mcmaO2`7qS%ruBkb}}bmcYOfk_$QOj zR4`VE(iJe4o5JeQ$hc~xj5c;pZsf?^gq5SQ_fBKCrbg0s4McJ@a7Y?i0YPxiX>=3! zf{{1^I7KtzzWslXwNv2sz6Ad7r_kBr7mUq9aHM14d6&bT_5sg)a~P+6fuc7Vt~tnJ zWE-s6=irGyfT!~do@>_so>&dygw>k|E3zl7+A$DswSt(tGrKp8q#|7k??l|V6-(}-Pvv7Y6fdPB*@^q@YoVI`oD2!fl*P2WW&Go3&!0W7*%&* z%pFJeLNvb?exk{+j|_#e(HR(jjch)}|J?oD|M~6ffJl%7F8TuS%)bK{=L`5P@4@eS z{eNVd!w?HLGCdo+Z8Y}U_>zuwpnwvhqc=;D^ce&5JCq`?v1NH}cd ztfT{FqLE(S*!!uG7rPh6-x^?JEOIw?ahMA0OrQUe8ylJ9-ar&m0$<3<)HAiPDit!2 z<79p?U*Op$!1W#+sLvpdcmeOlGnbhA@Z29VkKxa2%q{rnE`0O?K066spWthL|L+A7 z6!=aRObt^Bqr(YnT_dkZ0Wpd1|45N7V4O5|6B`a^oW?b8HLy1J16Sh~aG)Al7GM6y z7inaw*nw)&{D01NFW9Fyv1d!adW4DpL1pYKb&y^F_WPx10(&ES5zNe3w3d_0RdcU$ zUT_m3c51_&gd>(L6%Yo%%LKBCd6DCm@Ai=%`gW}G9k<~xlT=aPG1(;1>{X;EExg+2QOr>|3ZAox(A4Lb)Q&8zhu90SF_F zY-l?~cy?yCJC8L7*5_bU$M$CxGg7($a0Jt+Pw?}cqq3-rtS|1b2*b$%y%i7X%G&MT z!IabA>14Vd_Nu0E!urH2MP{-$Ae-H9AZiR}oOA#qp}k;DSw+obrm_5CEnUT$3+yS# zHq+U3C%O;3Ho$7)$@Fv&02;$maR2+d?=oUo>kb2LYap{5at`C@#Z0CfMG9cPFM>U5 zqPvs{blaJK^dV3OW>JywQLa0h73Yp*UO*JUf>UmHdJWYcd`w@d-SiMxdxp^$DH#z)UXk%cJaHG?;%0Ixb%1IhpOP2I6NG|%3vulr>H?Wa z)DmgLLO3UXCAHL9f=zTME)y%r_2e1IhCGA#wj7^A#FAaf9>j6{I5r83g7XMMejzWD zPNFk03%`UP!qebv+8lf5@`f|bEIfo5OO)WJaZj9rsP_%F1?Ldc@Nrmch;-|)llVIP z1UAL>*R>ZjnvK0R1zHn;FgYR`5 zSK;IF3vimp@SPCrCK7K51CfX4;j8gjVk1#R^n+6%MvkY~GP7X^@rJ1WH&p-;&^9`a z@q)FW1MJ}a*b8A@>%jg5D`p2Earm$mXcc;yGY4YE8s0?yUjB9d2!0p7AAc*qlb}kl zPuNQoE&d_qN={0eOBYMKOZP|;!3}2>8N{>2^Ta9Qa5%T5i4)=M(pLOX_*YQRKgw&v z^XGZ-LU<|MgIqRu160`8pi^K(Pi6IUAA?UI zd_@M)hinQXYz^#mQz$N#N}eOTk(Y_hq#jOkcbSebd-u3;@GC{q+v(phBCPazSc^hn z&ecL}W2C)cAN6E5QRNVoT_HKtdtxAQoV-N&5b<~)sI5LDmJoCBNm#IRJSgM+?Iy@$ zY8;i$>)^Wl&k(5(X;EBWzTt~Eb zIc)E()0_eH7xqX&u>7|+!gHPXJCAt7GtVC$q(_=(iud!t2Vr#)U7HSz+TBDLJ}Rtf z=%SGBf#-ea_+)x@_p0(ZsaNY~>6fUV$RZ@01i73+tW;8huXA3t|8Y#h=HOqjevZkO zZ)V1*tq-YPP&2bW)zsgb)Y;b=jhl! zsVux#&~5)rAKc@YW{sqry8*e4ud%i>{i=W55N1AZ+EjO-vRl#W?9418<6~-3TI;O3 z{Cy>OS<^CE$v~*yg_jPjyj)9|(wqmGO0=5yM}P}Q%Y$^Y_1`_Jd@l#RihLXMvc>h- z?QL3hoY|$go3v+ruMs^$dra!#*(0XMyBf2dE-ynuefi&e_#nCKYRWN{66T%lZsmGa_weQt*NBwLupu9YULJt zFZz=x*|0rC7QWu+h5m^a)t~dZ7pM+<9aGa4?KZq!b^GvE&L%&bZis4#X%Tz3NpRDp z5hWqBf|3ng^j;ocRefda1v;cP{C9tKES%#HJ_U3Zco34O8OLZzJ!)%sBRc2FK_?BLDnF4 z=z>W9=oif+TfA(!wbkD?Gg|w#JJ6v|`-l$L+oiN&x0c7&HrF@LX|^*8X*M-td3g8G z;eN&Xc(q+=R1TDvLACF?;1*8{B#A5J3tWzE#&|?4{0F9V4X|H!7CT$p%k6}77v6>3 zMC^l=e<7JqUuIgfzC(l?!~TMPK}RC1*+bBH_Cn4}_EdBre+BxPX$L%^-^4{nEp`)E zI_rQGbIN$CE~4I8Hlx&D$*=5LT31XKR^^0bru}J_J}pC@v^TXX?O)>Vq&2^9eS7o~ ze`oxh^r`E+lOKnDT=}v5`@GV6bFaMp`QoB`sqB=*9C0O4`OvAAcnJHc0u3{Hg>11{ zt9+Pqdxd%G9J|%a5%Ut!}j&*=BL`z-CENrf64$H2ia5u-7i# z3+Z;jS}t&<>0i_i_ju-+V|ByGvWA>4nFX1xb53V9%as-NDST4UG^RKtSZB@OYdSm_i+C%ko>}QA|Rx;?kMvD(AKB%|rW_wKXEc59d92A5F%#FOz zY-#h|9qpYvb@Y zORFc^#6^Fz2A5?NnDRn%rT?z~8=3wl`QH!2H&()sPYd38eOvl-b@o`NzI<(ptY~ja z_KS5tK4o1hdSQv7V>#8*C}DuJw7#A32o<3S_6iE93$--Y##{+355yxE2OJCP&_dYu zZTqOUnr70b52J#jH^ppdCXM(La?Y31tyd-jbZoaQRZ^pxt*mFOaKiMZqCTe@EUkhOiuUNdv(V(IrI!xMheeC{r<2qsOYhw?F zEen(c9rSzTIn493A061z*GF3`oh_cl%RqqD&a9zo@TE?dwUceKrL?|T{iOPrwa+UL zl#VLiSp2eNXIYD?MYXIt&)O!{Nfo-XVdZsDHQrTo)w~l60M1E(<(^g8K$@F7wwV{h zY#QtkTDA6S%Z0l9hTgSp>f8-s74^lR$`+M3EgV<0q%b2pEc-@Ac*?y*N%pnk9OF^v zvhv8HVFl5k=ewMrSbn$eEa@f4;YG0l1q8=?7IB3rLmR0>SK+Y>u1;fsJdH0SEm@)Sw`D7;ND~^ z@rN2pqsU(V2hnB821T?wLi1D??%mXXT~K`Z?T8Ld$47fK^NCy;`911c^Hs5FE#}0e z$MlGK6Fnd*CURSdJE&!lIAEXe8qeFhB85 zC#xD%w!EZGQJbRl;&~eE5229scfjaUzSw#rKG4tR2o>2k=LbQQ_#~P>iUBc9s$MBofeIIqG%IuOB2kdib~T&DKxT5BI+AGbVr? z;tm`du+sOJ$1HDf4FtUKTj1fL8Yk^2sumpJw*tmbo^-9Wt&ntwJFgid>!XaHn|ZYZH7fXkF9QTD>k-y>t-QM z(QrrTl;9SD`~4pJWccQLR(nqJSfib!jaLYz)xa$qL{G&(IK!+yR-JiMO>E`s;^ukX zv+rk4&hX7B{nsz^a%RlG%NhO|L(@g+L(;}(?EbguUsz73ocx><1q}tPg6#!E3Loa| z&36~=$oDPWTspG2bJ2xTPQj@BoKpYlDw7G$qGsdw+BhSpKFEY)1DQPD7G0r-Qh7#p zOJqS)MB_Y~`v(LOA-4ls`TYs`6d136;lc5j2lNYU5je>sUv*t0Q=d_34Ab==WIsh- z{9mjo%vU;^3PpJQB5oEk*L?vSY?)&@V3j*gk-0!)=DX{tL$21&=aB1<18096G6Tgq zFWC9+D7HWM9QQH5NLVW{^1TIn1>(g3^P2I=$dO1G*10c?*P98fiXcx zg7`r_{a5*g__gw#=X1&TF1VN5`!4iKHSi6;3@t(b)kOA0dHIdl0lXWkx?)%Gd2SvILNfA z#qoulg8l^uvwG!h&Fz*w6J}pR=BvMS+Jv;)r17coX;=TY%)eOKw!VL5RpHS*N4BBx zWL0>>4|^(8%jqj9fO<_1oP`4}DP z>wD=%KBq#gVNZhRd(Kt=0R7KiS%v7BWT7Ho6`>uY=qpTMf2FUxE0E2QJ8h&%bwCy{ zRn!ym7nMOTU~KdTGJxzy3?T}r!$1i8%=U&1e>}Sds0T)%1_2_uFPk8LrwC9vl%NAs z7OJjlcj&V`?s>#{F85@Ek~iGj?zPsl%3#)c>WrF8>J)XldaZVzwz+Dee3Imm$XnIg6z3F8Ekp{_^UvqM%H5uQCVN8Wri@pA6Vgtn@5)+Hh?Mp&Je@x+ zXF`rwL8szPr4uS%)owH8*v^9fgkx`tDd;|^MW7Xo;?3m+h)&2YiZQBonyH!(`ktN^ z!(r_U_}yk|=X)f2HFyp1;22Z})KH|`p%JNDtHqjT+KJj4wL_6AU!-^i72jE)M!P3| zB3dGvE1bng`2qZe{Ih(3)p56RDk0}Q0WD@npwH3XoORq^Tsbe2_kcSO7Vk^j6^T;P zL;M>wa`S}@)b;-HJ^3r(yWZ!#LHDtTB0?6=J)3z-pP)97fy7FziR-8%%humI)x6ZS z+?Y}Sw(d=hrK+-Wb;Y%Eud>l4eT$0=uNQpF=jIjUc;y&!R^|%w*5u{qEXg^L7nNUI zFt})EQ6!vgLaP?mp0EAa;9)*tR@=Wj`Z&M3g0S`Ee&&I@JIduu5q^Fi=PV zk*}LHTHa3KtJtNuuDk@=(#~p=dYopECQ*~79S`dI2yLZik9L6eo+e%srrxaTuNJCE zMUuiK|0@raua@1H?Uq_310ZA2MiMC5C~hU*F4`>$20eHRs1Y}bzlnc}FNueWvqk$w z4@AjOVVxwZ6P*xkgjW;bI8{USb&%j9{}6vN{{`;^Z!s^9mj@2~Pn<=ZV(`fnLG7&- z@*S#m>CkZo%>Ix&3Z+D3HJ%Bv)DP!($3^>UTZHwMd5Wp1;e5SU-RK&1_5RAQzLb zIXhrIh&5y#H4+)c{)_hJ59Oa0>=LyTi=~65&1Gc@gHo!B(3ERNYAV#3hoO0AZN4y zSQUso66h$uloQo~FrqeTqqV8pH0^WEY+xP4YT9dtXkMx_RKVzw^W<%%wUXYFBccJK zgThMw8J?8;lVj(+Mbps_oVA?GoShs$&UIh}IJlp9XL#56mj$(g5@9=0l=zb*Q_?|d zl9ozQ(D-bX<;(Q)5XB_LDtWBjPrg-_E8QUN3aZwol6c@kHo)G)=YQj#<7@+p<#Sdg z6GtB*f8f`#ovw68AA6?N*V4x{y&=8sZf#r*Q}v{>s63+VVM%S#%EImiz4Axq<>idY zHfJ{dckr((t<|4isi(kYsQj}m-RG}=Ml{?pe(vBrZQ-KQvE>ofS~zhIv8tUZ&K%bq zyc^XXvaD&W4V)y-P2MBHVxdADDw!bDD8z~a<#lzuI#+W^SEC1|zP3blS8-9{D=(Br$#Os^I7Q47Ef>xZB=JuQLP3k)S+HI33TD(0LAqcU ztXy|RBSp(Z=`hzui%Z4%Vm4@^0znQT%UzHj>vX zpIbPg#J{3jb+mDeO=sWY^d}}lZlN`~p1A^Kh1DD}|1B>`utUTWzZCzG_Ep|frYQrp zV|C4RR{dd*KAsI8nH~!~zZmrTm-;w;7I+ByYX@rHtJW#r$W}_lpwH?E+Pp)e7jP$h zgkr%WL03>)_=`$mjk+#o#6u;=C1a)6WXa$;7$NTiYnn>YPH|3gTiF3XxucY=mCcoZ z72TCzl^>NYRXbFevJCV?GZaqwb@_UEs_eG(gXFCEjj)+O&u_)+z+HjPKz6%_(-X;l zct4lTam1cy&vRIofz#JG~0X20grJs*BW@)PGc;RI%#Ms!-Ko5r@^N!_|WJPDr&4^ELnr2D8n6fdMNSd8|J~i&|Q1A!6&AC)?rdSC(5D zSF^X?YPoJ(WiN#y)Dg(Jf5#tzR^%io_*2*ooZ0+U!cF2=($}EGy{Vd@*{74~y$rb? z8J@|WZ9Qu})_6?Q$LgYVb(%m;faZdF6F6U9DrAbL@@KM%@_boGS&A$Sbc5mYXn8#7 z%SM4>u)pG#f&=5_sWML0L483zMx|5zQ0-E+QYWbQsRwBMv@-1zja_|Ny+bX6J2g-3 zrEaC3tX`#htvsUCDu*gI$`fHct(8=XlREh7RS*1hEwpP;Bf;zM5ijD2K>`ZgEauzyx&Rv)rA5F%& zhqJqK5_zPcP_#nwM&_YB3na%w8lg5)yIa@OaK>;LR;GixMVgDMJF1bYOG>NqqUtjc z1}qQ@CCk__qdLgTWII4b*;Rg07A89fO7|wfAvr9sk+p*OVZHo^{FUOV^1RZa_@*S4 z8Y6+59+PzooY%|plqw$r0`a(lfResmUWPFBtyh? z!eYTr!7Kh7-Z<`Ev@N?ms};khlOX?JiTA{2JL~MtY!56kkhQtnkXXO6&Q+tTzEF9x zVtBb<8Nbw6{I^J4=*VlDyD!^2>+`?v8QQ-))0+K>`ZFZ0RmO+x8M$S-!wQ}iS&D3h zp2h3R*;PPlF!Qa5HP&|7I?CE$^>p0F<`F09ChTI!oks9m3ZICcNSyNKO1{#lnxmPb zS*D$-zhNlTYxM2(F}ehJ_M;;AAFM$QlAJ7tODio6K4 zssXY#GD^BiW|rBZCi6miPg*WrEk_j`d7L~@fhs~2bqXJqT0KntK;2L6ugX@I!!vrL zOjmiT=c@Xu)+j?1EX5m8Kz4%r)eBsxQprH^G0{!oQ{hITpP&xn(Ihm6oxytRwlZRwW?}UHG8&?VBT`RUx~va;=2H1k`3|`Llc&kGijVTg@&)p@a3Ef>9G7%;65^sdNH$u2VK=2g2Q{3kqGuj1h!v4j&?%u?-r8`l_NE6YE z*o<$&{&S6Xo_GAVv+Z4MYpi1}YIC}AK|^xg@Y=`K7_bN2rM-%$7hK7;W~FB=O>2?j zkvQU~&-b_lGQscH*Q9YN1*xg&f&c#glV_@Pjuqq<_pfYT_p$z^vD~!Ulw%rUW~}R+ z8?dodJJxA-40jt}Cn}JT@_2P;-3r5fuVy~$d`I|?3&ex+f~E!K2l)pj1o#Kg{z$-o z{uO?OeouWT0A=ki(7V>@w(EN8ytOh-wEDI(MSe@>Een+%mRyj`m-H5A2rD6)n9QHg z*YJILM|r{gIv&FBz;7)8cb>3HxLZ^$8Y-S4z91&Vw8cfyMc+FB`Jr`A|wHlC1NwU;C>4?2>pel1;_ZQyxrWXoVjQvROn}crV)Cr zy46f8x&t+ftS7vPhu~JIaZI(xTa(PKP1hQd>NvGut5_9HN;emM$p4bV&RUqUDs6m9 z$?t1FHht^(x%`9p!_p6bKi&DZ_Q&R51&IYI)<0Eg0U52bujUOa9#ffGlUb)V?l+A! zmzrJHA&{>qAP2IJp|5!tg})_V<_Pf% z*%%xc90{4`p8?PPXZg1FZt$S=6Ld?p64jIjDi9;EMb3-%d)BAsXOJhWuBokB zU6EAkSKOiCPp(Hcogq&Dm0FPeAo1u=V?y%R{7;)d^nBm{J^SO+&t1P|eUJG4GFOx*N}$)>$03{?1%%8qG#7qwjfZgd-#$fGB%O%kl8@ zHu-+{{}ZSSITE@!3<yT-zChm@mp6&wc**jc{hhb;If-&_`>6L6?>(?C#d&S;#69MC)ET4(Os4>sr&FB=D{ql}knE!* zP`q2{@){PJx>*L;t~y#`G%=o8!5+qo5pEUFk!31H8jWs` zhtB7x-{OE-L2W}vgd-6vBHBg{h+G`GDspJ#tcWh*d%|vnYC^UIjSR5({`AH@PZ~xZ@A_C@+B%%Nc==XM?(q^~SxA89?u%%E)3u59*1rpzQ5{?R6b@ z6#>&`Hr5Wy!kQB6$SG7do#gglPv`XF%@AaZzDg2gm5MIv=US1zx8ajVcQ0S>Yu z1ipWL$NIhXEAo5h_r$NO|9(G(|4qMIzY~6#Z?yj?mH^i+s*KslrlInB7qmyurEJ+@si%C8;<^l9g#^|5<&(V0}kl;Nt~sx8XF@?uE`SdH6?77EUD z|8PpVE%;*LdO-xYHF}dh3T@8`<8t9?Jw|!prYYd`N7u2%sFkyce}+GV*O#{+D$-{} zJw#g2s&AEzR1%77Dn@Zcm8yBKd+*5uQP4ZXQbVS`nMXCK7`}Pl@Eqh~JK?xER9x1sleJwc!?|&4Qz)8O? zUkYwVl^~TL&9`xnp;q={SP4I|MGnGp##Z_A?mQ-je9Obgggvu3_d&;_% z|1H~4R$TF{BCT9nexXEKdbTXDEU9c~$%>*6#YrVurBjON!Y4(?i&~X56iN$X^7rK2 z$nBHYI_GU}bk6>q)y4OU3i8I~`eYU5tS?Ng@UD7Xom%(C=xy>dw=-Qa-LUN;XL8m7 zwR8@*EhrCfa4UI>1UV2rx6-WA9aG=tx-SHsI^y=yA(ZT2Jm(d06M)&+E08@vPjYgsON>?E_*L*C%Gw|DBT6C zUySga&A)o1Abn%`j_St_|R>-{koIt8pS_(u4=h{n5WQd zyypY`VU3sCOYuv7TN$O!*Nrgz(2?qJ#U1H%$wui_nO5;$HAp2_N)U}CT&LnNIgO~o%nJwPke0=&l$L|LN!z>su++itVqtH4VT0`u<$Zy|3QFN*hq zg8;V(I-tHA2h^N|j7)c!qeCjIE7n%+t=?PfSwF3zSsh*DSsPIEuXb>~v0i2VZ0T!02)?CH zrfB00(;bTt*h8;uQT823$f-D<5fd(?fb?NPU|}g>Q-z z3-%SAt(afE$Yi$cu?)1fF(JT#o?~8y&!eJQW5BWDiM|G%-vh*j4iNkklhW0yo~m&Q zn>t&_oz68;tP$*Bd@~weg&8=;pD-W3zXlkH#n3XO=hV)6;jV&jH^$pI~p9 zkIv_-Pf!23{vZ7Z`e*pEp?&%|pFKXSfG;%1@2+2&-*R6spZ?&LU+=ZqdzbGx-%~!t z-f14&46q&QuIVT0i}Wf(Uwv~OOWjQITeeI(SZ0zxSM-+O6K@jy$2-W&<~`sw@Fww! zIpt^#y8#@Sdgz)q1^VxBk&fINAFK8-WvcIrInhU3A9 zvyj=uoCDqdUAi^X-Mx;4M(brIsaL3Z5U z$~nmKz}n0H+fnG)VBccxV;yUavu-d`7K`nb>8iPf88u2QcdP?V>+ACxN*jWWGV@N; zw|aZ+#M(Oz-NA)g(@O4K~d!6)J zZ{PuS?}qNI-VOPSMOugEm{y^~R2${~Pz#BYdMPl)HF=a&B5DDrnGwJbTFjFPDuf#G z646A#AkG|crpnnACz{`zKaFQY6Wu@P$52fSf~;IJD*${VpksykJCU3~8fhVGENJR$ zK?BNWZqp6SJmwoc0=gV6X0Hd=@;=Z44+qunJ@-P;40i?H@mf#_3mZFiA#Fe{*pBJI z(C%)?L8Kka#_*X{OdYs)#xmcj2s)o6KsCug65f>LK{;l;B(2NM}O-iB-rg z_7i3g;RD*yR`fOG5IU9nfn7kx68mu>-jtfh>c9puAvjwuQ6Gpp@cfmLFF-^4AM+CG z{e8h_a)xb0uDh$L81TjYVAdczAc`6Sd*yCK<9%~X;oB3bFC!}%(SNT95C!9e3G?n@WDA44owgP55G6bqb(YpA*IE1>G1=srvx1*+<9dM@*x zP5=kmQgSMNl6g(PCF8*lmw|7=!*JR~I}=@PaT(m_FRmtzByio!!AaQy9B_VQG&LQU zx&oc3^B=yFyaywv8$JVDi}xp$L@#VR=1(l9Gu6gyO`<5T0oDbt}>_Tq0~!yAo!zhg2y(Gh0yJBGdYv#237tz zTI7BJ-3Ptg9m$3G1L8aWmHJ4IJN-&`32sOD8Q~0a~$`K zTywQ;wW~jLOCw!9Y@IE4E#vJ+9rgAK=LKg|P&(UfN6bwi0`_otgOBhFrY9yjX4wWh z%N@P&7|@yjrZdS}tPAlIci?@QH_%&ognJkz!4Bce$TX@^Lm$WthI!r{A4r1Jiq!?2 z4Y7V&`y@dF!E9*?pdg69nTaKvW@iOJ^xo zDLX@zZIE=bw3l+K@_-uAZqX_X@w#bhr7~5HD|hLp8s-`5btAOF>JF;g%GQbvpm8#) zrfU3EhvemwZNLGzEg{7NMcKk5qF=&H0Tf9EcX<@=J0}7yWv^ws&?ejqoD{YkISby0 z=ir!G(D*Ff9>{F)XtV~FgAXmFKfs(L+|xk2D@KapvtHB5RA=buAw~~^`?D8o3v(CL z`Y783>+g7ABJ^X2F?;D~@LF^N&t4$wJKdaGO7*1$?j#@qq%$L_cp?JU9xe1{?c*Ly zl@hNh4nw-vu{OKDH{f3UyHC5fVx8IW+A{(e zTaR?3)!&uH$^(j_%4bmdXsT|lh>}{wy+z-JwD2G3BQ^+I3V!j=^DDUdoN*AZ3*gUq zv=5xhULz!HsJjSup+ZpJ51^Ic2dM|oqYw3scnKApjZodp11DtznTK}-ulFw4`zB&x z5Eq`NI+ClfzW8ul4L*k$!VGTjW>hgaFiOC4F&`qTKUg_Fp5lY6BZ^Uww_F-mCoBMV zQYT$U^2n#I$&Od9L8J#A;x3^J;mH$@2-iE;37jN5QDwvy-0fP0^T_26wu8hX@f)rh z*Kw?=bA&C%?zUgIcLFU?uw$ED1|7_%SUX#u8RuF~TKigBIyCrP$7kCo`xjt9Ry3?K zf3**TPHF+J9gZL7OXlI$Znj%Ci<83UILdJk?2p~;*ltcSk8wS6)no1HT~J$ZaK%~E z>|!0baYxDHOJ!gC0^LF^i{MCLReJA@a^-l4O^uoMP`KRXSTx<$zm-4bRpK1cYmtxmzEmxDN~0wvQKewLV6dnw^yX?1 zRR~9k-+?CNrzBs}OwwJv2*@gzMFCLPXeBFFoRJTf{gFnA3xsb(IpQFxk3315C(aSG zg%^0ec+0unKm`)R{{{3L6kU!yc3)?t%r1I16T#BDx%51;4xHmla9{kZtMR{VM=~%Z z${hf_bnb9mak4-c(%HEHs5Jstb5~2}F&Bl6aJ_cj!`8X}gS{&cTS|1HuaWNwHN;*c zsaom_oNCSz6u~3gF^AcUk=x)oc~4AmJa_$~Qr$du0(3c!$30v>UG#5v~v^m zZpccX#vWR?TUI-7*gIo~$Q{HIvX5(#&0rg1@-yZ+im@k7V9!3q>WyiZXXa(L zR6>E@vq#&Gn%9_)S(e!u>j>**P`L&}-Q}ls6^t{bt;*cN(b0O+MB1`k`y9BX3LD@W zU`cRZvFWVeieIZ>1&tR&9<)25l&wsfLs7%mmQY6hPKA`CdS^` z!E&5~DEO#-owWp*Vg_JyuEA{p;McLHyonSB)SNZkST168eZ1=L47 z$}43}LE%14m@e4PMZqDzA6#I*Chl9z z3bz(nf~-N0aT3{AxD)sT1a^L^fP_C^0DtVFtgAw<^49o44}|Zk0Vx$S?N4tPqnoHpHG5~3msV3INQ4x*oRsl0!d|&^|r0E{keU%Eyl6OIo#oZv&Tx? z1nY0;y>w5cKdt&{2O9#gqyALoo*4UfbC)wkSJaY?cZ(DPyR+ZRC zSvy$=L(kB9qtIAqx&zLgL$+}16?49AH@=-1M9(6h+jJI%h3#y~jN|u}E*JlWJlciu8X&o$Ky@md_%r5+2Bmw4dj9SOelDgpF?kPBY4(xm_P8O zIN<%NWJ-`G=yY@qhh}d?SnRJz8EXQ20B13`N{|e_XqEw!2LnCX5?(C-wa_e4!zfrJ zX(l=bT96t1dfsk+iD0bgqU?(8=@WuYh`qeVr*24bDF$!w;xI<=lTEE&) zIBT4}p!3oPQ0k<*Zn-$Fdl0on0_UXzaf?(@DR_JQE4JR10UVgmWGngsoS_#GS@?Qa zD`GP>pPYr45;&oP-ZMX)di)f&j#x|K5V6k2c7al(gT2NPME(ans)dY_=|MRiFCD9} zdoC-s3UY=gXoDLro!1m_9n;`T_rHs>ziT(ZDVxciTz9l;R9ec z-N$~py4!uBuhIs`KF1xWm#u^OnC&F2ihA32^A>XtQ+Ko7I>-?V?#z1_h3Q>N8_U^> zctRT8{kW@m$9O4F*J{P@$J@it6P=ZCRq-05R%3XsnV{ON5-1NT^3=C=VUPhktKXpY zR&k-L$~l=?atk4h0Vb3u-Jg>wZhWX+SOKPyJwAoY;Cg5;0gs2keWCQ=dyjo z1R{re47$VnKueju#?8QkoavUi6+lnWgO~peoXvb#caYz3GTMd!xq^HM z`))OKE)xNxLJzvw6Vzj3zRSzC5WJdOiAz*1vx=2QEhhFm8+8}^?31vO5JOLZ9%r-2 zP@HguVBK8H;cU8-+KtRbJb)M_p(==%WDQxwyB#_rf zfd~Dp+XwEdmFChoA*b>oqMJzai1y@PsvQ{*e$-@`Gpn7Uj#*AV*3snzhmSWtiV)&P z$2jLISk;~In**^KasbWop`eG(!X(%XY%h5UbQC_sX^e0cK?HXh{|4voTjX~7He^d5 zQKj^6O{jp z`}_>f1PQ>gTg@LNXakI-v7!^gDwr(+{3kpKFPWRcX~S6#ERccFue3k92i*?rgE#K3 ztaqr9H33cz2pY(7L9geFTp!3PZ{(+nxFWacoa8@opu{9zD*gp^gSK*(daO#L>M!3Y zs|1~eMjR)q5nmCvkmie1pr`8-&OUYsYdcUg-1L9$2KP5`0rSZOmj=2r*4rmJmN*wW zZB8Bb%@OVLc8sy(4uRvb!^hdrc>+@qkMOs|c_IfoowJ#r?jztopGQ32?4d~mp(do=F@&K?lnt(HZ3E7j% zU_K$$Km>B3%UB*PsXKwOB17r1X2&) zXgys?PjKHr7$DP}f!xhz$lxcDjQb%tI&QHP;P9WpN??V%hq8{q`Tq-$bNVqT^omk( zz9VBGi{Fj36RpS{H)Af{D3|sVa-Q&bI%G_3R_5G#kqV7 zU(fU7*GdS{f3kmyS5l2EQ9W6n4%*Exl6=uF;Q@|I5DP>(r9>-v%ij;35w0LyWUD)w z%)ozRuU#+k0r3lk=qR?wX>!i7|8mv>?ei(V0e1DfIGik9gMb^; z6S}IRZV3~N1R#Bo`RD*-9P26)0=0-eoYfo;-UZ%JGyrMFQo1|3)v&U7v!1(m)BC9R zR1>lbX~ezp)4+lo4@9?Y_|EHG;f}NR7WQiUODEK_f#x^NVYXG-G5b5mMb{D6Z`d>Y zV=G~e^oK}oEIp9%1EK@yr06+NC0YViAwAQX31sYU2{Meekv)x71ayl6ili#2XIKm1 zHr=C!1Apie(iQzoM-hR9%(crg8d!K`)LC{5VigvM^^z}#j!?b)~-Z}6Cnu!f(Hoh z?heJ>t+-2Z*WylbcXxM(Vgcgrc4ck$oxZOhrL>gfuFU*%^f|*`qixosjoa*z(Tjf9 zyCPv^B4&s>`c?I{(GzJZ1#}eh51JVB)OLEj@{nCdBKva3 z6f9-hlw zZy+~y3)r5gXdc|>`%h52z-hj`SE5TZbAb7~6lv;;^vVmSt?*P`k+Pu2x@u&ppP-;& z&TUF9#RMF91F)Q$%3DoFCYGDtMCmK%%2VXi+Hb>yJ0x~9jWo@%6j;10^~`@tZJ^l~ zOiwJaZUagbl<@WvJ*T;5S=K?V>5yu}xiHLB*?hXhK9MKb6E`qP zE41tCMD>kwO6>sbdZDrzJjp%zr1Dj@=sd8vD~3OvfUKrt$m%&k?;;WCDfkb6Qyt4t zs|Az)zhj>#?^lJ-i^r7@-)(|q%KbB5`)^b1KVF~o_Tf(nr`aPw-LwiTSz!EtwLxJ zri>o!DQ3cxN}jw&D@0!1ab=fdwxhfAk-e+D($T^ZXP;&ZM;_-y$1=>{d+`Q+wRe%7 zjzw}K^*Rz9BayJzOG(krAXjG&Niz;Ga29MQ3pNbxq5e^`08ea8Jo$;-Hd>ktM;gi| zocE)UQPu+Ptjq`{nF_rcwp`H(GH1+=r$%qC z9lLEFX8xDvTE2-ntc7r$4%9Ex?c6rq6ZvBo^oqhB@<^q^7&cl@7aJN2^b`CGwW1tu zoOGU-V`(#e0C5WSj2k4DBqK}arlr&E$2HwU?Y;dCk&1*VD^C3Q1hO7f5w)e%Wi$Mi#tup6pB*5^X@ zH!0NPjK$1P7fA&T=oh)CZv>X&PbY~jah2Y1Xa^s67ew$Yde~g~<(eCh^)Z(9be2?X-h%JBD%Q6q#&xAKdB-}C z81pZFg8883IX%Oj&|l*$a9F8l93hvf8N5jhpNlM;Kw84shP==An!l2YY^Q8}zi@!` zGZ^2U`I0sC2HQbzu`R3|p5zdui$#$W{3)ZUaG!6b-<4XB8|nlxNNuPd=WfX!lLS2mU-4UZv@#Gl(n{?b3;oX}Cvpk~9Vf9JbEar;H= zCAY-cezwEM6mL{k>X>)4p++sq8+h+`nyK#MGWgrQ2 zmqLuT0~w^R)j)AHcS`vr>_(c8S-7ip;cn3x%%87BGciqEvTMMfyXpC~3mK z%mZ&NGq&>4nw@!2Z)H3&^SfDj(+6QbpKqRT?Bk%%aW24Bp5PROM0SefEDHb2dRXd4 zx{KYQ!d;-pI4T)_NS&%>I$?@}T6&dqgddfVcyQNc z*a$;M;?xe(gq}8H$z8hDc%}aakJVC7Lk3oR?F~Fw=k?v3Kl{P`66WY$d<=Q6`5GJ9 zC-u5|h z^>*r?NHn;v?w}FsP-B<$k(Lx4W`oK?F#k-C;=dv{Zwl8_^CC(7RuaI6ao@;Ca*S2M z`Jy@4?^y06ca22zySO#Uc~Z0^nX|Vw z#<;-swAl20KtoTdA4y;Rfij%lR>vq0=tcFuT0nMa92;#+*W<|&#_6-TPFf><3SFXn zM;crkwL0yEgw=!e8lL>4qyVRv-%(a+JI#nQ<$nnfvL<;Be_Rfe>Id&l(*_5 zX@>F;$$MAyc)km{%Y6CL+Bhg0?mBB5RcLi(EwF(IViEUXPMuG>Av>cF)?Bli#~zWT z+BI$w`B%G1+h{+OR=NvT=uWJsQip^IU)d;VZI^1jNfD3%rgjIe6^S`wKX#nEq$^)V zoXpmMEe%%B>%YJspVwT_EZ84&=oU5u?bos7oH3iBSW0V!bJJt>f<0VL*JnHS>%(Yc zeE{_|u47Ug;~1|jcX$;)avX7Nvv+_NDiBF&Rp<>albjD!Mw;H6R+wIjL*cOf#XE&u@hu%C91<6iEMW<+v0*&gXz`Rb zQx8GGTTe+cQq|hpSh@)-?ou)a&-g^6iuMaBZP(!1n4||A!L$h*%{Rums*O;Y{ehKd zHg>6>bOVy*Tm+<~vTDL9%)oizA?0j#3_nf{{}sSe`_PUZ`lMeB$2 zktbJ~FU9Igs6t{3#M00-e5CGLJpTg7V;RW-y>-S*4_wx4xa2m$|EJ(aYeZ#FLj=k!*L>i zqddg_lP&+DPPgxKj5a6gRXFExK#xlt}rSzLeagjrbFS z3Qpq!_k;%WcD{q?Zca5XaTy5D(-I(U!`-*R1)gf&X|8DQDBUnUmhQpjJ4c{!(_g1u zXg%YZ=BuS5N2^fo0EKi{poaj~XLdMp8Aq@kWGD%xtKd(+C%8(x zO%KeQEYE-;UNO63YR-|~N}r_j(2!*TPpXEj=>P8KV(u{YfM4?k`-bG_j_^=4hGMS- zl5s`N3vB@Bz(M!dp8!KMga7A{9=!-0brf)JUtl$P;&Zr?B~ywt!qm=G+f)NsPn2n? zDIQ-r*8ITA%#}>D&3;la@`~eymDmSMn0`qfQdO5>W~b??xrvy`pFoD{MUulpAW)=8 z_-cYg&A&KL{wwKid}TKMi<+)o)&~JW-K+;gc{2bk{|P8+T(wP#ubQN+R#qq#wRLKS zQc}xM+9T^ZL0zT)qpDh()>VD3W#HuIh1B0yWUn5A2_uEH<2G>zF%w@h2yh}Fc1CYX z_UKi}VlGA(g^Ao!a+|M)bNB|d6yMQ0%X5`;T9!N*3dLvodQ4#L;Kt~}2I@%YbGX}| z74>#(v_&~DX+zcXBnuAo-a;c9Zv0C$bZhKWyFmSw500CX59$vigUdH6igTgCQTP^6 zQt{{?Pt+@8T?kOI+aM2bIv9#woGL!$6G$ZWXN%+$vI4}WzdFY_$-=m?!aS}Y7^(5d z!`!M4mzyYl_J$6tI>y;e8%rv&!So!zn>@vxT|k=fyVxD$0=uR>(5ox6l)d^`wFFkp z;WR)<5M6;NcM}KT8T*&LfSMyjX{xki7xWYKlHlMji_5r|NS*Woifl)E{BV7%{9DV0 zGa(kM-&80)cR@v!pt%`m*e2Es3bXpai-$y&WB+oXm%Yb#mW8|Mryo~x|hA;H2LwFH+)tLWi8 zW?ZxYjyFb%T`;Xy5U(K(aFZ}le?uBdpZGaqJ!B?C3rX06yBb}U9ni0pbk5QL%g;Q* z53pPnh&04y8{0x^antQhwS&$w@-tPF8w-`_4?fc^L_F>i=~AEXX=*{<>4&tz>V7aE z&pB72C4bbl2U6ZmF4>k)WSWdO%-SVn-Pd z_7iBkQ5{Zn{?z1Jp`WFvxt!R@(p*}rWE*|igZv?i8}qkyA;-B`G6jrNWpiz0N)(8{ zwd-h*y6-rwdh7o3O6@k0H6J6Ln?P?82e(wHY?L$7>3O-lGEKVy6<7tNHSj(jqc{nMj)-gmvAmcD|n)N`y4DC>~5ziWsQS6JHpqo`s`(d?E zw%aObZPek4h*Mh{DI?xO4)RI*hrZqzq3)G4*g@wO^)H&KF)mnK&hL?W3SPo4;4w!r z%kN_6pfLJj@LUy~51t9UIE-@QKJF1o;X;rj@(qkYEhIvB6yHiCpg8=)JYCudOuH8n z5k{I?3Wm^58YbKq3q?0+nY2=DE437-^3}wp{8A_;H`7bT1I!t9jc;T)xeo2m6lBJ3 zW9y7zB%6(ansfv5f)|jf!fno1C=`QnCwB`E`4r%fF=BtD=|3{H6U&>vnw@9~-zc4f zbNVFr3YykPG8f)@4sDoP zk~~A^OD+C2w-G486z&re(uWyg+AXX|r&v0zgGr>SI2|X|UHon?74Oyv_DyT2F9w?X z11H73v>r1d8}+`H$l7X?p!=MsZf7E|#Fi;+D0h^?W-*uWa{^)QDPb23T6eXV5dW#@rAU8AS?Q~Dde z72U61;Z7ncI*6Q99;yTNxk`lkTo2H6^$I(vE=9`uIK^H2ONn>(RTeqI@U*mXwgV5Y zLkl?o>_tUwiaJ$^lW#f8IoCU)or~nB$}-(VHqcO>7aIzjguCK1{s~`7(72M=8w!MX zNC~x>ds{in7`V&tSn9dNSdwuXzb6$469pfB8xl(|f{zL37GwSB!i}X5xy{r}cM~0I z@+x|+`UZUN>^5|^Fy z9OuTr;NJ*az~ROsLH{tnkDtmXalg3rNb$eUuMnb;-Y+A=f10>P=mng(P`u?b#x%o} z=8_}%nM;^wz;zTMzOytixta&NR3`@+EJgdMRRKF$K|QRb zsn?Y-bY3-4=dr=iF|=b>@arI@67+Kwp>?eQ-r*m^L!V(RCYAK-%m%$*fcnL;$9V{> z?`7=(p7!yiz3~BWcq=VJU92@g|3RB5BXRvX~!ZqusZZNn3C znEA+~l`i@pn7Eo47V3$fs!(Ae=Rm{49!%?{l-mwTo+S(F5d92`=Ra~4gl=XTpEQc^ zp+8gO)K>C1;_HK&ON@Gqo*p4TqOP2}@xS!fHx!OTt8+?hx` z=|Jw35Xy7H3hp!#k;@|6zl2&{84X;yn~{WujJ{HsxsG@QtZ+xHHgR+SW{#QI`y=!u zOpeV+Aor1u;HGeG_+?Q0jo{B2eGE7Ju8%7%mT5;FtH?TM>*i*+S z^eKE*?vNzX3UA9VVXvs@ZG;E3qrAoV1=gdD#;XHZv_8dnMfad#_cE!>Mnc^jf(+hA z=t5b+df?Mu&`a@Ux$9~KZAFIYUyMEEU#^4^2uIkzY%85(P%TEAsQ9P@WPdeOCu%*T z32n^Qa{Ku5Y$6|sU0X38vH_Yep1~MBOj``JYcCjW6Ensm{L($iPV7!T6H|1BS~i( zf|SdTTzR3raEiZzH*lJ8N=O!JNxW3gbRYds9+m@^>DHqbB&b>+SWZ}`BbPYXTwt1M zx+i&?{*-D;EhMKnMOr9zk$NCW!A%mSFys^`q5WbNx18GuuI(b1&Y8j1{}LXGo?@_A z6MpzwczXPh>)!}HT;~ipg27!MhIjKm5U2*E6W7Uz2Pg7I>rPJL`E_S!-=V*#eo8aVYYM?QXCG#jGffxV-5fWb88g21{g!qnOwC;y7H9aJ+;Pg0)LiwT z)&RY_0F`r_!qDEqp?T7_Ng*9)pa;h0xA#?exqRlHQ);yhq) zpjjQW?RC_)wubhta+d9`eYkuMD#Ud8uKZYit<+O8wOAz(llDSojaorFr}oEN6UmOz z_h3;Mh@mDg>;RL{$hOTg4Rw{COw587^$U3ZNoXzU&TZq~03Y1@|Fdpw;WwIc_6e8Jc=BFYkGJ+;cpSzEZ-sQh zQ>-c;#*UBzhWotO4jDqP#oOp*OBOFme$pYamE?wSt!7n=OU7?U*Z#lK;OMlDk*Leu8LO#Ub-d*N*_$Rcmw)b59z5i9Q)G< z;X2gY0qDQDz}MxAc^CdQe+Qo+1^p{`Xe~J1`=X_y8=3$zjHzf?aMSNXS5i$~56y`~ zsfMicO!X;zAdisxX2qSZsFa1GW*F3C;c_-K2Ue&Ho;#1hOC>wQkv#7A|6KA>$Rh3F z+>f@(sg8Dzd-nPE8R+l3jYhv7_D#?vWIGjj4_YeSi}V{H)Ki(`j?;gY(+OVRyE0ytZF(D)v-Ij9gOQr-mV?zA{d^ z<8TfwhYbD(z;Ss_1aj7b-v{OK66}lByc)-=l*t8Up1HBOK|2ChjEcaE&!9u*EvB0V`bp@C2BEbkfGO%q{EF&r(faXU4*XNNgHEE~F$AmR8nvEd zs_Kdi`zgAoY{K*K%+XS9h;-l{YG3e=?n-BM9{9;MNa$+@J?l?pAT<83)stESc#)23 z8t&;}?2_lPFRzlfX@AQ-zymc^=i-N(K2-an@_HHhrkaN~rhe*3CxYJ86Y@w+#46KS zJL#;i&Q`9XpS-&A32F4h;sgM$wA2J}w9aLU>Zjyzct$xCAsbD{0gj28km z$x-S}ec(s?O%CDg>cfQ!1Gyzos$7FcBT4um`am&yg}*5H;|=eP+^_{`+%tqK_@_EZ z@$<)>+HC3}-IXetDw{frlT1IQ%VKNuaPhslwduE5&l)AQM`Pm!Xvm(4cTCl#b*8VT z7;!(CzHz23tOYGh(?kP}OHCvd8ifwhLrIZ7qNhm{0t9sWiPQ0vp5k+aYeF_ROB%wt z2y4Xebgy)Y))W=qn*u9>uFQ_C=5*RyxWJNxc}5;x&baQPWy@71jWSJxJrVgK7wL z8{lY-)e3SmjC`elJXJmd{~4}aXN|Q4br+MBDtZI`t@=0A_usUhss*i(?bJ-P9&Iur zmABA!WXN^&Vl@==+AcXotEs=0_o5HNg88fg_K@ppQ@BcM!r3_r`*a#IXtQ*SR*YG7 zzgC0gs6SY+wo$DKN8~lVKh6`?3`JFGEB$~TNqeX}$qn>jya#JL=|5cxy1Bm7ZknC0 z#E#jKw$o$aGfE?KSr_VMOd|7*8Dtth-2}QGJlkjb1^?|ZznazHR`4gZ*W5cpq+_|) z^b4yCwq_lssM@3+nlQE)I%Y^sm$}zy3pytZC0n?C#(K1f`~;T#&KSv$X79Q6IPq1+ zy;0GZ5KUePbJ4rOi*vaX@YY129cUQ84vD4vpv-E=4d*KHw}6iKK&Q}nek55b)Z`}c z>-YkGC#_Ge;m2Me??LP>Tw-}7R6JpH6Me-M{35=jxQb3Uu5r7`BI6G5VG5>!(hk(! z_`oO9O(YYjeFJiiYYI&B7gr4&&TVLM{v~ttsW=G*KwXf^hZ>`4S?;ZV00^B8j*>TA zXT1eij=s?U;C|`lji*|K`K)-2PP8&@pV65{fxnsyj!0t#I2YHzTpvLWXbs3?y@#VdBz{pUm?F>HqK9{;9fKIb6_?(w3<#w|4V1^C1qJ8e)U8B zsLx^}!JVDL_q5Mwp>-i;I11;ekI_dT3an{?evEyTw{Rtx7gREPFlV|$?bVF5)OzX1 zxiQ)Ui;VqJ7w2n5MhVt~VN}I;dJznM2~yt}!L2tw7*F|5A z?i;S$24OdNn6>&915G#D4Zb5wGQP9wYzQsGZlZ@q2$Ey=%Ni7y!~iKHhQS?EohqcXSoy zdtSghUdr&IKJ0Js-ZRnal7hB|O<>378BX*FIoM3%$4QutpMs5fL;|5zOJqgr6flYx zkU`IImdRveH3!&%*6cfWzRAjDB5I>(Husl4nvKKyCMiTHV^n4{`L=pt-v}z zK}mEGs<1;qq$|*8VkhYVI-_P$NvE`ju~7R)FB)q|BfhIq78qnCdxw6%FwLF((!;RM z{Kayq3p`kD#k26ky$3({iwopWaj6U^e&dL07JlGQcNOO9HL)M%>({xfbOf6%tmE(C z#23O%X6HC>yzTGkV6G-P%X&bC>J6S^6piI#p)TrycBtE=8Fz)BXSAUiTpwzO4_apP zXe98m)1;Q+!@cCc(bsw*ond5hg5U@B>;NF21`>jXslByw#&GQ{7p$Mw^2kWFxwHyb z;t%PhzLv8{ceFl)5pVkut-rxJ%cCo8zcC#78CP_j4imfU>9m-&mYW)Oy@wLQ-POyI z1>#-Jg^$8J+nkMoKGaXQvjF1MBcYx8pbx-j?xZ=5N_q-w1h(BoAvK0y^} zACjeg5KhuZ#z6EqZH9-|3uon5w2MBGyDgkj40MELD4pp%y&>M;T~Hdf0(10`_0deg zhg{UN=sE1m-U`P!O`l+V5eI2)q1(19b-5|>ahA#J2FF>=D!IZR=AB^drh|!gqTwl* zuSV+P)Y}NBTAi(>gVai^0T$Y;6w-kP;2Y}0$SGtHZ zxJS-0{AYEPl1^8ucEewEp%;P94l>$^L983!9-n6*_MaXq+}Z5B+=d^Gvq>8Nh^wxZ z6xK0A9dC|TztCR5LF-An$y-7cudzZkv}(3;q>OPujxv_2F0>|@F4nbn<_Ds$^gZj& z9mX59mvvVz&}~qn1OtD|g;qJk7{~3ED)CLkP<=rs zt*q1tyREpBBs`VfNHPDy5XF|*4Z3id#um<7E31BI3za^!9b0M;C|U*?tKq<$gVycA zWGMXby;v~NnJjHQ^kDr}Mc+oM8XLF?#v-*Hd##)UqqBh46n1NGgbqNu=E1*Po#l|v z>JqgF{p$RP?yGi8=I-d4;LTQ$lEQ2C1+O|+fIE#+`@tc40q&+2ID@z6H?dFBU@i-5 zyrO@He>@6}TlcjsMkDP3v;qcjTqjh3+mU2=U+ZaV4d3}8F+_R5H71RmUev5^m*mIBS|uc5_$xtvW+X*Ft3~bZc9+O)LmZOaU1syfptItQI<%j`0fo zmp!rf?lU%02Yic{w2z8}7TO(Tk+>GC%?45ysyV_P5-ww9xP^00tbPamSikHG9c9p) zb%I<`41F5CuFqqwOuw+o_A!qlEd}0iVg^l@J0l^Zl-dTo=M`uz|1qA>gJe4Q$<$B^ z5d17d#P(R@_UrGJT=f{Ng=7le?Do(&?&eR56QLQ1QoM_AXgBPmY_i-1KAz>Q8=M7A z`D+}M^`@4Vmcnm7otj9L+Se{?>+rkW*ga(eJq|VDMe{Dcxv9P2$wiqr34{1}(_AP- z`;#VW1kktf&Lc&Qp&3aK4e0Lc2qnm7AepYxE*{u3!3?84gPJ^2?xRhRcgZ&#e#%C9 zj%FrLXcN56-^Kb^@n(WW`2lRmU%4kw*Q*0-{h+1F=^8P1vip2jzPAy;pB6@O5$3DJt%1I=&-4M9k#}M}VU>{%Zu%!@0w?9CFfh^Ei-tP?(#ZjS|q#0*OOLCi!5pU4XDCG z$q)WJ-j@qTE##|g)heqy^@m{k_mgP2X@6O^d~0NX$6R z-@z=U=^eC>aEJz5#VFyfvJu|T1$QO0s0!P&{*lte@16x6QJKQ;Mtc!1LH)8 zXA&1qCqUQT2CQIXLTM4S37Moj8IS+-v3?zj=0~h9SnbZFITvRvrRj7vDWt>b0AMEH z8O}S<#_a^7F&vmkv=NKu(z?(?6ZVP)fekyO*Vo2FBc0=@=NRftb4-UnagzRyl{db+qpVfU)RjTt;_k9=uFmKs9+OLb_t| zK{I`M>mJJv^H^Y<#aJs|LV+WLeOxKr7RmrgH~~B})Z*sS)~dMNc0K1(#;uO)J(nog z%WwvKGXIov;Wzj$IZW$K#iq&TI{3ZYEMb-yVBF&@d6uJ4H5+12-2KvM+^f$w;p|)y z{EyXvS5|_~q@FYdSXoKYj^@a#JP#ylKKBz!q>(uPHh>2r9-QEMV+WKN!_dZh9rN%{ z+~>cwTWI=5TeqXKy}JEYalb+hj)I*9&5J)4rWS20Zc#Y4D6$|Re`7(N{PG21c@MI) za`H3BW*p9JoPIt15@H%Yq}NaPPxZ(MNQ+C&&v=>IDD75y@028D%=b==PwJB38yBAt z6nEe^m*5e9B=KzOv5e(~3!Ky0EQ^4U)1QG=O1%zy5q>AUQR!Zx9wB-VALv`Me88NN zT+ly3H$!%nDh#!Ubq%`|{#WVw;q}9Bho*%b3@qt8xP*`E6=AtiUD;hcDeqKPC=%ct zX{OAIIahN>73?m)YPUOM!DbwVo9<6EpeLwQJEaa%H=s9BVvWg8x>Oi#+G}m&ao1~| zU+JK{kZNJ^;e$(il^z`yT&g%IAt1tcidUS+D7U7rAueu6;LLEj;X23dfP0bqUXL%H z(Ix7Xm|o%s^5#FeA4J~TMr*!FFrSrrgL#-OO|WEH-Cg56YI$w+zUDg<30&#^g?`Qb zr~BRXx$N!bbr@;Bo81l}L)_vr#9CyrTEAFS%NFZcB&jV&VrYap3)y{_pvo(&ly!s^ zzd){;ORj&;vg~KsLvo^X8>2DpZhjTG%C-Dn`F?Qg`sBoC6=eEnY)mhiR-BTV`Y`#= zHQCpSnsAGapX`m5>B6W?clt@drizrQ|J{qo@RADcKbn@bOmm=;0HxRjAgw+RagO$_q$PxLzG{?RfI?PLR#Uqw@L>ttKgPZZp&Vnb0fE5UhRBm-kQ8Ic{%u0uX7sYl*_x8KeVu%ZI9!uGC?0p`Unx`wJx1K zEBpBP`v3zH%2jh6>$)0w|Hmv( zOlQR#aJqB^zb(pt+DjJ81 u<%Z|<&t95!FY8RUDVNJXUAVxO?(7aN%tc|lX^zWl zj{_yP`>gPf3-}V)E4UyeJv1sTEBsW%+KAAytZd(Mb0a%M?v1P!S-0H3Wm6;Wg$IOH z40&Jjq2Iq=wLHRII+}uoNhDV9scwLy`w;Ycp+(~h*A*Pg_s&0_cR%-WPSu>X+1}ah zv-)L?$(o&2CA)KWhnz0CNx6^n&g3sH=wCR$@M_`lB1GsGDTT8N+zPMd&(7(e8=KuZ zy;AxhG%UF!UXF{3^N(E~GyZ4GU;aPLF)XHE%!im^F}X4Kf1ZwM^>fy*iNE&8c1@U? z;+~nByTmq8`N}qmspe0vLWxHuO8UG)cJec%hxrB!39JzmAN)t?%Fwf6=fk=1?P2G_ z{tCSoGA1}I&?TTh(jBXLUUQvn`7CPmkycUmvW?8|jXuoo8R6;Y)5fR$*UEM&UC!v3 zIWhBC=BmtYnL{$0WBC4KH(cCoa#|+4o^|i<+0Uzw zPf1_NZ;9Ut-&~*P-riofJqLN*a*K59>bl%zu?z2V*gDI4-;!f)WS%TN652oowVMU& z?(itja87WnwddO=*aq5yk??WCR>^+X{>j0dflzKv(5o7a=?|`x;E#3<4fy>ymxFE< z-Kl3;?~123@(XPJ(r@S=Bv#`IUwxx&-vi5y?BqA&<9qdPY6v``dW zgg@W~d?}rQt}Mg;bBSLk^g;KBhg1eBkKR%f@Lu1L9WxQ?iH1UBF#;^xY||w38}m2w z7V~(Z?%%~9!e9JddWoc>N%wNrUrIK4S;kuXyR>v|?)J#d!@Z+BaX;xc)h*X`hU*A)(oA-} z<~q%7p8Fy9`yT&z{_Poq%=Am1Pd)PRJCj^zyNpJI%66~=b;TrpANOAt#5yAu`}#~c ztw(8o;7bD4=}L@T1$^2-M^F29+pOaCMILZK_bY5ts227wDlY11>u0a*nCjdkYjQpH zJX~liwccoQe+<9fRcI}Tg3CRtGG)H<3;y9`rw3TldeHQ=LhtVuo;E99h{0nPD7Elwv zU05qNGygJ2Sl7BtaINFE+0E*n?Kanag4=YrWv>4qrR|=1Ip)`6{A4s&BO?Zy*CF!X z&ex99j_kZvY&TU zk<+1?oXnztC~oJ&#m&-d(+~4QF!0~eAUp$3-YD#V0d7m(GTpkvd2V-q;=a*+fcsAD za3@`+TLUf0rt4_Gi4tn?Re`vBk)CijEz=9oQeU8MRr{#5)i%hzJEHzlqcQOxf)B8f zJ{-!2Y`rIX#kU({Nf&58_CtTWO*|=WHI*}OH@jHsTFP0u#(0grV`}NXDRb0(3vMBH-?b$v37*4R2;HKQJKi4dpEBvU@ zXhvEC-mnd>-eWZrz1X$XM(R)KcxI?U>MtcwT?b7cbobekqtkcYMvQ#w@am&H_sn&tDUUfh9jGwKDyN@}x&bGd;Dx}0sTyNf#qATCu<>42i;bN zAUAXvvh!BT%=stMzKWf^{93Lh`^gOzSLLvL07{hON~)3s4m@2cP*u2XPiVWLo~sAO z{4H(B&xS6mHcl>#`wq0=PjDR+UxA`+egM#dH*n@fL#b6?P@$b3i)?UBa1rOhlU{@G z&iR2KYDOQCyCj|dLeslHcM>jPj{C_~;yQwz_r)2+%V-7!x8;H4kniwPYlfFsQC6noy>AuMqPva)eL_=A1>OZ|a8L9PKovdF8qJ)|aUCL+sY-V=2&_Zm{atYSznxp1xsI`ptB!|` z7^GsIbhefQluyc8q-uq;HR!k3>1{ap7l~aZ9?v~BotNfHR%s?U(7%N{!adn&2z(66Xd)LV=dwq4n2ncmq}_^YB$=`4e;{qi_N$Rz9npv3B**i}dz* z?qp!?(Z&v(nHrO^aCxLdJ)?0dIC`hB9_(mcXd031Sf7*lcF3Gg2JU?m zPN0Q&{$GLT?}g;E2CO!63d@7t`Hu73IQS{*Li?Deap?LUq94{vu_N$1?l2;8XS&nx zv@Q3F>y494E?-t?BvcUus0ugnJ$O$fSIp&la5Xp|oXm=7A^iX?#VA?^`$}!{0%y<{ z$gKOM55)Si0cUHQ`VfD;P;Wz1QBRu-PjfKV(!~G2S^I#yB;mO!WwbViB9rfqL7`z= zO)djFu8&lrZE%-9r!VPMx`yrmzf=I;qb>ZSd!ew}jL&ibr%qp_>fMJPy-;3WPBG2{Gv8=t2oQZ!z$3v3Gd)yu+3`~k^zlYo^KVx8EAHMBR<4+r3@9D#?n zj6My?1OaRiyTt^g4sJGHqbaBbFwbq|F*0;=h=p3X9&h~u=E?SQ_nyW&ldI&=W>cCVk+33&xSj$#VoM(G5AxfK+*Ci z-H7`ZL%q2Y+&A1mZ*D4g7OJaEq@3IZN86g4!j+eUBF zYxsFG9YWh8ZR9iV;AB!C-s?A*E#^RH6$+%}HNN{t>=e|#H`rnPd;!zPD{xq!m?xa# zO>m`GgC!NA37$;Wg8{VAM&K|H(*!6S!?{T4@|tsHIX5mAPuync?!)j2wvcT80J(3p z7dX0v1(B9;(cjcpL3#HWxf2v_nBkhJ&(x__oh50XaMwoaA+#@8&pE`x@{P;jvwp)l zegp}*y$xUEo<1F^xfS3>O2+r{5<0Znz&vN*&ix?A;eUAvMPC7~eJIwtAa;)BK`Clt z$V7nZK26)e@{t1UqFuakik!w^S z{{SEEP5vgI!T#+etBt>CQ{V$HfpAoW+FHf6zhexb+wn9M<6hlhbMQTl0=xAOaHQ_6 zBeFGyprdk<9;AA*S^5?=8}5o;c%$>+Ypn`Qrl}DKHQh!d+(-thgeLj__(EV2RyG^= z;1Qb--f>q*6n# zb`9>T6-cgQm4cN| z=F9ULbPOi?6k-8}b%70mip-|ffaWFy=aa5VHulsujtC^yg4n~JOZlU-qJf!xeS zQtnhG3yl)x|WR-Z;$@%ps;`z{rb*k=Ts|{veGY4#R4+V!e@z-%M}J3Zan- zgwA`YrtAKgm;TW+*+i`1x8MmKXXG2Dv5upk66^O$>>CyIok;UJtS`{U=^fcJB^;lt zw7OKitYxWxvCdipWaHEp zN=dy;hfUQiuh5y=4H~?au3KEkxE*w>;y%djzN?IOvn{R(u5VmDToq{aHoFXR@wYy( z{%KV$l`Y-TlUmEv-1JjCB-R$ram_HZpC>mkg};Habr&>8Ey0oP*L!ODr?Cxm-|5^eW{@R(1yYr}P1bVmBnJ66n5CURao_JdqZ}!=&rZWJ%=X0okK;OK zvwrdsXcnuel`+fU^umw>WCZFHp}{Mq4tBh=eX)JAkF*c7eXvb&e6(M8bar-DM?rT{ zmg~$n5T{x;BG>hlg-3H?iwSXvk>Jis-2N5?`}xSNf_^c14cg|BZE8wf4QQ{xAa z>j>~KTg9qq0l8*5X^n7gik#!d?tR>!d!(cB@t|9jivvA#g~EPe8+OcGvOu4yy^@D3 z@8ntb369Q=Z?;>-?nUQ{^U;#9rN~k^G;c^j=ll@`T7G(7KlIgZD>z%=Qgp_4$yTNK zgLAASxA-EGfa=SsNJ@)QYbzU^y_My-UK5cF9IQW4ZaSYkk0MuYcku<=Yul{CZ}~U! zYvzUL?9KE9VzdQV(tz}aXs}hY>K0rq8s_*%xd6}e9QOM2zGH?AG`P4_o$prVguEU_!TyI*529#*N0osep5{AU6ZhAxY ziF1>~*AeOna5P8X(bA%Rh3$&g6bIS&J5MR^wfgYntQICCORbEDzxM#YvjNWo4+k$V zbv$%l*pjdsVYNaXAv+K1;#Rm$1qnn3R-rFKZWlG!agxX4?XY1qYbmR#4* z-Z%V{Lfe!(Sn*1g=qk}w{iD`aeOa|mm9Z6vl+(g@l(Ge`^J`gxcRykoEso{BV|7}g zK9?a?gvz4&0!hvHz{0o!7GLh-=D&IL#FPZr!P7*$ZZuxi25d`rQf1)qy{+1@*5 zK+Cj+t8O|BY`&-WD*rYi+rvniqR6mH$yLTx9apt|)$x_ZN}DSFE`Ouk$od-l zj%T)6wjD)Ni`Eq$D_l^pA%91H)%=9K;rYAs#^$xiDa!tknV;d4c`x%q#$Op#GY+RN zO-oG)OOw;TrMsjJORJP>O1+*crnk=Y&GgRbli3zDdGrAe|iadP~u_(!omv5wdiv4dlG#AYXSN_EY;l=rE~>_~Lp z&~Az!+8(x^u`2e?RYw?nA8OO_Rh?TpoL?o<~|vf-TQB&%OkhTW8x*+cKLE zvdui;o4qH;D>^vN*62o<19jM0ZYbYN^hA=7%_5`i*3*TezjUx`nrln!!4?-!OAFHj zaXeg~{@f$uy|xQZx{Ae5k!bofw{gzy%oZ8JX-`w8Cp#0hgg+A=#>d7TjawGiC2n<` zOPm<*h~pFXB$Z6@N%Kl;n|?KYRpz>^7dijt+Y8SZ^emiH)UD`1AuaA;yJP!R*aICR zpYl%S9LNsOZJaePeM)NA)N0A;3G)(YQk$ft*f+6*4{h@gA0{F1~KXo|XJk zf)0kql^+l_Ir?+$%k}-5ENI@X)wH&MwhL=}x>etnW19OliD*z<>q?Yc`G3L(1~>I5 zUN2m+f75yD3CCgEisH1Q;9{ZpbK&!XhK0k6&)O@>7U)QG^>=y+WJ3<;`tvDphjtP^ z@au&g;w9j@-6fmo6v`nfus7bfx~5X5cj&81<#^toFUfU=yR^1;-nq)Y%jQz-S-2@r z%Ke^sHN9urozzDubCb6wZc5yp5FLL#zG>pr#0j{v(nHcGd{zS zwmRb&Ivbm(E>DR`zLZRp+a_}<*OGoD?o0|!8kw{_v2$WhB1ya#|400o1XDtbgv`Ww z$(vFaq`k>}kn^*khwZ%l92|FhQ?%>z5^;VL0%w&n!Y7t-jm#*2ppstsbY;GB)k<3{ zbcie}Gc;mvSXxL?;5`3x-m*s>w*^+wyh~iqdvIPxG?ENTs6&C>o^+0PesYX-OmVD0 zMrIqwV8;$efg{=ZUt+~UoC4FZkGxy-VH+sSr-fJ*ikKs$#TQXJ!4A zhOv!(noeofv00O*4Vr9jlu& z*E5`E%rGY6Y-WN3D%IGHQ+G#T70Yk}b^tR-2HSszEGBuF{XK!i>;jTjfj^7642pcP zp*(uGKa2mpi94W0SW5O9pW$&>@7QV^S=c_mZtj@uu9>ycTc%z}zLHp&5Rxz~zE50- zxbboG;~K{ej&mezi{F@7G07#lSL)2v=V_PI#-s6zX3oT_8lE#Y>k3$ta#^*(pM1}p znq4h>W%ixyfb17p8?$?4*3M4J&dO?o|NcGmLB@jgg=t^YTvF$!SW;FdO--DU;2-}u zF824r-#)*;{`~7_?U?m3?_-RgbALaLdz;ui<$l`DjLNzBg%;-`y)Rc_dg1Z|tvqf4 z?L$_Cb&t4F)+6#(`7h;jD=ezer$W#2cguAyd#CiU@UNjUAzuRD2e|uB^Qq?5)HB*6 z3!PdP>v?lsQzfwvJVBkfKWH&dgfq1}YLRRQ{<+C9 zC0qFxrVG{~?t8s{`W^~ASjsOVBl3KuXHgw$IBPAd<6eJJgTRJL!;cNb2IcGJ)Gmzf zTK#mD@fCWPJry=DxI(~d?;W14+_qUyz}Ik8utQst23OKL_!L$`DbT?5!gSC)(Y(;q zONzzbmJVdJ4)PGT0yn7x{KFp_)V=gAa2y2=!sEE3Lj}ga=U)H;IYHNwsYVyJNn4@3 zat1rxY#WOX7rf6an_E5mVCJ|CXL?~;VOm&vwRC5?l<_)aUFMd|4p~B0bY{6suT1yM z%9#6lWX{3At(RFoOUO>jJfGDmYe!bc>_J&+cxSt1eadK<@n?GZv>9nnQa+`mr1Vbd zo-!*rBl%~tPfCTg zce?-I0VM;M26hRu1PMXagK|nfFPRy@m3-$f`S0=F;gjR7drk3*^!ka`=hL1SJj$YN z{-b;9|2R6w0K3k$4KMAr?J+YG+qP}nc7w)f8rx=LHcn$UY-~)DnaN<=Sp4qu{m5xg zGwJSLc=5c?eO-pl2DM?4zB5*a-_$b9#5>uRitbEnnxe{}7qt#(xHG^o?ME~quYpDI zN-;|LQtj7l(%&)0SnkHe$E{AdlsGlHR?6ViL1`V+*Q9Sr@1FiNjZSNmIzOd%a(0q8 zu|(qbggOcR6FMdQig(8^jNcZ&C9ZAU(l|W~Z^vWSS^HU9n=TvQ=-24dwAIv2lo^Ug z^l6+#Or@%$7FPkC^P`CpsN9W(i6mR1#ku^|=#=Pntoz@B3jarRdgLs|778a+og;(6 zdu#$0%ICnd02?X-?%?$BuaF~1gwBUP2fK!9gz^KE@Vhkw=L3)Y(|j8|Z{3BiF|OvW zSI&p7Mecs?A?~W~_Ab)(yR*FGl4HH2yknjt%W>JU&%WN?$X>-3EKDk_To^5wRj{bw zdO?-K9<~JgB8S3R(>c)jr*n|AxAUa)gJX*`)3Y<7IJU&NVdL17x6ffe~|!RUNtexLl#1wsL9TW5c3Z|gkn6kT6D`+fiUzXqM* z-qCuzLOdXSL8o;oW*X~M-ctA1D)e@}+wj9Uz@#=;HlH?+w|ubZVV^#2ypP%Z zB;L5sm|!Yu+F|m;zcIq%f;D0{pWO7}r&}EB~{jBLzN2>IBZZQ%DsL@=I|l z)CDAp-qFGQ98fn-g?|tDeL4Pd{#yY>a5{MR&;0HE#k`|@55c>CAMp7-{%gK$&jQbU z_eWnN-yLr?&j%;te(Wyd&Gw$~4fmb#{P0Zj4)YH5m++5)KfA4`g?p{b;q2gf=BB-0 zyh}a5ygNMsx57Kcv(&TSy#N+Kjk}7sq{jtQ5bs{^dGAqpYl1tU>3QnPb`8c8R?%Gz zpK!%}+F_IM@+TiaN(EW=>l{@qMlnph5+$D1Y7B2yW2Rnu`(E&PeAcv|!^U*Uj%k9*w2 znqm2d6X?yx)o^lOF_t#9hrQQf9A!Lc%+_Bu1ob@(%Z*tEttkONk!A4d>SE_V(y-L9 zMt{-x9+r$*`rX=DdW~+K?zkogxy3BCRZFNGIQ^N+K1F9_3QpHQD=#Sq;H10)_P#ek zV`&V=!f3GPuSt7}_Rd#2*n_fE2L<)x6_!MKCn_i;v)gD#D%S zKZAi>~r{v z1%P~%4}poH zj*(^2rov>YCn&NJGDO`2rMexv66wBo%08+|>Qb88ntM3c?uc{kQo3Hcd-`%l$rxvL zV7ynG`&oLy$(L%qWC>Xe*8A2U*4l96Za4W2KMkWmVz=oC-8XFy*wLnH>uM`&*XV9& zzoGtFSMxw~NpnZr%1}!`P+vtiTbm8zYJp~~HbGxUpQx{@pQC%D_Zf^Ta7~bf!^sRJu{e9zNgHitkrh*5` znX37!pQ_5B+iO+$CLn^3Z#(&DY#}oAa@Hkx^+?716NA}F{wezyx zMt%)Opx=Gh-Iv@KUGtn9oNi~5_kwR~pgAlIb)!zMF}i0{r5a#_G^1V2c=n>&qzM7O zdQ4MGdqZu|aS#3FL*7Ibn-_;}=uIdB&eBHn3FFdHr*D_j0 zJw$ay)m%LsKPST)FkD?(b4@c+yG`fS1<>DeSbI-XL8HWfQ!6+u4(N92Y2z*95|h)o z(=@}l!L4vuRAfP z@z-@RhS;C6#<+*Eon!aMjK9DVm_v-LSjyf~)KeC+dF)q}AH8f{HT^YQ=58Cf8af{G;@s+O@S?w7utOw1^o462 zP7mljlN?uFyX`us-^SQ#Inr(I3V*Y8vX^nxv|q7(a&@z}bf0pqa*Yq9`VzgLoHrb8 zTv~f4_hQHAf{3kc{yqBR%TU%dXK-;YREx$hOTxNk>)T+gnZ*Unbl-91n< zTvmKY1=$IjHu}ba!^O6T9R*3H%`^MbDn53VnRx5SPW}FBg=bOTscRQd6tnd#H=prk*51b&Q^|sNyHDL#ay-=D3AHdf$DT^7v10b48t%(*pO|UZTe|Svy8BAi`@>k z{>X%9@&CpRjBQ{YYHnk47>>ZHSys~>ohjv2Ta;Us9hKWu^)wH(oFU2bIA&D*(4^IA zeTwKZHf7YvxL)K$>ZxQ~qA7kv>`Kc;(-A`n!+98Qju`gqqlWpW(&kmxKP>w!@m3>N zv!6_L47c@9HQ8Wp<&wkXcyT1(i7Um=j$Y;9!{lGXld%qNilykwUIylOb)>_ez+1XA z@+@@0KgT!5GuGF_Yj@A~H}fv?zX^|v%tXERHa8mdsdFGpRs$T3^B^T6|J+QhJ>tfg<%Y|I5hDc#+Y%2v?+8k&fv*(_5vzNvWNhma-;!UdoS@eyP^< z#_93t!L*Ah>ZA{`olI{vbC{^OJhH@>=9*}CV2?`|>@CQ#m2k+;dS1bQB*btHk>Ku2 zPgg8ZZcs1O+`w8Rs9pz-%0SggRleG<-C{Uoeir*8u{NsdgNqg|o?hZf@n*%p7afq1 zmOd(_TVlod4l%DyAv_f>n1TDK#;coa_UjgyI$1eXu98!Hsb$iZrDP|yOUR4Ku(%B> z-DpjMx|!-va5>zV6Ax=k>U$aX8kd?@nm<}9TC2pYk5R;Qu=Fr*F!si2Q&ZMNk0aiR zf?xnc?FwuFlSPAEiFhGD1NG;Mtd$GJio&F5O8C!!!+XT-arzu(>;vs(ZQX1q9o1bb z&wO9^KrlEXd?WH-WIRrj9)>LN`dQI~o8;`~XyNSREa_6Y2fGh>2=r9uxNmz)cn7$K zx{kQ+ICxuAd$`b@|0RE0!Qs45zmDe($*-M%CO^S8*gnNE+x5w{&OIMbOA+TEHeW%% z!b1gF`FHa)s4!H`uUk;9uyaA(f{le8@`+ztb1UVg=Pk|on44PgGVeZ~f|a>va_49F z`l0^5C#zi6(XZm?PoFn^Y5m#oY4xXZpG$tb_v2~qGyBKjLFo*;%4AM_n)bD*wdC89 z{Y%~}p)UF;olo15tV@g}u1Kns^eTB)+TQeUMSK}2GP-7N%WRu*HvMPngyf3}6Jz=t zx~s#?c99OP@jSMd%9nEzvcG1(`?afJyJNj~S7;PJMh<}?H4qm1CkkFMl}!aXIa}FJ zJwba{|IPTHWo+!O_?bzsQjHmJi~du>Seh-{v)rk&gUWl=xq8rYy}$CC2&&HRTGpQ z+4r;>y)!L^1<_|XCpzJaz|_9bb=kSaxzM%6-3$A`#lETm0dCRfp-S*eTnWj6o&I!R zBTt_5ABWLV)PCLG+?nOr?P}u7#4dslYzbOJJ%S$te+DlFF8G>zkGyd=KX4x*T(h<^T#{KP3Jz>Nlyaa_{N^nSTXi<7q^uvEM;#{cq+Gl?wZ^eIYo0H z=bX;@k)PsNRXD{F?_2G<XmWp?cxe{TKn zbARmp`PbL8pZ9$({%P5Vqwnv1{P6Wz&IDVgzqRy9`OTD-L}WBBeyQxfa;atZl^$Ip zrl`3{uarAU7m|CW3`^;mUMq8P(LnK;rD955EcLDQ$&#_f%|&;nmPvdXvs$02{6?PV z?)oRXMi*FfTcQG9D?8&CUHHP8?7tH(4S(q_#cOt$s)0IIHAuOcJq%x#Mb%C%X-63i z*0S+^lIirOMJJW?mHt__L51uJb1KxW@V(p|oXor`+C6<65;k7r50G6yD@LJ?>tJ`N zvbCK}b7FpvU!8Ow8H}Z=f#h$ASL3c&SDQxZE5T*b0PD%(y1GW6IVVPwa5i~!TFLZ( zi~N~!tB5&$cPgD+EPj`jH2JkHRht!6!Hq0NJwX2S2Q!wf2HW3KbqsPZKiL7yH?lf1 zBUQsg1GF#BQ`~*nH5L8P&AqdcOj3rHqaSi^q(byVWKyJ2Brnu3I3mFN8hg8YcDmoV z%DBtBH@OFTL*8}%7O>BL3YCfMiCQ8RL+`@b!7bhfICbiZWLxuHiT3a7mnyrPs ztgEhbsl6CRigk9vR>dA)sQr+T_;mg2xE~D)$NQd0=~}|tF8yiAv*o^3(pOHYP_~?<#M-o^q(-q* zEh9}8O*gEYVs<3(se{sIWvWWFEwQz@v-p_I?6i#J?J?a=S2edOgETzy-qX|mXTF$i z`Z@21`{%Sg&Q{htBw~@bDH5~>V=eQ0>|NIwR%#|I>nr}ESgIY}Tam6ltdFrAiSL#= zBx6YNN2MB-X;M~Sc6sT)N|rCaAmd!>&cyXG`9`g_F}sTz1iSPFaFa=K26`r%GD+~~ zwbwq>7cq3#x6(I8&fuQ855Cpc1yq_@nRRD4W{Hzh8WxKg}Iu@afv z(g&n=OP&BD?a`PKmWigP*una>mo?~%K%$`(PVdLCw-wir^1P07))7d#)(G7X-0-Qr zcTqi9=pFB?5NLvRd}1U54=5WQ7HKCdZeND_^I%&e2kFSSuXa6mjddC^gRFF{ak!An zXoj6ZY(bgAB6#oH71nn4c67H}olP*$7~M@#-F)S_<1OQ!;2rAS?aXm^c2}}l9VX|{ zf~K~&_Av#`?A4I)NOEY9bGq%k?flQN%QnF7Do8>St6ARkocx^iIgNgH$w|%`nLR#- z$@!2yG3Rvl)t?u0?&QYi*3PeOALxz-#tE6+%GuBCCArDR+| zV$)Qj$oNcavFu`h7rR=tdS+~qGpSFK{)nFz)5W|~AE(Js)?f~i>p)QKD%xT1X^1rT zD`|;5nD~dd2JYrj`7pYUU!Vh%0yR9${!+Iy4vdLSe45fT!&5v`;%=!iWrmlYT546v z3&jT)ZImIT<)$c-9>v>Z^DNuUcZ_xQEp;O_qt%RxMGv)wsY%_3SK=xEGV(N-gZ`8e z9-CWs5A=-mHuKL2hC*`W9Y{wG@jiMBS!p18*+vVe&;c+JT~6n@8n9VC<;SD*i4vp2 zcJVm6Uk;)Re;8Q1$3rV{#yG`O+_lS5+R+oMLzR8EZJPb0eG<-^dOOmbvz*JFe%CbD zE+^}9IM&${9Y5^%?4_K7^SxW^8SFjcxA;H#D+L<{s|K02UJ<4PEbaq$v^z_Yi*K}ulw5W2NbKJInv-P*V zwr#bEwy(}zj`j8tsC#a7V+9;+3ak3|@QUz)02_E2eC#`iG@#k7b2fJzvDdTRw{>(B z*oQf%dAA}BnJHYOo3L)pC+m>7kMSE*=cjH?j!pfUSUK*CwTn4lUryT})rqL~hHeAXI{r}$T&HB0evD&|p(D)76pK9t6nx(q1VV0SQ(Zy9t*p_%Nu{=&y7RS$t z?G&@da>}&Nct(FiyHImZ)m)j4bCb(-7FCV%BN6?E?gfJ1VfJrjbL9r53Z5dT@~Enp zy1ur9{(ylu-L;;Kof`KxZhQRoxJb-ZYn;VsDr@Xy7^3ThR7?&1V_jIM!#;Hxp6rfF z1zR8Is-uxB8b+6AW-=$3O^P5G5C!NLT8jS1(dZs+N}NDKy9zyxNoTe&cj^DgCu9m( z)E~wAVgdZ4GsS@-h4h9E{LQP8|Ds#bUpW*G^4XyQ;c20Z;l|<4A$xFHaB1*B;3)RE zeS#AM?Ltq(O~L7F9*Kvy+7oUTxdCrhf1IKy0yTq{+&O zOE?Itg=V8xS~T#~cg26vr}S&#j_nXA8K@I1KsU{4{CkP;^H6r=2yBQP`uO6}*OwP< z06XLR$o0s_NE+9Xdl{((mt887Az5I4S;an*7rEF9@)PhGZFmsO=?R0zs^iXc>r04DQpG^!zM*B#bgnak5rT&14H{l zuBmW6ngi?lj!+&fb!pK|Zh3eiS0s8mv<9sC_0bpMYEqoEBve3di!=?rAvOh*qDSB$ z{}cVM!-WvY!3ja5k z6}cMeA>IV9`@Wn4`&ns_56|!^Fb2$u)QTPx^xS8pFB}n*+=u&<`vKnF3UN2#;v2w9 zt>i!RMzJjDA+_Y9gkRL5@A+u7hO7tgm?ed%H^N9e;MIcXCi*!#B-jUB){JOxae8#3 z&`|mYwpk5y@?Bw8!-d&XS)RFt_3|gW35X)+nC1!rj@DwzKFT}l`l=_`^?X#GMuono zDyX`z_Ngx`hpX1;Z-PlbNUO)r{VBVgzNAtqMk?E=j?;s|NgvIe1xdRMvx1&Ze*|$& zr}&LINIzBXz`wf+>!WruRhXef{ZPi%OyGT=~f8NJVL{R5{XwoJBkipJpQA-9ioNRiq_hz9FMN(*m; z_DDbJarjx}kQfN}M(4W@`G|euIsSTZz1ShVIMh1aJoJ+I7^oW(s8(>j%?Bl_U9=Rv zBh-+;!_5o)B|1X`B4>o-!C+*uG>Dtcn<$n$g5;hjyj|XoE}ivaEPp7vHoA@L8<-SH zebjiefKke$_@cyPK@KEK)1&*tB|uGn%oi8t(L+QJpTls` z2Ed@olKhUeSFO(qF!%Whb7%eoU_EAUW+Z0Kx zlP3{A`1dY>s;HsvD|S(P6=}$WEKp8~n5Z7?o@hWdf?6TmVY8)Kgh>H9JTpl7Uf#!= z6g&A|%2|p7QATBC_C!x9uE8)+UJ)ebQ(8qBc`p^75`qfDvojqYsLn((+UQKy<_`OlSgx$9SVEOW?fg9*t0+OoG0Tv*JU&n*FLa^a<4`!K(b5V&o(2obVlf7uyQ1nBNu0qVuJlil~&t(o`9A z+26uDoyJurM~J_~1lr8^1h0z}PthgB#>@q(t*}&Gnw~FswI_sk(H|-RiTIh!Lh%Fg zsvVHgX)G=ScW#JiqYhK2xbo^()LZVprUdd%-Iz@v!rmi})3c)`WG6E!JX_9HJmF$A ztz{`ZU4KFTDu2`}i6rF_*g0fHdC^0q$(3X~lZ4d5EpixLfop@jL`{KK`l$n4Q_XjB zdHB7`CG_N-s&;|3{6FmRP(idjs_{?69oi8@s#sb5R8k2&*&&LSV0a#+^590gM?8&| zk@m}4AzvP)Ji@{_20A>PIFB-M^u<%1Vt1>B$NkLtI(G}dF?GQdCO=JEN zjpSb8J@YyorspUJgyR(N(eaf}uP27f?UZGtb7Eh*mRy(I4c}anC{Q-~cTgjHG8BOJWgrh#al>LgNNi4 z+@bx*>Pnr^N9x4ZjqGJdlEGlEdP&5__0cYju9Hf`&|H=)LE_vu_y@998@La|O$?7y#}smOXR6|9O*)K318 zq7>a+SfIEe9gfZ-D@C&~#x{qKw7#gNI}o=NE@mM$irB2a5be)3(Qgb_mvYH(p_hs= zp#a~TS`c0V{uUc(hjDAa-zCQ?*NRH%nBoCdld1tO+SbT5`3G~&yIygYHt}zmy5bgo zDj9=S!X)M$l`oK>YWBeQJS)v)Tzq%(J$`vu2e&~vS6HpAMa>F+P~G81gZF#TKbP%I zo{5B{Fy9~DHDn}?kdUXTk8!jiWf6W7EbF8O%g5>0;1Wg&qp(0|Af*%Ign9CLavL=U zY^*(GFU3Q0CaF?PpySAgSZ8!$jKly5xo1>HIy0wflXQs+@GHb#iaco}2#BB9@A478 zzoMm_$1aq%Di;z@!xJ=5L+|8K>N(+mgj(ba%$cLvvqU_rd6;x zkw~uRGl>bBVd3haGc*k!(TLGMLET>^eN?7NvlZI#QDrj`Bn3UH?b^C|Gz(VtwOlpuQVFWI+y_359sCvZT;Y^BNmI(VTJ@Cwp9Ag% zzf!v3FQMmQLf)(y6j-aMj&!P02H62#%8%NE+(|_tca^*q-hk&R7^c+ffRc$5uYd;o zCD=b^i+d04)6{SuFihYHkU{4NM&>u}0I`LhNhHZ{h}P((9w5K>-vk}3XJjfnB>aV_ z!88=N()XAJ;8N@*(wMKj9x1Mip&4=;VGlC%G&Nan2}d3cTCXIxqv|6YYZar!dPTOc zFQZXT2(=>@lEe9tTn!>em>!%;e-|4B9Gczilz>w6KspnyLtlp7vptf*Iw~MEr&fry z!4X_2y@ZW?BX({Vq+`gpUsr6GO#E%#VD516w5oXItXzSy)%I3{EwZv>a4 zzik{}fwWPVxD46`!fK&%j?@!0WRZ)=C!?+9A>>z(5f_0^I88ZTu}glS+y(NMPUPXh zYsYPouk$4_VqKN@h!63(4T!UJx5y~@5VHZc#1PewycL5On^GIYX3%6^{vOKaM0|7&J4@A9 z(1Y>58jQ!Kkp-aPZVMz6`^i3`waP^52;WlGJo1aENBu#+QwHSc>?`m!=Fpq5CThS; zBKq*XiPB^x@&mt#F{(;Md#c{D`9eqaZf1%yNVl3<9YRMgh4CNX! zl_;g1f|?L`f-pJnLZUAh?#XxL7g(OBu#NcNMT>e$^u4T6?Tyx;y`Z)aCsL`c)e-m-kCjD{e(r z(*^2P(fdlL^gL9cJ0IG{ZB<-~Sg@OI$_yl*QKOW3QP9JseH5#7`rB*w(9QgZ4M*gZ z=yUZ&*+jmSCz7>^%ltM)l2}Q&r(P1O3&U+F`c#=Jkji;1 zzqW;6N%^`N9Na7#o`0nnuF#7nIH)#9k5l{TR}rIzm+OdsVbv5%*OgvVUxeb+7J8!K zrFYQfh&=u)-H@&osH*;pY7}nE-WHn*`1kSh*2g8UF8R=iqdj=vZ5t=>1)Va=>uY{qA9T|_?M!Ov@JTAX5=7wf_{v#@C`AT z9WON%ZYVG+N*3@4D^kb7gnb47(`Y=w%fPO#EB!{VCJD(*E69*&C!51Nyfd6blvjae z4PLiNu@$V$cCtks;QZ1qby@L1R7r0W=SME7#(?L4fhmtsny5ce+~psvAd^^qMx zmJjcyai78+B^~pEn~VLxY<>^^bXVyFy&4(wkaV6R&}%qGeJrp)oS<$VIU$rG*5V33 zCKk}y(nu+RvGC)iUG%FEO|Bt3Mt+B7?E?RnSWcA@PfE#545y&3Gt;OtF78T?a z{x(^itrzVnX_?FXHK`@x657!HVI=P<^&*$>$&q;FdsNhBXgfu3h&uKH#>H=R7sARN zU}^}nxOa*-kpglLtq>CEckpe`kyjuc>O#*(OED@Pmye1Y1rs{xk8wUGmZ-w_X7^&3 z_#d%|EC(Y4ORYl=z8JXO4zZcgPmv|8l0tMt{vBM^t@*~{IyzfCC6!ea<2X2bljUS$ zD0zTv3WwZ8u{5z;-a?n+eo(uKrd)s?D~+Rn@PZUWPmq%2eavH`ZFrdWkT{bcZ!8*Y z$=6X`#0h*=)InY0+594`QeP7PX@*43(jUpSoP+Hj{axHm|KKzrja*cg#NJ2E9v9mx zI>Wu`q&J5Bpq19cXubw1^sm_KYNJK`w3V@Gw5Ql-IB~}d8T+3 z4z~oxD7>VDnBT4JTW~8M$QH2~QH)+K{1R8GYRR>^-!&FuQ>3QGPY#PJz=_Y34@Hhh z>zKWf4PX*1mv+-Ni4WuwIBx~)b!X83%A@En#B8xYcu})pH!G#Q%pJl#R&(v;JjzEr z;A^rB-H=nl=G08SDLqfLy0QN#|0%yHGxkjL< z!}2UbNs~+;*#UPcTqZaNI#3BoW&NoPuJ zluF@mdJj2W+(TCrUXpHPw^@20C|id>!B_^SLkP2nMxH^}6^{^y6`%NPAgH{P+=^79 zfmn|&Mz)08>I4Y5)8NA1L?6LEWjZsGSCO5VG9WX2L2l;+SP==#UE(*YE{N{SKuVq_ z9md+@1g8e`>Ts;9gL5$wtWa?*MJ@ zH=b6sL!GZL>{8=d5xkGK)bVI$l0PpkwtWOz?6_!aqz2p&!+h*b-f!XvO^^n&7{%uVT2c z9aV+%;tj$>KL?|J8nHu;$9L>VRge>zR7poY6yFLr=~Uq?v4~EL#41LBP`;F1CO$#F zT94|(J5i4cO@K?GPl3hcV~&fPL4JRT-QhpPU(#Zcq$>*(`5CMV?CS4qC+QG>61AC! z(nG2REDT}#Kd}y~3txoksPvWuXZ8-SL^iRqXoXQ?7&ip)fFmfv%G zkq%0ac$v9G*U*0|f;Vwzm>0M<7vOFGDOAI3wHCILt>gxA8lQvOMm9H)?MqjR^rN0C zu13sKLwY)&#-C*+xqvIJxKF-^n=qZ`_;12q@;=Plnfx#&i%TRx1CRc$Xev7*ZRo3H zIqo;mTBnOE_?2Xu{2Q;KS|MS5fln6_n2qvop{$}axfmJW?Yxm)5M3iyVJ<|+P*UVO zxF#h~ZU4%TVhG$}HSs=DLhP&Pjpv{~JsM_&_EK?rm-tg`DlO-Ska_%PST(-MRnW(H zfG7uA5F_uUjzycp_^gjU0F`)-JU-fmjFP?h#>71Gr1Uo0o{pFHMXNAjtW>h$a|g$Z zAAz^w7H?rJs0DUsJ_%Q&znF9}!Y`yQlOS75JIRk+39yqs!o=ngbBPT6$>ZYD$Rd1i zz1Uk!q8GvSGlsq{u8EA5PLdfMSieLc)MZ_AZQ=)9rQ4z>rJ3~TC?lu9XhDIhaZ?oF zef(FRD%Hh$Tq7;xMkqeQ`!bo`$XSUN(nxhvUhug{3 zsHm7FHDV|jPgaU6gaP1U_a&=y4cI#5iEv%2H(?bT(P3h&U<8}^y6_%6_(E=j#7JGi z^&cnughA33q7qn0Etx0auDntdfkSWxsz(LVa%QV=LO#uF`TdG6(OzU3<{Ii!_h~&w z-kl1kkR|e#Vq0M?^$b?&^Flu8!Rw^A3>*^NXZpFwafQ?==@#7bMX_$1 zESI3y34>sMjPS3>+tLRZM1Kf#i1);wMF&ZN&r%Lt>H3Gx;09bHBWz+d6Sx+Y$( zPF0BJD;Cg4BI8sGW8B6x%Rb3U<5Rp*hqU#XM;& zwVoKupd}E=9ZPhI!b+}yLm-!!FV(^CPL|`SlHy1?N6uhdifg1>%-^^djj0sk42aDk zc@ExaB{hO72GWj5Kc_YcVG7J&aRk*zG84m;gTy?3xYEpZ$EWzBW@ZUkN;<+qhUw05 z*AIfV^&+Yr^P~)gN$MiDW0f)is#*(aD`^EE=`eebv=C=lu<@i^cp(O}{X*SqpFNi_fN79|4o&|Jtv2;N5D^eMC*hMY6#Y(y%kx}@f1aYk41^7E)S!Y!!u!o z!*H>Dlw3|X0^O=NxlC%t%S<9>NsTH2{=ooqPH2Vq@U*a6pxG5-SCMAVF$}kZo-Xrz z5^7&-q=t$UWEo-)Q<|7TohD@LyE7JnToei7fIqIRuooljxdVj|y2kc{A>N zNntIn$CF4SqNZBph7*~nweBXYiV)Y4YD&R8AbY6QFk`P*Rg2t_waS{{K=o(uOM0pu zdmL9x%j^|XrP9nus;)SZ>A-B{yHW?~J%S9v#5b7UQ^<>|ve9%#2WC`RMQiR)#w=}v z31zf6kUA>Vrk6;gNsiuwddw|)G5a!lgLG4R?ljgDzo-&oXT^0mUnawQ@Q$7gf6EeZ zi$}>z;oE#o=SLPR+AyBb9>q;jg01*6d6?cJJz-u*tx?C{ATCnw6cprQwqJB1_84=; zsq_y}R{O%Bxmez%SVMe)31u7HBr@u<)8ABRvRv#f60Pe-v5xj7P&z z^Mh$id?%+d>&0)d;tUqLkg(%m&sUz=OI{c2D$Wo+#C*kixGofmH^M>5#tao|&^Fm2 zUsT?T)MP*si>9&LU_=R`9_=C$r0a~8wGacCTOe*dp)#cb%vs_j)?Khm^35p%-rJ7k zCbAv%nCOBIhaK_`N=uFwbc#DjM7oFt;t6?!*qgWk`q&jBPQv*ow*Ymzzok-CAL=}; zN(yvNG#68qZnC$O1CIYxF@&8+Tb>0$XD@z_m9JwRHJF&8yeYdB_31PavicEKQQw=y ztdT}?UC4m2I`WN9l7{j77`^-(b(;2(m57!Y-It^GI8&ZNl!otQ57|$cNw;UZO4sQC zh;vQl*+{Q>L7clxPKSTF1btDiCyzz~{T<$svaV*VX9Qrdv)xmS1q3waVU(y?Fw9z&Jxx%i9hh?8U$?ypm*M%+b3cM07c)~-sh z7Pn`9$iwJq_)IS|3e+-^`HKq+TjV_0N;2@AM~SI&NpzO1!d*PCqE55{>?3bUR##@Ctg`ac7J#Vf@$D7?(kCVoO+2W zd5oCMU4dPw9={A^{m;ZAu;mm~0dauZ#4LoPyDsiuOPuGDL=UVz{}qlAOF_1E6JNnM z+z2l0bIAcyK^1x^YH8m<=oE3D+Xi$#zw`qY(MI6w^pcB6bH#LV67sB0t`nGcZ$UHL z1*T3WXlyq`7mQ{n(GhV0IRis?8Ji0P!!SH(kE9OM9HfV*MSY7^? z{0(c(HelFQ$JyZojPTVlU$h2s?}k(hMwIRFr&N(L#5!_<`%tq_VPL?HM9pgv-1LNSj?HykDq?F9mVZjLPICoPp1fHo&XUP`-MF3cnI^LAN|5j)gN#FRzm(pp$1l=yn6}rui{W z)sugLITeHNnSma=17Z(QBDdq+oI}K;yLT_VZN;emFt$!1Z-MsM8N5af_T*uVgBf5? zHYJvVN;y~#igjQ=y$IH;Pb9z-%mY2In{*A_z%;z|S8=Zwg1R>f?72n61(484>}2Z@ z)v)^1f-d?IHI_uI#qNPhF$+}G)07XU`PbA$dMf7H-PCrFESlJ@EZsf|go`EWpp~CQ@;FJ{bJU zeHbmD;@!@}&b2k(;{dM4dR(E(D(Ijlq7gaCgcAdmJVR`SO_=VQMo2& z!16>RBu<+Wb8$z05EDT+eGPKrBlL6}llzSW;_EnX*+V;Tjcrh zr^bK=8UQus%azoUfxAP@l;o)CFo6 zHJkba9n%I>bi2|@`aG`LDG*J|IL4%7pM)z zW4HJR=#W`>-=@MMvl3*isd!(`fa5B`sj>u2y3xXad@etZ9}8ylZf*?M0$l3*(QNo= zK7$A1L&wo3xPR5)5obo0MoLC{g9Y;jeBjC9wD9WiXYi6dNES)Z#Dm;c~{r=2G6;_ zwoo%wQwHApA?h;fM(T^IYTy?i2b<>!+X&q2SMaJ%W&cF->@PM3Cv;I(MxMTwvc2-E zvc0N`Do@!{wN=$gbyp>WP!OlS1zPuYRbwQR&#DO3IIy$3vfmXg6+ZM=_Q&~05Cq(R z2-MVZhrft((c_jWl*N_k!Ud!KK&)IDSrRS_4*OWRj1Pc>_P;#mzy6N+wZz}TKil`f z_ZB_>q%Ypv*rWC|b!WJGJO6dmairUK7B(n+mOm|T270b9{Hl^)xu8oysX~R_@2KSd z=1UDJ;U)h zJ?jOdR<}oWn<2?bVnfaysu9TaUBf37wPAK~gV}K*R4!6H(j{^k%nu#c6K3Ib)T*ipKhX#M5dB9_#h;*J zhCzUi=l_HK_hj_<$VkvW(t_vwVefNf?L)5Z&NB|q_N%aM;i1AUh4Tsq70UR#?}d9D zUEB{n3jdejrpQL5(T0$Zm=j8~_Nrl=sh6cpY`OUD2^EsEl9#3222~|FxmMDtgfj6f zV$-ZOOlm_*?OfF)#SV0zRS-{dN00-Y88Cv4U)1{qZjeWAgJ+CK@{|HSptD~Q41_jC zCUBL6o#IuHRB=KGmz1A9pz5Ojt~P7)wN71E!|%qXrfa4=(^Yev<#$UXbetMu>RF3f zFIX;_pQ6XKk8zQHnf8^sl&T6_6E>h+@LNlWNBCmsbo~b=gc89ofy}@O{~Uj|uao~T ze=q+Ke*-*qfxx@qozP^wW!)oRB7a0DZ~|9CSitA-qlE;}-D|)AkjtOtuW%2eEu#P8 zeoR9OKF|Nk+uM_XXT)aL*iIH~$a|F6A}{&ZnqM(_mb@Ro4&=|Y^>DOxAMi#3FT#Jq zjn|3FRV>y#GAuM3V>ZMkCtgl&o_4Cpn9Ng|shKx3GBYL?8IfKrEi1V};-I+OmMzBZ z+Wo2sJcE)nnI8~|3C{4H2WjK5)8de9>yUD&Z%eSBv5#^b-tvqt>_7TTlgda8fm#a>72sp;VJCW-@KD{mDo0!zpH(AQw);Qhepz?eX@z>q*X;5z+HI05$6Xjq&^ zh0=nZ{m;Gc-8Y;GNN3b7_%DB1-r-;Duim*SxjS;o{Bq>oE;w!5?&#$12!GM#@IL;a ze2y-q?4oI_k2N*29*Rpz+Ltmut$X^{^gczd6-mjsTI6K930JXZ;_SG))={Qq`q9WT zOr}@JEzze~EmYwD;9cq-+H)MZrP*)ZHrLr5O;>s64^SMx=oc}eg`XhE7nQVz$*V;xW!-M21dVyCxF5D z(C70cx{o=3A;VD5cBgQ2!T$WydExw~g;i}o?X=VF>fxE{`w*-Zy)FzPX3+ImNhNC2 zjnge9;>g4<$y-xartVDJn?AotT1NSd8bw~EJx;9P7*x!()cgECcLj0#*%=v9`0J2ACuD14TGNQ1S<&RU>^uk-%vGLyU|? zJl|apozERz9J}qqz_C(0V_hlkex9q|a(*`WIlPJ+D2^ba)C$FE{fd~vIEJYQ>bmEBxew+OLK``)OyClR#mIDS-Sg%QKmWO zr50sOax4)yC~iyKrZ{)(%-EMPEn)^)6_$gh{|sMsPK{61UfEW$k-kIDk$b>6{5PB> zs&I1X-@sk}THg(C3uGymgVh&G`sn6v>6-7X;OOl* z>A3Gma{8T>+=-rn-kSc=!TDh!dO&C)?<6DiHnxE}L3hc}#oW;PBW4ovEa{2)iAR&- zk_(c$B+W_uBY}vo9DCN9Z9Z?z)Nj#*lsd&qRAp#s5nneN6Rr^K0h&Z4)Qlsy$^g_1Fb`H_+<1re;=Lo)v4W#UP-Bg8Ua-2I;L~x=GH?oD`R_rx-~E^ z6uU9DTkP?eH()uvGyOE&)Crne>RzxR)~AbNC%;5k1A1%sus?VkOcc`h8J{@8y~_RB zz0Do#v3MGIjNTjGI{tEj!$E&&PdGkWgxikug@|}xK1WPKmF5LCh`vmBW{xoP;eFc* z=S3|r`Of1!uOXb4ZTOqfDUqh(8^Py+BVcfi@xBFDcbDg`C(-*KvU278Ljy^n4`Eld zkpCbC5Z-IdtY_oY!?oM=!;GYPw&k_8Tx^TDlJT?S55zZ(zZ@sWX2u#}Kk98BX6&Xf ztu?5>vagwEFh;MGh6+P5GiQb!!B>F?aN)G~ef3`Re(*N(J@@_QZxvV*d>2wi&P8jY zuJ9E#6cXkXwep2(m!^|WWjJW8Y+h`+YaJK!A;uh=9&3sH6tgepsr8Aa+5ge>72s_n zUDsyIl9|KM6jNs2GNepfW@ct?x6DnMVav?8g_I#pnl#MlIF92uX8w-zzyC9jWmzU^ zG;`<9opaBT91|G?t@($^GQz_tf+l-?+$p^)&#}>7Xsuk-@wEF4QJxpb$@s?0z&qO)qJ8!whx%YTdJWIUYz~#RVs)k3wXZIMh z$ys2;3iQOdh2Rq}w12SWScTSbOC9qI(wlz#4^rxC^Y^- z^+JZ%JLrMRBZXi*TsGKxL2z;lZ5e{L8X3`1CHm zhaR8nt5fPYVzXLCnCqEzQ1%&HwZAArl$ zVOCjBS-;xu+h;%*=B9HYW+F|YW^vFx)y;K3bZv4KI{$U90cS@Kh(seM z#6hCE!ukRQ-$tI}g%Wc(pTU!7pkJa*@=f)&^(cY&-{K^lHyooK3mo$u#~d{9zdK=D zDCBEYfQ5YyeJ$fT)|K_ha6ZcWLw4nN1)IrY(JApt$va73=?>{`=_cuT=?dv{iA~%~ z+*s67n9u)6eL@bQnLC0bWnY88Qcm0ETLV?GH{iRS2k-fy?I`?YhGnTa*%WHhRQIfI zUQJcMFw+zFm+}={h;xPYh zd1i@+D%mz`o>c=*`QP>mM-SINL{qGYt83}sn1|TQxXpMkNCm&4V7>5`h>~PU-bowF z`pSZ2PHB>~sq~&CPy85d%E@0x>3F?}Bu)$Vb*7I#ohJ9a^qhBJMCOaqInCk1T}ZTN z+M7BKL0@W_bBODwTMlgtF4i7jLS4Yin#0W|NOC%LkzXR12)?0L@QHUzPf8M~WPlBv|mN6SblDQ+ob^5c28xNX_dKtzxjv6j0k9sk*{*xp(bk$?BM zCB(AcQfj#fg)<>q>oagXy>s71^llD)FjK=yV>jWx!GkK z%!5UlgoO~=maFnP~=&yK^BP|R`>&Y+_GwB zb+~bA)m}pb!`#XgeWCt7I7qJP=jg}jW%}CsCi;!~ANnu)R^TgXT-l{^LgmHE4V4Qk z|EoMu8B@sthsowjd*xn3!zz`rFKc>A=1|~8%*4-?QEOMx(2N@ z1inxY%YKW%dc%6kcFLaOsBmVx7J7#GVlnRALiW-I&K)k39LeVhLPh(;6QvVm?c{Rh z2W7B2Mok1<3b3dTs{5)dR4bH`ied7X(oy2`82<{$&crje9XjZ%eH_m`*L%lu8)1wo=o36%AvEz{Kq?KWvX=#Ao zJIC_aVzF$n9*06(lI^AaBRFf8KwWyVH-NU3p3UeAevRXVmA8u@Dr_olF1;nYqE+3ReKHszRSSqC~&T&X|>Rxo#Qo00eU*+QMTnJi5Hk7BP< zqDoTzqw23(rL3b!mbH<#5Pua;=Kr8R5Vw(czl%`|O5RrFzm0G`bFOu?aV)W42Xp;4 z>t*XOYj?WKhYqlJ_N!lK!RR zOOI$KfUV_W+2yin+U8oNZfeD1{ZNC-7-#BinQq(W+~he8H0OQxeBKvoi?B#MNj4T< z+%^@d{;BQ}@UMEC`hjX7>giQ62A;l;j-lPF*<0G8_+0VyqIrc)ihdNnDLhvAt|+iLu4r`0 zgOawT;hL_R0?l>J42`X{M6*h}N_$Xurt*1JsA;&hzq5(=K7AkSU&6v~BfcOTr+OM_ z3jP?{H|%JbBP=&`Mo3)Hp1>yR<%$8avEt3bV^j=L%|6F$OgDp1RpT~;7jcb~25Yhv zaoGy12R@$F+#4h1A=3)PC`+oFn<~*3bF41=WmiKWQ*@jwWFZugPb!lFJBQv4{~kHF z_9*as{H$Y*I$x__gfC=n&~Ft>9*L-gfQ;g9Vh1y)V|7LjHJd(&kEeJ}dV!BY+?wI&bh&VG;faFn1>C}3g|&)$7xyozQyQz;Q5Ih| zS-VYJOM9g3vt~f)fYL)6g+>8RsoVN%hF#TXZP#66X@yJ=ccRcPxuqB%Xb)W!p@|Hu zeKu-k)YDqpa7Ng};K>1UDn#B>pCeJ6~gjTej((`$37b**EF=RSQWXE^mmWK}i@>JZwz*6unf^*YsW z8QnH|U;VB1>eXITYj)VSAdWgu)>OQJN+uH72}~w9LLz`xT!Xa)13d!TTz47Yv4*k- zH8dKQeUn3FU2A$;MX%)R29+h03QA&&?1ia?or^9O9V$Lp5?lI6bGs}-JF`3v5nHM5 zLir2rZKx;fils%Vh3ulXP_OQ+8CCw5eqVK^eUjJA+(Oud+vQ(XgM-h8w~Ojk`+J?f zb@_EdYga}@houA+sw!k!(KmhyuQ{P*k70FTIT59$mK0$xO`OMcyZYo~}Cq;~ps;=8PdThh% z(X!~%4c68@Q0rvalHexlt+K7+(Sj?am%EJ9ncW@S+~1%i)f-vIo7unFkJzJ-#eR;N zj6B46_hZKa>v~g$A-y8Md|}y`(%~fmCBKSK7e|!1Oa9RWmQ|L$DnG5W=tfl-Dt3ZB zVq1A-S&U|OasR?r`M&Ig%tKj?v#(@{@{SiDFUzX9VO|Zi^;6=Fu!CYnK!XrX__$i1 zqV!Q;qt-+o2xo>q3W!kdm->WJh)EW4POulVJFo+=PB4aB1Qx?(yy3({tQkO)mQmz$ zcpf+xTaDEk!?OyuwgS9rnZ=@#C8cY%$16m}W!6KkGI}z1i6CAoP!9;67B;um?%Ip$ zsOmE6j;nLH)`*CXVfjIndawM5^qc6opb@_})s~tMy`?SWJ#qv2jdz3h6KnuV&Kg!W zV+pdSUb})F{jCE{zp8py4y_no?k*cu_D++Cw(lu>gUFY;{DLm3!lk>Vi`7}d5ENfl zskvWDD>+egwxDc<}7N0qTK@jbKSyBrK!~pojGboB)@2*Wi!;!{-Rk2-^tj@z0WPi89t( zn$6wU8DhO`bX8ugs9(NYQ&^f@dJf7gleG^k>Q_xS4|Oc}wqb4Mg$QR#Pb%vNgatki zc85$2F@`WgpNDJ+UKZRpXno-8fXAvk%E^ky@^ezLbe`0UsO@0!N0D4uM^Hh<@Xm6d zv)GJ!z7Os%&f#_yw6sLfYnTcJlRi}eRr?H!;fLz<=k@*Ie+B4Qf@izCPL1)mNFy!X zRBR|(Q8>DwIR8jKKmS_Z@4VJ|BXc8izvSiRM;2C;^eC4b`kD$IM`)`!H0p=&qil-8 zqBN+qYAV30YNcMF9H{(PA(YpX)sj|93WY<3`GR-+75s_(Oo4?zRq#Y`PH>BVn_S8Z zu&GR)t8VBS73a#i<*mz4mWNaf)mK$U8%LYRBDNv-?qV3(J$c^+SFqAh zEWf1urJSsqq+Y7-uD+n262J&-8yFqfI4Cq|dSIgfB4ClKiDIri6A>E@I8m2?y>bY_ z;CzJg$q@LgznsS$MsSz5wWfo&=br6oVoNvQGiJh5no~Ky(pkCPkYnIgA2+EiEc+=Z!&{$jVU-hd{xs1L z$q+@RVxaPZdQU)qbt~1_z@_R^Wk>aPwM6w!k)*tWXropdCP@-a6uuU`0yk_Maw6g7 zF6X#eTbZkYR(gk7IoJ>EBW*MbVP0&yQQg3}vg&13q@hD)y8bbI`W_V$y|&_Tg{I=X z{<2=KPtsq~yDDa274DtxZTVR+|Ig62(>?zY2BEm9FtVFD!f%I>1txpgKNH&UQ3!4g$BSy3Zyr#8yMcmiWJWc1c;gH-noCHn^ z=Ow2Rr!MBlF1iW%YPDQE2Vsjhzc)3k-f0|TJdG&TRxn5LEpgWS_J*#n;L+U8ywCB$ z^Z9}qLxlXNB1|2jo}eBb5EBrr&Q{G+A5jJ>XDimpAA{?^tz?TR4|-g!`Q4~R3as6}V+XLVGT30c=nCBLK``=uw8z-P!KF6SVRy85seq}O3%uJY)+^38 z;wx$7zY`{iMo1)x6?6x4EnWUu`T-pEUU8y0S{x>-6jTVl^1o5(lo*O`iM$!SeS`+< z#P>Klc3U>`7@&SY(Vf0>Z$Hm+_aRqZX9veo`*Zlu6R?^w9vn$hi^V(`J?);Q9_AMh zti5fqwwtzE_C9tklnA~-1J`EFw?uFq=xB)#!F;sU;8QF~IP;*_wn8r{6BQcUO70eI{<{V}za|Y`xs|P!WJrZ@%mfIa8 z{a7x^{m3~4PMv$~Ww=s4N;#{H)sj6AO2MC@B`xN3k=N0EBlw$to)Z)am z&*Oa=I|sEgo7I99&hm3RBr#txKQb>f|7Bi>4*XB#x{w3K`L|qM&@`jw%_qcb=$y8 z69tBvA)c8~9>41O=<#@DP`B)e48dt=L#v?vxe6I+J%M)eL4)@sly-Z27(hv#gKlF7 zcNlczjjnu`7SU6`#Gde-jI++>D8pGPhddD)Lowj86VNYaFVozmHVUI_jieQVtSCt7b?*!PZ#z7f5 zj3t4#OFHut^C5CJ_kt&8A#*&l2ehu`OcRh$cX7vtF~S*IDCDlDcLSp51<co)ljHsb#qIsj6rKug;V$l$VEJi)Ya9q|j4^10Id()gd1L0?3|4o)Mmwz^>6f zdenG4^ow7+<52&J?hIsk5T06|Zuq_Z&^Rwa-FHK6JwSFqm~WJC8+2SE{c^#_W;M`+8z?$w4C zhK#5@-%p@(&Vf;6g>RN`0`ACgu&hYUqJJ<0MfNFRv99&+8T^{Rb$_uqZ14^90dvqtP=vbNOac&3G@g2 zc;CHmfDn8PEzJ|&y+9hS^v?Irfl~Gu^!ve><$th%4MpI}2RebLb;~!Bu}pNdzNFBJ5uR@ZRr$v%OBc zg3lbJ?FSNTAryYc0#)0d)&eLvDHKV|f%N(cm6)?oWM1oA235nkzOm3I9*k>^K^ctw zG0;k$3*6ZzurXc689W09D;e1dCgcrBXhA@Fb;sEZhmP_lC_|k<9{fGpU7$q&L)&{! zdq}&6|33lz-wNQ@hSIv?-1#)KFBAU@{4w;JC!>}dqu!XlJnvWFz|P~!Hv=C!70AXI zs9g3!?R3L-uy+vZW&!$TtoH)yG98F97V5t}?&ucW-B-Y0>3tGl;QHXsFR3|~8_?eU zgjz|1aur-OC`lWkT~$WQLTx3Zmf~=Jk8sAvfh60A+2wrNEZPLx2wDuzvpedk8Lb_9 zPa|4=)K@t0!*zh_li?x7HLy^)$p7#H-AboI0?w|i&-l(~w7g0|aQ7+NW(@x>} zI9SZ~quvhTclY6O5ZhQh_M`6|$950y`5K@;S0Dpo5o&fGZ3f!xIFzB_2k3_yZi)J> zk2(!Qee$4y?)6pr3Nade^u32B`9rj}BdCwXXd#1fmA%kc>*MOBJ`NZK^sp5rcw~E1 zz`5}oy?i{?l>192i^^RLr?RM@Hb=&C!3mqxX;X&4evjjrvBu81BM# z+>K{I6u$Dsp}eiB!6ejTlJ6VNCmFpY2Q465Y)5?^tm&QmQD;X{qbKlu3iTL^ zqq~8E-HLX(61C^AyCL|67+@Zc(j?3D5LOt3Xaag`*ZP_ht}t>n{~bo=<(}d6*j^i z@5b*OK`RBb6;!sbqTEI6eS-gbi~oF!|4u+n{itc(DX6VXlx(#5V$_ih?a+zd!A7kF zpiaV}G~NWI8Lq@%AERNDCZaZ0!$R#qt2u;gKaW;+4wmZyTI&;7k`E|f(32DJ@b4v} z|E5BV-rt53(Z7G9MSsHQzJP%=4p!$0diB3Gzi}G1>Ti|XY5$-#E<%}y;%9yw37a(( z8CWAxr~d5_^!uUMA6TF0R$1fTJrx&N9U;Ttz_ z{kLkqeHy=XuBM$FLeJWVe>s9@e=G6#x9w;t`)lgX-)8)6%HOI^pfw#wZSKc+kJPlK zjX1ZXICdNs*?;T=%0bvkf1Mx2Z|=tSXwCWi|6x;2+uMleefa$Tnzp$e-#U-i{=U5) z-}SGzAH@~^i=#(tuG{|~{@=eyrR`obf*^^98S8;m99E z_DXO^OJI#OHP+gQZBC8NF2J4+dQN7V9rns^sfzIFT$Bvl_tct48n$V$=$~=_{k8cN zHSD)sSMa!tGdzTIJ%p?BTls~wxwvnWaZg8puXZRNebA@c!Oqu(J(8jDX)d%`YQx{x}Q$aTfaGGL(hrk(T`V_cr7NN!ONg5UJ4&38+qVau*)gf`w3s@6RdOsvOj*-Jg2}`XW;!G zIO0F{8~b@Ul8vn%zf)tgZ>wub3U^wc1CTe*a%FG(ux46dkt;Kd(&DE~O-Sk_) zz3^3HVf~Jye*EPO_Wk$$GV01-^JmZwj-%EN;&Vr8u=58{b9?dKtu-xU6D;X!Jl5ft z7yFiA&u^EP;6Eqh|NRzt0q*zQntz>+I-ZFm6Y%g`=TS8U*_$=3$#28||NCSAXVa(P zn+woN7UOw2esvXE$rk+g#+v`}p9wOGaDEqXuK%KK+{Yb%fHrUwd(Y4^-k@E-M;rf! zvG*&=FN}Oi7>l#f@{3@_i)#9T9X^e=X2jE@Zk!M?LglDVhj{ z4_?SH6eTRbzxX{T{|HbUen}*b*RAoZ0%*Z_6#`G74qivr9IK1{NPI3BpKgi~qY*~Y zhBYHcL-f)17y(-&SHa(FJK&Xn+X>s2INH5tl2qOO|w++-H2|Ek>(2SpE-q=Mx@&Z}|~^{SmCyE&TpP z{G0#z6s+kfoR5E0jK!I4g_Q*(3(j&qJh~b1C8oog&Bfk)?2W@%Itq3TY!kTV;jnD} z@ipduzDIYAX#G%6|F3kf@o-yWF3=9%iN7v8p(Z=wsNYxViP9CtU%US29ypGiRMc=g zY+K{F-wW}-*9CiB@E!k=E~xSDcdklPWXiXjO*LskOU?T&@Vova{xfTed+qn1YNLdsUHI*D2=?pLY{T$e`+w~#sHUyS;AQa94uv&a zf9n(BAw_HSAN70g6#O#3ZwBc$ybeIiRMZp@^JCwCoWgGq@T%zW?lJktH~oc;2`XNeF!B~!Xz+vWE#FQp5moTNwUd*}R)ZE032BT9fGlm((G$7+lgEgZVL=a-Z zC2|Sebaxro5P_-LU);0Fj0>%qn0I+asP+#iW5P)P#(h=FIj? zE%OkoIp-_UnYh4R%W2G>!upNur5t)8GRo@E3AzGVU^D2s^byE)+KKH``hS?0e?|+9 z1T!OpmW}b`J*?F}*us;@=UNZXVk5kVIGPLNIv;w&@9@n4`d{>0^nS=s3If-|Qm{ej zkR{a*SzHV0Ymkk5hn`1YPwz<|i_cZl#Pn~70dxRM?GsoGiEkMrme8}>vKMje?1wB1 zy_(^mkER8BvYlR+6Dk3jjzx}B&KT!4XA0QA`Xj?A!R}kIUNb^FYaA#yE`58jvJm2$R`w`y>fj2V&0MOo#a(}+rxr>MV{jiPpWsi zPwQ=fyr{>R?e|3N+2L8^Cfsv@b03P5>{f#N=c4PDOY9kq`M%tH1W}XQ$YhK} zwq5}uCtF*yTuCzGEYy>Uxl#*@13E-s)m_7%kw(yzw@Kua@f_jXZ^&z8qRKTH7Po&{#wfIP`J!#`FrD zv)i-Nn-2CAr*Al|0X>A?oIxRu{2r0hf9P@WO7-wj>(E|fEM{TuYW7Od+b1IW5<`1P z_kv9#2od2itn28zbg(p?r-#r6dru$^H^6<{{m@h85&ME+@eiT>uV%hshB7^j10xfg6P$vIseE#}Q3sGMg}>8UN95BX@c^vgk}`5vSY} zJzuOHic+3x4AEiMW?iB{wb_Cn0&FyfBm5#jp} z`$s(IJ^l#o6+~b6fe-m2`qd*a9(M4QxG%ZKx!1VQy6K)No`;@29s{y$(tuFy?#^+A zfj9bwqqCFd+U|-$W-Q%3&-E7Z@k8#1?z(Od*g>13&K|lc&t3GZm(GUBtQ+oJ3qEhD z>!-8G*&Tf8J>18^p4-srvoEoKxAPsl9odeF4!eClbS|d@U5Q*epcfw6rr3YlLmez9 z(>d7D)iK-A!CCH1a;|a~Iv&Dy{O#bvdi-+S!Wx;=8fLp=`(bMVOq0d3#q!QF(s~V9 zdC9g>wkqpg+XoNaC(Xvm9Va#>Cp^1th}0; z$2f+3(WmsCjH^sGiwzqd3{G(i>m)Q4UUALbQtmFI8!rN@c)NJixdE1xltCD-Asv1oJHOD>IR~ly#5AXX}87f5583XMQmUGRH8F zGcPdfF<)Y>f`;$3!?6#_S5!%aW_8$eSO2R z&7pO)aU(_JQ^QdzGz|?W}dA^}U4;+^fU<#C!#q-+kuB&_b@V ztg@P|LAFh{Ao~sb3f#kW&Iit~POo#li*WPYo!njBXI-tZKGVYW$$8g#3Od_a&L(K5 zpHYUv`Um+s(N=?bu_0EH_OqK|B_tbbQroe9lE-<0x;V!^gEgWHEIX?cdmwu>`vkiq z=PDKme0LYnZ9-~oRrpUEfrh150bUn+}g#IMhv!LNk=byGo%KqP!A zycdY2+t1vxCLkf3yFTjLt+r`1Me!7 zPCr14^bdKSyh&~*Taur7{dsz{H!XKMx0-Vp3Jo02f9wLRZi+brI6FBkZVZ=2>;R*F zS6)}17P@sCp+t3oJVOqEcI#W-d){hZ6t9?wB}Bws?lGw0{>6>wtmO3J{DmCUOYCjP zLoHxkXZ2&%#%f9%C?|)pCbITomC(W3%B~9ywK2FC&AI)N_d1z7lk0}!;!LbP=dxR{ znfT^jmWWk}*4mq?fR|-rykitH;*c|R4C^^x8Gjfbu?Ay92Dbrg6t%%883wQOI%3YH zzDUeN2;{xyd*=Za1a@&St?JuZiYJqR&a&;6jou{BR95- z>!VZbigji?WLWnb=IDmCKanHfzT9rNJ+>XO8DV>-Kq>74a)R4Iourk`Wc}MZ*}BiV z+}gmp3h363c&&w9`hZel2}E|;SnC$+AnR)DMLeRc&8?AEzV)1?E)btAtI6`m^4apm z@(6hg6_&4-LwKEHDY5K=Zf=Zigzb$@ir#V!x?EOAN2o64L0@``E7qlVb;lazX!r{u zSaF-`If6W7l{XY4?n&=(%t*6+6y^jLL~c~zQAvjXW}>%5&ecRlTlk_I8AkYcnOM)s zLH5;oJio-6&qaDVGP4AXX7Ej8fv)<>;J`)a6?&vXQ85Zw-622-4b>)>%`97}F_#`kF6V}lImHj5 zt*5>ltc=XBlUgqz zIzSY2kSu*|ac1zT!6UrjboV z%b)?bC1_%3Wq3r?<~m>N#W#2q?TH=}{h@x7daddfMQx637``QVeLz#?dTEKM1r%65 zaT`H7tPVZX7wugNG{_^T(0R#T$KJ_)$kxb~jI86SmY?PuW}2m=CDEK_E;i3V=I3$q zM{^%!-I>kb%xhpbFPTS~3G-J|J)nHA8ogEjR2?(a2HGi1w_IDG>0jCaD817Bhq>dj z>t$X{+nn+=dCu>z-}G@aZ(~D)W6>j`V`|TctQ~eS;Jc!o>?-st=MocH z9(r@%J@<0&Vc!K8>73!1YLYM}Y6Wxjb))y7J;evbBMsirilw4~TvcqhM4!QX{P z3lO9us{pXA-$k@Io4^5(kbB?HqYLvRPzsWQWMS$h{G&aA(-1 zklg`RWo^ZGX|QCKXq4~+|1@&z(|K*km%Jam=HzzXXkHnw3AvJ#!QLI>5xk~EEutfL z5ob5FkO~>&z%H@@+!rNu511{i;Kd2#3`XAmSfFKA2`Sjxwc=$Go%ogLo-jc0iCj+H z=P1CHei1y@_1%x4yR1bX?s(|djWKBT4=dOeJ9O{MCus>_0q+-&Dq2-Ix3GJ`Klx|# zcH}P0k!SbIK9|)gt6}E(jL{j3(o51-W&cyKyl8=Tso}n5hP{{fE>|kfQS1%~j$Br^ zPW|l-spi5~_ggYsOmF6HvcJL8+F?=aA~?Z$%7(HmVOy$@^ABq#(@V?3e4>fFw=3RL z=KT#nvH>zuWyoUd?cU~T?VjLlI{#`?aKO0kMj-mR6FDC0_$5-fpJdN4ro?B(zn(B z1AN%L^26olfZ|`Jb(FO&>#FHh8d3VDt%8^|V;d39_~5v2wP0NWl2uYvGF` z280_hBHo%hZKWnt`A{dHg|e3lB{PX>+%M3q zZ;Jof3s$3bu8)oh_5rqa*3K56X^3eWRN`gEC}Vt8vSDo{U(c@iRK8i8R<=&FsANsi z{({`RDY;*=r)G`KT$8>$wO7j8KU?BnOF_6nxhE;32DDXzarw>{T%Vb z`%{@T<4qoaJyhW*#&+@Pu?zKBqtb=kpawoVBbCr?$IwEj!SF%Z84i>LDhJK@kZ5l!z1Lg zY&4v&YF(XbdTn`cbJ;h#UPCKo9H&25$)k}o$tBbg{&&F}{#pTUm?fw=)_rI+QL}Xj4*n-H{Lzn)!aG8 zvEBZct=RI9Ikr05$T2RfQXAUo!*u<%n>6~80$@P;($=8#8$rFVtS9v>}DOLmUlq*(g>Y%y7l@T`UPF0BT%u z(Cs?Fox+pyHb7k_idl;`$W!9{%em2Mb$IRBw&TDyHFfNBwe+UaZZW5_k8+yv+nK+Gn!s zajtWh5Eps(c%L!PIm5d~&LxLai}*+Qzo_AqnHnwBi(ZR6NM*8VvMOn*tf!)-@{4k> z(yo{+?~Et+SgJxf*>9W6MJ-#`C+-toN4xx;eKWoKsn$&zLd&UR&4 zvP_xvGIwPZroT=vPV1Hyo09jZYqC16Bx_}!q3n=hoVnb3-;>5(OMMo{-vue>hd*>sH^BR$#v*2r4>Gq_zr1vlD8*q>x}7132tC^{rg5l1WNwgOvlf2;4Kkkm&U?;VNL}Z55p3kw;x|Q} z(j+RF-$RhgS3q-gAD_-I<;M#rh<77+*~o27#!GsWqm`J(#* z8vhW<=l$YTvWBpxF%L87^yNN@_n`ZYYn>~`#f0alvqf0VP}MtbvKl*8J=0IqE!EmI za?Pv~qWEN?J^w{+hn!21Pe|*Wm zlaD2ZC%4MT&K_PkRe!_$+&&oU2aSb&W%bodL-kQ*welk8)EQcL4U|y3g&hnUtLP?( zrg~Aei9Kuv5lXs%3S7voB-WE-C>!(=uL&Cn7K*M4gQ!(R3HuiFu8-j$jAI9KOmnjUm*9&tK`e&0rJVR3Q4IrLG)cPfPbDWA`-ai&`^m5I(D`1nAe9f zzm+q@5o}*;8x6*ULFRs@nCi>MW>wECkLtJRHfuFyI?b8V?Inwf8x-v=D9(G9+ak9j z`)c-&?B`i2ne8&0Xa3CCmeD%nMta}$W@)!mMXAP=>_2n<=#qD)adXz>|I*H>CTueI z1;$iLD_*Z?5jZkj6nP=6O+;a>dl4-{VnRL#&Q%x$_jvosyTlZ3CRa>pcwe~+iV!Rk zeiCh$go796rudNLru3KaFy-XBSQ}|0-2+?`9Zi9Hra6KgD=(DC zU4V>cxyfU^V92XTL1xf>*pQ5(NrkWTpXEw&re*)iqGwsN%Cnkf1!P^$?4B8%S(uTO zF+L+VttfR`>hhF($qSR4CtXVVnjVs!T)1EV!gSmIofeJUAe&659vH%p{2AULba8l4 zctXHh^%rG|bf=&#H;}!Q{eqpv{>J;te?YwyT#|g2s^xh~xALw^t{fvvkUbZtQct+U znQm`AC(ypgjZe0AwI6V{MubyCBRL*2TEvm|k{?j!DO;qKPyGQi9j%6dg?qdEy5 zi`&WDC?6^RQ@oKMm8FU63g1wjdFR-B8HefbeUH6ez`2>v+QyCN4HwK6*OESy1uIJB zrScgvmSl-2PS6E9;cGcD%#(B@?LJ-19L2J5*7G)kF(Xy%lI>IYlv%14YEm^^5iK7j zy(XR}JkDQ2Jwco>3}fgrG5~oW+l0HtX7Oi9Kj{p~1u;#uga4A3$Jxp}PZM~HoE}?q z>k@NoQ%1E7Y!3@e2Td$<2lI5(p6XA==~Zt1ah+0IQW8?c$dAme%vzo?5msVIYS+}) zsVmZaX=~Cir&F2xGWKLF$f%w1E3JO&oD^O%BPlBJc*^jsjNAj-HC1oT_Z^Su%{c{R zC-HviM#U|KO4>^NOd2VYP>qBSz)!UonEzp%gS;3~x!A7g9I#m#6?hMPH`~?!DUZo^ ziR+18^V)I)IFslTd@^rKkI(J&zNDwJrto%>3;403KZ3i0%c2PJC1Ed&M?TIT_HI@u zaC;17PGZhu9bh-&Tqj0Tae`(NhSUYU?yi!v5kL~nKswDy|&j(YES1_7aa6cPDS?`kkBXL|(TajLLeEOb-3;e171 z?Y7c=#oG%z6pYS4m^TaUcy0cT0;+Ib!PJ6P`Bk|MbDWvWGD=f9DdUpIC%O}M{mPHm z|GJc*O{h+MlJq9slYh75MAcaPbC=zdz{p`TSWO7%!4dtVA9tq>95?&i9*}vz$C2A90sGfqW$oJkSSk2$gzk;kG zF?wn_#z+?PmG7CiJ$)*p8?!NcEl0ymA_obwgvTXDNwTDcWS=NeFpO+MuHdP;8dIGr=U&Ve>q39TV~hI+Mcx^ zb~!z3Q|7CTj_LE$KBWl$Y)WqZyC9)Yg8SF0UlV>EOYD`tHm9(7Qsvz0j@Cfef8Il$ zGWu+=a=qj@Ift;4)Q#JNca3-}Y9XE>DNv@!Yb)xias#4NGSy~Pxa0~(?{$J+m=#|l zKf&tK#S6qK(q*#s;2H@L&g0Did*4$|e{fJ#V~vSPk3}y05tbX?G#A&po^KW82zK-7 z)KcCKVmV^|{keTPYq{Hr8ANlkBStp9V7KsZp-5n(Zj*g@M|o>_sYD>Jn3sm}m_-GF ze_=F6#Z#he;RE4EK@Pt=b)FXqRo$M5=e{9NQVIM@!53kPa02+qG~yA6Mb8wS7PJHZ z*dE?1LQV`Nx)TSu0`MISV;^U|Wo~EeLA$Q&>FDy=yIAces`_`;KxC84^fM|Fbhj`s z->RXNeksl@3@m7w*D=SExiI5V+JzKW3h&R|Wa*#Mqf}8t+pWtsWK@r_q+6ET!DNdWubb0@HG*Bhag$trH_^Yc6%wCfhhh|1=T!0` zauQw>R~#selPDy+!2|ZM5AZbB0`uY@ zE}*^C#6pa0DDJcZ8jV zl;9G-FBMK|$-~re{t>j^D*_|d6(R+0cqr?U-xI|f#$|G1Stl7?p(n5sE56IHdR*m7 zaUQYfT8d36VEB?$zOE4IkiA&8y7W%*@S^tx>G_I$SI&>@;H>c(XJCB=e^P%x|NSme zo$xl{(yzbbKmP3cv-6K8KbNEgWw`Ppv_Hyv=r$Q#h8g;v<}Fy2Q_<~2H1d%gg8m|% zU@-Vmz6v|ZjqfAi8w18E~K zkv`a|d&FHLfmp#iM$)M~s7Q~drtmxRV<|O1P@oZP7Pb<#5VaAm7YyMK;Aits34(=N zgh|3Utd#tu`ck{7kyH+)r<(GI@{dvXseM#)Sc+{Fn?H^}nZF9_C~`_n9wK&eO`Oe~ zdYmWhyDS4I(M_2vk;zUgyC|ZS?bU}> zWeTQ>gABQ8s`09?@<+0v(g(ubNbr$S?MN3^11BItl*n??Q+#)@s?pg!-O~}uv6JW> zfZEs%q`ehbG9DwA>0{|R;e?F49&3t|I9yIQ?nY!^mGPp> zL9w7nXb?UW)DpB8tPq3?<)RIuhNADnL5Ro=Lhe~0J{?c)qK;Bw)G9I)S+?WAH2M;~ zN63ROgPCM&qCS_;Il^MGrZBS@w~@c(qFwdTpp1Ik>9!Z!?t*3Ek=a=tZ%i=%Nl8lO2!u33PFA}Q=C+8%)5vzpJmGK_BRDaW5Ko>-N!`vaT z4FX$T>w1gb9AyTn-ALVI1|ASJHlSq@z^z$ z_J?(n9E6ePkU&hW=Dp%ZaSdck;Xp}id0oX}X|(phiJYMC!Rq@}@Ju&j1R+wtnH|Ep$B}R+aXs81=)+@$lPsrh^LJpC z^o!sl@+U_NZvv9`O2igBMPEeCMM=Uz$cc6d)(H0q`w9h!x-R1HqOOp+yg^tO7)qSw zCP2ye6VB-r^ce?un|WHgBV0eg4E)>L7fh|&t1lVvR_P7tm9hGr6@+e!c4}Dz4Ov=U zoKTcdIJn?iUcKC9*{;m(8D(kTQa2+u%1aTaP${}pPWrx#_E~MR=jC3?6BlTT9+%J#k;u>pjIYm0cYi*fsEY z;H|);fzd%Xg0=>44!II?Fr;;eD)>O)H?>untN0{e15VpwNh|SA;Rya>G8EjUZ&?eO zuzA3`41!Lw8miXG$gInEoyPhAvV(#8c!7+uE?^|vh}^OMo}S2o3Ia;pMcWBv*)~RZ zc$L%HvpLJSr-*P~1+Oo80MlPFW7rsZSN6fyDV2ykf7qcuYO zGmer`bIA9IGpdQ<+_})J{tW%4@mQ^pZ7TA&tf=(W1OpITN#EGqck>r{7P1m_8|eL}r_;+gZ=DYv<(W z#^!CyUtP4W=wie8Whwah#vS%{h#W*GFh=w@eMJVn=+TA zhq#w;IKL~oikQxAgmKLR%~&tybROjVz4t_ThF~@M5Y}gBxjVZ*xOzdue2(*uv%)zX z48XKE#5`qBewvf+#6(fCNWOachJWB zxX^=t;5zJl>bMU6u4~qtmWSr;;G8^ZJYJP%sHmh>Hb(Bfr<_$@QuaafytJs~PqDs8 zU-+}2F#lEF$=usHYjd9F#N@2a?Sl2zl>G35hJ`bVl8X+P^wTWWR_i|McUC>AwwoN5 z7~5?7Gbh`9#WSBajSTltrYA>&hvfAKhi$3gR-BpI`ZBCl;(gX!DjJmlKtdhMdSd!XhcJoUUSfRnpL z8x1tiPSpB2=0JFup`0vEIc7~_-aej~>_vVhIn+XUaU=LU_;>kn{9D-T!go?%s5ev$ zJc=x`4|$y@;sN)FeB1)gV2%m7wNF^}SR0sc7(V2JUID^&8T1`U?;6i-tb^A>oNhhX z##`9$f?sEl^_?Zv+zEHVP<&KE8uwP~Od9ia>qFZ?`*+84uurGD3p_O6 zR+@v}mN|iSn!OkZAwKbzSjxMPS=MU)L%}>D9TD8SSdr6;#)=P#KZ_?wHc2i?5+!Re z^NJJq701C3Xdy}m@}-t=q2M83kE_-IF|wTa#+`ySufH%VB>>SemiduUm+=U>w%ve7 zEkV?11dzVhfZ{&@Z2UH$-ZP=PYXcsE2fS}xV2zK%w(x-`Jr0y9A86KLjJZH{(19(w z%hWULu|`2jXA9P0HnHMZxri(EVE1K@MV9VFBzPaC z?yc_sBk3%_qqyEKKECd8hv4q+QrsyH#ogVCyA`+MTKMDcR!H%pad)@X*|G1W-;*bh zmSi(KbLY;z?|aVg6o|{AF5QX@hIN>44?|bG3zYm_lt|5>^~`v9kw>yS*+TH-H*#;F z#u=$v2|j~cC95LUt7YkS#BOL#O++bH4~yY(C6vo>{B}mZ z$aQHsoav$1P2B>Cy}K9!U;j;EH`Iyqgh4`oVW`kcXe)FO`UnGrIl_AU%M-jW6VyRN zp_F_Km8DH;2W{YesYGfBWy=wGWmNDs4#P-#jdvFT-TxZ+2;$+FG9zuG8xk{j!7cxi z{DD=AhSF0i=;bJir9@P%bMb%Qk$2&)UIhhrJ1DQq;aIqWvp*F&{$L#cTewFS!!Oek z+B%1v0p0E;jFH(;boQ6qLvI@aLKWimBwi|nTGfLD6EAe1b|iclWiwRJ_2mvAeUHU8 zTnPp7O{}4^;N__VU(NvNf_B4;5(lT13LcMMQ1JZ;k3<5}D{A7nCqoN#0c3(4q*g>g ztKJ(c`FYd^Xo0Rld-a)$gYHVAjC3#*?$w|PtVY*?=Dh(_>MfA$+W?xv%FxSNK{Qa& zZYmd4gAcf`FQHeu{Xe!rAEaB>g|ueYE)+KJye2fFeeICE8?W>?@x`wmCP zb11ov!+o&;nxc{LRaE$co8Xu$90XQqizWC23ZaOt(DQ$XBKbMq*IhV+PGiJvgbQfi zf6l1^@SnDXE2*+#fs?5O{?tVHonFKD`~vwj=Wu>@qcXPz%IxLvK+lJ|cM3kn!3{kH z8su5fK+pg0bFRD!-&LWlI)gL%0QoPU;hHK$@&gAadNp{18)3~l20o%i_}%+(1uy^S z)qD#lLMogGd2sKL@Hs%@2OnuLK11NC2*!Ky!&g=DRZYx{b>PKl@SoqJA--+`Uq)xR zIQqgU{DE~`2^{Zy z0cb{{F^%&*7$>tcpGVbksho2u}qn2=S&!ddRXPh?|lR^cJ9 za7f?%&xv>yPQ+svVgK(r`+paCh2yNkP5uY8_Z6 zF`Qo?;79+65&QE$M@q#}Rd`*zaB8UWyDP>;@PFQ$3J+F&xOY1K=MCzBxuFjnM#JDb z8V6r&g|B-&zOL}YPR7^c@HrCWZHDREqcU z4)5+RuGVQ>y`7lZm*QTY4zJ)$+|OHaB`@IIgyRlu4=2z?t=5!nRU2Q+N6KsDEh;$Y zZl%t({^|3VzJ(lJt|}f~79;$@rht^zfK8NsL4Pwt$RQle5-y$ot{A-Az~K2QePEMx zRZItUsmyDnh0{Vb@hi7W(@wpNE*E3?8NxHA8FPtqagC@rd7$^G(1?y=Qt?dxK_x>W z_>_N#42Du_Cf$-rCMwCpFxtn%`OuzxqqL-^k^AWZtRM8TP*gWftr({28M`wTG4yV;l)*9&Ju5Mzf%B`De+0KbnCZFfb z6nZfh?KSg3Qw_%7y+}|i8<@eSDq0U&!#>irT^vm<)a}tsA{BSBy@C4)u@DIbBZ<52 zQ)ON3Go@dsC-hX}GVd-QT7KShN(!M#rG@K!Nxf1eg$U&&NC<+$bNm&)O#H>W2VRDc zuG5ajo@nI@F@@%thP+Vz&i=zQkGw!_RBmAf_nXHi){*}r8VeE4m^o{>lm_>$5 z_g#(M`}r)X3dzuA;vna$@}cf>=xdje`-PQmmS4rcb-k2&(wty%zb)%%f8{lk7m$>4 zNSFtT^WXe^#a9Y<2TF_NDbhU9Rt89y`2%vc_ZHtm)>75k(WF87=9=jC9F2~k4W673WrH-3iWHxV>*RTSN3uCHkWHqxV%7MZkde0fmDnL~ z7A||AA|q};bAb9NE~1_(q=zA?@P%}Re8Vi@cBxl$%|M6gtO!JF z_KbEiTa|95tQ2=6ui+BWnY%!{=~Ijr&y1E>b=M>>%9)?!wdd z8C8m8qF2lVq;@&PlTte6!?jeMQ)N+e;I=tR<%7S~0Q%tuQhjAN6UIH))Y0TJXOYqD z5w8)yv74Ajc&63G)2#tJfa#+u*Z8yHWN&G;_)=D5W~fYz11q;5RfXKd3_&;L1)Bh7 z`witIv5fvibVVg6*t0=C$^>$!v`e^I)BbB6|W*2<2*5xS*>c!TqKI* zVd6)*8Jz}tj-Oq3r{ zu6f(LLq(hgyvAqr{CZJtTBmx772$R<+*4Qh-SJGA4YueYg+?a*a6Hu>$WF2;IZV3f zG4aDie@Ul|rWeVylnG}=Dq(@Q#6fizJMm|s?)*zkCr{JKawoDL()!ATrpi5}YPrbH z@({eAOlcuC7wHm*ql=ZF?k3#_e9D=*0BLYNSy>=xz=9jQ>9 zkMV>~DM1d#6Y-6BMEHRFWvSaE7E))BL$QT=BCS(C@Drq)@(5);`O1?eR;&-IlV9j# z++nOb+LE(f8^kQ{CS@YEUwK5%K%(vwaxzs3X`PF_ZK09R6az8FtI{4k!6kX8pi>4& zZh9^pgNKAx?qKBIe`!`crP}Xdwo;i$hx-8PWM$kLOPQrO&H`ej@{Wj?Mv;xl<>DJ= z2=4y&=#O7xG}xJb!#EzITo#^TgcVZ1tMkRBOe?I8p3z1n0C_x}g`e)OauX(wct#a5 zDd>nCpdvx5Xw0k-Hu5W#14up|K@FoRJh4|(&87BaKTb!yQ(DT?psk%r{A7P&7SX&^ zjR*$A&WC9WN7ycDAyS<`5i>Qrs4+?Oj&aBdMT+Hc zZ#$xe@=kb7ens-aR+$rGsl!OiUBpaMwlQnySlXFXMpkpQZeT)|7bDUPBx$a9F(>;vTK*2R9mGJS|xNB@r)DGWfW$tCF?afKYf zY?qsmi!eTZXZ}LQ*LU(AagJI?{YE^L+EX^sz>}yP^?)aHA}1jKt1BuxhnYC!*AA8c zl4AIM%1xS6g>qp?Kwb&Qb~KgDJX7+qj;t+LgCeUY`@7(y{4I_1v{kk;!{N!oH;lw#x}Btw z7xV4pFs6vUNiC%7D0hUvg;it+c`6dxKf=>IflP+u@~sdcpYhfd2ceEKirUD|g?n~2 z!HRRph0@=6kFua=WO1In5TmoV+!Y$s5@f>qDjU2Hu-3037t>zlf!Lh^2T5E)1YthC zp(H`Y2tR=LtF()}Bp#%!N(}kTJBj>^Jm%J_DDl3)Q8qqK>>n4s1anL8sk}@$*eiXgPi0&#iCU=AE zy$Y_{%1jNaDR%%V;c3z>WdfV)-AcWeBRo6kO==rCM^i*3i^1AIykqHfwxbwGP^?$p zOtjO4iz#dk>VZ3tKBT;%o-y4}`3NNan8{Rq^%U&a;<%$?6Y?@`p@Qgi`UKp``-#!= zWeSxRYBh0%8q38h+sR_4Dz%XOrhF&&BQHJ;gt`mh()FNUP*>SW%2ah7T1AA=k&;a3 z5HGN|h(hZ8PNc|;QC@;{J(y_61(6$2M{6&)r{0Te++W!=q7i2KOkyc=f-WlO=qU0m zYD(Rd({wQLoyn$(g%)y(GM1eKkAEGevLaKlQljb)dL(&6^$B+-E>7WKcr|RqIaikaN56-JF<(2e%xc` zgwzsQTMs2sZpf0NL%s=L+bE`-2v=9+Xn1_Uu%-3N4aO{uVoq=*zED_Y-f( z>r~^#U}$()x0h~2bDM7BLqD!n066`JhM)+P3VgV|Inlr8YdM^i7bngL?~j<>B;uGpXH z$Ye_#I)d}0&EjiSI-g9eQI31Y(0#<8tP?E#K&DX6B$iVqZjJOwl6b}*NmP@5Bbw12 zg$vRT?`O6jai813u9SO|zw(2LQ$i(aCOO{IL{9cZi2cwt7$a7fDXh3ov)71lM)azk z!@b!~Ur8mTVkhzm~ti z$M6lk&*66;BOXDHcA~ewa?|@v(93#xgUIu}M6+-oD=?>cfDiK4k`r8Qm6d`+Zi@BC zM!J^oa69gT997vBI3B`H{l80OQ?zGYa_4S zc~dTS*AzD(KlTDDFEhn)UXSOjbb=Tm1PIfpS^O(^1E!8^rA)C`ol}_}a=Ocp+FW`_ z`CGlD{CDOvJH*+8ec);2txRn2?uM^CO5w3KxDX{>l>S8ReB4kTr1JLELmu6YeS-_$j0$ zEECUREaj0a=*w~=%9m`a*s;gI$8RG`;C0+VN%9z_9cv;6%B{6G_|C*0MrSAJdt#it zMm2$-z;dWC+PNC^HPI^gk>iE7=ztxgwlSZ`L-YqKfW9G*V8Z0iVs+}1w4Z%UU63wO za6S`z*^$gvrH{HT87Mv_&B_(&Bj$xu%Malf)>m~ZNj%zPzL#glR3N4gf)Z>`A;8A1!L)AyHBMw(Tpw?l|a8XNzc=A5!$2#Efy3ID1w5V(+VrLhnI;7ex zbz}mRLF{F2xU8aQ5WgrNsGhP9Rq>i1>E+Z{La$BZnozwpKe5J7WX6aMk^LNxedt;? znAk*n=&9a2ObTk~chvQX%j$;QOdMY;FhY(}1HBI6AE~35#~SEmPy)RdtI$Ez2<*@A z%d@1G@^4Ick!AlTP2zaAm3W7FBY&obkW0aFJt?+jyG!%ow|YpN5Kf4r*`LZd)lepj z9EuKhb#k?6V=s}l`IeYxiUnV~JGst1nBF4)k9M)|;dP%PwGuar_m#HdFy=4upi+fw zE{q|b$}a^9wWB2Bxw4k1js)^}&rM{os4m8U z85^y9^`^)@q}9Z4;viu>K?`;`Z;vwn^R$zDxQFox6z|%{NDA%VNH&vuiG7$O@)e4Z zU%LCrZ-gj*HK7yT@>``5e^NX^b`WkMN72vwm++d}=h-Ihqni_Ngl)oQd5XI)4ez#d z22swJxORe8@uyHpo{!b@B4!K`DSkl(@u0lPJ6CGSL?N&2quiO8AU9TB5xx9Fwu}2W zjF{HWL9!Zkgivw@m>D;fFshH7ELOrEe7pd*q}W5+izmV_@y z@q+$<{Q>HDx5*kzfc%}v7miU8(MPnZX9_{9L0G&?WKPO_@9&=!d$HO8cHqcWby+YPxj_|lTEz8ksYZq z-c?}#WV%^0TE5M{2355ebz03Kk{wwUPXhos*4&<1$b*v zucd)f7B7gOh}Ygo>Zj66T!__hrspB?ot*6+ObnvCiN}dbsx3-{hNLS=2US(EBLi8= zyGA64o7AslQ zCw7B8UeSv)&?k@e_NL>!$H=p63;7;KO*^@#QVhTRH)nI~E2cPqa6Zg_PY3){ehiJ#htWL9?fde z!`j1a^&$#DRbL)X4x|!1UFfDP=gm=R0*ff@(y*eI~o#uLZ6FP?_f7Nw1FpRFZ1aNYWNA5c4S2CFe! zP-Ag%+vNxJYGNuJN@oQPS&Ir$Hgl7i_uh-3O!250SH^5_$K85Ot}Biath8FlBooLp zd>;2otU;Y;Z($WfNwcXiCQ`IYQ^=u8C&@y$hi+*s`45!cbJ#=ZSW(OjCQH1XQOXYCYFmA$Q*^jc+n{% zNtX3j8lWqEour^!pFv-w?}0g!NiGvNQq#RZm;ydT`9=OB9;Xw$V>z@m6&7`hWO=9P zM&;wG#C!L%$=*Fu8}5^P77;4t2)(F4b~)8oDJ1X6RlT9&Z^RD%jxvb6AT{B>QfCEU z;)=A_I{`hO9C3j79V?2aWB_L5`*JtWDoRkk@~MOitBcpl3i+sa32NcF)I&lNHj=^O zH)*=}K6d$_gp;)~y3WyRK35v4Z1othLR>Dbq4Us_c~7qOyo5^df5LFGzTAr5Dyva# zuOtul{6&nFCc|;FQp}LAkfq8Ya)mojnNE*{FSrNxn77Ew;G8c4L3zA1!8?a27Uz4O z(_vV5?e#8_yA#diaI!#rqjVu6$g0E}r4aXTAXJ$%FjqCiyLg4%+XCkd<%zhS_M?2{ zKggQsU54T(ZaAv(Q#e#TXI$l`+V-e9f^c+@Y`U#VWi-_ilR;tb+3_5|qM3SI!z1`z*J;Mq%;#GA?}@}SaMYJnNFju54s zmzrTevP@d6oDgpDeH0V=Z2!X%{{=!yUCBoNl-FQC@mLrSMo~Bzh>yWSD^@lT^~Egw z%LocrSRR8;cnx_m(H1O-Wn>>@whfZTN@`R*UZD?4V!RGSU-BiiOm|VQs^CA3MN$z- z{124M&I9ZolKkF3zBnW-!2_1p!$q88xL>!lBPM*1oRs4wkeNxEzKJT(5UMIsDA$DIZ4mmTW1%Y; zN}M6xR34NHlgUV^YiE%>)f)u)T&f|uR+Gsy=xRTpj^VYU$m-}r8R)toW(1<*aULDm zialsFaTUG5Iq0wrBcjOeR4OzvKf$egNOVD^TS8}YA^I-+!G21VOOOmuD*XkTSDJKB zj>S1TBjtl|KMX9ftN*2=ok3dlQui-z+8yp*>sp8WlmQNtqm?7X5sYNSKIN~NzSo+s+Nz+|ghPA%+7M*$=o=XN8R8Aiq5bqRUW2cB7<8&T^y_r@p_4kV z@za#4--7`2Sd|45(HriViqS-9A8C*3n(ICK+Hg_{rf25E$cjH~y?}&Y2HGwS(m&SQ zs@fuKudTABfu%WINo9sHhDG`;-5 zE3w{*@Gtk`FM4|6e7am?L1Os~{>rzC1Zzhovgk|71MIYY1@bR$muHmqF5O>px8!Z{ z?4qVcbBlq?g=)O6DqvYN$wlt%l zZk{*WFY9z>{p^49<`>s>ppJ~%ze&?t*T8VrxE4O*Xh*?@-Oc$crJ1eTwu+?0lua*Xo|;OmkdyE^|NR8%RhD;EHq; z&C#~+zODR*_TMs)d`x0U>)7jUDo!}bWjzc zjuANKa=<4zk5PF*IciiCg&)Jy}UjWQ#ED7 zOWT(IS@ui$xbm}hgR`M)1sr*6J(F-`1@`o^;NqgZ`Pppdvs5N2Dn2&0Q%r93!|1v( zY^*cZ6Q7o_CGkO0vt)#fBxw?A#9sbU?fcxA-l;nZGTgPeZ9WIWc2o_h`?AU8R`xc} z+eWpHZWdL)dUb2ab_1cDDhtY@5+6iQ{IT`tfw(=%Tho_k-N+kLddD?KJVr6cZ}t8c6Mu0~qb4-rj*TH1DL z0||}mUeWxVYZ-&m6Vw0B{Fw8+U_dGDR0|iVD%xW6HQ%iP-GaUal?Mjo#UrIO>|66>Y zxLdI;<7Oq~C90A~r9Mi3k(rmdA!|mKDYHs?$CM+96XJSBFOB}0ct1DFv6vig3J%>_ zEwQ${(d?FyZ5p>OXuiDB&RRsJnZ7Gk`R)PvGgAViJAzx$`N#aY390pR7ZyXo>W!c- zsOIRyEh)AmKJBfu45?}mJe7mqU{|iAn`@<)A~alo-Eh-JTUWoxz>?r?p_f8`1eNwCputoV90%iw%^((Z6nB#PB zxj<@=asf|cBvnx7)$fhBeBK9th*(x7t(v#`&l*3fo~ra2te5lVr`G8?!SnjP8? zx&eln2DM?5ZZ_y(TU967@$??jf@*3tsg|e+M@6^LOYACj0ySF1lfABZ0GZS*m`9ym z^_)reK>JYmH*S=r0DrOB{;z$kW2~bD*_?q6bmdFA(*DH<^CR*`<~GYYkkut)ZN`}N z7imA!+NGUM`ZT+#l~WsBC21-o7J<6k)E+xz1Q+1k+uu7@G=Z0ZcRR=Z1o&M?3*Px}uyfogy?ak)3& zV@F2oez%>sONW`I#)P&EpBbpPH8=d^ZV{QHlP`9k@c4;= z0=Fo3{1* zbXx2|aHLNxeqS^jDarewCe|V^Ik>o4apMx9xMuN-LRWtOyahQ!vg&4}r^Tn92Ae1% z=?>;TN0MLCuEe;cxTMjEY4Kt42V)+sO?HN_W!}j|g*R4Ud+gg~Fvp4fiwkJQdG{(8gTc2>U8+9ElM|}tS&G#8( zPS&tYHSx0jWnrVdy?Jl*9Ql72`IO&uFOhyQTMPxhme8}6Gph`(s;yEz{ApkVYq~au z9YPM2|0gW+x4pOJ$DnQ5Rlb@aOCy5QkI^0MuFltx zk2A{+j|0wCs$64#?MHP0*{V6K%Ia{xz>l`e`mgL5^p|J2Y1dX)srNSZOj}|(;{Q+Z z=dc6ey~EkiMS<=8zF0o!qL^{=Hs0Xc=J0V@yz$Bb?xCK5i#aZ!XUNsCKf_;yEf0Me z;zIX2zzpDXG(vLlQINv*KRH zPKaX@h9GW~FT@?2FpJ(^%K{P~iT`ZENP& znO$dKHABQK|MSLKTs7&s<7CK$=b zc;xuK!Tk${cZ) zcPkXp``ljsihLOQj!5{BkHOcPPM1@ch^q23p{wVf6a5hTX-5+zOpimKBhs+V&l7Q> zMpo?+bsTjy^$yjzUinlA@3+Q0Otp=uB#Z)0vMbMuEZITb*ktk1_!|R>;G4m!;NAWc zt^USCnwxZ%Sk-gT)yy5jZxFYUdsRz}cWu0%BXC1#%SuNox2aSmtWr=P-^wOZJ(`>; z9`;Okek>nWwy=x`zingr0(*DoD|dVEeqo}}!@CJ9iUW>~<&DbTmFz6$ik=tTDX{13 z3t|caiyjyEEGaIHKsH~WlJ-Rf1)BVnTw~6t%<%M9DIJn}!Xfx9_FBx#*h6vZ_|@^P z686W>i(3=(?q~V;`(M6)Zu`A7d2?=$vP418r5PeY(pVn4tr-A8PHjrmAjCgnc>_DZZkWZ zUO|0_mv_2cRlMW<(=*YPhCKHVF4B|g9Y$W)tgy}r*%EoY`tF*~Yn-exs#|B%LEeqqytTl#Obp4JDelF4b(Ja3fyi8I16$gZMwf}tI-Ine~z2+eO8S3k|;2RHs8oR$zUG4)dLJSkFiqvLl`fxR< zKfq9^?rH1DwMW5$(S;X09lRtekss(Pnogz&-}wOz0!@Kg{&#&{w!bW6O&Piz^-PtA zSxt2#tWXWRl!vIs1#_LbTTCqV1$)b?&?gR1no9G9ubvw&#d+IR)1x4Z^BxndyJu|` zXb!&8eVd;Xh30j{=D3m*_P}T*{pB1BTT;H~cdKx&&)oj)Fu4BxU1I~wNIRq8vwbVo0A#b?)Tsdf^7oc;WjpXpb&}xknz9O@J zl!R0&Hb(0-hx%^_wudc>2#Yut&W1e-SrZiLx54_WvA6b_>IciI3e?-d1Z!_PV=1vk z`q*v2=vYr!?5588P>qRQPVAO9p+dA<_9|oOOYCRWEbR%s(R9LGXt`lcvF2OaBhq1% zT4oNAuRslJCUo-tz^?VGt1G-Ef4Fj7gWOZ$FNtuya3q!AE-flvQPiq%U;f{@M9#^q z(V4w77iQJT7IKR6MiqDq8Wc`1EGX!Z|2}tWPJUKOW@zT_^qHyOLCb5LJU{7CQlI4f zyw2I%BcspxwVdwH7VE~=18LaPNTw)FNUE){Ax2J86d9J>Rwj;VKmC=!G3Nrj| zY6dtDg={W2Slv?H3!K@WY6Dk`93|fq_w%FOyWLIr?}8Q{gnHz7dI7si^GLVEI07`O z&)ENOx6CpFe(%(GD=w}!O@@5+IH)=`M;Dh);a�csG4e<>DLu|bKR|SpTgeqm8&Y8Kug^g_ZHU^=Lu9|YL|5@`L*aB7&T?N2XmT& zxw$_xIYY_#o~g>V=KPkkEhi_(nKLRUC3|)D+^n*Ul^OZz8&g-OJV*{n&Pf`U=$Cjm z!5BXw?o~|nxR~^!0+aoa@Gt$hcDw1G?<}Ur2nhrhn0&)nI4bIqdka~mIiPS!#o6=vN3zGXC>6UmH zmBy3O38jb{tGcIuYkufk5I8fWbLf|l)ghTd)dT$lg8UBIyrvh14E;UrBJBa)F+;fN z0thdgEGFxC%SZEk^As@t_UUBxC2lEQi#h}!FhwrH{<00X6;Hi!x@Lw!Mz?8>xf@tc z9}V+#Zq*2;8(B$C5=!|e*kRtqJaoo+_^a4m%I>Vvb3V?ue`(xBq}f_=zT9+TZD zD=0H2y+eBE^lj-G8C$c8vVY{v&HbDkg4s%&^-sp)^upAxDJPQMNwX98$A5{P7<)aY zK}_!$|Jd^AJwL6Fj5KnC7Af`25b6tRq7RvV z;GBHcRsmo6tnPxYn*NsN8<#{UNEi4&Jua8%yyp7b6M;V6ed;O~tPR)anO;~2`_>6q z5n%T3?3-nA8guk5wY$_?RQEv?9Ix7mkA12WDi?IKL%BStOLNG+N|Cr&=-{pI?cl8p zl|mI%e|J+H*Ie7)c*^?T?_!`OxN7i~p!|SDznZ>xZ6~bNE$>WiQO$`4uX?1Zh26z}_?-1i378*RDZF#=cK;=|J%-Pk*^CBG1FJMh*3lxK#|B+C zNLXJ@zZ%<`o*8|#omHF3CEgfMqI0CZ+#cp^$j8bnNMANvJJ#^YTpQG@7QvlEzT(;Z z$~VNa$yCp9804B;s$WzO!SsBG`)HJQt?n0n5xk&XG@ZC8x*@Sx`cvE|EI@|DOQBr! zk+&!_=zglxQI#({j9C0!PRy_ueH&tUf^SBY~7Dkgg!Mn`n{EbJ(56kREJ3zp%$?30=MGfWw| zX%AC9DGyTqPMMYZBy~hu?eraKSJF16@hR0(x+MQh+?;SZt~_>bY`vI1(WigD|MB&E zJy?Yn6!dirmpM&0^8(-F{@wsOyjSIt(7A!feO?%*X{ymL;5&OLOp>Nh)tE)9uDVO` z_9lU8cfja19?|~bl8AjCwX;+ChSGgyZ|xLcMOi_;WNq4xhD7sMzny_Z$kfmiLCXTp z`wTbD)mPH?QQNuSxnZim)U7lb>R&Z2v>$c-PSNPd3DG6Dh%Cb-x;xni4ESeWHC@ z`P{LMwVpF+3=g$+z$uwSZzsD$N75eI9skPD$u0<(jL^)}Hq%YiEkT|6o>tUc2HBz) zTaSKDtd{SJ(}YHFh7R;b@dG`ZTwfi(+DppDlzuGM7MTi)b2G9-v+icZr^{*e(s!g! zO#hZ%KV6&FJ9TUFpyd0>my$*&jY~R~I60van0wt5-o{UiubU7bZ;8*09UqhQ^YxF; z-$ur(^25qC(j-kyv*6n$a8A&+h+9?mR5~5nGGLAQgYFluCH&=e;8p!fjbM-k3_|== zeP@%~s54KqtP(T_LYK&8KqK2kk{VTpa>1#cez)musUBow3H z#uABCAU9BxWB6eG;Ts$f5?Cj|&;OjyI_oxz(_GE6(tOnz3;yo{u>THNj@zF4_VU~A zo98p#_J?_kVY^nPZqIdRW>X`{A|(S_o;adDoy@jXch$DkwbnJ%-PAtO+yehX;SAh) zxEbn^=z+rt#)~3SRo)6)y^B2aTt^)T%HNk#B~yyZ3$pXR!E?xut%M<-QDI-FQL>2>0$#2twn67I%djh_==9N!?p9e*pKePVFJia2x31S;j zw(I%^n%1hx@c93L87;}Q+q}+{ZT!cmGOI14&D*hh?9Of>`%1OFA@Ciid%<{Aim^Yb z%2m^B(M>YkuuKEtk-=`-w%Fu{A0i@ zSZ({=?A8BIV^ck0RwA?FA1JSSQORUIY92j}ovB)-`K0yN_0nEZS5^&U?MOD>2kmtx zvQ|zKx0IhyEg0djA0_=Q1_-VBORi}9k}`G4FGY0Uebqzp;gn=mk;XS^$JdqP=4&4kSfM-%=RpA`Ql{(O8%LhbksF>ybye~%Nqc8Al8S*;fRm8l?Q>ve+(mix=U~StFT^iR4N`tjX;vtk?>Kghm<4dc? zy2>)s`p&f3NEw{mC{iOHa?~mPTd<)%v44e#=^5(OWlI_Lwuwm5g09HufShQgV0~xxYAjxD~#iq#=K0 z!c-UFoZ6+|W$b8aX8VR$O0!fomm8nx3$-J4r}P7j3(WT{eQbaEZ1frF`@3JUZ?q5N zGt*jR%F!RgD$EUa<5c7=Z6sSF)%!kWLDEJ|_AU5n_dqT2QA^web{3;a* zmOv%4hxi6XyFYwH9|X$##2xFP%R81VD{Pd%Jhx8Ho$R66Q?v3i<%|dEA?f|o>ZSEa z`;iir>`LMjCdP-wnPShyxTC$%^Ws*-UyVH(7mtd>W^|;&@ptvu0Y80yBz^0j)Ul+G z=Lj8OI~4RaY+&URm4;O5RqbX(t>9sP+YA%cJLt9GDO5%Fb_SKG?yr~3`Ia2ZpH>%G z;Kz08(8yjFvz(@~ZpE_-CzM${hou$lSRHHhT5Y}yd_MW<{jI*VPmWn@+F|OT+rcG+ zj&Pe>qC0LHZTs$X)29JANThkcexj~S)rcA^eL#NwBKK>D)tTb1EBcZSCQh{)Yw`2? zO5iV?#9FYg@rz-yVZFY+u7>Us$Y#yV53S!onEc28wSR$Mkl%XzzU!8WroRk3bUW1p zxd3S750hu9xr`4;F;)Ldn26;5VR^P2y9f>hf9gFcA{)U&`cO6LO6)1zlQUEc!FTTh zPtPKGhFH(L&7F_ArfykA$(`c%sG<%j8dcOD`>ekC-E&uF*Uns+J~k~YWnyxRB>zM% zv3KIvgmwwz6RIcHNt%+>GU-Z^H*s&mfrS2vLz9T)dPyS^TP4hoi;VpzrhCkT&Bo5p$)@FhGm2sD~E-b1U~f1H7wWk;u?VS^a*~S!Rjl91?Ci+&GynX&3IgU zku!4bF~5z8L|0V5NZLK0hqIjmA z7k3e~pT*z~xYhpJv%2mEjnQU`v^cGIeV+y71U?OX67b2dna@H?k+HJDM}JCt9aN|Q z%;HvcOXRJTf*NyAGgdQ4b&35#OYnFnABg` zv9yL9C$mz3_!i9Owx0g3NN2hIXn8@|(lW8MSE;+WtZ-8P-JB1ZQRz+7$kcNwKa=+* zkHjjzddj7g=#;gnj+)aZr)H&0Nv@IPlQ=Wp8hcEP}tMZX`x%g4Pgrc*ZTH1r)j6F7m~KJL45n;P z{R^YYlI_zvXkSR*kn4dBd>$JQYn!lLi2k5hUGv@nujG_a9lpW;5%1x08=~2xJ*Do% zCDMyg5sD=ASfl)+F4ZO*ds|Lgd-=ZbpAj$~G@uaylwWOYh;fJRgk~{lD~~jfwUzXO z-l=E61-z)Orw!3OSJmTuXe(iquZU~K5%PSZJN1@X%MDgh>J`{yxVis<4^fl-28i}w zL^jwRynScPY0tdrJ12?4#&_*UdbX zek`qe>e=Kj$(7MVxRO{c`7Ex~`A4Ji-50ntxNGn~fqQ~_23PkVZk=a5r7mY@GM~x+(O*>lx`XB=woI#H zIbg_C$3j=y3OwG;o_Oc4s47P|<2~O56KT=DGuN=4wAHoDHJs87Rkr}c{S3R7o2U(kKfOj1R%yQ!{3IXvqfP}!x)<&;0Wg$qNH(?9GL5IKfW+sNOD^~xDg zD|pZynkwUnpYWm9rZcEvul!T#pCz-3Cl|R2&lFB5Y+Sgephy1o z+^X3NQPnw;+B&sL>eG}fDg9DfrZi6dHzhb_QOd34p2;7QPbJSyawWA*mXbyxKP4hH zAgy8Aytu=1l zU4%OBG`}+JRrymY<%72$KSdDv^PYX^kJzb$oAw-xc8n&Z-ljcpyxD_c}^qAC(d;Pz0pl}TiLo2P4TsYIpD4} z$QhiyBJ+9%pYbCjB2&!RlX*A8n_ew_LF)a~Vaan7MI1dmf|^GKdnx685vAQI-9D; z8a2Nfci8s(9`KcX{sH+a&PTSLuyipVM9#$z^-)#4`jRFG#N7FqK@#dId{w_-xsR{LEkb3#FKF3`|U&` zbt*_8-J#mtD%X?7UO*T7vcJL!4yr{pq_!|tX;AXDfQ z?v>jqa>}^$1%>mxWpFONLq^9uYl3BtrHOU6Edb=|LZi#@%3NlXtXnJ!>X_aA{QYI>mwWvDf`5|^ntAQ|mvW%zGiLVK2peAZ&9@&056s+Su^*j#>1 z{eyic+a7}12KK6Pw*I!Rj(!qyB<31>nnxHH8rOqHnAk}TT)<3VsiDA(}sK^L)Pf?+I*yozN4%ZoXIicKVDllln5% zTV$IImRAdHWLinVRnUWi(G+@!61kP4NLu-`vIaT!YpK4piP^*LP|wgEGSRr6bFEJ- zRV)oHyAfl%-w}H9L8Y=!bQT3Mz}g zn95KWegiQjk1{d~nC)B!(&Tc~=RhSqr;j#tH7+zAH+3}kht{o&af9(77^c%rxyC|c z9i$MPfih#0euQo_2>&xw)43*a>3^de(IenyiHDkP59lv9mG_{D?Gc}Y9yL;^Ecgo+ z@T%T{*r(vxR0pZ4^Z17z6_U1oI(s_i*)NpODlaNOSN<3rZ&&fSqWFS8@_ceCWpBtT z&opJ`XFSg6nGu-LEWJ)zT5_kP1&NtB|9ul~BqYYKOn8L&+yhC~lHVlP#oZE{6qp*5 zx5+hD;;F$%Cj6{58EPZ$H`-ikF1O^_I{W_X-!wQaC?Y62a8zJ}pt!)kzLDqwQaDdr znRV20swMrAl8F<_1$hK$Ei2{2(gNWXd{4W@$x^JmjCe^^;o?;`O&!p2JAl$ZRsX_J zV$z_NBb%Fp?E9C`YTu!@O12oEo;JJrgK3-LI4o0rF|%C-#e2W{vkJs85TOWUFGNr+ zkSqFv0wjqVK(}E&U|qj~yTt7WQ8QbWrcTxrY9H%sn2OC7^IN0Ylw%^y^Gx%M8eEH2 z#;=BH20thq&*&3ChODK{QSVj}TsU(XS!YwI0??ZBKzezi+>ndJNDveM5Q0J0JILSQ zdw>tR(({MAfxAAQ5A9u}ogy?cX2%ozlyXCvsjM3EFYLvaiv||`TM&~sE7zKHI=e7y z1g^(2?4UE!E~S1?(WDMbF{UIW^+?Q%zZz$Y-5VPa-y|VFo=SL{a4f-?5R&*l^Fx`( z2fQ{D^wDWBfn!0 z5+^5kYk6DoP@uUVBVV$+<6Zg4@=fK-a4(dUzAfoivc4q0_;t~QqHTp^3bOOtHq8eUXmB<%gmiScdnchs=BT0rJAhlr&z6;tRgDOvgx(!YF^e1tQlVQvpS->XZ4$^ zb5-ifS(QoENx$y?8ei2y%`$zqCAh0xOT0r+pVx(WlvGLW&!F-6f;D1epfdPC$nD_C zA$7rH!(zhgLz+ltiFXMX2pEERUVDB^!6b0btz#9@dXcXJwWZuM(skMO&ArLH#Cy&y z0~*;fF!P{da52olS zeW|^uH>f`93FPtK1zYB4`d5Y-*v!`%SD1gWH()Oj%zev|0c&y-+s7J+RYSw1vX(Iq z0v&q> zSu5Fa>1!!Vo-GTIN66x3E2Lkfsq(&>@s{njY3{B-#VB=;1@^`lZ(r22i5NxfN4#`l ztuS3&DJ&EZ6Y<1X0)HYV&K6PxPm$w#ir<Sxv2#%1vqV^x37Sp$sf0M-E3c2+JB+E=lMvPJ>BZzTN|?Hjc}bpth; z_L0VdpF#zszA>m%>jiYYMCwiIKH4E#DOUN@v<*OWT*LUlsAD8D6^v8B0_5YrUtug| zZUi6acwm*+&;|&!ea+hXlXWq5`_(Mu^JS>cDK{w;^2_q}^49WPd5rw3?4hho{#||uynU0T zY+0VPUMiC2$XN0w@*=rj_C;Q&jBa>k9pab^p6V1-LKFejib0r9R#CdM-g7SqZUy`q zm?Y^F5E;;0{6+kSX|1?w}% z3@_MypP5F1(Pw7Hg zAL(T2X6Z#~7uj>^QS3p6%5STt>XJ=EtXmxU$eXBk{fUZ7uxJq5Qr6JVvcK?(1?$CU z#e)M915*N-l9oX+K}#f|L8}4_#e{%D@eZ*U$9T~o;Z*(@ZW#Ls;~DVuJCYuOC$u~9 zA+Tk0y{Vo&cRjK)-Z)1&+koR2dFo(-u6C|><+!z|?;DMgIgZ+pslfco0v_jA@_p(c z`gO)U)Cv5HjQ%d{FYMWz#T-8O19u^>3egAnTEIuJA5!4tlLbTh|MDhq$8p-QGgw=g ziOdEtAjdHd(I3$cQM*xQlE0BU!Di(G!Ey`m=gNTjwh0#N46xge0s7q=A{i^r0%R5R zh39)JIg+vkn4+D5FDb`}^`_X#4dgv!9{Cli8|=6wcxoS_5-`X+3HeQRt~<_rF#gop z=2`n#=75!$YWZr)H7QVI(b72Ia0*z^m3n@|Y<)NV8{HyZ3te9=6C9>LGO;R6Xq#eGRia_&Bcd_Ciia2&M|m=#`F$oIe)y5M~Hdfz)afEEk+d)N>7IH+wki zUwGYy(g)BC6c0HR7$TA2L?^+kaK-2KcJuD`5Iu9i?41t_zQ5~~OYNe#$D_LaggYB3 z16zO%L_oz#H-d~1L3}}^lD?BJf)BG7RYEJIjRg~MM@Co1UPc4s5VJKapEZ=Vnzfa6 zhBcSfg=J)3WX@wop&Dcl<2S}BdMNz?ErRC8dUBbXL_Li80TdS?L~2PVz~FlZA)NxoN>lj7pQGOF6?q2vKk|C;{*ENi#jy(b0UyZQfM5;$E)t()L2XSb z>T)q|WQ+x(OEl}1h)LHb23=Z=IGoJOGHIQAw9tynX zrC{&q1>G{9r{ebFCUR$Regh*xS7;bHlgiZ5bLdNGPk?ml$Id^5y5Tjzs{IrBD08sm zo%JS_Sk_2db710b6x2n30bHJvJWw z7eyI>3TZM>Ukj5x=YXB#38N3=HGMrir7?5` zR^Y?Hd={YUXdksB_V#lq8DLC*NbUu1oej9s%}4_56_h~S+=%^Q4bTd6f!)6s-lS2) z_r#OHSY@KdECjf)V@cac|H3NPf>VAuxRqC621v;sass6dR*Dgng3TX`KBW$KIL{!8Acl3vkKp2J` zMH{eu_hIj8^c?ZzAQpQXd(NHiC}5mFcm3n)?(%?veU_8yyyaNyFxZ0}e}nTO$sTAU z*jTnyYj^8MOJ~dPs95_K`GXfs(@YNI8;lgi=rf!#+%j|m0=CpR%(UH9W@4I0o41%% z=INHxmhRSKYliKWZIJz=eXb+PdDpqrwH63XYrI>19SD_#RjAZEN(uybeH{1}+EUNM zG8zHfRY7xu?_xat5O#ZBI*;L`zowt1PX%*M6S|Z3opusDAj4@H;D9fso`k&T!0Sz- zR)WuUJN79(!Gy`9l#`3_y{Eu?m})0H z8vx(VZ{#K9VOUKz<8x=p2grBH*U8_Z)dKL1k-)UBChw%|#GKMmc2Q2F=k*v#A0-dA zT^w~fv_MDdTA*C-rXC04^;PK3k$AKf&rHWvX5-aP)M82~^#JyLYcM!+spIK02;Cfxm!!Xdp;gm=t|dGco5z? z7XblJ;~3&db9@2cbg=!LZIVr4eSoa^dP{|62j*RzrPMsnyu!>hm!qa|ylI3f&}23i z8h0BH7|$3T#vZ1HrVpk#)D`|^{%Q`gOt*Ztgjsi6$+j7&0UBxlW}oS(aIAEuyGYQO zS22Fgv40*4yyUr9o9B@Osgdoz?|2>^(-yO(W3jX)4T4H+)-n zXqWN)K4`M7w7;=K&8Ib?#bX|l;KS8`6Ym;4#^cdbAyg@5%@N8B%4p1^1d0>BxBXxP z%K;9C6S&0RNISrRK8rL87{_gZV8REFc^r_5b73d61~zU#Jo_6EaK~b$xVEPq@(pM@HTWnLd1(2)vVx-PeR%6r; zQ*ZUn||3i$4KgQ>oVqyTPp5aw(@@+|V2#+iK!Gt-KhnGE?DiP^LWNaP2=!0?t* zN~x!qv2xOYKQ5p$sAg~if5n=3k+K%@)Dfd>BEKc?gic5X!rygBNFHpG8^9rtAb!Al z+?G)1JM0_kqx+r#Jv!R^9oWTrsOCKf4D~?wE7xLIb61^nKQJxm@Y((4$OS9FHDJrf z+JAwed!#MgCbb^NNVc(xtt!i1%LdCVOMBF(QZ04nH|87WeVAwC%^jgjC1$Uw-t^V< z9JbAIjPDxL5z{x5vT+P2m}i=IgDbw?9AFuRk$h{3w$8P_v}V~(*~Iq6cB_4hqnXp} zJnk9_oWy&cUS6$tB~ZaPpz3-tBH~NHjW7=LG6xb|O=$u4R2j7kZ6grn$7(h( zu-9D%UZGXUQ#=kV&qMV6$XQ%YpGlttr22u-w<+`>I*ne2QC^9$7t?+~tL>uxg^^98 z5~<&?`W!}|{z;jNb#5qS6vnw9r2{1&M;jbHAwjF46>d=;0Oh6<+Cf6i#acHOI${g; z9QJpwsgH2`3>Ho)wH7(ZdU!7g&;>qleVM2pDv8E~Zs1@qO`ut+I_gL2X?*fed_xz+ zIxUnUXo&HYJPHx^@FdB4H64O1*+uNqsCYi}+ zl9+Q~0nIjV#5=#4?dELDe9L|0zD}|h0k!skEyRA*-ohbw9Ch{tD*thJXUxwXsC+N* zwIaMAbOQ?F1X2}g2>Agy9MWDwX^qu|PWuDC9x}ZReHKuD3g|`P4^YzW^gzUA+JRr7 z2V)TI`R0r~unTm6{hyD=;S4MNE&UiW0@~1-7~NaY$BQt3{dVLJjLQz{YU+IIAFxe^ zQ3u1$Xp4PvD3y;9XHWyN8_1)!#GA#z^{G0Qji|cvgdVJ)-W1g*zPUWi%|Ia>@-z%t%UHqeZidJOt}?Ep+!Z zQVi_SWx$V>68!8GV&v9r@crha`5plEJss8e7d+E|E?Vh60Inwwyy{bdRr(EetNok; z=XWqb3`C@<1Pl;^?P0(Veg-Rbge?QL`0p@N7g>9Q_d#s6S}IXVe#3Ieve7aZ_D(lT zt|ikFYT;TOuxLumFHsG48o8RwFf&J+2VpFmm}AYMU~Wpm$acXqW6bk0x+l!9&E;mF zIT`c*riBR`xBz+PTWuu!Dtox&remU$2Rn9-TYx<5UO=HAhkd{VpytoVj%^)$-ZP=u z+Cf`|QrBS~4955hU=5w3|DY2wcI|+3w}^2XI7?1O2(vY_531{zGIub~F&{ENfi1_6G_tiCNG1#(2fJ#n{JK#TWyL$YLZi0vQa3hi*o0M==;F&d}G;XVZt$ zJ3%7yG4K2003#e)2ejVwLC_ZCkqI&stvkIbJ(w=0(=n(ldH@~=(IXnq33NBscq^KU zRt2WLSI`@4u`;v2#E4T&ja^gn7bC&ZRq2UE;9UKn=snJ z&I-)mIgTz4mZRK$&i*$>JlbxzmD+CDw%Zm!JNLGwgZaZ_t+jr_xLmOAu>ObK^QqR~ zt-o1wt+Cc1D+O7({#Il80(rP&IROp65EYxF*1*l&WMChvX1^7>%s!81|6_-YD@2dmQQa@Pr!&9 zX^RZ&%xo*caHF+llrP+cDc>TR%i|h?wU`t&6Rl ztP-mUR`Em28Os*T@$r@cKt9iajV!SE%m#BMX88+LNL?`>!7gU4c`3B@81n$wV0pml zV4|{J3r+qJcGhLnUbG{o!={6{Jq;cI9M8*4BXt_WcXVCh}s6mNEgWVNXn$C-)SnpdP4MZpgSA%+58n!6XN-9@xCPdLbs(A`1xLe7UjoK` zBkJ}VfH}7n+PxDH+1`WCr6&+oN?fPm0dJ0|>0RJ~BxC<`25WY_Lu0>ZUvD2|Pqf=? zZ?U4zu=TOU*{IkLJqDl83|M4BtI_fad((CB29B_FvE*U>mS7Ik5bHCVHRd|lTXklc z`Iq^F`Mvogj_2m5u-e|EeKP;V@dbONPdG{-ElPaCfX^AB3%q85CCHKlX-dPo(b3Y& zG7R>?EX&`PWzZjcESF$2$SneBhn?URYLBZU+GpA;?L!cZ1^>9GYPDYtiM@TSq-cN_7ucY z4PZ(i!&wjfm-ieM$H1W>78;HCMGiL|_hPOM@t-o}>JMgU?+k7@5{GH9gDlyx>8T#uHhLOp7W?5I^)d*c3LxJCwBpkQ5Vt!QXA}Nvnf|7DX^6zXxkCZn2QKTH)Iuxn3I|J zm~=$`cC!k>)!G!*6>A_fU)U^84re5|icfH!aEdreju8yHRIU%%LrTssNXtb;cNcRe zfxk766AKnh1N#&E3VRcK9J@WZ)GAq*Sc_Oap=D&u8?Z(Cz>BP6Jcp&^_cE|?M;CCGa zKgclTa0P-z{WBtWI|;K1{lWKQ2iwU-Sb{Tsy?p7&aj66LMS*vx_b;IKSW?6hsILQ4O>b98x(hhD!>G*#V}%c21#F<#^*b z;#df*p+s;~{IcJ+Z^!8LfHmQ?eFT=x9#|8-ZKzs@EupcNU}ts{*2Hm)_)1h3&bQ99 z&azH{G)={$CGf`Vg>>DrzO(+cs;vYY+m>i+V;hY6&*iX7&O%!$fQ}=!H?xn#-tGYY z+fTdC9^q&Qz4ou;wBxly?yTZYHHNrL1wFJ35cU|vXWv)8F zNif`gMz17yOEBtJIF1I>G9Y(=O2F(mG)W@e3Zqh1fapv2VD5 z9Z_#e2>gB*;Fsx4ro)=t0WXjkl@M2n^RdG4;ZeJd-SH^s6bW|tg~0Dx@0)>LL^5hp zRfsmupmzw0{U+Kdqz13zmvMvUOnIKLxP_>gnGb1pdQ zx;tAV*FoZ>!G=*eDjnZo$=rdqx`B+88;+}BMLFMiT*cWvw3m+0jw(lk!{-n?ln15!sT)$;)=#0;(FZu+Fj$Of#0&NXILY@ z$z^c8sXQETj`aX%);xHu_INJ?{Zrw!fsH8?tFWINE7#Y;*9oncZ-@`5PC%HO2+Qn$ zZM1K?Z#MdI0a&{>qaDDEd=C58?h8gw55<~(h46*oBV+(6`Vd&GLP)(aGQ}i1IUnBE z`#_d$1=Pk@6b`(8E1(zb)Li(JuEG1A4i>zfSncfeXt3n1WL&~N#s!Ob04(Ai&}6@u zdM1mN3|puhblPwh5G7eNSQF7ku==rDvzoBdSP?8HY#$}6ny)aoFlRD5BmT%>RzkM+ zFlI5@LOa#c&tdn|pB_cm(r&`DJOGmQ1wOAa)EMNboJ9n_Ibw{D!A95_yl6%6_KkzT zh>S}AzV7Tj*EReg|cMOb!y@RCgi&s%SA7jO%-^tSLeg|(C94fRHX1gid7G1h?@DpCK~%CISR)od?smFPf`jh^SO)E27ffi>L^Clv4x?JW z1XoPhz|^^nVxU?IES z*nfKvM_US6-3|2Y?_LKGc-w=kY$7T!SD@dwfkmzWY)=i~dm2XA2u=+dA&}S=JB!VT zJj#fC@G#GTSGbnMg$H*w`3U(X*-1`?-(e~A$Y+X-LW6&+FZ@h<;n(Khsnzje@G*<$v@y9_2A}`^32ccAF}pfopF~1N+Xt|Hoq|4_2ft-oFhGX+xQH64!QA!?eRK%XpKajC zn+J&-0oJuHkhff*xTe8BErGPrU{4w0ZBl#Wo+==WmwLVcX}!?%4BXgvp!cr9zj6$* zqum&-Rp6=^iye3`aJ#j`s@eo{$o06~a%h$>7~ks{M)4xf9ZL3!H-AfrV-H60sr(;V;YtOHOl0)>v2=Gtjpy ze9LewM?dd?jGcp&Jp;Gg5Ad!vfCG+55Q5#L3HaW6BC}{XG{S1=g|mc5SXa#i8Zip| zs^gJQw;!X4niQf0Hg7k?DAte;VMp{0aTFSCpZ4&2O@+2OfOz|3axr3w4P+vPgQ!y? zr6r{sqQd>K%NkD^huzjBXs1~?W+2K4-Vw?)cw9$Q1|g^AH{?JiBJ(}~zELf?3g2^= zd>oOm#n^EV1g~l&nF~~(3eqdA^xNQN{~b9P(THYhh+p6d--Q)(EUq>V>}&O~xb7kU z;~(UI^n`Cb0W7dqaNoVg`ndyrF$<^;ogs6{J`qsyjg4CFf%gu$8Mi^l&4JHmB7AgX zVUZ1huJiL+{tdf>hU88$`ajl7MlN&!s#Tb<);t~_9{a$CsR1u0*{g#7`vHlpfjqu} z#s3<|eQ3i^IJ*Y4&67BGdp0*pUt_d7_%Blkntg2y4PetBI5hSZtvX|M~2y9dK=8U)#G1vzRBU78JBDhFrHaNDv` zyS8e)&&R7h@cyBQ$IZs)*W!B)LlZxDf5m^XxY-^_qn7LM8G|d^3`Edp_&pZ<#w2hb z4h19VMpzto8)c3NOpIjoVt?Nx^yz*?+n@VB18dYTQ*nqWHOJ~Q3Q?M+@MmuXF6Jr1 z4eXGfg6XClj5#*=lGs>@;-Me%py`G~&Srw)(a#}y5$ze`)jwd@S&0OoYepc#k_X8f z01JCAX&GrfX)CPlGvK3mhDgg#k{od|8;OEAsh=q{4STiU$lVby8;1DXRPsa|^I&5w zg)A*cp2A9Sx~?a0K$gc!+%HF4fmi+Sn2Yz12H$Hhe7-&WJSp(;QDECE;KeQmAL#|i z{TAdCO+yZepII~xF?Rw<1qplx?wGys3rqm|YkTkj27#l}h+p~|7Tj_C{=dPO*azqV zso+BN^Z(UB9*Zy=FQVU$A-i`4kXffg6ZY}-M!sKD^lSq7Qv-cGNGK6{(d?~5`w1S{ zZ?OA{z3+h#^d652fj|BO?G2tQ!MOauEg*>St`e^lHW4`Wyhd*WA}tzl;~H`6KXU+0 z*a3T~0fz&a-Tr6%kKA4-ng@>@&?zR|HvIo5jChy-@PCK@{TdurJobNz4Bf-Pl?LN_ z!@v@d3mw!3zhelj^eM2kmNwd1JE4Cr!)kv8MA|y=tdPJ&3RVn4Yg9lDB}_mMZ3808 zdFY<^u)ZoAnMOHaQw#@7YisnXpP|vu(fAjnWF2t>@Ndr$Ph;laBi={;z)S48J|W&v zj#i1BJslAxa>#}7OAiMbE=(}@GQlPqgovvUEd=-e=4U32fqaGH88KqAT-*xregT^Q zQ*?Zqjz|8+gtpW8hsLh_$p6JPqw& z!KEVF9swRpANZc?U@Mdo-azBsM$YbG)UW&tzRAgux53EhX^EMbfZYQF7|>SmZI*y> z>;afTPa)pB6H>PnR`4XS$PV>&M{5JiIT6gY0fvNisc17`b1nS;6>%>(Hcz3QgGb^eB=a4tlFw)r z;25n2*RP)=l@7*EKYMNjWHsggIda=VyXAu?H6MHWp|IFSLVo?ss58*-bHH%?C*iL~ z9@6EI>gCXY3vl}v9{XQeK==di^K+<9!#K^x*);H~j>F;qo-ued2J6-&oDWAEjbkwW z$55kZ8D>*&#Qkg{8i{d)v6|2VYVen_03#dbaH-A!Pp*@oulJza;h zt$6gmbN_pG;IaRmd+=y09`8ikfqmNE#&6k!+r!`nD*_W-RM?4{md;P#J1@5ZAq?t9}|KXYz3R9kh#`WkG%t z8oxgp^Dzv}zkViI0wTRcFx8u|a{AeSYjKTAT%{D(TZUdQ#t3{z{}*E37NgH!qp#nh z*Zo|%{t@e%+F8kzxV$dzisLtMSpvM<{#5{X#U^! z39Sfy;6MCK#-DNj6}?l1aYgA6B-8)s2VPNNbZa5o8uY0aXLY!5K+}PP+=l!zCwjX9 zubQyqcLCqai(50!Wq5}P?=aw1{|HDMzuC{l?7u$$l|I067spFHx`p1mg5w%~wV$ck ze|8Fecm`($c-H^QZM6G1^UsO9_=ZpTCjZD(qkU+caV7XK8eFXc*J8qd#)1X;i3e9@ zCE*W}i$Ek^aLpucVza3~S;>?F`hSA6KvNQjoJrgeGDq}CayfWV%;D%B5$^(Yf=vRa zV1Q&zm^9);#ISH{bCKh# zwLt%uc8j)!uBn!-YopWF&(s{({H{~#-3=X(6Mjoys*f=oX?SPq0dHYj+gkez+bUZf zyf?310@NglvB&=DKJV#EI!F=?>OP6`E>55us)g2$BsbvP}dLp4WK%mHcvA@wpQB)*uA#Swh`8HWKC7yeHwFhWi_*{)Zvo3Oe0*yD9Vv~?t50O@bi1h6{fK{qOWN?yAb+!5V@XGLm2~(ilNA@2_T0dCy_+F3BTG?Z<$X3 zA8u#jUcz{+kgu>_p7A#GJacVxYMnZFiMJZeVFL)SyfZy3T-}`gT_4!eJt!?pSx$lJ6h=M?f&SwgSBvz_m%rS(0=Il z{?;GXi}15=w{x6M=P>vujNr^Fw>S(6LzHo~p^N#wMTJ=X4f9~rIAft9&6sJp+E8Yc z8wZ$c455ZMh6=D1pET@fC}_wvF4ynU4>ZOay`}~$(X`3%!W3ah)t}X7Yohdn4Ob0s zj9ZO!%)|=k~S-AW7fcosVTJhQ8Cf6zsJu{H#VJ=dpY-5PV20gwCTws6R*YX zj+>S!OpZ@pm2xsUEKV7@FYJ0qVn|lV+2C)I!-3-hdy7vA()f3He{=n@p}x$m^h{K8 z2hfhghrfV+pWYG~whCHP%0SXq;#~53>KH~_&K7Py>WU)RnaoOLP!}*Ca*p%&h~9~l zfe~mIlym2yHsCc^jfzY+uwRAA8YM@4QRS@hlnwtTDmn8_@l9W5EhAJMQt@mPjnT>j@|?=c zpT`tuSFO-EEK7YpnvPsgETL37HtJRy9geo-H^Plkk@2&V`Xq$P`A}jkL?z4`5v0jSY&v9-$941~vo69$R=)uu?Er zaFXAJ`<~UDQ9?Lx&vt}3D7FEP#XwG*M|esy5M*|Pu?97wT@0seqXB$gak#MjP4WDH}++uHSBBn;)tl2(!_6R`I!w_Q<^-;F31Q=Uy(8@ z@kCrh@c2k2YX)6_{Si)uz)f6c$zwz?)-s%8QB7PQJP@(TGdRdW3V?R?!J z{d8S{uD#ZyexSarE~+1{BkG*`ekPUavT?6IT}RTFse9M4b>*7j+Is!`hH%Xt^&jf@ zH93`{>YU1Z)eGc$`A2Ebsyt^X^>ALDV>-X#4=@;wk zwZ*zs`Z@Y;n(gY-^>6F)Cxy}PjPt?13jS5uyP9!-9->~V|7yPmu&KCc>FG57m|vckd}&sIP0^S*z{rLAnTD{X9C@31|=rz2?5U9!||Rp}r%NR_>`_SM8F8X(ySSj;`THv!5<=uqNl_r$8AV#k^DN9kkK=9Puw~#@@fm(K<2|X?bKkKX z2R|zD#{y`slB_+i$PAz54!tPiB3;ROS3q^x07K z^S$XA`_a0>y``3)8JZdHP+wnqny;;Ch^9uf#PfnSU;H@iT+G3=ujmnO+OHULe9M$6 zDUN1OTTNnnvvT#V2`%R4lx5sZU6jI$uZ?aVwllCxMEmGxVY<+Wz$nf- zBFz$`+h_RKY`0x?bg?kaTTNXx!=(>?y({sS#8jND>7;0{nx*clJS2BXAIZa%chzgP zXBvV`4$B-5op1xI*G^g)pCJ*4ObexjKM4O4nH^IXZ%y+yJ)OsC715@oWq7L>c}<%I zD+8Xo}Ty)Q9V9wA&hTO)XL7zsxX3pRdo-5gRmy z3#c=nZME54mr~y`1p;=<~NvS3WrUaDTy+yI1a=yOU7x?Cq7(^(AjU zhZn6b&MTbva_9Tr#mutV(oV+Zu4}|UQT6@ZbJe|oe33m}*gg0}NEev{Hl4h%#sy?jDlgXu5rF~^$`7V{Vez|&z%nS^b!rC(mX+6bs z!ST&YCdvse@)$}uC5?K4K9KK_EDC=Z-8)7To0~K)xj5xwh9Env>G`HJo9eUGSue9D zW*y4?yQYq;b9@t~AF-RLyTQ zsq}ZX!?jnnqjf3z6?&7dQX8VlR}WCmRCHD5s%q=j>3SI)MitO7;u{$H;SG6)>xPZS z9P?(>qScsAn${VIntq!9KyFWrX}{ruUR!rUm8qJeo}wP56e*5NU)OH@HMjiV(ji|5 ze&-ig6}|nk=hO1{nQy+o`2EH4r?($Zc{=!UuUFKcs-LCBn~Se~-CIU1I{#i(`dh{L z>f!2Xns6{UcK0kMbaM-7L--caL&+!6#=yAHi{aA~^AozKd~EWiSy{{KmK*cj&2zJ7 zXADgLnDQZWdQQ7$ZL-9vOOyF2(Mjder$Uwl?F?KezQ!NTbWvxJjGncK{>}qip0qWxVC=F4+RKL~u%KF4aGlrSGru+61xGo+4+aW0buau)co)S>c|(rEYzlSoNo}L}^gXQPSiKrG0A_ z)m*FnBHN&7tISkiSBupz)QNQ|b#|phbqzI`6SN^<$Y`R?)I@0Z>*nc<+RvKJ^+k0e zZswF=AR` z|MaNLw^=2bA2N0(FH9(mI}&?2{?CMku|q@Hfrj7@A-lu!LV5;_4}2t!7Ix)&$d|l4 zSE2Q-t*>*OYrT7ptB@g)P7_-Zs<0CMHp@v-}Ep#e8V69szU1>dBWfXn;2v#INutc;j4xH5B@7KSMVFVD=?FKQ4fQ?HrL1X{B&-#cQS`I{HK-GeN}Z+ zu2PmMl4QTt2G|llrFFR zuXbwfc!aV7YVs?~D-KnRu54DpEB~kbaM|+mnLiema7q`IT>G~1>xbf;ulv3>s}RY$ z%JemDWfk?`8oq0WX)ZS8S~eiJDb!Kxyx`qLS&8c42>u^IcOs5N=7%4ObjQq2h)<#> zOiDbQ{49A+(&G3<2@Oe(q{^gmNyC#;lCLNAOV}Gt2|F8nB}gSn517sMG562|D04`Y z!1>+V*ACHEzFXi}VmoTxZ0=%6Z}{8L&Tz-@3LIaZ48siNmYLw&E(AK~MDkp!2|O@; zQ77%-t`e*e{t8e8^TNpCqKIKJa}qL>-=rE+&!t{TotgS5^>K)R=1=Aj*xj_nS{TKDu>s0FP z$|>?DvXQ{_nqE7)c3$m+n&6uJ8gA`0>A$kGsKl!OpDL7|sVm4B1%ka2*uBa;@@ z4ypOQT3mCd=129ts%6!Es#{id|F!UEMfsre1!b*&oc=uJv$8zAIvyCpucg^_3{