mirror of
https://github.com/k2-fsa/icefall.git
synced 2025-12-09 14:05:33 +00:00
Merge branch 'k2-fsa:master' into dev/k2ssl
This commit is contained in:
Yifan Yang
GitHub
commit
893fee43d4
@ -333,6 +333,7 @@ We provide a Colab notebook to test the pre-trained model: [;
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
"""
|
||||
This file computes fbank features of the musan dataset.
|
||||
It looks for manifests in the directory `src_dir` (default is data/manifests).
|
||||
|
||||
The generated fbank features are saved in data/fbank.
|
||||
"""
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import torch
|
||||
from lhotse import (
|
||||
CutSet,
|
||||
Fbank,
|
||||
FbankConfig,
|
||||
LilcomChunkyWriter,
|
||||
MonoCut,
|
||||
WhisperFbank,
|
||||
WhisperFbankConfig,
|
||||
combine,
|
||||
)
|
||||
from lhotse.recipes.utils import read_manifests_if_cached
|
||||
|
||||
from icefall.utils import get_executor, str2bool
|
||||
|
||||
# Torch's multithreaded behavior needs to be disabled or
|
||||
# it wastes a lot of CPU and slow things down.
|
||||
# Do this outside of main() in case it needs to take effect
|
||||
# even when we are not invoking the main (e.g. when spawning subprocesses).
|
||||
torch.set_num_threads(1)
|
||||
torch.set_num_interop_threads(1)
|
||||
|
||||
|
||||
def is_cut_long(c: MonoCut) -> bool:
|
||||
return c.duration > 5
|
||||
|
||||
|
||||
def compute_fbank_musan(
|
||||
src_dir: str = "data/manifests",
|
||||
num_mel_bins: int = 80,
|
||||
whisper_fbank: bool = False,
|
||||
output_dir: str = "data/fbank",
|
||||
):
|
||||
src_dir = Path(src_dir)
|
||||
output_dir = Path(output_dir)
|
||||
num_jobs = min(15, os.cpu_count())
|
||||
|
||||
dataset_parts = (
|
||||
"music",
|
||||
"speech",
|
||||
"noise",
|
||||
)
|
||||
prefix = "musan"
|
||||
suffix = "jsonl.gz"
|
||||
manifests = read_manifests_if_cached(
|
||||
dataset_parts=dataset_parts,
|
||||
output_dir=src_dir,
|
||||
prefix=prefix,
|
||||
suffix=suffix,
|
||||
)
|
||||
assert manifests is not None
|
||||
|
||||
assert len(manifests) == len(dataset_parts), (
|
||||
len(manifests),
|
||||
len(dataset_parts),
|
||||
list(manifests.keys()),
|
||||
dataset_parts,
|
||||
)
|
||||
|
||||
musan_cuts_path = output_dir / "musan_cuts.jsonl.gz"
|
||||
|
||||
if musan_cuts_path.is_file():
|
||||
logging.info(f"{musan_cuts_path} already exists - skipping")
|
||||
return
|
||||
|
||||
logging.info("Extracting features for Musan")
|
||||
|
||||
if whisper_fbank:
|
||||
extractor = WhisperFbank(
|
||||
WhisperFbankConfig(num_filters=num_mel_bins, device="cuda")
|
||||
)
|
||||
else:
|
||||
extractor = Fbank(FbankConfig(num_mel_bins=num_mel_bins))
|
||||
|
||||
with get_executor() as ex: # Initialize the executor only once.
|
||||
# create chunks of Musan with duration 5 - 10 seconds
|
||||
musan_cuts = (
|
||||
CutSet.from_manifests(
|
||||
recordings=combine(part["recordings"] for part in manifests.values())
|
||||
)
|
||||
.cut_into_windows(10.0)
|
||||
.filter(is_cut_long)
|
||||
.compute_and_store_features(
|
||||
extractor=extractor,
|
||||
storage_path=f"{output_dir}/musan_feats",
|
||||
num_jobs=num_jobs if ex is None else 80,
|
||||
executor=ex,
|
||||
storage_type=LilcomChunkyWriter,
|
||||
)
|
||||
)
|
||||
musan_cuts.to_file(musan_cuts_path)
|
||||
|
||||
|
||||
def get_args():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"--src-dir",
|
||||
type=str,
|
||||
default="data/manifests",
|
||||
help="Source manifests directory.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--num-mel-bins",
|
||||
type=int,
|
||||
default=80,
|
||||
help="""The number of mel bins for Fbank""",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--whisper-fbank",
|
||||
type=str2bool,
|
||||
default=False,
|
||||
help="Use WhisperFbank instead of Fbank. Default: False.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-dir",
|
||||
type=str,
|
||||
default="data/fbank",
|
||||
help="Output directory. Default: data/fbank.",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
formatter = "%(asctime)s %(levelname)s [%(filename)s:%(lineno)d] %(message)s"
|
||||
|
||||
logging.basicConfig(format=formatter, level=logging.INFO)
|
||||
args = get_args()
|
||||
compute_fbank_musan(
|
||||
src_dir=args.src_dir,
|
||||
num_mel_bins=args.num_mel_bins,
|
||||
whisper_fbank=args.whisper_fbank,
|
||||
output_dir=args.output_dir,
|
||||
)
|
||||
1
egs/ksponspeech/ASR/local/compute_fbank_musan.py
Symbolic link
1
egs/ksponspeech/ASR/local/compute_fbank_musan.py
Symbolic link
@ -0,0 +1 @@
|
||||
../../../librispeech/ASR/local/compute_fbank_musan.py
|
||||
@ -1,157 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""
|
||||
This script removes short and long utterances from a cutset.
|
||||
|
||||
Caution:
|
||||
You may need to tune the thresholds for your own dataset.
|
||||
|
||||
Usage example:
|
||||
|
||||
python3 ./local/filter_cuts.py \
|
||||
--bpe-model data/lang_bpe_5000/bpe.model \
|
||||
--in-cuts data/fbank/speechtools_cuts_test.jsonl.gz \
|
||||
--out-cuts data/fbank-filtered/speechtools_cuts_test.jsonl.gz
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
import sentencepiece as spm
|
||||
from lhotse import CutSet, load_manifest_lazy
|
||||
from lhotse.cut import Cut
|
||||
|
||||
|
||||
def get_args():
|
||||
parser = argparse.ArgumentParser()
|
||||
|
||||
parser.add_argument(
|
||||
"--bpe-model",
|
||||
type=Path,
|
||||
help="Path to the bpe.model",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--in-cuts",
|
||||
type=Path,
|
||||
help="Path to the input cutset",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--out-cuts",
|
||||
type=Path,
|
||||
help="Path to the output cutset",
|
||||
)
|
||||
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def filter_cuts(cut_set: CutSet, sp: spm.SentencePieceProcessor):
|
||||
total = 0 # number of total utterances before removal
|
||||
removed = 0 # number of removed utterances
|
||||
|
||||
def remove_short_and_long_utterances(c: Cut):
|
||||
"""Return False to exclude the input cut"""
|
||||
nonlocal removed, total
|
||||
# Keep only utterances with duration between 1 second and 20 seconds
|
||||
#
|
||||
# Caution: There is a reason to select 20.0 here. Please see
|
||||
# ./display_manifest_statistics.py
|
||||
#
|
||||
# You should use ./display_manifest_statistics.py to get
|
||||
# an utterance duration distribution for your dataset to select
|
||||
# the threshold
|
||||
total += 1
|
||||
if c.duration < 1.0 or c.duration > 20.0:
|
||||
logging.warning(
|
||||
f"Exclude cut with ID {c.id} from training. Duration: {c.duration}"
|
||||
)
|
||||
removed += 1
|
||||
return False
|
||||
|
||||
# In pruned RNN-T, we require that T >= S
|
||||
# where T is the number of feature frames after subsampling
|
||||
# and S is the number of tokens in the utterance
|
||||
|
||||
# In ./pruned_transducer_stateless2/conformer.py, the
|
||||
# conv module uses the following expression
|
||||
# for subsampling
|
||||
if c.num_frames is None:
|
||||
num_frames = c.duration * 100 # approximate
|
||||
else:
|
||||
num_frames = c.num_frames
|
||||
|
||||
T = ((num_frames - 1) // 2 - 1) // 2
|
||||
# Note: for ./lstm_transducer_stateless/lstm.py, the formula is
|
||||
# T = ((num_frames - 3) // 2 - 1) // 2
|
||||
|
||||
# Note: for ./pruned_transducer_stateless7/zipformer.py, the formula is
|
||||
# T = ((num_frames - 7) // 2 + 1) // 2
|
||||
|
||||
tokens = sp.encode(c.supervisions[0].text, out_type=str)
|
||||
|
||||
if T < len(tokens):
|
||||
logging.warning(
|
||||
f"Exclude cut with ID {c.id} from training. "
|
||||
f"Number of frames (before subsampling): {c.num_frames}. "
|
||||
f"Number of frames (after subsampling): {T}. "
|
||||
f"Text: {c.supervisions[0].text}. "
|
||||
f"Tokens: {tokens}. "
|
||||
f"Number of tokens: {len(tokens)}"
|
||||
)
|
||||
removed += 1
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
# We use to_eager() here so that we can print out the value of total
|
||||
# and removed below.
|
||||
ans = cut_set.filter(remove_short_and_long_utterances).to_eager()
|
||||
ratio = removed / total * 100
|
||||
logging.info(
|
||||
f"Removed {removed} cuts from {total} cuts. {ratio:.3f}% data is removed."
|
||||
)
|
||||
return ans
|
||||
|
||||
|
||||
def main():
|
||||
args = get_args()
|
||||
logging.info(vars(args))
|
||||
|
||||
if args.out_cuts.is_file():
|
||||
logging.info(f"{args.out_cuts} already exists - skipping")
|
||||
return
|
||||
|
||||
assert args.in_cuts.is_file(), f"{args.in_cuts} does not exist"
|
||||
assert args.bpe_model.is_file(), f"{args.bpe_model} does not exist"
|
||||
|
||||
sp = spm.SentencePieceProcessor()
|
||||
sp.load(str(args.bpe_model))
|
||||
|
||||
cut_set = load_manifest_lazy(args.in_cuts)
|
||||
assert isinstance(cut_set, CutSet)
|
||||
|
||||
cut_set = filter_cuts(cut_set, sp)
|
||||
logging.info(f"Saving to {args.out_cuts}")
|
||||
args.out_cuts.parent.mkdir(parents=True, exist_ok=True)
|
||||
cut_set.to_file(args.out_cuts)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
formatter = "%(asctime)s %(levelname)s [%(filename)s:%(lineno)d] %(message)s"
|
||||
|
||||
logging.basicConfig(format=formatter, level=logging.INFO)
|
||||
|
||||
main()
|
||||
1
egs/ksponspeech/ASR/local/filter_cuts.py
Symbolic link
1
egs/ksponspeech/ASR/local/filter_cuts.py
Symbolic link
@ -0,0 +1 @@
|
||||
../../../librispeech/ASR/local/filter_cuts.py
|
||||
@ -1,115 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright 2021 Xiaomi Corp. (authors: Fangjun Kuang)
|
||||
#
|
||||
# See ../../../../LICENSE for clarification regarding multiple authors
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
# You can install sentencepiece via:
|
||||
#
|
||||
# pip install sentencepiece
|
||||
#
|
||||
# Due to an issue reported in
|
||||
# https://github.com/google/sentencepiece/pull/642#issuecomment-857972030
|
||||
#
|
||||
# Please install a version >=0.1.96
|
||||
|
||||
import argparse
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from typing import Dict
|
||||
|
||||
import sentencepiece as spm
|
||||
|
||||
|
||||
def get_args():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"--lang-dir",
|
||||
type=str,
|
||||
help="""Input and output directory.
|
||||
The generated bpe.model is saved to this directory.
|
||||
""",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--transcript",
|
||||
type=str,
|
||||
help="Training transcript.",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--vocab-size",
|
||||
type=int,
|
||||
help="Vocabulary size for BPE training",
|
||||
)
|
||||
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def generate_tokens(lang_dir: Path):
|
||||
"""
|
||||
Generate the tokens.txt from a bpe model.
|
||||
"""
|
||||
sp = spm.SentencePieceProcessor()
|
||||
sp.load(str(lang_dir / "bpe.model"))
|
||||
token2id: Dict[str, int] = {sp.id_to_piece(i): i for i in range(sp.vocab_size())}
|
||||
with open(lang_dir / "tokens.txt", "w", encoding="utf-8") as f:
|
||||
for sym, i in token2id.items():
|
||||
f.write(f"{sym} {i}\n")
|
||||
|
||||
|
||||
def main():
|
||||
args = get_args()
|
||||
vocab_size = args.vocab_size
|
||||
lang_dir = Path(args.lang_dir)
|
||||
|
||||
model_type = "unigram"
|
||||
|
||||
model_prefix = f"{lang_dir}/{model_type}_{vocab_size}"
|
||||
train_text = args.transcript
|
||||
character_coverage = 1.0
|
||||
input_sentence_size = 100000000
|
||||
|
||||
user_defined_symbols = ["<blk>", "<sos/eos>"]
|
||||
unk_id = len(user_defined_symbols)
|
||||
# Note: unk_id is fixed to 2.
|
||||
# If you change it, you should also change other
|
||||
# places that are using it.
|
||||
|
||||
model_file = Path(model_prefix + ".model")
|
||||
if not model_file.is_file():
|
||||
spm.SentencePieceTrainer.train(
|
||||
input=train_text,
|
||||
vocab_size=vocab_size,
|
||||
model_type=model_type,
|
||||
model_prefix=model_prefix,
|
||||
input_sentence_size=input_sentence_size,
|
||||
character_coverage=character_coverage,
|
||||
user_defined_symbols=user_defined_symbols,
|
||||
unk_id=unk_id,
|
||||
bos_id=-1,
|
||||
eos_id=-1,
|
||||
)
|
||||
else:
|
||||
print(f"{model_file} exists - skipping")
|
||||
return
|
||||
|
||||
shutil.copyfile(model_file, f"{lang_dir}/bpe.model")
|
||||
|
||||
generate_tokens(lang_dir)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
1
egs/ksponspeech/ASR/local/train_bpe_model.py
Symbolic link
1
egs/ksponspeech/ASR/local/train_bpe_model.py
Symbolic link
@ -0,0 +1 @@
|
||||
../../../librispeech/ASR/local/train_bpe_model.py
|
||||
@ -1,101 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""
|
||||
This script checks the following assumptions of the generated manifest:
|
||||
|
||||
- Single supervision per cut
|
||||
- Supervision time bounds are within cut time bounds
|
||||
|
||||
We will add more checks later if needed.
|
||||
|
||||
Usage example:
|
||||
|
||||
python3 ./local/validate_manifest.py \
|
||||
./data/fbank/speechtools_cuts_train.jsonl.gz
|
||||
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
from lhotse import CutSet, load_manifest_lazy
|
||||
from lhotse.cut import Cut
|
||||
from lhotse.dataset.speech_recognition import validate_for_asr
|
||||
|
||||
|
||||
def get_args():
|
||||
parser = argparse.ArgumentParser()
|
||||
|
||||
parser.add_argument(
|
||||
"manifest",
|
||||
type=Path,
|
||||
help="Path to the manifest file",
|
||||
)
|
||||
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def validate_one_supervision_per_cut(c: Cut):
|
||||
if len(c.supervisions) != 1:
|
||||
raise ValueError(f"{c.id} has {len(c.supervisions)} supervisions")
|
||||
|
||||
|
||||
def validate_supervision_and_cut_time_bounds(c: Cut):
|
||||
tol = 2e-3 # same tolerance as in 'validate_for_asr()'
|
||||
s = c.supervisions[0]
|
||||
|
||||
# Supervision start time is relative to Cut ...
|
||||
# https://lhotse.readthedocs.io/en/v0.10_e/cuts.html
|
||||
if s.start < -tol:
|
||||
raise ValueError(
|
||||
f"{c.id}: Supervision start time {s.start} must not be negative."
|
||||
)
|
||||
if s.start > tol:
|
||||
raise ValueError(
|
||||
f"{c.id}: Supervision start time {s.start} is not at the beginning of the Cut. Please apply `lhotse cut trim-to-supervisions`."
|
||||
)
|
||||
if c.start + s.end > c.end + tol:
|
||||
raise ValueError(
|
||||
f"{c.id}: Supervision end time {c.start+s.end} is larger "
|
||||
f"than cut end time {c.end}"
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
args = get_args()
|
||||
|
||||
manifest = args.manifest
|
||||
logging.info(f"Validating {manifest}")
|
||||
|
||||
assert manifest.is_file(), f"{manifest} does not exist"
|
||||
cut_set = load_manifest_lazy(manifest)
|
||||
assert isinstance(cut_set, CutSet)
|
||||
|
||||
for c in cut_set:
|
||||
validate_one_supervision_per_cut(c)
|
||||
validate_supervision_and_cut_time_bounds(c)
|
||||
|
||||
# Validation from K2 training
|
||||
# - checks supervision start is 0
|
||||
# - checks supervision.duration is not longer than cut.duration
|
||||
# - there is tolerance 2ms
|
||||
validate_for_asr(cut_set)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
formatter = "%(asctime)s %(levelname)s [%(filename)s:%(lineno)d] %(message)s"
|
||||
|
||||
logging.basicConfig(format=formatter, level=logging.INFO)
|
||||
|
||||
main()
|
||||
1
egs/ksponspeech/ASR/local/validate_manifest.py
Symbolic link
1
egs/ksponspeech/ASR/local/validate_manifest.py
Symbolic link
@ -0,0 +1 @@
|
||||
../../../librispeech/ASR/local/validate_manifest.py
|
||||
@ -1 +0,0 @@
|
||||
This recipe implements Zipformer model.
|
||||
@ -138,7 +138,7 @@ def get_parser():
|
||||
parser.add_argument(
|
||||
"--save-every-n",
|
||||
type=int,
|
||||
default=1,
|
||||
default=5,
|
||||
help="""Save checkpoint after processing this number of epochs"
|
||||
periodically. We save checkpoint to exp-dir/ whenever
|
||||
params.cur_epoch % save_every_n == 0. The checkpoint filename
|
||||
@ -1093,14 +1093,14 @@ def run(rank, world_size, args):
|
||||
rank=rank,
|
||||
)
|
||||
|
||||
# if not params.print_diagnostics:
|
||||
# scan_pessimistic_batches_for_oom(
|
||||
# model=model,
|
||||
# train_dl=train_dl,
|
||||
# optimizer_g=optimizer_g,
|
||||
# optimizer_d=optimizer_d,
|
||||
# params=params,
|
||||
# )
|
||||
if not params.print_diagnostics:
|
||||
scan_pessimistic_batches_for_oom(
|
||||
model=model,
|
||||
train_dl=train_dl,
|
||||
optimizer_g=optimizer_g,
|
||||
optimizer_d=optimizer_d,
|
||||
params=params,
|
||||
)
|
||||
|
||||
scaler = GradScaler(enabled=params.use_fp16, init_scale=1.0)
|
||||
if checkpoints and "grad_scaler" in checkpoints:
|
||||
|
||||
@ -45,12 +45,11 @@ if [ $stage -le 1 ] && [ $stop_stage -ge 1 ]; then
|
||||
# to $dl_dir/LibriTTS
|
||||
mkdir -p data/manifests
|
||||
if [ ! -e data/manifests/.libritts.done ]; then
|
||||
lhotse prepare libritts --num-jobs 32 $dl_dir/LibriTTS data/manifests
|
||||
lhotse prepare libritts --num-jobs ${nj} $dl_dir/LibriTTS data/manifests
|
||||
touch data/manifests/.libritts.done
|
||||
fi
|
||||
fi
|
||||
|
||||
|
||||
if [ $stage -le 2 ] && [ $stop_stage -ge 2 ]; then
|
||||
log "Stage 2: Compute Spectrogram for LibriTTS"
|
||||
mkdir -p data/spectrogram
|
||||
@ -64,7 +63,7 @@ if [ $stage -le 2 ] && [ $stop_stage -ge 2 ]; then
|
||||
if [ ! -f data/spectrogram/libritts_cuts_train-all-shuf.jsonl.gz ]; then
|
||||
cat <(gunzip -c data/spectrogram/libritts_cuts_train-clean-100.jsonl.gz) \
|
||||
<(gunzip -c data/spectrogram/libritts_cuts_train-clean-360.jsonl.gz) \
|
||||
<(gunzip -c /data/spectrogramlibritts_cuts_train-other-500.jsonl.gz) | \
|
||||
<(gunzip -c data/spectrogramlibritts_cuts_train-other-500.jsonl.gz) | \
|
||||
shuf | gzip -c > data/spectrogram/libritts_cuts_train-all-shuf.jsonl.gz
|
||||
fi
|
||||
|
||||
|
||||
51
egs/libritts/TTS/README.md
Normal file
51
egs/libritts/TTS/README.md
Normal file
@ -0,0 +1,51 @@
|
||||
# Introduction
|
||||
|
||||
LibriTTS is a multi-speaker English corpus of approximately 585 hours of read English speech at 24kHz sampling rate, prepared by Heiga Zen with the assistance of Google Speech and Google Brain team members.
|
||||
The LibriTTS corpus is designed for TTS research. It is derived from the original materials (mp3 audio files from LibriVox and text files from Project Gutenberg) of the LibriSpeech corpus.
|
||||
The main differences from the LibriSpeech corpus are listed below:
|
||||
1. The audio files are at 24kHz sampling rate.
|
||||
2. The speech is split at sentence breaks.
|
||||
3. Both original and normalized texts are included.
|
||||
4. Contextual information (e.g., neighbouring sentences) can be extracted.
|
||||
5. Utterances with significant background noise are excluded.
|
||||
For more information, refer to the paper "LibriTTS: A Corpus Derived from LibriSpeech for Text-to-Speech", Heiga Zen, Viet Dang, Rob Clark, Yu Zhang, Ron J. Weiss, Ye Jia, Zhifeng Chen, and Yonghui Wu, arXiv, 2019. If you use the LibriTTS corpus in your work, please cite this paper where it was introduced.
|
||||
|
||||
> [!CAUTION]
|
||||
> The next-gen Kaldi framework provides tools and models for generating high-quality, synthetic speech (Text-to-Speech, TTS).
|
||||
> While these recipes has the potential to advance various fields such as accessibility, language education, and AI-driven solutions, it also carries certain ethical and legal responsibilities.
|
||||
>
|
||||
> By using this framework, you agree to the following:
|
||||
> 1. Legal and Ethical Use: You shall not use this framework, or any models derived from it, for any unlawful or unethical purposes. This includes, but is not limited to: Creating voice clones without the explicit, informed consent of the individual whose voice is being cloned. Engaging in any form of identity theft, impersonation, or fraud using cloned voices. Violating any local, national, or international laws regarding privacy, intellectual property, or personal data.
|
||||
>
|
||||
> 2. Responsibility of Use: The users of this framework are solely responsible for ensuring that their use of voice cloning technologies complies with all applicable laws and ethical guidelines. We explicitly disclaim any liability for misuse of the technology.
|
||||
>
|
||||
> 3. Attribution and Use of Open-Source Components: This project is provided under the Apache 2.0 license. Users must adhere to the terms of this license and provide appropriate attribution when required.
|
||||
>
|
||||
> 4. No Warranty: This framework is provided “as-is,” without warranty of any kind, either express or implied. We do not guarantee that the use of this software will comply with legal requirements or that it will not infringe the rights of third parties.
|
||||
|
||||
|
||||
# VITS
|
||||
|
||||
This recipe provides a VITS model trained on the LibriTTS dataset.
|
||||
|
||||
Pretrained model can be found [here](https://huggingface.co/zrjin/icefall-tts-libritts-vits-2024-10-30).
|
||||
|
||||
The training command is given below:
|
||||
```
|
||||
export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
|
||||
./vits/train.py \
|
||||
--world-size 4 \
|
||||
--num-epochs 400 \
|
||||
--start-epoch 1 \
|
||||
--use-fp16 1 \
|
||||
--exp-dir vits/exp \
|
||||
--max-duration 500
|
||||
```
|
||||
|
||||
To inference, use:
|
||||
```
|
||||
./vits/infer.py \
|
||||
--exp-dir vits/exp \
|
||||
--epoch 400 \
|
||||
--tokens data/tokens.txt
|
||||
```
|
||||
1
egs/libritts/TTS/local/compute_spectrogram_libritts.py
Symbolic link
1
egs/libritts/TTS/local/compute_spectrogram_libritts.py
Symbolic link
@ -0,0 +1 @@
|
||||
../../CODEC/local/compute_spectrogram_libritts.py
|
||||
1
egs/libritts/TTS/local/prepare_token_file.py
Symbolic link
1
egs/libritts/TTS/local/prepare_token_file.py
Symbolic link
@ -0,0 +1 @@
|
||||
../../../ljspeech/TTS/local/prepare_token_file.py
|
||||
89
egs/libritts/TTS/local/prepare_tokens_libritts.py
Executable file
89
egs/libritts/TTS/local/prepare_tokens_libritts.py
Executable file
@ -0,0 +1,89 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright 2023 Xiaomi Corp. (authors: Zengwei Yao,
|
||||
# Zengrui Jin,)
|
||||
# 2024 Tsinghua University (authors: Zengrui Jin,)
|
||||
#
|
||||
# See ../../../../LICENSE for clarification regarding multiple authors
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
"""
|
||||
This file reads the texts in given manifest and save the new cuts with phoneme tokens.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
import tacotron_cleaner.cleaners
|
||||
from lhotse import CutSet, load_manifest
|
||||
from piper_phonemize import phonemize_espeak
|
||||
from tqdm.auto import tqdm
|
||||
|
||||
|
||||
def remove_punc_to_upper(text: str) -> str:
|
||||
text = text.replace("‘", "'")
|
||||
text = text.replace("’", "'")
|
||||
tokens = set("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'")
|
||||
s_list = [x.upper() if x in tokens else " " for x in text]
|
||||
s = " ".join("".join(s_list).split()).strip()
|
||||
return s
|
||||
|
||||
|
||||
def prepare_tokens_libritts():
|
||||
output_dir = Path("data/spectrogram")
|
||||
prefix = "libritts"
|
||||
suffix = "jsonl.gz"
|
||||
partitions = (
|
||||
"dev-clean",
|
||||
"dev-other",
|
||||
"test-clean",
|
||||
"test-other",
|
||||
"train-all-shuf",
|
||||
"train-clean-460",
|
||||
# "train-clean-100",
|
||||
# "train-clean-360",
|
||||
# "train-other-500",
|
||||
)
|
||||
|
||||
for partition in partitions:
|
||||
cut_set = load_manifest(output_dir / f"{prefix}_cuts_{partition}.{suffix}")
|
||||
|
||||
new_cuts = []
|
||||
for cut in tqdm(cut_set):
|
||||
# Each cut only contains one supervision
|
||||
assert len(cut.supervisions) == 1, (len(cut.supervisions), cut)
|
||||
text = cut.supervisions[0].text
|
||||
# Text normalization
|
||||
text = tacotron_cleaner.cleaners.custom_english_cleaners(text)
|
||||
# Convert to phonemes
|
||||
tokens_list = phonemize_espeak(text, "en-us")
|
||||
tokens = []
|
||||
for t in tokens_list:
|
||||
tokens.extend(t)
|
||||
cut.tokens = tokens
|
||||
cut.supervisions[0].normalized_text = remove_punc_to_upper(text)
|
||||
|
||||
new_cuts.append(cut)
|
||||
|
||||
new_cut_set = CutSet.from_cuts(new_cuts)
|
||||
new_cut_set.to_file(
|
||||
output_dir / f"{prefix}_cuts_with_tokens_{partition}.{suffix}"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
formatter = "%(asctime)s %(levelname)s [%(filename)s:%(lineno)d] %(message)s"
|
||||
logging.basicConfig(format=formatter, level=logging.INFO)
|
||||
|
||||
prepare_tokens_libritts()
|
||||
1
egs/libritts/TTS/local/validate_manifest.py
Symbolic link
1
egs/libritts/TTS/local/validate_manifest.py
Symbolic link
@ -0,0 +1 @@
|
||||
../../../ljspeech/TTS/local/validate_manifest.py
|
||||
134
egs/libritts/TTS/prepare.sh
Executable file
134
egs/libritts/TTS/prepare.sh
Executable file
@ -0,0 +1,134 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# fix segmentation fault reported in https://github.com/k2-fsa/icefall/issues/674
|
||||
export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
|
||||
|
||||
set -eou pipefail
|
||||
|
||||
stage=0
|
||||
stop_stage=100
|
||||
sampling_rate=24000
|
||||
nj=32
|
||||
|
||||
dl_dir=$PWD/download
|
||||
|
||||
. shared/parse_options.sh || exit 1
|
||||
|
||||
# All files generated by this script are saved in "data".
|
||||
# You can safely remove "data" and rerun this script to regenerate it.
|
||||
mkdir -p data
|
||||
|
||||
log() {
|
||||
# This function is from espnet
|
||||
local fname=${BASH_SOURCE[1]##*/}
|
||||
echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*"
|
||||
}
|
||||
|
||||
log "dl_dir: $dl_dir"
|
||||
|
||||
if [ $stage -le -1 ] && [ $stop_stage -ge -1 ]; then
|
||||
log "Stage -1: build monotonic_align lib"
|
||||
if [ ! -d vits/monotonic_align/build ]; then
|
||||
cd vits/monotonic_align
|
||||
python setup.py build_ext --inplace
|
||||
cd ../../
|
||||
else
|
||||
log "monotonic_align lib already built"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ $stage -le 0 ] && [ $stop_stage -ge 0 ]; then
|
||||
log "Stage 0: Download data"
|
||||
|
||||
# If you have pre-downloaded it to /path/to/LibriTTS,
|
||||
# you can create a symlink
|
||||
#
|
||||
# ln -sfv /path/to/LibriTTS $dl_dir/LibriTTS
|
||||
#
|
||||
if [ ! -d $dl_dir/LibriTTS ]; then
|
||||
lhotse download libritts $dl_dir
|
||||
fi
|
||||
|
||||
if [ ! -d $dl_dir/xvector_nnet_1a_libritts_clean_460 ]; then
|
||||
log "Downloading x-vector"
|
||||
|
||||
git clone https://huggingface.co/datasets/zrjin/xvector_nnet_1a_libritts_clean_460 $dl_dir/xvector_nnet_1a_libritts_clean_460
|
||||
|
||||
mkdir -p exp/xvector_nnet_1a/
|
||||
cp -r $dl_dir/xvector_nnet_1a_libritts_clean_460/* exp/xvector_nnet_1a/
|
||||
fi
|
||||
|
||||
fi
|
||||
|
||||
if [ $stage -le 1 ] && [ $stop_stage -ge 1 ]; then
|
||||
log "Stage 1: Prepare LibriTTS manifest"
|
||||
# We assume that you have downloaded the LibriTTS corpus
|
||||
# to $dl_dir/LibriTTS
|
||||
mkdir -p data/manifests
|
||||
if [ ! -e data/manifests/.libritts.done ]; then
|
||||
lhotse prepare libritts --num-jobs ${nj} $dl_dir/LibriTTS data/manifests
|
||||
touch data/manifests/.libritts.done
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ $stage -le 2 ] && [ $stop_stage -ge 2 ]; then
|
||||
log "Stage 2: Compute Spectrogram for LibriTTS"
|
||||
mkdir -p data/spectrogram
|
||||
if [ ! -e data/spectrogram/.libritts.done ]; then
|
||||
./local/compute_spectrogram_libritts.py --sampling-rate $sampling_rate
|
||||
touch data/spectrogram/.libritts.done
|
||||
fi
|
||||
|
||||
# Here we shuffle and combine the train-clean-100, train-clean-360 and
|
||||
# train-other-500 together to form the training set.
|
||||
if [ ! -f data/spectrogram/libritts_cuts_train-all-shuf.jsonl.gz ]; then
|
||||
cat <(gunzip -c data/spectrogram/libritts_cuts_train-clean-100.jsonl.gz) \
|
||||
<(gunzip -c data/spectrogram/libritts_cuts_train-clean-360.jsonl.gz) \
|
||||
<(gunzip -c data/spectrogramlibritts_cuts_train-other-500.jsonl.gz) | \
|
||||
shuf | gzip -c > data/spectrogram/libritts_cuts_train-all-shuf.jsonl.gz
|
||||
fi
|
||||
|
||||
# Here we shuffle and combine the train-clean-100, train-clean-360
|
||||
# together to form the training set.
|
||||
if [ ! -f data/spectrogram/libritts_cuts_train-clean-460.jsonl.gz ]; then
|
||||
cat <(gunzip -c data/spectrogram/libritts_cuts_train-clean-100.jsonl.gz) \
|
||||
<(gunzip -c data/spectrogram/libritts_cuts_train-clean-360.jsonl.gz) | \
|
||||
shuf | gzip -c > data/spectrogram/libritts_cuts_train-clean-460.jsonl.gz
|
||||
fi
|
||||
|
||||
if [ ! -e data/spectrogram/.libritts-validated.done ]; then
|
||||
log "Validating data/spectrogram for LibriTTS"
|
||||
./local/validate_manifest.py \
|
||||
data/spectrogram/libritts_cuts_train-all-shuf.jsonl.gz
|
||||
touch data/spectrogram/.libritts-validated.done
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ $stage -le 3 ] && [ $stop_stage -ge 3 ]; then
|
||||
log "Stage 3: Prepare phoneme tokens for LibriTTS"
|
||||
# We assume you have installed piper_phonemize and espnet_tts_frontend.
|
||||
# If not, please install them with:
|
||||
# - piper_phonemize:
|
||||
# refer to https://github.com/rhasspy/piper-phonemize,
|
||||
# could install the pre-built wheels from https://github.com/csukuangfj/piper-phonemize/releases/tag/2023.12.5
|
||||
# - espnet_tts_frontend:
|
||||
# `pip install espnet_tts_frontend`, refer to https://github.com/espnet/espnet_tts_frontend/
|
||||
if [ ! -e data/spectrogram/.libritts_with_token.done ]; then
|
||||
./local/prepare_tokens_libritts.py
|
||||
touch data/spectrogram/.libritts_with_token.done
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ $stage -le 4 ] && [ $stop_stage -ge 4 ]; then
|
||||
log "Stage 4: Generate token file"
|
||||
# We assume you have installed piper_phonemize and espnet_tts_frontend.
|
||||
# If not, please install them with:
|
||||
# - piper_phonemize:
|
||||
# refer to https://github.com/rhasspy/piper-phonemize,
|
||||
# could install the pre-built wheels from https://github.com/csukuangfj/piper-phonemize/releases/tag/2023.12.5
|
||||
# - espnet_tts_frontend:
|
||||
# `pip install espnet_tts_frontend`, refer to https://github.com/espnet/espnet_tts_frontend/
|
||||
if [ ! -e data/tokens.txt ]; then
|
||||
./local/prepare_token_file.py --tokens data/tokens.txt
|
||||
fi
|
||||
fi
|
||||
1
egs/libritts/TTS/shared
Symbolic link
1
egs/libritts/TTS/shared
Symbolic link
@ -0,0 +1 @@
|
||||
../../../icefall/shared/
|
||||
1
egs/libritts/TTS/vits/duration_predictor.py
Symbolic link
1
egs/libritts/TTS/vits/duration_predictor.py
Symbolic link
@ -0,0 +1 @@
|
||||
../../../ljspeech/TTS/vits/duration_predictor.py
|
||||
1
egs/libritts/TTS/vits/flow.py
Symbolic link
1
egs/libritts/TTS/vits/flow.py
Symbolic link
@ -0,0 +1 @@
|
||||
../../../ljspeech/TTS/vits/flow.py
|
||||
1
egs/libritts/TTS/vits/generator.py
Symbolic link
1
egs/libritts/TTS/vits/generator.py
Symbolic link
@ -0,0 +1 @@
|
||||
../../../ljspeech/TTS/vits/generator.py
|
||||
1
egs/libritts/TTS/vits/hifigan.py
Symbolic link
1
egs/libritts/TTS/vits/hifigan.py
Symbolic link
@ -0,0 +1 @@
|
||||
../../../ljspeech/TTS/vits/hifigan.py
|
||||
280
egs/libritts/TTS/vits/infer.py
Executable file
280
egs/libritts/TTS/vits/infer.py
Executable file
@ -0,0 +1,280 @@
|
||||
#!/usr/bin/env python3
|
||||
#
|
||||
# Copyright 2023 Xiaomi Corporation (Author: Zengwei Yao,
|
||||
# Zengrui Jin,)
|
||||
#
|
||||
# See ../../../../LICENSE for clarification regarding multiple authors
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""
|
||||
This script performs model inference on test set.
|
||||
|
||||
Usage:
|
||||
./vits/infer.py \
|
||||
--epoch 1000 \
|
||||
--exp-dir ./vits/exp \
|
||||
--max-duration 500
|
||||
"""
|
||||
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
from typing import List
|
||||
|
||||
import k2
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torchaudio
|
||||
from lhotse.features.io import KaldiReader
|
||||
from tokenizer import Tokenizer
|
||||
from train import get_model, get_params
|
||||
from tts_datamodule import LibrittsTtsDataModule
|
||||
|
||||
from icefall.checkpoint import load_checkpoint
|
||||
from icefall.utils import AttributeDict, setup_logger
|
||||
|
||||
|
||||
def get_parser():
|
||||
parser = argparse.ArgumentParser(
|
||||
formatter_class=argparse.ArgumentDefaultsHelpFormatter
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--epoch",
|
||||
type=int,
|
||||
default=1000,
|
||||
help="""It specifies the checkpoint to use for decoding.
|
||||
Note: Epoch counts from 1.
|
||||
""",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--exp-dir",
|
||||
type=str,
|
||||
default="vits/exp",
|
||||
help="The experiment dir",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--tokens",
|
||||
type=str,
|
||||
default="data/tokens.txt",
|
||||
help="""Path to vocabulary.""",
|
||||
)
|
||||
|
||||
return parser
|
||||
|
||||
|
||||
def infer_dataset(
|
||||
dl: torch.utils.data.DataLoader,
|
||||
subset: str,
|
||||
params: AttributeDict,
|
||||
model: nn.Module,
|
||||
tokenizer: Tokenizer,
|
||||
speaker_map: KaldiReader,
|
||||
) -> None:
|
||||
"""Decode dataset.
|
||||
The ground-truth and generated audio pairs will be saved to `params.save_wav_dir`.
|
||||
|
||||
Args:
|
||||
dl:
|
||||
PyTorch's dataloader containing the dataset to decode.
|
||||
params:
|
||||
It is returned by :func:`get_params`.
|
||||
model:
|
||||
The neural model.
|
||||
tokenizer:
|
||||
Used to convert text to phonemes.
|
||||
"""
|
||||
|
||||
# Background worker save audios to disk.
|
||||
def _save_worker(
|
||||
subset: str,
|
||||
batch_size: int,
|
||||
cut_ids: List[str],
|
||||
audio: torch.Tensor,
|
||||
audio_pred: torch.Tensor,
|
||||
audio_lens: List[int],
|
||||
audio_lens_pred: List[int],
|
||||
):
|
||||
for i in range(batch_size):
|
||||
torchaudio.save(
|
||||
str(params.save_wav_dir / subset / f"{cut_ids[i]}_gt.wav"),
|
||||
audio[i : i + 1, : audio_lens[i]],
|
||||
sample_rate=params.sampling_rate,
|
||||
)
|
||||
torchaudio.save(
|
||||
str(params.save_wav_dir / subset / f"{cut_ids[i]}_pred.wav"),
|
||||
audio_pred[i : i + 1, : audio_lens_pred[i]],
|
||||
sample_rate=params.sampling_rate,
|
||||
)
|
||||
|
||||
device = next(model.parameters()).device
|
||||
num_cuts = 0
|
||||
log_interval = 5
|
||||
|
||||
try:
|
||||
num_batches = len(dl)
|
||||
except TypeError:
|
||||
num_batches = "?"
|
||||
|
||||
futures = []
|
||||
with ThreadPoolExecutor(max_workers=1) as executor:
|
||||
for batch_idx, batch in enumerate(dl):
|
||||
batch_size = len(batch["tokens"])
|
||||
|
||||
tokens = batch["tokens"]
|
||||
tokens = tokenizer.tokens_to_token_ids(
|
||||
tokens, intersperse_blank=True, add_sos=True, add_eos=True
|
||||
)
|
||||
tokens = k2.RaggedTensor(tokens)
|
||||
row_splits = tokens.shape.row_splits(1)
|
||||
tokens_lens = row_splits[1:] - row_splits[:-1]
|
||||
tokens = tokens.to(device)
|
||||
tokens_lens = tokens_lens.to(device)
|
||||
# tensor of shape (B, T)
|
||||
tokens = tokens.pad(mode="constant", padding_value=tokenizer.pad_id)
|
||||
|
||||
audio = batch["audio"]
|
||||
audio_lens = batch["audio_lens"].tolist()
|
||||
cut_ids = [cut.id for cut in batch["cut"]]
|
||||
sids = ["_".join(cut_id.split("_")[:2]) for cut_id in cut_ids]
|
||||
spembs = (
|
||||
torch.Tensor(np.array([speaker_map.read(sid) for sid in sids]))
|
||||
.squeeze(1)
|
||||
.to(device)
|
||||
)
|
||||
|
||||
audio_pred, _, durations = model.inference_batch(
|
||||
text=tokens,
|
||||
text_lengths=tokens_lens,
|
||||
spembs=spembs,
|
||||
)
|
||||
audio_pred = audio_pred.detach().cpu()
|
||||
# convert to samples
|
||||
audio_lens_pred = (
|
||||
(durations.sum(1) * params.frame_shift).to(dtype=torch.int64).tolist()
|
||||
)
|
||||
|
||||
futures.append(
|
||||
executor.submit(
|
||||
_save_worker,
|
||||
subset,
|
||||
batch_size,
|
||||
cut_ids,
|
||||
audio,
|
||||
audio_pred,
|
||||
audio_lens,
|
||||
audio_lens_pred,
|
||||
)
|
||||
)
|
||||
|
||||
num_cuts += batch_size
|
||||
|
||||
if batch_idx % log_interval == 0:
|
||||
batch_str = f"{batch_idx}/{num_batches}"
|
||||
|
||||
logging.info(
|
||||
f"batch {batch_str}, cuts processed until now is {num_cuts}"
|
||||
)
|
||||
# return results
|
||||
for f in futures:
|
||||
f.result()
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def main():
|
||||
parser = get_parser()
|
||||
LibrittsTtsDataModule.add_arguments(parser)
|
||||
args = parser.parse_args()
|
||||
args.exp_dir = Path(args.exp_dir)
|
||||
|
||||
params = get_params()
|
||||
params.update(vars(args))
|
||||
|
||||
params.suffix = f"epoch-{params.epoch}"
|
||||
|
||||
params.res_dir = params.exp_dir / "infer" / params.suffix
|
||||
params.save_wav_dir = params.res_dir / "wav"
|
||||
params.save_wav_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
setup_logger(f"{params.res_dir}/log-infer-{params.suffix}")
|
||||
logging.info("Infer started")
|
||||
|
||||
device = torch.device("cpu")
|
||||
if torch.cuda.is_available():
|
||||
device = torch.device("cuda", 0)
|
||||
|
||||
tokenizer = Tokenizer(params.tokens)
|
||||
params.blank_id = tokenizer.pad_id
|
||||
params.vocab_size = tokenizer.vocab_size
|
||||
|
||||
# we need cut ids to display recognition results.
|
||||
args.return_cuts = True
|
||||
libritts = LibrittsTtsDataModule(args)
|
||||
|
||||
logging.info(f"Device: {device}")
|
||||
logging.info(params)
|
||||
|
||||
logging.info("About to create model")
|
||||
model = get_model(params)
|
||||
|
||||
load_checkpoint(f"{params.exp_dir}/epoch-{params.epoch}.pt", model)
|
||||
|
||||
model.to(device)
|
||||
model.eval()
|
||||
|
||||
num_param_g = sum([p.numel() for p in model.generator.parameters()])
|
||||
logging.info(f"Number of parameters in generator: {num_param_g}")
|
||||
num_param_d = sum([p.numel() for p in model.discriminator.parameters()])
|
||||
logging.info(f"Number of parameters in discriminator: {num_param_d}")
|
||||
logging.info(f"Total number of parameters: {num_param_g + num_param_d}")
|
||||
|
||||
test_clean_cuts = libritts.test_clean_cuts()
|
||||
test_clean_speaker_map = libritts.test_clean_xvector()
|
||||
test_clean_dl = libritts.test_dataloaders(test_clean_cuts)
|
||||
|
||||
dev_clean_cuts = libritts.dev_clean_cuts()
|
||||
dev_clean_speaker_map = libritts.dev_clean_xvector()
|
||||
dev_clean_dl = libritts.dev_dataloaders(dev_clean_cuts)
|
||||
|
||||
infer_sets = {
|
||||
"test-clean": (test_clean_dl, test_clean_speaker_map),
|
||||
"dev-clean": (dev_clean_dl, dev_clean_speaker_map),
|
||||
}
|
||||
|
||||
for subset, data in infer_sets.items():
|
||||
save_wav_dir = params.res_dir / "wav" / subset
|
||||
save_wav_dir.mkdir(parents=True, exist_ok=True)
|
||||
dl, speaker_map = data
|
||||
|
||||
logging.info(f"Processing {subset} set, saving to {save_wav_dir}")
|
||||
|
||||
infer_dataset(
|
||||
dl=dl,
|
||||
subset=subset,
|
||||
params=params,
|
||||
model=model,
|
||||
tokenizer=tokenizer,
|
||||
speaker_map=speaker_map,
|
||||
)
|
||||
|
||||
logging.info(f"Wav files are saved to {params.save_wav_dir}")
|
||||
logging.info("Done!")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
1
egs/libritts/TTS/vits/loss.py
Symbolic link
1
egs/libritts/TTS/vits/loss.py
Symbolic link
@ -0,0 +1 @@
|
||||
../../../ljspeech/TTS/vits/loss.py
|
||||
1
egs/libritts/TTS/vits/monotonic_align
Symbolic link
1
egs/libritts/TTS/vits/monotonic_align
Symbolic link
@ -0,0 +1 @@
|
||||
../../../ljspeech/TTS/vits/monotonic_align
|
||||
1
egs/libritts/TTS/vits/posterior_encoder.py
Symbolic link
1
egs/libritts/TTS/vits/posterior_encoder.py
Symbolic link
@ -0,0 +1 @@
|
||||
../../../ljspeech/TTS/vits/posterior_encoder.py
|
||||
1
egs/libritts/TTS/vits/residual_coupling.py
Symbolic link
1
egs/libritts/TTS/vits/residual_coupling.py
Symbolic link
@ -0,0 +1 @@
|
||||
../../../ljspeech/TTS/vits/residual_coupling.py
|
||||
141
egs/libritts/TTS/vits/test_onnx.py
Executable file
141
egs/libritts/TTS/vits/test_onnx.py
Executable file
@ -0,0 +1,141 @@
|
||||
#!/usr/bin/env python3
|
||||
#
|
||||
# Copyright 2023-2024 Xiaomi Corporation (Author: Zengwei Yao,
|
||||
# Zengrui Jin,)
|
||||
#
|
||||
# See ../../../../LICENSE for clarification regarding multiple authors
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""
|
||||
This script is used to test the exported onnx model by vits/export-onnx.py
|
||||
|
||||
Use the onnx model to generate a wav:
|
||||
./vits/test_onnx.py \
|
||||
--model-filename vits/exp/vits-epoch-1000.onnx \
|
||||
--tokens data/tokens.txt
|
||||
"""
|
||||
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
import onnxruntime as ort
|
||||
import torch
|
||||
import torchaudio
|
||||
from tokenizer import Tokenizer
|
||||
|
||||
|
||||
def get_parser():
|
||||
parser = argparse.ArgumentParser(
|
||||
formatter_class=argparse.ArgumentDefaultsHelpFormatter
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--model-filename",
|
||||
type=str,
|
||||
required=True,
|
||||
help="Path to the onnx model.",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--speakers",
|
||||
type=Path,
|
||||
default=Path("data/speakers.txt"),
|
||||
help="Path to speakers.txt file.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--tokens",
|
||||
type=str,
|
||||
default="data/tokens.txt",
|
||||
help="""Path to vocabulary.""",
|
||||
)
|
||||
|
||||
return parser
|
||||
|
||||
|
||||
class OnnxModel:
|
||||
def __init__(self, model_filename: str):
|
||||
session_opts = ort.SessionOptions()
|
||||
session_opts.inter_op_num_threads = 1
|
||||
session_opts.intra_op_num_threads = 4
|
||||
|
||||
self.session_opts = session_opts
|
||||
|
||||
self.model = ort.InferenceSession(
|
||||
model_filename,
|
||||
sess_options=self.session_opts,
|
||||
providers=["CPUExecutionProvider"],
|
||||
)
|
||||
logging.info(f"{self.model.get_modelmeta().custom_metadata_map}")
|
||||
|
||||
def __call__(
|
||||
self, tokens: torch.Tensor, tokens_lens: torch.Tensor, speaker: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Args:
|
||||
tokens:
|
||||
A 1-D tensor of shape (1, T)
|
||||
Returns:
|
||||
A tensor of shape (1, T')
|
||||
"""
|
||||
noise_scale = torch.tensor([0.667], dtype=torch.float32)
|
||||
noise_scale_dur = torch.tensor([0.8], dtype=torch.float32)
|
||||
alpha = torch.tensor([1.0], dtype=torch.float32)
|
||||
|
||||
out = self.model.run(
|
||||
[
|
||||
self.model.get_outputs()[0].name,
|
||||
],
|
||||
{
|
||||
self.model.get_inputs()[0].name: tokens.numpy(),
|
||||
self.model.get_inputs()[1].name: tokens_lens.numpy(),
|
||||
self.model.get_inputs()[2].name: noise_scale.numpy(),
|
||||
self.model.get_inputs()[3].name: alpha.numpy(),
|
||||
self.model.get_inputs()[4].name: noise_scale_dur.numpy(),
|
||||
self.model.get_inputs()[5].name: speaker.numpy(),
|
||||
},
|
||||
)[0]
|
||||
return torch.from_numpy(out)
|
||||
|
||||
|
||||
def main():
|
||||
args = get_parser().parse_args()
|
||||
|
||||
tokenizer = Tokenizer(args.tokens)
|
||||
|
||||
with open(args.speakers) as f:
|
||||
speaker_map = {line.strip(): i for i, line in enumerate(f)}
|
||||
args.num_spks = len(speaker_map)
|
||||
|
||||
logging.info("About to create onnx model")
|
||||
model = OnnxModel(args.model_filename)
|
||||
|
||||
text = "I went there to see the land, the people and how their system works, end quote."
|
||||
tokens = tokenizer.texts_to_token_ids(
|
||||
[text], intersperse_blank=True, add_sos=True, add_eos=True
|
||||
)
|
||||
tokens = torch.tensor(tokens) # (1, T)
|
||||
tokens_lens = torch.tensor([tokens.shape[1]], dtype=torch.int64) # (1, T)
|
||||
speaker = torch.tensor([1], dtype=torch.int64) # (1, )
|
||||
audio = model(tokens, tokens_lens, speaker) # (1, T')
|
||||
|
||||
torchaudio.save(str("test_onnx.wav"), audio, sample_rate=22050)
|
||||
logging.info("Saved to test_onnx.wav")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
formatter = "%(asctime)s %(levelname)s [%(filename)s:%(lineno)d] %(message)s"
|
||||
logging.basicConfig(format=formatter, level=logging.INFO)
|
||||
main()
|
||||
1
egs/libritts/TTS/vits/text_encoder.py
Symbolic link
1
egs/libritts/TTS/vits/text_encoder.py
Symbolic link
@ -0,0 +1 @@
|
||||
../../../ljspeech/TTS/vits/text_encoder.py
|
||||
1
egs/libritts/TTS/vits/tokenizer.py
Symbolic link
1
egs/libritts/TTS/vits/tokenizer.py
Symbolic link
@ -0,0 +1 @@
|
||||
../../../ljspeech/TTS/vits/tokenizer.py
|
||||
1015
egs/libritts/TTS/vits/train.py
Executable file
1015
egs/libritts/TTS/vits/train.py
Executable file
File diff suppressed because it is too large
Load Diff
1
egs/libritts/TTS/vits/transform.py
Symbolic link
1
egs/libritts/TTS/vits/transform.py
Symbolic link
@ -0,0 +1 @@
|
||||
../../../ljspeech/TTS/vits/transform.py
|
||||
432
egs/libritts/TTS/vits/tts_datamodule.py
Normal file
432
egs/libritts/TTS/vits/tts_datamodule.py
Normal file
@ -0,0 +1,432 @@
|
||||
# Copyright 2021 Piotr Żelasko
|
||||
# Copyright 2022-2024 Xiaomi Corporation (Authors: Mingshuang Luo,
|
||||
# Zengwei Yao,
|
||||
# Zengrui Jin,)
|
||||
#
|
||||
# See ../../../../LICENSE for clarification regarding multiple authors
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
import torch
|
||||
from lhotse import CutSet, Spectrogram, SpectrogramConfig, load_manifest_lazy
|
||||
from lhotse.dataset import ( # noqa F401 for PrecomputedFeatures
|
||||
CutConcatenate,
|
||||
DynamicBucketingSampler,
|
||||
PrecomputedFeatures,
|
||||
SimpleCutSampler,
|
||||
SpeechSynthesisDataset,
|
||||
)
|
||||
from lhotse.dataset.input_strategies import ( # noqa F401 For AudioSamples
|
||||
AudioSamples,
|
||||
OnTheFlyFeatures,
|
||||
)
|
||||
from lhotse.features.io import KaldiReader
|
||||
from lhotse.utils import fix_random_seed
|
||||
from torch.utils.data import DataLoader
|
||||
|
||||
from icefall.utils import str2bool
|
||||
|
||||
|
||||
class _SeedWorkers:
|
||||
def __init__(self, seed: int):
|
||||
self.seed = seed
|
||||
|
||||
def __call__(self, worker_id: int):
|
||||
fix_random_seed(self.seed + worker_id)
|
||||
|
||||
|
||||
LIBRITTS_SAMPLING_RATE = 24000
|
||||
|
||||
|
||||
class LibrittsTtsDataModule:
|
||||
"""
|
||||
DataModule for tts experiments.
|
||||
It assumes there is always one train and valid dataloader,
|
||||
but there can be multiple test dataloaders (e.g. LibriSpeech test-clean
|
||||
and test-other).
|
||||
|
||||
It contains all the common data pipeline modules used in ASR
|
||||
experiments, e.g.:
|
||||
- dynamic batch size,
|
||||
- bucketing samplers,
|
||||
- cut concatenation,
|
||||
- on-the-fly feature extraction
|
||||
|
||||
This class should be derived for specific corpora used in ASR tasks.
|
||||
"""
|
||||
|
||||
def __init__(self, args: argparse.Namespace):
|
||||
self.args = args
|
||||
|
||||
@classmethod
|
||||
def add_arguments(cls, parser: argparse.ArgumentParser):
|
||||
group = parser.add_argument_group(
|
||||
title="TTS data related options",
|
||||
description="These options are used for the preparation of "
|
||||
"PyTorch DataLoaders from Lhotse CutSet's -- they control the "
|
||||
"effective batch sizes, sampling strategies, applied data "
|
||||
"augmentations, etc.",
|
||||
)
|
||||
group.add_argument(
|
||||
"--full-libri",
|
||||
type=str2bool,
|
||||
default=False,
|
||||
help="""When enabled, use the entire LibriTTS training set.
|
||||
Otherwise, use the 460h clean subset.""",
|
||||
)
|
||||
group.add_argument(
|
||||
"--manifest-dir",
|
||||
type=Path,
|
||||
default=Path("data/spectrogram"),
|
||||
help="Path to directory with train/valid/test cuts.",
|
||||
)
|
||||
group.add_argument(
|
||||
"--speaker-embeds",
|
||||
type=Path,
|
||||
default=Path("exp/xvector_nnet_1a/"),
|
||||
help="Path to directory with speaker embeddings.",
|
||||
)
|
||||
group.add_argument(
|
||||
"--max-duration",
|
||||
type=int,
|
||||
default=200.0,
|
||||
help="Maximum pooled recordings duration (seconds) in a "
|
||||
"single batch. You can reduce it if it causes CUDA OOM.",
|
||||
)
|
||||
group.add_argument(
|
||||
"--bucketing-sampler",
|
||||
type=str2bool,
|
||||
default=True,
|
||||
help="When enabled, the batches will come from buckets of "
|
||||
"similar duration (saves padding frames).",
|
||||
)
|
||||
group.add_argument(
|
||||
"--num-buckets",
|
||||
type=int,
|
||||
default=30,
|
||||
help="The number of buckets for the DynamicBucketingSampler"
|
||||
"(you might want to increase it for larger datasets).",
|
||||
)
|
||||
|
||||
group.add_argument(
|
||||
"--on-the-fly-feats",
|
||||
type=str2bool,
|
||||
default=False,
|
||||
help="When enabled, use on-the-fly cut mixing and feature "
|
||||
"extraction. Will drop existing precomputed feature manifests "
|
||||
"if available.",
|
||||
)
|
||||
group.add_argument(
|
||||
"--shuffle",
|
||||
type=str2bool,
|
||||
default=True,
|
||||
help="When enabled (=default), the examples will be "
|
||||
"shuffled for each epoch.",
|
||||
)
|
||||
group.add_argument(
|
||||
"--drop-last",
|
||||
type=str2bool,
|
||||
default=True,
|
||||
help="Whether to drop last batch. Used by sampler.",
|
||||
)
|
||||
group.add_argument(
|
||||
"--return-cuts",
|
||||
type=str2bool,
|
||||
default=True,
|
||||
help="When enabled, each batch will have the "
|
||||
"field: batch['cut'] with the cuts that "
|
||||
"were used to construct it.",
|
||||
)
|
||||
group.add_argument(
|
||||
"--num-workers",
|
||||
type=int,
|
||||
default=8,
|
||||
help="The number of training dataloader workers that "
|
||||
"collect the batches.",
|
||||
)
|
||||
|
||||
group.add_argument(
|
||||
"--input-strategy",
|
||||
type=str,
|
||||
default="PrecomputedFeatures",
|
||||
help="AudioSamples or PrecomputedFeatures",
|
||||
)
|
||||
|
||||
def train_dataloaders(
|
||||
self,
|
||||
cuts_train: CutSet,
|
||||
sampler_state_dict: Optional[Dict[str, Any]] = None,
|
||||
) -> DataLoader:
|
||||
"""
|
||||
Args:
|
||||
cuts_train:
|
||||
CutSet for training.
|
||||
sampler_state_dict:
|
||||
The state dict for the training sampler.
|
||||
"""
|
||||
logging.info("About to create train dataset")
|
||||
train = SpeechSynthesisDataset(
|
||||
return_text=True,
|
||||
return_tokens=True,
|
||||
return_spk_ids=True,
|
||||
feature_input_strategy=eval(self.args.input_strategy)(),
|
||||
return_cuts=self.args.return_cuts,
|
||||
)
|
||||
|
||||
if self.args.on_the_fly_feats:
|
||||
sampling_rate = LIBRITTS_SAMPLING_RATE
|
||||
config = SpectrogramConfig(
|
||||
sampling_rate=sampling_rate,
|
||||
frame_length=1024 / sampling_rate, # (in second),
|
||||
frame_shift=256 / sampling_rate, # (in second)
|
||||
use_fft_mag=True,
|
||||
)
|
||||
train = SpeechSynthesisDataset(
|
||||
return_text=True,
|
||||
return_tokens=True,
|
||||
return_spk_ids=True,
|
||||
feature_input_strategy=OnTheFlyFeatures(Spectrogram(config)),
|
||||
return_cuts=self.args.return_cuts,
|
||||
)
|
||||
|
||||
if self.args.bucketing_sampler:
|
||||
logging.info("Using DynamicBucketingSampler.")
|
||||
train_sampler = DynamicBucketingSampler(
|
||||
cuts_train,
|
||||
max_duration=self.args.max_duration,
|
||||
shuffle=self.args.shuffle,
|
||||
num_buckets=self.args.num_buckets,
|
||||
buffer_size=self.args.num_buckets * 2000,
|
||||
shuffle_buffer_size=self.args.num_buckets * 5000,
|
||||
drop_last=self.args.drop_last,
|
||||
)
|
||||
else:
|
||||
logging.info("Using SimpleCutSampler.")
|
||||
train_sampler = SimpleCutSampler(
|
||||
cuts_train,
|
||||
max_duration=self.args.max_duration,
|
||||
shuffle=self.args.shuffle,
|
||||
)
|
||||
logging.info("About to create train dataloader")
|
||||
|
||||
if sampler_state_dict is not None:
|
||||
logging.info("Loading sampler state dict")
|
||||
train_sampler.load_state_dict(sampler_state_dict)
|
||||
|
||||
# 'seed' is derived from the current random state, which will have
|
||||
# previously been set in the main process.
|
||||
seed = torch.randint(0, 100000, ()).item()
|
||||
worker_init_fn = _SeedWorkers(seed)
|
||||
|
||||
train_dl = DataLoader(
|
||||
train,
|
||||
sampler=train_sampler,
|
||||
batch_size=None,
|
||||
num_workers=self.args.num_workers,
|
||||
persistent_workers=False,
|
||||
worker_init_fn=worker_init_fn,
|
||||
)
|
||||
|
||||
return train_dl
|
||||
|
||||
def dev_dataloaders(self, cuts_valid: CutSet) -> DataLoader:
|
||||
logging.info("About to create dev dataset")
|
||||
if self.args.on_the_fly_feats:
|
||||
sampling_rate = LIBRITTS_SAMPLING_RATE
|
||||
config = SpectrogramConfig(
|
||||
sampling_rate=sampling_rate,
|
||||
frame_length=1024 / sampling_rate, # (in second),
|
||||
frame_shift=256 / sampling_rate, # (in second)
|
||||
use_fft_mag=True,
|
||||
)
|
||||
validate = SpeechSynthesisDataset(
|
||||
return_text=True,
|
||||
return_tokens=True,
|
||||
return_spk_ids=True,
|
||||
feature_input_strategy=OnTheFlyFeatures(Spectrogram(config)),
|
||||
return_cuts=self.args.return_cuts,
|
||||
)
|
||||
else:
|
||||
validate = SpeechSynthesisDataset(
|
||||
return_text=True,
|
||||
return_tokens=True,
|
||||
return_spk_ids=True,
|
||||
feature_input_strategy=eval(self.args.input_strategy)(),
|
||||
return_cuts=self.args.return_cuts,
|
||||
)
|
||||
dev_sampler = DynamicBucketingSampler(
|
||||
cuts_valid,
|
||||
max_duration=self.args.max_duration,
|
||||
shuffle=False,
|
||||
)
|
||||
logging.info("About to create valid dataloader")
|
||||
dev_dl = DataLoader(
|
||||
validate,
|
||||
sampler=dev_sampler,
|
||||
batch_size=None,
|
||||
num_workers=2,
|
||||
persistent_workers=False,
|
||||
)
|
||||
|
||||
return dev_dl
|
||||
|
||||
def test_dataloaders(self, cuts: CutSet) -> DataLoader:
|
||||
logging.info("About to create test dataset")
|
||||
if self.args.on_the_fly_feats:
|
||||
sampling_rate = LIBRITTS_SAMPLING_RATE
|
||||
config = SpectrogramConfig(
|
||||
sampling_rate=sampling_rate,
|
||||
frame_length=1024 / sampling_rate, # (in second),
|
||||
frame_shift=256 / sampling_rate, # (in second)
|
||||
use_fft_mag=True,
|
||||
)
|
||||
test = SpeechSynthesisDataset(
|
||||
return_text=True,
|
||||
return_tokens=True,
|
||||
return_spk_ids=True,
|
||||
feature_input_strategy=OnTheFlyFeatures(Spectrogram(config)),
|
||||
return_cuts=self.args.return_cuts,
|
||||
)
|
||||
else:
|
||||
test = SpeechSynthesisDataset(
|
||||
return_text=True,
|
||||
return_tokens=True,
|
||||
return_spk_ids=True,
|
||||
feature_input_strategy=eval(self.args.input_strategy)(),
|
||||
return_cuts=self.args.return_cuts,
|
||||
)
|
||||
test_sampler = DynamicBucketingSampler(
|
||||
cuts,
|
||||
max_duration=self.args.max_duration,
|
||||
shuffle=False,
|
||||
)
|
||||
logging.info("About to create test dataloader")
|
||||
test_dl = DataLoader(
|
||||
test,
|
||||
batch_size=None,
|
||||
sampler=test_sampler,
|
||||
num_workers=self.args.num_workers,
|
||||
)
|
||||
return test_dl
|
||||
|
||||
@lru_cache()
|
||||
def train_all_shuf_cuts(self) -> CutSet:
|
||||
logging.info(
|
||||
"About to get the shuffled train-clean-100, \
|
||||
train-clean-360 and train-other-500 cuts"
|
||||
)
|
||||
return load_manifest_lazy(
|
||||
self.args.manifest_dir / "libritts_cuts_with_tokens_train-all-shuf.jsonl.gz"
|
||||
)
|
||||
|
||||
@lru_cache()
|
||||
def train_clean_460_cuts(self) -> CutSet:
|
||||
logging.info(
|
||||
"About to get the shuffled train-clean-100 and train-clean-360 cuts"
|
||||
)
|
||||
return load_manifest_lazy(
|
||||
self.args.manifest_dir
|
||||
/ "libritts_cuts_with_tokens_train-clean-460.jsonl.gz"
|
||||
)
|
||||
|
||||
@lru_cache()
|
||||
def dev_clean_cuts(self) -> CutSet:
|
||||
logging.info("About to get dev-clean cuts")
|
||||
return load_manifest_lazy(
|
||||
self.args.manifest_dir / "libritts_cuts_with_tokens_dev-clean.jsonl.gz"
|
||||
)
|
||||
|
||||
@lru_cache()
|
||||
def dev_other_cuts(self) -> CutSet:
|
||||
logging.info("About to get dev-other cuts")
|
||||
return load_manifest_lazy(
|
||||
self.args.manifest_dir / "libritts_cuts_with_tokens_dev-other.jsonl.gz"
|
||||
)
|
||||
|
||||
@lru_cache()
|
||||
def test_clean_cuts(self) -> CutSet:
|
||||
logging.info("About to get test-clean cuts")
|
||||
return load_manifest_lazy(
|
||||
self.args.manifest_dir / "libritts_cuts_with_tokens_test-clean.jsonl.gz"
|
||||
)
|
||||
|
||||
@lru_cache()
|
||||
def test_other_cuts(self) -> CutSet:
|
||||
logging.info("About to get test-other cuts")
|
||||
return load_manifest_lazy(
|
||||
self.args.manifest_dir / "libritts_cuts_with_tokens_test-other.jsonl.gz"
|
||||
)
|
||||
|
||||
@lru_cache()
|
||||
def train_all_shuf_xvector(self) -> KaldiReader:
|
||||
raise NotImplementedError(
|
||||
"Please implement the method to load speaker embeddings."
|
||||
)
|
||||
|
||||
@lru_cache()
|
||||
def train_clean_460_xvector(self) -> KaldiReader:
|
||||
logging.info("About to get speaker embeddings for train-clean-460")
|
||||
return KaldiReader(
|
||||
str(self.args.speaker_embeds / "xvectors_train_clean_460" / "feats.scp")
|
||||
)
|
||||
|
||||
@lru_cache()
|
||||
def train_clean_100_xvector(self) -> KaldiReader:
|
||||
raise NotImplementedError(
|
||||
"Please implement the method to load speaker embeddings."
|
||||
)
|
||||
|
||||
@lru_cache()
|
||||
def train_clean_360_xvector(self) -> KaldiReader:
|
||||
raise NotImplementedError(
|
||||
"Please implement the method to load speaker embeddings."
|
||||
)
|
||||
|
||||
@lru_cache()
|
||||
def train_other_500_xvector(self) -> KaldiReader:
|
||||
raise NotImplementedError(
|
||||
"Please implement the method to load speaker embeddings."
|
||||
)
|
||||
|
||||
@lru_cache()
|
||||
def dev_clean_xvector(self) -> KaldiReader:
|
||||
logging.info("About to get speaker embeddings for dev-clean")
|
||||
return KaldiReader(
|
||||
str(self.args.speaker_embeds / "xvectors_dev_clean" / "feats.scp")
|
||||
)
|
||||
|
||||
@lru_cache()
|
||||
def dev_other_xvector(self) -> KaldiReader:
|
||||
raise NotImplementedError(
|
||||
"Please implement the method to load speaker embeddings."
|
||||
)
|
||||
|
||||
@lru_cache()
|
||||
def test_clean_xvector(self) -> KaldiReader:
|
||||
logging.info("About to get speaker embeddings for test-clean")
|
||||
return KaldiReader(
|
||||
str(self.args.speaker_embeds / "xvectors_test_clean" / "feats.scp")
|
||||
)
|
||||
|
||||
@lru_cache()
|
||||
def test_other_xvector(self) -> KaldiReader:
|
||||
raise NotImplementedError(
|
||||
"Please implement the method to load speaker embeddings."
|
||||
)
|
||||
1
egs/libritts/TTS/vits/utils.py
Symbolic link
1
egs/libritts/TTS/vits/utils.py
Symbolic link
@ -0,0 +1 @@
|
||||
../../../ljspeech/TTS/vits/utils.py
|
||||
1
egs/libritts/TTS/vits/vits.py
Symbolic link
1
egs/libritts/TTS/vits/vits.py
Symbolic link
@ -0,0 +1 @@
|
||||
../../../ljspeech/TTS/vits/vits.py
|
||||
1
egs/libritts/TTS/vits/wavenet.py
Symbolic link
1
egs/libritts/TTS/vits/wavenet.py
Symbolic link
@ -0,0 +1 @@
|
||||
../../../ljspeech/TTS/vits/wavenet.py
|
||||
@ -409,7 +409,12 @@ class VITSGenerator(torch.nn.Module):
|
||||
g = self.global_emb(sids.view(-1)).unsqueeze(-1)
|
||||
if self.spk_embed_dim is not None:
|
||||
# (B, global_channels, 1)
|
||||
g_ = self.spemb_proj(F.normalize(spembs.unsqueeze(0))).unsqueeze(-1)
|
||||
if spembs.ndim == 2:
|
||||
g_ = self.spemb_proj(F.normalize(spembs)).unsqueeze(-1)
|
||||
elif spembs.ndim == 1:
|
||||
g_ = self.spemb_proj(F.normalize(spembs.unsqueeze(0))).unsqueeze(-1)
|
||||
else:
|
||||
raise ValueError("spembs should be 1D or 2D (batch mode) tensor.")
|
||||
if g is None:
|
||||
g = g_
|
||||
else:
|
||||
|
||||
@ -542,13 +542,13 @@ def train_one_epoch(
|
||||
tb_writer, "train/valid_", params.batch_idx_train
|
||||
)
|
||||
tb_writer.add_audio(
|
||||
"train/valdi_speech_hat",
|
||||
"train/valid_speech_hat",
|
||||
speech_hat,
|
||||
params.batch_idx_train,
|
||||
params.sampling_rate,
|
||||
)
|
||||
tb_writer.add_audio(
|
||||
"train/valdi_speech",
|
||||
"train/valid_speech",
|
||||
speech,
|
||||
params.batch_idx_train,
|
||||
params.sampling_rate,
|
||||
|
||||
@ -622,6 +622,8 @@ class VITS(nn.Module):
|
||||
text: torch.Tensor,
|
||||
text_lengths: torch.Tensor,
|
||||
sids: Optional[torch.Tensor] = None,
|
||||
spembs: Optional[torch.Tensor] = None,
|
||||
lids: Optional[torch.Tensor] = None,
|
||||
durations: Optional[torch.Tensor] = None,
|
||||
noise_scale: float = 0.667,
|
||||
noise_scale_dur: float = 0.8,
|
||||
@ -635,6 +637,8 @@ class VITS(nn.Module):
|
||||
text (Tensor): Input text index tensor (B, T_text).
|
||||
text_lengths (Tensor): Input text index tensor (B,).
|
||||
sids (Tensor): Speaker index tensor (B,).
|
||||
spembs (Optional[Tensor]): Speaker embedding tensor (B, spk_embed_dim).
|
||||
lids (Tensor): Language index tensor (B,).
|
||||
noise_scale (float): Noise scale value for flow.
|
||||
noise_scale_dur (float): Noise scale value for duration predictor.
|
||||
alpha (float): Alpha parameter to control the speed of generated speech.
|
||||
@ -650,6 +654,8 @@ class VITS(nn.Module):
|
||||
text=text,
|
||||
text_lengths=text_lengths,
|
||||
sids=sids,
|
||||
spembs=spembs,
|
||||
lids=lids,
|
||||
noise_scale=noise_scale,
|
||||
noise_scale_dur=noise_scale_dur,
|
||||
alpha=alpha,
|
||||
|
||||
@ -597,13 +597,13 @@ def train_one_epoch(
|
||||
tb_writer, "train/valid_", params.batch_idx_train
|
||||
)
|
||||
tb_writer.add_audio(
|
||||
"train/valdi_speech_hat",
|
||||
"train/valid_speech_hat",
|
||||
speech_hat,
|
||||
params.batch_idx_train,
|
||||
params.sampling_rate,
|
||||
)
|
||||
tb_writer.add_audio(
|
||||
"train/valdi_speech",
|
||||
"train/valid_speech",
|
||||
speech,
|
||||
params.batch_idx_train,
|
||||
params.sampling_rate,
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user