diff --git a/egs/zipvoice/README.md b/egs/zipvoice/README.md index 0eed8f540..a80025564 100644 --- a/egs/zipvoice/README.md +++ b/egs/zipvoice/README.md @@ -21,7 +21,33 @@ ZipVoice is a high-quality zero-shot TTS model with a small model size and fast ## Installation + +* Clone icefall repository and change to zipvoice directory: + +```bash +git clone https://github.com/k2-fsa/icefall.git +cd icefall/egs/zipvoice ``` + +* Create a Python virtual environment (optional but recommended): + +```bash +python3 -m venv venv +source venv/bin/activate +``` + +* Install the required packages: + +```bash +# Install pytorch and k2. +# If you want to use different versions, please refer to https://k2-fsa.org/get-started/k2/ for details. +# For users in China mainland, please refer to https://k2-fsa.org/zh-CN/get-started/k2/ + +pip install torch==2.5.1 torchaudio==2.5.1 --index-url https://download.pytorch.org/whl/cu121 +pip install k2==1.24.4.dev20250208+cuda12.1.torch2.5.1 -f https://k2-fsa.github.io/k2/cuda.html + +# Install other dependencies. +pip install piper_phonemize -f https://k2-fsa.github.io/icefall/piper_phonemize.html pip install -r requirements.txt ``` @@ -31,12 +57,21 @@ To generate speech with our pre-trained ZipVoice or ZipVoice-Distill models, use ### 1. Inference of a single sentence: ```bash +# Chinese example python3 zipvoice/zipvoice_infer.py \ --model-name "zipvoice_distill" \ - --prompt-wav prompt.wav \ - --prompt-text "I am the transcription of the prompt wav." \ - --text "I am the text to be synthesized." \ - --res-wav-path result.wav + --prompt-wav assets/prompt-zh.wav \ + --prompt-text "对,这就是我,万人敬仰的太乙真人,虽然有点婴儿肥,但也掩不住我逼人的帅气。" \ + --text "欢迎使用我们的语音合成模型,希望它能给你带来惊喜!" \ + --res-wav-path result-zh.wav + +# English example +python3 zipvoice/zipvoice_infer.py \ + --model-name "zipvoice_distill" \ + --prompt-wav assets/prompt-en.wav \ + --prompt-text "Some call me nature, others call me mother nature. I've been here for over four point five billion years, twenty two thousand five hundred times longer than you." \ + --text "Welcome to use our tts model, have fun!" \ + --res-wav-path result-en.wav ``` ### 2. Inference of a list of sentences: @@ -46,6 +81,7 @@ python3 zipvoice/zipvoice_infer.py \ --test-list test.tsv \ --res-dir results/test ``` + - `--model-name` can be `zipvoice` or `zipvoice_distill`, which are models before and after distillation, respectively. - Each line of `test.tsv` is in the format of `{wav_name}\t{prompt_transcription}\t{prompt_wav}\t{text}`. @@ -357,4 +393,4 @@ on three test sets, i.e., LibriSpeech-PC test-clean, Seed-TTS test-en and Seed-T journal={arXiv preprint arXiv:2506.13053}, year={2025}, } -``` \ No newline at end of file +``` diff --git a/egs/zipvoice/assets/prompt-en.wav b/egs/zipvoice/assets/prompt-en.wav new file mode 100644 index 000000000..b7047ce9b Binary files /dev/null and b/egs/zipvoice/assets/prompt-en.wav differ diff --git a/egs/zipvoice/assets/prompt-zh.wav b/egs/zipvoice/assets/prompt-zh.wav new file mode 100644 index 000000000..af1366329 Binary files /dev/null and b/egs/zipvoice/assets/prompt-zh.wav differ diff --git a/egs/zipvoice/local/compute_fbank_libritts.py b/egs/zipvoice/local/compute_fbank_libritts.py old mode 100755 new mode 100644 diff --git a/egs/zipvoice/local/validate_manifest.py b/egs/zipvoice/local/validate_manifest.py old mode 100755 new mode 100644 diff --git a/egs/zipvoice/local/evaluate.sh b/egs/zipvoice/scripts/evaluate.sh similarity index 100% rename from egs/zipvoice/local/evaluate.sh rename to egs/zipvoice/scripts/evaluate.sh diff --git a/egs/zipvoice/scripts/prepare.sh b/egs/zipvoice/scripts/prepare.sh new file mode 100755 index 000000000..011301f2d --- /dev/null +++ b/egs/zipvoice/scripts/prepare.sh @@ -0,0 +1,232 @@ +#!/usr/bin/env bash + +# fix segmentation fault reported in https://github.com/k2-fsa/icefall/issues/674 +export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python + +# add icefall to PYTHONPATH +export PYTHONPATH=../../../:$PYTHONPATH + +set -eou pipefail + +stage=0 +stop_stage=100 + +token_type=bpe # bpe, letter, phone +bpe_vocab_size=500 + +nj=32 + +dl_dir=$PWD/download + +. shared/parse_options.sh || exit 1 + +# All files generated by this script are saved in "data". +# You can safely remove "data" and rerun this script to regenerate it. +mkdir -p data + +log() { + # This function is from espnet + local fname=${BASH_SOURCE[1]##*/} + echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*" +} + +log "dl_dir: $dl_dir" + + +if [ $stage -le -2 ] && [ $stop_stage -ge -2 ]; then + + if [ ! -d $dl_dir/xvector_nnet_1a_libritts_clean_460 ]; then + log "Downloading x-vector" + + git clone https://huggingface.co/datasets/zrjin/xvector_nnet_1a_libritts_clean_460 $dl_dir/xvector_nnet_1a_libritts_clean_460 + + mkdir -p exp/xvector_nnet_1a/ + cp -r $dl_dir/xvector_nnet_1a_libritts_clean_460/* exp/xvector_nnet_1a/ + fi + +fi + +if [ $stage -le -1 ] && [ $stop_stage -ge -1 ]; then + log "Stage -1: build monotonic_align lib" + if [ ! -d vits/monotonic_align/build ]; then + cd vits/monotonic_align + python setup.py build_ext --inplace + cd ../../ + else + log "monotonic_align lib already built" + fi +fi + +if [ $stage -le 0 ] && [ $stop_stage -ge 0 ]; then + log "Stage 0: Download data" + + # If you have pre-downloaded it to /path/to/LibriTTS, + # you can create a symlink + # + # ln -sfv /path/to/LibriTTS $dl_dir/LibriTTS + # + if [ ! -d $dl_dir/LibriTTS ]; then + lhotse download libritts $dl_dir + fi + +fi + +if [ $stage -le 1 ] && [ $stop_stage -ge 1 ]; then + log "Stage 1: Prepare LibriTTS manifest" + # We assume that you have downloaded the LibriTTS corpus + # to $dl_dir/LibriTTS + mkdir -p data/manifests + if [ ! -e data/manifests/.libritts.done ]; then + lhotse prepare libritts --num-jobs ${nj} $dl_dir/LibriTTS data/manifests + touch data/manifests/.libritts.done + fi +fi + +if [ $stage -le 2 ] && [ $stop_stage -ge 2 ]; then + log "Stage 2: Compute Fbank for LibriTTS" + mkdir -p data/fbank + + for subset in train-clean-100 train-clean-360 train-other-500 dev-clean test-clean; do + python local/compute_fbank.py --dataset libritts --subset ${subset} + done + + # Here we shuffle and combine the train-clean-100, train-clean-360 and + # train-other-500 together to form the training set. + if [ ! -f data/fbank/libritts_cuts_train-all-shuf.jsonl.gz ]; then + cat <(gunzip -c data/fbank/libritts_cuts_train-clean-100.jsonl.gz) \ + <(gunzip -c data/fbank/libritts_cuts_train-clean-360.jsonl.gz) \ + <(gunzip -c data/fbank/libritts_cuts_train-other-500.jsonl.gz) | \ + shuf | gzip -c > data/fbank/libritts_cuts_train-all-shuf.jsonl.gz + fi + + if [ ! -f data/fbank/libritts_cuts_train-clean-460.jsonl.gz ]; then + cat <(gunzip -c data/fbank/libritts_cuts_train-clean-100.jsonl.gz) \ + <(gunzip -c data/fbank/libritts_cuts_train-clean-360.jsonl.gz) | \ + shuf | gzip -c > data/fbank/libritts_cuts_train-clean-460.jsonl.gz + fi + + if [ ! -e data/fbank/.libritts-validated.done ]; then + log "Validating data/fbank for LibriTTS" + ./local/validate_manifest.py \ + data/fbank/libritts_cuts_train-all-shuf.jsonl.gz + touch data/fbank/.libritts-validated.done + fi +fi + +if [ $stage -le 3 ] && [ $stop_stage -ge 3 ]; then + log "Stage 3: Prepare tokens.txt" + + if [ $token_type == "bpe" ] || [ $token_type == "letter" ]; then + if [ ! -e data/texts.txt ]; then + ./local/export_normalized_texts.py --output data/texts.txt \ + --manifests data/fbank/libritts_cuts_train-all-shuf.jsonl.gz + fi + fi + + if [ $token_type == "bpe" ]; then + mkdir -p data/lang_bpe_${bpe_vocab_size} + if [ ! -e data/lang_bpe_${bpe_vocab_size}/tokens.txt ]; then + ./local/train_bpe_model.py --transcript data/texts.txt \ + --lang-dir data/lang_bpe_${bpe_vocab_size} \ + --vocab-size $bpe_vocab_size + fi + fi + + if [ $token_type == "phone" ]; then + mkdir -p data/lang_phone + ./local/export_tokens.py --token-type phone \ + --output data/lang_phone/tokens.txt + fi + + if [ $token_type == "letter" ]; then + mkdir -p data/lang_letter + ./local/export_tokens.py --token-type letter \ + --texts data/texts.txt \ + --output data/lang_letter/tokens.txt + fi +fi + +if [ $stage -le 4 ] && [ $stop_stage -ge 4 ]; then + log "Stage 4: Download and prepare librispeech-pc test clean for testing." + + if [ ! -e $dl_dir/test-clean.tar.gz ]; then + wget https://huggingface.co/datasets/k2-fsa/LibriSpeech/resolve/main/test-clean.tar.gz -P $dl_dir + fi + # For China users. + if [ ! -e $dl_dir/test-clean.tar.gz ]; then + wget https://hf-mirror.com/datasets/k2-fsa/LibriSpeech/resolve/main/test-clean.tar.gz -P $dl_dir + fi + + if [ ! -d $dl_dir/LibriSpeech/test-clean ]; then + tar -xvf $dl_dir/test-clean.tar.gz -C $dl_dir + fi + + mkdir -p $dl_dir/LibriSpeech-PC + if [ ! -e $dl_dir/LibriSpeech-PC/test-clean.json ]; then + wget https://us.openslr.org/resources/145/manifests.tar.gz -P $dl_dir/LibriSpeech-PC + tar -xvf $dl_dir/LibriSpeech-PC/manifests.tar.gz -C $dl_dir/LibriSpeech-PC + fi + + python local/compute_fbank.py --dataset librispeech --subset test-clean + python local/prepare_prompts_librispeech_test_clean.py +fi + +if [ $stage -le 5 ] && [ $stop_stage -ge 5 ]; then + log "Stage 5: Compute Spectrogram for LibriTTS (for VITS system)" + mkdir -p data/spectrogram + if [ ! -e data/spectrogram/.libritts.done ]; then + ./local/compute_spectrogram_libritts.py --sampling-rate $sampling_rate + touch data/spectrogram/.libritts.done + fi + + # Here we shuffle and combine the train-clean-100, train-clean-360 and + # train-other-500 together to form the training set. + if [ ! -f data/spectrogram/libritts_cuts_train-all-shuf.jsonl.gz ]; then + cat <(gunzip -c data/spectrogram/libritts_cuts_train-clean-100.jsonl.gz) \ + <(gunzip -c data/spectrogram/libritts_cuts_train-clean-360.jsonl.gz) \ + <(gunzip -c data/spectrogram/libritts_cuts_train-other-500.jsonl.gz) | \ + shuf | gzip -c > data/spectrogram/libritts_cuts_train-all-shuf.jsonl.gz + fi + + # Here we shuffle and combine the train-clean-100, train-clean-360 + # together to form the training set. + if [ ! -f data/spectrogram/libritts_cuts_train-clean-460.jsonl.gz ]; then + cat <(gunzip -c data/spectrogram/libritts_cuts_train-clean-100.jsonl.gz) \ + <(gunzip -c data/spectrogram/libritts_cuts_train-clean-360.jsonl.gz) | \ + shuf | gzip -c > data/spectrogram/libritts_cuts_train-clean-460.jsonl.gz + fi + + if [ ! -e data/spectrogram/.libritts-validated.done ]; then + log "Validating data/spectrogram for LibriTTS" + ./local/validate_manifest.py \ + data/spectrogram/libritts_cuts_train-all-shuf.jsonl.gz + touch data/spectrogram/.libritts-validated.done + fi +fi + +audio_feats_dir=data/tokenized +dataset_parts="--dataset-parts all" # debug "-p dev-clean -p test-clean" +if [ $stage -le 6 ] && [ $stop_stage -ge 6 ]; then + log "Stage 6: Tokenize/Fbank LibriTTS for valle" + mkdir -p ${audio_feats_dir} + if [ ! -e ${audio_feats_dir}/.libritts.tokenize.done ]; then + python3 ./local/compute_neural_codec_and_prepare_text_tokens.py --dataset-parts "${dataset_parts}" \ + --audio-extractor "Encodec" \ + --batch-duration 400 \ + --src-dir "data/manifests" \ + --output-dir "${audio_feats_dir}" + fi + touch ${audio_feats_dir}/.libritts.tokenize.done + + lhotse combine \ + ${audio_feats_dir}/libritts_cuts_train-clean-100.jsonl.gz \ + ${audio_feats_dir}/libritts_cuts_train-clean-360.jsonl.gz \ + ${audio_feats_dir}/libritts_cuts_train-other-500.jsonl.gz \ + ${audio_feats_dir}/cuts_train.jsonl.gz + lhotse copy \ + ${audio_feats_dir}/libritts_cuts_dev-clean.jsonl.gz \ + ${audio_feats_dir}/cuts_dev.jsonl.gz + lhotse copy \ + ${audio_feats_dir}/libritts_cuts_test-clean.jsonl.gz \ + ${audio_feats_dir}/cuts_test.jsonl.gz +fi diff --git a/egs/zipvoice/local/prepare_libritts.sh b/egs/zipvoice/scripts/prepare_libritts.sh similarity index 99% rename from egs/zipvoice/local/prepare_libritts.sh rename to egs/zipvoice/scripts/prepare_libritts.sh index b35065bb1..da35eec7b 100755 --- a/egs/zipvoice/local/prepare_libritts.sh +++ b/egs/zipvoice/scripts/prepare_libritts.sh @@ -85,4 +85,4 @@ if [ $stage -le 3 ] && [ $stop_stage -ge 3 ]; then if [ ! -e data/tokens_libritts.txt ]; then ./local/prepare_token_file_libritts.py --tokens data/tokens_libritts.txt fi -fi \ No newline at end of file +fi diff --git a/egs/zipvoice/scripts/run_eval.sh b/egs/zipvoice/scripts/run_eval.sh new file mode 100644 index 000000000..c4f00548d --- /dev/null +++ b/egs/zipvoice/scripts/run_eval.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash + +export PYTHONPATH=../../../:$PYTHONPATH + +stage=1 +stop_stage=10 +generated_wav_path="flow-matching/exp/generated_wavs" + +. shared/parse_options.sh || exit 1 + + +log() { + # This function is from espnet + local fname=${BASH_SOURCE[1]##*/} + echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*" +} + +if [ $stage -le -2 ] && [ $stop_stage -ge -2 ]; then + log "Stage -2: Install dependencies and download models" + + pip install -r requirements-eval.txt + pip install git+https://github.com/sarulab-speech/UTMOSv2.git + modelscope download --model k2-fsa/TTS_eval_models --local_dir TTS_eval_models +fi + + +if [ $stage -le -1 ] && [ $stop_stage -ge -1 ]; then + log "Stage -1: Prepare evaluation data." + + mkdir -p data/reference/librispeech-test-clean + + gunzip -c data/fbank/librispeech_cuts_with_prompts_test-clean.jsonl.gz | \ + jq -r '"\(.recording.sources[0].source)"' | \ + awk '{split($1, a, "/"); cmd="cp "$1" data/reference/librispeech-test-clean/"a[length(a)]; print cmd; system(cmd)}' + + + mkdir -p data/reference/librispeech-test-clean-prompt + gunzip -c data/fbank/librispeech_cuts_with_prompts_test-clean.jsonl.gz | \ + jq -r '"\(.custom.prompt.recording.sources[0].source) \(.recording.sources[0].source)"' | \ + awk '{split($2, a, "/"); cmd="cp "$1" data/reference/librispeech-test-clean-prompt/"a[length(a)]; print cmd; system(cmd)}' +fi + + +if [ $stage -le 1 ] && [ $stop_stage -ge 1 ]; then + log "Stage 1: Evaluate the model with FSD." + + python local/evaluate_fsd.py --real-path data/reference/librispeech-test-clean \ + --eval-path $generated_wav_path +fi + +if [ $stage -le 2 ] && [ $stop_stage -ge 2 ]; then + log "Stage 2: Evaluate the model with SIM." + + python local/evaluate_sim.py --real-path data/reference/librispeech-test-clean \ + --eval-path $generated_wav_path +fi + +if [ $stage -le 3 ] && [ $stop_stage -ge 3 ]; then + log "Stage 3: Evaluate the model with UTMOS." + + python local/evaluate_utmos.py --wav-path $generated_wav_path +fi + +if [ $stage -le 4 ] && [ $stop_stage -ge 4 ]; then + log "Stage 4: Evaluate the model with UTMOSv2." + + python local/evaluate_utmosv2.py --wav-path $generated_wav_path +fi + +if [ $stage -le 5 ] && [ $stop_stage -ge 5 ]; then + log "Stage 5: Evaluate the model with WER." + + python local/evaluate_wer_hubert.py --wav-path $generated_wav_path \ + --decode-path $generated_wav_path/decode +fi diff --git a/egs/zipvoice/zipvoice/generate_averaged_model.py b/egs/zipvoice/zipvoice/generate_averaged_model.py old mode 100755 new mode 100644 diff --git a/egs/zipvoice/zipvoice/zipvoice_infer.py b/egs/zipvoice/zipvoice/zipvoice_infer.py index 472ad700d..16cf8e039 100644 --- a/egs/zipvoice/zipvoice/zipvoice_infer.py +++ b/egs/zipvoice/zipvoice/zipvoice_infer.py @@ -23,7 +23,7 @@ This script generates speech with our pre-trained ZipVoice or Usage: Note: If you having trouble connecting to HuggingFace, - you try switch endpoint to mirror site: + try switching endpoint to mirror site: export HF_ENDPOINT=https://hf-mirror.com @@ -55,7 +55,6 @@ import os import numpy as np import safetensors.torch -import soundfile as sf import torch import torch.nn as nn import torchaudio @@ -115,15 +114,20 @@ def get_parser(): "--res-dir", type=str, default="results", - help="Path name of the generated wavs dir, " - "used when decdode-list is not None", + help=""" + Path name of the generated wavs dir, + used when test-list is not None + """, ) parser.add_argument( "--res-wav-path", type=str, default="result.wav", - help="Path name of the generated wav path, " "used when decdode-list is None", + help=""" + Path name of the generated wav path, + used when test-list is None + """, ) parser.add_argument( @@ -456,8 +460,7 @@ def generate_sentence( # Adjust wav volume if necessary if prompt_rms < target_rms: wav = wav * prompt_rms / target_rms - wav = wav[0].cpu().numpy() - sf.write(save_path, wav, sampling_rate) + torchaudio.save(save_path, wav.cpu(), sample_rate=sampling_rate) return metrics