mirror of
https://github.com/k2-fsa/icefall.git
synced 2025-08-09 18:12:19 +00:00
270 lines
18 KiB
HTML
270 lines
18 KiB
HTML
<!DOCTYPE html>
|
||
<html class="writer-html5" lang="en">
|
||
<head>
|
||
<meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||
<title>Finetune from a supervised pre-trained Zipformer model — icefall 0.1 documentation</title>
|
||
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=fa44fd50" />
|
||
<link rel="stylesheet" type="text/css" href="../../../_static/css/theme.css?v=19f00094" />
|
||
|
||
|
||
<!--[if lt IE 9]>
|
||
<script src="../../../_static/js/html5shiv.min.js"></script>
|
||
<![endif]-->
|
||
|
||
<script src="../../../_static/jquery.js?v=5d32c60e"></script>
|
||
<script src="../../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||
<script data-url_root="../../../" id="documentation_options" src="../../../_static/documentation_options.js?v=e031e9a9"></script>
|
||
<script src="../../../_static/doctools.js?v=888ff710"></script>
|
||
<script src="../../../_static/sphinx_highlight.js?v=4825356b"></script>
|
||
<script src="../../../_static/js/theme.js"></script>
|
||
<link rel="index" title="Index" href="../../../genindex.html" />
|
||
<link rel="search" title="Search" href="../../../search.html" />
|
||
<link rel="next" title="Contributing" href="../../../contributing/index.html" />
|
||
<link rel="prev" title="Fine-tune a pre-trained model" href="../index.html" />
|
||
</head>
|
||
|
||
<body class="wy-body-for-nav">
|
||
<div class="wy-grid-for-nav">
|
||
<nav data-toggle="wy-nav-shift" class="wy-nav-side">
|
||
<div class="wy-side-scroll">
|
||
<div class="wy-side-nav-search" >
|
||
|
||
|
||
|
||
<a href="../../../index.html" class="icon icon-home">
|
||
icefall
|
||
</a>
|
||
<div role="search">
|
||
<form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
|
||
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
||
<input type="hidden" name="check_keywords" value="yes" />
|
||
<input type="hidden" name="area" value="default" />
|
||
</form>
|
||
</div>
|
||
</div><div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu">
|
||
<p class="caption" role="heading"><span class="caption-text">Contents:</span></p>
|
||
<ul>
|
||
<li class="toctree-l1"><a class="reference internal" href="../../../for-dummies/index.html">Icefall for dummies tutorial</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../../../installation/index.html">Installation</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../../../docker/index.html">Docker</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../../../faqs.html">Frequently Asked Questions (FAQs)</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../../../model-export/index.html">Model export</a></li>
|
||
</ul>
|
||
<ul class="current">
|
||
<li class="toctree-l1 current"><a class="reference internal" href="../../index.html">Recipes</a><ul class="current">
|
||
<li class="toctree-l2"><a class="reference internal" href="../../Non-streaming-ASR/index.html">Non Streaming ASR</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../../Streaming-ASR/index.html">Streaming ASR</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../../RNN-LM/index.html">RNN-LM</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../../TTS/index.html">TTS</a></li>
|
||
<li class="toctree-l2 current"><a class="reference internal" href="../index.html">Fine-tune a pre-trained model</a><ul class="current">
|
||
<li class="toctree-l3 current"><a class="current reference internal" href="#">Finetune from a supervised pre-trained Zipformer model</a><ul>
|
||
<li class="toctree-l4"><a class="reference internal" href="#data-preparation">Data preparation</a></li>
|
||
<li class="toctree-l4"><a class="reference internal" href="#model-preparation">Model preparation</a></li>
|
||
<li class="toctree-l4"><a class="reference internal" href="#fine-tune">Fine-tune</a></li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
<ul>
|
||
<li class="toctree-l1"><a class="reference internal" href="../../../contributing/index.html">Contributing</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../../../huggingface/index.html">Huggingface</a></li>
|
||
</ul>
|
||
<ul>
|
||
<li class="toctree-l1"><a class="reference internal" href="../../../decoding-with-langugage-models/index.html">Decoding with language models</a></li>
|
||
</ul>
|
||
|
||
</div>
|
||
</div>
|
||
</nav>
|
||
|
||
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"><nav class="wy-nav-top" aria-label="Mobile navigation menu" >
|
||
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
|
||
<a href="../../../index.html">icefall</a>
|
||
</nav>
|
||
|
||
<div class="wy-nav-content">
|
||
<div class="rst-content">
|
||
<div role="navigation" aria-label="Page navigation">
|
||
<ul class="wy-breadcrumbs">
|
||
<li><a href="../../../index.html" class="icon icon-home" aria-label="Home"></a></li>
|
||
<li class="breadcrumb-item"><a href="../../index.html">Recipes</a></li>
|
||
<li class="breadcrumb-item"><a href="../index.html">Fine-tune a pre-trained model</a></li>
|
||
<li class="breadcrumb-item active">Finetune from a supervised pre-trained Zipformer model</li>
|
||
<li class="wy-breadcrumbs-aside">
|
||
<a href="https://github.com/k2-fsa/icefall/blob/master/docs/source/recipes/Finetune/from_supervised/finetune_zipformer.rst" class="fa fa-github"> Edit on GitHub</a>
|
||
</li>
|
||
</ul>
|
||
<hr/>
|
||
</div>
|
||
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
|
||
<div itemprop="articleBody">
|
||
|
||
<section id="finetune-from-a-supervised-pre-trained-zipformer-model">
|
||
<h1>Finetune from a supervised pre-trained Zipformer model<a class="headerlink" href="#finetune-from-a-supervised-pre-trained-zipformer-model" title="Permalink to this heading"></a></h1>
|
||
<p>This tutorial shows you how to fine-tune a supervised pre-trained <strong>Zipformer</strong>
|
||
transducer model on a new dataset.</p>
|
||
<div class="admonition hint">
|
||
<p class="admonition-title">Hint</p>
|
||
<p>We assume you have read the page <a class="reference internal" href="../../../installation/index.html#install-icefall"><span class="std std-ref">Installation</span></a> and have setup
|
||
the environment for <code class="docutils literal notranslate"><span class="pre">icefall</span></code>.</p>
|
||
</div>
|
||
<div class="admonition hint">
|
||
<p class="admonition-title">Hint</p>
|
||
<p>We recommend you to use a GPU or several GPUs to run this recipe</p>
|
||
</div>
|
||
<p>For illustration purpose, we fine-tune the Zipformer transducer model
|
||
pre-trained on <a class="reference external" href="https://www.openslr.org/12">LibriSpeech</a> on the small subset of <a class="reference external" href="https://github.com/SpeechColab/GigaSpeech">GigaSpeech</a>. You could use your
|
||
own data for fine-tuning if you create a manifest for your new dataset.</p>
|
||
<section id="data-preparation">
|
||
<h2>Data preparation<a class="headerlink" href="#data-preparation" title="Permalink to this heading"></a></h2>
|
||
<p>Please follow the instructions in the <a class="reference external" href="https://github.com/k2-fsa/icefall/tree/master/egs/gigaspeech/ASR">GigaSpeech recipe</a>
|
||
to prepare the fine-tune data used in this tutorial. We only require the small subset in GigaSpeech for this tutorial.</p>
|
||
</section>
|
||
<section id="model-preparation">
|
||
<h2>Model preparation<a class="headerlink" href="#model-preparation" title="Permalink to this heading"></a></h2>
|
||
<p>We are using the Zipformer model trained on full LibriSpeech (960 hours) as the intialization. The
|
||
checkpoint of the model can be downloaded via the following command:</p>
|
||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>$<span class="w"> </span><span class="nv">GIT_LFS_SKIP_SMUDGE</span><span class="o">=</span><span class="m">1</span><span class="w"> </span>git<span class="w"> </span>clone<span class="w"> </span>https://huggingface.co/Zengwei/icefall-asr-librispeech-zipformer-2023-05-15
|
||
$<span class="w"> </span><span class="nb">cd</span><span class="w"> </span>icefall-asr-librispeech-zipformer-2023-05-15/exp
|
||
$<span class="w"> </span>git<span class="w"> </span>lfs<span class="w"> </span>pull<span class="w"> </span>--include<span class="w"> </span><span class="s2">"pretrained.pt"</span>
|
||
$<span class="w"> </span>ln<span class="w"> </span>-s<span class="w"> </span>pretrained.pt<span class="w"> </span>epoch-99.pt
|
||
$<span class="w"> </span><span class="nb">cd</span><span class="w"> </span>../data/lang_bpe_500
|
||
$<span class="w"> </span>git<span class="w"> </span>lfs<span class="w"> </span>pull<span class="w"> </span>--include<span class="w"> </span>bpe.model
|
||
$<span class="w"> </span><span class="nb">cd</span><span class="w"> </span>../../..
|
||
</pre></div>
|
||
</div>
|
||
<p>Before fine-tuning, let’s test the model’s WER on the new domain. The following command performs
|
||
decoding on the GigaSpeech test sets:</p>
|
||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>./zipformer/decode_gigaspeech.py<span class="w"> </span><span class="se">\</span>
|
||
<span class="w"> </span>--epoch<span class="w"> </span><span class="m">99</span><span class="w"> </span><span class="se">\</span>
|
||
<span class="w"> </span>--avg<span class="w"> </span><span class="m">1</span><span class="w"> </span><span class="se">\</span>
|
||
<span class="w"> </span>--exp-dir<span class="w"> </span>icefall-asr-librispeech-zipformer-2023-05-15/exp<span class="w"> </span><span class="se">\</span>
|
||
<span class="w"> </span>--use-averaged-model<span class="w"> </span><span class="m">0</span><span class="w"> </span><span class="se">\</span>
|
||
<span class="w"> </span>--max-duration<span class="w"> </span><span class="m">1000</span><span class="w"> </span><span class="se">\</span>
|
||
<span class="w"> </span>--decoding-method<span class="w"> </span>greedy_search
|
||
</pre></div>
|
||
</div>
|
||
<p>You should see the following numbers:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">For</span> <span class="n">dev</span><span class="p">,</span> <span class="n">WER</span> <span class="n">of</span> <span class="n">different</span> <span class="n">settings</span> <span class="n">are</span><span class="p">:</span>
|
||
<span class="n">greedy_search</span> <span class="mf">20.06</span> <span class="n">best</span> <span class="k">for</span> <span class="n">dev</span>
|
||
|
||
<span class="n">For</span> <span class="n">test</span><span class="p">,</span> <span class="n">WER</span> <span class="n">of</span> <span class="n">different</span> <span class="n">settings</span> <span class="n">are</span><span class="p">:</span>
|
||
<span class="n">greedy_search</span> <span class="mf">19.27</span> <span class="n">best</span> <span class="k">for</span> <span class="n">test</span>
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
<section id="fine-tune">
|
||
<h2>Fine-tune<a class="headerlink" href="#fine-tune" title="Permalink to this heading"></a></h2>
|
||
<p>Since LibriSpeech and GigaSpeech are both English dataset, we can initialize the whole
|
||
Zipformer model with the checkpoint downloaded in the previous step (otherwise we should consider
|
||
initializing the stateless decoder and joiner from scratch due to the mismatch of the output
|
||
vocabulary). The following command starts a fine-tuning experiment:</p>
|
||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>$<span class="w"> </span><span class="nv">use_mux</span><span class="o">=</span><span class="m">0</span>
|
||
$<span class="w"> </span><span class="nv">do_finetune</span><span class="o">=</span><span class="m">1</span>
|
||
|
||
$<span class="w"> </span>./zipformer/finetune.py<span class="w"> </span><span class="se">\</span>
|
||
<span class="w"> </span>--world-size<span class="w"> </span><span class="m">2</span><span class="w"> </span><span class="se">\</span>
|
||
<span class="w"> </span>--num-epochs<span class="w"> </span><span class="m">20</span><span class="w"> </span><span class="se">\</span>
|
||
<span class="w"> </span>--start-epoch<span class="w"> </span><span class="m">1</span><span class="w"> </span><span class="se">\</span>
|
||
<span class="w"> </span>--exp-dir<span class="w"> </span>zipformer/exp_giga_finetune<span class="si">${</span><span class="nv">do_finetune</span><span class="si">}</span>_mux<span class="si">${</span><span class="nv">use_mux</span><span class="si">}</span><span class="w"> </span><span class="se">\</span>
|
||
<span class="w"> </span>--use-fp16<span class="w"> </span><span class="m">1</span><span class="w"> </span><span class="se">\</span>
|
||
<span class="w"> </span>--base-lr<span class="w"> </span><span class="m">0</span>.0045<span class="w"> </span><span class="se">\</span>
|
||
<span class="w"> </span>--bpe-model<span class="w"> </span>data/lang_bpe_500/bpe.model<span class="w"> </span><span class="se">\</span>
|
||
<span class="w"> </span>--do-finetune<span class="w"> </span><span class="nv">$do_finetune</span><span class="w"> </span><span class="se">\</span>
|
||
<span class="w"> </span>--use-mux<span class="w"> </span><span class="nv">$use_mux</span><span class="w"> </span><span class="se">\</span>
|
||
<span class="w"> </span>--master-port<span class="w"> </span><span class="m">13024</span><span class="w"> </span><span class="se">\</span>
|
||
<span class="w"> </span>--finetune-ckpt<span class="w"> </span>icefall-asr-librispeech-zipformer-2023-05-15/exp/pretrained.pt<span class="w"> </span><span class="se">\</span>
|
||
<span class="w"> </span>--max-duration<span class="w"> </span><span class="m">1000</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>The following arguments are related to fine-tuning:</p>
|
||
<ul class="simple">
|
||
<li><dl class="simple">
|
||
<dt><code class="docutils literal notranslate"><span class="pre">--base-lr</span></code></dt><dd><p>The learning rate used for fine-tuning. We suggest to set a <strong>small</strong> learning rate for fine-tuning,
|
||
otherwise the model may forget the initialization very quickly. A reasonable value should be around
|
||
1/10 of the original lr, i.e 0.0045.</p>
|
||
</dd>
|
||
</dl>
|
||
</li>
|
||
<li><dl class="simple">
|
||
<dt><code class="docutils literal notranslate"><span class="pre">--do-finetune</span></code></dt><dd><p>If True, do fine-tuning by initializing the model from a pre-trained checkpoint.
|
||
<strong>Note that if you want to resume your fine-tuning experiment from certain epochs, you
|
||
need to set this to False.</strong></p>
|
||
</dd>
|
||
</dl>
|
||
</li>
|
||
<li><dl class="simple">
|
||
<dt><code class="docutils literal notranslate"><span class="pre">--finetune-ckpt</span></code></dt><dd><p>The path to the pre-trained checkpoint (used for initialization).</p>
|
||
</dd>
|
||
</dl>
|
||
</li>
|
||
<li><dl class="simple">
|
||
<dt><code class="docutils literal notranslate"><span class="pre">--use-mux</span></code></dt><dd><p>If True, mix the fine-tune data with the original training data by using <a class="reference external" href="https://lhotse.readthedocs.io/en/latest/api.html#lhotse.supervision.SupervisionSet.mux">CutSet.mux</a>
|
||
This helps maintain the model’s performance on the original domain if the original training
|
||
is available. <strong>If you don’t have the original training data, please set it to False.</strong></p>
|
||
</dd>
|
||
</dl>
|
||
</li>
|
||
</ul>
|
||
<p>After fine-tuning, let’s test the WERs. You can do this via the following command:</p>
|
||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>$<span class="w"> </span><span class="nv">use_mux</span><span class="o">=</span><span class="m">0</span>
|
||
$<span class="w"> </span><span class="nv">do_finetune</span><span class="o">=</span><span class="m">1</span>
|
||
$<span class="w"> </span>./zipformer/decode_gigaspeech.py<span class="w"> </span><span class="se">\</span>
|
||
<span class="w"> </span>--epoch<span class="w"> </span><span class="m">20</span><span class="w"> </span><span class="se">\</span>
|
||
<span class="w"> </span>--avg<span class="w"> </span><span class="m">10</span><span class="w"> </span><span class="se">\</span>
|
||
<span class="w"> </span>--exp-dir<span class="w"> </span>zipformer/exp_giga_finetune<span class="si">${</span><span class="nv">do_finetune</span><span class="si">}</span>_mux<span class="si">${</span><span class="nv">use_mux</span><span class="si">}</span><span class="w"> </span><span class="se">\</span>
|
||
<span class="w"> </span>--use-averaged-model<span class="w"> </span><span class="m">1</span><span class="w"> </span><span class="se">\</span>
|
||
<span class="w"> </span>--max-duration<span class="w"> </span><span class="m">1000</span><span class="w"> </span><span class="se">\</span>
|
||
<span class="w"> </span>--decoding-method<span class="w"> </span>greedy_search
|
||
</pre></div>
|
||
</div>
|
||
<p>You should see numbers similar to the ones below:</p>
|
||
<div class="highlight-text notranslate"><div class="highlight"><pre><span></span>For dev, WER of different settings are:
|
||
greedy_search 13.47 best for dev
|
||
|
||
For test, WER of different settings are:
|
||
greedy_search 13.66 best for test
|
||
</pre></div>
|
||
</div>
|
||
<p>Compared to the original checkpoint, the fine-tuned model achieves much lower WERs
|
||
on the GigaSpeech test sets.</p>
|
||
</section>
|
||
</section>
|
||
|
||
|
||
</div>
|
||
</div>
|
||
<footer><div class="rst-footer-buttons" role="navigation" aria-label="Footer">
|
||
<a href="../index.html" class="btn btn-neutral float-left" title="Fine-tune a pre-trained model" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left" aria-hidden="true"></span> Previous</a>
|
||
<a href="../../../contributing/index.html" class="btn btn-neutral float-right" title="Contributing" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right" aria-hidden="true"></span></a>
|
||
</div>
|
||
|
||
<hr/>
|
||
|
||
<div role="contentinfo">
|
||
<p>© Copyright 2021, icefall development team.</p>
|
||
</div>
|
||
|
||
Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
|
||
<a href="https://github.com/readthedocs/sphinx_rtd_theme">theme</a>
|
||
provided by <a href="https://readthedocs.org">Read the Docs</a>.
|
||
|
||
|
||
</footer>
|
||
</div>
|
||
</div>
|
||
</section>
|
||
</div>
|
||
<script>
|
||
jQuery(function () {
|
||
SphinxRtdTheme.Navigation.enable(true);
|
||
});
|
||
</script>
|
||
|
||
</body>
|
||
</html> |