From 1abf2863bbcd35ca02f7282154e304424ddddeec Mon Sep 17 00:00:00 2001 From: Teo Wen Shen <36886809+teowenshen@users.noreply.github.com> Date: Sun, 30 Oct 2022 23:47:21 +0900 Subject: [PATCH] fix typos (#639) --- egs/csj/ASR/.gitignore | 5 +- egs/csj/ASR/local/compute_fbank_musan.py | 128 ++++++++++++++++++++++- egs/csj/ASR/prepare.sh | 24 ++--- 3 files changed, 142 insertions(+), 15 deletions(-) mode change 120000 => 100644 egs/csj/ASR/local/compute_fbank_musan.py diff --git a/egs/csj/ASR/.gitignore b/egs/csj/ASR/.gitignore index c0a162e20..5d965832e 100644 --- a/egs/csj/ASR/.gitignore +++ b/egs/csj/ASR/.gitignore @@ -1,7 +1,8 @@ -librispeech_*.* +librispeech_* todelete* lang* notify_tg.py finetune_* misc.ini -.vscode/* \ No newline at end of file +.vscode/* +offline/* \ No newline at end of file diff --git a/egs/csj/ASR/local/compute_fbank_musan.py b/egs/csj/ASR/local/compute_fbank_musan.py deleted file mode 120000 index 5833f2484..000000000 --- a/egs/csj/ASR/local/compute_fbank_musan.py +++ /dev/null @@ -1 +0,0 @@ -../../../librispeech/ASR/local/compute_fbank_musan.py \ No newline at end of file diff --git a/egs/csj/ASR/local/compute_fbank_musan.py b/egs/csj/ASR/local/compute_fbank_musan.py new file mode 100644 index 000000000..44a33c4eb --- /dev/null +++ b/egs/csj/ASR/local/compute_fbank_musan.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 +# Copyright 2021 Xiaomi Corp. (authors: Fangjun Kuang) +# +# See ../../../../LICENSE for clarification regarding multiple authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import logging +import os +from pathlib import Path + +import torch +from lhotse import CutSet, Fbank, FbankConfig, LilcomChunkyWriter, combine +from lhotse.recipes.utils import read_manifests_if_cached + +from icefall.utils import get_executor + + +ARGPARSE_DESCRIPTION = """ +This file computes fbank features of the musan dataset. +It looks for manifests in the directory data/manifests. + +The generated fbank features are saved in data/fbank. +""" + +# Torch's multithreaded behavior needs to be disabled or +# it wastes a lot of CPU and slow things down. +# Do this outside of main() in case it needs to take effect +# even when we are not invoking the main (e.g. when spawning subprocesses). +torch.set_num_threads(1) +torch.set_num_interop_threads(1) + + +def compute_fbank_musan(manifest_dir: Path, fbank_dir: Path): + # src_dir = Path("data/manifests") + # output_dir = Path("data/fbank") + num_jobs = min(15, os.cpu_count()) + num_mel_bins = 80 + + dataset_parts = ( + "music", + "speech", + "noise", + ) + prefix = "musan" + suffix = "jsonl.gz" + manifests = read_manifests_if_cached( + dataset_parts=dataset_parts, + output_dir=manifest_dir, + prefix=prefix, + suffix=suffix, + ) + assert manifests is not None + + assert len(manifests) == len(dataset_parts), ( + len(manifests), + len(dataset_parts), + list(manifests.keys()), + dataset_parts, + ) + + musan_cuts_path = fbank_dir / "musan_cuts.jsonl.gz" + + if musan_cuts_path.is_file(): + logging.info(f"{musan_cuts_path} already exists - skipping") + return + + logging.info("Extracting features for Musan") + + extractor = Fbank(FbankConfig(num_mel_bins=num_mel_bins)) + + with get_executor() as ex: # Initialize the executor only once. + # create chunks of Musan with duration 5 - 10 seconds + musan_cuts = ( + CutSet.from_manifests( + recordings=combine( + part["recordings"] for part in manifests.values() + ) + ) + .cut_into_windows(10.0) + .filter(lambda c: c.duration > 5) + .compute_and_store_features( + extractor=extractor, + storage_path=f"{fbank_dir}/musan_feats", + num_jobs=num_jobs if ex is None else 80, + executor=ex, + storage_type=LilcomChunkyWriter, + ) + ) + musan_cuts.to_file(musan_cuts_path) + + +def get_args(): + parser = argparse.ArgumentParser( + description=ARGPARSE_DESCRIPTION, + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + + parser.add_argument( + "--manifest-dir", type=Path, help="Path to save manifests" + ) + parser.add_argument( + "--fbank-dir", type=Path, help="Path to save fbank features" + ) + + return parser.parse_args() + + +if __name__ == "__main__": + args = get_args() + formatter = ( + "%(asctime)s %(levelname)s [%(filename)s:%(lineno)d] %(message)s" + ) + + logging.basicConfig(format=formatter, level=logging.INFO) + compute_fbank_musan(args.manifest_dir, args.fbank_dir) diff --git a/egs/csj/ASR/prepare.sh b/egs/csj/ASR/prepare.sh index 269c1ec9a..052748ca6 100755 --- a/egs/csj/ASR/prepare.sh +++ b/egs/csj/ASR/prepare.sh @@ -2,7 +2,7 @@ # We assume the following directories are downloaded. # # - $csj_dir -# CSJ is assumed to be the USB-type directory, which should contain the following subdirectories:- +# CSJ is assumed to be the USB-type directory, which should contain the following subdirectories:- # - DATA (not used in this script) # - DOC (not used in this script) # - MODEL (not used in this script) @@ -30,7 +30,7 @@ # - music # - noise # - speech -# +# # By default, this script produces the original transcript like kaldi and espnet. Optionally, you # can generate other transcript formats by supplying your own config files. A few examples of these # config files can be found in local/conf. @@ -58,15 +58,15 @@ log() { echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*" } -if [ $stage -le 1 ] && [ $stop_stage -ge 1 ]; then +if [ $stage -le 1 ] && [ $stop_stage -ge 1 ]; then log "Stage 1: Prepare CSJ manifest" # If you want to generate more transcript modes, append the path to those config files at c. # Example: lhotse prepare csj $csj_dir $trans_dir $csj_manifest_dir -c local/conf/disfluent.ini # NOTE: In case multiple config files are supplied, the second config file and onwards will inherit - # the segment boundaries of the first config file. - if [ ! -e $csj_manifest_dir/.librispeech.done ]; then + # the segment boundaries of the first config file. + if [ ! -e $csj_manifest_dir/.csj.done ]; then lhotse prepare csj $csj_dir $trans_dir $csj_manifest_dir -j 4 - touch $csj_manifest_dir/.librispeech.done + touch $csj_manifest_dir/.csj.done fi fi @@ -85,20 +85,20 @@ if [ $stage -le 3 ] && [ $stop_stage -ge 3 ]; then python local/compute_fbank_csj.py --manifest-dir $csj_manifest_dir \ --fbank-dir $csj_fbank_dir parts=( - train + train valid eval1 eval2 eval3 ) - for part in ${parts[@]}; do + for part in ${parts[@]}; do python local/validate_manifest.py --manifest $csj_manifest_dir/csj_cuts_$part.jsonl.gz done touch $csj_fbank_dir/.csj-validated.done fi fi -if [ $stage -le 4 ] && [ $stop_stage -ge 4 ]; then +if [ $stage -le 4 ] && [ $stop_stage -ge 4 ]; then log "Stage 4: Prepare CSJ lang" modes=disfluent @@ -117,14 +117,14 @@ if [ $stage -le 5 ] && [ $stop_stage -ge 5 ]; then log "Stage 5: Compute fbank for musan" mkdir -p $musan_fbank_dir - if [ ! -e $musan_fbank_dir/.musan.done ]; then + if [ ! -e $musan_fbank_dir/.musan.done ]; then python local/compute_fbank_musan.py --manifest-dir $musan_manifest_dir --fbank-dir $musan_fbank_dir touch $musan_fbank_dir/.musan.done fi fi -if [ $stage -le 6 ] && [ $stop_stage -ge 6 ]; then +if [ $stage -le 6 ] && [ $stop_stage -ge 6 ]; then log "Stage 6: Show manifest statistics" python local/display_manifest_statistics.py --manifest-dir $csj_manifest_dir > $csj_manifest_dir/manifest_statistics.txt cat $csj_manifest_dir/manifest_statistics.txt -fi \ No newline at end of file +fi