Add BPE training

egs/bengaliai_speech/ASR/local/prepare_lang_bpe.py

@@ -0,0 +1 @@
+../../../librispeech/ASR/local/prepare_lang_bpe.py

egs/bengaliai_speech/ASR/local/preprocess_bengaliai_speech.py

@@ -38,8 +38,8 @@ def get_args():
 
 
 def normalize_text(utt: str) -> str:
-    punc = "~`!#$%^&*()_+-=|';\":/.,?><~·！@#￥%……&*（）——+-=“：’；、。，？》《{}"
+    opr_and_punc = "=\+\-\*\/%<>×÷" + "।,;:\?!'\.\"-\[\]\{\}\(\)–—―~"
-    return re.sub(r"[{0}]+".format(punc), "", utt).upper()
+    return re.sub(r"[{0}]+".format(opr_and_punc), "", utt).upper()
 
 
 def preprocess_bengaliai_speech(
@@ -109,7 +109,9 @@ def preprocess_bengaliai_speech(
 
 
 def main():
-    formatter = "%(asctime)s %(levelname)s [%(filename)s:%(lineno)d] %(message)s"
+    formatter = (
+        "%(asctime)s %(levelname)s [%(filename)s:%(lineno)d] %(message)s"
+    )
 
     logging.basicConfig(format=formatter, level=logging.INFO)
     args = get_args()

egs/bengaliai_speech/ASR/local/train_bpe_model.py

@@ -0,0 +1 @@
+../../../librispeech/ASR/local/train_bpe_model.py

egs/bengaliai_speech/ASR/local/validate_bpe_lexicon.py

@@ -0,0 +1 @@
+../../../librispeech/ASR/local/validate_bpe_lexicon.py

egs/bengaliai_speech/ASR/prepare.sh

@@ -135,7 +135,7 @@ if [ $stage -le 6 ] && [ $stop_stage -ge 6 ]; then
       --num-workers $nj \
       --batch-duration 600 \
       --start 0 \
-      --num-splits 300
+      --num-splits 233
     touch data/fbank/.bengaliai_speech_train.done
   fi
 fi