From 1b9e4f0fea2e3482affb52e345fe122e96f674b9 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Wed, 27 Oct 2021 19:54:28 +0800 Subject: [PATCH 1/5] WIP: Decoding scripts using conformer LM. --- icefall/lm/rescore.py | 284 ++++++++++++++++++++++++++++++++++++++++ test/lm/test_rescore.py | 133 +++++++++++++++++++ 2 files changed, 417 insertions(+) create mode 100644 icefall/lm/rescore.py create mode 100755 test/lm/test_rescore.py diff --git a/icefall/lm/rescore.py b/icefall/lm/rescore.py new file mode 100644 index 000000000..fa15cc621 --- /dev/null +++ b/icefall/lm/rescore.py @@ -0,0 +1,284 @@ +# Copyright 2021 Xiaomi Corp. (authors: Fangjun Kuang) +# +# See ../../../../LICENSE for clarification regarding multiple authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This file contains rescoring code for NN LMs, e.g., conformer LM. + +Here are the ideas about preparing the inputs for the conformer LM model +from an Nbest object. + +Given an Nbest object `nbest`, we have: + - nbest.fsa + - nbest.shape, whose axes are [utt][path] + +We can get `tokens` from nbest.fsa. The resulting `tokens` will have +2 axes [path][token]. Note, we should remove 0s from `tokens`. + +We can generate the following inputs for the conformer LM model from `tokens`: + - masked_src + - src + - tgt +by using `k2.levenshtein_alignment`. +""" + +from typing import Tuple + +import k2 +import torch + +from icefall.decode import Nbest + + +def concat( + ragged: k2.RaggedTensor, value: int, direction: str +) -> k2.RaggedTensor: + """Prepend a value to the beginning of each sublist or append a value. + to the end of each sublist. + + Args: + ragged: + A ragged tensor with two axes. + value: + The value to prepend or append. + direction: + It can be either "left" or "right". If it is "left", we + prepend the value to the beginning of each sublist; + if it is "right", we append the value to the end of each + sublist. + + Returns: + Return a new ragged tensor, whose sublists either start with + or end with the given value. + + >>> a = k2.RaggedTensor([[1, 3], [5]]) + >>> a + [ [ 1 3 ] [ 5 ] ] + >>> concat(a, value=0, direction="left") + [ [ 0 1 3 ] [ 0 5 ] ] + >>> concat(a, value=0, direction="right") + [ [ 1 3 0 ] [ 5 0 ] ] + + """ + dtype = ragged.dtype + device = ragged.device + + assert ragged.num_axes == 2, f"num_axes: {ragged.num_axes}" + pad_values = torch.full( + size=(ragged.tot_size(0),), + fill_value=value, + device=device, + dtype=dtype, + ) + pad_shape = k2.ragged.regular_ragged_shape(ragged.tot_size(0), 1).to(device) + pad = k2.RaggedTensor(pad_shape, pad_values) + + if direction == "left": + ans = k2.ragged.cat([pad, ragged], axis=1) + elif direction == "right": + ans = k2.ragged.cat([ragged, pad], axis=1) + else: + raise ValueError( + f'Unsupported direction: {direction}. " \ + "Expect either "left" or "right"' + ) + return ans + + +def add_bos(ragged: k2.RaggedTensor, bos_id: int) -> k2.RaggedTensor: + """Add BOS to each sublist. + + Args: + ragged: + A ragged tensor with two axes. + bos_id: + The ID of the BOS symbol. + + Returns: + Return a new ragged tensor, where each sublist starts with BOS. + + >>> a = k2.RaggedTensor([[1, 3], [5]]) + >>> a + [ [ 1 3 ] [ 5 ] ] + >>> add_bos(a, bos_id=0) + [ [ 0 1 3 ] [ 0 5 ] ] + + """ + return concat(ragged, bos_id, direction="left") + + +def add_eos(ragged: k2.RaggedTensor, eos_id: int) -> k2.RaggedTensor: + """Add EOS to each sublist. + + Args: + ragged: + A ragged tensor with two axes. + bos_id: + The ID of the EOS symbol. + + Returns: + Return a new ragged tensor, where each sublist ends with EOS. + + >>> a = k2.RaggedTensor([[1, 3], [5]]) + >>> a + [ [ 1 3 ] [ 5 ] ] + >>> add_eos(a, eos_id=0) + [ [ 1 3 0 ] [ 5 0 ] ] + + """ + return concat(ragged, eos_id, direction="right") + + +def make_hyp_to_ref_map(row_splits: torch.Tensor): + """ + TODO: Add documentation. + + >>> row_splits = torch.tensor([0, 3, 5], dtype=torch.int32) + >>> make_hyp_to_ref_map(row_splits) + tensor([0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4], dtype=torch.int32) + + """ + device = row_splits.device + sizes = (row_splits[1:] - row_splits[:-1]).tolist() + offsets = row_splits[:-1] + + map_tensor_list = [] + for size, offset in zip(sizes, offsets): + # Explanation of the following operations + # assume size is 3, offset is 2 + # torch.arange() + offset is [2, 3, 4] + # expand() is [[2, 3, 4], [2, 3, 4], [2, 3, 4]] + # t() is [[2, 2, 2], [3, 3, 3], [4, 4, 4]] + # reshape() is [2, 2, 2, 3, 3, 3, 4, 4, 4] + map_tensor = ( + (torch.arange(size, dtype=torch.int32, device=device) + offset) + .expand(size, size) + .t() + .reshape(-1) + ) + map_tensor_list.append(map_tensor) + + return torch.cat(map_tensor_list) + + +def make_repeat_map(row_splits: torch.Tensor): + """ + TODO: Add documentation. + + >>> row_splits = torch.tensor([0, 3, 5], dtype=torch.int32) + >>> make_repeat_map(row_splits) + tensor([0, 1, 2, 0, 1, 2, 0, 1, 2, 3, 4, 3, 4], dtype=torch.int32) + + """ + device = row_splits.device + sizes = (row_splits[1:] - row_splits[:-1]).tolist() + offsets = row_splits[:-1] + + map_tensor_list = [] + for size, offset in zip(sizes, offsets): + # Explanation of the following operations + # assume size is 3, offset is 2 + # torch.arange() + offset is [2, 3, 4] + # expand() is [[2, 3, 4], [2, 3, 4], [2, 3, 4]] + # reshape() is [2, 3, 4, 2, 3, 4, 2, 3, 4] + map_tensor = ( + (torch.arange(size, dtype=torch.int32, device=device) + offset) + .expand(size, size) + .reshape(-1) + ) + map_tensor_list.append(map_tensor) + + return torch.cat(map_tensor_list) + + +def make_repeat(tokens: k2.RaggedTensor) -> k2.RaggedTensor: + """Repeat the number of paths of an utterance to the number that + equals to the number of paths in the utterance. + + For instance, if an utterance contains 3 paths: [path1 path2 path3], + after repeating, this utterance will contain 9 paths: + [path1 path2 path3] [path1 path2 path3] [path1 path2 path3] + + >>> tokens = k2.RaggedTensor([ [[1, 2, 3], [4, 5], [9]], [[5, 8], [10, 1]] ]) + >>> tokens + [ [ [ 1 2 3 ] [ 4 5 ] [ 9 ] ] [ [ 5 8 ] [ 10 1 ] ] ] + >>> make_repeat(tokens) + [ [ [ 1 2 3 ] [ 4 5 ] [ 9 ] [ 1 2 3 ] [ 4 5 ] [ 9 ] [ 1 2 3 ] [ 4 5 ] [ 9 ] ] [ [ 5 8 ] [ 10 1 ] [ 5 8 ] [ 10 1 ] ] ] + + TODO: Add documentation. + + """ + assert tokens.num_axes == 3, f"num_axes: {tokens.num_axes}" + if True: + indexes = make_repeat_map(tokens.shape.row_splits(1)) + return tokens.index(axis=1, indexes=indexes)[0] + else: + # This branch produces the same result as the above branch. + # It's more readable. Will remove it later. + repeated = [] + for p in tokens.tolist(): + repeated.append(p * len(p)) + return k2.RaggedTensor(repeated).to(tokens.device) + + +def compute_alignments( + tokens: k2.RaggedTensor, + shape: k2.RaggedShape, +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """ + TODO: Add documentation. + + Args: + tokens: + A ragged tensor with two axes: [path][token]. + shape: + A ragged shape with two axes: [utt][path] + """ + assert tokens.tot_size(0) == shape.tot_size(1) + device = tokens.device + utt_path_shape = shape.compose(tokens.shape) + utt_path_token = k2.RaggedTensor(utt_path_shape, tokens.values) + utt_path_token_repeated = make_repeat(utt_path_token) + path_token_repeated = utt_path_token_repeated.remove_axis(0) + + refs = k2.levenshtein_graph(tokens, device=device) + hyps = k2.levenshtein_graph(path_token_repeated, device=device) + + hyp_to_ref_map = make_hyp_to_ref_map(utt_path_shape.row_splits(1)) + alignment = k2.levenshtein_alignment( + refs=refs, hyps=hyps, hyp_to_ref_map=hyp_to_ref_map + ) + return alignment + + +def conformer_lm_rescore( + nbest: Nbest, + model: torch.nn.Module, + # TODO: add other arguments if needed +) -> k2.RaggedTensor: + """Rescore an Nbest object with a conformer_lm model. + + Args: + nbest: + It contains linear FSAs to be rescored. + model: + A conformer lm model. See "conformer_lm/train.py" + + Returns: + Return a ragged tensor containing scores for each path + contained in the nbest. Its shape equals to `nbest.shape`. + """ + assert hasattr(nbest.fsa, "tokens") + # TODO: diff --git a/test/lm/test_rescore.py b/test/lm/test_rescore.py new file mode 100755 index 000000000..fd7457a7e --- /dev/null +++ b/test/lm/test_rescore.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 +# Copyright 2021 Xiaomi Corp. (authors: Fangjun Kuang) +# +# See ../../LICENSE for clarification regarding multiple authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import k2 +import torch + +from icefall.lm.rescore import ( + add_bos, + add_eos, + make_repeat_map, + make_hyp_to_ref_map, + make_repeat, + compute_alignments, +) + + +def test_add_bos(): + bos_id = 100 + ragged = k2.RaggedTensor([[1, 2], [3], [0]]) + bos_ragged = add_bos(ragged, bos_id) + expected = k2.RaggedTensor([[bos_id, 1, 2], [bos_id, 3], [bos_id, 0]]) + assert str(bos_ragged) == str(expected) + + +def test_add_eos(): + eos_id = 30 + ragged = k2.RaggedTensor([[1, 2], [3], [], [5, 8, 9]]) + ragged_eos = add_eos(ragged, eos_id) + expected = k2.RaggedTensor( + [[1, 2, eos_id], [3, eos_id], [eos_id], [5, 8, 9, eos_id]] + ) + + +def test_pad(): + bos_id = 10 + eos_id = 100 + ragged = k2.RaggedTensor([[1, 2, 3], [5], [9, 8]]) + bos_ragged = add_bos(ragged, bos_id) + bos_ragged_eos = add_eos(bos_ragged, eos_id) + blank_id = -1 + padded = bos_ragged_eos.pad(mode="constant", padding_value=blank_id) + expected = torch.tensor( + [ + [bos_id, 1, 2, 3, eos_id], + [bos_id, 5, eos_id, blank_id, blank_id], + [bos_id, 9, 8, eos_id, blank_id], + ] + ).to(padded) + assert torch.all(torch.eq(padded, expected)) + + +def test_make_hyp_to_ref_map(): + a = k2.RaggedTensor([[[1, 2], [], [3]], [[1, 3], [2], [4], [5]]]) + row_splits = a.shape.row_splits(1) + repeat_map = make_hyp_to_ref_map(row_splits) + # fmt: off + expected = torch.tensor([0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, + 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6]).to(repeat_map) + # fmt: on + assert torch.all(torch.eq(repeat_map, expected)) + + +def test_make_repeat_map(): + a = k2.RaggedTensor([[[1, 2], [], [3]], [[1, 3], [2], [4], [5]]]) + row_splits = a.shape.row_splits(1) + repeat_map = make_repeat_map(row_splits) + # fmt: off + expected = torch.tensor([0, 1, 2, 0, 1, 2, 0, 1, 2, + 3, 4, 5, 6, 3, 4, 5, 6, 3, 4, 5, 6, + 3, 4, 5, 6]).to(repeat_map) + # fmt: on + assert torch.all(torch.eq(repeat_map, expected)) + + +def test_make_repeat(): + # fmt: off + a = k2.RaggedTensor([ + [[1, 3, 5], [2, 6]], + [[1, 2, 3, 4], [2], [], [9, 10, 11]], + ]) + b = make_repeat(a) + expected = k2.RaggedTensor([ + [[1, 3, 5], [2, 6], [1, 3, 5], [2, 6]], + [[1, 2, 3, 4], [2], [], [9, 10, 11], + [1, 2, 3, 4], [2], [], [9, 10, 11], + [1, 2, 3, 4], [2], [], [9, 10, 11], + [1, 2, 3, 4], [2], [], [9, 10, 11]], + ]) + # fmt: on + assert str(b) == str(expected) + + +def test_compute_alignments(): + # fmt: off + tokens = k2.RaggedTensor([ + # utt 0 + [1, 3], [8], [2], + # utt 1 + [1, 5], [9], + ]) + shape = k2.RaggedShape('[[x x x] [x x]]') + # fmt: on + alignment = compute_alignments(tokens, shape) + print("maksed_src:", alignment.labels) + print("src:", alignment.hyp_labels) + print("tgt:", alignment.ref_labels) + + +def main(): + test_add_bos() + test_add_eos() + test_pad() + test_make_repeat_map() + test_make_hyp_to_ref_map() + test_make_repeat() + test_compute_alignments() + + +if __name__ == "__main__": + main() From 3441634f34691721302e37337edc4c5acdd873c6 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Mon, 1 Nov 2021 21:34:22 +0800 Subject: [PATCH 2/5] Finish preparing the inputs for conformer lm from an nbest object. --- icefall/lm/rescore.py | 138 ++++++++++++++++++++++++++++++++++++++-- test/lm/test_rescore.py | 57 +++++++++++++---- 2 files changed, 178 insertions(+), 17 deletions(-) diff --git a/icefall/lm/rescore.py b/icefall/lm/rescore.py index fa15cc621..be2b3c929 100644 --- a/icefall/lm/rescore.py +++ b/icefall/lm/rescore.py @@ -42,6 +42,28 @@ import torch from icefall.decode import Nbest +def make_key_padding_mask(lengths: torch.Tensor): + """ + TODO: add documentation + + >>> make_key_padding_mask(torch.tensor([3, 1, 4])) + tensor([[False, False, False, True], + [False, True, True, True], + [False, False, False, False]]) + """ + assert lengths.dim() == 1 + + bs = lengths.numel() + max_len = lengths.max().item() + device = lengths.device + seq_range = torch.arange(0, max_len, device=device) + seq_range_expand = seq_range.unsqueeze(0).expand(bs, max_len) + + seq_length_expand = lengths.unsqueeze(-1) + mask = seq_range_expand >= seq_length_expand + return mask + + def concat( ragged: k2.RaggedTensor, value: int, direction: str ) -> k2.RaggedTensor: @@ -77,13 +99,12 @@ def concat( assert ragged.num_axes == 2, f"num_axes: {ragged.num_axes}" pad_values = torch.full( - size=(ragged.tot_size(0),), + size=(ragged.tot_size(0), 1), fill_value=value, device=device, dtype=dtype, ) - pad_shape = k2.ragged.regular_ragged_shape(ragged.tot_size(0), 1).to(device) - pad = k2.RaggedTensor(pad_shape, pad_values) + pad = k2.RaggedTensor(pad_values) if direction == "left": ans = k2.ragged.cat([pad, ragged], axis=1) @@ -233,10 +254,10 @@ def make_repeat(tokens: k2.RaggedTensor) -> k2.RaggedTensor: return k2.RaggedTensor(repeated).to(tokens.device) -def compute_alignments( +def compute_alignment( tokens: k2.RaggedTensor, shape: k2.RaggedShape, -) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: +) -> k2.Fsa: """ TODO: Add documentation. @@ -263,9 +284,90 @@ def compute_alignments( return alignment +def prepare_conformer_lm_inputs( + alignment: k2.Fsa, + bos_id: int, + eos_id: int, + blank_id: int, + unmasked_weight: float = 0.25, +) -> Tuple[ + torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor +]: + """ + TODO: add documentation. + + Args: + alignments: + It is computed by :func:`compute_alignment` + """ + # alignment.arcs.shape has axes [fsa][state][arc] + # we remove axis 1, i.e., state, here + labels_shape = alignment.arcs.shape().remove_axis(1) + + masked_src = k2.RaggedTensor(labels_shape, alignment.labels.contiguous()) + masked_src = masked_src.remove_values_eq(-1) + bos_masked_src = add_bos(masked_src, bos_id=bos_id) + bos_masked_src_eos = add_eos(bos_masked_src, eos_id=eos_id) + bos_masked_src_eos_pad = bos_masked_src_eos.pad( + mode="constant", padding_value=blank_id + ) + + src = k2.RaggedTensor(labels_shape, alignment.hyp_labels) + src = src.remove_values_eq(-1) + bos_src = add_bos(src, bos_id=bos_id) + bos_src_eos = add_eos(bos_src, eos_id=eos_id) + bos_src_eos_pad = bos_src_eos.pad(mode="constant", padding_value=blank_id) + + tgt = k2.RaggedTensor(labels_shape, alignment.ref_labels) + # TODO: Do we need to remove 0s from tgt ? + tgt = tgt.remove_values_eq(-1) + tgt_eos = add_eos(tgt, eos_id=eos_id) + + # add a blank here since tgt_eos does not start with bos + # assume blank id is 0 + tgt_eos = add_eos(tgt_eos, eos_id=blank_id) + + row_splits = tgt_eos.shape.row_splits(1) + lengths = row_splits[1:] - row_splits[:-1] + src_key_padding_mask = make_key_padding_mask(lengths) + + tgt_eos_pad = tgt_eos.pad(mode="constant", padding_value=blank_id) + + weight = torch.full( + (tgt_eos_pad.size(0), tgt_eos_pad.size(1) - 1), + fill_value=1, + dtype=torch.float32, + ) + + # find unmasked positions + unmasked_positions = bos_src_eos_pad[:, 1:] == tgt_eos_pad[:, :-1] + weight[unmasked_positions] = unmasked_weight + + # set weights for paddings + weight[src_key_padding_mask[:, 1:]] = 0 + zeros = torch.zeros(weight.size(0), 1).to(weight) + + weight = torch.cat((weight, zeros), dim=1) + + # all other positions are assumed to be masked and + # have the default weight 1 + + return ( + bos_masked_src_eos_pad, + bos_src_eos_pad, + tgt_eos_pad, + src_key_padding_mask, + weight, + ) + + def conformer_lm_rescore( nbest: Nbest, model: torch.nn.Module, + bos_id: int, + eos_id: int, + blank_id: int, + unmasked_weight: float = 0.25, # TODO: add other arguments if needed ) -> k2.RaggedTensor: """Rescore an Nbest object with a conformer_lm model. @@ -281,4 +383,28 @@ def conformer_lm_rescore( contained in the nbest. Its shape equals to `nbest.shape`. """ assert hasattr(nbest.fsa, "tokens") - # TODO: + utt_path_shape = nbest.shape + # nbest.fsa.arcs.shape() has axes [path][state][arc] + # We remove the state axis here + path_token_shape = nbest.fsa.arcs.shape().remove_axis(1) + + path_token = k2.RaggedTensor(path_token_shape, nbest.fsa.tokens) + path_token = path_token.remove_values_leq(0) + + alignment = compute_alignment(path_token, utt_path_shape) + ( + masked_src, + src, + tgt, + src_key_padding_mask, + weight, + ) = prepare_conformer_lm_inputs( + alignment, + bos_id=bos_id, + eos_id=eos_id, + blank_id=blank_id, + unmasked_weight=unmasked_weight, + ) + return masked_src, src, tgt, src_key_padding_mask, weight + # TODO: pass masked_src, src, tgt, src_key_padding_mask, and weight + # to the given model diff --git a/test/lm/test_rescore.py b/test/lm/test_rescore.py index fd7457a7e..bade45b5d 100755 --- a/test/lm/test_rescore.py +++ b/test/lm/test_rescore.py @@ -17,13 +17,16 @@ import k2 import torch +from icefall.decode import Nbest from icefall.lm.rescore import ( add_bos, add_eos, - make_repeat_map, + compute_alignment, + conformer_lm_rescore, make_hyp_to_ref_map, make_repeat, - compute_alignments, + make_repeat_map, + prepare_conformer_lm_inputs, ) @@ -103,20 +106,51 @@ def test_make_repeat(): assert str(b) == str(expected) -def test_compute_alignments(): +def test_compute_alignment(): # fmt: off tokens = k2.RaggedTensor([ # utt 0 - [1, 3], [8], [2], + [1, 3, 5, 8], [1, 5, 8], [2, 8, 3, 2], # utt 1 - [1, 5], [9], + [2, 3], [2], ]) - shape = k2.RaggedShape('[[x x x] [x x]]') # fmt: on - alignment = compute_alignments(tokens, shape) - print("maksed_src:", alignment.labels) - print("src:", alignment.hyp_labels) - print("tgt:", alignment.ref_labels) + shape = k2.RaggedShape("[[x x x] [x x]]") + alignment = compute_alignment(tokens, shape) + ( + masked_src, + src, + tgt, + src_key_padding_mask, + weight, + ) = prepare_conformer_lm_inputs(alignment, bos_id=10, eos_id=20, blank_id=0) + + # print("masked src", masked_src) + # print("src", src) + # print("tgt", tgt) + # print("src_key_padding_mask", src_key_padding_mask) + # print("weight", weight) + + +def test_conformer_lm_rescore(): + path00 = k2.linear_fsa([1, 2, 0, 3, 0, 5]) + path01 = k2.linear_fsa([1, 0, 5, 0]) + path10 = k2.linear_fsa([9, 8, 0, 3, 0, 2]) + path11 = k2.linear_fsa([9, 8, 0, 0, 3, 2]) + path12 = k2.linear_fsa([9, 0, 8, 4, 0, 2, 3]) + + fsa = k2.Fsa.from_fsas([path00, path01, path10, path11, path12]) + fsa.tokens = fsa.labels.clone() + shape = k2.RaggedShape("[[x x] [x x x]]") + nbest = Nbest(fsa, shape) + masked_src, src, tgt, src_key_padding_mask, weight = conformer_lm_rescore( + nbest, model=None, bos_id=10, eos_id=20, blank_id=0 + ) + print("masked src", masked_src) + print("src", src) + print("tgt", tgt) + print("src_key_padding_mask", src_key_padding_mask) + print("weight", weight) def main(): @@ -126,7 +160,8 @@ def main(): test_make_repeat_map() test_make_hyp_to_ref_map() test_make_repeat() - test_compute_alignments() + test_compute_alignment() + test_conformer_lm_rescore() if __name__ == "__main__": From 19828cbf22904c8932886ccf534d754326d317a2 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Mon, 1 Nov 2021 21:58:43 +0800 Subject: [PATCH 3/5] Add files form Dan. See https://github.com/k2-fsa/icefall/pull/54 --- .../ASR/conformer_lm/asr_datamodule.py | 1 + egs/librispeech/ASR/conformer_lm/conformer.py | 1484 +++++++++++++++++ egs/librispeech/ASR/conformer_lm/dataset.py | 823 +++++++++ egs/librispeech/ASR/conformer_lm/madam.py | 1256 ++++++++++++++ .../ASR/conformer_lm/test_conformer.py | 156 ++ .../ASR/conformer_lm/test_dataset.py | 32 + .../ASR/conformer_lm/test_dataset_empty.py | 39 + egs/librispeech/ASR/conformer_lm/train.py | 623 +++++++ 8 files changed, 4414 insertions(+) create mode 120000 egs/librispeech/ASR/conformer_lm/asr_datamodule.py create mode 100644 egs/librispeech/ASR/conformer_lm/conformer.py create mode 100644 egs/librispeech/ASR/conformer_lm/dataset.py create mode 100644 egs/librispeech/ASR/conformer_lm/madam.py create mode 100644 egs/librispeech/ASR/conformer_lm/test_conformer.py create mode 100644 egs/librispeech/ASR/conformer_lm/test_dataset.py create mode 100644 egs/librispeech/ASR/conformer_lm/test_dataset_empty.py create mode 100755 egs/librispeech/ASR/conformer_lm/train.py diff --git a/egs/librispeech/ASR/conformer_lm/asr_datamodule.py b/egs/librispeech/ASR/conformer_lm/asr_datamodule.py new file mode 120000 index 000000000..fa1b8cca3 --- /dev/null +++ b/egs/librispeech/ASR/conformer_lm/asr_datamodule.py @@ -0,0 +1 @@ +../tdnn_lstm_ctc/asr_datamodule.py \ No newline at end of file diff --git a/egs/librispeech/ASR/conformer_lm/conformer.py b/egs/librispeech/ASR/conformer_lm/conformer.py new file mode 100644 index 000000000..fe0a5eec9 --- /dev/null +++ b/egs/librispeech/ASR/conformer_lm/conformer.py @@ -0,0 +1,1484 @@ +# Copyright (c) 2021 University of Chinese Academy of Sciences (author: Han Zhu) +# Apache 2.0 + +import copy +import math +from typing import Dict, List, Optional, Tuple + +import torch +import torch.nn as nn +from torch import Tensor +from torch.nn.utils.rnn import pad_sequence + +# Note: TorchScript requires Dict/List/etc. to be fully typed. +Supervisions = Dict[str, torch.Tensor] + + +class MaskedLmConformer(nn.Module): + def __init__( + self, + num_classes: int, + d_model: int = 256, + nhead: int = 4, + dim_feedforward: int = 2048, + num_encoder_layers: int = 6, + num_decoder_layers: int = 6, + dropout: float = 0.1, + cnn_module_kernel: int = 31, + ) -> None: + """ + Args: + num_classes: + The input and output dimension of the model (inputs and outputs are + both discrete) + d_model: + Attention dimension. + nhead: + Number of heads in multi-head attention. + Must satisfy d_model // nhead == 0. + dim_feedforward: + The output dimension of the feedforward layers in encoder/decoder. + num_encoder_layers: + Number of encoder layers. + num_decoder_layers: + Number of decoder layers. + dropout: + Dropout in encoder/decoder. + """ + super(MaskedLmConformer, self).__init__() + + self.num_classes = num_classes + + # self.embed is the embedding used for both the encoder and decoder. + self.embed_scale = d_model ** 0.5 + self.embed = nn.Embedding( + num_embeddings=self.num_classes, embedding_dim=d_model, + _weight=torch.randn(self.num_classes, d_model) * (1 / self.embed_scale) + ) + + self.encoder_pos = RelPositionalEncoding(d_model, dropout) + + encoder_layer = MaskedLmConformerEncoderLayer( + d_model, + nhead, + dim_feedforward, + dropout, + cnn_module_kernel, + ) + self.encoder = MaskedLmConformerEncoder(encoder_layer, num_encoder_layers, + norm=nn.LayerNorm(d_model)) + + if num_decoder_layers > 0: + + decoder_layer = RelPosTransformerDecoderLayer( + d_model=d_model, + nhead=nhead, + dim_feedforward=dim_feedforward, + dropout=dropout, + ) + + # Projects the embedding of `src`, to be added to `memory` + self.src_linear = torch.nn.Linear(d_model, d_model) + + decoder_norm = nn.LayerNorm(d_model) + self.decoder = RelPosTransformerDecoder( + decoder_layer=decoder_layer, + num_layers=num_decoder_layers, + norm=decoder_norm, + ) + + self.decoder_output_layer = torch.nn.Linear( + d_model, self.num_classes + ) + + + def forward( + self, + masked_src_symbols: torch.Tensor, + key_padding_mask: torch.Tensor = None + ) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Args: + masked_src_symbols: + The input symbols to be embedded (will actually have query positions + masked), as a Tensor of shape (batch_size, seq_len) and dtype=torch.int64. + I.e. shape (N, T) + key_padding_mask: + Either None, or a Tensor of shape (batch_size, seq_len) i.e. (N, T), + and dtype=torch.bool which has True in positions to be masked in attention + layers and convolutions because they represent padding at the ends of + sequences. + + + Returns: + Returns (memory, pos_emb), where: + `memory` is a Tensor containing the encoded data; it is of shape (N, T, C) + where C is the embedding_dim. + `pos_emb` is a Tensor containing the relative positional encoding, of + shape (1, 2*T-1, C) + """ + + x = self.embed(masked_src_symbols) * self.embed_scale # (N, T, C) + x, pos_emb = self.encoder_pos(x) # pos_emb: (1, 2*T-1, C) + x = x.permute(1, 0, 2) # (N, T, C) -> (T, N, C) + + x = self.encoder(x, pos_emb, key_padding_mask=key_padding_mask) # (T, N, C) + + return x, pos_emb + + def decoder_nll( + self, + memory: torch.Tensor, + pos_emb: torch.Tensor, + src_symbols: torch.Tensor, + tgt_symbols: torch.Tensor, + key_padding_mask: torch.Tensor + ) -> torch.Tensor: + """ + Args: + memory: + The output of the encoder, with shape (T, N, C) + pos_emb: + Relative positional embedding, of shape (1, 2*T-1, C), as + returned from the encoder + src_symbols: + The un-masked src symbols, a LongTensor of shape (N, T). + Can be used to predict the target + only in a left-to-right manner (otherwise it's cheating). + tgt_symbols: + Target symbols, a LongTensor of shape (N, T). + The same as src_symbols, but shifted by one (and also, + without symbol randomization, see randomize_proportion + in dataloader) + key_padding_mask: + A BoolTensor of shape (N, T), with True for positions + that correspond to padding at the end of source and + memory sequences. The same mask is used for self-attention + and cross-attention, since the padding is the same. + + Returns: + Returns a tensor of shape (N, T), containing the negative + log-probabilities for the target symbols at each position + in the target sequence. + """ + (T, N, C) = memory.shape + + attn_mask = generate_square_subsequent_mask(T, memory.device) + + x = self.embed(src_symbols) * self.embed_scale # (N, T) -> (N, T, C) + x = x.permute(1, 0, 2) # (N, T, C) -> (T, N, C) + + x = memory + self.src_linear(x) # (T, N, C) + + # This is a little confusing, how "tgt" is set to src. "src" is the + # symbol sequence without masking but with padding and randomization. + # "tgt" is like "src" but shifted by one. + pred = self.decoder( + x, + pos_emb, + memory=memory, + attn_mask=attn_mask, + key_padding_mask=key_padding_mask) + # (T, N, C) + + pred = pred.permute(1, 0, 2) # (T, N, C) -> (N, T, C) + pred = self.decoder_output_layer(pred) # (N, T, C) + + # nll: negative log-likelihood + nll = torch.nn.functional.cross_entropy( + pred.view(-1, self.num_classes), + tgt_symbols.reshape(-1), + reduction="none", + ) + nll = nll.view(N, T) + return nll + + + + +class RelPosTransformerDecoder(nn.Module): + r"""RelPosTransformerDecoder is a stack of N decoder layers. + This is modified from nn.TransformerDecoder to support relative positional + encoding. + + Args: + decoder_layer: an instance of the RelPosTransformerDecoderLayer() class (required). + num_layers: the number of sub-decoder-layers in the decoder (required). + norm: the layer normalization component (optional). + + Examples:: + >>> decoder_layer = nn.RelPosTransformerDecoderLayer(d_model=512, nhead=8) + >>> transformer_decoder = nn.RelPosTransformerDecoder(decoder_layer, num_layers=6) + >>> memory = torch.rand(10, 32, 512) + >>> tgt = torch.rand(20, 32, 512) + >>> pos_enc = torch.rand() + >>> out = transformer_decoder(tgt, memory) + """ + __constants__ = ['norm'] + + def __init__(self, decoder_layer, num_layers, norm=None): + super(RelPosTransformerDecoder, self).__init__() + self.layers = _get_clones(decoder_layer, num_layers) + self.num_layers = num_layers + self.norm = norm + + def forward(self, x: Tensor, + pos_emb: Tensor, + memory: Tensor, + attn_mask: Optional[Tensor] = None, + key_padding_mask: Optional[Tensor] = None) -> Tensor: + r"""Pass the inputs (and mask) through the decoder layer in turn. + + Args: + x: the input embedding sequence to the decoder (required): shape = (T, N, C). + Will be an embedding of `src_symbols` in practice + pos_emb: + A torch.Tensor with dtype=torch.float and shape (1, 2*T-1, C) with c==num_channels, + representing the relative positional encoding. + memory: the sequence from the last layer of the encoder (required): + shape = (T, N, C) + attn_mask: the mask for the `x` sequence's attention to itself, + of shape (T, T); in practice, will ensure that no + position can attend to later positions. A torch.Tensor with dtype=torch.float + or dtype=torch.bool. + key_padding_mask: the key-padding mask for both the memory and x sequences, + a torch.Tensor with dtype=bool and shape (N, T): true for masked + positions after the ends of sequences. + """ + for mod in self.layers: + x = mod(x, pos_emb, memory, attn_mask=attn_mask, + key_padding_mask=key_padding_mask) + + if self.norm is not None: + x = self.norm(x) + + return x + + +class RelPosTransformerDecoderLayer(nn.Module): + """ + Modified from torch.nn.TransformerDecoderLayer. + Add it to use normalize_before (hardcoded to True), i.e. use layer_norm before the first block; + to use relative positional encoding; and for some changes/simplifications in interface + because both sequences are the same length and have the same mask. + + Args: + d_model: + the number of expected features in the input (required). + nhead: + the number of heads in the multiheadattention models (required). + dim_feedforward: + the dimension of the feedforward network model (default=2048). + dropout: + the dropout value (default=0.1). + activation: + the activation function of intermediate layer, relu or + gelu (default=relu). + + Examples:: + >>> decoder_layer = nn.RelPosTransformerDecoderLayer(d_model=512, nhead=8) + >>> memory = torch.rand(10, 32, 512) + >>> tgt = torch.rand(20, 32, 512) + >>> pos_emb = torch.rand(1, 20*2+1, 512) + >>> out = decoder_layer(tgt, pos_emb, memory) + """ + + def __init__( + self, + d_model: int, + nhead: int, + dim_feedforward: int = 2048, + dropout: float = 0.1, + activation: str = "relu", + ) -> None: + super(RelPosTransformerDecoderLayer, self).__init__() + self.self_attn = RelPositionMultiheadAttention(d_model, nhead, dropout=0.0) + self.src_attn = RelPositionMultiheadAttention(d_model, nhead, dropout=0.0) + # Implementation of Feedforward model + self.linear1 = nn.Linear(d_model, dim_feedforward) + self.dropout = nn.Dropout(dropout) + self.linear2 = nn.Linear(dim_feedforward, d_model) + + self.norm1 = nn.LayerNorm(d_model) + self.norm2 = nn.LayerNorm(d_model) + self.norm3 = nn.LayerNorm(d_model) + self.dropout1 = nn.Dropout(dropout) + self.dropout2 = nn.Dropout(dropout) + self.dropout3 = nn.Dropout(dropout) + + self.activation = _get_activation_fn(activation) + + + def __setstate__(self, state): + if "activation" not in state: + state["activation"] = nn.functional.relu + super(RelPosTransformerDecoderLayer, self).__setstate__(state) + + def forward( + self, + x: torch.Tensor, + pos_emb: torch.Tensor, + memory: torch.Tensor, + attn_mask: Optional[torch.Tensor] = None, + key_padding_mask: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + """Pass the inputs (and mask) through the decoder layer. + + Args: + x + The input embedding, to be added to by the forward function, of shape (T, N, C). + Attention within x will be left-to-right only (causal), thanks to attn_mask. + pos_emb: + A torch.Tensor with dtype=torch.float and shape (1, 2*T-1, C) with c==num_channels, + containing the relative positional encoding. + memory: + the sequence from the last layer of the encoder (required). Shape = (T, N, C) + attn_mask: + the mask for the x, to enforce causal (left to right) attention (optional). + Shape == (T, T); may be bool or float. The first T pertains to the output, + the second T to the input. + key_padding_mask: + the key-padding mask to use for both the x and memory sequences. Shep == (N, T); + may be bool (True==masked) or float (to be added to attention scores). + + Returns: + Returns 'x plus something', a torch.Tensor with dtype the same as x (e.g. float), + and shape (T, N, C). + """ + residual = x + x = self.norm1(x) + self_attn = self.self_attn(x, x, x, + pos_emb=pos_emb, + key_padding_mask=key_padding_mask, + need_weights=False, + attn_mask=attn_mask, + )[0] + x = residual + self.dropout1(self_attn) + + residual = x + x = self.norm2(x) + src_attn = self.src_attn(x, memory, memory, + pos_emb=pos_emb, + key_padding_mask=key_padding_mask, + need_weights=False, + )[0] + x = residual + self.dropout2(src_attn) + + residual = x + x = self.norm3(x) + ff = self.linear2(self.dropout(self.activation(self.linear1(x)))) + x = residual + self.dropout3(ff) + return x + + +def _get_activation_fn(activation: str): + if activation == "relu": + return nn.functional.relu + elif activation == "gelu": + return nn.functional.gelu + + raise RuntimeError( + "activation should be relu/gelu, not {}".format(activation) + ) + + +class PositionalEncoding(nn.Module): + """This class implements the positional encoding + proposed in the following paper: + + - Attention Is All You Need: https://arxiv.org/pdf/1706.03762.pdf + + PE(pos, 2i) = sin(pos / (10000^(2i/d_modle)) + PE(pos, 2i+1) = cos(pos / (10000^(2i/d_modle)) + + Note:: + + 1 / (10000^(2i/d_model)) = exp(-log(10000^(2i/d_model))) + = exp(-1* 2i / d_model * log(100000)) + = exp(2i * -(log(10000) / d_model)) + """ + + def __init__(self, d_model: int, dropout: float = 0.1) -> None: + """ + Args: + d_model: + Embedding dimension. + dropout: + Dropout probability to be applied to the output of this module. + """ + super().__init__() + self.d_model = d_model + self.xscale = math.sqrt(self.d_model) + self.dropout = nn.Dropout(p=dropout) + self.pe = None + + def extend_pe(self, x: torch.Tensor) -> None: + """Extend the time t in the positional encoding if required. + + The shape of `self.pe` is [1, T1, d_model]. The shape of the input x + is [N, T, d_model]. If T > T1, then we change the shape of self.pe + to [N, T, d_model]. Otherwise, nothing is done. + + Args: + x: + It is a tensor of shape [N, T, C]. + Returns: + Return None. + """ + if self.pe is not None: + if self.pe.size(1) >= x.size(1): + if self.pe.dtype != x.dtype or self.pe.device != x.device: + self.pe = self.pe.to(dtype=x.dtype, device=x.device) + return + pe = torch.zeros(x.size(1), self.d_model, dtype=torch.float32) + position = torch.arange(0, x.size(1), dtype=torch.float32).unsqueeze(1) + div_term = torch.exp( + torch.arange(0, self.d_model, 2, dtype=torch.float32) + * -(math.log(10000.0) / self.d_model) + ) + pe[:, 0::2] = torch.sin(position * div_term) + pe[:, 1::2] = torch.cos(position * div_term) + pe = pe.unsqueeze(0) + # Now pe is of shape [1, T, d_model], where T is x.size(1) + self.pe = pe.to(device=x.device, dtype=x.dtype) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """ + Add positional encoding. + + Args: + x: + Its shape is [N, T, C] + + Returns: + Return a tensor of shape [N, T, C] + """ + self.extend_pe(x) + x = x * self.xscale + self.pe[:, : x.size(1), :] + return self.dropout(x) + + +class Noam(object): + """ + Implements Noam optimizer. + + Proposed in + "Attention Is All You Need", https://arxiv.org/pdf/1706.03762.pdf + + Modified from + https://github.com/espnet/espnet/blob/master/espnet/nets/pytorch_backend/transformer/optimizer.py # noqa + + Args: + params: + iterable of parameters to optimize or dicts defining parameter groups + model_size: + attention dimension of the transformer model + factor: + learning rate factor + warm_step: + warmup steps + """ + + def __init__( + self, + params, + model_size: int = 256, + factor: float = 10.0, + warm_step: int = 25000, + weight_decay=0, + ) -> None: + """Construct an Noam object.""" + self.optimizer = torch.optim.Adam( + params, lr=0, betas=(0.9, 0.98), eps=1e-9, weight_decay=weight_decay + ) + self._step = 0 + self.warmup = warm_step + self.factor = factor + self.model_size = model_size + self._rate = 0 + + @property + def param_groups(self): + """Return param_groups.""" + return self.optimizer.param_groups + + def step(self): + """Update parameters and rate.""" + self._step += 1 + rate = self.rate() + for p in self.optimizer.param_groups: + p["lr"] = rate + self._rate = rate + self.optimizer.step() + + def rate(self, step=None): + """Implement `lrate` above.""" + if step is None: + step = self._step + return ( + self.factor + * self.model_size ** (-0.5) + * min(step ** (-0.5), step * self.warmup ** (-1.5)) + ) + + def zero_grad(self): + """Reset gradient.""" + self.optimizer.zero_grad() + + def state_dict(self): + """Return state_dict.""" + return { + "_step": self._step, + "warmup": self.warmup, + "factor": self.factor, + "model_size": self.model_size, + "_rate": self._rate, + "optimizer": self.optimizer.state_dict(), + } + + def load_state_dict(self, state_dict): + """Load state_dict.""" + for key, value in state_dict.items(): + if key == "optimizer": + self.optimizer.load_state_dict(state_dict["optimizer"]) + else: + setattr(self, key, value) + + +class LabelSmoothingLoss(nn.Module): + """ + Label-smoothing loss. KL-divergence between q_{smoothed ground truth prob.}(w) + and p_{prob. computed by model}(w) is minimized. + Modified from + https://github.com/espnet/espnet/blob/master/espnet/nets/pytorch_backend/transformer/label_smoothing_loss.py # noqa + + Args: + size: the number of class + padding_idx: padding_idx: ignored class id + smoothing: smoothing rate (0.0 means the conventional CE) + normalize_length: normalize loss by sequence length if True + criterion: loss function to be smoothed + """ + + def __init__( + self, + size: int, + padding_idx: int = -1, + smoothing: float = 0.1, + normalize_length: bool = False, + criterion: nn.Module = nn.KLDivLoss(reduction="none"), + ) -> None: + """Construct an LabelSmoothingLoss object.""" + super(LabelSmoothingLoss, self).__init__() + self.criterion = criterion + self.padding_idx = padding_idx + assert 0.0 < smoothing <= 1.0 + self.confidence = 1.0 - smoothing + self.smoothing = smoothing + self.size = size + self.true_dist = None + self.normalize_length = normalize_length + + def forward(self, x: torch.Tensor, target: torch.Tensor) -> torch.Tensor: + """ + Compute loss between x and target. + + Args: + x: + prediction of dimension + (batch_size, input_length, number_of_classes). + target: + target masked with self.padding_id of + dimension (batch_size, input_length). + + Returns: + A scalar tensor containing the loss without normalization. + """ + assert x.size(2) == self.size + # batch_size = x.size(0) + x = x.view(-1, self.size) + target = target.view(-1) + with torch.no_grad(): + true_dist = x.clone() + true_dist.fill_(self.smoothing / (self.size - 1)) + ignore = target == self.padding_idx # (B,) + total = len(target) - ignore.sum().item() + target = target.masked_fill(ignore, 0) # avoid -1 index + true_dist.scatter_(1, target.unsqueeze(1), self.confidence) + kl = self.criterion(torch.log_softmax(x, dim=1), true_dist) + # denom = total if self.normalize_length else batch_size + denom = total if self.normalize_length else 1 + return kl.masked_fill(ignore.unsqueeze(1), 0).sum() / denom + + + +def generate_square_subsequent_mask(sz: int, device: torch.device = torch.device('cpu')) -> torch.Tensor: + """Generate a square mask for the sequence. The masked positions are + filled with float('-inf'). Unmasked positions are filled with float(0.0). + The mask can be used for masked self-attention. + + For instance, if sz is 3, it returns:: + + tensor([[0., -inf, -inf], + [0., 0., -inf], + [0., 0., 0]]) + + Args: + sz: mask size + + Returns: + A square mask of dimension (sz, sz) + """ + mask = (torch.triu(torch.ones(sz, sz, device=device)) == 1).transpose(0, 1) + mask = ( + mask.float() + .masked_fill(mask == 0, float("-inf")) + .masked_fill(mask == 1, float(0.0)) + ) + return mask + + +def add_sos(token_ids: List[List[int]], sos_id: int) -> List[List[int]]: + """Prepend sos_id to each utterance. + + Args: + token_ids: + A list-of-list of token IDs. Each sublist contains + token IDs (e.g., word piece IDs) of an utterance. + sos_id: + The ID of the SOS token. + + Return: + Return a new list-of-list, where each sublist starts + with SOS ID. + """ + ans = [] + for utt in token_ids: + ans.append([sos_id] + utt) + return ans + + +def add_eos(token_ids: List[List[int]], eos_id: int) -> List[List[int]]: + """Append eos_id to each utterance. + + Args: + token_ids: + A list-of-list of token IDs. Each sublist contains + token IDs (e.g., word piece IDs) of an utterance. + eos_id: + The ID of the EOS token. + + Return: + Return a new list-of-list, where each sublist ends + with EOS ID. + """ + ans = [] + for utt in token_ids: + ans.append(utt + [eos_id]) + return ans + + + +class MaskedConvolutionModule(nn.Module): + """ + This is used in the MaskedLmConformerLayer. It is the same as the ConvolutionModule + of theConformer code, but with key_padding_mask supported to make the output independent + of the batching. + + Modified, ultimately, from https://github.com/espnet/espnet/blob/master/espnet/nets/pytorch_backend/conformer/convolution.py + + Args: + channels (int): The number of channels of conv layers. + kernel_size (int): Kernerl size of conv layers. + bias (bool): Whether to use bias in conv layers (default=True). + """ + + def __init__( + self, channels: int, kernel_size: int, bias: bool = True + ) -> None: + """Construct a MaskedConvolutionModule object.""" + super(MaskedConvolutionModule, self).__init__() + # kernerl_size should be a odd number for 'SAME' padding + assert (kernel_size - 1) % 2 == 0 + + self.pointwise_conv1 = nn.Conv1d( + channels, + 2 * channels, + kernel_size=1, + stride=1, + padding=0, + bias=bias, + ) + self.depthwise_conv = nn.Conv1d( + channels, + channels, + kernel_size, + stride=1, + padding=(kernel_size - 1) // 2, + groups=channels, + bias=bias, + ) + self.norm = nn.LayerNorm(channels) + self.pointwise_conv2 = nn.Conv1d( + channels, + channels, + kernel_size=1, + stride=1, + padding=0, + bias=bias, + ) + self.activation = Swish() + + def forward(self, x: Tensor, key_padding_mask: Optional[Tensor]) -> Tensor: + """Compute convolution module. + + Args: + x: Input tensor (T, N, C) == (#time, batch, channels). + key_padding_mask: if supplied, a Tensor with dtype=torch.Bool and + shape (N, T), with True for positions that correspond to + padding (and should be zeroed in convolutions). + + Returns: + Tensor: Output tensor (T, N, C) + + """ + # exchange the temporal dimension and the feature dimension + x = x.permute(1, 2, 0) # (#batch, channels, time). + + # GLU mechanism + x = self.pointwise_conv1(x) # (batch, 2*channels, time) + x = nn.functional.glu(x, dim=1) # (batch, channels, time) + + # Logical-not key_padding_mask, unsqueeze to shape (N, 1, T) and convert + # to float. Then we can just multiply by it when we need to apply + # masking, i.e. prior to the convolution over time. + if key_padding_mask is not None: + x = x * torch.logical_not(key_padding_mask).unsqueeze(1).to(dtype=x.dtype) + + # 1D Depthwise Conv + x = self.depthwise_conv(x) + x = x.transpose(1, 2) # (batch, time, channel) + x = self.norm(x) # LayerNorm requires channel be last dim. + x = x.transpose(1, 2) # (batch, channel, time) + x = self.activation(x) + x = self.pointwise_conv2(x) # (batch, channel, time) + + return x.permute(2, 0, 1) # (time, batch, channel) + + +class Swish(torch.nn.Module): + """Construct an Swish object.""" + + def forward(self, x: Tensor) -> Tensor: + """Return Swich activation function.""" + return x * torch.sigmoid(x) + + + +class MaskedLmConformerEncoderLayer(nn.Module): + """ + MaskedLmConformerEncoderLayer is made up of self-attn, feedforward and convolution + networks. It's a simplified version of the conformer code we were previously + using, with pre-normalization hard-coded, relative positional encoding, + LayerNorm instead of BatchNorm in the convolution layers, and the key_padding_mask + applied also in the convolution layers so the computation is independent of + how the sequences are batched. + + See: "Conformer: Convolution-augmented Transformer for Speech Recognition", for + the basic conformer. + + Args: + d_model: the number of expected features in the input (required). + nhead: the number of heads in the multiheadattention models (required). + dim_feedforward: the dimension of the feedforward network model (default=2048). + dropout: the dropout value (default=0.1). + cnn_module_kernel (int): Kernel size of convolution module. + + Examples:: + >>> encoder_layer = ConformerEncoderLayer(d_model=512, nhead=8) + >>> src = torch.rand(10, 32, 512) + >>> pos_emb = torch.rand(32, 19, 512) + >>> out = encoder_layer(src, pos_emb) + """ + + def __init__( + self, + d_model: int, + nhead: int, + dim_feedforward: int = 2048, + dropout: float = 0.1, + cnn_module_kernel: int = 31, + ) -> None: + super(MaskedLmConformerEncoderLayer, self).__init__() + self.self_attn = RelPositionMultiheadAttention( + d_model, nhead, dropout=0.0 + ) + + self.feed_forward = nn.Sequential( + nn.Linear(d_model, dim_feedforward), + Swish(), + nn.Dropout(dropout), + nn.Linear(dim_feedforward, d_model), + ) + + self.feed_forward_macaron = nn.Sequential( + nn.Linear(d_model, dim_feedforward), + Swish(), + nn.Dropout(dropout), + nn.Linear(dim_feedforward, d_model), + ) + + self.conv_module = MaskedConvolutionModule(d_model, cnn_module_kernel) + + self.norm_ff_macaron = nn.LayerNorm( + d_model + ) # for the macaron style FNN module + self.norm_ff = nn.LayerNorm(d_model) # for the FNN module + self.norm_mha = nn.LayerNorm(d_model) # for the MHA module + + self.ff_scale = 0.5 + + self.norm_conv = nn.LayerNorm(d_model) # for the CNN module + self.norm_final = nn.LayerNorm( + d_model + ) # for the final output of the block + + self.dropout = nn.Dropout(dropout) + + def forward( + self, + x: Tensor, + pos_emb: Tensor, + attn_mask: Optional[Tensor] = None, + key_padding_mask: Optional[Tensor] = None, + ) -> Tensor: + """ + Pass the input through the encoder layer. + + Args: + x: the sequence to the encoder layer (required). + pos_emb: Positional embedding tensor (required). + attn_mask: the mask for the x sequence's attention to itself (optional); + of shape (T, T) + key_padding_mask: the mask for the src keys per batch (optional). + + Shape: + x: (T, N, C) i.e. (seq_len, batch_size, num_channels) + pos_emb: (1, 2*T-1, C) + attn_mask: (T, T) or (N*num_heads, T, T), of dtype torch.bool or torch.float, where + the 1st S is interpreted as the target sequence (output) and the 2nd as the source + sequence (input). + key_padding_mask: (N, T), of dtype torch.bool + + T is the sequence length, N is the batch size, C is the number of channels. + Return: + Returns x with something added to it, of shape (T, N, C) + """ + + # macaron style feed forward module + residual = x + x = self.norm_ff_macaron(x) + x = residual + self.ff_scale * self.dropout( + self.feed_forward_macaron(x) + ) + + # multi-headed self-attention module + residual = x + x = self.norm_mha(x) + self_attn = self.self_attn(x, x, x, + pos_emb=pos_emb, + attn_mask=attn_mask, + key_padding_mask=key_padding_mask, + need_weights=False + )[0] + x = residual + self.dropout(self_attn) + + # convolution module + residual = x + x = self.norm_conv(x) + + x = residual + self.dropout(self.conv_module(x, key_padding_mask=key_padding_mask)) + + # feed forward module + residual = x + x = self.norm_ff(x) + x = residual + self.ff_scale * self.dropout(self.feed_forward(x)) + + x = self.norm_final(x) + + return x + + +def _get_clones(module, N): + return torch.nn.ModuleList([copy.deepcopy(module) for i in range(N)]) + +class MaskedLmConformerEncoder(nn.Module): + r"""MaskedLmConformerEncoder is a stack of N encoder layers, modified from + torch.nn.TransformerEncoder. The only differences are some name + changes for parameters. + + Args: + encoder_layer: an instance of the MaskedLmConformerEncoderLayer() class (required). + num_layers: the number of sub-encoder-layers in the encoder (required). + norm: the layer normalization component (optional). + + Examples:: + >>> encoder_layer = MaskedLmConformerEncoderLayer(d_model=512, nhead=8) + >>> conformer_encoder = MaskedLmConformerEncoder(encoder_layer, num_layers=6) + >>> src = torch.rand(10, 32, 512) + >>> src, pos_emb = self.encoder_pos(src) + >>> out = conformer_encoder(src, pos_emb) + """ + __constants__ = ['norm'] + + def __init__(self, encoder_layer: nn.Module, num_layers: int, + norm: Optional[nn.Module] = None): + super(MaskedLmConformerEncoder, self).__init__() + self.layers = _get_clones(encoder_layer, num_layers) + self.num_layers = num_layers + self.norm = norm + + + def forward( + self, + x: Tensor, + pos_emb: Tensor, + attn_mask: Optional[Tensor] = None, + key_padding_mask: Optional[Tensor] = None, + ) -> Tensor: + r"""Pass the input through the encoder layers in turn. + Args + x: input of shape (T, N, C), i.e. (seq_len, batch, channels) + pos_emb: positional embedding tensor of shape (1, 2*T-1, C), + attn_mask (optional, likely not used): mask for self-attention of + x to itself, of shape (T, T) + key_padding_mask (optional): mask of shape (N, T), dtype must be bool. + Returns: + Returns a tensor with the same shape as x, i.e. (T, N, C). + """ + for mod in self.layers: + x = mod( + x, + pos_emb, + attn_mask=attn_mask, + key_padding_mask=key_padding_mask, + ) + + if self.norm is not None: + x = self.norm(x) + + return x + + +class RelPositionalEncoding(torch.nn.Module): + """Relative positional encoding module. + + See : Appendix B in "Transformer-XL: Attentive Language Models Beyond a Fixed-Length Context" + Modified from https://github.com/espnet/espnet/blob/master/espnet/nets/pytorch_backend/transformer/embedding.py + + Args: + d_model: Embedding dimension. + dropout_rate: Dropout rate. + max_len: Maximum input length. + + """ + + def __init__( + self, d_model: int, dropout_rate: float, max_len: int = 5000 + ) -> None: + """Construct an PositionalEncoding object.""" + super(RelPositionalEncoding, self).__init__() + self.d_model = d_model + self.dropout = torch.nn.Dropout(p=dropout_rate) + self.pe = None + self.extend_pe(torch.tensor(0.0).expand(1, max_len)) + + def extend_pe(self, x: Tensor) -> None: + """Reset the positional encodings.""" + if self.pe is not None: + # self.pe contains both positive and negative parts + # the length of self.pe is 2 * input_len - 1 + if self.pe.size(1) >= x.size(1) * 2 - 1: + # Note: TorchScript doesn't implement operator== for torch.Device + if self.pe.dtype != x.dtype or str(self.pe.device) != str( + x.device + ): + self.pe = self.pe.to(dtype=x.dtype, device=x.device) + return + # Suppose `i` means to the position of query vector and `j` means the + # position of key vector. We use position relative positions when keys + # are to the left (i>j) and negative relative positions otherwise (i Tuple[Tensor, Tensor]: + """Add positional encoding. + + Args: + x (torch.Tensor): Input tensor (batch, time, C). + + Returns (x, pos_enc): + x: torch.Tensor: x itself, with dropout added: (batch, time, C). + pos_enc: torch.Tensor: Relative positional encoding as tensor of shape (1, 2*time-1, C). + + """ + self.extend_pe(x) + pos_emb = self.pe[ + :, + self.pe.size(1) // 2 + - x.size(1) + + 1 : self.pe.size(1) // 2 + + x.size(1), + ] + return self.dropout(x), self.dropout(pos_emb) + + +class RelPositionMultiheadAttention(nn.Module): + r"""Multi-Head Attention layer with relative position encoding + + See reference: "Transformer-XL: Attentive Language Models Beyond a Fixed-Length Context" + + Args: + embed_dim: total dimension of the model. + num_heads: parallel attention heads. + dropout: a Dropout layer on attn_output_weights. Default: 0.0. + + Examples:: + + >>> rel_pos_multihead_attn = RelPositionMultiheadAttention(embed_dim, num_heads) + >>> attn_output, attn_output_weights = rel_pos_multihead_attn(query, key, value, pos_emb) + """ + + def __init__( + self, + embed_dim: int, + num_heads: int, + dropout: float = 0.0, + ) -> None: + super(RelPositionMultiheadAttention, self).__init__() + self.embed_dim = embed_dim + self.num_heads = num_heads + self.dropout = dropout + self.head_dim = embed_dim // num_heads + assert ( + self.head_dim * num_heads == self.embed_dim + ), "embed_dim must be divisible by num_heads" + + self.in_proj = nn.Linear(embed_dim, 3 * embed_dim, bias=True) + self.out_proj = nn.Linear(embed_dim, embed_dim, bias=True) + + # linear transformation for positional encoding. + self.linear_pos = nn.Linear(embed_dim, embed_dim, bias=False) + # these two learnable bias are used in matrix c and matrix d + # as described in "Transformer-XL: Attentive Language Models Beyond a Fixed-Length Context" Section 3.3 + self.pos_bias_u = nn.Parameter(torch.Tensor(num_heads, self.head_dim)) + self.pos_bias_v = nn.Parameter(torch.Tensor(num_heads, self.head_dim)) + + self._reset_parameters() + + def _reset_parameters(self) -> None: + nn.init.xavier_uniform_(self.in_proj.weight) + nn.init.constant_(self.in_proj.bias, 0.0) + nn.init.constant_(self.out_proj.bias, 0.0) + + nn.init.xavier_uniform_(self.pos_bias_u) + nn.init.xavier_uniform_(self.pos_bias_v) + + def forward( + self, + query: Tensor, + key: Tensor, + value: Tensor, + pos_emb: Tensor, + key_padding_mask: Optional[Tensor] = None, + need_weights: bool = True, + attn_mask: Optional[Tensor] = None, + ) -> Tuple[Tensor, Optional[Tensor]]: + r""" + Args (see below for shapes): + query, key, value: map a query and a set of key-value pairs to an output. + pos_emb: Positional embedding tensor + key_padding_mask: if provided, specified padding elements in the key will + be ignored by the attention. When given a binary mask and a value is True, + the corresponding value on the attention layer will be ignored. When given + a byte mask and a value is non-zero, the corresponding value on the attention + layer will be ignored + need_weights: if true, return (output, attn_output_weights); else, (output, None). + + attn_mask: 2D or 3D mask that prevents attention to certain positions. A 2D mask will be broadcasted for all + the batches while a 3D mask allows to specify a different mask for the entries of each batch. + + Shape: + - Inputs: + - query: :math:`(T, N, C)` where T is the output sequence length, N is the batch size, C is + the embedding dimension (number of channels). + - key: :math:`(S, N, C)`, where S is the input sequence length. + - value: :math:`(S, N, C)` + - pos_emb: :math:`(N, 2*T-1, C)` or :math:`(1, 2*T-1, C)`. Note: this assumes T == S, which it will be, but + still we use different letters because S relates to the input position, T to the + output posision. + - key_padding_mask: :math:`(N, S)` where N is the batch size, S is the input sequence length. + If a ByteTensor is provided, the non-zero positions will be ignored while the position + with the zero positions will be unchanged. If a BoolTensor is provided, the positions with the + value of ``True`` will be ignored while the position with the value of ``False`` will be unchanged. + - attn_mask: 2D mask :math:`(T, S)` where T is the output sequence length, S is the input sequence length. + 3D mask :math:`(N*num_heads, T, S)` where N is the batch size, where T is the output sequence length, + S is the input sequence length. attn_mask ensure that position i is allowed to attend the unmasked + positions. If a ByteTensor is provided, the non-zero positions are not allowed to attend + while the zero positions will be unchanged. If a BoolTensor is provided, positions with ``True`` + is not allowed to attend while ``False`` values will be unchanged. If a FloatTensor + is provided, it will be added to the attention weight. + + Return: + (output, attn_output_weights) if need_weights==True, else (output, None), where: + + - output: :math:`(T, N, C)` where T is the output sequence length, N is the batch size, + C is the embedding/channel dimension. + - attn_output_weights: :math:`(N, T, S)` where N is the batch size, + T is the output sequence length, S is the input sequence length (actually + S and T are the same number). + """ + return self.multi_head_attention_forward( + query, + key, + value, + pos_emb, + self.embed_dim, + self.num_heads, + self.in_proj.weight, + self.in_proj.bias, + self.dropout, + self.out_proj.weight, + self.out_proj.bias, + training=self.training, + key_padding_mask=key_padding_mask, + need_weights=need_weights, + attn_mask=attn_mask, + ) + + def rel_shift(self, x: Tensor) -> Tensor: + """Compute relative positional encoding. + + Args: + x: Input tensor (batch, head, time1, 2*time1-1). + time1 means the length of query vector. + + Returns: + Tensor: tensor of shape (batch, head, time1, time2) + (note: time2 has the same value as time1, but it is for + the key, while time1 is for the query). + """ + (batch_size, num_heads, time1, n) = x.shape + assert n == 2 * time1 - 1 + # Note: TorchScript requires explicit arg for stride() + batch_stride = x.stride(0) + head_stride = x.stride(1) + time1_stride = x.stride(2) + n_stride = x.stride(3) + return x.as_strided( + (batch_size, num_heads, time1, time1), + (batch_stride, head_stride, time1_stride - n_stride, n_stride), + storage_offset=n_stride * (time1 - 1), + ) + + def multi_head_attention_forward( + self, + query: Tensor, + key: Tensor, + value: Tensor, + pos_emb: Tensor, + embed_dim_to_check: int, + num_heads: int, + in_proj_weight: Tensor, + in_proj_bias: Tensor, + dropout_p: float, + out_proj_weight: Tensor, + out_proj_bias: Tensor, + training: bool = True, + key_padding_mask: Optional[Tensor] = None, + need_weights: bool = True, + attn_mask: Optional[Tensor] = None, + ) -> Tuple[Tensor, Optional[Tensor]]: + r""" + Args: + query, key, value: map a query and a set of key-value pairs to an output. + pos_emb: Positional embedding tensor + embed_dim_to_check: total dimension of the model. + num_heads: parallel attention heads. + in_proj_weight, in_proj_bias: input projection weight and bias. + dropout_p: probability of an element to be zeroed. + out_proj_weight, out_proj_bias: the output projection weight and bias. + training: apply dropout if is ``True``. + key_padding_mask: if provided, specified padding elements in the key will + be ignored by the attention. This is an binary mask. When the value is True, + the corresponding value on the attention layer will be filled with -inf. + need_weights: output attn_output_weights. + attn_mask: 2D or 3D mask that prevents attention to certain positions. A 2D mask will be broadcasted for all + the batches while a 3D mask allows to specify a different mask for the entries of each batch. + + Shape: + Inputs: + - query: :math:`(L, N, E)` where L is the target sequence length, N is the batch size, E is + the embedding dimension. + - key: :math:`(S, N, E)`, where S is the source sequence length, N is the batch size, E is + the embedding dimension. + - value: :math:`(S, N, E)` where S is the source sequence length, N is the batch size, E is + the embedding dimension. + - pos_emb: :math:`(N, 2*L-1, E)` or :math:`(1, 2*L-1, E)` where L is the target sequence + length, N is the batch size, E is the embedding dimension. + - key_padding_mask: :math:`(N, S)` where N is the batch size, S is the source sequence length. + If a ByteTensor is provided, the non-zero positions will be ignored while the zero positions + will be unchanged. If a BoolTensor is provided, the positions with the + value of ``True`` will be ignored while the position with the value of ``False`` will be unchanged. + - attn_mask: 2D mask :math:`(L, S)` where L is the target sequence length, S is the source sequence length. + 3D mask :math:`(N*num_heads, L, S)` where N is the batch size, L is the target sequence length, + S is the source sequence length. attn_mask ensures that position i is allowed to attend the unmasked + positions. If a ByteTensor is provided, the non-zero positions are not allowed to attend + while the zero positions will be unchanged. If a BoolTensor is provided, positions with ``True`` + are not allowed to attend while ``False`` values will be unchanged. If a FloatTensor + is provided, it will be added to the attention weight. + + Outputs: + - attn_output: :math:`(L, N, E)` where L is the target sequence length, N is the batch size, + E is the embedding dimension. + - attn_output_weights: :math:`(N, L, S)` where N is the batch size, + L is the target sequence length, S is the source sequence length. + """ + + tgt_len, bsz, embed_dim = query.size() + assert embed_dim == embed_dim_to_check + assert key.size(0) == value.size(0) and key.size(1) == value.size(1) + + head_dim = embed_dim // num_heads + assert ( + head_dim * num_heads == embed_dim + ), "embed_dim must be divisible by num_heads" + scaling = float(head_dim) ** -0.5 + + if torch.equal(query, key) and torch.equal(key, value): + # self-attention + q, k, v = nn.functional.linear( + query, in_proj_weight, in_proj_bias + ).chunk(3, dim=-1) + + elif torch.equal(key, value): + # encoder-decoder attention + # This is inline in_proj function with in_proj_weight and in_proj_bias + _b = in_proj_bias + _start = 0 + _end = embed_dim + _w = in_proj_weight[_start:_end, :] + if _b is not None: + _b = _b[_start:_end] + q = nn.functional.linear(query, _w, _b) + # This is inline in_proj function with in_proj_weight and in_proj_bias + _b = in_proj_bias + _start = embed_dim + _end = None + _w = in_proj_weight[_start:, :] + if _b is not None: + _b = _b[_start:] + k, v = nn.functional.linear(key, _w, _b).chunk(2, dim=-1) + + else: + # This is inline in_proj function with in_proj_weight and in_proj_bias + _b = in_proj_bias + _start = 0 + _end = embed_dim + _w = in_proj_weight[_start:_end, :] + if _b is not None: + _b = _b[_start:_end] + q = nn.functional.linear(query, _w, _b) + + # This is inline in_proj function with in_proj_weight and in_proj_bias + _b = in_proj_bias + _start = embed_dim + _end = embed_dim * 2 + _w = in_proj_weight[_start:_end, :] + if _b is not None: + _b = _b[_start:_end] + k = nn.functional.linear(key, _w, _b) + + # This is inline in_proj function with in_proj_weight and in_proj_bias + _b = in_proj_bias + _start = embed_dim * 2 + _end = None + _w = in_proj_weight[_start:, :] + if _b is not None: + _b = _b[_start:] + v = nn.functional.linear(value, _w, _b) + + #if not self.is_espnet_structure: + # q = q * scaling + + if attn_mask is not None: + assert ( + attn_mask.dtype == torch.float32 + or attn_mask.dtype == torch.float64 + or attn_mask.dtype == torch.float16 + or attn_mask.dtype == torch.uint8 + or attn_mask.dtype == torch.bool + ), "Only float, byte, and bool types are supported for attn_mask, not {}".format( + attn_mask.dtype + ) + if attn_mask.dtype == torch.uint8: + warnings.warn( + "Byte tensor for attn_mask is deprecated. Use bool tensor instead." + ) + attn_mask = attn_mask.to(torch.bool) + + if attn_mask.dim() == 2: + attn_mask = attn_mask.unsqueeze(0) + if list(attn_mask.size()) != [1, query.size(0), key.size(0)]: + raise RuntimeError( + "The size of the 2D attn_mask is not correct." + ) + elif attn_mask.dim() == 3: + if list(attn_mask.size()) != [ + bsz * num_heads, + query.size(0), + key.size(0), + ]: + raise RuntimeError( + "The size of the 3D attn_mask is not correct." + ) + else: + raise RuntimeError( + "attn_mask's dimension {} is not supported".format( + attn_mask.dim() + ) + ) + # attn_mask's dim is 3 now. + + # convert ByteTensor key_padding_mask to bool + if ( + key_padding_mask is not None + and key_padding_mask.dtype == torch.uint8 + ): + warnings.warn( + "Byte tensor for key_padding_mask is deprecated. Use bool tensor instead." + ) + key_padding_mask = key_padding_mask.to(torch.bool) + + q = q.contiguous().view(tgt_len, bsz, num_heads, head_dim) + k = k.contiguous().view(-1, bsz, num_heads, head_dim) + v = v.contiguous().view(-1, bsz * num_heads, head_dim).transpose(0, 1) + + src_len = k.size(0) + + if key_padding_mask is not None: + assert key_padding_mask.size(0) == bsz, "{} == {}".format( + key_padding_mask.size(0), bsz + ) + assert key_padding_mask.size(1) == src_len, "{} == {}".format( + key_padding_mask.size(1), src_len + ) + + q = q.transpose(0, 1) # (batch, time1, head, d_k) + + pos_emb_bsz = pos_emb.size(0) + assert pos_emb_bsz in (1, bsz) # actually it is 1 + p = self.linear_pos(pos_emb).view(pos_emb_bsz, -1, num_heads, head_dim) + p = p.transpose(1, 2) # (batch, head, 2*time1-1, d_k) + + q_with_bias_u = (q + self.pos_bias_u).transpose( + 1, 2 + ) # (batch, head, time1, d_k) + + q_with_bias_v = (q + self.pos_bias_v).transpose( + 1, 2 + ) # (batch, head, time1, d_k) + + # compute attention score + # first compute matrix a and matrix c + # as described in "Transformer-XL: Attentive Language Models Beyond a Fixed-Length Context" Section 3.3 + k = k.permute(1, 2, 3, 0) # (batch, head, d_k, time2) + matrix_ac = torch.matmul( + q_with_bias_u, k + ) # (batch, head, time1, time2) + + # compute matrix b and matrix d + matrix_bd = torch.matmul( + q_with_bias_v, p.transpose(-2, -1) + ) # (batch, head, time1, 2*time1-1) + matrix_bd = self.rel_shift(matrix_bd) + + #if not self.is_espnet_structure: + # attn_output_weights = ( + # matrix_ac + matrix_bd + # ) # (batch, head, time1, time2) + #else: + + attn_output_weights = ( + matrix_ac + matrix_bd + ) * scaling # (batch, head, time1, time2) + + attn_output_weights = attn_output_weights.view( + bsz * num_heads, tgt_len, -1 + ) + + assert list(attn_output_weights.size()) == [ + bsz * num_heads, + tgt_len, + src_len, + ] + + if attn_mask is not None: + if attn_mask.dtype == torch.bool: + attn_output_weights.masked_fill_(attn_mask, float("-inf")) + else: + attn_output_weights += attn_mask + + if key_padding_mask is not None: + attn_output_weights = attn_output_weights.view( + bsz, num_heads, tgt_len, src_len + ) + attn_output_weights = attn_output_weights.masked_fill( + key_padding_mask.unsqueeze(1).unsqueeze(2), + float("-inf"), + ) + attn_output_weights = attn_output_weights.view( + bsz * num_heads, tgt_len, src_len + ) + + attn_output_weights = nn.functional.softmax(attn_output_weights, dim=-1) + attn_output_weights = nn.functional.dropout( + attn_output_weights, p=dropout_p, training=training + ) + + attn_output = torch.bmm(attn_output_weights, v) + assert list(attn_output.size()) == [bsz * num_heads, tgt_len, head_dim] + attn_output = ( + attn_output.transpose(0, 1) + .contiguous() + .view(tgt_len, bsz, embed_dim) + ) + attn_output = nn.functional.linear( + attn_output, out_proj_weight, out_proj_bias + ) + + if need_weights: + # average attention weights over heads + attn_output_weights = attn_output_weights.view( + bsz, num_heads, tgt_len, src_len + ) + return attn_output, attn_output_weights.sum(dim=1) / num_heads + else: + return attn_output, None diff --git a/egs/librispeech/ASR/conformer_lm/dataset.py b/egs/librispeech/ASR/conformer_lm/dataset.py new file mode 100644 index 000000000..6c28c21ca --- /dev/null +++ b/egs/librispeech/ASR/conformer_lm/dataset.py @@ -0,0 +1,823 @@ +import torch +import torch.distributed as dist +import k2 +import _k2 +import logging +import sentencepiece as spm +from pathlib import Path +from typing import Optional, List, Tuple, Union + + + +class LmDataset(torch.utils.data.Dataset): + """ + Torch dataset for language modeling data. This is a map-style dataset. + The indices are integers. + """ + def __init__(self, sentences: k2.RaggedTensor, + words: k2.RaggedTensor): + super(LmDataset, self).__init__() + self.sentences = sentences + self.words = words + + + def __len__(self): + # Total size on axis 0, == num sentences + return self.sentences.tot_size(0) + + def __getitem__(self, i: int): + """ + Return the i'th sentence, as a list of ints (representing BPE pieces, without + bos or eos symbols). + """ + # in future will just do: + #return self.words[self.sentences[i]].tolist() + + # It would be nicer if we could just return self.sentences[i].tolist(), + # but for now that operator on k2.RaggedInt does not support when the + # ragged has only 2 axes. + row_splits = self.sentences.shape.row_splits(1) + (begin, end) = row_splits[i:i+2].tolist() + sentence = self.sentences.data[begin:end] + sentence, _ = self.words.index(sentence, axis=0, need_value_indexes=False) + return sentence.data.tolist() + + +def load_train_test_lm_dataset(archive_fn: Union[str,Path], + test_proportion: float = 0.025) -> Tuple[LmDataset, LmDataset]: + """ + returns (train_lm_dataset, test_lm_dataset) + """ + + d = torch.load(archive_fn) + words = d['words'] # a k2.RaggedTensor with 2 axes, maps from word-ids to sequences of BPE pieces + sentences = d['data'] # a k2.RaggedTensor + + with torch.random.fork_rng(devices=[]): + g = torch.manual_seed(0) + num_sentences = sentences.tot_size(0) + # probably the generator (g) argument to torch.randperm below is not necessary. + sentence_perm = torch.randperm(num_sentences, generator=g, dtype=torch.int32) + sentences, _ = sentences.index(sentence_perm, axis=0, need_value_indexes=False) + + num_test_sentences = int(num_sentences * test_proportion) + + axis=0 + train_sents = sentences.arange(axis, num_test_sentences, num_sentences) + test_sents = sentences.arange(axis, 0, num_test_sentences) + + return LmDataset(train_sents, words), LmDataset(test_sents, words) + + +def mask_and_pad(sentence: List[int], + seq_len: int, + bos_sym: int, + eos_sym: int, + blank_sym: int, + mask_proportion: float, + padding_proportion: float, + inv_mask_length: float, + unmasked_weight: float) -> Tuple[List[int], List[int], List[int], List[float]]: + """ + This function contains part of the logic of collate_fn, broken out. It is responsible + for inserting masking and padding into the sequence `sentence`. Most of the arguments + are documented for `collate_fn` below. + Other args: + sentence: The original sentence to be masked and padded. + seq_len: The desired length of the lists to be returned + bos_sym, eos_sym, blank_sym, mask_proportion, + padding_proportion, inv_mask_length, unmasked_weight: see their documentation + as args to `collate_fn` below. + + + Return: a tuple (src, masked_src, tgt, weight, randomizable, attn_mask), all lists of length `seq_len`, + where: + `src` is: [bos] + [the sentence after inserting blanks in place of padding + after regions to be masked] + [eos] + [blank padding to seq_len]. + `src_masked` is as `src` but the masked regions have their values replaced with blank, + i.e. they are actually masked. + `tgt` is: [the original sentence, without masking] + [eos] + [blank] + [blank padding to seq_len] + `weight` is the weight at the nnet output, which is: `unmasked_weight` for un-masked + positions, 1.0 for masked and padded positions, and 0.0 for positions that + correspond to blank-padding after the final [eos]. + `randomizable` is a bool that is True for positions where the symbol in + in `src_masked` is not bos or eos or blank. + `attn_mask` is a bool that is False for positions in `src` and `src_masked` that + are between the initial [bos] and final [eos] inclusive; and True for + positions after the final [eos]. + """ + sent_len = len(sentence) + assert sent_len + 3 <= seq_len + + for w in sentence: + assert w not in [bos_sym, eos_sym, blank_sym] + + num_mask = int(torch.binomial(count=torch.tensor([sent_len * 1.0]), + prob=torch.tensor([mask_proportion])).item()) + num_pad = int(torch.poisson(torch.tensor([sent_len * padding_proportion])).item()) + # Ensure the total length after bos, padding of masked sequences, and eos, is + # no greater than seq_len + num_pad -= max(0, sent_len + 2 + num_pad - seq_len) + + if num_mask + num_pad == 0: + num_pad += 1 + + # num_split_points is the number of times we split the (masked+padded) + # region, so the total number of (masking+padding) subsequences will be + # num_split_points + 1. If num_mask positions are masked, then the + # remaining number of words is `sent_len - num_mask`, and any two + # masked regions must have at least one non-masked word between them, + # so num_split_points == number of masked regions - 1, must be + # no greater than `sent_len - num_mask`. The formula about + # mask_proportion * inv_mask_length / (1.0 - mask_proportion) + # is what's required (I think) so that inv_mask_length is the expected + # length of masked regions. + num_split_points = int(torch.binomial(count=torch.tensor([float(sent_len - num_mask)]), + prob=torch.tensor([mask_proportion * inv_mask_length / (1.0 - mask_proportion)])).item()) + # Somehow this assertion failed, debugging it below. + assert num_split_points <= sent_len - num_mask + + assert isinstance(num_split_points, int) + + def split_into_subseqs(length: int , num_subseqs: int) -> List[int]: + """Splits a sequence of `length` items into `num_subseqs` possibly-empty + subsequences. The length distributions are geometric, not Poisson, i.e. + we choose the split locations with uniform probability rather than + randomly assigning each word to one subsequences. This gives us more + shorter/longer subsequences. + Require num_subseqs > 0 + """ + boundaries = [0] + sorted(torch.randint(low=0, high=length + 1, size=(num_subseqs - 1,)).tolist()) + [length] + return [ boundaries[i + 1] - boundaries[i] for i in range(num_subseqs) ] + + mask_lengths = split_into_subseqs(num_mask, num_split_points + 1) + pad_lengths = split_into_subseqs(num_pad, num_split_points + 1) + # mask_pad_lengths contains only the (mask, pad) length pairs for which mask + pad > 0. + # From this point we only refer to the mask_pad_lengths. + mask_pad_lengths = [ (mask, pad) for (mask, pad) in zip(mask_lengths, pad_lengths) if mask+pad > 0 ] + num_subseqs = len(mask_pad_lengths) + assert num_subseqs > 0 + + # Now figure out how to distribute these subsequences throughout the actual + # sentence. The subsequences, if there are more than one, must not touch, + # i.e. there must be an actual word in between each subsequence, where the + # number of such "mandatory" words equals num_subseqs - 1. We also have to + # subtract `num_mask` words, since obviously the masked words cannot separate + # the masked regions. + reduced_len = sent_len - num_mask - (num_subseqs - 1) + assert reduced_len >= 0 + # unmasked_lengths will be the lengths of the un-masked regions between the masked + # regions. + unmasked_lengths = split_into_subseqs(reduced_len, num_subseqs + 1) + for i in range(1, num_subseqs): + # Unmasked regions between masked regions must have length at least 1, + # we add 1 to unmasked regions that are not initial/final. + unmasked_lengths[i] = unmasked_lengths[i] + 1 + assert sum(unmasked_lengths) + sum(mask_lengths) == sent_len + + + # src_positions will be: for each position in the masked+padded sentence, + # the corresponding position in the source sentence `sentence`; or -1 + # if this was padding. + src_positions = [] + # `masked` will be: for each position in the masked+padded sentence, True if + # it was masked and False otherwise. (Note: it is False for padding + # locations, although this will not matter in the end). + masked = [] + + cur_pos = 0 # current position in source sentence + for i in range(num_subseqs + 1): + for j in range(unmasked_lengths[i]): + src_positions.append(cur_pos) + masked.append(False) + cur_pos += 1 + if i < num_subseqs: + (mask_len, pad_len) = mask_pad_lengths[i] + for j in range(mask_len): + src_positions.append(cur_pos) + masked.append(True) + cur_pos += 1 + for j in range(pad_len): + src_positions.append(-1) + masked.append(False) + assert cur_pos == len(sentence) + + + src = [] + src_masked = [] + tgt = [] + weight = [] + randomizable = [] + + src.append(bos_sym) + src_masked.append(bos_sym) + randomizable.append(False) + for i, src_pos in enumerate(src_positions): + is_masked = masked[i] + if src_pos >= 0: + src_word = sentence[src_pos] + src_masked.append(blank_sym if masked[i] else src_word) + src.append(src_word) + tgt.append(src_word) + weight.append(1.0 if masked[i] else unmasked_weight) + randomizable.append(not masked[i]) + else: + # Padding inside a masked region + src_masked.append(blank_sym) + src.append(blank_sym) + tgt.append(blank_sym) + weight.append(1.0) + randomizable.append(False) + src.append(eos_sym) + src_masked.append(eos_sym) + tgt.append(eos_sym) + weight.append(unmasked_weight) + tgt.append(blank_sym) + weight.append(0.0) + randomizable.append(False) + + attn_mask = ([False] * len(src)) + ([True] * (seq_len - len(src))) + + for i in range(seq_len - len(src)): + src.append(blank_sym) + src_masked.append(blank_sym) + tgt.append(blank_sym) + weight.append(0.0) + randomizable.append(False) + + return (src, src_masked, tgt, weight, randomizable, attn_mask) + + +# dataset.mask_and_pad(list(range(10, 20)), seq_len=16, bos_sym=1, eos_sym=2, blank_sym=0, mask_proportion=0.2, padding_proportion=0.2, inv_mask_length=0.33, unmasked_weight=0.444) + +# dataset.collate_fn(sentences=[ list(range(10, 20)), list(range(30, 45))], bos_sym=1, eos_sym=2, blank_sym=0, mask_proportion=0.2, padding_proportion=0.2, randomize_proportion=0.05, inv_mask_length=0.33, unmasked_weight=0.444) + +def collate_fn(sentences: List[List[int]], + bos_sym: int, + eos_sym: int, + blank_sym: int, + mask_proportion: float = 0.15, + padding_proportion: float = 0.15, + randomize_proportion: float = 0.05, + inv_mask_length: float = 0.25, + unmasked_weight: float = 0.25, + debug: bool = False) -> Tuple[torch.Tensor, torch.Tensor, + torch.Tensor, torch.Tensor, + torch.Tensor]: + """ + Caution, this is not the collate_fn we give directly to the dataloader, + we give it a lambda: collate_fn=(lambda x: dataset.collate_fn(x, [other args])) + This formats a list-of-lists-of-int into 5 Tensors, explained below. + The key thing is that we mask out subsequences of random length within + these sentences, and force the network to predict the masked-out + subsequences (which have blanks appended to them to prevent the model + from knowing the exact length of the sequences it has to predict). + So it's like BERT but at the level of sequences rather than individual + words. + + Args: + bos_sym: the integer id of the beginning-of-sentence symbol, e.g. 2. + Is allowed be the same as eos_sym (we are not necessarily + saying it will work best that way). + eos_sym: the integer id of the end-of-sentence symbol, e.g. 2. + blank_sym: the integer id of the blank symbol, e.g. 0 or 1. + mask_proportion: The proportion of words in each sentence that + are masked, interpreted as (roughly) the probability of any given + word being masked, although the masked locations will + tend to be in contiguous sequences (they are not independent). + padding_proportion: Like mask_proportion, but determines the + number of extra, blank symbols that are inserted as padding + at the end of masked regions (this ensures that the model + cannot know exactly how many words need to be inserted in + any given masked region. + randomize_proportion: The probability with which we replace + words that were not masked with randomly chosen words. + Like BERT, this is intended to force the model to predict + something reasonable at non-masked positions, and to make + this task harder than simply repeating the input. + inv_mask_length: This number determines how many separate + sub-sequences the (masked + padded) proportion of a sentence is split up + into, interpreted as the inverse of the expected length of + each *masked* region. + unmasked_weight: The weight to be applied to the log-likelihoods of + un-masked positions in sentences (predicting un-masked + positions is not completely trivial if randomize_proportion > 0). + Will be reflected in the returned tgt_weights tensor. + + Returns a tuple (masked_src_symbols, src_symbols, + tgt_symbols, src_key_padding_mask, + tgt_weights), + all with 2 axes and the same shape: (num_sent, seq_len). + Their dtypes will be, respectively, + (torch.int64, torch.int64, + torch.int64, torch.bool, + torch.float) + masked_src_symbols: The sentences, with bos_symbol prepended and eos_symbol + appended, masked regions (including padding) replaced with blank, + and `randomize_proportion` non-masked symbols replaced with + symbols randomly taken from elsewhere in the sentences of this + minibatch. Then padded to a fixed length with blank. + src_symbols: Like masked_src_symbols, except with the masked symbols replaced + with the original symbols (but the padding that follows each + masked sub-sequence will still be blank) + tgt_symbols: The original sentences, with eos_symbol appended, and then + padded with blank to the same length as masked_symbols and + src_symbols. + src_key_padding_mask: Masking tensor for masked_src_symbols and src_symbols, to + account for all the sentence lengths not being identical + (makes each sentence's processing independent of seq_len). + Tensor of Bool of shape (num_sent, seq_len), with True + for masked positions (these are the blanks that follow the + eos_symbol in masked_src_symbols), False for un-masked positions. + tgt_weights: Weights that will be applied to the log-probabilities at + the output of the network. Will have 1.0 in positions + in `tgt_symbols` that were masked (including blank + padding at the end of masked regions), `unmasked_weight` + in other positions in the original sentences (including + terminating eos_symbol); and 0.0 in the remaining positions + corresponding to blank padding after the ends of + sentences. + """ + assert blank_sym not in [bos_sym, eos_sym] + max_sent_len = max([ len(s) for s in sentences]) + #logging.info(f"Sentence lengths: {[ len(s) for s in sentences]}") + + typical_mask_and_pad = int(max_sent_len * (mask_proportion + padding_proportion)) + + # The following formula gives roughly 1 standard deviation above where we'd + # expect the maximum sentence length to be with masking and padding.. we use + # this as a hard upper limit, to prevent outliers from affecting the batch + # size too much. We use this as the size `seq_len`. + # The "+ 4" is to ensure there is always room for the BOS, EOS and at least + # two padding symbols. + seq_len = max_sent_len + 4 + typical_mask_and_pad + int(typical_mask_and_pad ** 0.5) + + + # srcs, srcs_masked, tgts and weights will be lists of the lists returned + # from `mask_and_pad`, one per sentence. + srcs = [] + srcs_masked = [] + tgts = [] + weights = [] + randomizables = [] + attn_masks = [] + for s in sentences: + (src, src_masked, tgt, + weight, randomizable, + attn_mask) = mask_and_pad(s, seq_len, bos_sym, eos_sym, + blank_sym, mask_proportion, padding_proportion, + inv_mask_length, unmasked_weight) + srcs.append(src) + srcs_masked.append(src_masked) + tgts.append(tgt) + weights.append(weight) + randomizables.append(randomizable) + attn_masks.append(attn_mask) + + src_symbols = torch.tensor(srcs, dtype=torch.int64) + masked_src_symbols = torch.tensor(srcs_masked, dtype=torch.int64) + tgt_symbols = torch.tensor(tgts, dtype=torch.int64) + src_key_padding_mask = torch.tensor(attn_masks, dtype=torch.bool) + tgt_weights = torch.tensor(weights, dtype=torch.float) + + attn_mask_sum = torch.sum(torch.logical_not(src_key_padding_mask), dim=0).tolist() + while attn_mask_sum[-1] == 0: # Remove always-masked positions at the endof the lists. + attn_mask_sum.pop() + if len(attn_mask_sum) < seq_len: + seq_len = len(attn_mask_sum) + (src_symbols, masked_src_symbols, + tgt_symbols, src_key_padding_mask, tgt_weights) = (src_symbols[:,:seq_len], masked_src_symbols[:,:seq_len], + tgt_symbols[:,:seq_len], src_key_padding_mask[:,:seq_len], + tgt_weights[:,:seq_len]) + + if randomize_proportion > 0.0: + randomizable_tensor = torch.tensor(randomizables, dtype=torch.bool) + randomizable_indexes = torch.nonzero(randomizable_tensor) # (num_randomizable, 2) + num_randomizable = randomizable_indexes.shape[0] + + to_randomize_indexes = torch.nonzero(torch.rand(num_randomizable) < randomize_proportion, as_tuple=True)[0] + num_to_randomize = to_randomize_indexes.numel() + + # older versions of torch don't have tensor_split, so fake a simplified version of it. + # we'd be calling it as xxx.tensor_split(dim=1) if really in torc. + def tensor_split(t): + return (t[:,0], t[:,1]) + + random_src_locations = torch.randperm(num_randomizable)[:num_to_randomize] + + random_symbols = src_symbols[tensor_split(randomizable_indexes[random_src_locations])] + random_indexes_tuple= tensor_split(randomizable_indexes[to_randomize_indexes]) + src_symbols[random_indexes_tuple] = random_symbols + masked_src_symbols[random_indexes_tuple] = random_symbols + + + # I set this to true and tested with: + # python3 -c 'import dataset; dataset.collate_fn(sentences=[ list(range(100, 200)), list(range(300, 450)), list(range(500,600))], bos_sym=1, eos_sym=2, blank_sym=0, mask_proportion=0.2, padding_proportion=0.2, randomize_proportion=0.05, inv_mask_length=0.33, unmasked_weight=0.444)' + #.. and ran a few times to check the values printed looked about right, and that no assertions failed. + if debug: + check_collated_tensors(sentences, bos_sym, eos_sym, blank_sym, + unmasked_weight, + masked_src_symbols, src_symbols, + tgt_symbols, src_key_padding_mask, tgt_weights) + return (masked_src_symbols, src_symbols, + tgt_symbols, src_key_padding_mask, tgt_weights) + + + +def check_collated_tensors(sentences: List[List[int]], + bos_sym: int, + eos_sym: int, + blank_sym: int, + unmasked_weight: float, + masked_src_symbols, src_symbols, + tgt_symbols, src_key_padding_mask, + tgt_weights): + """ + This function checks the output of collate_fn, consider it test code. Please see + the documentation of collate_fn to understand the args. + """ + for t in src_symbols, tgt_symbols, src_key_padding_mask, tgt_weights: + assert t.shape == masked_src_symbols.shape + + tot_positions = src_symbols.numel() + + masked_src_symbols, src_symbols, tgt_symbols, src_key_padding_mask, tgt_weights = ( + masked_src_symbols.tolist(), src_symbols.tolist(), tgt_symbols.tolist(), + src_key_padding_mask.tolist(), tgt_weights.tolist()) + assert len(sentences) == len(masked_src_symbols) + + tot_masked_positions = 0 + tot_padded_positions = 0 + tot_unmasked_positions = 0 # all un-masked, non-blank postions, including eos + tot_randomized_positions = 0 + num_masked_subseqs = 0 + tot_symbols = 0 # original symbols in sentences, no bos/eos + + assert unmasked_weight > 0.001 # or this test code won't work.. + + for i in range(len(sentences)): + reconstructed_sent = list(filter(lambda x: x not in [bos_sym,eos_sym,blank_sym], tgt_symbols[i])) + if sentences[i] != reconstructed_sent: + print(f"Error: sentence {i}={sentences[i]} differs from {reconstructed_sent}") + (masked_src, src, tgt, src_mask, weights) = (masked_src_symbols[i], src_symbols[i], + tgt_symbols[i], src_key_padding_mask[i], tgt_weights[i]) + + assert src[0] == masked_src[0] == bos_sym + for j in range(len(masked_src)): + assert masked_src[j] == blank_sym or masked_src[j] == src[j] + + if src[j] not in [bos_sym, eos_sym, blank_sym]: + tot_symbols += 1 + + if j > 0: + assert (src[j] == eos_sym) == (masked_src[j] == eos_sym) == (tgt[j-1] == eos_sym) + if masked_src[j] == blank_sym: # masked or padding of masked subseq, or post-eos padding.. + assert src[j] == tgt[j - 1] # masked symbols are not randomized. + assert weights[j - 1] in [0.0, 1.0] # 0.0 for final blank padding + if weights[j - 1] == 1.0: # Not final blank padding... + if tgt[j - 1] == blank_sym: + tot_padded_positions += 1 + else: + tot_masked_positions += 1 + if masked_src[j + 1] != blank_sym: + num_masked_subseqs += 1 + else: + assert weights[j - 1] == 0 or abs(weights[j-1] - unmasked_weight) < 0.001 + if abs(weights[j - 1]-unmasked_weight) < 0.001: + tot_unmasked_positions += 1 + if tgt[j - 1] != src[j]: + tot_randomized_positions += 1 + + if src_mask[j]: # if masked.. + assert src[j] == blank_sym + + assert tot_symbols == sum(len(x) for x in sentences) + + assert tot_unmasked_positions + tot_masked_positions == tot_symbols + len(sentences) + + print(f"{tot_unmasked_positions} + {tot_masked_positions} == {tot_symbols} + {len(sentences)}") + print(f"tot_symbols / tot_positions = {tot_symbols/tot_positions} (rest is bos,eos,padding)") + + print(f"Masking/tot_symbols = {tot_masked_positions/tot_symbols}, Padding/tot_symbols = {tot_padded_positions/tot_symbols}") + print(f"Randomization/tot_non_masked_symbols = {tot_randomized_positions/(tot_symbols-tot_masked_positions)}") + print(f"Mean masking length = {tot_masked_positions/num_masked_subseqs}, Mean padding length = {tot_padded_positions/num_masked_subseqs}") + + + +# This shows some useful code about the BPE encoding. +# import sentencepiece as spm +# sp = spm.SentencePieceProcessor() +# sp.load(bpe_model_fn) # bpe.model +# sp.GetPieceSize(..) +# sp.Decode(...) +# sp.Encode(...) + + +# import dataset +# import torch +# train,test = dataset.load_train_test_lm_dataset('../data/lm_training_5000/lm_data.pt') + + +# train_dl = torch.utils.data.DataLoader(train, batch_size=10, shuffle=True, collate_fn=(lambda x: train.collate_fn(x))) +# x = iter(train_dl) +# str(next(x)) +# '[ [ 10 38 651 593 3 1343 31 780 6 4172 112 788 1696 24 289 24 3 403 6 4493 162 92 71 328 417 217 338 14 5 3 1876 154 21 23 2237 43 3 1535 92 71 2816 7 1031 31 2318 92 2528 4806 14 206 3 954 1373 6 525 4 631 447 2639 ] [ 1014 336 171 209 795 10 16 90 27 787 139 53 45 2817 ] [ 11 980 51 22 1748 14 91 105 363 428 6 8 2887 3305 2525 2297 70 3 4651 6 27 282 335 426 134 292 5 193 3 539 2250 584 127 ] [ 9 3 1858 4 18 2257 4 6 41 748 10 304 7 229 83 2793 4 9 981 7 1484 33 3 103 7 539 5 477 3195 18 64 39 82 1034 6 3 4128 ] [ 17 147 22 7 708 60 133 174 105 4111 4 6 3 1384 65 50 1051 9 2953 6 3 461 180 1142 23 5 36 888 8 131 173 390 78 23 266 2822 715 46 182 65 22 1739 33 3 700 1450 14 233 4 ] [ 80 10 16 67 279 7 1827 264 96 3 187 2851 2108 ] [ 1473 48 106 227 9 160 2011 4 674 ] [ 3 954 762 29 85 228 33 8 940 40 4952 36 486 390 595 3 81 225 6 1440 125 346 134 296 126 419 1017 3824 4 8 179 184 11 33 580 1861 ] [ 30 22 245 15 117 8 2892 28 1204 145 7 3 236 3417 6 3 3839 5 3106 155 198 30 228 2555 46 15 32 41 747 72 9 25 977 ] [ 222 466 6 3157 ] ]' +# +# or: +# import k2 +# k2.ragged.to_list(next(x)) +# [shows something similar]. +# +# You'd really do something like: +# for epoch in range(max_epochs): +# for minibatch in train_dl: + + +# .. How to process data? Suppose we have a sentence like [259, 278, 45, 11, 303, 1319, 34, 15, 396, 3435, 7, 44]. +# +# First: we randomly choose one or more starting positins for a masked segment. +# Each sentence must have at least one masked segment (or there is no contribution to the loss function). +# We choose to have: +# num_masked_segments = max(1, len(sent) // 15) +# +# The length of the masked segment (this is the target for prediction), we set to the geometric +# distribution with the probability of success set to 3: +# +# g = torch.distributions.geometric.Geometric(probs=0.3) # <-- expected value is 3.333 +# Example of sampling: +# g.sample(sample_shape=torch.Size([10])) +# +# We now we randomly compute the location of the masked segments (length computed above) as follows: +# First, the masked segments must be separated by at least one non-masked word (else they would be +# a single segment). So for n masked segments, there are n-1 words required for minimal separation. +# If tot-length-of-segments + n-1 is greater than the sentence length, we just have the entire +# sentence be masked. Otherwise, we randomly divide the remaining number of words between the n+1 +# positions where they can appear (e.g. for 2 segments, this would be at the start, between the 2 segments, +# and at the end). This is the multinomial distribution, but we can more easily compute this +# directly using rand() and cutoffs, rather than creating a torch.distributions.Multinomial(). +# + +# Next we need to compute a random amount of blank padding (>= 0) for each of the masked regions; +# this is done so the model never knows the exact length of the masked region. We can just use the +# same distribution as for the length of the masked regions, i.e. geometric with success-prob=0.3 +# (expected padding length is 3). +# +# At this point we know where the masked regions are and how much padding they have. We can format +# the result as three lists, of the same length: +# +# sent: contains the words in the sentence with, in masked +# positions, the original (target) words, then with +# blank in the blank-padding after masked positions. +# +# sent_augmented: `sent` with, at a small defined percentage of positions +# that were *not* masked, the real token replaced with a +# token randomly chosen from the tokens in the minibatch. +# (like BERT, we use this type of augmentation, so the model +# has to predict the original token). +# +# masked_sent_augmented: List[int], contains the words in `sent_augmented`, except +# with masked positions and the blank padding after the masked regions +# both replaced with blank. +# +# +# +# The way these will be processed is as follows: +# +# masked_sent_in = [bos] + masked_sent_augmented + [eos] <-- so we know the sentence ended, distinguish it from truncated ones. +# sent_in = [bos] + sent_augmented + [eos] +# +# sent_out = sent + [eos] + [eos] #<--- the predicted targets at each point, although +# # we only really care about this in masked regions. +# # The extra eos is so that the length is the same as +# # masked_sent_in and sent_in. +# +# out_scale = (masked_sent==blk ? 1.0 : non_masked_scale) # e.g. non_masked_scale = 1.0 is fine, +# # this is a choice; we can perhaps +# # report these 2 parts of the loss +# # separately though. +# # <-- can also set the last element +# # of out_scale to a smaller number, since +# # it's a repeated eos. +# +# +# OK, how do we combine these into a minibatch? Firstly, we truncate sentences to a maximum +# length, e.g. 128, if `masked_sent_in`/`sent_in` have length longer than that. We choose randomly +# in each case to truncate the beginning or end, truncating both masked_sent_in/sent_in and sent_out +# from the same side. Caution: this means that these sentences may lack bos and/or eos symbols. +# +# Next, we combine shorter utterances by appending them ( all of: masked_sent_in, sent_in, out_scale) +# as long as doing so would keep the total length under 128. We then pad (masked_sent_in, sent_in, sent_out, out_scale) +# with: (,,, 0) up to the maximum length of any sentence in the minibatch <- or could use +# +# +# +# +# +# +# +# # i.e. ones where masked_sent is blank and zeros elsewhere; +# # this pertains to positions in `sent_out`. +# +# +# +# +# +# +# +# +# +# +# torch.distributions.gamma.Gamma(concentration=1.0, rate=1.0/5) + + + + +class LmBatchSampler(torch.utils.data.Sampler): + """ + A sampler that returns a batch of integer indexes as a list, intended for use + with class LmDataset. The sentences returned in each batch will all be about + the same size, and the batch size is specified as a number of words (we also + provide an option that allows you to limit the max memory consumed by transformers) + + Has support for distributed operation. + """ + def __init__(self, dataset: LmDataset, + symbols_per_batch: int, + length_ceil: float = 200.0, + length_floor: float = 4.0, + world_size: Optional[int] = None, + rank: int = None, + seed: int = 0, + delay_init: bool = False): + """ + Constructor documentation: + dataset: the LmDataset object that we are sampling from. This + class does not retain a reference to the LmDataset. + symbols_per_batch: The number of BPE symbols desired in each minibatch + length_floor: When the sentence length gets less than about this much, + the batch size stops increasing inversely with sentence + length. Prevent OOM on batches with short sentences. + length_ceil: After the sentence length gets more than about + this much, the batch size will start decreasing + as 1/(sentence-length^2). This is a mechanism to + avoid excessive memory consumption in transformers, when + sentence length gets long. + world_size: The world size for distributed operation; if None, + will be worked out from torch.distributed. + rank: The rank of this sampler/process for distributed operation; if None, + will be worked out from torch.distributed. + seed: The random seed + delay_init: If true, will omit calling self.set_epoch(0) at the + end of the __init__ function. In this case the caller + must call set_epoch(0). [Setting this option is necessary + to work with data-loader worker processes plus DDP, since + set_epoch() will use ddp, which I believe is a no-no prior + to initializing data-loaders.] + """ + self.seed = seed + self.symbols_per_batch = symbols_per_batch + self.length_floor = length_floor + self.quadratic_constant = 1.0 / length_ceil + self._maybe_init_distributed(world_size=world_size, rank=rank) + + # a configuration constant we don't expose. + self.multiplicative_random_length = 0.05 + + # "indexes" is the subset of indexes into LmDataset that this + # sampler is reponsible for (all of them, in the non-distributed case). + data_indexes = torch.arange(self.rank, len(dataset), self.world_size, dtype=torch.int32) # dtype=torch.int32 + + word_row_splits = dataset.words.shape.row_splits(1) # dtype=torch.int32 + word_lengths = word_row_splits[1:] - word_row_splits[:-1] # dtype=torch.int32 + + # the sentences this sampler is responsible for, as sequences of words. + # It's a ragged tensor of int32 + sentences, _ = dataset.sentences.index(data_indexes, axis=0) + + # sentence_lengths is a k2.RaggedTensor like `sentences`, but with the words replaced + # with their respective lengths, in BPE pieces. + sentence_lengths = k2.ragged.index(word_lengths, sentences) + del sentences # save memory + assert isinstance(sentence_lengths, k2.RaggedTensor) + + # convert to float so sum_per_sublist() will work (TODO: sum_per_sublist() will eventually + # support int32.) + sentence_lengths = sentence_lengths.to(dtype=torch.float32) + + # Convert into a simple tensor of float by adding lengths of words. + sentence_lengths = sentence_lengths.sum() + + assert isinstance(sentence_lengths, torch.Tensor) + assert sentence_lengths.dtype == torch.float32 + + # self.sentence_lengths is a Tensor with dtype=torch.float32. It + # contains the lengths, in BPE tokens, of the sentences that this + # sampler is responsible for, whose real indexes are in + # `data_indexes` above (this is not stored, as we know the formula). + self.sentence_lengths = sentence_lengths + + if not delay_init: + self.set_epoch(0) # this is responsible for setting self.sorted_data_indexes + + def _sync_sizes(self, device: torch.device = torch.device('cuda')): + # Calling this on all copies of a DDP setup will sync the sizes so that + # all copies have the exact same number of batches. I think + # this needs to be called with the GPU device, not sure if it would + # work otherwise. + if self.world_size > 1: + min_size = torch.tensor([len(self.batch_indices)], device=device, dtype=torch.int64) + dist.all_reduce(min_size, op=dist.ReduceOp.MIN) + min_size = min_size.to('cpu').item() + logging.info(f"world_size={self.world_size}, rank={self.rank}: reducing batch indices from {len(self.batch_indices)} to {min_size}") + self.batch_indices = self.batch_indices[0:min_size] + + def _maybe_init_distributed(self, world_size: Optional[int], rank: Optional[int]): + if world_size is not None: + assert world_size >= 1 + if rank is not None: + assert rank >= 0 + if not dist.is_available() or not dist.is_initialized(): + self.world_size = 1 if world_size is None else world_size + self.rank = 0 if rank is None else rank + return + self.world_size = dist.get_world_size() if world_size is None else world_size + self.rank = dist.get_rank() if rank is None else rank + assert self.rank < self.world_size + + + def set_epoch(self, epoch: int): + """ + Must be called at the beginning of each epoch, before initializing the DataLoader, + to re-shuffle the data. If this is not done, this sampler will give you the same batches + each time it is called. + """ + g = torch.manual_seed(self.rank + self.seed + epoch) + + sentence_lengths = (self.sentence_lengths * + (1.0 + torch.rand(*self.sentence_lengths.shape, generator=g) * self.multiplicative_random_length)) + + # This mechanism regulates the batch size so that we don't get OOM in transformers + # when the sentences are long. + sentence_lengths = (sentence_lengths + (sentence_lengths ** 2) * self.quadratic_constant) + self.length_floor + + values, indices = torch.sort(sentence_lengths) # values,indices dtypes: torch.float,torch.int64 + + # map to the original indexes into the dataset (the original sentence + # indexes), see torch.arange expression in the constructor. save as + # int32 just to save a little memory. self.indices are indexes into the + # LmDataset, just including the subset of indices that this sampler is + # responsible for (in terms of rank and world_size), and sorted by + # length with a small amount of randomization specific to the epoch. + self.indices = ((indices * self.world_size) + self.rank).to(dtype=torch.int32) + + # now `batch_ids` will be: [0, 0, 0, 0, .., 0, 1, 1, 1, ... 1, 2, ... ], + # saying which batch each element of values/indices belongs to. + batch_ids = (torch.cumsum(values.to(dtype=torch.double), dim=0) * (1.0 / self.symbols_per_batch)).to(dtype=torch.int32) + + batch_boundaries = torch.nonzero(batch_ids[1:] - batch_ids[:-1], as_tuple=True)[0] + batch_boundaries.add_(1) + self.batch_boundaries = torch.cat((torch.zeros(1, dtype=torch.int32), batch_boundaries), dim=0) + + num_batches = self.batch_boundaries.numel() - 1 + + # self.batch_indices is a permutation of [0, 1, ... num_batches - + # 1]; it determines the order in which we access the batches. It's + # necessary to randomize the order of these, to avoid returning batches + # from shortest to longest sentences. + self.batch_indices = torch.randperm(num_batches, generator=g, dtype=torch.int32).tolist() + self._sync_sizes() + + + def __len__(self): + return len(self.batch_indices) + + def __iter__(self): + """ + Iterator that yields lists of indices (i.e., integer indices into the LmDataset) + """ + for batch_idx in self.batch_indices: + batch_start = self.batch_boundaries[batch_idx].item() + batch_end = self.batch_boundaries[batch_idx + 1].item() + yield self.indices[batch_start:batch_end].tolist() + + +class CollateFn: + def __init__(self, **kwargs): + self.extra_args = kwargs + + def __call__(self, sentences: List[List[int]]): + return collate_fn(sentences, **self.extra_args) + + + + + +# train,test = dataset.load_train_test_lm_dataset('../data/lm_training_5000/lm_data.pt') +# sampler = dataset.LmBatchSampler(test, symbols_per_batch=1000, world_size=2, rank=0) +# a = iter(sampler) +# print(str(next(a))) + +# collate_fn=(lambda x:dataset.collate_fn(x, bos_sym=1, eos_sym=1, blank_sym=0, debug=True)) +# train_dl = torch.utils.data.DataLoader(test, batch_sampler=sampler, collate_fn=collate_fn) +# x = iter(train_dl) +# print(str(next(x))) diff --git a/egs/librispeech/ASR/conformer_lm/madam.py b/egs/librispeech/ASR/conformer_lm/madam.py new file mode 100644 index 000000000..b7bb77ec9 --- /dev/null +++ b/egs/librispeech/ASR/conformer_lm/madam.py @@ -0,0 +1,1256 @@ +import logging +import math +import random +import torch +from torch import nn +from torch import Tensor +from torch.optim.optimizer import Optimizer +from typing import List, Tuple + + + +# After this many warnings about infinite gradients we'll die. +inf_grad_count = 0 +inf_grad_max_count = 20 + +class Madam(Optimizer): + r"""Madam is a modification of the Adam algorithm, with various changes + intended to support certain "common-sense" ideas and solve common + pathologies that can happen particularly in transformer-type models that + have multiplication of parameters (particularly, key and query matrices)-- + these can be vulnerable to "subspace loss" where, if you have any l2 + regularization, certain subspaces in the key/query space might get + regularized towards zero. We solve this with a special formula that + changes how the l2/weight-decay is done (see compute_l2_grad()). + I'll try to write the math down at some point. This formula only + applies to tensors that have at least two dimensions; for one-dimensional + tensors we simply won't do l2 regularization. + + One more thing-- there is a special pathology that can sometimes afflict + models like LSTMs, where a particular element of a minibatch experiences + gradient blowup in the backward pass. We'd like to identify such cases and + fix it somehow, e.g. by removing or scaling down the gradient for that + particular minibatch. We can identify and somewhat fix this by seeing that the + gradient norm (computed over all the parameters in a parameter group) is + much more than on previous minibatches, and limiting it to (the preceding + average step size times some constant). + + Like most optimization algorithms, for this to work well you need to + have an appropriate learning rate schedule, either decreasing with + time, or increasing (warm-up) and then decreasing. The LR schedule may + possibly need to decrease a little more aggressively than you would with + Adam, or at least have smaller values overall than Adam, because + the smaller parameters will mean the effective (relative) learning + rate is higher. + + This is modified from PyTorch's optim/adam.py + + + Args: + params (iterable): iterable of parameters to optimize or dicts defining + parameter groups + lr (float, optional): learning rate (default: 1e-3) + betas (Tuple[float, float], optional): coefficients used for computing + running averages of gradient and its square (default: (0.9, 0.999)) + eps (float, optional): term added to the denominator to improve + numerical stability (default: 1e-8) + grad_norm_buffer_size (int, optional): Buffer size used in detecting + minibatches with unusually large gradients and scaling them down. + limit_grad_factor (float): factor by which we don't allow the + gradient to be greater than the average of previous gradients + (we'll scale the gradient down, over the whole param-group, + to enforce this). Must be greater than 1. Set to float('inf') + to disable norm clipping. + min_target_rms: A floor on the "target rms" of each Tensor, so + that Tensors that, when initialized, have less than this + rms value will have their target rms value floored to this + l2: True to enable l2 regularization + l2_period: You may set this to a value greater than one to save + computation by only periodically doing the l2 update. + We include a scaling factor in the formula so that, as far + as possible (for small learning rates) this shouldn't affect + the results. (Note: this probably isn't necessary to set, + since it turns out the update is quite fast, at least on GPU, + and the gradient clipping is actually more of a problem) + + + .. _Adam\: A Method for Stochastic Optimization: + https://arxiv.org/abs/1412.6980 + .. _Decoupled Weight Decay Regularization: + https://arxiv.org/abs/1711.05101 + .. _On the Convergence of Adam and Beyond: + https://openreview.net/forum?id=ryQu7f-RZ + + """ + + def __init__(self, params, + lr: float = 1e-3, + betas: Tuple[float, float] = (0.9, 0.999), + eps: float = 1e-8, + grad_norm_buffer_size: int = 8, + limit_grad_factor: float = 2.0, + min_target_rms: float = 0.05, + l2: bool = True, + l2_period: int = 1): + if not 0.0 <= lr: + raise ValueError("Invalid learning rate: {}".format(lr)) + if not 0.0 <= eps: + raise ValueError("Invalid epsilon value: {}".format(eps)) + if not 0.0 <= betas[0] < 1.0: + raise ValueError("Invalid beta parameter at index 0: {}".format(betas[0])) + if not 0.0 <= betas[1] < 1.0: + raise ValueError("Invalid beta parameter at index 1: {}".format(betas[1])) + if not (isinstance(grad_norm_buffer_size, int) and grad_norm_buffer_size > 1): + raise ValueError("Invalid grad_norm_buffer_size value: {}".format(grad_norm_buffer_size)) + if not limit_grad_factor > 1.0: + raise ValueError("Invalid limit_grad_factor: {}".format(limit_grad_factor)) + if not isinstance(l2, bool): + raise ValueError("Invalid l2 value: {}".format(l2)) + if not l2_period >= 1: + raise ValueError("Invalid l2_period value: {}".format(l2_period)) + defaults = dict(lr=lr, betas=betas, eps=eps, + grad_norm_buffer_size=grad_norm_buffer_size, + limit_grad_factor=limit_grad_factor, + l2=l2, l2_period=l2_period, + min_target_rms=min_target_rms) + super(Madam, self).__init__(params, defaults) + + + @torch.no_grad() + def step(self, closure=None): + """Performs a single optimization step. + + Args: + closure (callable, optional): A closure that reevaluates the model + and returns the loss. + """ + loss = None + if closure is not None: + with torch.enable_grad(): + loss = closure() + + for group in self.param_groups: + + beta1, beta2 = group['betas'] + grad_norm_buffer_size = group['grad_norm_buffer_size'] + limit_grad_factor = group['limit_grad_factor'] + min_target_rms = group['min_target_rms'] + + # The next 5 lists are part of the original Adam optimizer + params_with_grad = [] + grads = [] + exp_avgs = [] + exp_avg_sqs = [] + state_steps = [] + + # The next 3 lists are not part of the original Adam optimizer. + target_rms_values = [] # relates to weight decay. Target root-mean-square + # values of the elements of each parameter + # we are optimizing + prev_norm_stats = [] # contains Tensor with 2 elements each, the sum + # of the [sum_squared, count] of + # this parameter on previous minibatches (up to + # grad_norm_buffer_size minibatches) + cur_grad_norms = [] # and `cur_grad_norms` contains the squared l2 + # norm norm of this step's gradient for this + # parameter, as a Tensor. + + + for p in group['params']: + if p.grad is not None: + params_with_grad.append(p) + if p.grad.is_sparse: + raise RuntimeError('Adam does not support sparse gradients, please consider SparseAdam instead') + grads.append(p.grad) + + state = self.state[p] + # Lazy state initialization + if len(state) == 0: + state['step'] = 0 + # Exponential moving average of gradient values + state['exp_avg'] = torch.zeros_like(p, memory_format=torch.preserve_format) + # Exponential moving average of squared gradient values + state['exp_avg_sq'] = torch.zeros_like(p, memory_format=torch.preserve_format) + + # The things below are not part of original Adam, they are the Madam extension.. + state['target_rms'] = _get_target_rms(p, min_target_rms) + # grad_norm_buf is a rotating buffer containing (grad_norm**2, count), where + # count is 1 for grad_norms that are set and 0 for those that are not set because + # we're near step 0 or because they were infinite. + state['grad_norm_buf'] = torch.zeros(grad_norm_buffer_size, 2, device=p.device) + + exp_avgs.append(state['exp_avg']) + exp_avg_sqs.append(state['exp_avg_sq']) + + target_rms_values.append(state['target_rms']) + + cur_step = state['step'] + if limit_grad_factor != float('inf'): + grad_norm_buf = state['grad_norm_buf'] + cur_grad_norm = (p.grad ** 2).sum() # actually squared nom + prev_mean_norm = grad_norm_buf.sum(0) # prev_mean_norm is a Tensor [ tot_norm_squared, count ] + grad_norm_buf[cur_step % grad_norm_buffer_size][0] = cur_grad_norm + grad_norm_buf[cur_step % grad_norm_buffer_size][1].fill_(1.0) + prev_norm_stats.append(prev_mean_norm) + cur_grad_norms.append(cur_grad_norm) + + # update the steps for each param group update + cur_step += 1 + state['step'] = cur_step + # record the step after step update + state_steps.append(cur_step) + + if limit_grad_factor != float('inf'): + self._apply_grad_norm_clipping(group['params'], + prev_norm_stats, cur_grad_norms, grads, + limit_grad_factor, grad_norm_buffer_size) + + _madam(params_with_grad, + grads, + exp_avgs, + exp_avg_sqs, + state_steps, + target_rms_values, + beta1=beta1, + beta2=beta2, + lr=group['lr'], + eps=group['eps'], + l2=group['l2'], + l2_period=group['l2_period']) + + return loss + + + def _apply_grad_norm_clipping(self, + params_list, + prev_norm_stats: List[Tensor], + cur_grad_norms: List[Tensor], + grads: List[Tensor], + limit_grad_factor: float, + grad_norm_buffer_size: int) -> None: + """ + This function applies gradient norm clipping for this parameter group if this + minibatch has substantially larger gradients in this param group than + recent minibatches. The idea is to catch cases like where an LSTM + happens to blow up in the backward pass, or some code bug causes very + large or infinite gradients on a particular minibatch; so we scale + down any very large gradients and zero infinite ones. + + Args: + params_list: some kind of iterable or list of params in this group + prev_norm_stats: a list which, for each parameter in this group + with a grad, contains a Tensor with 2 elements each, containing + # the [sum, count] of up to `grad_norm_buffer_size` + # norms of this parameter on previous minibatches; + cur_grad_norms: a list of Tensor containing, for each parameter in this group, + the norm of this step's gradient for this parameter. + grads: List of gradients with the same order as prev_norm_stats and + cur_grad_norms + limit_grad_factor: a float >1.0 (e.g. 4.0) that dictates + how-much-larger-than-average gradients we allow before clipping. + grad_norm_buffer_size: an int that determines the rolling buffer size over which + we store gradient norms + """ + num_params = len(prev_norm_stats) + assert len(grads) == num_params + + all_prev_norm_stats, all_cur_grad_norms = _to_device('cpu', + torch.stack(prev_norm_stats), + torch.stack(cur_grad_norms)) + assert all_prev_norm_stats.shape == (num_params, 2) + assert all_cur_grad_norms.shape == (num_params,) + + # divide totals by counts (i.e. counts of iterations were we stored + # a finite grad) + all_prev_grad_norms = all_prev_norm_stats[:,0] / all_prev_norm_stats[:,1] + # prev_norm and cur_norm are floats, they are actually squared norms. + prev_norm = all_prev_grad_norms.sum().item() + cur_norm = all_cur_grad_norms.sum().item() + + if prev_norm - prev_norm != 0.0: + # There were zero counts; fix this by using the current grad norm + # for affected parameters, and recompute all_prev_grad_norms and + # prev_norm. + for i in range(num_params): + if all_prev_norm_stats[i][1] == 0.0: + # if count is 0 and cur norm is finite, use cur norm as our estimate + # of previous norms. This would only be useful if some but not + # all params were in this situation of having no previous estimates. + cur = all_cur_grad_norms[i] + if cur - cur == 0.0: # finite.. + all_prev_norm_stats[i][0] = cur + all_prev_norm_stats[i][1] = 1.0 + else: + # 0.0 is a default; likely won't matter, as if we + # get infinite `cur`, we'll abandon this minibatch. + all_prev_norm_stats[i][0] = 0.0 + all_prev_grad_norms = all_prev_norm_stats[:,0] / all_prev_norm_stats[:,1] + prev_norm = all_prev_grad_norms.sum().item() + + # Deal with infinite gradients. + if cur_norm - cur_norm != 0: # cur_norm is infinite or NaN + global inf_grad_count + logging.warning(f'Infinite gradient-norm detected (cur/prev: {cur_norm}/{prev_norm}): will ' + f'zero grad ({inf_grad_count}/{inf_grad_max_count} times until dying)') + inf_grad_count += 1 + if inf_grad_count >= inf_grad_max_count: + assert 0, "Reached max count of infinite gradient-norm stats" + # Zero all gradients in this group + for g in grads: + g[:] = 0. + # .. and zero the stored gradient norms in grad_norm_buf (so + # that infinities don't ruin our stats of previous batches) + for p in params_list: + if p.grad is not None: + state = self.state[p] + grad_norm_buf = state['grad_norm_buf'] + # cur_step is the location where we would have written the grad_norm. + # We didn't check if it was infinity before, because we didn't want to + # incur lots of GPU->CPU transfers. + cur_step = state['step'] - 1 + # Remove this 'bad' step from the buffer. + grad_norm_buf[cur_step % grad_norm_buffer_size][:] = 0.0 + else: + # cur_norm is finite. Check whether we have to clip this iteration's grad. + # we always remove infinities/NaNs from the buffer, so prev_norm should not + # be infinite or NaN. + assert prev_norm - prev_norm == 0.0 + # cur_norm and prev_norm are actually squared norms, so we need to + # square limit_grad_factor.. + limit_grad_factor2 = limit_grad_factor ** 2 + if cur_norm > prev_norm * limit_grad_factor2: + grad_factor2 = (prev_norm * limit_grad_factor2) / cur_norm + grad_factor = grad_factor2 ** 0.5 + cur_norm_f, prev_norm_f, grad_factor_f = ('%.2g' % cur_norm, '%.2g' % prev_norm, + '%.2g' % grad_factor) + logging.warning(f'Gradient norm exceeds average of last {grad_norm_buffer_size} ' + f'gradients times {limit_grad_factor}: cur/prev {cur_norm_f}/{prev_norm_f}: ' + f'scaling it by {grad_factor_f}.') + for g in grads: + g[:] *= grad_factor + # .. and scale down the stored gradient norms in grad_norm_buf, to + # avoid the bound getting too loose too quickly. + for p in params_list: + if p.grad is not None: + state = self.state[p] + grad_norm_buf = state['grad_norm_buf'] + cur_step = state['step'] - 1 + # the buffer contains squared norms, so multiply by grad_factor2 + grad_norm_buf[cur_step % grad_norm_buffer_size][0] *= grad_factor2 + + +def _to_device(device, *args): + """ + Transfers a tuple of Tensors from one device to another, using a single transfer. Must have + same dtype but may have different shapes. + E.g. + (cpu_tensor_a, cpu_tensor_b) = _to_device('cpu', gpu_tensor_a, gpu_tensor_b) + """ + if device == args[0].device: + return args + else: + arg0 = args[0] + combined_src = torch.cat([ x.reshape(-1) for x in args ]) + combined_dest = combined_src.to(device) + dests = [] + offset = 0 + for src in args: + numels = src.numel() + dests.append(combined_dest[offset:offset+numels].reshape(src.shape)) + offset += numels + return tuple(dests) + + + +def _get_target_rms(x: Tensor, min_target_rms: float) -> Tensor: + """ + Returns Tensor with one element, representing a target root-mean-square + value of elements of x, that we consider "reasonable", and will use a + as a "target rms" in our modified weight-decay formula. It returns + the maximum of the current RMS of the values of x, and `min_target_rms`, + as a Tensor on the same device as x. + """ + with torch.no_grad(): + # `norm` is the 2-norm of x currently (and this function should be + # called right after parameter initialization) + rms = ((x ** 2).sum() / x.numel()).sqrt() + largest_dim = max(list(x.shape)) + numel = x.numel() + if min_target_rms > 0.0: + rms = rms.clamp(min=min_target_rms) + if x.ndim > 1 and __name__ == '__main__': # will only be used for x.ndim > 1. + print("Target rms = ", rms) # Print this in testing only. + return rms + + +def _madam(params: List[Tensor], + grads: List[Tensor], + exp_avgs: List[Tensor], + exp_avg_sqs: List[Tensor], + state_steps: List[int], + target_rms_values: List[Tensor], + *, + beta1: float, + beta2: float, + lr: float, + eps: float, + l2: bool, + l2_period: int): + r"""This is a modification of adam() from torch's optim/_functional.py. + + It has been modified to: + (i) remove the amsgrad option; this shouldn't be as necessary due to + the adaptive gradient norm clipping we have added + (ii) add our special formula for l2 regularization. This doesn't have + any tunable parameters, other than the target standard deviation + of the elements of the tensor (which is passed in as target_rms). + Args: + params: list of Tensor, containing the parameters to be optimized + grads: list of Tensor, containing the gradients corresponding to + each of the params (grads[i] should correspond to params[i].grad, + although it may have undergone gradient clipping). + exp_avgs: list of Tensor, containing tensors with the same dimensions + as params and grads, that contain the moving-averages of + `grads`. + exp_avg_sqs: list of Tensor, containing tensors with the same dimensions + as params and grads, that contain the moving-averages of + `grads ** 2`. + state_steps: list of int, containing the step for each parameter (step >= 1) + target_rms_values: list of Tensor with one element each, containing the + target root-mean-square values of each parameter tensor in `params` + l2: a bool, where if true we will activate the l2 regularization + formula. + l2_period: an integer that determines how often (i.e. every how many + minibatches) we apply the l2 update. We include a scaling factor + so that as far as possible the result will not be too sensitive + to the value of this. + + beta1: decay factor for gradients, e.g. 0.9 + beta2: decay factor for gradients squared, e.g. 0.999 + lr: learning rate, e.g. 0.0001 + eps: a small constant used to prevent division by zero, e.g. 1.0e-8 + + See :class:`~torch.optim.Adam` for details. + """ + assert len(params) == len(grads) == len(state_steps) == len(exp_avgs) == len(exp_avg_sqs) == len(target_rms_values) + + for i, param in enumerate(params): + + grad = grads[i] + + exp_avg = exp_avgs[i] + exp_avg_sq = exp_avg_sqs[i] + step = state_steps[i] + target_rms = target_rms_values[i] + + bias_correction1 = 1 - beta1 ** step + bias_correction2 = 1 - beta2 ** step + + do_l2 = param.ndim > 1 and l2 and step % l2_period == 0 + + if do_l2: + # This represents just the "noise term" of the gradient, i.e. the grad minus the + # running mean. We'll later divide by denom. + cur_grad_noise = (grad - exp_avg) + + # Decay the first and second moment running average coefficient + exp_avg.mul_(beta1).add_(grad, alpha=1 - beta1) + exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value=1 - beta2) + + denom = (exp_avg_sq.sqrt() / math.sqrt(bias_correction2)).add_(eps) + + step_size = lr / bias_correction1 + + if not do_l2: + param.addcdiv_(exp_avg, denom, value=-step_size) + else: + # We can treat "pseudo_grad" as if it were a gradient (even though it's + # actually a gradient times a per-element learning rate). The analysis + # that we used to figure out what the l2 should be did not use the fact + # that the gradients were actually gradients, it simply analyzed it as a + # quantity that can be treated as close to zero-mean and with a certain + # structure of variance, and added to the param with the formula: + # + # param -= step_size * grad + # + # The original analysis assumed the gradients were independent from frame + # to frame; in fact these are not, but the important difference can be captured + # in a scalar `grad_scale` that expresses the scale of pseudo_grad relative + # to the independent gradients that we are effectively adding on each frame + # (but with a delay). + + pseudo_grad = exp_avg / denom + cur_pseudo_grad = cur_grad_noise / denom + + # grad_scale expresses the expected size of cur_pseudo_grad relative to the + # original grads if we had not done the moving-average; it is the sqrt of + # the sum of the squares of coefficients of previous gradients: + # c_n = (1-beta1) beta1^n, for + # n = 0, 1, .. + # .. plus one which is the sumsq of the coefficient of 'grad' itself in + # (grad - exp_avg). + # It is relevant that the sum of the coefficients (i.e. not squared) is 1; + # if this were not so we'd have to incorporate that into the formula for l2. + grad_scale = (((1 - beta1)**2) / (1 - beta1**2) + 1) ** 0.5 + + with torch.no_grad(): + l2_grad = _compute_l2_grad(param, cur_pseudo_grad, target_rms, + rho=step_size, grad_scale=grad_scale, + period_scale=l2_period, + eps=eps, safe=True) + + # TODO: could alternate computing l2 on only, say, odd frames, and scale it + # up by 2, to save time. + param.add_(pseudo_grad + l2_grad, alpha=-step_size) + + + +def _view_as_matrix(x: Tensor, dim: int) -> Tensor: + """ + Returns a Tensor of shape (n, x.shape[dim]), where n is the product + of the sizes of the other dimensions of x. This may involve a copy, + if x cannot be reshaped in this way. + """ + ndim = x.ndim + assert ndim > 1 and dim >= 0 and dim < ndim + # Move the dim to the last position in x.. + if dim != ndim - 1: + x = x.transpose(dim, ndim - 1) + return x.reshape(-1, x.shape[-1]) + + +def _outer_product(x: Tensor, dim: int) -> Tensor: + """ + Returns a Tensor of shape (x.shape[dim], x.shape[dim]) formed by + summing the outer products of all the vectors in x of size + `x.shape[dim]`, that we get by indexing x with all tuples of dimensions + on other axes. E.g. if x is a matrix and dim == 0, this would + be torch.matmul(x, x.transpose(0, 1)). + + Note: x must have at least 2 dimensions, x.ndim >= 2. + """ + x = _view_as_matrix(x, dim) + return torch.matmul(x.transpose(0, 1), x) + +def _multiply_on_dim(x: Tensor, m: Tensor, dim: int) -> Tensor: + """ + Multiplies x by the matrix m which must be of shape: + (x.shape[dim], n)), with `dim` as the dimension/axis on + x to be multiplied. + + Caution: result may not have the same layout/strides as x, + although it will have the same shape. + + Args: + x: Tensor to be multiplied; must have ndim >= 2 + m: Symmetric matrix to multiply x by; must have + m.shape == (x.shape[dim], x.shape[dim]) + dim: Dimension of x to multiply on, with 0 <= dim < x.ndim + Return: + The matrix product, of the same shape as + x, except with the size on dimension `dim` being n. + """ + ndim = x.ndim + if dim != ndim - 1: + x = x.transpose(dim, ndim - 1) + ans = torch.matmul(x, m) + if dim != ndim - 1: + # Swap the dimensions back to what they were originally. + ans = ans.transpose(dim, ndim - 1) + return ans + + +def _multiply_product_combined(l2: Tensor, grad: Tensor, dim: int, + need_grad_sumsq: bool): + """ + This function is an optimized version of the following code: + outer_prod = _outer_product(grad, dim) + l2 = _multiply_on_dim(l2, outer_prod, dim) + if dim == 0: # could choose any dim for this + grad_sumsq = torch.trace(outer_prod) + Args: + l2: The l2 matrix which starts out as the parameter tensor x, must have >= 2 diims + grad: The gradient tensor (or a gradient-like quantity); must + have same shape as l2. + dim: The dimension of l2 and grad that we want this to + act on, with 0 <= dim < l2.ndim. We multiply l2, on + this dim, by a symmetric quantity of shape + (l2.shape[dim], l2.shape[dim]), that is formed + by a product and sum on grad (this is a matrix + product, if there are 2 axes). + Returns: + Returns (l2, grad_sumsq), where l2 is the result of + multiplying l2 by the product mentioned above, and + grad_sumsq is either None, or a Tensor representing + the sum-of-squares of `grad`; for at least one + dim with 0 <= dim < l2.ndim, we guarantee to + return such a Tensor. + """ + grad = _view_as_matrix(grad, dim) + if grad.shape[1] <= grad.shape[0]: + # Minimize the size of the intermediate product, which will probably well reflect + # the compute time since memory access can be limiting on CUDA.a + grad_product = torch.matmul(grad.transpose(0, 1), grad) + l2 = _multiply_on_dim(l2, grad_product, dim) + if need_grad_sumsq: + grad_sumsq = torch.trace(grad_product) + else: + grad_sumsq = None + return (l2, grad_sumsq) + else: + l2 = _multiply_on_dim(l2, grad.transpose(0, 1), dim) + l2 = _multiply_on_dim(l2, grad, dim) + # This branch does not compute grad_sumsq, but we're bound to + # take the other branch on at least one occasion. + return (l2, None) + + + +def _compute_l2_grad(x: Tensor, grad: Tensor, target_stddev: float, rho: float, + grad_scale: float = 1.0, period_scale: int = 1, + eps: float = 1.0e-08, + safe: bool = True) -> Tensor: + """ + Returns the l2 gradient of x, which will be added to 'grad'. + This is a more principled replacement for the typical l2 regularization + formula where we do: + grad += weight_decay * x. + (Note: this must only be called if x.ndim >= 2). + + For x with 2 axes, we instead do this: + + grad += (rho / (2*target_stddev**2)) * (grad grad^T) x (grad^T grad) / trace(grad^T grad), + + where the implicit multiplication above refers to matrix multiplication; note, x means + the variable x. We'll have to write the justification of this, which is a little + complicated, separately; it has to do with using exactly the amount of l2 in each + subspace of each dimension of x, to to cancel out the gradient noise. + + Args: + x: parameter to be updated. MUST HAVE x.ndim >= 2. + grad: Gradient for x on this iteration (or at least, something that + is treated like a gradient in the update formula) +target_stddev: The target standard deviation (uncentered), of elements of x. + This is our estimate of what standard deviation these elements would + have in a well-trained model; it is set by some kind of heuristic. + rho: The learning rate we are going to use, as in: x -= (grad + l2) * rho. + grad_scale: A scale whereby the caller asserts that `grad` is some + quantity that is distributed like the real + gradient times `grad_scale` (this is useful when the provided `grad` + is really a moving average gradient). Because the l2 term's magnitude + is proportional to the gradient squared, we need to divide it by the + square of grad_scale, so this function uses 1/grad_scale^2 as a scaling + factor. +period_scale: An integer scale that we use to compensate for the fact that this + weight decay is only applied periodically, once every + `period_scale` minibatches. Accordingly, we make the l2 term + that many times larger. + eps: A small constant used to avoid division by zero + safe: If true, use a safe version of the formula that checks for + 'overshoot' of l2 regularization and fixes the issue (might + be an issue for models that are getting unstable or have high + learning rate) + + + Returns: + Returns l2 pseudo-gradient (term to be added to `grad`). + """ + assert x.shape == grad.shape + assert x.ndim >= 2 + + l2 = x + grad_sumsq = None + num_ignored_dims = 0 # for an optimization for when size=1 on some dim. + for dim in range(x.ndim): + # The code below is an optimization of the following few lines, + # which were perhaps easier to understand: + # outer_prod = _outer_product(grad, dim) + # l2 = _multiply_on_dim(l2, outer_prod, dim) + # if dim == 0: # could choose any dim for this + # grad_sumsq = torch.trace(outer_prod) + if x.shape[dim] <= 1: + num_ignored_dims += 1 + continue + (l2, maybe_grad_sumsq) = _multiply_product_combined(l2, grad, dim, + grad_sumsq is None) + if maybe_grad_sumsq is not None: + grad_sumsq = maybe_grad_sumsq + if grad_sumsq is None: + # We shouldn't reach here, except if at some point we start calling this + # code for tensors with ndim <= 1, or with numel() == 1. + grad_sumsq = (grad ** 2).sum() + + # l2 is the amount of l2, we'll subtract this from x, as in: + # x -= rho * (grad + l2). + + factor = rho * period_scale / (2.0 * (target_stddev * grad_scale)**2) + l2 = l2 * (factor / (grad_sumsq ** (x.ndim - 1 - num_ignored_dims) + eps)) + + if safe and rho > 0: + #x2_sum = (x ** 2).sum() + l2_sum = (l2 ** 2).sum() * (rho * rho) + cross_sum = (x * l2).sum() * rho + alpha = cross_sum / (l2_sum + eps) + # We want to minimize the sum-of-squares of (x - alpha * rho * l2), where alpha + # is a constant in [0,1] that we are about to estimate, intended to prevent + # instability by scaling down our weight decay formula. Right now (and treating + # things as if they were scalars for brevity): + # x2_sum = x * x + # l2_sum = rho * rho * l2 * l2 + # cross_sum = x * rho * l2 + # We want to minimize the sum-sq of (x - alpha * rho * l2), + # i.e. we want to choose alpha to minimize: + # x2_sum - 2 * alpha * cross_sum + alpha^2 * l2_sum + # d/dalpha of this, is: + # -2*cross_sum + 2 * alpha * l2_sum + # and setting this to zero and solving for alpha, we have: + # alpha = cross_sum / l2_sum. + # If it turns out that alpha >= 1, then we just use alpha=1 + # (the original formula), as there is no problem with + # instability/overshoot. + l2.mul_(alpha.clamp(max=1.0)) + if random.random() < 0.001 and alpha < 1.0: + logging.info(f'madam optimizer: alpha={alpha}, shape={tuple(x.shape)}') + return l2 + + + +class Moam(object): + """ + Implements Moam optimizer. This is a modified version of the Noam optimizer + which was proposed in "Attention Is All You Need", https://arxiv.org/pdf/1706.03762.pdf, + but changed to use Madam (see above) instead of Adam as the base optimizer. + Modified from https://github.com/espnet/espnet/blob/master/espnet/nets/pytorch_backend/transformer/optimizer.py + + Caution: you probably want to set 'factor' to a smaller value than you would typically + use for a corresponding Noam optimizer, because Moam does a kind of l2 regularization which + keeps the parameters fairly small, so the relative changes in model parameters + will be larger than Noam, for any given learning rate. + + Args: + params (iterable): iterable of parameters to optimize or dicts defining parameter groups + model_size: attention dimension of the transformer model + factor: learning rate factor, that multiplies the output of the + formula based on model size + warm_step: number of warmup steps before the learning rate starts to decrease + (it increases until this point). + min_target_rms: this is a parameter of the Madam optimizer; it represents a floor + on the "target root-mean-square value" that is used when the initialization + of a tensor is zero or below this value. It may be worth optimizing. + Don't worry about tensors with fewer than 2 dimensions when setting this, + these are not subject to our l2 formula. + limit_grad_factor: you can set this to a finite value, e.g. 2.0, to activate + a mechanism that limits the norms of larger-than-usual gradients. + This seems to cause a slowdown, likely due to GPU->CPU transfers. + l2_period: mechanism to improve the optimization speed, by only applying the l2 + regularization (which is a complicated formula) every this-many + minibatches. E.g. can set it to 2 or 4. + """ + + def __init__(self, params, model_size: int = 256, + factor: float = 2.0, warm_step: int = 25000, + min_target_rms: float = 0.05, + limit_grad_factor: float = float('inf'), + l2_period: int = 1) -> None: + """Construct an Noam object.""" + self.optimizer = Madam(params, lr=0, betas=(0.9, 0.98), eps=1e-9, + min_target_rms=min_target_rms, + limit_grad_factor=limit_grad_factor, + l2_period=l2_period) + self._step = 0 + self.warmup = warm_step + self.factor = factor + self.model_size = model_size + self._rate = 0 + + @property + def param_groups(self): + """Return param_groups.""" + return self.optimizer.param_groups + + def step(self): + """Update parameters and rate.""" + self._step += 1 + rate = self.rate() + for p in self.optimizer.param_groups: + p["lr"] = rate + self._rate = rate + self.optimizer.step() + + def rate(self, step=None): + """Implement `lrate` above.""" + if step is None: + step = self._step + return ( + self.factor + * self.model_size ** (-0.5) + * min(step ** (-0.5), step * self.warmup ** (-1.5)) + ) + + def zero_grad(self): + """Reset gradient.""" + self.optimizer.zero_grad() + + def state_dict(self): + """Return state_dict.""" + return { + "_step": self._step, + "warmup": self.warmup, + "factor": self.factor, + "model_size": self.model_size, + "_rate": self._rate, + "optimizer": self.optimizer.state_dict(), + } + + def load_state_dict(self, state_dict): + """Load state_dict.""" + for key, value in state_dict.items(): + if key == "optimizer": + self.optimizer.load_state_dict(state_dict["optimizer"]) + else: + setattr(self, key, value) + + +class Foam(object): + """ + + This code was modified from https://github.com/espnet/espnet/blob/master/espnet/nets/pytorch_backend/transformer/optimizer.py + + + Args: + params (iterable): iterable of parameters to optimize or dicts defining parameter groups + + warm_step: number of warmup steps before the learning rate starts to decrease + (it increases until this point). + max_lrate: The learning rate at its maximum, on step `warm_step` + knee_factor: The multiple of `max_lrate` after which the learning rate will + start to decrease more like 1/x. It increases linearly from 0 to + `warm_step`, then decreases approximately as 1/sqrt(x) from + `warm_step` to `warm_step * knee_factor`, then decreases + approximately as 1/x from `warm_step * knee_factor` onwards. + + min_target_rms: this is a parameter of the Madam optimizer; it represents a floor + on the "target root-mean-square value" that is used when the initialization + of a tensor is zero or below this value. It may be worth optimizing. + Don't worry about tensors with fewer than 2 dimensions when setting this, + these are not subject to our l2 formula. + limit_grad_factor: Another parameter of Madam, you can set this to a finite + value, e.g. 2.0, to activate a mechanism that limits the norms of + larger-than-usual gradients. This seems to cause a slowdown, likely due + to GPU->CPU transfers, and it is disabled by setting it to infinity. + l2_period: mechanism to improve the optimization speed, by only applying the l2 + regularization (which is a complicated formula) every this-many + minibatches. E.g. can set it to 2 or 4. + """ + + def __init__(self, + params, + max_lrate: float = 5.0e-04, + warm_step: int = 25000, + knee_factor: float = 5.0, + min_target_rms: float = 0.05, + limit_grad_factor: float = float('inf'), + l2_period: int = 1) -> None: + """Construct an Noam object.""" + self.optimizer = Madam(params, lr=0, betas=(0.9, 0.98), eps=1e-9, + min_target_rms=min_target_rms, + limit_grad_factor=limit_grad_factor, + l2_period=l2_period) + self._step = 0 + + self._max_lrate = max_lrate + self._warm_step = warm_step + self._knee_factor = knee_factor + self._rate = 0 + + + @property + def param_groups(self): + """Return param_groups.""" + return self.optimizer.param_groups + + def step(self): + """Update parameters and rate.""" + self._step += 1 + rate = self.rate() + for p in self.optimizer.param_groups: + p["lr"] = rate + self._rate = rate + self.optimizer.step() + + + def rate(self, step=None): + """ + Suppose the step of optimization is 's', i.e. with s = 0, 1, 2... + We define 't = s / warm_step', i.e. t is the step s, normalized so that it + is 1.0 at warm_step. Our formula for the learning rate as a function of + t is: + rate = max_lrate * (t <= 1.0 ? t : + sqrt((2 + alpha) / (1 + t + alpha t^2))) + where alpha is chosen so that the 't' and 'alpha t^2' terms are identical + at t == knee_factor (this means alpha = 1.0/knee_factor). So the + learning rate increases linearly from t=00 to t=1, and decreases + after that. You can see + that sqrt((2 + alpha) / (1 + t + alpha t^2))) is 1.0 when t == 1, + which is why the line and the curve meet at that point. + + On the denominator of that ratio, the "t" term makes it decrease a + bit like 1/sqrt(t) in 1 <= t <= warm_step; the "alpha t^2" term + makes it decrease a bit like 1/t for t > warm_step; and the "1" + term makes it decrease a bit slower than 1/sqrt(t) when t is quite + close to 1.0 (so we linger a little, near the maximum learning rate). + + This learning rate schedule ultimately decreases more aggressively + than Noam, i.e. as 1 / t instead of 1 / sqrt(t). The reason we + feel this will work better in conjunction with Madam, is that Madam + keeps the norms of the parameters approximately constant throughout + training; whereas with Noam, if there is no weight decay, these + norms tend to increase as training progresses (although rather + unevenly across different parameter tensors). + As the norms of the parameters increase, the relative changes + in parameters get smaller (the step sizes don't change because + Adam normalizes the gradient magnitudes; they'd get smaller otherwise). + So Noam doesn't have to decrease the learning rate too aggressively + because even with a fixed learning rate, the effective learning rate + would be decreasing (again, this only applies without weight decay). + """ + if step is None: + step = self._step + t = step / self._warm_step # floating point division.. t is the normalized step. + alpha = 1.0 / self._knee_factor + return self._max_lrate * (t if t <= 1.0 else + ((2 + alpha) / (1 + t + alpha * t * t)) ** 0.5) + + def zero_grad(self): + """Reset gradient.""" + self.optimizer.zero_grad() + + def state_dict(self): + """Return state_dict.""" + return { + "_step": self._step, + } + + def load_state_dict(self, state_dict): + """Load state_dict. This is compatible with reading a Moam state_dict""" + for key, value in state_dict.items(): + if key == "optimizer": + self.optimizer.load_state_dict(state_dict["optimizer"]) + elif key == '_step': + self._step = value + + + +class Gloam(object): + """ + Implements Gloam optimizer. This is a modified version of the Noam optimizer + which was proposed in "Attention Is All You Need", https://arxiv.org/pdf/1706.03762.pdf, + but changed to use Madam (see above) instead of Adam as the base optimizer, and then + to change the learning rate schedule and how it is specified. We have + a warm-up stage, but after it gets to `max_lrate` it stays constant for the + rest of the 1st epoch, and after that, only changes on epoch boundaries. + + CAUTION: you have to call set_epoch() every epoch, to set the epoch. If you don't do this, + this won't work! + + + This code was modified from https://github.com/espnet/espnet/blob/master/espnet/nets/pytorch_backend/transformer/optimizer.py + + + Args: + params (iterable): iterable of parameters to optimize or dicts defining parameter groups + + warm_step: number of warmup steps before the learning rate starts to decrease + (it increases until this point). + max_lrate: The learning rate at its maximum, on step `warm_step` + first_decrease_epoch: The epoch number on which to start decreasing the + learning rate. + decay_per_epoch: + min_target_rms: this is a parameter of the Madam optimizer; it represents a floor + on the "target root-mean-square value" that is used when the initialization + of a tensor is zero or below this value. It may be worth optimizing. + Don't worry about tensors with fewer than 2 dimensions when setting this, + these are not subject to our l2 formula. + limit_grad_factor: Another parameter of Madam, you can set this to a finite + value, e.g. 2.0, to activate a mechanism that limits the norms of + larger-than-usual gradients. This seems to cause a slowdown, likely due + to GPU->CPU transfers, and it is disabled by setting it to infinity. + l2_period: mechanism to improve the optimization speed, by only applying the l2 + regularization (which is a complicated formula) every this-many + minibatches. E.g. can set it to 2 or 4. + """ + + def __init__(self, + params, + max_lrate: float = 5.0e-04, + warm_step: int = 25000, + first_decrease_epoch: int = 1, + decay_per_epoch: float = 0.85, + min_target_rms: float = 0.05, + limit_grad_factor: float = float('inf'), + l2_period: int = 1) -> None: + """Construct an Noam object.""" + self.optimizer = Madam(params, lr=0, betas=(0.9, 0.98), eps=1e-9, + min_target_rms=min_target_rms, + limit_grad_factor=limit_grad_factor, + l2_period=l2_period) + self._step = 0 + + self._max_lrate = max_lrate + self._warm_step = warm_step + self._first_decrease_epoch = first_decrease_epoch + self._decay_per_epoch = decay_per_epoch + self._rate = 0 + self._epoch = 0 + + + @property + def param_groups(self): + """Return param_groups.""" + return self.optimizer.param_groups + + def set_epoch(self, epoch: int): + self._epoch = epoch + + def step(self): + """Update parameters and rate.""" + self._step += 1 + rate = self.rate() + for p in self.optimizer.param_groups: + p["lr"] = rate + self._rate = rate + self.optimizer.step() + + + def rate(self, step=None): + """ + Suppose the step of optimization is 's', i.e. with s = 0, 1, 2... + We define 't = s / warm_step', i.e. t is the step s, normalized so that it + is 1.0 at warm_step. Our formula for the learning rate as a function of + t is: + base_rate = max_lrate * (t <= 1.0 ? t : t ** -0.5) + epoch_rate = [starts at 1.0 but from first_decrease_epoch, start decreasing it + by a factor of decay_per_epoch] + rate = base_rate * epoch_rate + + """ + if step is None: + step = self._step + t = step / self._warm_step # floating point division.. t is the normalized step. + base_rate = self._max_lrate * (t if t <= 1.0 else t ** -0.5) + epoch_rate = self._decay_per_epoch ** max(0, self._epoch + 1 - self._first_decrease_epoch) + return base_rate * epoch_rate + + + def zero_grad(self): + """Reset gradient.""" + self.optimizer.zero_grad() + + def state_dict(self): + """Return state_dict.""" + return { + "_step": self._step, + "_epoch": self._epoch, + } + + def load_state_dict(self, state_dict): + """Load state_dict. This is compatible with reading a Moam state_dict""" + for key, value in state_dict.items(): + if key == "optimizer": + self.optimizer.load_state_dict(state_dict["optimizer"]) + elif key == '_step': + self._step = value + elif key == '_epoch': + self._epoch = value + + + +class TestModel(torch.nn.Module): + """Class for testing the Madam optimizer""" + def __init__(self): + super(TestModel, self).__init__() + self.first_layers = torch.nn.Sequential( + torch.nn.Linear(100, 200), + torch.nn.ReLU(), + torch.nn.Linear(200, 300), + torch.nn.ReLU()) + self.conv1 = torch.nn.Conv1d(in_channels=300, out_channels=200, + kernel_size=1) + self.relu = torch.nn.ReLU() + self.conv2 = torch.nn.Conv1d(in_channels=200, out_channels=250, + kernel_size=3) + + + def forward(self, x): + # from (B, T, 100) to (B, T, 200) + x = self.first_layers(x) + # B, T, C -> B, C, T + x = x.transpose(1, 2) + x = self.conv2(self.relu(self.conv1(x))) + # B, C, T -> B, T, C + x = x.transpose(1, 2) + return x + +def test_madam(): + print("Testing Madam optimizer") + global inf_grad_max_count + inf_grad_max_count = 200 + if torch.cuda.is_available(): + devices_and_l2 = [(torch.device('cuda'), True), + (torch.device('cuda'), False), + (torch.device('cpu'), True), + (torch.device('cpu'), False)] + else: + devices_and_l2 = [(torch.device('cpu'), True), + (torch.device('cpu'), False)] + + + for (device, l2) in devices_and_l2: + model = TestModel().to(device) + # min_target_rms=0.01 is for testing, so the target equals the initial RMS + # and we can more easily tell whether our update has the desired effect. + # I also tested this with betas=(0.1, 0.98), to check that the effect of + # `grad_scale` was correct (it only makes much difference for small beta). + optimizer = Madam(model.parameters(), lr=0.0005, betas=(0.9, 0.98), + l2=l2, min_target_rms=0.01, l2_period=1) + #optimizer = torch.optim.Adam(model.parameters()) + + def get_elems_rms(x: Tensor) -> Tensor: + return ((x ** 2).sum() / x.numel()).sqrt().item() + + for i in range(1000): + if i % 100 == 0: + rms_values = (get_elems_rms(model.first_layers[0].weight), + get_elems_rms(model.first_layers[2].weight), + get_elems_rms(model.conv1.weight), + get_elems_rms(model.conv2.weight)) + print(f"Iter {i}, l2={l2}, device={device}: stddevs = {rms_values} ") + B = 4 + T = 20 + x = torch.randn(B, T, 100).to(device) + y = model(x) + yderiv = torch.randn_like(y) + if i % 190 <= 3 and i > 0: + yderiv *= 100.0 + if i % 550 == 0 and i > 0: + yderiv *= float('inf') + + y.backward(gradient=yderiv) + optimizer.step() + model.zero_grad() + print("") + +def test_moam(): + print("Testing Moam optimizer") + model = TestModel() + # min_target_rms=0.01 is for testing, so the target equals the initial RMS + # and we can more easily tell whether our update has the desired effect. + optimizer = Moam(model.parameters(), factor=1.0, warm_step=300, + min_target_rms=0.01) + + + def get_elems_rms(x: Tensor) -> Tensor: + return ((x ** 2).sum() / x.numel()).sqrt().item() + + for i in range(1000): + if i % 100 == 0: + rms_values = (get_elems_rms(model.first_layers[0].weight), + get_elems_rms(model.first_layers[2].weight), + get_elems_rms(model.conv1.weight), + get_elems_rms(model.conv2.weight)) + print(f"Iter {i} (Moam): stddevs = {rms_values} ") + B = 4 + T = 20 + x = torch.randn(B, T, 100) + y = model(x) + yderiv = torch.randn_like(y) + if i % 190 <= 3 and i > 0: + yderiv *= 100.0 + if i % 550 == 0 and i > 0: + yderiv *= float('inf') + + y.backward(gradient=yderiv) + optimizer.step() + model.zero_grad() + print("") + + +def test_foam(): + print("Testing Foam optimizer") + model = TestModel() + # min_target_rms=0.01 is for testing, so the target equals the initial RMS + # and we can more easily tell whether our update has the desired effect. + optimizer = Foam(model.parameters(), + max_lrate=1.0e-03, warm_step=300, + min_target_rms=0.01, + limit_grad_factor=4.0) + + + def get_elems_rms(x: Tensor) -> Tensor: + return ((x ** 2).sum() / x.numel()).sqrt().item() + + for i in range(1000): + if i % 100 == 0: + rms_values = (get_elems_rms(model.first_layers[0].weight), + get_elems_rms(model.first_layers[2].weight), + get_elems_rms(model.conv1.weight), + get_elems_rms(model.conv2.weight)) + print(f"Iter {i} (Foam): stddevs = {rms_values} ") + B = 4 + T = 20 + x = torch.randn(B, T, 100) + y = model(x) + yderiv = torch.randn_like(y) + if i % 190 <= 3 and i > 0: + yderiv *= 100.0 + if i % 550 == 0 and i > 0: + yderiv *= float('inf') + + y.backward(gradient=yderiv) + optimizer.step() + model.zero_grad() + print("") + + state_dict = optimizer.state_dict() + step = optimizer._step + optimizer._step = 0 + optimizer.load_state_dict(state_dict) + assert optimizer._step == step + + +def test_to_device(): + if not torch.cuda.is_available(): + return + a_gpu = torch.ones(1,2,3,4, device='cuda') + b_gpu = torch.zeros(3,8, device='cuda') + (a_cpu, b_cpu) = _to_device('cpu', a_gpu, b_gpu) + print("a_cpu,b_cpu = ", a_cpu, b_cpu) + (a_gpu2, b_gpu2) = _to_device('cuda', a_cpu, b_cpu) + print("a_gpu2,b_gpu2 = ", a_gpu2, b_gpu2) + +# Caution: this testing code is not very automated, it reqires looking at the output to +# make sure it looks right. The main thing is that with l2=True, the printed stddevs stay close +# to the "Target rms" values, which are printed out; while with l2=False, the stddevs +# increase to significantly higher than that. +# +# The test of the Moam optimizer is mainly to make sure it runs; the scale of the +# gradients, and the learning rate, are such that one of the rms's stays quite a bit +# above the target value, i.e. (0.047, 0.044, 0.047), vs. targets of +# (0.057, 0.04, 0.019), I think this has to do with the alpha<1 stability mechanism being +# activated, the l2 does have an effect, as I verified by changing the code to set +# l2=False. +def main(): + # Set number of threads to 1, or Torch can do weird things that make it extremely slow. + torch.set_num_threads(1) + torch.set_num_interop_threads(1) + #test_to_device() + random.seed(0) + torch.random.manual_seed(0) + test_foam() + test_moam() + test_madam() + + + +if __name__ == '__main__': + main() diff --git a/egs/librispeech/ASR/conformer_lm/test_conformer.py b/egs/librispeech/ASR/conformer_lm/test_conformer.py new file mode 100644 index 000000000..45b50a6ea --- /dev/null +++ b/egs/librispeech/ASR/conformer_lm/test_conformer.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python3 +# run with: +# python3 -m pytest test_conformer.py + +import torch +import dataset # from . +from conformer import ( + RelPosTransformerDecoder, + RelPosTransformerDecoderLayer, + MaskedLmConformer, + MaskedLmConformerEncoder, + MaskedLmConformerEncoderLayer, + RelPositionMultiheadAttention, + RelPositionalEncoding, + generate_square_subsequent_mask, +) + +from torch.nn.utils.rnn import pad_sequence + + +def test_rel_position_multihead_attention(): + # Also tests RelPositionalEncoding + embed_dim = 256 + num_heads = 4 + T = 25 + N = 4 + C = 256 + pos_emb_module = RelPositionalEncoding(C, dropout_rate=0.0) + rel_pos_multihead_attn = RelPositionMultiheadAttention(embed_dim, num_heads) + + x = torch.randn(N, T, C) + #pos_emb = torch.randn(1, 2*T-1, C) + x, pos_emb = pos_emb_module(x) + x = x.transpose(0, 1) # (T, N, C) + attn_output, attn_output_weights = rel_pos_multihead_attn(x, x, x, pos_emb) + + +def test_masked_lm_conformer_encoder_layer(): + # Also tests RelPositionalEncoding + embed_dim = 256 + num_heads = 4 + T = 25 + N = 4 + C = 256 + pos_emb_module = RelPositionalEncoding(C, dropout_rate=0.0) + encoder_layer = MaskedLmConformerEncoderLayer(embed_dim, num_heads) + + + x = torch.randn(N, T, C) + x, pos_emb = pos_emb_module(x) + x = x.transpose(0, 1) # (T, N, C) + key_padding_mask = (torch.randn(N, T) > 0.0) # (N, T) + y = encoder_layer(x, pos_emb, key_padding_mask=key_padding_mask) + + +def test_masked_lm_conformer_encoder(): + # Also tests RelPositionalEncoding + embed_dim = 256 + num_heads = 4 + T = 25 + N = 4 + C = 256 + pos_emb_module = RelPositionalEncoding(C, dropout_rate=0.0) + encoder_layer = MaskedLmConformerEncoderLayer(embed_dim, num_heads) + norm = torch.nn.LayerNorm(embed_dim) + encoder = MaskedLmConformerEncoder(encoder_layer, num_layers=4, + norm=norm) + + + x = torch.randn(N, T, C) + x, pos_emb = pos_emb_module(x) + x = x.transpose(0, 1) # (T, N, C) + key_padding_mask = (torch.randn(N, T) > 0.0) # (N, T) + y = encoder(x, pos_emb, key_padding_mask=key_padding_mask) + + +def test_transformer_decoder_layer_rel_pos(): + embed_dim = 256 + num_heads = 4 + T = 25 + N = 4 + C = 256 + pos_emb_module = RelPositionalEncoding(C, dropout_rate=0.0) + decoder_layer = RelPosTransformerDecoderLayer(embed_dim, num_heads) + + + x = torch.randn(N, T, C) + x, pos_emb = pos_emb_module(x) + x = x.transpose(0, 1) # (T, N, C) + key_padding_mask = (torch.randn(N, T) > 0.0) # (N, T) + attn_mask = generate_square_subsequent_mask(T) + memory = torch.randn(T, N, C) + y = decoder_layer(x, pos_emb, memory, attn_mask=attn_mask, key_padding_mask=key_padding_mask) + + + +def test_transformer_decoder_rel_pos(): + embed_dim = 256 + num_heads = 4 + T = 25 + N = 4 + C = 256 + pos_emb_module = RelPositionalEncoding(C, dropout_rate=0.0) + decoder_layer = RelPosTransformerDecoderLayer(embed_dim, num_heads) + decoder_norm = torch.nn.LayerNorm(embed_dim) + decoder = RelPosTransformerDecoder(decoder_layer, num_layers=6, norm=decoder_norm) + + x = torch.randn(N, T, C) + x, pos_emb = pos_emb_module(x) + x = x.transpose(0, 1) # (T, N, C) + key_padding_mask = (torch.randn(N, T) > 0.0) # (N, T) + attn_mask = generate_square_subsequent_mask(T) + memory = torch.randn(T, N, C) + y = decoder(x, pos_emb, memory, attn_mask=attn_mask, key_padding_mask=key_padding_mask) + + +def test_masked_lm_conformer(): + + num_classes = 87 + d_model = 256 + + model = MaskedLmConformer(num_classes,d_model) + + + N = 31 + + + (masked_src_symbols, src_symbols, + tgt_symbols, src_key_padding_mask, + tgt_weights) = dataset.collate_fn(sentences=[ list(range(10, 20)), list(range(30, 45)), list(range(50,68))], bos_sym=1, eos_sym=2, + blank_sym=0) + + # test forward() of MaskedLmConformer + memory, pos_emb = model(masked_src_symbols, src_key_padding_mask) + nll = model.decoder_nll(memory, pos_emb, src_symbols, tgt_symbols, + src_key_padding_mask) + print("nll = ", nll) + loss = (nll * tgt_weights).sum() + print("loss = ", loss) + + + +def test_generate_square_subsequent_mask(): + s = 5 + mask = generate_square_subsequent_mask(s, torch.device('cpu')) + inf = float("inf") + expected_mask = torch.tensor( + [ + [0.0, -inf, -inf, -inf, -inf], + [0.0, 0.0, -inf, -inf, -inf], + [0.0, 0.0, 0.0, -inf, -inf], + [0.0, 0.0, 0.0, 0.0, -inf], + [0.0, 0.0, 0.0, 0.0, 0.0], + ] + ) + assert torch.all(torch.eq(mask, expected_mask)) diff --git a/egs/librispeech/ASR/conformer_lm/test_dataset.py b/egs/librispeech/ASR/conformer_lm/test_dataset.py new file mode 100644 index 000000000..4cadaa939 --- /dev/null +++ b/egs/librispeech/ASR/conformer_lm/test_dataset.py @@ -0,0 +1,32 @@ +import k2 +import torch +import _k2 +import dataset +import os +from torch import multiprocessing as mp +import torch.distributed as dist + +def local_collate_fn(sentences): + return dataset.collate_fn(sentences, bos_sym=1, eos_sym=1, blank_sym=0, debug=True) + +if __name__ == '__main__': + + #mp.set_start_method('spawn') + os.environ["MASTER_ADDR"] = "localhost" + os.environ["MASTER_PORT"] = "12344" + + dist.init_process_group(backend="nccl", group_name="main", + rank=0, world_size=1) + + train,test = dataset.load_train_test_lm_dataset('../data/lm_training_5000/lm_data.pt') + sampler = dataset.LmBatchSampler(test, symbols_per_batch=5000, world_size=2, rank=0) + print("len(sampler) = ", len(sampler)) + + a = iter(sampler) + print(str(next(a))) + + train_dl = torch.utils.data.DataLoader(test, batch_sampler=sampler, + collate_fn=local_collate_fn, + num_workers=2) + x = iter(train_dl) + print(str(next(x))) diff --git a/egs/librispeech/ASR/conformer_lm/test_dataset_empty.py b/egs/librispeech/ASR/conformer_lm/test_dataset_empty.py new file mode 100644 index 000000000..7e933f07b --- /dev/null +++ b/egs/librispeech/ASR/conformer_lm/test_dataset_empty.py @@ -0,0 +1,39 @@ +import k2 +import torch +import _k2 +import dataset +from dataset import LmDataset +import os +from torch import multiprocessing as mp +import torch.distributed as dist + +def local_collate_fn(sentences): + return dataset.collate_fn(sentences, bos_sym=1, eos_sym=1, blank_sym=0, debug=False) + +x = _k2.RaggedInt('[[1]]') # make sure library initialized? + +if __name__ == '__main__': + + mp.set_start_method('spawn') + os.environ["MASTER_ADDR"] = "localhost" + os.environ["MASTER_PORT"] = "12344" + + dist.init_process_group(backend="nccl", group_name="main", + rank=0, world_size=1) + + words = k2.RaggedInt('[[0][1 2]]') + sentences = k2.RaggedInt('[[1][][][][][]]') + + train = LmDataset(sentences, words) + + + sampler = dataset.LmBatchSampler(train, symbols_per_batch=10, world_size=1, rank=0) + + a = iter(sampler) + print(str(next(a))) + + train_dl = torch.utils.data.DataLoader(train, batch_sampler=sampler, + collate_fn=local_collate_fn, + num_workers=0) + x = iter(train_dl) + print(str(next(x))) diff --git a/egs/librispeech/ASR/conformer_lm/train.py b/egs/librispeech/ASR/conformer_lm/train.py new file mode 100755 index 000000000..2d1c1a4c3 --- /dev/null +++ b/egs/librispeech/ASR/conformer_lm/train.py @@ -0,0 +1,623 @@ +#!/usr/bin/env python3 +# Copyright 2021 Xiaomi Corp. (authors: Fangjun Kuang, Daniel Povey) +# +# +# See ../../../../LICENSE for clarification regarding multiple authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import argparse +import logging +from pathlib import Path +from shutil import copyfile +from typing import Optional, Tuple + +import k2 +import dataset # from . +import madam # from . +import torch +import torch.distributed as dist +import torch.multiprocessing as mp +import torch.nn as nn +from conformer import MaskedLmConformer +from lhotse.utils import fix_random_seed +from torch.nn.parallel import DistributedDataParallel as DDP +from torch.nn.utils import clip_grad_norm_ +from torch.utils.tensorboard import SummaryWriter +from madam import Gloam + +from icefall.checkpoint import load_checkpoint +from icefall.checkpoint import save_checkpoint as save_checkpoint_impl +from icefall.dist import cleanup_dist, setup_dist + +from icefall.utils import ( + AttributeDict, + setup_logger, + str2bool, +) + + +def get_parser(): + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + + parser.add_argument( + "--world-size", + type=int, + default=1, + help="Number of GPUs for DDP training.", + ) + + parser.add_argument( + "--master-port", + type=int, + default=12354, + help="Master port to use for DDP training.", + ) + + parser.add_argument( + "--tensorboard", + type=str2bool, + default=True, + help="Should various information be logged in tensorboard.", + ) + + return parser + + +def get_params() -> AttributeDict: + """Return a dict containing training parameters. + + All training related parameters that are not passed from the commandline + is saved in the variable `params`. + + Commandline options are merged into `params` after they are parsed, so + you can also access them via `params`. + + Explanation of options saved in `params`: + + - exp_dir: It specifies the directory where all training related + files, e.g., checkpoints, log, etc, are saved + + - lr: It specifies the initial learning rate + + - feature_dim: The model input dim. It has to match the one used + in computing features. + + - start_epoch: If it is not zero, load checkpoint `start_epoch-1` + and continue training from that checkpoint. + + - num_epochs: Number of epochs to train. + + - num_valid_batches: Number of batches of validation data to use each + time we compute validation loss + + - symbols_per_batch: Number of symbols in each batch (sampler will + choose the number of sentences to satisfy this contraint). + + - best_train_loss: Best training loss so far. It is used to select + the model that has the lowest training loss. It is + updated during the training. + + - best_valid_loss: Best validation loss so far. It is used to select + the model that has the lowest validation loss. It is + updated during the training. + + - best_train_epoch: It is the epoch that has the best training loss. + + - best_valid_epoch: It is the epoch that has the best validation loss. + + - batch_idx_train: Used to writing statistics to tensorboard. It + contains number of batches trained so far across + epochs. + + - log_interval: Print training loss if batch_idx % log_interval` is 0 + + - valid_interval: Run validation if batch_idx % valid_interval is 0 + + - reset_interval: Reset statistics if batch_idx % reset_interval is 0 + + """ + params = AttributeDict( + { + # exp_3, vs. exp_2, is using 5e-04 not 2d-04 as max learning rate. + # exp_4, vs. exp_3, is using the Gloam optimizer with + # in exp_5, vs. exp_4, we change Gloam to have a 1/sqrt(t) factor + # as well as the exponential part. + # exp_6, we change the decay from 0.85 to 0.9. + "exp_dir": Path("conformer_lm/exp_6"), + "lm_dataset": Path("data/lm_training_5000/lm_data.pt"), + "num_tokens": 5000, + "blank_sym": 0, + "bos_sym": 1, + "eos_sym": 1, + "start_epoch": 2, + "num_epochs": 20, + "num_valid_batches": 200, + "symbols_per_batch": 5000, + "best_train_loss": float("inf"), + "best_valid_loss": float("inf"), + "best_train_epoch": -1, + "best_valid_epoch": -1, + "batch_idx_train": 0, + "log_interval": 10, + "reset_interval": 200, + "valid_interval": 3000, + "beam_size": 10, + "accum_grad": 1, + "attention_dim": 512, + "nhead": 8, + "num_decoder_layers": 6, + "max_lrate": 5.0e-04 + } + ) + + return params + + +def load_checkpoint_if_available( + params: AttributeDict, + model: nn.Module, + optimizer: Optional[torch.optim.Optimizer] = None, + scheduler: Optional[torch.optim.lr_scheduler._LRScheduler] = None, +) -> None: + """Load checkpoint from file. + + If params.start_epoch is positive, it will load the checkpoint from + `params.start_epoch - 1`. Otherwise, this function does nothing. + + Apart from loading state dict for `model`, `optimizer` and `scheduler`, + it also updates `best_train_epoch`, `best_train_loss`, `best_valid_epoch`, + and `best_valid_loss` in `params`. + + Args: + params: + The return value of :func:`get_params`. + model: + The training model. + optimizer: + The optimizer that we are using. + scheduler: + The learning rate scheduler we are using. + Returns: + Return None. + """ + if params.start_epoch <= 0: + return + + filename = params.exp_dir / f"epoch-{params.start_epoch-1}.pt" + saved_params = load_checkpoint( + filename, + model=model, + optimizer=optimizer, + scheduler=scheduler, + ) + + keys = [ + "best_train_epoch", + "best_valid_epoch", + "batch_idx_train", + "best_train_loss", + "best_valid_loss", + ] + for k in keys: + params[k] = saved_params[k] + + return saved_params + + +def save_checkpoint( + params: AttributeDict, + model: nn.Module, + optimizer: Optional[torch.optim.Optimizer] = None, + scheduler: Optional[torch.optim.lr_scheduler._LRScheduler] = None, + rank: int = 0, +) -> None: + """Save model, optimizer, scheduler and training stats to file. + + Args: + params: + It is returned by :func:`get_params`. + model: + The training model. + """ + if rank != 0: + return + filename = params.exp_dir / f"epoch-{params.cur_epoch}.pt" + save_checkpoint_impl( + filename=filename, + model=model, + params=params, + optimizer=optimizer, + scheduler=scheduler, + rank=rank, + ) + + if params.best_train_epoch == params.cur_epoch: + best_train_filename = params.exp_dir / "best-train-loss.pt" + copyfile(src=filename, dst=best_train_filename) + + if params.best_valid_epoch == params.cur_epoch: + best_valid_filename = params.exp_dir / "best-valid-loss.pt" + copyfile(src=filename, dst=best_valid_filename) + + +def compute_loss( + model: nn.Module, + batch: Tuple, + is_training: bool, +): + + """ + Compute training or validation loss given the model and its inputs + (this corresponds to log-prob of the targets, with weighting + of 1.0 for masked subsequences + (including padding blanks), and something smaller, e.g. 0.25, + for non-masked positions (this is not totally trivial due to + a small amount of randomization of symbols). + + This loss is not normalized; you can divide by batch[4].sum() + to get a normalized loss (i.e. divide by soft-count). + + Args: + params: + Parameters for training. See :func:`get_params`. + model: + The model for training. It is an instance of MaskedLmConformer in our case. + batch: + A batch of data, actually a tuple of 5 tensors (on the device), as returned + by collate_fn in ./dataset.py. + is_training: + True for training. False for validation. When it is True, this + function enables autograd during computation; when it is False, it + disables autograd. + + Returns: + Returns the loss as a scalar tensor. + """ + (masked_src_symbols, src_symbols, + tgt_symbols, src_key_padding_mask, tgt_weights) = batch + + with torch.set_grad_enabled(is_training): + memory, pos_emb = model(masked_src_symbols, src_key_padding_mask) + decoder_nll_func = model.module.decoder_nll if isinstance(model, DDP) else model.decoder_nll + tgt_nll = decoder_nll_func(memory, pos_emb, src_symbols, + tgt_symbols, src_key_padding_mask) + loss = (tgt_nll * tgt_weights).sum() + + assert loss.requires_grad == is_training + + return loss + + +def compute_validation_loss( + device: torch.device, + params: AttributeDict, + model: nn.Module, + valid_dl: torch.utils.data.DataLoader, + world_size: int = 1, +) -> None: + """Run the validation process. The validation loss + is saved in `params.valid_loss`. + """ + model.eval() + + tot_loss = 0.0 + tot_frames = 0.0 + for batch_idx, batch in enumerate(valid_dl): + if batch_idx == params.num_valid_batches: + break + batch = tuple(x.to(device) for x in batch) + + + loss = compute_loss(model, batch, is_training=False) + num_frames = batch[4].sum() + + assert loss.requires_grad is False + + loss_cpu = loss.detach().cpu().item() + num_frames_cpu = num_frames.cpu().item() + + tot_loss += loss_cpu + tot_frames += num_frames_cpu + + + if world_size > 1: + s = torch.tensor( + [tot_loss, tot_frames], + device=loss.device, + ) + dist.all_reduce(s, op=dist.ReduceOp.SUM) + (tot_loss, tot_frames) = s.cpu().tolist() + + params.valid_loss = tot_loss / tot_frames + + if params.valid_loss < params.best_valid_loss: + params.best_valid_epoch = params.cur_epoch + params.best_valid_loss = params.valid_loss + + +def train_one_epoch( + device: torch.device, + params: AttributeDict, + model: nn.Module, + optimizer: torch.optim.Optimizer, + train_dl: torch.utils.data.DataLoader, + valid_dl: torch.utils.data.DataLoader, + tb_writer: Optional[SummaryWriter] = None, + world_size: int = 1, +) -> None: + """Train the model for one epoch. + + The training loss from the mean of all frames is saved in + `params.train_loss`. It runs the validation process every + `params.valid_interval` batches. + + Args: + device: + The device to use for training (model must be on this device) + params: + It is returned by :func:`get_params`. + model: + The model for training. + optimizer: + The optimizer we are using. + train_dl: + Dataloader for the training dataset. + valid_dl: + Dataloader for the validation dataset. + tb_writer: + Writer to write log messages to tensorboard. + world_size: + Number of nodes in DDP training. If it is 1, DDP is disabled. + """ + model.train() # training mode + + tot_loss = 0.0 # sum of losses over all batches + tot_frames = 0.0 # sum of frames over all batches + + params.tot_loss = 0.0 + params.tot_frames = 0.0 + for batch_idx, batch in enumerate(train_dl): + params.batch_idx_train += 1 + batch = tuple(x.to(device) for x in batch) + + try: + loss = compute_loss( + model=model, + batch=batch, + is_training=True, + ) + + optimizer.zero_grad() + loss.backward() + # We are not normalizing by the num-frames, but Adam/Madam are insensitive to the total + # gradient scale so this should not matter. + # clip_grad_norm_(model.parameters(), 5.0, 2.0) + optimizer.step() + except RuntimeError as e: + print(f"Error on batch of shape (N,T) = {batch[0].shape}") + raise e + + + loss_cpu = loss.detach().cpu().item() + num_frames_cpu = batch[4].sum().cpu().item() + + tot_loss += loss_cpu + tot_frames += num_frames_cpu + + params.tot_frames += num_frames_cpu + params.tot_loss += loss_cpu + + tot_avg_loss = tot_loss / tot_frames + + if batch_idx % params.log_interval == 0: + logging.info( + f"Epoch {params.cur_epoch}, batch {batch_idx}, " + f"batch avg loss {loss_cpu/num_frames_cpu:.4f}, " + f"total avg loss: {tot_avg_loss:.4f}, " + f"batch shape: {tuple(batch[0].shape)}") + + + if tb_writer is not None: + tb_writer.add_scalar( + "train/current_loss", + loss_cpu / num_frames_cpu, + params.batch_idx_train, + ) + tb_writer.add_scalar( + "train/tot_avg_loss", + tot_avg_loss, + params.batch_idx_train, + ) + if batch_idx > 0 and batch_idx % params.reset_interval == 0: + tot_loss = 0.0 # sum of losses over all batches + tot_frames = 0.0 # sum of frames over all batches + + if batch_idx > 0 and batch_idx % params.valid_interval == 0: + compute_validation_loss( + device=device, + params=params, + model=model, + valid_dl=valid_dl, + world_size=world_size, + ) + model.train() + logging.info( + f"Epoch {params.cur_epoch}, " + f"valid loss {params.valid_loss:.4f}," + f" best valid loss: {params.best_valid_loss:.4f} " + f"best valid epoch: {params.best_valid_epoch}" + ) + if tb_writer is not None: + tb_writer.add_scalar( + "train/valid_loss", + params.valid_loss, + params.batch_idx_train, + ) + + params.train_loss = params.tot_loss / params.tot_frames + + if params.train_loss < params.best_train_loss: + params.best_train_epoch = params.cur_epoch + params.best_train_loss = params.train_loss + + +def run(rank, world_size, args): + """ + Args: + rank: + It is a value between 0 and `world_size-1`, which is + passed automatically by `mp.spawn()` in :func:`main`. + The node with rank 0 is responsible for saving checkpoint. + world_size: + Number of GPUs for DDP training. + args: + The return value of get_parser().parse_args() + """ + params = get_params() + params.update(vars(args)) + + fix_random_seed(42) + if world_size > 1: + setup_dist(rank, world_size, params.master_port) + + setup_logger(f"{params.exp_dir}/log/log-train") + logging.info("Training started") + logging.info(params) + + if args.tensorboard and rank == 0: + tb_writer = SummaryWriter(log_dir=f"{params.exp_dir}/tensorboard") + else: + tb_writer = None + + num_tokens = params.num_tokens + + device = torch.device("cpu") + if torch.cuda.is_available(): + device = torch.device("cuda", rank) + + + logging.info("About to create model") + model = MaskedLmConformer( + num_classes=params.num_tokens, + d_model=params.attention_dim, + nhead=params.nhead, + num_decoder_layers=params.num_decoder_layers, + ) + + checkpoints = load_checkpoint_if_available(params=params, model=model) + + model.to(device) + if world_size > 1: + model = DDP(model, device_ids=[rank]) + + # Caution: don't forget to do optimizer.set_epoch() with Gloam! + # Don't remove this warning! + optimizer = Gloam( + model.parameters(), + max_lrate=params.max_lrate, + first_decrease_epoch=1, + decay_per_epoch=0.9 + ) + + if checkpoints: + optimizer.load_state_dict(checkpoints["optimizer"]) + + train,test = dataset.load_train_test_lm_dataset(params.lm_dataset) + + collate_fn=dataset.CollateFn(bos_sym=params.bos_sym, + eos_sym=params.eos_sym, + blank_sym=params.blank_sym, + mask_proportion=0.15, + padding_proportion=0.15, + randomize_proportion=0.05, + inv_mask_length=0.25, + unmasked_weight=0.25) + + train_sampler = dataset.LmBatchSampler(train, + symbols_per_batch=params.symbols_per_batch, + world_size=world_size, rank=rank) + test_sampler = dataset.LmBatchSampler(test, + symbols_per_batch=params.symbols_per_batch, + world_size=world_size, rank=rank) + + train_dl = torch.utils.data.DataLoader(train, + batch_sampler=train_sampler, + collate_fn=collate_fn) + + valid_dl = torch.utils.data.DataLoader(test, + batch_sampler=test_sampler, + collate_fn=collate_fn) + + for epoch in range(params.start_epoch, params.num_epochs): + train_sampler.set_epoch(epoch) + optimizer.set_epoch(epoch) # Caution: this is specific to the Gloam + # optimizer. + + cur_lr = optimizer._rate + if tb_writer is not None: + tb_writer.add_scalar( + "train/learning_rate", cur_lr, params.batch_idx_train + ) + tb_writer.add_scalar("train/epoch", epoch, params.batch_idx_train) + + if rank == 0: + logging.info("epoch {}, learning rate {}".format(epoch, cur_lr)) + + params.cur_epoch = epoch + + train_one_epoch( + device=device, + params=params, + model=model, + optimizer=optimizer, + train_dl=train_dl, + valid_dl=valid_dl, + tb_writer=tb_writer, + world_size=world_size, + ) + + save_checkpoint( + params=params, + model=model, + optimizer=optimizer, + rank=rank, + ) + + logging.info("Done!") + + if world_size > 1: + torch.distributed.barrier() + cleanup_dist() + + +def main(): + parser = get_parser() + args = parser.parse_args() + + world_size = args.world_size + assert world_size >= 1 + if world_size > 1: + mp.spawn(run, args=(world_size, args), nprocs=world_size, join=True) + else: + run(rank=0, world_size=1, args=args) + + +torch.set_num_threads(1) +torch.set_num_interop_threads(1) + +if __name__ == "__main__": + main() From 1ac9bb3fd71dc79eeaf6166f2302cbad9b612976 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Mon, 1 Nov 2021 22:01:12 +0800 Subject: [PATCH 4/5] WIP: Begin to add decoding scripts. --- egs/librispeech/ASR/conformer_lm/decode.py | 693 +++++++++++++++++++++ 1 file changed, 693 insertions(+) create mode 100755 egs/librispeech/ASR/conformer_lm/decode.py diff --git a/egs/librispeech/ASR/conformer_lm/decode.py b/egs/librispeech/ASR/conformer_lm/decode.py new file mode 100755 index 000000000..bfeec1454 --- /dev/null +++ b/egs/librispeech/ASR/conformer_lm/decode.py @@ -0,0 +1,693 @@ +#!/usr/bin/env python3 +# Copyright 2021 Xiaomi Corporation (Author: Liyong Guo, Fangjun Kuang) +# +# See ../../../../LICENSE for clarification regarding multiple authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import argparse +import logging +from collections import defaultdict +from pathlib import Path +from typing import Dict, List, Optional, Tuple + +import k2 +import sentencepiece as spm +import torch +import torch.nn as nn +from asr_datamodule import LibriSpeechAsrDataModule +from conformer import MaskedLmConformer + +from icefall.bpe_graph_compiler import BpeCtcTrainingGraphCompiler +from icefall.checkpoint import average_checkpoints, load_checkpoint +from icefall.decode import ( + get_lattice, + nbest_decoding, + nbest_oracle, + one_best_decoding, + rescore_with_attention_decoder, + rescore_with_n_best_list, + rescore_with_whole_lattice, +) +from icefall.lexicon import Lexicon +from icefall.utils import ( + AttributeDict, + get_env_info, + get_texts, + setup_logger, + store_transcripts, + str2bool, + write_error_stats, +) + + +def get_parser(): + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + + parser.add_argument( + "--epoch", + type=int, + default=19, + help="It specifies the checkpoint to use for decoding." + "Note: Epoch counts from 0.", + ) + parser.add_argument( + "--avg", + type=int, + default=5, + help="Number of checkpoints to average. Automatically select " + "consecutive checkpoints before the checkpoint specified by " + "'--epoch'. ", + ) + + parser.add_argument( + "--method", + type=str, + default="attention-decoder", + help="""Decoding method. + Supported values are: + - (0) ctc-decoding. Use CTC decoding. It uses a sentence piece + model, i.e., lang_dir/bpe.model, to convert word pieces to words. + It needs neither a lexicon nor an n-gram LM. + - (1) 1best. Extract the best path from the decoding lattice as the + decoding result. + - (2) nbest. Extract n paths from the decoding lattice; the path + with the highest score is the decoding result. + - (3) nbest-rescoring. Extract n paths from the decoding lattice, + rescore them with an n-gram LM (e.g., a 4-gram LM), the path with + the highest score is the decoding result. + - (4) whole-lattice-rescoring. Rescore the decoding lattice with an + n-gram LM (e.g., a 4-gram LM), the best path of rescored lattice + is the decoding result. + - (5) attention-decoder. Extract n paths from the LM rescored + lattice, the path with the highest score is the decoding result. + - (6) nbest-oracle. Its WER is the lower bound of any n-best + rescoring method can achieve. Useful for debugging n-best + rescoring method. + """, + ) + + parser.add_argument( + "--num-paths", + type=int, + default=100, + help="""Number of paths for n-best based decoding method. + Used only when "method" is one of the following values: + nbest, nbest-rescoring, attention-decoder, and nbest-oracle + """, + ) + + parser.add_argument( + "--nbest-scale", + type=float, + default=0.5, + help="""The scale to be applied to `lattice.scores`. + It's needed if you use any kinds of n-best based rescoring. + Used only when "method" is one of the following values: + nbest, nbest-rescoring, attention-decoder, and nbest-oracle + A smaller value results in more unique paths. + """, + ) + + parser.add_argument( + "--export", + type=str2bool, + default=False, + help="""When enabled, the averaged model is saved to + conformer_ctc/exp/pretrained.pt. Note: only model.state_dict() is saved. + pretrained.pt contains a dict {"model": model.state_dict()}, + which can be loaded by `icefall.checkpoint.load_checkpoint()`. + """, + ) + + parser.add_argument( + "--exp-dir", + type=str, + default="conformer_lm/exp", + help="The experiment dir", + ) + + parser.add_argument( + "--lang-dir", + type=str, + default="data/lang_bpe_5000", + help="The lang dir", + ) + + return parser + + +def get_params() -> AttributeDict: + params = AttributeDict( + { + "lm_dir": Path("data/lm"), + "num_tokens": 5000, + "blank_sym": 0, + "bos_sym": 1, + "eos_sym": 1, + # parameters for conformer + "attention_dim": 512, + "nhead": 8, + "num_decoder_layers": 6, + # parameters for decoding + "search_beam": 20, + "output_beam": 8, + "min_active_states": 30, + "max_active_states": 10000, + "use_double_scores": True, + "env_info": get_env_info(), + } + ) + return params + + +def decode_one_batch( + params: AttributeDict, + model: nn.Module, + HLG: Optional[k2.Fsa], + H: Optional[k2.Fsa], + bpe_model: Optional[spm.SentencePieceProcessor], + batch: dict, + word_table: k2.SymbolTable, + sos_id: int, + eos_id: int, + G: Optional[k2.Fsa] = None, +) -> Dict[str, List[List[str]]]: + """Decode one batch and return the result in a dict. The dict has the + following format: + + - key: It indicates the setting used for decoding. For example, + if no rescoring is used, the key is the string `no_rescore`. + If LM rescoring is used, the key is the string `lm_scale_xxx`, + where `xxx` is the value of `lm_scale`. An example key is + `lm_scale_0.7` + - value: It contains the decoding result. `len(value)` equals to + batch size. `value[i]` is the decoding result for the i-th + utterance in the given batch. + Args: + params: + It's the return value of :func:`get_params`. + + - params.method is "1best", it uses 1best decoding without LM rescoring. + - params.method is "nbest", it uses nbest decoding without LM rescoring. + - params.method is "nbest-rescoring", it uses nbest LM rescoring. + - params.method is "whole-lattice-rescoring", it uses whole lattice LM + rescoring. + + model: + The neural model. + HLG: + The decoding graph. Used only when params.method is NOT ctc-decoding. + H: + The ctc topo. Used only when params.method is ctc-decoding. + bpe_model: + The BPE model. Used only when params.method is ctc-decoding. + batch: + It is the return value from iterating + `lhotse.dataset.K2SpeechRecognitionDataset`. See its documentation + for the format of the `batch`. + word_table: + The word symbol table. + sos_id: + The token ID of the SOS. + eos_id: + The token ID of the EOS. + G: + An LM. It is not None when params.method is "nbest-rescoring" + or "whole-lattice-rescoring". In general, the G in HLG + is a 3-gram LM, while this G is a 4-gram LM. + Returns: + Return the decoding result. See above description for the format of + the returned dict. + """ + if HLG is not None: + device = HLG.device + else: + device = H.device + feature = batch["inputs"] + assert feature.ndim == 3 + feature = feature.to(device) + # at entry, feature is (N, T, C) + + supervisions = batch["supervisions"] + + nnet_output, memory, memory_key_padding_mask = model(feature, supervisions) + # nnet_output is (N, T, C) + + supervision_segments = torch.stack( + ( + supervisions["sequence_idx"], + supervisions["start_frame"] // params.subsampling_factor, + supervisions["num_frames"] // params.subsampling_factor, + ), + 1, + ).to(torch.int32) + + if H is None: + assert HLG is not None + decoding_graph = HLG + else: + assert HLG is None + assert bpe_model is not None + decoding_graph = H + + lattice = get_lattice( + nnet_output=nnet_output, + decoding_graph=decoding_graph, + supervision_segments=supervision_segments, + search_beam=params.search_beam, + output_beam=params.output_beam, + min_active_states=params.min_active_states, + max_active_states=params.max_active_states, + subsampling_factor=params.subsampling_factor, + ) + + if params.method == "ctc-decoding": + best_path = one_best_decoding( + lattice=lattice, use_double_scores=params.use_double_scores + ) + # Note: `best_path.aux_labels` contains token IDs, not word IDs + # since we are using H, not HLG here. + # + # token_ids is a lit-of-list of IDs + token_ids = get_texts(best_path) + + # hyps is a list of str, e.g., ['xxx yyy zzz', ...] + hyps = bpe_model.decode(token_ids) + + # hyps is a list of list of str, e.g., [['xxx', 'yyy', 'zzz'], ... ] + hyps = [s.split() for s in hyps] + key = "ctc-decoding" + return {key: hyps} + + if params.method == "nbest-oracle": + # Note: You can also pass rescored lattices to it. + # We choose the HLG decoded lattice for speed reasons + # as HLG decoding is faster and the oracle WER + # is only slightly worse than that of rescored lattices. + best_path = nbest_oracle( + lattice=lattice, + num_paths=params.num_paths, + ref_texts=supervisions["text"], + word_table=word_table, + nbest_scale=params.nbest_scale, + oov="", + ) + hyps = get_texts(best_path) + hyps = [[word_table[i] for i in ids] for ids in hyps] + key = f"oracle_{params.num_paths}_nbest_scale_{params.nbest_scale}" # noqa + return {key: hyps} + + if params.method in ["1best", "nbest"]: + if params.method == "1best": + best_path = one_best_decoding( + lattice=lattice, use_double_scores=params.use_double_scores + ) + key = "no_rescore" + else: + best_path = nbest_decoding( + lattice=lattice, + num_paths=params.num_paths, + use_double_scores=params.use_double_scores, + nbest_scale=params.nbest_scale, + ) + key = f"no_rescore-nbest-scale-{params.nbest_scale}-{params.num_paths}" # noqa + + hyps = get_texts(best_path) + hyps = [[word_table[i] for i in ids] for ids in hyps] + return {key: hyps} + + assert params.method in [ + "nbest-rescoring", + "whole-lattice-rescoring", + "attention-decoder", + ] + + lm_scale_list = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7] + lm_scale_list += [0.8, 0.9, 1.0, 1.1, 1.2, 1.3] + lm_scale_list += [1.4, 1.5, 1.6, 1.7, 1.8, 1.9, 2.0] + + if params.method == "nbest-rescoring": + best_path_dict = rescore_with_n_best_list( + lattice=lattice, + G=G, + num_paths=params.num_paths, + lm_scale_list=lm_scale_list, + nbest_scale=params.nbest_scale, + ) + elif params.method == "whole-lattice-rescoring": + best_path_dict = rescore_with_whole_lattice( + lattice=lattice, + G_with_epsilon_loops=G, + lm_scale_list=lm_scale_list, + ) + elif params.method == "attention-decoder": + # lattice uses a 3-gram Lm. We rescore it with a 4-gram LM. + rescored_lattice = rescore_with_whole_lattice( + lattice=lattice, + G_with_epsilon_loops=G, + lm_scale_list=None, + ) + # TODO: pass `lattice` instead of `rescored_lattice` to + # `rescore_with_attention_decoder` + + best_path_dict = rescore_with_attention_decoder( + lattice=rescored_lattice, + num_paths=params.num_paths, + model=model, + memory=memory, + memory_key_padding_mask=memory_key_padding_mask, + sos_id=sos_id, + eos_id=eos_id, + nbest_scale=params.nbest_scale, + ) + else: + assert False, f"Unsupported decoding method: {params.method}" + + ans = dict() + if best_path_dict is not None: + for lm_scale_str, best_path in best_path_dict.items(): + hyps = get_texts(best_path) + hyps = [[word_table[i] for i in ids] for ids in hyps] + ans[lm_scale_str] = hyps + else: + for lm_scale in lm_scale_list: + ans[f"{lm_scale}"] = [[] * lattice.shape[0]] + return ans + + +def decode_dataset( + dl: torch.utils.data.DataLoader, + params: AttributeDict, + model: nn.Module, + HLG: Optional[k2.Fsa], + H: Optional[k2.Fsa], + bpe_model: Optional[spm.SentencePieceProcessor], + word_table: k2.SymbolTable, + sos_id: int, + eos_id: int, + G: Optional[k2.Fsa] = None, +) -> Dict[str, List[Tuple[List[str], List[str]]]]: + """Decode dataset. + + Args: + dl: + PyTorch's dataloader containing the dataset to decode. + params: + It is returned by :func:`get_params`. + model: + The neural model. + HLG: + The decoding graph. Used only when params.method is NOT ctc-decoding. + H: + The ctc topo. Used only when params.method is ctc-decoding. + bpe_model: + The BPE model. Used only when params.method is ctc-decoding. + word_table: + It is the word symbol table. + sos_id: + The token ID for SOS. + eos_id: + The token ID for EOS. + G: + An LM. It is not None when params.method is "nbest-rescoring" + or "whole-lattice-rescoring". In general, the G in HLG + is a 3-gram LM, while this G is a 4-gram LM. + Returns: + Return a dict, whose key may be "no-rescore" if no LM rescoring + is used, or it may be "lm_scale_0.7" if LM rescoring is used. + Its value is a list of tuples. Each tuple contains two elements: + The first is the reference transcript, and the second is the + predicted result. + """ + results = [] + + num_cuts = 0 + + try: + num_batches = len(dl) + except TypeError: + num_batches = "?" + + results = defaultdict(list) + for batch_idx, batch in enumerate(dl): + texts = batch["supervisions"]["text"] + + hyps_dict = decode_one_batch( + params=params, + model=model, + HLG=HLG, + H=H, + bpe_model=bpe_model, + batch=batch, + word_table=word_table, + G=G, + sos_id=sos_id, + eos_id=eos_id, + ) + + for lm_scale, hyps in hyps_dict.items(): + this_batch = [] + assert len(hyps) == len(texts) + for hyp_words, ref_text in zip(hyps, texts): + ref_words = ref_text.split() + this_batch.append((ref_words, hyp_words)) + + results[lm_scale].extend(this_batch) + + num_cuts += len(batch["supervisions"]["text"]) + + if batch_idx % 100 == 0: + batch_str = f"{batch_idx}/{num_batches}" + + logging.info( + f"batch {batch_str}, cuts processed until now is {num_cuts}" + ) + return results + + +def save_results( + params: AttributeDict, + test_set_name: str, + results_dict: Dict[str, List[Tuple[List[int], List[int]]]], +): + if params.method == "attention-decoder": + # Set it to False since there are too many logs. + enable_log = False + else: + enable_log = True + test_set_wers = dict() + for key, results in results_dict.items(): + recog_path = params.exp_dir / f"recogs-{test_set_name}-{key}.txt" + store_transcripts(filename=recog_path, texts=results) + if enable_log: + logging.info(f"The transcripts are stored in {recog_path}") + + # The following prints out WERs, per-word error statistics and aligned + # ref/hyp pairs. + errs_filename = params.exp_dir / f"errs-{test_set_name}-{key}.txt" + with open(errs_filename, "w") as f: + wer = write_error_stats( + f, f"{test_set_name}-{key}", results, enable_log=enable_log + ) + test_set_wers[key] = wer + + if enable_log: + logging.info( + "Wrote detailed error stats to {}".format(errs_filename) + ) + + test_set_wers = sorted(test_set_wers.items(), key=lambda x: x[1]) + errs_info = params.exp_dir / f"wer-summary-{test_set_name}.txt" + with open(errs_info, "w") as f: + print("settings\tWER", file=f) + for key, val in test_set_wers: + print("{}\t{}".format(key, val), file=f) + + s = "\nFor {}, WER of different settings are:\n".format(test_set_name) + note = "\tbest for {}".format(test_set_name) + for key, val in test_set_wers: + s += "{}\t{}{}\n".format(key, val, note) + note = "" + logging.info(s) + + +@torch.no_grad() +def main(): + parser = get_parser() + LibriSpeechAsrDataModule.add_arguments(parser) + args = parser.parse_args() + args.exp_dir = Path(args.exp_dir) + args.lang_dir = Path(args.lang_dir) + + params = get_params() + params.update(vars(args)) + + setup_logger(f"{params.exp_dir}/log-{params.method}/log-decode") + logging.info("Decoding started") + logging.info(params) + + lexicon = Lexicon(params.lang_dir) + max_token_id = max(lexicon.tokens) + num_classes = max_token_id + 1 # +1 for the blank + assert num_classes == params.num_tokens + + device = torch.device("cpu") + if torch.cuda.is_available(): + device = torch.device("cuda", 0) + + logging.info(f"device: {device}") + + graph_compiler = BpeCtcTrainingGraphCompiler( + params.lang_dir, + device=device, + sos_token="", + eos_token="", + ) + sos_id = graph_compiler.sos_id + eos_id = graph_compiler.eos_id + + if params.method == "ctc-decoding": + HLG = None + H = k2.ctc_topo( + max_token=max_token_id, + modified=False, + device=device, + ) + bpe_model = spm.SentencePieceProcessor() + bpe_model.load(str(params.lang_dir / "bpe.model")) + else: + H = None + bpe_model = None + HLG = k2.Fsa.from_dict( + torch.load(f"{params.lang_dir}/HLG.pt", map_location="cpu") + ) + HLG = HLG.to(device) + assert HLG.requires_grad is False + + if not hasattr(HLG, "lm_scores"): + HLG.lm_scores = HLG.scores.clone() + + if params.method in ( + "nbest-rescoring", + "whole-lattice-rescoring", + "attention-decoder", + ): + if not (params.lm_dir / "G_4_gram.pt").is_file(): + logging.info("Loading G_4_gram.fst.txt") + logging.warning("It may take 8 minutes.") + with open(params.lm_dir / "G_4_gram.fst.txt") as f: + first_word_disambig_id = lexicon.word_table["#0"] + + G = k2.Fsa.from_openfst(f.read(), acceptor=False) + # G.aux_labels is not needed in later computations, so + # remove it here. + del G.aux_labels + # CAUTION: The following line is crucial. + # Arcs entering the back-off state have label equal to #0. + # We have to change it to 0 here. + G.labels[G.labels >= first_word_disambig_id] = 0 + G = k2.Fsa.from_fsas([G]).to(device) + G = k2.arc_sort(G) + torch.save(G.as_dict(), params.lm_dir / "G_4_gram.pt") + else: + logging.info("Loading pre-compiled G_4_gram.pt") + d = torch.load(params.lm_dir / "G_4_gram.pt", map_location="cpu") + G = k2.Fsa.from_dict(d).to(device) + + if params.method in ["whole-lattice-rescoring", "attention-decoder"]: + # Add epsilon self-loops to G as we will compose + # it with the whole lattice later + G = k2.add_epsilon_self_loops(G) + G = k2.arc_sort(G) + G = G.to(device) + + # G.lm_scores is used to replace HLG.lm_scores during + # LM rescoring. + G.lm_scores = G.scores.clone() + else: + G = None + + model = MaskedLmConformer( + num_classes=params.num_tokens, + d_model=params.attention_dim, + nhead=params.nhead, + num_decoder_layers=params.num_decoder_layers, + ) + + if params.avg == 1: + load_checkpoint(f"{params.exp_dir}/epoch-{params.epoch}.pt", model) + else: + start = params.epoch - params.avg + 1 + filenames = [] + for i in range(start, params.epoch + 1): + if start >= 0: + filenames.append(f"{params.exp_dir}/epoch-{i}.pt") + logging.info(f"averaging {filenames}") + model.to(device) + model.load_state_dict(average_checkpoints(filenames, device=device)) + + if params.export: + logging.info(f"Export averaged model to {params.exp_dir}/pretrained.pt") + torch.save( + {"model": model.state_dict()}, f"{params.exp_dir}/pretrained.pt" + ) + return + print("TODO: Implement me!") + # [ ] Add an option to use conformer lm for rescoring + # [ ] Load conformer_lm only when that options is activated + # [ ] Load conformer model + return + + model.to(device) + model.eval() + num_param = sum([p.numel() for p in model.parameters()]) + logging.info(f"Number of model parameters: {num_param}") + + librispeech = LibriSpeechAsrDataModule(args) + # CAUTION: `test_sets` is for displaying only. + # If you want to skip test-clean, you have to skip + # it inside the for loop. That is, use + # + # if test_set == 'test-clean': continue + # + test_sets = ["test-clean", "test-other"] + for test_set, test_dl in zip(test_sets, librispeech.test_dataloaders()): + results_dict = decode_dataset( + dl=test_dl, + params=params, + model=model, + HLG=HLG, + H=H, + bpe_model=bpe_model, + word_table=lexicon.word_table, + G=G, + sos_id=sos_id, + eos_id=eos_id, + ) + + save_results( + params=params, test_set_name=test_set, results_dict=results_dict + ) + + logging.info("Done!") + + +torch.set_num_threads(1) +torch.set_num_interop_threads(1) + +if __name__ == "__main__": + main() From cdd539e55c63d21df91bfad9dd9cccdc306842a9 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Wed, 3 Nov 2021 20:59:54 +0800 Subject: [PATCH 5/5] First version using conformer lm for rescoring (not tested) --- .../ASR/conformer_ctc/conformer.py | 6 +- egs/librispeech/ASR/conformer_ctc/decode.py | 129 +++- egs/librispeech/ASR/conformer_ctc/export.py | 2 +- .../ASR/conformer_ctc/pretrained.py | 2 +- .../ASR/conformer_ctc/test_transformer.py | 4 +- .../ASR/conformer_ctc/transformer.py | 2 +- egs/librispeech/ASR/conformer_lm/decode.py | 693 ------------------ icefall/decode.py | 201 +++++ icefall/lm/rescore.py | 57 +- test/lm/test_rescore.py | 31 +- 10 files changed, 331 insertions(+), 796 deletions(-) delete mode 100755 egs/librispeech/ASR/conformer_lm/decode.py diff --git a/egs/librispeech/ASR/conformer_ctc/conformer.py b/egs/librispeech/ASR/conformer_ctc/conformer.py index b19b94db1..367906d69 100644 --- a/egs/librispeech/ASR/conformer_ctc/conformer.py +++ b/egs/librispeech/ASR/conformer_ctc/conformer.py @@ -21,8 +21,12 @@ import warnings from typing import Optional, Tuple import torch +from conformer_ctc.transformer import ( + Supervisions, + Transformer, + encoder_padding_mask, +) from torch import Tensor, nn -from transformer import Supervisions, Transformer, encoder_padding_mask class Conformer(Transformer): diff --git a/egs/librispeech/ASR/conformer_ctc/decode.py b/egs/librispeech/ASR/conformer_ctc/decode.py index c5ae3ad7d..860b31b08 100755 --- a/egs/librispeech/ASR/conformer_ctc/decode.py +++ b/egs/librispeech/ASR/conformer_ctc/decode.py @@ -26,8 +26,9 @@ import k2 import sentencepiece as spm import torch import torch.nn as nn -from asr_datamodule import LibriSpeechAsrDataModule -from conformer import Conformer +from conformer_ctc.asr_datamodule import LibriSpeechAsrDataModule +from conformer_ctc.conformer import Conformer +from conformer_lm.conformer import MaskedLmConformer from icefall.bpe_graph_compiler import BpeCtcTrainingGraphCompiler from icefall.checkpoint import average_checkpoints, load_checkpoint @@ -37,6 +38,7 @@ from icefall.decode import ( nbest_oracle, one_best_decoding, rescore_with_attention_decoder, + rescore_with_conformer_lm, rescore_with_n_best_list, rescore_with_whole_lattice, ) @@ -94,7 +96,10 @@ def get_parser(): is the decoding result. - (5) attention-decoder. Extract n paths from the LM rescored lattice, the path with the highest score is the decoding result. - - (6) nbest-oracle. Its WER is the lower bound of any n-best + - (6) conformer-lm. In addition to attention-decoder rescoring, it + also uses conformer lm for rescoring. See the model in the + directory ./conformer_lm + - (7) nbest-oracle. Its WER is the lower bound of any n-best rescoring method can achieve. Useful for debugging n-best rescoring method. """, @@ -106,7 +111,8 @@ def get_parser(): default=100, help="""Number of paths for n-best based decoding method. Used only when "method" is one of the following values: - nbest, nbest-rescoring, attention-decoder, and nbest-oracle + nbest, nbest-rescoring, attention-decoder, conformer-lm, + and nbest-oracle """, ) @@ -117,8 +123,8 @@ def get_parser(): help="""The scale to be applied to `lattice.scores`. It's needed if you use any kinds of n-best based rescoring. Used only when "method" is one of the following values: - nbest, nbest-rescoring, attention-decoder, and nbest-oracle - A smaller value results in more unique paths. + nbest, nbest-rescoring, attention-decoder, conformer_lm, + and nbest-oracle. A smaller value results in more unique paths. """, ) @@ -147,6 +153,35 @@ def get_parser(): help="The lang dir", ) + parser.add_argument( + "--conformer-lm-exp-dir", + type=str, + default="conformer_lm/exp", + help="""The conformer lm exp dir. + Used only when method is conformer_lm. + """, + ) + + parser.add_argument( + "--conformer-lm-epoch", + type=int, + default=19, + help="""Used only when method is conformer_lm. + It specifies the checkpoint to use for the conformer + lm model. + """, + ) + + parser.add_argument( + "--conformer-lm-avg", + type=int, + default=1, + help="""Used only when method is conformer_lm. + It specifies number of checkpoints to average for + the conformer lm model. + """, + ) + return parser @@ -177,6 +212,7 @@ def get_params() -> AttributeDict: def decode_one_batch( params: AttributeDict, model: nn.Module, + masked_lm_model: Optional[nn.Module], HLG: Optional[k2.Fsa], H: Optional[k2.Fsa], bpe_model: Optional[spm.SentencePieceProcessor], @@ -334,6 +370,7 @@ def decode_one_batch( "nbest-rescoring", "whole-lattice-rescoring", "attention-decoder", + "conformer-lm", ] lm_scale_list = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7] @@ -354,7 +391,7 @@ def decode_one_batch( G_with_epsilon_loops=G, lm_scale_list=lm_scale_list, ) - elif params.method == "attention-decoder": + elif params.method in ("attention-decoder", "conformer-lm"): # lattice uses a 3-gram Lm. We rescore it with a 4-gram LM. rescored_lattice = rescore_with_whole_lattice( lattice=lattice, @@ -364,16 +401,32 @@ def decode_one_batch( # TODO: pass `lattice` instead of `rescored_lattice` to # `rescore_with_attention_decoder` - best_path_dict = rescore_with_attention_decoder( - lattice=rescored_lattice, - num_paths=params.num_paths, - model=model, - memory=memory, - memory_key_padding_mask=memory_key_padding_mask, - sos_id=sos_id, - eos_id=eos_id, - nbest_scale=params.nbest_scale, - ) + if params.method == "attention-decoder": + best_path_dict = rescore_with_attention_decoder( + lattice=rescored_lattice, + num_paths=params.num_paths, + model=model, + memory=memory, + memory_key_padding_mask=memory_key_padding_mask, + sos_id=sos_id, + eos_id=eos_id, + nbest_scale=params.nbest_scale, + ) + else: + # It uses: + # attention_decoder + conformer_lm + best_path_dict = rescore_with_conformer_lm( + lattice=rescored_lattice, + num_paths=params.num_paths, + model=model, + masked_lm_model=masked_lm_model, + memory=memory, + memory_key_padding_mask=memory_key_padding_mask, + sos_id=sos_id, + eos_id=eos_id, + blank_id=0, # TODO(fangjun): pass it as an argument + nbest_scale=params.nbest_scale, + ) else: assert False, f"Unsupported decoding method: {params.method}" @@ -393,6 +446,7 @@ def decode_dataset( dl: torch.utils.data.DataLoader, params: AttributeDict, model: nn.Module, + masked_lm_model: Optional[nn.Module], HLG: Optional[k2.Fsa], H: Optional[k2.Fsa], bpe_model: Optional[spm.SentencePieceProcessor], @@ -449,6 +503,7 @@ def decode_dataset( hyps_dict = decode_one_batch( params=params, model=model, + masked_lm_model=masked_lm_model, HLG=HLG, H=H, bpe_model=bpe_model, @@ -584,6 +639,7 @@ def main(): "nbest-rescoring", "whole-lattice-rescoring", "attention-decoder", + "conformer-lm", ): if not (params.lm_dir / "G_4_gram.pt").is_file(): logging.info("Loading G_4_gram.fst.txt") @@ -607,7 +663,11 @@ def main(): d = torch.load(params.lm_dir / "G_4_gram.pt", map_location="cpu") G = k2.Fsa.from_dict(d).to(device) - if params.method in ["whole-lattice-rescoring", "attention-decoder"]: + if params.method in [ + "whole-lattice-rescoring", + "attention-decoder", + "conformer-lm", + ]: # Add epsilon self-loops to G as we will compose # it with the whole lattice later G = k2.add_epsilon_self_loops(G) @@ -655,6 +715,38 @@ def main(): num_param = sum([p.numel() for p in model.parameters()]) logging.info(f"Number of model parameters: {num_param}") + if params.method == "conformer-lm": + logging.info("Loading conformer lm model") + # Note: If the parameters does not match + # the one used to save the checkpoint, it will + # throw while calling `load_state_dict`. + masked_lm_model = MaskedLmConformer( + num_classes=num_classes, + d_model=params.attention_dim, + nhead=params.nhead, + num_decoder_layers=params.num_decoder_layers, + ) + if params.conformer_lm_avg == 1: + load_checkpoint( + f"{params.conformer_lm_exp_dir}/epoch-{params.conformer_lm_epoch}.pt", # noqa + masked_lm_model, + ) + else: + start = params.conformer_lm_epoch - params.conformer_lm_avg + 1 + filenames = [] + for i in range(start, params.conformer_lm_epoch + 1): + if start >= 0: + filenames.append( + f"{params.conformer_lm_exp_dir}/epoch-{i}.pt" + ) + logging.info(f"averaging {filenames}") + masked_lm_model.to(device) + masked_lm_model.load_state_dict( + average_checkpoints(filenames, device=device) + ) + else: + masked_lm_model = None + librispeech = LibriSpeechAsrDataModule(args) # CAUTION: `test_sets` is for displaying only. # If you want to skip test-clean, you have to skip @@ -668,6 +760,7 @@ def main(): dl=test_dl, params=params, model=model, + masked_lm_model=masked_lm_model, HLG=HLG, H=H, bpe_model=bpe_model, diff --git a/egs/librispeech/ASR/conformer_ctc/export.py b/egs/librispeech/ASR/conformer_ctc/export.py index 79e026dac..b83b7b9f7 100755 --- a/egs/librispeech/ASR/conformer_ctc/export.py +++ b/egs/librispeech/ASR/conformer_ctc/export.py @@ -24,7 +24,7 @@ import logging from pathlib import Path import torch -from conformer import Conformer +from conformer_ctc.conformer import Conformer from icefall.checkpoint import average_checkpoints, load_checkpoint from icefall.lexicon import Lexicon diff --git a/egs/librispeech/ASR/conformer_ctc/pretrained.py b/egs/librispeech/ASR/conformer_ctc/pretrained.py index beed6f73b..aad777f52 100755 --- a/egs/librispeech/ASR/conformer_ctc/pretrained.py +++ b/egs/librispeech/ASR/conformer_ctc/pretrained.py @@ -27,7 +27,7 @@ import kaldifeat import sentencepiece as spm import torch import torchaudio -from conformer import Conformer +from conformer_ctc.conformer import Conformer from torch.nn.utils.rnn import pad_sequence from icefall.decode import ( diff --git a/egs/librispeech/ASR/conformer_ctc/test_transformer.py b/egs/librispeech/ASR/conformer_ctc/test_transformer.py index 667057c51..932b7a07a 100644 --- a/egs/librispeech/ASR/conformer_ctc/test_transformer.py +++ b/egs/librispeech/ASR/conformer_ctc/test_transformer.py @@ -17,8 +17,7 @@ import torch -from torch.nn.utils.rnn import pad_sequence -from transformer import ( +from conformer_ctc.transformer import ( Transformer, add_eos, add_sos, @@ -26,6 +25,7 @@ from transformer import ( encoder_padding_mask, generate_square_subsequent_mask, ) +from torch.nn.utils.rnn import pad_sequence def test_encoder_padding_mask(): diff --git a/egs/librispeech/ASR/conformer_ctc/transformer.py b/egs/librispeech/ASR/conformer_ctc/transformer.py index 3e6abb695..e15b1d500 100644 --- a/egs/librispeech/ASR/conformer_ctc/transformer.py +++ b/egs/librispeech/ASR/conformer_ctc/transformer.py @@ -20,7 +20,7 @@ from typing import Dict, List, Optional, Tuple import torch import torch.nn as nn -from subsampling import Conv2dSubsampling, VggSubsampling +from conformer_ctc.subsampling import Conv2dSubsampling, VggSubsampling from torch.nn.utils.rnn import pad_sequence # Note: TorchScript requires Dict/List/etc. to be fully typed. diff --git a/egs/librispeech/ASR/conformer_lm/decode.py b/egs/librispeech/ASR/conformer_lm/decode.py deleted file mode 100755 index bfeec1454..000000000 --- a/egs/librispeech/ASR/conformer_lm/decode.py +++ /dev/null @@ -1,693 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2021 Xiaomi Corporation (Author: Liyong Guo, Fangjun Kuang) -# -# See ../../../../LICENSE for clarification regarding multiple authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import argparse -import logging -from collections import defaultdict -from pathlib import Path -from typing import Dict, List, Optional, Tuple - -import k2 -import sentencepiece as spm -import torch -import torch.nn as nn -from asr_datamodule import LibriSpeechAsrDataModule -from conformer import MaskedLmConformer - -from icefall.bpe_graph_compiler import BpeCtcTrainingGraphCompiler -from icefall.checkpoint import average_checkpoints, load_checkpoint -from icefall.decode import ( - get_lattice, - nbest_decoding, - nbest_oracle, - one_best_decoding, - rescore_with_attention_decoder, - rescore_with_n_best_list, - rescore_with_whole_lattice, -) -from icefall.lexicon import Lexicon -from icefall.utils import ( - AttributeDict, - get_env_info, - get_texts, - setup_logger, - store_transcripts, - str2bool, - write_error_stats, -) - - -def get_parser(): - parser = argparse.ArgumentParser( - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - - parser.add_argument( - "--epoch", - type=int, - default=19, - help="It specifies the checkpoint to use for decoding." - "Note: Epoch counts from 0.", - ) - parser.add_argument( - "--avg", - type=int, - default=5, - help="Number of checkpoints to average. Automatically select " - "consecutive checkpoints before the checkpoint specified by " - "'--epoch'. ", - ) - - parser.add_argument( - "--method", - type=str, - default="attention-decoder", - help="""Decoding method. - Supported values are: - - (0) ctc-decoding. Use CTC decoding. It uses a sentence piece - model, i.e., lang_dir/bpe.model, to convert word pieces to words. - It needs neither a lexicon nor an n-gram LM. - - (1) 1best. Extract the best path from the decoding lattice as the - decoding result. - - (2) nbest. Extract n paths from the decoding lattice; the path - with the highest score is the decoding result. - - (3) nbest-rescoring. Extract n paths from the decoding lattice, - rescore them with an n-gram LM (e.g., a 4-gram LM), the path with - the highest score is the decoding result. - - (4) whole-lattice-rescoring. Rescore the decoding lattice with an - n-gram LM (e.g., a 4-gram LM), the best path of rescored lattice - is the decoding result. - - (5) attention-decoder. Extract n paths from the LM rescored - lattice, the path with the highest score is the decoding result. - - (6) nbest-oracle. Its WER is the lower bound of any n-best - rescoring method can achieve. Useful for debugging n-best - rescoring method. - """, - ) - - parser.add_argument( - "--num-paths", - type=int, - default=100, - help="""Number of paths for n-best based decoding method. - Used only when "method" is one of the following values: - nbest, nbest-rescoring, attention-decoder, and nbest-oracle - """, - ) - - parser.add_argument( - "--nbest-scale", - type=float, - default=0.5, - help="""The scale to be applied to `lattice.scores`. - It's needed if you use any kinds of n-best based rescoring. - Used only when "method" is one of the following values: - nbest, nbest-rescoring, attention-decoder, and nbest-oracle - A smaller value results in more unique paths. - """, - ) - - parser.add_argument( - "--export", - type=str2bool, - default=False, - help="""When enabled, the averaged model is saved to - conformer_ctc/exp/pretrained.pt. Note: only model.state_dict() is saved. - pretrained.pt contains a dict {"model": model.state_dict()}, - which can be loaded by `icefall.checkpoint.load_checkpoint()`. - """, - ) - - parser.add_argument( - "--exp-dir", - type=str, - default="conformer_lm/exp", - help="The experiment dir", - ) - - parser.add_argument( - "--lang-dir", - type=str, - default="data/lang_bpe_5000", - help="The lang dir", - ) - - return parser - - -def get_params() -> AttributeDict: - params = AttributeDict( - { - "lm_dir": Path("data/lm"), - "num_tokens": 5000, - "blank_sym": 0, - "bos_sym": 1, - "eos_sym": 1, - # parameters for conformer - "attention_dim": 512, - "nhead": 8, - "num_decoder_layers": 6, - # parameters for decoding - "search_beam": 20, - "output_beam": 8, - "min_active_states": 30, - "max_active_states": 10000, - "use_double_scores": True, - "env_info": get_env_info(), - } - ) - return params - - -def decode_one_batch( - params: AttributeDict, - model: nn.Module, - HLG: Optional[k2.Fsa], - H: Optional[k2.Fsa], - bpe_model: Optional[spm.SentencePieceProcessor], - batch: dict, - word_table: k2.SymbolTable, - sos_id: int, - eos_id: int, - G: Optional[k2.Fsa] = None, -) -> Dict[str, List[List[str]]]: - """Decode one batch and return the result in a dict. The dict has the - following format: - - - key: It indicates the setting used for decoding. For example, - if no rescoring is used, the key is the string `no_rescore`. - If LM rescoring is used, the key is the string `lm_scale_xxx`, - where `xxx` is the value of `lm_scale`. An example key is - `lm_scale_0.7` - - value: It contains the decoding result. `len(value)` equals to - batch size. `value[i]` is the decoding result for the i-th - utterance in the given batch. - Args: - params: - It's the return value of :func:`get_params`. - - - params.method is "1best", it uses 1best decoding without LM rescoring. - - params.method is "nbest", it uses nbest decoding without LM rescoring. - - params.method is "nbest-rescoring", it uses nbest LM rescoring. - - params.method is "whole-lattice-rescoring", it uses whole lattice LM - rescoring. - - model: - The neural model. - HLG: - The decoding graph. Used only when params.method is NOT ctc-decoding. - H: - The ctc topo. Used only when params.method is ctc-decoding. - bpe_model: - The BPE model. Used only when params.method is ctc-decoding. - batch: - It is the return value from iterating - `lhotse.dataset.K2SpeechRecognitionDataset`. See its documentation - for the format of the `batch`. - word_table: - The word symbol table. - sos_id: - The token ID of the SOS. - eos_id: - The token ID of the EOS. - G: - An LM. It is not None when params.method is "nbest-rescoring" - or "whole-lattice-rescoring". In general, the G in HLG - is a 3-gram LM, while this G is a 4-gram LM. - Returns: - Return the decoding result. See above description for the format of - the returned dict. - """ - if HLG is not None: - device = HLG.device - else: - device = H.device - feature = batch["inputs"] - assert feature.ndim == 3 - feature = feature.to(device) - # at entry, feature is (N, T, C) - - supervisions = batch["supervisions"] - - nnet_output, memory, memory_key_padding_mask = model(feature, supervisions) - # nnet_output is (N, T, C) - - supervision_segments = torch.stack( - ( - supervisions["sequence_idx"], - supervisions["start_frame"] // params.subsampling_factor, - supervisions["num_frames"] // params.subsampling_factor, - ), - 1, - ).to(torch.int32) - - if H is None: - assert HLG is not None - decoding_graph = HLG - else: - assert HLG is None - assert bpe_model is not None - decoding_graph = H - - lattice = get_lattice( - nnet_output=nnet_output, - decoding_graph=decoding_graph, - supervision_segments=supervision_segments, - search_beam=params.search_beam, - output_beam=params.output_beam, - min_active_states=params.min_active_states, - max_active_states=params.max_active_states, - subsampling_factor=params.subsampling_factor, - ) - - if params.method == "ctc-decoding": - best_path = one_best_decoding( - lattice=lattice, use_double_scores=params.use_double_scores - ) - # Note: `best_path.aux_labels` contains token IDs, not word IDs - # since we are using H, not HLG here. - # - # token_ids is a lit-of-list of IDs - token_ids = get_texts(best_path) - - # hyps is a list of str, e.g., ['xxx yyy zzz', ...] - hyps = bpe_model.decode(token_ids) - - # hyps is a list of list of str, e.g., [['xxx', 'yyy', 'zzz'], ... ] - hyps = [s.split() for s in hyps] - key = "ctc-decoding" - return {key: hyps} - - if params.method == "nbest-oracle": - # Note: You can also pass rescored lattices to it. - # We choose the HLG decoded lattice for speed reasons - # as HLG decoding is faster and the oracle WER - # is only slightly worse than that of rescored lattices. - best_path = nbest_oracle( - lattice=lattice, - num_paths=params.num_paths, - ref_texts=supervisions["text"], - word_table=word_table, - nbest_scale=params.nbest_scale, - oov="", - ) - hyps = get_texts(best_path) - hyps = [[word_table[i] for i in ids] for ids in hyps] - key = f"oracle_{params.num_paths}_nbest_scale_{params.nbest_scale}" # noqa - return {key: hyps} - - if params.method in ["1best", "nbest"]: - if params.method == "1best": - best_path = one_best_decoding( - lattice=lattice, use_double_scores=params.use_double_scores - ) - key = "no_rescore" - else: - best_path = nbest_decoding( - lattice=lattice, - num_paths=params.num_paths, - use_double_scores=params.use_double_scores, - nbest_scale=params.nbest_scale, - ) - key = f"no_rescore-nbest-scale-{params.nbest_scale}-{params.num_paths}" # noqa - - hyps = get_texts(best_path) - hyps = [[word_table[i] for i in ids] for ids in hyps] - return {key: hyps} - - assert params.method in [ - "nbest-rescoring", - "whole-lattice-rescoring", - "attention-decoder", - ] - - lm_scale_list = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7] - lm_scale_list += [0.8, 0.9, 1.0, 1.1, 1.2, 1.3] - lm_scale_list += [1.4, 1.5, 1.6, 1.7, 1.8, 1.9, 2.0] - - if params.method == "nbest-rescoring": - best_path_dict = rescore_with_n_best_list( - lattice=lattice, - G=G, - num_paths=params.num_paths, - lm_scale_list=lm_scale_list, - nbest_scale=params.nbest_scale, - ) - elif params.method == "whole-lattice-rescoring": - best_path_dict = rescore_with_whole_lattice( - lattice=lattice, - G_with_epsilon_loops=G, - lm_scale_list=lm_scale_list, - ) - elif params.method == "attention-decoder": - # lattice uses a 3-gram Lm. We rescore it with a 4-gram LM. - rescored_lattice = rescore_with_whole_lattice( - lattice=lattice, - G_with_epsilon_loops=G, - lm_scale_list=None, - ) - # TODO: pass `lattice` instead of `rescored_lattice` to - # `rescore_with_attention_decoder` - - best_path_dict = rescore_with_attention_decoder( - lattice=rescored_lattice, - num_paths=params.num_paths, - model=model, - memory=memory, - memory_key_padding_mask=memory_key_padding_mask, - sos_id=sos_id, - eos_id=eos_id, - nbest_scale=params.nbest_scale, - ) - else: - assert False, f"Unsupported decoding method: {params.method}" - - ans = dict() - if best_path_dict is not None: - for lm_scale_str, best_path in best_path_dict.items(): - hyps = get_texts(best_path) - hyps = [[word_table[i] for i in ids] for ids in hyps] - ans[lm_scale_str] = hyps - else: - for lm_scale in lm_scale_list: - ans[f"{lm_scale}"] = [[] * lattice.shape[0]] - return ans - - -def decode_dataset( - dl: torch.utils.data.DataLoader, - params: AttributeDict, - model: nn.Module, - HLG: Optional[k2.Fsa], - H: Optional[k2.Fsa], - bpe_model: Optional[spm.SentencePieceProcessor], - word_table: k2.SymbolTable, - sos_id: int, - eos_id: int, - G: Optional[k2.Fsa] = None, -) -> Dict[str, List[Tuple[List[str], List[str]]]]: - """Decode dataset. - - Args: - dl: - PyTorch's dataloader containing the dataset to decode. - params: - It is returned by :func:`get_params`. - model: - The neural model. - HLG: - The decoding graph. Used only when params.method is NOT ctc-decoding. - H: - The ctc topo. Used only when params.method is ctc-decoding. - bpe_model: - The BPE model. Used only when params.method is ctc-decoding. - word_table: - It is the word symbol table. - sos_id: - The token ID for SOS. - eos_id: - The token ID for EOS. - G: - An LM. It is not None when params.method is "nbest-rescoring" - or "whole-lattice-rescoring". In general, the G in HLG - is a 3-gram LM, while this G is a 4-gram LM. - Returns: - Return a dict, whose key may be "no-rescore" if no LM rescoring - is used, or it may be "lm_scale_0.7" if LM rescoring is used. - Its value is a list of tuples. Each tuple contains two elements: - The first is the reference transcript, and the second is the - predicted result. - """ - results = [] - - num_cuts = 0 - - try: - num_batches = len(dl) - except TypeError: - num_batches = "?" - - results = defaultdict(list) - for batch_idx, batch in enumerate(dl): - texts = batch["supervisions"]["text"] - - hyps_dict = decode_one_batch( - params=params, - model=model, - HLG=HLG, - H=H, - bpe_model=bpe_model, - batch=batch, - word_table=word_table, - G=G, - sos_id=sos_id, - eos_id=eos_id, - ) - - for lm_scale, hyps in hyps_dict.items(): - this_batch = [] - assert len(hyps) == len(texts) - for hyp_words, ref_text in zip(hyps, texts): - ref_words = ref_text.split() - this_batch.append((ref_words, hyp_words)) - - results[lm_scale].extend(this_batch) - - num_cuts += len(batch["supervisions"]["text"]) - - if batch_idx % 100 == 0: - batch_str = f"{batch_idx}/{num_batches}" - - logging.info( - f"batch {batch_str}, cuts processed until now is {num_cuts}" - ) - return results - - -def save_results( - params: AttributeDict, - test_set_name: str, - results_dict: Dict[str, List[Tuple[List[int], List[int]]]], -): - if params.method == "attention-decoder": - # Set it to False since there are too many logs. - enable_log = False - else: - enable_log = True - test_set_wers = dict() - for key, results in results_dict.items(): - recog_path = params.exp_dir / f"recogs-{test_set_name}-{key}.txt" - store_transcripts(filename=recog_path, texts=results) - if enable_log: - logging.info(f"The transcripts are stored in {recog_path}") - - # The following prints out WERs, per-word error statistics and aligned - # ref/hyp pairs. - errs_filename = params.exp_dir / f"errs-{test_set_name}-{key}.txt" - with open(errs_filename, "w") as f: - wer = write_error_stats( - f, f"{test_set_name}-{key}", results, enable_log=enable_log - ) - test_set_wers[key] = wer - - if enable_log: - logging.info( - "Wrote detailed error stats to {}".format(errs_filename) - ) - - test_set_wers = sorted(test_set_wers.items(), key=lambda x: x[1]) - errs_info = params.exp_dir / f"wer-summary-{test_set_name}.txt" - with open(errs_info, "w") as f: - print("settings\tWER", file=f) - for key, val in test_set_wers: - print("{}\t{}".format(key, val), file=f) - - s = "\nFor {}, WER of different settings are:\n".format(test_set_name) - note = "\tbest for {}".format(test_set_name) - for key, val in test_set_wers: - s += "{}\t{}{}\n".format(key, val, note) - note = "" - logging.info(s) - - -@torch.no_grad() -def main(): - parser = get_parser() - LibriSpeechAsrDataModule.add_arguments(parser) - args = parser.parse_args() - args.exp_dir = Path(args.exp_dir) - args.lang_dir = Path(args.lang_dir) - - params = get_params() - params.update(vars(args)) - - setup_logger(f"{params.exp_dir}/log-{params.method}/log-decode") - logging.info("Decoding started") - logging.info(params) - - lexicon = Lexicon(params.lang_dir) - max_token_id = max(lexicon.tokens) - num_classes = max_token_id + 1 # +1 for the blank - assert num_classes == params.num_tokens - - device = torch.device("cpu") - if torch.cuda.is_available(): - device = torch.device("cuda", 0) - - logging.info(f"device: {device}") - - graph_compiler = BpeCtcTrainingGraphCompiler( - params.lang_dir, - device=device, - sos_token="", - eos_token="", - ) - sos_id = graph_compiler.sos_id - eos_id = graph_compiler.eos_id - - if params.method == "ctc-decoding": - HLG = None - H = k2.ctc_topo( - max_token=max_token_id, - modified=False, - device=device, - ) - bpe_model = spm.SentencePieceProcessor() - bpe_model.load(str(params.lang_dir / "bpe.model")) - else: - H = None - bpe_model = None - HLG = k2.Fsa.from_dict( - torch.load(f"{params.lang_dir}/HLG.pt", map_location="cpu") - ) - HLG = HLG.to(device) - assert HLG.requires_grad is False - - if not hasattr(HLG, "lm_scores"): - HLG.lm_scores = HLG.scores.clone() - - if params.method in ( - "nbest-rescoring", - "whole-lattice-rescoring", - "attention-decoder", - ): - if not (params.lm_dir / "G_4_gram.pt").is_file(): - logging.info("Loading G_4_gram.fst.txt") - logging.warning("It may take 8 minutes.") - with open(params.lm_dir / "G_4_gram.fst.txt") as f: - first_word_disambig_id = lexicon.word_table["#0"] - - G = k2.Fsa.from_openfst(f.read(), acceptor=False) - # G.aux_labels is not needed in later computations, so - # remove it here. - del G.aux_labels - # CAUTION: The following line is crucial. - # Arcs entering the back-off state have label equal to #0. - # We have to change it to 0 here. - G.labels[G.labels >= first_word_disambig_id] = 0 - G = k2.Fsa.from_fsas([G]).to(device) - G = k2.arc_sort(G) - torch.save(G.as_dict(), params.lm_dir / "G_4_gram.pt") - else: - logging.info("Loading pre-compiled G_4_gram.pt") - d = torch.load(params.lm_dir / "G_4_gram.pt", map_location="cpu") - G = k2.Fsa.from_dict(d).to(device) - - if params.method in ["whole-lattice-rescoring", "attention-decoder"]: - # Add epsilon self-loops to G as we will compose - # it with the whole lattice later - G = k2.add_epsilon_self_loops(G) - G = k2.arc_sort(G) - G = G.to(device) - - # G.lm_scores is used to replace HLG.lm_scores during - # LM rescoring. - G.lm_scores = G.scores.clone() - else: - G = None - - model = MaskedLmConformer( - num_classes=params.num_tokens, - d_model=params.attention_dim, - nhead=params.nhead, - num_decoder_layers=params.num_decoder_layers, - ) - - if params.avg == 1: - load_checkpoint(f"{params.exp_dir}/epoch-{params.epoch}.pt", model) - else: - start = params.epoch - params.avg + 1 - filenames = [] - for i in range(start, params.epoch + 1): - if start >= 0: - filenames.append(f"{params.exp_dir}/epoch-{i}.pt") - logging.info(f"averaging {filenames}") - model.to(device) - model.load_state_dict(average_checkpoints(filenames, device=device)) - - if params.export: - logging.info(f"Export averaged model to {params.exp_dir}/pretrained.pt") - torch.save( - {"model": model.state_dict()}, f"{params.exp_dir}/pretrained.pt" - ) - return - print("TODO: Implement me!") - # [ ] Add an option to use conformer lm for rescoring - # [ ] Load conformer_lm only when that options is activated - # [ ] Load conformer model - return - - model.to(device) - model.eval() - num_param = sum([p.numel() for p in model.parameters()]) - logging.info(f"Number of model parameters: {num_param}") - - librispeech = LibriSpeechAsrDataModule(args) - # CAUTION: `test_sets` is for displaying only. - # If you want to skip test-clean, you have to skip - # it inside the for loop. That is, use - # - # if test_set == 'test-clean': continue - # - test_sets = ["test-clean", "test-other"] - for test_set, test_dl in zip(test_sets, librispeech.test_dataloaders()): - results_dict = decode_dataset( - dl=test_dl, - params=params, - model=model, - HLG=HLG, - H=H, - bpe_model=bpe_model, - word_table=lexicon.word_table, - G=G, - sos_id=sos_id, - eos_id=eos_id, - ) - - save_results( - params=params, test_set_name=test_set, results_dict=results_dict - ) - - logging.info("Done!") - - -torch.set_num_threads(1) -torch.set_num_interop_threads(1) - -if __name__ == "__main__": - main() diff --git a/icefall/decode.py b/icefall/decode.py index 62d27dd68..bc59a96d6 100644 --- a/icefall/decode.py +++ b/icefall/decode.py @@ -20,6 +20,11 @@ from typing import Dict, List, Optional, Union import k2 import torch +from icefall.lm.rescore import ( + compute_alignment, + make_hyp_to_ref_map, + prepare_conformer_lm_inputs, +) from icefall.utils import get_texts @@ -224,6 +229,7 @@ class Nbest(object): else: word_seq = lattice.aux_labels.index(path) word_seq = word_seq.remove_axis(word_seq.num_axes - 2) + word_seq = word_seq.remove_values_leq(0) # Each utterance has `num_paths` paths but some of them transduces # to the same word sequence, so we need to remove repeated word @@ -889,3 +895,198 @@ def rescore_with_attention_decoder( key = f"ngram_lm_scale_{n_scale}_attention_scale_{a_scale}" ans[key] = best_path return ans + + +def rescore_with_conformer_lm( + lattice: k2.Fsa, + num_paths: int, + model: torch.nn.Module, + masked_lm_model: torch.nn.Module, + memory: torch.Tensor, + memory_key_padding_mask: Optional[torch.Tensor], + sos_id: int, + eos_id: int, + blank_id: int, + nbest_scale: float = 1.0, + ngram_lm_scale: Optional[float] = None, + attention_scale: Optional[float] = None, + masked_lm_scale: Optional[float] = None, + use_double_scores: bool = True, +) -> Dict[str, k2.Fsa]: + """This function extracts `num_paths` paths from the given lattice and uses + an attention decoder to rescore them. The path with the highest score is + the decoding output. + + Args: + lattice: + An FsaVec with axes [utt][state][arc]. + num_paths: + Number of paths to extract from the given lattice for rescoring. + model: + A transformer model. See the class "Transformer" in + conformer_ctc/transformer.py for its interface. + memory: + The encoder memory of the given model. It is the output of + the last torch.nn.TransformerEncoder layer in the given model. + Its shape is `(T, N, C)`. + memory_key_padding_mask: + The padding mask for memory with shape `(N, T)`. + sos_id: + The token ID for SOS. + eos_id: + The token ID for EOS. + nbest_scale: + It's the scale applied to `lattice.scores`. A smaller value + leads to more unique paths at the risk of missing the correct path. + ngram_lm_scale: + Optional. It specifies the scale for n-gram LM scores. + attention_scale: + Optional. It specifies the scale for attention decoder scores. + masked_lm_scale: + Optional. It specifies the scale for conformer_lm scores. + Returns: + A dict of FsaVec, whose key contains a string + ngram_lm_scale_attention_scale and the value is the + best decoding path for each utterance in the lattice. + """ + nbest = Nbest.from_lattice( + lattice=lattice, + num_paths=num_paths, + use_double_scores=use_double_scores, + nbest_scale=nbest_scale, + ) + # nbest.fsa.scores are all 0s at this point + + nbest = nbest.intersect(lattice) + # Now nbest.fsa has its scores set. + # Also, nbest.fsa inherits the attributes from `lattice`. + assert hasattr(nbest.fsa, "lm_scores") + + am_scores = nbest.compute_am_scores() + ngram_lm_scores = nbest.compute_lm_scores() + + # The `tokens` attribute is set inside `compile_hlg.py` + assert hasattr(nbest.fsa, "tokens") + assert isinstance(nbest.fsa.tokens, torch.Tensor) + + path_to_utt_map = nbest.shape.row_ids(1).to(torch.long) + # the shape of memory is (T, N, C), so we use axis=1 here + expanded_memory = memory.index_select(1, path_to_utt_map) + + if memory_key_padding_mask is not None: + # The shape of memory_key_padding_mask is (N, T), so we + # use axis=0 here. + expanded_memory_key_padding_mask = memory_key_padding_mask.index_select( + 0, path_to_utt_map + ) + else: + expanded_memory_key_padding_mask = None + + # remove axis corresponding to states. + tokens_shape = nbest.fsa.arcs.shape().remove_axis(1) + tokens = k2.RaggedTensor(tokens_shape, nbest.fsa.tokens) + tokens = tokens.remove_values_leq(0) + + alignment = compute_alignment(tokens, nbest.shape) + ( + masked_src_symbols, + src_symbols, + tgt_symbols, + src_key_padding_mask, + tgt_weights, + ) = prepare_conformer_lm_inputs( + alignment, + bos_id=sos_id, + eos_id=eos_id, + blank_id=blank_id, + unmasked_weight=0.0, + ) + + masked_src_symbols = masked_src_symbols.to(torch.int64) + src_symbols = src_symbols.to(torch.int64) + tgt_symbols = tgt_symbols.to(torch.int64) + + masked_lm_memory, masked_lm_pos_emb = masked_lm_model( + masked_src_symbols, src_key_padding_mask + ) + + tgt_nll = masked_lm_model.decoder_nll( + masked_lm_memory, + masked_lm_pos_emb, + src_symbols, + tgt_symbols, + src_key_padding_mask, + ) + + # nll means negative log-likelihood + # ll means log-likelihood + tgt_ll = -1 * (tgt_nll * tgt_weights).sum(dim=-1) + + # Note: log-likelihood for those pairs that have identical src/tgt are 0 + # since their tgt_weights is 0 + + # TODO(fangjun): Add documentation about why we do the following + tgt_ll_shape_row_ids = make_hyp_to_ref_map(nbest.shape.row_splits(1)) + tgt_ll_shape = k2.ragged.create_ragged_shape2( + row_splits=None, + row_ids=tgt_ll_shape_row_ids, + cached_tot_size=tgt_ll_shape_row_ids.numel(), + ) + ragged_tgt_ll = k2.RaggedTensor(tgt_ll_shape, tgt_ll) + + ragged_tgt_ll = ragged_tgt_ll.remove_values_eq(0) + masked_lm_scores = ragged_tgt_ll.max() + + # TODO(fangjun): Support passing a ragged tensor to `decoder_nll` directly. + token_ids = tokens.tolist() + + nll = model.decoder_nll( + memory=expanded_memory, + memory_key_padding_mask=expanded_memory_key_padding_mask, + token_ids=token_ids, + sos_id=sos_id, + eos_id=eos_id, + ) + assert nll.ndim == 2 + assert nll.shape[0] == len(token_ids) + + attention_scores = -nll.sum(dim=1) + + if ngram_lm_scale is None: + ngram_lm_scale_list = [0.01, 0.05, 0.08] + ngram_lm_scale_list += [0.1, 0.3, 0.5, 0.6, 0.7, 0.9, 1.0] + ngram_lm_scale_list += [1.1, 1.2, 1.3, 1.5, 1.7, 1.9, 2.0] + else: + ngram_lm_scale_list = [ngram_lm_scale] + + if attention_scale is None: + attention_scale_list = [0.01, 0.05, 0.08] + attention_scale_list += [0.1, 0.3, 0.5, 0.6, 0.7, 0.9, 1.0] + attention_scale_list += [1.1, 1.2, 1.3, 1.5, 1.7, 1.9, 2.0] + else: + attention_scale_list = [attention_scale] + + if masked_lm_scale is None: + masked_lm_scale_list = [0.01, 0.05, 0.08] + masked_lm_scale_list += [0.1, 0.3, 0.5, 0.6, 0.7, 0.9, 1.0] + masked_lm_scale_list += [1.1, 1.2, 1.3, 1.5, 1.7, 1.9, 2.0] + else: + masked_lm_scale_list = [masked_lm_scale] + + ans = dict() + for n_scale in ngram_lm_scale_list: + for a_scale in attention_scale_list: + for m_scale in masked_lm_scale_list: + tot_scores = ( + am_scores.values + + n_scale * ngram_lm_scores.values + + a_scale * attention_scores + + m_scale * masked_lm_scores + ) + ragged_tot_scores = k2.RaggedTensor(nbest.shape, tot_scores) + max_indexes = ragged_tot_scores.argmax() + best_path = k2.index_fsa(nbest.fsa, max_indexes) + + key = f"ngram_lm_scale_{n_scale}_attention_scale_{a_scale}_masked_lm_scale_{m_scale}" # noqa + ans[key] = best_path + return ans diff --git a/icefall/lm/rescore.py b/icefall/lm/rescore.py index be2b3c929..e9984f5f6 100644 --- a/icefall/lm/rescore.py +++ b/icefall/lm/rescore.py @@ -32,6 +32,8 @@ We can generate the following inputs for the conformer LM model from `tokens`: - src - tgt by using `k2.levenshtein_alignment`. + +TODO(fangjun): Add more doc about rescoring with masked conformer-lm. """ from typing import Tuple @@ -39,8 +41,6 @@ from typing import Tuple import k2 import torch -from icefall.decode import Nbest - def make_key_padding_mask(lengths: torch.Tensor): """ @@ -236,7 +236,7 @@ def make_repeat(tokens: k2.RaggedTensor) -> k2.RaggedTensor: >>> tokens [ [ [ 1 2 3 ] [ 4 5 ] [ 9 ] ] [ [ 5 8 ] [ 10 1 ] ] ] >>> make_repeat(tokens) - [ [ [ 1 2 3 ] [ 4 5 ] [ 9 ] [ 1 2 3 ] [ 4 5 ] [ 9 ] [ 1 2 3 ] [ 4 5 ] [ 9 ] ] [ [ 5 8 ] [ 10 1 ] [ 5 8 ] [ 10 1 ] ] ] + [ [ [ 1 2 3 ] [ 4 5 ] [ 9 ] [ 1 2 3 ] [ 4 5 ] [ 9 ] [ 1 2 3 ] [ 4 5 ] [ 9 ] ] [ [ 5 8 ] [ 10 1 ] [ 5 8 ] [ 10 1 ] ] ] # noqa TODO: Add documentation. @@ -300,6 +300,7 @@ def prepare_conformer_lm_inputs( alignments: It is computed by :func:`compute_alignment` """ + device = alignment.device # alignment.arcs.shape has axes [fsa][state][arc] # we remove axis 1, i.e., state, here labels_shape = alignment.arcs.shape().remove_axis(1) @@ -337,6 +338,7 @@ def prepare_conformer_lm_inputs( (tgt_eos_pad.size(0), tgt_eos_pad.size(1) - 1), fill_value=1, dtype=torch.float32, + device=device, ) # find unmasked positions @@ -359,52 +361,3 @@ def prepare_conformer_lm_inputs( src_key_padding_mask, weight, ) - - -def conformer_lm_rescore( - nbest: Nbest, - model: torch.nn.Module, - bos_id: int, - eos_id: int, - blank_id: int, - unmasked_weight: float = 0.25, - # TODO: add other arguments if needed -) -> k2.RaggedTensor: - """Rescore an Nbest object with a conformer_lm model. - - Args: - nbest: - It contains linear FSAs to be rescored. - model: - A conformer lm model. See "conformer_lm/train.py" - - Returns: - Return a ragged tensor containing scores for each path - contained in the nbest. Its shape equals to `nbest.shape`. - """ - assert hasattr(nbest.fsa, "tokens") - utt_path_shape = nbest.shape - # nbest.fsa.arcs.shape() has axes [path][state][arc] - # We remove the state axis here - path_token_shape = nbest.fsa.arcs.shape().remove_axis(1) - - path_token = k2.RaggedTensor(path_token_shape, nbest.fsa.tokens) - path_token = path_token.remove_values_leq(0) - - alignment = compute_alignment(path_token, utt_path_shape) - ( - masked_src, - src, - tgt, - src_key_padding_mask, - weight, - ) = prepare_conformer_lm_inputs( - alignment, - bos_id=bos_id, - eos_id=eos_id, - blank_id=blank_id, - unmasked_weight=unmasked_weight, - ) - return masked_src, src, tgt, src_key_padding_mask, weight - # TODO: pass masked_src, src, tgt, src_key_padding_mask, and weight - # to the given model diff --git a/test/lm/test_rescore.py b/test/lm/test_rescore.py index bade45b5d..49016d1f5 100755 --- a/test/lm/test_rescore.py +++ b/test/lm/test_rescore.py @@ -17,12 +17,10 @@ import k2 import torch -from icefall.decode import Nbest from icefall.lm.rescore import ( add_bos, add_eos, compute_alignment, - conformer_lm_rescore, make_hyp_to_ref_map, make_repeat, make_repeat_map, @@ -45,6 +43,7 @@ def test_add_eos(): expected = k2.RaggedTensor( [[1, 2, eos_id], [3, eos_id], [eos_id], [5, 8, 9, eos_id]] ) + assert str(ragged_eos) == str(expected) def test_pad(): @@ -71,7 +70,7 @@ def test_make_hyp_to_ref_map(): repeat_map = make_hyp_to_ref_map(row_splits) # fmt: off expected = torch.tensor([0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, - 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6]).to(repeat_map) + 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6]).to(repeat_map) # noqa # fmt: on assert torch.all(torch.eq(repeat_map, expected)) @@ -82,8 +81,8 @@ def test_make_repeat_map(): repeat_map = make_repeat_map(row_splits) # fmt: off expected = torch.tensor([0, 1, 2, 0, 1, 2, 0, 1, 2, - 3, 4, 5, 6, 3, 4, 5, 6, 3, 4, 5, 6, - 3, 4, 5, 6]).to(repeat_map) + 3, 4, 5, 6, 3, 4, 5, 6, 3, 4, 5, 6, # noqa + 3, 4, 5, 6]).to(repeat_map) # noqa # fmt: on assert torch.all(torch.eq(repeat_map, expected)) @@ -132,27 +131,6 @@ def test_compute_alignment(): # print("weight", weight) -def test_conformer_lm_rescore(): - path00 = k2.linear_fsa([1, 2, 0, 3, 0, 5]) - path01 = k2.linear_fsa([1, 0, 5, 0]) - path10 = k2.linear_fsa([9, 8, 0, 3, 0, 2]) - path11 = k2.linear_fsa([9, 8, 0, 0, 3, 2]) - path12 = k2.linear_fsa([9, 0, 8, 4, 0, 2, 3]) - - fsa = k2.Fsa.from_fsas([path00, path01, path10, path11, path12]) - fsa.tokens = fsa.labels.clone() - shape = k2.RaggedShape("[[x x] [x x x]]") - nbest = Nbest(fsa, shape) - masked_src, src, tgt, src_key_padding_mask, weight = conformer_lm_rescore( - nbest, model=None, bos_id=10, eos_id=20, blank_id=0 - ) - print("masked src", masked_src) - print("src", src) - print("tgt", tgt) - print("src_key_padding_mask", src_key_padding_mask) - print("weight", weight) - - def main(): test_add_bos() test_add_eos() @@ -161,7 +139,6 @@ def main(): test_make_hyp_to_ref_map() test_make_repeat() test_compute_alignment() - test_conformer_lm_rescore() if __name__ == "__main__":