# Copyright 2021-2023 Xiaomi Corp. (authors: Xiaoyu Yang, # # See ../../../../LICENSE for clarification regarding multiple authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from typing import Tuple import torch import torch.nn as nn from encoder_interface import EncoderInterface from icefall.utils import make_pad_mask class AudioTaggingModel(nn.Module): def __init__( self, encoder_embed: nn.Module, encoder: EncoderInterface, encoder_dim: int = 384, num_events: int = 527, ): """An audio tagging model Args: encoder_embed: It is a Convolutional 2D subsampling module. It converts an input of shape (N, T, idim) to an output of of shape (N, T', odim), where T' = (T-3)//2-2 = (T-7)//2. encoder: It is the transcription network in the paper. Its accepts two inputs: `x` of (N, T, encoder_dim) and `x_lens` of shape (N,). It returns two tensors: `logits` of shape (N, T, encoder_dim) and `logit_lens` of shape (N,). encoder_dim: Dimension of the encoder. num_event: The number of classes. """ super().__init__() assert isinstance(encoder, EncoderInterface), type(encoder) self.encoder_embed = encoder_embed self.encoder = encoder self.encoder_dim = encoder_dim self.classifier = nn.Sequential( nn.Dropout(0.1), nn.Linear(encoder_dim, num_events), ) # for multi-class classification self.criterion = torch.nn.BCEWithLogitsLoss(reduction="sum") def forward_encoder( self, x: torch.Tensor, x_lens: torch.Tensor, ) -> Tuple[torch.Tensor, torch.Tensor]: """Compute encoder outputs. Args: x: A 3-D tensor of shape (N, T, C). x_lens: A 1-D tensor of shape (N,). It contains the number of frames in `x` before padding. Returns: encoder_out: Encoder output, of shape (N, T, C). encoder_out_lens: Encoder output lengths, of shape (N,). """ # logging.info(f"Memory allocated at entry: {torch.cuda.memory_allocated() // 1000000}M") x, x_lens = self.encoder_embed(x, x_lens) # logging.info(f"Memory allocated after encoder_embed: {torch.cuda.memory_allocated() // 1000000}M") src_key_padding_mask = make_pad_mask(x_lens) x = x.permute(1, 0, 2) # (N, T, C) -> (T, N, C) encoder_out, encoder_out_lens = self.encoder(x, x_lens, src_key_padding_mask) encoder_out = encoder_out.permute(1, 0, 2) # (T, N, C) ->(N, T, C) assert torch.all(encoder_out_lens > 0), (x_lens, encoder_out_lens) return encoder_out, encoder_out_lens def forward( self, x: torch.Tensor, x_lens: torch.Tensor, target: torch.Tensor, ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: """ Args: x: A 3-D tensor of shape (N, T, C). x_lens: A 1-D tensor of shape (N,). It contains the number of frames in `x` before padding. target: The ground truth label of audio events, could be many hot Returns: Return the binary crossentropy loss """ assert x.ndim == 3, x.shape assert x_lens.ndim == 1, x_lens.shape # Compute encoder outputs encoder_out, encoder_out_lens = self.forward_encoder(x, x_lens) # Forward the speaker module logits = self.forward_audio_tagging( encoder_out=encoder_out, encoder_out_lens=encoder_out_lens ) # (N, num_classes) loss = self.criterion(logits, target) return loss def forward_audio_tagging(self, encoder_out, encoder_out_lens): """ Args: encoder_out: A 3-D tensor of shape (N, T, C). encoder_out_lens: A 1-D tensor of shape (N,). It contains the number of frames in `x` before padding. Returns: A 3-D tensor of shape (N, num_classes). """ logits = self.classifier(encoder_out) # (N, T, num_classes) padding_mask = make_pad_mask(encoder_out_lens) logits[padding_mask] = 0 logits = logits.sum(dim=1) # mask the padding frames logits = logits / (~padding_mask).sum(dim=1).unsqueeze(-1).expand_as( logits ) # normalize the logits return logits