Add baseline for the PeLU expt, keeping only the small normalization-related changes.

This commit is contained in:
Daniel Povey 2022-03-02 18:27:08 +08:00
parent 9ed7d55a84
commit 3fb559d2f0
3 changed files with 6 additions and 6 deletions

View File

@ -47,11 +47,11 @@ class Conv2dSubsampling(nn.Module):
nn.Conv2d(
in_channels=1, out_channels=odim, kernel_size=3, stride=2
),
PeLU(cutoff=-1.0),
nn.ReLU(),
nn.Conv2d(
in_channels=odim, out_channels=odim, kernel_size=3, stride=2
),
PeLU(cutoff=-5.0),
nn.ReLU(),
)
self.out = nn.Linear(odim * (((idim - 1) // 2 - 1) // 2), odim)
self.out_norm = nn.LayerNorm(odim, elementwise_affine=False)

View File

@ -156,14 +156,14 @@ class ConformerEncoderLayer(nn.Module):
self.feed_forward = nn.Sequential(
nn.Linear(d_model, dim_feedforward),
PeLU(),
Swish(),
nn.Dropout(dropout),
nn.Linear(dim_feedforward, d_model),
)
self.feed_forward_macaron = nn.Sequential(
nn.Linear(d_model, dim_feedforward),
PeLU(),
Swish(),
nn.Dropout(dropout),
nn.Linear(dim_feedforward, d_model),
)
@ -882,7 +882,7 @@ class ConvolutionModule(nn.Module):
padding=0,
bias=bias,
)
self.activation = PeLU()
self.activation = Swish()
def forward(self, x: Tensor) -> Tensor:
"""Compute convolution module.

View File

@ -110,7 +110,7 @@ def get_parser():
parser.add_argument(
"--exp-dir",
type=str,
default="transducer_stateless/specaugmod_baseline_randcombine1_pelu",
default="transducer_stateless/specaugmod_baseline_randcombine1_pelu_base",
help="""The experiment dir.
It specifies the directory where all training related
files, e.g., checkpoints, log, etc, are saved