Update train.py

This commit is contained in:
zr_jin 2024-11-29 14:33:34 +08:00
parent 18fa6a0fec
commit 732737ece2

View File

@ -974,7 +974,16 @@ def run(rank, world_size, args):
logging.info("Using DDP")
model = DDP(model, device_ids=[rank], find_unused_parameters=True)
optimizer = ScaledAdam(model.parameters(), lr=params.base_lr, clipping_scale=2.0)
parameters_names = []
parameters_names.append(
[name_param_pair[0] for name_param_pair in model.named_parameters()]
)
optimizer = ScaledAdam(
model.parameters(),
lr=params.base_lr,
clipping_scale=2.0,
parameters_names=parameters_names,
)
scheduler = Eden(optimizer, params.lr_batches, params.lr_epochs)