Pytorch中的gradient_accumulate_steps、warmup、lr_decay、optimizer和scheduler等问题的解答

# 该函数的实现包括了warmup和lr_decay
def warmup_linear(x, warmup = 0.002):
    if x < warmup:
        return x/warmup
    return 1.0 - x
 
global_step_th = int(len(train_examples)/batch_size/gradient_accumulation_steps * start_epoch)
 
for epoch in range(start_epoch, total_train_epoch):
    train_loss = 0
    train_start = time.time()
    model.train()
    optimizer.zero_grad()
    for step, batch in enumerate(train_dataloader):
        create input and output for batch object_loss
        # loss regularization
        if gradient_accumulation_steps > 1:
            object_loss = object_loss / gradient_accumulation_steps
 
        # Implementation of backpropagation
        object_loss.backward()
        train_loss = train_loss + object_loss.item()
        if (step+1) % gradient_accumulation_steps == 0:
            # modifying and update the learning rate with warm up which bert uses.
            lr_this_step = learning_rate*warmup_linear(global_step_th/total_train_steps)
            # for the params in optimizer, we update the learning rate
            for param_group in optimizer.param_groups:
                param_group['lr'] = lr_this_step
            optimizer.step()
            optimizer.zero_grad()
            global_step_th = global_step_th + 1

手动调节学习率：（需要将改变后的学习率手动更新到模型中）
EXAMPLE.1(该样例的实现包含了gradient_accumulate)
# 该函数的实现包括了warmup和lr_decay
def warmup_linear(x, warmup = 0.002):
    if x < warmup:
        return x/warmup
    return 1.0 - x
 
for epoch in range(start_epoch, total_train_epoch):
    train_loss = 0
    train_start = time.time()
    model.train()
    optimizer.zero_grad()
    for step, batch in enumerate(train_dataloader):
        create input and output for batch object_loss
        # loss regularization
        if gradient_accumulation_steps > 1:
            object_loss = object_loss / gradient_accumulation_steps
 
        # Implementation of backpropagation
        object_loss.backward()
        train_loss = train_loss + object_loss.item()
        if (step+1) % gradient_accumulation_steps == 0:
            # modifying and update the learning rate with warm up which bert uses.
            lr_this_step = learning_rate*warmup_linear(global_step_th/total_train_steps)
            # for the params in optimizer, we update the learning rate
            for param_group in optimizer.param_groups:
                param_group['lr'] = lr_this_step
            optimizer.step()
            optimizer.zero_grad()
            global_step_th = global_step_th + 1
 
 
EXAMPLE.2 Facebook提出的gradual warmup
import numpy as np
warmup_steps = 2500
init_lr = 0.1  
# 模拟训练15000步
max_steps = 15000
for train_steps in range(max_steps):
    # 实现warmup
　　　　if warmup_steps and train_steps < warmup_steps:
        warmup_percent_done = train_steps / warmup_steps
        warmup_learning_rate = init_lr * warmup_percent_done  #gradual warmup_lr
        learning_rate = warmup_learning_rate
    # 实现lr_decay
    else:
        #learning_rate = np.sin(learning_rate)  #预热学习率结束后,学习率呈sin衰减
        learning_rate = learning_rate**1.0001 #预热学习率结束后,学习率呈指数衰减(近似模拟指数衰减)

库函数调节：
常见的学习率调节库函数：
torch.optim.lr_scheduler.StepLR(optimizer, step_size, gamma=0.1, last_epoch=-1)
torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones, gamma=0.1, last_epoch=-1)
torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma, last_epoch=-1)
torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max, eta_min=0, last_epoch=-1)
torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.1, patience=10, verbose=False, threshold=0.0001, threshold_mode='rel', cooldown=0, min_lr=0, eps=1e-08)
torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda, last_epoch=-1)
 
 
EXAMPLE.1(该代码仅实现了lr_decay)
optimizer = torch.optim.SGD(model.parameters(), lr=0.1) # 学习率初值0.1
scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=10, gamma=0.5) # 每十次迭代，学习率减半
for i in range(1,100):
    scheduler.step() # 学习率迭代次数+1
    arr.append(optimizer.state_dict()['param_groups'][0]['lr'])
    #arr.append(scheduler.get_lr())  # 与前一句功能相同，都是为了获取学习率的数值
 
 
EXAMPLE.2(该代码仅实现了lr_decay)
lambdaf = lambda epoch: 0.05 + (epoch) / 100
scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lambdaf)
for i in range(1,100):
    scheduler.step()# 学习率迭代次数+1,同时将迭代次数作为参数传给lambdaf
 
 
EXAMPLE.3(该代码仅实现了warmup)
def warmup_lr_scheduler(optimizer, warmup_iters, warmup_factor):
    def f(x): # x是step次数
        if x >= warmup_iters:
            return 1
        alpha = float(x) / warmup_iters # 当前进度 0-1
        return warmup_factor * (1 - alpha) + alpha
    return torch.optim.lr_scheduler.LambdaLR(optimizer, f)
 
if warmup:
    warmup_factor = 1. / 1000
    warmup_iters = min(1000, len(train_loader) - 1)
    lr_scheduler = warmup_lr_scheduler(optimizer, warmup_iters, warmup_factor)
for i in range(1,100):
    lr_scheduler.step() #将学习率迭代次数+1传给f
 
 
EXAMPLE.4(该代码实现warmup和lr_decay)
optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9, weight_decay=5e-4)
warmup_epoch = 5
scheduler = CosineAnnealingLR(optimizer, 100 - warmup_epoch)
warmup_scheduler = WarmUpLR(optimizer, iter_per_epoch * warmup_epoch)
for epoch in range(1, max_epoch+1):
        # lr_decay过程
        if epoch >= warmup_epoch:
            scheduler.step()
            learn_rate = scheduler.get_lr()[0]
            print("Learn_rate:%s" % learn_rate)
	# warmup过程
　　　　　　　　else:
            warmup_scheduler.step()
            warm_lr = warmup_scheduler.get_lr()
            print("warm_lr:%s" % warm_lr)
 
 
EXAMPLE.5(transformers.get_linear_schedule_with_warmup(),实现学习率预热和学习率下降)
total_steps = len(train_dataloader) * num_epoch
optimizer = AdamW(optimizer_grouped_parameters, lr=3e-5, eps=1e-8)
lr_scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps= warmup_rate* total_steps, num_training_steps = total_steps)
for __ in trange(num_epoch, desc = 'Epoch'):
    model.train()
    total_loss = 0
    for batch_iter, batch_dataloader in enumerate(train_dataloader):
        ......
        loss = outputs.loss
        loss.backward()
        total_loss = total_loss + loss.cpu().item() # item() return the value of tensor
        # update the parameters
        optimizer.step()
        # update the learning rate
        lr_scheduler.step()

# 将参数(权值)梯度全部置为０
optimizer.zero_grad()
# 通过输入，计算模型的输出
outputs = net(inputs)
# 计算损失
loss = criterion(outputs, labels)
# 通过损失，反向传播计算权值梯度
loss.backward()
# 更新权值
optimizer.step()