perf(train): 训练之前先step一次，保证学习率从最小值开始

Author: zjZSTU

py/densenet/train.py

@@ -135,8 +135,8 @@ def train_model(data_loaders, data_sizes, model_name, model, criterion, optimize
                 running_loss += loss.item() * inputs.size(0)
                 # running_corrects += torch.sum(preds == labels.data)
             if phase == 'train':
-                lr_scheduler.step()
                 print('lr: {}'.format(optimizer.param_groups[0]['lr']))
+                lr_scheduler.step()
 
             epoch_loss = running_loss / data_sizes[phase]
             epoch_top1_acc = running_top1_acc / len(data_loaders[phase])
@@ -200,6 +200,9 @@ def train_model(data_loaders, data_sizes, model_name, model, criterion, optimize
         optimizer = optim.Adam(model.parameters(), lr=3e-4, weight_decay=3e-5)
         scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, num_epochs - 5, eta_min=0)
         lr_scheduler = GradualWarmupScheduler(optimizer, multiplier=1, total_epoch=5, after_scheduler=scheduler)
+        optimizer.zero_grad()
+        optimizer.step()
+        lr_scheduler.step()
 
         util.check_dir('../data/models/')
         best_model, loss_dict, top1_acc_dict, top5_acc_dict = train_model(

py/resnet/train_resnet.py

@@ -140,8 +140,8 @@ def train_model(data_loaders, data_sizes, model_name, model, criterion, optimize
                 running_loss += loss.item() * inputs.size(0)
                 # running_corrects += torch.sum(preds == labels.data)
             if phase == 'train':
-                lr_scheduler.step(epoch + 1)
-                print('lr: {}'.format(lr_scheduler.get_lr()))
+                print('lr: {}'.format(optimizer.param_groups[0]['lr']))
+                lr_scheduler.step()
 
             epoch_loss = running_loss / data_sizes[phase]
             epoch_top1_acc = running_top1_acc / len(data_loaders[phase])
@@ -207,6 +207,9 @@ def train_model(data_loaders, data_sizes, model_name, model, criterion, optimize
         optimizer = optim.Adam(model.parameters(), lr=1e-3, weight_decay=1e-4)
         scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, num_epochs - 5, eta_min=1e-4)
         lr_scheduler = GradualWarmupScheduler(optimizer, multiplier=1, total_epoch=5, after_scheduler=scheduler)
+        optimizer.zero_grad()
+        optimizer.step()
+        lr_scheduler.step()
 
         util.check_dir('../data/models/')
         best_model, loss_dict, top1_acc_dict, top5_acc_dict = train_model(

py/resnet/train_resnet_v2.py

@@ -135,8 +135,8 @@ def train_model(data_loaders, data_sizes, model_name, model, criterion, optimize
                 running_loss += loss.item() * inputs.size(0)
                 # running_corrects += torch.sum(preds == labels.data)
             if phase == 'train':
-                lr_scheduler.step()
                 print('lr: {}'.format(optimizer.param_groups[0]['lr']))
+                lr_scheduler.step()
 
             epoch_loss = running_loss / data_sizes[phase]
             epoch_top1_acc = running_top1_acc / len(data_loaders[phase])
@@ -200,6 +200,9 @@ def train_model(data_loaders, data_sizes, model_name, model, criterion, optimize
         optimizer = optim.Adam(model.parameters(), lr=3e-4, weight_decay=3e-5)
         scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, num_epochs - 5, eta_min=0)
         lr_scheduler = GradualWarmupScheduler(optimizer, multiplier=1, total_epoch=5, after_scheduler=scheduler)
+        optimizer.zero_grad()
+        optimizer.step()
+        lr_scheduler.step()
 
         util.check_dir('../data/models/')
         best_model, loss_dict, top1_acc_dict, top5_acc_dict = train_model(