update 1 on comments

Author: tchaton

conf/config.yaml

@@ -6,7 +6,7 @@ defaults:
   - dataset: ???
     optional: True
 
-  - scheduler: lambda_rule
+  - lr_scheduler: multi_step
   - training: default
   - eval
 
@@ -17,4 +17,5 @@ defaults:
 
 model_name: ??? # Name of the specific model to load
 
+selection_stage: "test"
 pretty_print: False

conf/lr_scheduler/cosine.yaml

@@ -0,0 +1,4 @@
+lr_scheduler:
+  class: CosineAnnealingLR
+  params: 
+    T_max: 10

conf/lr_scheduler/cyclic.yaml

@@ -0,0 +1,5 @@
+lr_scheduler:
+  class: CyclicLR
+  params:
+    base_lr: ${training.optim.base_lr}
+    max_lr: 0.1

conf/lr_scheduler/exponential.yaml

@@ -0,0 +1,4 @@
+lr_scheduler:
+  class: ExponentialLR
+  params:
+    gamma: 0.1

conf/lr_scheduler/multi_step.yaml

@@ -0,0 +1,5 @@
+lr_scheduler:
+  class: MultiStepLR
+  params:
+    milestones: [10, 25, 50, 75, 100]
+    gamma: 0.1

conf/lr_scheduler/plateau.yaml

@@ -0,0 +1,4 @@
+lr_scheduler:
+  class: ReduceLROnPlateau
+    params: 
+      mode: "min"

conf/lr_scheduler/step.yaml

@@ -0,0 +1,5 @@
+lr_scheduler:
+  class: StepLR
+  params:
+    step_size: 10
+    gamma: 0.1

conf/scheduler/cosine.yaml

@@ -1,26 +1,26 @@
 # Those arguments defines the training hyper-parameters
 training:
-    shuffle: True
+    epochs: 100
     num_workers: 6
     batch_size: 16
+    shuffle: True
     cuda: 1
     precompute_multi_scale: False # Compute multiscate features on cpu for faster training / inference
-    epochs: 100
-    grad_clip: -1
-    base_lr: 0.001
-    accumulated_gradient: 10 # Accumulate gradient accumulated_gradient * batch_size
     optim:
+        base_lr: 0.001
+        accumulated_gradient: 10 # Accumulate gradient accumulated_gradient * batch_size
+        grad_clip: -1
         optimizer:
             class: Adam
             params:
-                lr: ${base_lr} # The path is cut from training
-        scheduler: ${scheduler}
+                lr: ${training.optim.base_lr} # The path is cut from training
+        scheduler: ${lr_scheduler}
         bn_scheduler:
             bn_policy: "step_decay"
             params:
-                bn_momentum: 0.9
-                bn_decay: 0.5
-                decay_step : 200000
+                bn_momentum: 0.1
+                bn_decay: 0.9
+                decay_step : 1024
                 bn_clip : 1e-2
     weight_name: "latest" # Used during resume, select with model to load from [miou, macc, acc..., latest]
     enable_cudnn: True
@@ -34,6 +34,7 @@ wandb:
     log: False
     notes:
     name:
+    public: True # It will be display the model within wandb log, else not.
 
     # parameters for TensorBoard Visualization
 tensorboard:

conf/scheduler/lambda_rule.yaml

@@ -1,26 +1,25 @@
 # Those arguments defines the training hyper-parameters
 training:
-    shuffle: True
+    epochs: 100
     num_workers: 4
     batch_size: 16
-    accumulated_gradient: 4 # Accumulate gradient accumulated_gradient * batch_size
+    shuffle: True
     cuda: 1
-    precompute_multi_scale: True # Compute multiscate features on cpu for faster training / inference
-    epochs: 100
-    grad_clip: -1
-    base_lr: 0.01
+    precompute_multi_scale: False # Compute multiscate features on cpu for faster training / inference
     optim:
+        base_lr:  0.01
+        grad_clip: -1
         optimizer:
             class: Adam
             params:
-                lr: ${base_lr} # The path is cut from training
-        scheduler: ${scheduler}
+                lr: ${training.optim.base_lr} # The path is cut from training
+        scheduler: ${lr_scheduler}
         bn_scheduler:
             bn_policy: "step_decay"
             params:
-                bn_momentum: 0.9
-                bn_decay: 0.5
-                decay_step : 200000
+                bn_momentum: 0.1
+                bn_decay: 0.9
+                decay_step : 1024
                 bn_clip : 1e-2
     weight_name: "latest" # Used during resume, select with model to load from [miou, macc, acc..., latest]
     enable_cudnn: True
@@ -34,6 +33,8 @@ wandb:
     log: True
     notes: "Multihead"
     name: "kpconv-multihead"
+    public: True # It will be display the model within wandb log, else not.
+
 
 # parameters for TensorBoard Visualization
 tensorboard:

conf/scheduler/plateau.yaml

@@ -68,7 +68,7 @@ def instantiate_bn_scheduler(model, bn_scheduler_opt, batch_size):
     Parameters:
         model          -- the nn network
         bn_scheduler_opt (option class) -- dict containing all the params to build the scheduler　
-                              opt.lr_policy is the name of learning rate policy: lambda_rule | step | plateau | cosine
+                              opt.bn_policy is the name of learning rate policy: lambda_rule | step | plateau | cosine
                               opt.params contains the scheduler_params to construct the scheduler
     See https://pytorch.org/docs/stable/optim.html for more details.
     """
@@ -79,13 +79,15 @@ def instantiate_bn_scheduler(model, bn_scheduler_opt, batch_size):
             * bn_scheduler_params.bn_decay ** (int(it * batch_size / bn_scheduler_params.decay_step)),
             bn_scheduler_params.bn_clip,
         )
+
     elif bn_scheduler_opt.bn_policy == "exponential_decay":
         bn_lambda = lambda e: max(
             eval(bn_scheduler_params.gamma) ** (e / bn_scheduler_params.decay_step),
             bn_scheduler_params.bn_clip / bn_scheduler_params.bn_momentum,
         )
+
     else:
-        return NotImplementedError("learning rate policy [%s] is not implemented", bn_scheduler_opt.lr_policy)
+        return NotImplementedError("bn_policy [%s] is not implemented", bn_scheduler_opt.bn_policy)
 
     bn_scheduler = BNMomentumScheduler(model, bn_lambda)
 

conf/training/default.yaml

@@ -1,17 +1,17 @@
 from torch.optim import lr_scheduler
-from functools import partial
 from omegaconf.dictconfig import DictConfig
 import logging
 from src.utils.config import merge_omega_conf
 
 
 log = logging.getLogger(__name__)
 
+
 def repr(self, scheduler_params={}):
     return "{}({})".format(self.__class__.__name__, scheduler_params)
 
-class SchedulerWrapper():
 
+class LRScheduler:
     def __init__(self, scheduler, scheduler_params):
         self._scheduler = scheduler
         self._scheduler_params = scheduler_params
@@ -22,7 +22,7 @@ def scheduler(self):
 
     @property
     def scheduler_opt(self):
-        return  self._scheduler._scheduler_opt
+        return self._scheduler._scheduler_opt
 
     def __repr__(self):
         return "{}({})".format(self._scheduler.__class__.__name__, self._scheduler_params)
@@ -36,6 +36,7 @@ def state_dict(self):
     def load_state_dict(self, state_dict):
         self._scheduler.load_state_dict(state_dict)
 
+
 def instantiate_scheduler(optimizer, scheduler_opt):
     """Return a learning rate scheduler
     Parameters:
@@ -45,37 +46,15 @@ def instantiate_scheduler(optimizer, scheduler_opt):
                               opt.params contains the scheduler_params to construct the scheduler
     See https://pytorch.org/docs/stable/optim.html for more details.
     """
-    base_lr = optimizer.defaults['lr']
+
+    scheduler_cls_name = getattr(scheduler_opt, "class")
+    scheduler_cls = getattr(lr_scheduler, scheduler_cls_name)
     scheduler_params = scheduler_opt.params
-    if scheduler_opt.lr_policy == 'lambda_rule':
-        if scheduler_opt.rule == "step_decay":
-            lr_lambda = lambda e: max(
-                scheduler_params.lr_decay ** (e // scheduler_params.decay_step),
-                scheduler_params.lr_clip / base_lr,
-            )
-        elif scheduler_opt.rule == "exponential_decay":
-            lr_lambda = lambda e: max(
-                eval(scheduler_params.gamma) ** (e / scheduler_params.decay_step),
-                scheduler_params.lr_clip / base_lr,
-            )            
-        else:
-            raise NotImplementedError
-        scheduler = lr_scheduler.LambdaLR(optimizer, lr_lambda=lr_lambda)
-    
-    elif scheduler_opt.lr_policy == 'step':
-        scheduler = lr_scheduler.StepLR(optimizer, **scheduler_params)
-    
-    elif scheduler_opt.lr_policy == 'plateau':
-        scheduler = lr_scheduler.ReduceLROnPlateau(optimizer, **scheduler_params)
-        scheduler_params = merge_omega_conf(scheduler_params, {"metric_name": scheduler_opt.metric_name})
-        setattr(scheduler, "metric_name", scheduler_opt.metric_name)
-
-    elif scheduler_opt.lr_policy == 'cosine':
-        scheduler = lr_scheduler.CosineAnnealingLR(optimizer, **scheduler_params)
-    else:
-        return NotImplementedError('learning rate policy [%s] is not implemented', scheduler_opt.lr_policy)
-    
+
+    if scheduler_cls_name.lower() == "ReduceLROnPlateau".lower():
+        raise NotImplementedError("This scheduler is not fully supported yet")
+
+    scheduler = scheduler_cls(optimizer, **scheduler_params)
     # used to re_create the scheduler
     setattr(scheduler, "_scheduler_opt", scheduler_opt)
-
-    return SchedulerWrapper(scheduler, scheduler_params)
+    return LRScheduler(scheduler, scheduler_params)

conf/training/kpconv.yaml

@@ -28,8 +28,6 @@ def get_model_checkpoint(
 ):
     """ Loads a model from a checkpoint or creates a new one.
     """
-    model.set_selection_stage(selection_stage)
-
     model_checkpoint: ModelCheckpoint = ModelCheckpoint(load_dir, check_name, resume, selection_stage)
 
     if resume:
@@ -55,7 +53,6 @@ def _initialize_objects(self):
         self._objects["model_state"] = None
         self._objects["stats"] = {"train": [], "test": [], "val": []}
         self._objects["optimizer"] = None
-        self._objects["lr_params"] = None
         self._filled = False
 
     def save_objects(self, models_to_save, model_state, stage, current_stat, optimizer, schedulers, **kwargs):