Transforms fixes + eval on multiple epochs (torch-points3d#233)

* Transforms fixes + eval on multiple epochs

* Adding some more tests

* Some missing changes

* Fix tests

Co-authored-by: chaton <thomas.chaton.ai@gmail.com>

Author: nicolas-chaulet

conf/eval.yaml

@@ -1,12 +1,13 @@
-num_workers: 2
-batch_size: 16
+num_workers: 6
+batch_size: 10
 cuda: 1
-weight_name: "latest" # Used during resume, select with model to load from [miou, macc, acc..., latest]
+weight_name: "latest"           # Used during resume, select with model to load from [miou, macc, acc..., latest]
 enable_cudnn: True
-checkpoint_dir: "" # "{your_path}/outputs/2020-01-28/11-04-13" for example
+checkpoint_dir: "/home/nicolas/deeppointcloud-benchmarks/outputs/2020-04-14/21-54-19" # "{your_path}/outputs/2020-01-28/11-04-13" for example
 model_name: KPConvPaper
-precompute_multi_scale: True # Compute multiscate features on cpu for faster training / inference
+precompute_multi_scale: True    # Compute multiscate features on cpu for faster training / inference
 enable_dropout: False
+voting_runs: 1
 
-
-
+tracker_options:                # Extra options for the tracker
+    full_res: True

conf/training/kpconv.yaml

@@ -1,9 +1,8 @@
 # Those arguments defines the training hyper-parameters
 training:
-    epochs: 200
-    num_workers: 10
-    batch_size: 10
-    shuffle: True
+    epochs: 550
+    num_workers: 8
+    batch_size: 8
     cuda: 1
     precompute_multi_scale: True # Compute multiscate features on cpu for faster training / inference
     optim:
@@ -33,7 +32,7 @@ wandb:
     project: s3dis
     log: True
     notes: "Fixed labels"
-    name: "kpconv-fixedlabels"
+    name: "kpconv-dynamicdataset-6"
     public: True # It will be display the model within wandb log, else not.
 
 

docs/src/api/transforms.rst

@@ -54,6 +54,6 @@ Transforms
 
 .. autofunction:: torch_points3d.core.data_transform.Random3AxisRotation
 
-.. autofunction:: src.core.data_transform.RandomCoordsFlip
+.. autofunction:: torch_points3d.core.data_transform.RandomCoordsFlip
 
 .. autofunction:: torch_points3d.core.data_transform.compute_planarity

eval.py

@@ -23,45 +23,79 @@
 log = logging.getLogger(__name__)
 
 
-def eval_epoch(model: BaseModel, dataset, device, tracker: BaseTracker, checkpoint: ModelCheckpoint):
+def eval_epoch(
+    model: BaseModel,
+    dataset,
+    device,
+    tracker: BaseTracker,
+    checkpoint: ModelCheckpoint,
+    voting_runs=1,
+    tracker_options={},
+):
     tracker.reset("val")
     loader = dataset.val_dataloader
-    with Ctq(loader) as tq_val_loader:
-        for data in tq_val_loader:
-            with torch.no_grad():
-                model.set_input(data, device)
-                model.forward()
+    for i in range(voting_runs):
+        with Ctq(loader) as tq_val_loader:
+            for data in tq_val_loader:
+                with torch.no_grad():
+                    model.set_input(data, device)
+                    model.forward()
 
-            tracker.track(model)
-            tq_val_loader.set_postfix(**tracker.get_metrics(), color=COLORS.VAL_COLOR)
+                tracker.track(model, **tracker_options)
+                tq_val_loader.set_postfix(**tracker.get_metrics(), color=COLORS.VAL_COLOR)
 
+    tracker.finalise(**tracker_options)
     tracker.print_summary()
 
 
-def test_epoch(model: BaseModel, dataset, device, tracker: BaseTracker, checkpoint: ModelCheckpoint):
+def test_epoch(
+    model: BaseModel,
+    dataset,
+    device,
+    tracker: BaseTracker,
+    checkpoint: ModelCheckpoint,
+    voting_runs=1,
+    tracker_options={},
+):
 
     loaders = dataset.test_dataloaders
 
     for loader in loaders:
         stage_name = loader.dataset.name
         tracker.reset(stage_name)
-        with Ctq(loader) as tq_test_loader:
-            for data in tq_test_loader:
-                with torch.no_grad():
-                    model.set_input(data, device)
-                    model.forward()
-
-                tracker.track(model)
-                tq_test_loader.set_postfix(**tracker.get_metrics(), color=COLORS.TEST_COLOR)
-
-            tracker.print_summary()
-
-
-def run(cfg, model, dataset: BaseDataset, device, tracker: BaseTracker, checkpoint: ModelCheckpoint):
+        for i in range(voting_runs):
+            with Ctq(loader) as tq_test_loader:
+                for data in tq_test_loader:
+                    with torch.no_grad():
+                        model.set_input(data, device)
+                        model.forward()
+
+                    tracker.track(model, **tracker_options)
+                    tq_test_loader.set_postfix(**tracker.get_metrics(), color=COLORS.TEST_COLOR)
+
+        tracker.finalise(**tracker_options)
+        tracker.print_summary()
+
+
+def run(
+    cfg,
+    model,
+    dataset: BaseDataset,
+    device,
+    tracker: BaseTracker,
+    checkpoint: ModelCheckpoint,
+    voting_runs=1,
+    tracker_options={},
+):
     if dataset.has_val_loader:
-        eval_epoch(model, dataset, device, tracker, checkpoint)
+        eval_epoch(
+            model, dataset, device, tracker, checkpoint, voting_runs=voting_runs, tracker_options=tracker_options
+        )
 
-    test_epoch(model, dataset, device, tracker, checkpoint)
+    if dataset.has_test_loaders:
+        test_epoch(
+            model, dataset, device, tracker, checkpoint, voting_runs=voting_runs, tracker_options=tracker_options
+        )
 
 
 @hydra.main(config_path="conf/eval.yaml")
@@ -98,7 +132,16 @@ def main(cfg):
     tracker: BaseTracker = dataset.get_tracker(model, dataset, False, False)
 
     # Run training / evaluation
-    run(cfg, model, dataset, device, tracker, checkpoint)
+    run(
+        cfg,
+        model,
+        dataset,
+        device,
+        tracker,
+        checkpoint,
+        voting_runs=cfg.voting_runs,
+        tracker_options=cfg.tracker_options,
+    )
 
 
 if __name__ == "__main__":

poetry.lock

@@ -35,7 +35,7 @@ markdown==3.2.1
 markupsafe==1.1.1
 matplotlib==3.2.1
 networkx==2.4
-numpy==1.18.2
+numpy==1.18.3
 nvidia-ml-py3==7.352.0
 oauthlib==3.1.0
 omegaconf==1.4.1

requirements.txt

@@ -30,6 +30,9 @@ def __init__(self):
         self._channels = [12, 12, 12, 12]
         self.nn = MLP(self._channels)
 
+    def set_input(self, a):
+        self.input = a
+
 
 class TestSimpleBatch(unittest.TestCase):
     def test_enable_dropout_eval(self):
@@ -46,5 +49,15 @@ def test_enable_dropout_eval(self):
             self.assertEqual(model.nn[i][2].training, False)
 
 
+class TestBaseModel(unittest.TestCase):
+    def test_getinput(self):
+        model = MockModel()
+        with self.assertRaises(AttributeError):
+            model.get_input()
+
+        model.set_input(1)
+        self.assertEqual(model.get_input(), 1)
+
+
 if __name__ == "__main__":
     unittest.main()

test/test_activate_dropout.py test/test_basemodel.py

@@ -70,7 +70,7 @@ def test_track(self):
             self.assertAlmostEqual(metrics[k], 0, 5)
 
     def test_ignore_label(self):
-        tracker = SegmentationTracker(MockDataset())
+        tracker = SegmentationTracker(MockDataset(), ignore_label=-100)
         tracker.reset("test")
         model = MockModel()
         model.iter = 3
@@ -79,6 +79,16 @@ def test_ignore_label(self):
         for k in ["test_acc", "test_miou", "test_macc"]:
             self.assertAlmostEqual(metrics[k], 100, 5)
 
+    def test_finalise(self):
+        tracker = SegmentationTracker(MockDataset(), ignore_label=-100)
+        tracker.reset("test")
+        model = MockModel()
+        model.iter = 3
+        tracker.track(model)
+        tracker.finalise()
+        with self.assertRaises(RuntimeError):
+            tracker.track(model)
+
 
 if __name__ == "__main__":
     unittest.main()

test/test_segmentationtracker.py

@@ -0,0 +1,59 @@
+import os
+import sys
+import unittest
+import numpy as np
+import torch
+from torch_geometric.data import Data
+
+ROOT = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..")
+sys.path.insert(0, ROOT)
+
+from torch_points3d.core.data_transform.transforms import RandomSphere, SphereSampling
+
+
+class TestRandomSphere(unittest.TestCase):
+    def setUp(self):
+
+        pos = np.array([[1, 0, 0], [0, 1, 0], [0, 0, 0], [0, 0, 1], [1, 1, 1], [0, 1, 1]])
+        labels = np.array([0, 0, 0, 0, 0, 0])
+
+        self.data = Data(pos=torch.from_numpy(pos).float(), labels=torch.from_numpy(labels))
+
+    def test_neighbour_found_under_random_sampling(self):
+        random_sphere = RandomSphere(0.1, strategy="RANDOM")
+        data = random_sphere(self.data.clone())
+        assert data.labels.shape[0] == 1
+
+        random_sphere = RandomSphere(3, strategy="RANDOM")
+        data = random_sphere(self.data.clone())
+        assert data.labels.shape[0] == 6
+
+
+class TestSphereSampling(unittest.TestCase):
+    def setUp(self):
+        pos = torch.tensor([[1, 0, 0], [0, 1, 0], [0, 0, 0], [0, 0, 1], [1, 1, 1], [0, 1, 1]])
+        labels = torch.tensor([0, 1, 2, 0, 0, 0])
+        self.data = Data(pos=pos.float(), labels=labels)
+
+    def test_sphere(self):
+        sphere_sampling = SphereSampling(0.1, [0, 0, 0])
+        sampled = sphere_sampling(self.data)
+
+        self.assertIn(SphereSampling.KDTREE_KEY, self.data)
+        self.assertEqual(len(sampled.labels), 1)
+        self.assertEqual(sampled.labels[0], 2)
+
+    def test_align(self):
+        sphere_sampling = SphereSampling(0.1, [1, 0, 0])
+        sampled = sphere_sampling(self.data)
+        torch.testing.assert_allclose(sampled.pos, torch.tensor([[0.0, 0, 0]]))
+        self.assertEqual(sampled.labels[0], 0)
+
+        sphere_sampling = SphereSampling(0.1, [1, 0, 0], align_origin=False)
+        sampled = sphere_sampling(self.data)
+        torch.testing.assert_allclose(sampled.pos, torch.tensor([[1.0, 0, 0]]))
+        self.assertEqual(sampled.labels[0], 0)
+
+
+if __name__ == "__main__":
+    unittest.main()