[benchmark] Parse setup time from verbose log

Author: palimondo

Diff for: benchmark/scripts/compare_perf_tests.py

@@ -236,6 +236,7 @@ def __init__(self, csv_row):
         self.max_rss = (                # Maximum Resident Set Size (B)
             int(csv_row[8]) if len(csv_row) > 8 else None)
         self.samples = None
+        self.setup = None
 
     def __repr__(self):
         """Short summary for debugging purposes."""
@@ -274,6 +275,7 @@ def minimum(a, b):  # work around None being less than everything
             return (min(filter(lambda x: x is not None, [a, b])) if any([a, b])
                     else None)
         self.max_rss = minimum(self.max_rss, r.max_rss)
+        self.setup = minimum(self.setup, r.setup)
 
 
 class ResultComparison(object):
@@ -318,7 +320,7 @@ def __init__(self):
     def _reset(self):
         """Reset parser to the default state for reading a new result."""
         self.samples, self.num_iters = [], 1
-        self.max_rss, self.mem_pages = None, None
+        self.setup, self.max_rss, self.mem_pages = None, None, None
         self.voluntary_cs, self.involuntary_cs = None, None
 
     # Parse lines like this
@@ -332,6 +334,7 @@ def _append_result(self, result):
         if len(columns) < 8:
             columns = result.split()
         r = PerformanceTestResult(columns)
+        r.setup = self.setup
         r.max_rss = r.max_rss or self.max_rss
         r.mem_pages = self.mem_pages
         r.voluntary_cs = self.voluntary_cs
@@ -360,6 +363,9 @@ def _store_memory_stats(self, max_rss, mem_pages):
          self.samples.append(
              Sample(int(i), int(self.num_iters), int(runtime)))),
 
+        re.compile(r'\s+SetUp (\d+)'):
+        (lambda self, setup: setattr(self, 'setup', int(setup))),
+
         # Environmental statistics: memory usage and context switches
         re.compile(r'\s+MAX_RSS \d+ - \d+ = (\d+) \((\d+) pages\)'):
         _store_memory_stats,

Diff for: benchmark/scripts/test_compare_perf_tests.py

@@ -221,23 +221,25 @@ def test_merge(self):
 1,AngryPhonebook,1,12270,12270,12270,0,12270,10498048""".split('\n')[1:]
         results = map(PerformanceTestResult,
                       [line.split(',') for line in tests])
+        results[2].setup = 9
+        results[3].setup = 7
 
         def as_tuple(r):
             return (r.num_samples, r.min, r.max, round(r.mean, 2),
-                    r.sd, r.median, r.max_rss)
+                    r.sd, r.median, r.max_rss, r.setup)
 
         r = results[0]
         self.assertEquals(as_tuple(r),
-                          (1, 12045, 12045, 12045, 0, 12045, None))
+                          (1, 12045, 12045, 12045, 0, 12045, None, None))
         r.merge(results[1])
-        self.assertEquals(as_tuple(r),  # drops SD and median
-                          (2, 12045, 12325, 12185, None, None, 10510336))
+        self.assertEquals(as_tuple(r),  # drops SD and median, +max_rss
+                          (2, 12045, 12325, 12185, None, None, 10510336, None))
         r.merge(results[2])
-        self.assertEquals(as_tuple(r),  # picks smaller of the MAX_RSS
-                          (3, 11616, 12325, 11995.33, None, None, 10502144))
+        self.assertEquals(as_tuple(r),  # picks smaller of the MAX_RSS, +setup
+                          (3, 11616, 12325, 11995.33, None, None, 10502144, 9))
         r.merge(results[3])
-        self.assertEquals(as_tuple(r),
-                          (4, 11616, 12325, 12064, None, None, 10498048))
+        self.assertEquals(as_tuple(r),  # picks smaller of the setup values
+                          (4, 11616, 12325, 12064, None, None, 10498048, 7))
 
 
 class TestResultComparison(unittest.TestCase):
@@ -384,6 +386,7 @@ def test_parse_results_verbose(self):
     Sample 2,11467
 1,AngryPhonebook,3,11467,13898,12392,1315,11812
 Running Array2D for 3 samples.
+    SetUp 14444
     Sample 0,369900
     Sample 1,381039
     Sample 2,371043
@@ -407,6 +410,7 @@ def test_parse_results_verbose(self):
             (r.name, r.min, r.max, int(r.mean), int(r.sd), r.median),
             ('Array2D', 369900, 381039, 373994, 6127, 371043)
         )
+        self.assertEquals(r.setup, 14444)
         self.assertEquals(r.num_samples, r.samples.num_samples)
         self.assertEquals(results[1].samples.all_samples,
                           [(0, 1, 369900), (1, 1, 381039), (2, 1, 371043)])