From 79dafd2cffca3c4b07c5d4164356a9ad049d4698 Mon Sep 17 00:00:00 2001
From: Martino Facchin <m.facchin@arduino.cc>
Date: Fri, 23 Aug 2019 12:50:07 +0200
Subject: [PATCH] Fix parallel compilation disregarding -jobs flag

The issue was due to the peculiar way concurrency and parallelism are handled in go.
We used to set GOMAXPROC to 1 to avoid parallelizing the WaitGroup execution.
This would work, in theory, unless the goroutines sleep.
In that case, another goroutine is allowed to start concurrently (only 1 goroutine running in parallel, so GOMAXPROC is happy).
Since our goroutines sleep (wait) after calling gcc, another task is started, without any hard limit, till the WaitGroup is completely spawned.
On systems with limited resources (as RaspberryPi) and cores with lots of files this can trigger an out of memory condition.
---
 arduino-builder/main.go |  9 +++++----
 builder_utils/utils.go  | 32 ++++++++++++++++++--------------
 types/context.go        |  3 +++
 3 files changed, 26 insertions(+), 18 deletions(-)

diff --git a/arduino-builder/main.go b/arduino-builder/main.go
index e55d1caf..9e912334 100644
--- a/arduino-builder/main.go
+++ b/arduino-builder/main.go
@@ -207,13 +207,14 @@ func main() {
 		return
 	}
 
+	ctx := &types.Context{}
+
 	if *jobsFlag > 0 {
-		runtime.GOMAXPROCS(*jobsFlag)
+		ctx.Jobs = *jobsFlag
 	} else {
-		runtime.GOMAXPROCS(runtime.NumCPU())
+		ctx.Jobs = runtime.NumCPU()
 	}
-
-	ctx := &types.Context{}
+	runtime.GOMAXPROCS(ctx.Jobs)
 
 	// place here all experimental features that should live under this flag
 	if *experimentalFeatures {
diff --git a/builder_utils/utils.go b/builder_utils/utils.go
index 9b4cb536..27912e27 100644
--- a/builder_utils/utils.go
+++ b/builder_utils/utils.go
@@ -170,23 +170,27 @@ func compileFilesWithRecipe(ctx *types.Context, objectFiles []string, sourcePath
 
 	ctx.Progress.Steps = ctx.Progress.Steps / float64(len(sources))
 	var wg sync.WaitGroup
-	wg.Add(len(sources))
 
-	for _, source := range sources {
-		go func(source string) {
-			defer wg.Done()
-			PrintProgressIfProgressEnabledAndMachineLogger(ctx)
-			objectFile, err := compileFileWithRecipe(ctx, sourcePath, source, buildPath, buildProperties, includes, recipe)
-			if err != nil {
-				errorsChan <- err
-			} else {
-				objectFilesChan <- objectFile
-			}
-		}(source)
-	}
+	par := ctx.Jobs
 
 	go func() {
-		wg.Wait()
+		for total := 0; total < len(sources); total += par {
+			for i := total; i < total+par && i < len(sources); i++ {
+				wg.Add(1)
+				go func(source string) {
+					defer wg.Done()
+					PrintProgressIfProgressEnabledAndMachineLogger(ctx)
+					objectFile, err := compileFileWithRecipe(ctx, sourcePath, source, buildPath, buildProperties, includes, recipe)
+					if err != nil {
+						errorsChan <- err
+					} else {
+						objectFilesChan <- objectFile
+					}
+				}(sources[i])
+			}
+			wg.Wait()
+		}
+
 		doneChan <- struct{}{}
 	}()
 
diff --git a/types/context.go b/types/context.go
index e9368c08..e90a1e53 100644
--- a/types/context.go
+++ b/types/context.go
@@ -103,6 +103,9 @@ type Context struct {
 
 	// Experimental: use arduino-preprocessor to create prototypes
 	UseArduinoPreprocessor bool
+
+	// Parallel processes
+	Jobs int
 }
 
 func (ctx *Context) ExtractBuildOptions() properties.Map {