diff --git a/src/arduino.cc/arduino-builder/main.go b/src/arduino.cc/arduino-builder/main.go
index 39a65c09..ae4cea95 100644
--- a/src/arduino.cc/arduino-builder/main.go
+++ b/src/arduino.cc/arduino-builder/main.go
@@ -37,6 +37,7 @@ import (
"io/ioutil"
"os"
"os/exec"
+ "runtime"
"strings"
"syscall"
@@ -79,6 +80,7 @@ const FLAG_LOGGER_HUMANTAGS = "humantags"
const FLAG_LOGGER_MACHINE = "machine"
const FLAG_VERSION = "version"
const FLAG_VID_PID = "vid-pid"
+const FLAG_JOBS = "jobs"
type foldersFlag []string
@@ -136,6 +138,7 @@ var warningsLevelFlag *string
var loggerFlag *string
var versionFlag *bool
var vidPidFlag *string
+var jobsFlag *int
func init() {
compileFlag = flag.Bool(FLAG_ACTION_COMPILE, false, "compiles the given sketch")
@@ -159,6 +162,7 @@ func init() {
loggerFlag = flag.String(FLAG_LOGGER, FLAG_LOGGER_HUMAN, "Sets type of logger. Available values are '"+FLAG_LOGGER_HUMAN+"', '"+FLAG_LOGGER_HUMANTAGS+"', '"+FLAG_LOGGER_MACHINE+"'")
versionFlag = flag.Bool(FLAG_VERSION, false, "prints version and exits")
vidPidFlag = flag.String(FLAG_VID_PID, "", "specify to use vid/pid specific build properties, as defined in boards.txt")
+ jobsFlag = flag.Int(FLAG_JOBS, 0, "specify how many concurrent gcc processes should run at the same time. Defaults to the number of available cores on the running machine")
}
func main() {
@@ -173,6 +177,12 @@ func main() {
return
}
+ if *jobsFlag > 0 {
+ runtime.GOMAXPROCS(*jobsFlag)
+ } else {
+ runtime.GOMAXPROCS(runtime.NumCPU())
+ }
+
ctx := &types.Context{}
if *buildOptionsFileFlag != "" {
diff --git a/src/arduino.cc/builder/builder_utils/utils.go b/src/arduino.cc/builder/builder_utils/utils.go
index 97b6e355..8636abe3 100644
--- a/src/arduino.cc/builder/builder_utils/utils.go
+++ b/src/arduino.cc/builder/builder_utils/utils.go
@@ -31,15 +31,16 @@ package builder_utils
import (
"bytes"
- "fmt"
"io"
"os"
"os/exec"
"path/filepath"
"strings"
+ "sync"
"arduino.cc/builder/constants"
"arduino.cc/builder/i18n"
+ "arduino.cc/builder/types"
"arduino.cc/builder/utils"
"arduino.cc/properties"
)
@@ -146,15 +147,47 @@ func findAllFilesInFolder(sourcePath string, recurse bool) ([]string, error) {
}
func compileFilesWithRecipe(objectFiles []string, sourcePath string, sources []string, buildPath string, buildProperties properties.Map, includes []string, recipe string, verbose bool, warningsLevel string, logger i18n.Logger) ([]string, error) {
+ if len(sources) == 0 {
+ return objectFiles, nil
+ }
+ objectFilesChan := make(chan string)
+ errorsChan := make(chan error)
+ doneChan := make(chan struct{})
+
+ var wg sync.WaitGroup
+ wg.Add(len(sources))
+
for _, source := range sources {
- objectFile, err := compileFileWithRecipe(sourcePath, source, buildPath, buildProperties, includes, recipe, verbose, warningsLevel, logger)
- if err != nil {
+ go func(source string) {
+ defer wg.Done()
+ objectFile, err := compileFileWithRecipe(sourcePath, source, buildPath, buildProperties, includes, recipe, verbose, warningsLevel, logger)
+ if err != nil {
+ errorsChan <- err
+ } else {
+ objectFilesChan <- objectFile
+ }
+ }(source)
+ }
+
+ go func() {
+ wg.Wait()
+ doneChan <- struct{}{}
+ }()
+
+ for {
+ select {
+ case objectFile := <-objectFilesChan:
+ objectFiles = append(objectFiles, objectFile)
+ case err := <-errorsChan:
return nil, i18n.WrapError(err)
+ case <-doneChan:
+ close(objectFilesChan)
+ for objectFile := range objectFilesChan {
+ objectFiles = append(objectFiles, objectFile)
+ }
+ return objectFiles, nil
}
-
- objectFiles = append(objectFiles, objectFile)
}
- return objectFiles, nil
}
func compileFileWithRecipe(sourcePath string, source string, buildPath string, buildProperties properties.Map, includes []string, recipe string, verbose bool, warningsLevel string, logger i18n.Logger) (string, error) {
@@ -361,10 +394,20 @@ func ExecRecipe(properties properties.Map, recipe string, removeUnsetProperties
}
if echoOutput {
- command.Stdout = os.Stdout
+ printToStdOut := func(data []byte) {
+ logger.UnformattedWrite(os.Stdout, data)
+ }
+ stdout := &types.BufferedUntilNewLineWriter{PrintFunc: printToStdOut, Buffer: bytes.Buffer{}}
+ defer stdout.Flush()
+ command.Stdout = stdout
}
- command.Stderr = os.Stderr
+ printToStdErr := func(data []byte) {
+ logger.UnformattedWrite(os.Stderr, data)
+ }
+ stderr := &types.BufferedUntilNewLineWriter{PrintFunc: printToStdErr, Buffer: bytes.Buffer{}}
+ defer stderr.Flush()
+ command.Stderr = stderr
if echoOutput {
err := command.Run()
@@ -396,7 +439,7 @@ func PrepareCommandForRecipe(buildProperties properties.Map, recipe string, remo
}
if echoCommandLine {
- fmt.Println(commandLine)
+ logger.UnformattedFprintln(os.Stdout, commandLine)
}
return command, nil
diff --git a/src/arduino.cc/builder/ctags_runner.go b/src/arduino.cc/builder/ctags_runner.go
index 2ab75638..f1e08a7b 100644
--- a/src/arduino.cc/builder/ctags_runner.go
+++ b/src/arduino.cc/builder/ctags_runner.go
@@ -30,7 +30,7 @@
package builder
import (
- "fmt"
+ "os"
"arduino.cc/builder/constants"
"arduino.cc/builder/ctags"
@@ -63,7 +63,7 @@ func (s *CTagsRunner) Run(ctx *types.Context) error {
verbose := ctx.Verbose
if verbose {
- fmt.Println(commandLine)
+ logger.UnformattedFprintln(os.Stdout, commandLine)
}
sourceBytes, err := command.Output()
diff --git a/src/arduino.cc/builder/i18n/i18n.go b/src/arduino.cc/builder/i18n/i18n.go
index ce85d22c..6532ccb1 100644
--- a/src/arduino.cc/builder/i18n/i18n.go
+++ b/src/arduino.cc/builder/i18n/i18n.go
@@ -38,12 +38,15 @@ import (
"regexp"
"strconv"
"strings"
+ "sync"
)
var PLACEHOLDER = regexp.MustCompile("{(\\d)}")
type Logger interface {
Fprintln(w io.Writer, level string, format string, a ...interface{})
+ UnformattedFprintln(w io.Writer, s string)
+ UnformattedWrite(w io.Writer, data []byte)
Println(level string, format string, a ...interface{})
Name() string
}
@@ -52,6 +55,10 @@ type NoopLogger struct{}
func (s NoopLogger) Fprintln(w io.Writer, level string, format string, a ...interface{}) {}
+func (s NoopLogger) UnformattedFprintln(w io.Writer, str string) {}
+
+func (s NoopLogger) UnformattedWrite(w io.Writer, data []byte) {}
+
func (s NoopLogger) Println(level string, format string, a ...interface{}) {}
func (s NoopLogger) Name() string {
@@ -62,13 +69,21 @@ type HumanTagsLogger struct{}
func (s HumanTagsLogger) Fprintln(w io.Writer, level string, format string, a ...interface{}) {
format = "[" + level + "] " + format
- fmt.Fprintln(w, Format(format, a...))
+ fprintln(w, Format(format, a...))
}
func (s HumanTagsLogger) Println(level string, format string, a ...interface{}) {
s.Fprintln(os.Stdout, level, format, a...)
}
+func (s HumanTagsLogger) UnformattedFprintln(w io.Writer, str string) {
+ fprintln(w, str)
+}
+
+func (s HumanTagsLogger) UnformattedWrite(w io.Writer, data []byte) {
+ write(w, data)
+}
+
func (s HumanTagsLogger) Name() string {
return "humantags"
}
@@ -76,20 +91,48 @@ func (s HumanTagsLogger) Name() string {
type HumanLogger struct{}
func (s HumanLogger) Fprintln(w io.Writer, level string, format string, a ...interface{}) {
- fmt.Fprintln(w, Format(format, a...))
+ fprintln(w, Format(format, a...))
}
func (s HumanLogger) Println(level string, format string, a ...interface{}) {
s.Fprintln(os.Stdout, level, format, a...)
}
+func (s HumanLogger) UnformattedFprintln(w io.Writer, str string) {
+ fprintln(w, str)
+}
+
+func (s HumanLogger) UnformattedWrite(w io.Writer, data []byte) {
+ write(w, data)
+}
+
func (s HumanLogger) Name() string {
return "human"
}
type MachineLogger struct{}
-func (s MachineLogger) printWithoutFormatting(w io.Writer, level string, format string, a []interface{}) {
+func (s MachineLogger) Fprintln(w io.Writer, level string, format string, a ...interface{}) {
+ printMachineFormattedLogLine(w, level, format, a)
+}
+
+func (s MachineLogger) Println(level string, format string, a ...interface{}) {
+ printMachineFormattedLogLine(os.Stdout, level, format, a)
+}
+
+func (s MachineLogger) UnformattedFprintln(w io.Writer, str string) {
+ fprintln(w, str)
+}
+
+func (s MachineLogger) Name() string {
+ return "machine"
+}
+
+func (s MachineLogger) UnformattedWrite(w io.Writer, data []byte) {
+ write(w, data)
+}
+
+func printMachineFormattedLogLine(w io.Writer, level string, format string, a []interface{}) {
a = append([]interface{}(nil), a...)
for idx, value := range a {
typeof := reflect.Indirect(reflect.ValueOf(value)).Kind()
@@ -97,20 +140,27 @@ func (s MachineLogger) printWithoutFormatting(w io.Writer, level string, format
a[idx] = url.QueryEscape(value.(string))
}
}
- fmt.Fprintf(w, "===%s ||| %s ||| %s", level, format, a)
- fmt.Fprintln(w)
+ fprintf(w, "===%s ||| %s ||| %s\n", level, format, a)
}
-func (s MachineLogger) Fprintln(w io.Writer, level string, format string, a ...interface{}) {
- s.printWithoutFormatting(w, level, format, a)
+var lock sync.Mutex
+
+func fprintln(w io.Writer, s string) {
+ lock.Lock()
+ defer lock.Unlock()
+ fmt.Fprintln(w, s)
}
-func (s MachineLogger) Println(level string, format string, a ...interface{}) {
- s.printWithoutFormatting(os.Stdout, level, format, a)
+func write(w io.Writer, data []byte) {
+ lock.Lock()
+ defer lock.Unlock()
+ w.Write(data)
}
-func (s MachineLogger) Name() string {
- return "machine"
+func fprintf(w io.Writer, format string, a ...interface{}) {
+ lock.Lock()
+ defer lock.Unlock()
+ fmt.Fprintf(w, format, a...)
}
func FromJavaToGoSyntax(s string) string {
diff --git a/src/arduino.cc/builder/types/accessories.go b/src/arduino.cc/builder/types/accessories.go
index 6601bbaa..4c87c30c 100644
--- a/src/arduino.cc/builder/types/accessories.go
+++ b/src/arduino.cc/builder/types/accessories.go
@@ -29,6 +29,11 @@
package types
+import (
+ "bytes"
+ "sync"
+)
+
type UniqueStringQueue []string
func (queue UniqueStringQueue) Len() int { return len(queue) }
@@ -74,3 +79,32 @@ func (queue *UniqueSourceFileQueue) Pop() SourceFile {
func (queue *UniqueSourceFileQueue) Empty() bool {
return queue.Len() == 0
}
+
+type BufferedUntilNewLineWriter struct {
+ PrintFunc PrintFunc
+ Buffer bytes.Buffer
+ lock sync.Mutex
+}
+
+type PrintFunc func([]byte)
+
+func (w *BufferedUntilNewLineWriter) Write(p []byte) (n int, err error) {
+ w.lock.Lock()
+ defer w.lock.Unlock()
+
+ writtenToBuffer, err := w.Buffer.Write(p)
+ return writtenToBuffer, err
+}
+
+func (w *BufferedUntilNewLineWriter) Flush() {
+ w.lock.Lock()
+ defer w.lock.Unlock()
+
+ remainingBytes := w.Buffer.Bytes()
+ if len(remainingBytes) > 0 {
+ if remainingBytes[len(remainingBytes)-1] != '\n' {
+ remainingBytes = append(remainingBytes, '\n')
+ }
+ w.PrintFunc(remainingBytes)
+ }
+}