Added build_utils.ArchiveCompiledFiles and changed core_builder

Moved some string literals into constants
Added a patch to AVR core (it's temporary: master AVR core is already patched
but not released)

Signed-off-by: Federico Fissore <f.fissore@arduino.cc>

Author: Federico Fissore

src/arduino.cc/builder/builder_utils/utils.go

@@ -158,6 +158,29 @@ func objFileIsUpToDateWithSourceFile(sourceFileStat, objectFileStat os.FileInfo)
 	return objectFileStat != nil && sourceFileStat.ModTime().Before(objectFileStat.ModTime())
 }
 
+func ArchiveCompiledFiles(buildPath string, archiveFile string, objectFiles []string, buildProperties map[string]string, verbose bool, logger i18n.Logger) (string, error) {
+	archiveFilePath := filepath.Join(buildPath, archiveFile)
+	if _, err := os.Stat(archiveFilePath); err == nil {
+		err = os.Remove(archiveFilePath)
+		if err != nil {
+			return "", utils.WrapError(err)
+		}
+	}
+
+	for _, objectFile := range objectFiles {
+		properties := utils.MergeMapsOfStrings(make(map[string]string), buildProperties)
+		properties[constants.BUILD_PROPERTIES_ARCHIVE_FILE] = filepath.Base(archiveFilePath)
+		properties[constants.BUILD_PROPERTIES_OBJECT_FILE] = objectFile
+
+		_, err := ExecRecipe(properties, constants.RECIPE_AR_PATTERN, false, verbose, verbose, logger)
+		if err != nil {
+			return "", utils.WrapError(err)
+		}
+	}
+
+	return archiveFilePath, nil
+}
+
 func ExecRecipe(properties map[string]string, recipe string, removeUnsetProperties bool, echoCommandLine bool, echoOutput bool, logger i18n.Logger) ([]byte, error) {
 	pattern := properties[recipe]
 	if pattern == constants.EMPTY_STRING {

src/arduino.cc/builder/constants/constants.go

@@ -32,6 +32,7 @@ package constants
 const BOARD_PROPERTIES_MENU = "menu"
 const BUILD_OPTIONS_FILE = "build.options.json"
 const BUILD_PROPERTIES_ARCHIVE_FILE = "archive_file"
+const BUILD_PROPERTIES_ARCHIVE_FILE_PATH = "archive_file_path"
 const BUILD_PROPERTIES_ARCH_OVERRIDE_CHECK = "architecture.override_check"
 const BUILD_PROPERTIES_BOOTLOADER_FILE = "bootloader.file"
 const BUILD_PROPERTIES_BOOTLOADER_NOBLINK = "bootloader.noblink"
@@ -45,6 +46,7 @@ const BUILD_PROPERTIES_BUILD_PROJECT_NAME = "build.project_name"
 const BUILD_PROPERTIES_BUILD_SYSTEM_PATH = "build.system.path"
 const BUILD_PROPERTIES_BUILD_VARIANT = "build.variant"
 const BUILD_PROPERTIES_BUILD_VARIANT_PATH = "build.variant.path"
+const BUILD_PROPERTIES_COMPILER_C_ELF_FLAGS = "compiler.c.elf.flags"
 const BUILD_PROPERTIES_COMPILER_CPP_FLAGS = "compiler.cpp.flags"
 const BUILD_PROPERTIES_COMPILER_PATH = "compiler.path"
 const BUILD_PROPERTIES_COMPILER_WARNING_FLAGS = "compiler.warning_flags"
@@ -70,6 +72,7 @@ const BUILD_PROPERTIES_VID = "vid"
 const COAN = "coan"
 const CTAGS = "ctags"
 const CTX_ACTUAL_PLATFORM = "actualPlatform"
+const CTX_ARCHIVE_FILE_PATH_CORE = "archiveFileCore"
 const CTX_BUILD_CORE = "buildCore"
 const CTX_BUILD_OPTIONS = "buildOptions"
 const CTX_BUILD_OPTIONS_JSON = "buildOptionsJson"
@@ -99,7 +102,6 @@ const CTX_LIBRARIES = "libraries"
 const CTX_LIBRARY_DISCOVERY_RECURSION_DEPTH = "libraryDiscoveryRecursionDepth"
 const CTX_LINE_OFFSET = "lineOffset"
 const CTX_LOGGER = "logger"
-const CTX_OBJECT_FILES_CORE = "objectFilesCore"
 const CTX_OBJECT_FILES_LIBRARIES = "objectFilesLibraries"
 const CTX_OBJECT_FILES_SKETCH = "objectFilesSketch"
 const CTX_PLATFORM_KEYS_REWRITE = "platformKeysRewrite"
@@ -173,6 +175,7 @@ const LIBRARY_ALL_ARCHS = "*"
 const LIBRARY_ARCHITECTURES = "architectures"
 const LIBRARY_AUTHOR = "author"
 const LIBRARY_CATEGORY = "category"
+const LIBRARY_DOT_A_LINKAGE = "dot_a_linkage"
 const LIBRARY_EMAIL = "email"
 const LIBRARY_FOLDER_ARCH = "arch"
 const LIBRARY_FOLDER_SRC = "src"
@@ -230,6 +233,8 @@ const PLATFORM_REWRITE_OLD = "old"
 const PLATFORM_URL = "url"
 const PLATFORM_VERSION = "version"
 const PROGRAMMER_NAME = "name"
+const RECIPE_AR_PATTERN = "recipe.ar.pattern"
+const RECIPE_C_COMBINE_PATTERN = "recipe.c.combine.pattern"
 const RECIPE_C_PATTERN = "recipe.c.o.pattern"
 const RECIPE_CPP_PATTERN = "recipe.cpp.o.pattern"
 const RECIPE_PREPROC_INCLUDES = "recipe.preproc.includes"

src/arduino.cc/builder/includes_finder_with_gcc.go

@@ -48,7 +48,7 @@ func (s *IncludesFinderWithGCC) Run(context map[string]interface{}) error {
 	verbose := context[constants.CTX_VERBOSE].(bool)
 	logger := context[constants.CTX_LOGGER].(i18n.Logger)
 
-	includesParams := ""
+	includesParams := constants.EMPTY_STRING
 	if utils.MapHas(context, constants.CTX_INCLUDE_FOLDERS) {
 		includes := context[constants.CTX_INCLUDE_FOLDERS].([]string)
 		includes = utils.Map(includes, utils.WrapWithHyphenI)

src/arduino.cc/builder/libraries_loader.go

@@ -191,6 +191,7 @@ func makeNewLibrary(libraryFolder string, debugLevel int, logger i18n.Logger) (*
 	library.Paragraph = strings.TrimSpace(properties[constants.LIBRARY_PARAGRAPH])
 	library.URL = strings.TrimSpace(properties[constants.LIBRARY_URL])
 	library.IsLegacy = false
+	library.DotALinkage = strings.TrimSpace(properties[constants.LIBRARY_DOT_A_LINKAGE]) == "true"
 
 	return library, nil
 }

src/arduino.cc/builder/phases/core_builder.go

@@ -35,8 +35,6 @@ import (
 	"arduino.cc/builder/i18n"
 	"arduino.cc/builder/utils"
 	"os"
-	"path/filepath"
-	"strings"
 )
 
 type CoreBuilder struct{}
@@ -53,17 +51,17 @@ func (s *CoreBuilder) Run(context map[string]interface{}) error {
 		return utils.WrapError(err)
 	}
 
-	objectFiles, err := compileCore(coreBuildPath, buildProperties, verbose, warningsLevel, logger)
+	archiveFile, err := compileCore(coreBuildPath, buildProperties, verbose, warningsLevel, logger)
 	if err != nil {
 		return utils.WrapError(err)
 	}
 
-	context[constants.CTX_OBJECT_FILES_CORE] = objectFiles
+	context[constants.CTX_ARCHIVE_FILE_PATH_CORE] = archiveFile
 
 	return nil
 }
 
-func compileCore(buildPath string, buildProperties map[string]string, verbose bool, warningsLevel string, logger i18n.Logger) ([]string, error) {
+func compileCore(buildPath string, buildProperties map[string]string, verbose bool, warningsLevel string, logger i18n.Logger) (string, error) {
 	var objectFiles []string
 	coreFolder := buildProperties[constants.BUILD_PROPERTIES_BUILD_CORE_PATH]
 	variantFolder := buildProperties[constants.BUILD_PROPERTIES_BUILD_VARIANT_PATH]
@@ -80,34 +78,19 @@ func compileCore(buildPath string, buildProperties map[string]string, verbose bo
 	if variantFolder != constants.EMPTY_STRING {
 		objectFiles, err = builder_utils.CompileFiles(objectFiles, variantFolder, true, buildPath, buildProperties, includes, verbose, warningsLevel, logger)
 		if err != nil {
-			return nil, utils.WrapError(err)
+			return "", utils.WrapError(err)
 		}
 	}
 
 	coreObjectFiles, err := builder_utils.CompileFiles([]string{}, coreFolder, true, buildPath, buildProperties, includes, verbose, warningsLevel, logger)
 	if err != nil {
-		return nil, utils.WrapError(err)
+		return "", utils.WrapError(err)
 	}
 
-	coreArchiveFilePath := filepath.Join(buildPath, "core.a")
-	if _, err := os.Stat(coreArchiveFilePath); err == nil {
-		err = os.Remove(coreArchiveFilePath)
-		if err != nil {
-			return nil, utils.WrapError(err)
-		}
-	}
-
-	for _, coreObjectFile := range coreObjectFiles {
-		properties := utils.MergeMapsOfStrings(make(map[string]string), buildProperties)
-		properties[constants.BUILD_PROPERTIES_INCLUDES] = strings.Join(includes, constants.SPACE)
-		properties[constants.BUILD_PROPERTIES_ARCHIVE_FILE] = filepath.Base(coreArchiveFilePath)
-		properties[constants.BUILD_PROPERTIES_OBJECT_FILE] = coreObjectFile
-
-		_, err := builder_utils.ExecRecipe(properties, "recipe.ar.pattern", false, verbose, verbose, logger)
-		if err != nil {
-			return nil, utils.WrapError(err)
-		}
+	archiveFile, err := builder_utils.ArchiveCompiledFiles(buildPath, "core.a", coreObjectFiles, buildProperties, verbose, logger)
+	if err != nil {
+		return "", utils.WrapError(err)
 	}
 
-	return objectFiles, nil
+	return archiveFile, nil
 }

src/arduino.cc/builder/phases/linker.go

@@ -34,6 +34,7 @@ import (
 	"arduino.cc/builder/constants"
 	"arduino.cc/builder/i18n"
 	"arduino.cc/builder/utils"
+	"path/filepath"
 	"strings"
 )
 
@@ -42,45 +43,50 @@ type Linker struct{}
 func (s *Linker) Run(context map[string]interface{}) error {
 	objectFilesSketch := context[constants.CTX_OBJECT_FILES_SKETCH].([]string)
 	objectFilesLibraries := context[constants.CTX_OBJECT_FILES_LIBRARIES].([]string)
-	objectFilesCore := context[constants.CTX_OBJECT_FILES_CORE].([]string)
 
 	var objectFiles []string
 	objectFiles = append(objectFiles, objectFilesSketch...)
 	objectFiles = append(objectFiles, objectFilesLibraries...)
-	objectFiles = append(objectFiles, objectFilesCore...)
+
+	coreArchiveFilePath := context[constants.CTX_ARCHIVE_FILE_PATH_CORE].(string)
 
 	buildProperties := context[constants.CTX_BUILD_PROPERTIES].(map[string]string)
 	verbose := context[constants.CTX_VERBOSE].(bool)
 	warningsLevel := context[constants.CTX_WARNINGS_LEVEL].(string)
 	logger := context[constants.CTX_LOGGER].(i18n.Logger)
 
-	err := link(objectFiles, buildProperties, verbose, warningsLevel, logger)
+	err := link(objectFiles, coreArchiveFilePath, buildProperties, verbose, warningsLevel, logger)
 	if err != nil {
 		return utils.WrapError(err)
 	}
 
 	return nil
 }
 
-func link(objectFiles []string, buildProperties map[string]string, verbose bool, warningsLevel string, logger i18n.Logger) error {
-	optRelax := constants.EMPTY_STRING
-	if buildProperties[constants.BUILD_PROPERTIES_BUILD_MCU] == "atmega2560" {
-		optRelax = ",--relax"
-	}
+func link(objectFiles []string, coreArchiveFilePath string, buildProperties map[string]string, verbose bool, warningsLevel string, logger i18n.Logger) error {
+	optRelax := addRelaxTrickIfATMEGA2560(buildProperties)
 
 	objectFiles = utils.Map(objectFiles, wrapWithDoubleQuotes)
 	objectFileList := strings.Join(objectFiles, constants.SPACE)
 
 	properties := utils.MergeMapsOfStrings(make(map[string]string), buildProperties)
-	properties["compiler.c.elf.flags"] = properties["compiler.c.elf.flags"] + optRelax
-	properties["compiler.warning_flags"] = properties["compiler.warning_flags."+warningsLevel]
-	properties[constants.BUILD_PROPERTIES_ARCHIVE_FILE] = "core.a"
+	properties[constants.BUILD_PROPERTIES_COMPILER_C_ELF_FLAGS] = properties[constants.BUILD_PROPERTIES_COMPILER_C_ELF_FLAGS] + optRelax
+	properties[constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS] = properties[constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS+"."+warningsLevel]
+	properties[constants.BUILD_PROPERTIES_ARCHIVE_FILE] = filepath.Base(coreArchiveFilePath)
+	properties[constants.BUILD_PROPERTIES_ARCHIVE_FILE_PATH] = coreArchiveFilePath
 	properties[constants.BUILD_PROPERTIES_OBJECT_FILES] = objectFileList
 
-	_, err := builder_utils.ExecRecipe(properties, "recipe.c.combine.pattern", false, verbose, verbose, logger)
+	_, err := builder_utils.ExecRecipe(properties, constants.RECIPE_C_COMBINE_PATTERN, false, verbose, verbose, logger)
 	return err
 }
 
 func wrapWithDoubleQuotes(value string) string {
 	return "\"" + value + "\""
 }
+
+func addRelaxTrickIfATMEGA2560(buildProperties map[string]string) string {
+	if buildProperties[constants.BUILD_PROPERTIES_BUILD_MCU] == "atmega2560" {
+		return ",--relax"
+	}
+	return constants.EMPTY_STRING
+}

src/arduino.cc/builder/props/properties.go

@@ -91,7 +91,7 @@ func loadSingleLine(properties map[string]string, line string) {
 		key := strings.TrimSpace(lineParts[0])
 		value := strings.TrimSpace(lineParts[1])
 
-		key = strings.Replace(key, "."+OSNAME, "", 1)
+		key = strings.Replace(key, "."+OSNAME, constants.EMPTY_STRING, 1)
 		properties[key] = value
 	}
 }

src/arduino.cc/builder/test/builder_test.go

@@ -264,3 +264,26 @@ func TestBuilderSketchNoFunctions(t *testing.T) {
 	err := command.Run(context)
 	require.Error(t, err)
 }
+
+/*
+func TestBuilderALinkage(t *testing.T) {
+	DownloadCoresAndToolsAndLibraries(t)
+
+	context := make(map[string]interface{})
+
+	buildPath := SetupBuildPath(t, context)
+	defer os.RemoveAll(buildPath)
+
+	context[constants.CTX_HARDWARE_FOLDERS] = []string{filepath.Join("..", "hardware"), "hardware", "downloaded_hardware", "downloaded_board_manager_stuff"}
+	context[constants.CTX_TOOLS_FOLDERS] = []string{"downloaded_tools", "downloaded_board_manager_stuff"}
+	context[constants.CTX_LIBRARIES_FOLDERS] = []string{"libraries", "downloaded_libraries"}
+	context[constants.CTX_FQBN] = "arduino:avr:leonardo"
+	context[constants.CTX_SKETCH_LOCATION] = filepath.Join("downloaded_libraries", "HID", "examples", "NKROKeyboard", "NKROKeyboard.ino")
+	context[constants.CTX_BUILD_PROPERTIES_RUNTIME_IDE_VERSION] = "10600"
+	context[constants.CTX_VERBOSE] = true
+
+	command := builder.Builder{}
+	err := command.Run(context)
+	NoError(t, err)
+}
+*/

src/arduino.cc/builder/test/downloaded_libraries/.gitignore

@@ -1,4 +1,5 @@
 A*
 B*
 C*
+H*
 R*

src/arduino.cc/builder/test/downloaded_stuff_patches/avr_platform_patch.patch

@@ -0,0 +1,15 @@
+--- downloaded_hardware/arduino/avr/platform.txt	2015-09-10 17:14:15.137942087 +0200
++++ downloaded_hardware/arduino/avr/platform.txt	2015-09-23 10:27:19.522085331 +0200
+@@ -77,6 +77,12 @@
+ recipe.size.regex.data=^(?:\.data|\.bss|\.noinit)\s+([0-9]+).*
+ recipe.size.regex.eeprom=^(?:\.eeprom)\s+([0-9]+).*
+ 
++## Preprocessor
++preproc.includes.flags=-w -x c++ -M -MG -MP
++recipe.preproc.includes="{compiler.path}{compiler.cpp.cmd}" {compiler.cpp.flags} {preproc.includes.flags} -mmcu={build.mcu} -DF_CPU={build.f_cpu} -DARDUINO={runtime.ide.version} -DARDUINO_{build.board} -DARDUINO_ARCH_{build.arch} {compiler.cpp.extra_flags} {build.extra_flags} {includes} "{source_file}"
++
++preproc.macros.flags=-w -x c++ -E -CC
++recipe.preproc.macros="{compiler.path}{compiler.cpp.cmd}" {compiler.cpp.flags} {preproc.macros.flags} -mmcu={build.mcu} -DF_CPU={build.f_cpu} -DARDUINO={runtime.ide.version} -DARDUINO_{build.board} -DARDUINO_ARCH_{build.arch} {compiler.cpp.extra_flags} {build.extra_flags} {includes} "{source_file}"
+ 
+ # AVR Uploader/Programmers tools
+ # ------------------------------

src/arduino.cc/builder/test/downloaded_stuff_patches/hid_library_properties.patch.disabled

@@ -0,0 +1,8 @@
+--- downloaded_libraries/HID/library.properties	2015-09-23 09:49:20.470004391 +0200
++++ downloaded_libraries/HID/library.properties.new	2015-09-23 11:04:13.206163949 +0200
+@@ -7,4 +7,4 @@
+ category=Device Control
+ url=https://github.com/NicoHood/HID
+ architectures=*
+-alinkage=true
++dot_a_linkage=true

src/arduino.cc/builder/test/helper_tools_downloader.go

@@ -51,6 +51,7 @@ const HARDWARE_FOLDER = "downloaded_hardware"
 const BOARD_MANAGER_FOLDER = "downloaded_board_manager_stuff"
 const TOOLS_FOLDER = "downloaded_tools"
 const LIBRARIES_FOLDER = "downloaded_libraries"
+const PATCHES_FOLDER = "downloaded_stuff_patches"
 
 type Tool struct {
 	Name    string
@@ -67,6 +68,7 @@ type OsUrl struct {
 type Library struct {
 	Name    string
 	Version string
+	Url     string
 }
 
 type Core struct {
@@ -125,8 +127,30 @@ func DownloadCoresAndToolsAndLibraries(t *testing.T) {
 		Library{Name: "Bridge", Version: "1.0.7"},
 		Library{Name: "CapacitiveSensor", Version: "0.5.0"},
 		Library{Name: "Robot IR Remote", Version: "1.0.2"},
+		//Library{Name: "HID", Version: "0.0.0", Url: "https://github.com/NicoHood/HID/archive/dev_2_4.zip"},
 	}
 
+	download(t, cores, boardsManagerCores, boardsManagerRedBearCores, tools, boardsManagerTools, boardsManagerRFduinoTools, libraries)
+
+	patch(t)
+}
+
+// FIXME: once patched cores are released, patching them will be unnecessary
+func patch(t *testing.T) {
+	files, err := ioutil.ReadDir(PATCHES_FOLDER)
+	NoError(t, err)
+
+	for _, file := range files {
+		if filepath.Ext(file.Name()) == ".patch" {
+			cmd := exec.Command("patch", "-N", "-p0", "-r", "-", "-i", filepath.Join(PATCHES_FOLDER, file.Name()))
+			cmd.CombinedOutput()
+			//output, _ := cmd.CombinedOutput()
+			//fmt.Println(string(output))
+		}
+	}
+}
+
+func download(t *testing.T, cores, boardsManagerCores, boardsManagerRedBearCores []Core, tools, boardsManagerTools, boardsManagerRFduinoTools []Tool, libraries []Library) {
 	if allCoresAlreadyDownloadedAndUnpacked(HARDWARE_FOLDER, cores) &&
 		allBoardsManagerCoresAlreadyDownloadedAndUnpacked(BOARD_MANAGER_FOLDER, boardsManagerCores) &&
 		allBoardsManagerCoresAlreadyDownloadedAndUnpacked(BOARD_MANAGER_FOLDER, boardsManagerRedBearCores) &&
@@ -626,6 +650,9 @@ func downloadLibraries(libraries []Library, index map[string]interface{}) error
 }
 
 func findLibraryUrl(index map[string]interface{}, library Library) (string, error) {
+	if library.Url != "" {
+		return library.Url, nil
+	}
 	libs := index["libraries"].([]interface{})
 	for _, l := range libs {
 		lib := l.(map[string]interface{})