Moved ctag preprocessor into proper location

Author: cmaglie

arduino/builder/preprocessor/ctags.go

@@ -16,8 +16,11 @@
 package preprocessor
 
 import (
+	"bufio"
+	"bytes"
 	"context"
 	"fmt"
+	"io"
 	"strconv"
 	"strings"
 
@@ -37,30 +40,74 @@ var tr = i18n.Tr
 // this is useful for unit-testing to provide more infos
 var DebugPreprocessor bool
 
-func CTags(sourceFile *paths.Path, targetFile *paths.Path, sketch *sketch.Sketch, lineOffset int, buildProperties *properties.Map) ([]byte, error) {
-	ctagsOutput, ctagsStdErr, err := RunCTags(sourceFile, buildProperties)
+// PreprocessSketchWithCtags performs preprocessing of the arduino sketch using CTags.
+func PreprocessSketchWithCtags(sketch *sketch.Sketch, buildPath *paths.Path, includes paths.PathList, lineOffset int, buildProperties *properties.Map, onlyUpdateCompilationDatabase bool) ([]byte, []byte, error) {
+	// Create a temporary working directory
+	tmpDir, err := paths.MkTempDir("", "")
 	if err != nil {
-		return ctagsStdErr, err
+		return nil, nil, err
+	}
+	defer tmpDir.RemoveAll()
+	ctagsTarget := tmpDir.Join("sketch_merged.cpp")
+
+	normalOutput := &bytes.Buffer{}
+	verboseOutput := &bytes.Buffer{}
+
+	// Run GCC preprocessor
+	sourceFile := buildPath.Join("sketch", sketch.MainFile.Base()+".cpp")
+	gccStdout, gccStderr, err := GCC(sourceFile, ctagsTarget, includes, buildProperties)
+	verboseOutput.Write(gccStdout)
+	verboseOutput.Write(gccStderr)
+	normalOutput.Write(gccStderr)
+	if err != nil {
+		if !onlyUpdateCompilationDatabase {
+			return normalOutput.Bytes(), verboseOutput.Bytes(), errors.WithStack(err)
+		}
+
+		// Do not bail out if we are generating the compile commands database
+		normalOutput.WriteString(fmt.Sprintf("%s: %s",
+			tr("An error occurred adding prototypes"),
+			tr("the compilation database may be incomplete or inaccurate")))
+		if err := sourceFile.CopyTo(ctagsTarget); err != nil {
+			return normalOutput.Bytes(), verboseOutput.Bytes(), errors.WithStack(err)
+		}
 	}
 
-	// func PrototypesAdder(sketch *sketch.Sketch, source string, ctagsStdout []byte, lineOffset int) string {
+	if src, err := ctagsTarget.ReadFile(); err == nil {
+		filteredSource := filterSketchSource(sketch, bytes.NewReader(src), false)
+		if err := ctagsTarget.WriteFile([]byte(filteredSource)); err != nil {
+			return normalOutput.Bytes(), verboseOutput.Bytes(), err
+		}
+	} else {
+		return normalOutput.Bytes(), verboseOutput.Bytes(), err
+	}
+
+	// Run CTags on gcc-preprocessed source
+	ctagsOutput, ctagsStdErr, err := RunCTags(ctagsTarget, buildProperties)
+	verboseOutput.Write(ctagsStdErr)
+	if err != nil {
+		return normalOutput.Bytes(), verboseOutput.Bytes(), err
+	}
+
+	// Parse CTags output
 	parser := &ctags.CTagsParser{}
 	prototypes, firstFunctionLine := parser.Parse(ctagsOutput, sketch.MainFile)
 	if firstFunctionLine == -1 {
 		firstFunctionLine = 0
 	}
 
+	// Add prototypes to the original sketch source
 	var source string
-	if sourceData, err := targetFile.ReadFile(); err != nil {
-		return nil, err
-	} else {
+	if sourceData, err := sourceFile.ReadFile(); err == nil {
 		source = string(sourceData)
+	} else {
+		return normalOutput.Bytes(), verboseOutput.Bytes(), err
 	}
 	source = strings.Replace(source, "\r\n", "\n", -1)
 	source = strings.Replace(source, "\r", "\n", -1)
 	sourceRows := strings.Split(source, "\n")
 	if isFirstFunctionOutsideOfSource(firstFunctionLine, sourceRows) {
-		return nil, nil
+		return normalOutput.Bytes(), verboseOutput.Bytes(), nil
 	}
 
 	insertionLine := firstFunctionLine + lineOffset - 1
@@ -84,8 +131,9 @@ func CTags(sourceFile *paths.Path, targetFile *paths.Path, sketch *sketch.Sketch
 		fmt.Println("#END OF PREPROCESSED SOURCE")
 	}
 
-	err = targetFile.WriteFile([]byte(preprocessedSource))
-	return ctagsStdErr, err
+	// Write back arduino-preprocess output to the sourceFile
+	err = sourceFile.WriteFile([]byte(preprocessedSource))
+	return normalOutput.Bytes(), verboseOutput.Bytes(), err
 }
 
 func composePrototypeSection(line int, prototypes []*ctags.Prototype) string {
@@ -157,3 +205,29 @@ func RunCTags(sourceFile *paths.Path, buildProperties *properties.Map) ([]byte,
 	stderr = append([]byte(args), stderr...)
 	return stdout, stderr, err
 }
+
+func filterSketchSource(sketch *sketch.Sketch, source io.Reader, removeLineMarkers bool) string {
+	fileNames := paths.NewPathList()
+	fileNames.Add(sketch.MainFile)
+	fileNames.AddAll(sketch.OtherSketchFiles)
+
+	inSketch := false
+	filtered := ""
+
+	scanner := bufio.NewScanner(source)
+	for scanner.Scan() {
+		line := scanner.Text()
+		if filename := cpp.ParseLineMarker(line); filename != nil {
+			inSketch = fileNames.Contains(filename)
+			if inSketch && removeLineMarkers {
+				continue
+			}
+		}
+
+		if inSketch {
+			filtered += line + "\n"
+		}
+	}
+
+	return filtered
+}

legacy/builder/add_additional_entries_to_context.go

@@ -26,10 +26,6 @@ type AddAdditionalEntriesToContext struct{}
 func (*AddAdditionalEntriesToContext) Run(ctx *types.Context) error {
 	if ctx.BuildPath != nil {
 		buildPath := ctx.BuildPath
-		preprocPath, err := buildPath.Join(constants.FOLDER_PREPROC).Abs()
-		if err != nil {
-			return errors.WithStack(err)
-		}
 		sketchBuildPath, err := buildPath.Join(constants.FOLDER_SKETCH).Abs()
 		if err != nil {
 			return errors.WithStack(err)
@@ -43,7 +39,6 @@ func (*AddAdditionalEntriesToContext) Run(ctx *types.Context) error {
 			return errors.WithStack(err)
 		}
 
-		ctx.PreprocPath = preprocPath
 		ctx.SketchBuildPath = sketchBuildPath
 		ctx.LibrariesBuildPath = librariesBuildPath
 		ctx.CoreBuildPath = coreBuildPath

legacy/builder/builder.go

@@ -20,6 +20,7 @@ import (
 	"time"
 
 	"github.com/arduino/arduino-cli/arduino/builder"
+	"github.com/arduino/arduino-cli/arduino/builder/preprocessor"
 	"github.com/arduino/arduino-cli/i18n"
 	"github.com/arduino/arduino-cli/legacy/builder/phases"
 	"github.com/arduino/arduino-cli/legacy/builder/types"
@@ -59,7 +60,7 @@ func (s *Builder) Run(ctx *types.Context) error {
 		&WarnAboutArchIncompatibleLibraries{},
 
 		utils.LogIfVerbose(false, tr("Generating function prototypes...")),
-		&PreprocessSketch{},
+		types.BareCommand(PreprocessSketch),
 
 		utils.LogIfVerbose(false, tr("Compiling sketch...")),
 		&RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.sketch.prebuild", Suffix: ".pattern"},
@@ -115,13 +116,19 @@ func (s *Builder) Run(ctx *types.Context) error {
 	return otherErr
 }
 
-type PreprocessSketch struct{}
-
-func (s *PreprocessSketch) Run(ctx *types.Context) error {
+func PreprocessSketch(ctx *types.Context) error {
 	if ctx.UseArduinoPreprocessor {
 		return PreprocessSketchWithArduinoPreprocessor(ctx)
 	} else {
-		return PreprocessSketchWithCtags(ctx)
+		normalOutput, verboseOutput, err := preprocessor.PreprocessSketchWithCtags(
+			ctx.Sketch, ctx.BuildPath, ctx.IncludeFolders, ctx.LineOffset,
+			ctx.BuildProperties, ctx.OnlyUpdateCompilationDatabase)
+		if ctx.Verbose {
+			ctx.WriteStdout(verboseOutput)
+		} else {
+			ctx.WriteStdout(normalOutput)
+		}
+		return err
 	}
 }
 
@@ -149,7 +156,7 @@ func (s *Preprocess) Run(ctx *types.Context) error {
 
 		&WarnAboutArchIncompatibleLibraries{},
 
-		&PreprocessSketch{},
+		types.BareCommand(PreprocessSketch),
 	}
 
 	if err := runCommands(ctx, commands); err != nil {

legacy/builder/constants/constants.go

@@ -34,7 +34,6 @@ const BUILD_PROPERTIES_RUNTIME_PLATFORM_PATH = "runtime.platform.path"
 const EMPTY_STRING = ""
 const FOLDER_BOOTLOADERS = "bootloaders"
 const FOLDER_CORE = "core"
-const FOLDER_PREPROC = "preproc"
 const FOLDER_SKETCH = "sketch"
 const FOLDER_TOOLS = "tools"
 const FOLDER_LIBRARIES = "libraries"

legacy/builder/container_add_prototypes.go

@@ -115,7 +115,7 @@ func (s *ExportProjectCMake) Run(ctx *types.Context) error {
 		fmt.Println(err)
 	}
 
-	if err := PreprocessSketchWithCtags(ctx); err != nil {
+	if err := PreprocessSketch(ctx); err != nil {
 		return err
 	}
 

legacy/builder/create_cmake_rule.go

@@ -31,12 +31,12 @@ import (
 )
 
 func PreprocessSketchWithArduinoPreprocessor(ctx *types.Context) error {
-	if err := ctx.PreprocPath.MkdirAll(); err != nil {
+	if err := ctx.BuildPath.Join("preproc").MkdirAll(); err != nil {
 		return errors.WithStack(err)
 	}
 
 	sourceFile := ctx.SketchBuildPath.Join(ctx.Sketch.MainFile.Base() + ".cpp")
-	targetFile := ctx.PreprocPath.Join("sketch_merged.cpp")
+	targetFile := ctx.BuildPath.Join("preproc", "sketch_merged.cpp")
 	gccStdout, gccStderr, err := preprocessor.GCC(sourceFile, targetFile, ctx.IncludeFolders, ctx.BuildProperties)
 	if ctx.Verbose {
 		ctx.WriteStdout(gccStdout)

legacy/builder/preprocess_sketch.go

@@ -31,7 +31,6 @@ func TestAddAdditionalEntriesToContextNoBuildPath(t *testing.T) {
 	command := builder.AddAdditionalEntriesToContext{}
 	NoError(t, command.Run(ctx))
 
-	require.Empty(t, ctx.PreprocPath)
 	require.Empty(t, ctx.SketchBuildPath)
 	require.Empty(t, ctx.LibrariesBuildPath)
 	require.Empty(t, ctx.CoreBuildPath)
@@ -48,7 +47,6 @@ func TestAddAdditionalEntriesToContextWithBuildPath(t *testing.T) {
 	command := builder.AddAdditionalEntriesToContext{}
 	NoError(t, command.Run(ctx))
 
-	require.Equal(t, Abs(t, paths.New("folder", constants.FOLDER_PREPROC)), ctx.PreprocPath)
 	require.Equal(t, Abs(t, paths.New("folder", constants.FOLDER_SKETCH)), ctx.SketchBuildPath)
 	require.Equal(t, Abs(t, paths.New("folder", "libraries")), ctx.LibrariesBuildPath)
 	require.Equal(t, Abs(t, paths.New("folder", constants.FOLDER_CORE)), ctx.CoreBuildPath)