Now compile --only-compilation-database will run all pre-* hooks (#1549)

* Clean up some useless constants
* made legacy/builder.go independent from legacy/constants
* Now 'compile --only-compilation-database' will run all pre-* hooks

Some platforms make sketch preparation see #1547

Fix #1547

Co-authored-by: per1234 <accounts@perglass.com>

Author: cmaglie

cli/compile/compile.go

@@ -112,7 +112,7 @@ func NewCommand() *cobra.Command {
 	command.RegisterFlagCompletionFunc("programmer", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
 		return arguments.GetInstalledProgrammers(), cobra.ShellCompDirectiveDefault
 	})
-	command.Flags().BoolVar(&compilationDatabaseOnly, "only-compilation-database", false, tr("Just produce the compilation database, without actually compiling."))
+	command.Flags().BoolVar(&compilationDatabaseOnly, "only-compilation-database", false, tr("Just produce the compilation database, without actually compiling. All build commands are skipped except pre* hooks."))
 	command.Flags().BoolVar(&clean, "clean", false, tr("Optional, cleanup the build folder and do not use any cached build."))
 	// We must use the following syntax for this flag since it's also bound to settings.
 	// This must be done because the value is set when the binding is accessed from viper. Accessing from cobra would only

docs/platform-specification.md

@@ -399,6 +399,10 @@ recipe.hooks.sketch.prebuild.02.pattern=echo 2
 recipe.hooks.sketch.prebuild.11.pattern=echo 11
 ```
 
+Note: all the `pre*` hooks are executed while producing the "compilation database" (a JSON file with the list of
+commands to run to compile the sketch), but the `post*` hooks and all compile commands are skipped. See the
+[`arduino-cli compile`](commands/arduino-cli_compile.md) command reference for more info.
+
 ## Global platform.txt
 
 Properties defined in a platform.txt created in the **hardware** subfolder of the Arduino IDE installation folder will

legacy/builder/builder.go

@@ -24,7 +24,6 @@ import (
 	"github.com/arduino/arduino-cli/arduino/sketch"
 	"github.com/arduino/arduino-cli/i18n"
 	"github.com/arduino/arduino-cli/legacy/builder/builder_utils"
-	"github.com/arduino/arduino-cli/legacy/builder/constants"
 	"github.com/arduino/arduino-cli/legacy/builder/phases"
 	"github.com/arduino/arduino-cli/legacy/builder/types"
 	"github.com/arduino/arduino-cli/legacy/builder/utils"
@@ -55,46 +54,46 @@ func (s *Builder) Run(ctx *types.Context) error {
 
 		&WarnAboutPlatformRewrites{},
 
-		&RecipeByPrefixSuffixRunner{Prefix: constants.HOOKS_PREBUILD, Suffix: constants.HOOKS_PATTERN_SUFFIX},
+		&RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.prebuild", Suffix: ".pattern"},
 
 		&ContainerMergeCopySketchFiles{},
 
-		utils.LogIfVerbose(constants.LOG_LEVEL_INFO, tr("Detecting libraries used...")),
+		utils.LogIfVerbose("info", tr("Detecting libraries used...")),
 		&ContainerFindIncludes{},
 
 		&WarnAboutArchIncompatibleLibraries{},
 
-		utils.LogIfVerbose(constants.LOG_LEVEL_INFO, tr("Generating function prototypes...")),
+		utils.LogIfVerbose("info", tr("Generating function prototypes...")),
 		&PreprocessSketch{},
 
-		utils.LogIfVerbose(constants.LOG_LEVEL_INFO, tr("Compiling sketch...")),
-		&RecipeByPrefixSuffixRunner{Prefix: constants.HOOKS_SKETCH_PREBUILD, Suffix: constants.HOOKS_PATTERN_SUFFIX},
+		utils.LogIfVerbose("info", tr("Compiling sketch...")),
+		&RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.sketch.prebuild", Suffix: ".pattern"},
 		&phases.SketchBuilder{},
-		&RecipeByPrefixSuffixRunner{Prefix: constants.HOOKS_SKETCH_POSTBUILD, Suffix: constants.HOOKS_PATTERN_SUFFIX},
+		&RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.sketch.postbuild", Suffix: ".pattern", SkipIfOnlyUpdatingCompilationDatabase: true},
 
-		utils.LogIfVerbose(constants.LOG_LEVEL_INFO, tr("Compiling libraries...")),
-		&RecipeByPrefixSuffixRunner{Prefix: constants.HOOKS_LIBRARIES_PREBUILD, Suffix: constants.HOOKS_PATTERN_SUFFIX},
+		utils.LogIfVerbose("info", tr("Compiling libraries...")),
+		&RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.libraries.prebuild", Suffix: ".pattern"},
 		&UnusedCompiledLibrariesRemover{},
 		&phases.LibrariesBuilder{},
-		&RecipeByPrefixSuffixRunner{Prefix: constants.HOOKS_LIBRARIES_POSTBUILD, Suffix: constants.HOOKS_PATTERN_SUFFIX},
+		&RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.libraries.postbuild", Suffix: ".pattern", SkipIfOnlyUpdatingCompilationDatabase: true},
 
-		utils.LogIfVerbose(constants.LOG_LEVEL_INFO, tr("Compiling core...")),
-		&RecipeByPrefixSuffixRunner{Prefix: constants.HOOKS_CORE_PREBUILD, Suffix: constants.HOOKS_PATTERN_SUFFIX},
+		utils.LogIfVerbose("info", tr("Compiling core...")),
+		&RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.core.prebuild", Suffix: ".pattern"},
 		&phases.CoreBuilder{},
-		&RecipeByPrefixSuffixRunner{Prefix: constants.HOOKS_CORE_POSTBUILD, Suffix: constants.HOOKS_PATTERN_SUFFIX},
+		&RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.core.postbuild", Suffix: ".pattern", SkipIfOnlyUpdatingCompilationDatabase: true},
 
-		utils.LogIfVerbose(constants.LOG_LEVEL_INFO, tr("Linking everything together...")),
-		&RecipeByPrefixSuffixRunner{Prefix: constants.HOOKS_LINKING_PRELINK, Suffix: constants.HOOKS_PATTERN_SUFFIX},
+		utils.LogIfVerbose("info", tr("Linking everything together...")),
+		&RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.linking.prelink", Suffix: ".pattern"},
 		&phases.Linker{},
-		&RecipeByPrefixSuffixRunner{Prefix: constants.HOOKS_LINKING_POSTLINK, Suffix: constants.HOOKS_PATTERN_SUFFIX},
+		&RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.linking.postlink", Suffix: ".pattern", SkipIfOnlyUpdatingCompilationDatabase: true},
 
-		&RecipeByPrefixSuffixRunner{Prefix: constants.HOOKS_OBJCOPY_PREOBJCOPY, Suffix: constants.HOOKS_PATTERN_SUFFIX},
-		&RecipeByPrefixSuffixRunner{Prefix: "recipe.objcopy.", Suffix: constants.HOOKS_PATTERN_SUFFIX},
-		&RecipeByPrefixSuffixRunner{Prefix: constants.HOOKS_OBJCOPY_POSTOBJCOPY, Suffix: constants.HOOKS_PATTERN_SUFFIX},
+		&RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.objcopy.preobjcopy", Suffix: ".pattern"},
+		&RecipeByPrefixSuffixRunner{Prefix: "recipe.objcopy.", Suffix: ".pattern", SkipIfOnlyUpdatingCompilationDatabase: true},
+		&RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.objcopy.postobjcopy", Suffix: ".pattern", SkipIfOnlyUpdatingCompilationDatabase: true},
 
 		&MergeSketchWithBootloader{},
 
-		&RecipeByPrefixSuffixRunner{Prefix: constants.HOOKS_POSTBUILD, Suffix: constants.HOOKS_PATTERN_SUFFIX},
+		&RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.postbuild", Suffix: ".pattern", SkipIfOnlyUpdatingCompilationDatabase: true},
 	}
 
 	mainErr := runCommands(ctx, commands)
@@ -149,7 +148,7 @@ func (s *Preprocess) Run(ctx *types.Context) error {
 
 		&ContainerBuildOptions{},
 
-		&RecipeByPrefixSuffixRunner{Prefix: constants.HOOKS_PREBUILD, Suffix: constants.HOOKS_PATTERN_SUFFIX},
+		&RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.prebuild", Suffix: ".pattern"},
 
 		&ContainerMergeCopySketchFiles{},
 
@@ -203,7 +202,7 @@ func runCommands(ctx *types.Context, commands []types.Command) error {
 
 func PrintRingNameIfDebug(ctx *types.Context, command types.Command) {
 	if ctx.DebugLevel >= 10 {
-		ctx.GetLogger().Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "Ts: {0} - Running: {1}", strconv.FormatInt(time.Now().Unix(), 10), reflect.Indirect(reflect.ValueOf(command)).Type().Name())
+		ctx.GetLogger().Fprintln(os.Stdout, "debug", "Ts: {0} - Running: {1}", strconv.FormatInt(time.Now().Unix(), 10), reflect.Indirect(reflect.ValueOf(command)).Type().Name())
 	}
 }
 

legacy/builder/constants/constants.go

@@ -47,31 +47,6 @@ const FOLDER_CORE = "core"
 const FOLDER_PREPROC = "preproc"
 const FOLDER_SKETCH = "sketch"
 const FOLDER_TOOLS = "tools"
-const hooks_core = hooks + ".core"
-const HOOKS_CORE_POSTBUILD = hooks_core + hooks_postbuild_suffix
-const HOOKS_CORE_PREBUILD = hooks_core + hooks_prebuild_suffix
-const hooks_libraries = hooks + ".libraries"
-const HOOKS_LIBRARIES_POSTBUILD = hooks_libraries + hooks_postbuild_suffix
-const HOOKS_LIBRARIES_PREBUILD = hooks_libraries + hooks_prebuild_suffix
-const hooks_linking = hooks + ".linking"
-const HOOKS_LINKING_POSTLINK = hooks_linking + hooks_postlink_suffix
-const HOOKS_LINKING_PRELINK = hooks_linking + hooks_prelink_suffix
-const hooks_objcopy = hooks + ".objcopy"
-const HOOKS_OBJCOPY_POSTOBJCOPY = hooks_objcopy + hooks_postobjcopy_suffix
-const HOOKS_OBJCOPY_PREOBJCOPY = hooks_objcopy + hooks_preobjcopy_suffix
-const HOOKS_PATTERN_SUFFIX = ".pattern"
-const HOOKS_POSTBUILD = hooks + hooks_postbuild_suffix
-const hooks_postbuild_suffix = ".postbuild"
-const hooks_postlink_suffix = ".postlink"
-const hooks_postobjcopy_suffix = ".postobjcopy"
-const HOOKS_PREBUILD = hooks + hooks_prebuild_suffix
-const hooks_prebuild_suffix = ".prebuild"
-const hooks_prelink_suffix = ".prelink"
-const hooks_preobjcopy_suffix = ".preobjcopy"
-const hooks = "recipe.hooks"
-const hooks_sketch = hooks + ".sketch"
-const HOOKS_SKETCH_POSTBUILD = hooks_sketch + hooks_postbuild_suffix
-const HOOKS_SKETCH_PREBUILD = hooks_sketch + hooks_prebuild_suffix
 const LIBRARY_ALL_ARCHS = "*"
 const LIBRARY_EMAIL = "email"
 const LIBRARY_FOLDER_ARCH = "arch"

legacy/builder/recipe_runner.go

@@ -29,8 +29,9 @@ import (
 )
 
 type RecipeByPrefixSuffixRunner struct {
-	Prefix string
-	Suffix string
+	Prefix                                string
+	Suffix                                string
+	SkipIfOnlyUpdatingCompilationDatabase bool
 }
 
 func (s *RecipeByPrefixSuffixRunner) Run(ctx *types.Context) error {
@@ -53,7 +54,7 @@ func (s *RecipeByPrefixSuffixRunner) Run(ctx *types.Context) error {
 			return errors.WithStack(err)
 		}
 
-		if ctx.OnlyUpdateCompilationDatabase {
+		if ctx.OnlyUpdateCompilationDatabase && s.SkipIfOnlyUpdatingCompilationDatabase {
 			if ctx.Verbose {
 				ctx.GetLogger().Println("info", tr("Skipping: {0}"), strings.Join(command.Args, " "))
 			}

legacy/builder/test/recipe_runner_test.go

@@ -19,10 +19,8 @@ import (
 	"testing"
 
 	"github.com/arduino/arduino-cli/legacy/builder"
-	"github.com/arduino/arduino-cli/legacy/builder/constants"
 	"github.com/arduino/arduino-cli/legacy/builder/types"
 	"github.com/arduino/go-properties-orderedmap"
-	"github.com/stretchr/testify/require"
 )
 
 // TODO
@@ -38,18 +36,11 @@ func TestRecipeRunner(t *testing.T) {
 
 	commands := []types.Command{
 		&builder.AddAdditionalEntriesToContext{},
-		&builder.RecipeByPrefixSuffixRunner{Prefix: constants.HOOKS_PREBUILD, Suffix: constants.HOOKS_PATTERN_SUFFIX},
+		&builder.RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.prebuild", Suffix: ".pattern"},
 	}
 
 	for _, command := range commands {
 		err := command.Run(ctx)
 		NoError(t, err)
 	}
 }
-
-func TestRecipesComposition(t *testing.T) {
-	require.Equal(t, "recipe.hooks.core.postbuild", constants.HOOKS_CORE_POSTBUILD)
-	require.Equal(t, "recipe.hooks.postbuild", constants.HOOKS_POSTBUILD)
-	require.Equal(t, "recipe.hooks.linking.prelink", constants.HOOKS_LINKING_PRELINK)
-	require.Equal(t, "recipe.hooks.objcopy.preobjcopy", constants.HOOKS_OBJCOPY_PREOBJCOPY)
-}

test/test_compile_part_4.py

@@ -292,6 +292,19 @@ def test_compile_with_esp8266_bundled_libraries(run_command, data_dir, copy_sket
     assert "\n".join(expected_output) not in res.stdout
 
 
+def test_generate_compile_commands_json_with_esp32(run_command, data_dir, copy_sketch):
+    # https://github.com/arduino/arduino-cli/issues/1547
+    assert run_command(["update"])
+
+    # Update index with esp32 core and install it
+    url = "https://raw.githubusercontent.com/espressif/arduino-esp32/gh-pages/package_esp32_index.json"
+    assert run_command(["core", "update-index", f"--additional-urls={url}"])
+    assert run_command(["core", "install", "esp32:esp32@2.0.1", f"--additional-urls={url}"])
+
+    sketch_path = copy_sketch("sketch_simple")
+    assert run_command(["compile", "-b", "esp32:esp32:featheresp32", "--only-compilation-database", sketch_path])
+
+
 def test_compile_sketch_with_tpp_file_include(run_command, copy_sketch):
     assert run_command(["update"])