[skip-changelog] builder: refactor utils functions (#2344)

arduino/builder/archive_compiled_files.go

+// This file is part of arduino-cli.
+//
+// Copyright 2023 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to license@arduino.cc.
+
+package builder
+
+import (
+	"github.com/arduino/go-paths-helper"
+	"github.com/pkg/errors"
+)
+
+// ArchiveCompiledFiles fixdoc
+func (b *Builder) archiveCompiledFiles(buildPath *paths.Path, archiveFile *paths.Path, objectFilesToArchive paths.PathList) (*paths.Path, error) {
+	archiveFilePath := buildPath.JoinPath(archiveFile)
+
+	if b.onlyUpdateCompilationDatabase {
+		if b.logger.Verbose() {
+			b.logger.Info(tr("Skipping archive creation of: %[1]s", archiveFilePath))
+		}
+		return archiveFilePath, nil
+	}
+
+	if archiveFileStat, err := archiveFilePath.Stat(); err == nil {
+		rebuildArchive := false
+		for _, objectFile := range objectFilesToArchive {
+			objectFileStat, err := objectFile.Stat()
+			if err != nil || objectFileStat.ModTime().After(archiveFileStat.ModTime()) {
+				// need to rebuild the archive
+				rebuildArchive = true
+				break
+			}
+		}
+
+		// something changed, rebuild the core archive
+		if rebuildArchive {
+			if err := archiveFilePath.Remove(); err != nil {
+				return nil, errors.WithStack(err)
+			}
+		} else {
+			if b.logger.Verbose() {
+				b.logger.Info(tr("Using previously compiled file: %[1]s", archiveFilePath))
+			}
+			return archiveFilePath, nil
+		}
+	}
+
+	for _, objectFile := range objectFilesToArchive {
+		properties := b.buildProperties.Clone()
+		properties.Set("archive_file", archiveFilePath.Base())
+		properties.SetPath("archive_file_path", archiveFilePath)
+		properties.SetPath("object_file", objectFile)
+
+		command, err := b.prepareCommandForRecipe(properties, "recipe.ar.pattern", false)
+		if err != nil {
+			return nil, errors.WithStack(err)
+		}
+
+		if err := b.execCommand(command); err != nil {
+			return nil, errors.WithStack(err)
+		}
+	}
+
+	return archiveFilePath, nil
+}

arduino/builder/builder.go

@@ -19,15 +19,20 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"os"
+	"path/filepath"
+	"strings"
 
 	"github.com/arduino/arduino-cli/arduino/builder/internal/compilation"
 	"github.com/arduino/arduino-cli/arduino/builder/internal/detector"
 	"github.com/arduino/arduino-cli/arduino/builder/internal/logger"
 	"github.com/arduino/arduino-cli/arduino/builder/internal/progress"
+	"github.com/arduino/arduino-cli/arduino/builder/internal/utils"
 	"github.com/arduino/arduino-cli/arduino/cores"
 	"github.com/arduino/arduino-cli/arduino/libraries"
 	"github.com/arduino/arduino-cli/arduino/libraries/librariesmanager"
 	"github.com/arduino/arduino-cli/arduino/sketch"
+	"github.com/arduino/arduino-cli/executils"
 	rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1"
 	"github.com/arduino/go-paths-helper"
 	"github.com/arduino/go-properties-orderedmap"
@@ -268,19 +273,16 @@ func (b *Builder) preprocess() error {
 		return err
 	}
 	b.Progress.CompleteStep()
-	b.Progress.PushProgress()
 
 	if err := b.RunRecipe("recipe.hooks.prebuild", ".pattern", false); err != nil {
 		return err
 	}
 	b.Progress.CompleteStep()
-	b.Progress.PushProgress()
 
 	if err := b.prepareSketchBuildPath(); err != nil {
 		return err
 	}
 	b.Progress.CompleteStep()
-	b.Progress.PushProgress()
 
 	b.logIfVerbose(false, tr("Detecting libraries used..."))
 	err := b.libsDetector.FindIncludes(
@@ -297,18 +299,15 @@ func (b *Builder) preprocess() error {
 		return err
 	}
 	b.Progress.CompleteStep()
-	b.Progress.PushProgress()
 
 	b.warnAboutArchIncompatibleLibraries(b.libsDetector.ImportedLibraries())
 	b.Progress.CompleteStep()
-	b.Progress.PushProgress()
 
 	b.logIfVerbose(false, tr("Generating function prototypes..."))
 	if err := b.preprocessSketch(b.libsDetector.IncludeFolders()); err != nil {
 		return err
 	}
 	b.Progress.CompleteStep()
-	b.Progress.PushProgress()
 
 	return nil
 }
@@ -337,11 +336,9 @@ func (b *Builder) Build() error {
 
 	b.libsDetector.PrintUsedAndNotUsedLibraries(buildErr != nil)
 	b.Progress.CompleteStep()
-	b.Progress.PushProgress()
 
 	b.printUsedLibraries(b.libsDetector.ImportedLibraries())
 	b.Progress.CompleteStep()
-	b.Progress.PushProgress()
 
 	if buildErr != nil {
 		return buildErr
@@ -350,13 +347,11 @@ func (b *Builder) Build() error {
 		return err
 	}
 	b.Progress.CompleteStep()
-	b.Progress.PushProgress()
 
 	if err := b.size(); err != nil {
 		return err
 	}
 	b.Progress.CompleteStep()
-	b.Progress.PushProgress()
 
 	return nil
 }
@@ -368,115 +363,155 @@ func (b *Builder) build() error {
 		return err
 	}
 	b.Progress.CompleteStep()
-	b.Progress.PushProgress()
 
-	if err := b.BuildSketch(b.libsDetector.IncludeFolders()); err != nil {
+	if err := b.buildSketch(b.libsDetector.IncludeFolders()); err != nil {
 		return err
 	}
 	b.Progress.CompleteStep()
-	b.Progress.PushProgress()
 
 	if err := b.RunRecipe("recipe.hooks.sketch.postbuild", ".pattern", true); err != nil {
 		return err
 	}
 	b.Progress.CompleteStep()
-	b.Progress.PushProgress()
 
 	b.logIfVerbose(false, tr("Compiling libraries..."))
 	if err := b.RunRecipe("recipe.hooks.libraries.prebuild", ".pattern", false); err != nil {
 		return err
 	}
 	b.Progress.CompleteStep()
-	b.Progress.PushProgress()
 
 	if err := b.removeUnusedCompiledLibraries(b.libsDetector.ImportedLibraries()); err != nil {
 		return err
 	}
 	b.Progress.CompleteStep()
-	b.Progress.PushProgress()
 
 	if err := b.buildLibraries(b.libsDetector.IncludeFolders(), b.libsDetector.ImportedLibraries()); err != nil {
 		return err
 	}
 	b.Progress.CompleteStep()
-	b.Progress.PushProgress()
 
 	if err := b.RunRecipe("recipe.hooks.libraries.postbuild", ".pattern", true); err != nil {
 		return err
 	}
 	b.Progress.CompleteStep()
-	b.Progress.PushProgress()
 
 	b.logIfVerbose(false, tr("Compiling core..."))
 	if err := b.RunRecipe("recipe.hooks.core.prebuild", ".pattern", false); err != nil {
 		return err
 	}
 	b.Progress.CompleteStep()
-	b.Progress.PushProgress()
 
 	if err := b.buildCore(); err != nil {
 		return err
 	}
 	b.Progress.CompleteStep()
-	b.Progress.PushProgress()
 
 	if err := b.RunRecipe("recipe.hooks.core.postbuild", ".pattern", true); err != nil {
 		return err
 	}
 	b.Progress.CompleteStep()
-	b.Progress.PushProgress()
 
 	b.logIfVerbose(false, tr("Linking everything together..."))
 	if err := b.RunRecipe("recipe.hooks.linking.prelink", ".pattern", false); err != nil {
 		return err
 	}
 	b.Progress.CompleteStep()
-	b.Progress.PushProgress()
 
 	if err := b.link(); err != nil {
 		return err
 	}
 	b.Progress.CompleteStep()
-	b.Progress.PushProgress()
 
 	if err := b.RunRecipe("recipe.hooks.linking.postlink", ".pattern", true); err != nil {
 		return err
 	}
 	b.Progress.CompleteStep()
-	b.Progress.PushProgress()
 
 	if err := b.RunRecipe("recipe.hooks.objcopy.preobjcopy", ".pattern", false); err != nil {
 		return err
 	}
 	b.Progress.CompleteStep()
-	b.Progress.PushProgress()
 
 	if err := b.RunRecipe("recipe.objcopy.", ".pattern", true); err != nil {
 		return err
 	}
 	b.Progress.CompleteStep()
-	b.Progress.PushProgress()
 
 	if err := b.RunRecipe("recipe.hooks.objcopy.postobjcopy", ".pattern", true); err != nil {
 		return err
 	}
 	b.Progress.CompleteStep()
-	b.Progress.PushProgress()
 
-	if err := b.MergeSketchWithBootloader(); err != nil {
+	if err := b.mergeSketchWithBootloader(); err != nil {
 		return err
 	}
 	b.Progress.CompleteStep()
-	b.Progress.PushProgress()
 
 	if err := b.RunRecipe("recipe.hooks.postbuild", ".pattern", true); err != nil {
 		return err
 	}
 	b.Progress.CompleteStep()
-	b.Progress.PushProgress()
 
 	if b.compilationDatabase != nil {
 		b.compilationDatabase.SaveToFile()
 	}
 	return nil
 }
+
+func (b *Builder) prepareCommandForRecipe(buildProperties *properties.Map, recipe string, removeUnsetProperties bool) (*executils.Process, error) {
+	pattern := buildProperties.Get(recipe)
+	if pattern == "" {
+		return nil, fmt.Errorf(tr("%[1]s pattern is missing"), recipe)
+	}
+
+	commandLine := buildProperties.ExpandPropsInString(pattern)
+	if removeUnsetProperties {
+		commandLine = properties.DeleteUnexpandedPropsFromString(commandLine)
+	}
+
+	parts, err := properties.SplitQuotedString(commandLine, `"'`, false)
+	if err != nil {
+		return nil, err
+	}
+
+	// if the overall commandline is too long for the platform
+	// try reducing the length by making the filenames relative
+	// and changing working directory to build.path
+	var relativePath string
+	if len(commandLine) > 30000 {
+		relativePath = buildProperties.Get("build.path")
+		for i, arg := range parts {
+			if _, err := os.Stat(arg); os.IsNotExist(err) {
+				continue
+			}
+			rel, err := filepath.Rel(relativePath, arg)
+			if err == nil && !strings.Contains(rel, "..") && len(rel) < len(arg) {
+				parts[i] = rel
+			}
+		}
+	}
+
+	command, err := executils.NewProcess(nil, parts...)
+	if err != nil {
+		return nil, err
+	}
+	if relativePath != "" {
+		command.SetDir(relativePath)
+	}
+
+	return command, nil
+}
+
+func (b *Builder) execCommand(command *executils.Process) error {
+	if b.logger.Verbose() {
+		b.logger.Info(utils.PrintableCommand(command.GetArgs()))
+		command.RedirectStdoutTo(b.logger.Stdout())
+	}
+	command.RedirectStderrTo(b.logger.Stderr())
+
+	if err := command.Start(); err != nil {
+		return err
+	}
+
+	return command.Wait()
+}

arduino/builder/compilation.go

+// This file is part of arduino-cli.
+//
+// Copyright 2023 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to license@arduino.cc.
+
+package builder
+
+import (
+	"bytes"
+	"fmt"
+	"runtime"
+	"strings"
+	"sync"
+
+	"github.com/arduino/arduino-cli/arduino/builder/internal/utils"
+	"github.com/arduino/arduino-cli/arduino/globals"
+	"github.com/arduino/go-paths-helper"
+	"github.com/pkg/errors"
+)
+
+func (b *Builder) compileFiles(
+	sourceDir *paths.Path,
+	buildPath *paths.Path,
+	recurse bool,
+	includes []string,
+) (paths.PathList, error) {
+	validExtensions := []string{}
+	for ext := range globals.SourceFilesValidExtensions {
+		validExtensions = append(validExtensions, ext)
+	}
+
+	sources, err := utils.FindFilesInFolder(sourceDir, recurse, validExtensions...)
+	if err != nil {
+		return nil, err
+	}
+
+	b.Progress.AddSubSteps(len(sources))
+	defer b.Progress.RemoveSubSteps()
+
+	objectFiles := paths.NewPathList()
+	var objectFilesMux sync.Mutex
+	if len(sources) == 0 {
+		return objectFiles, nil
+	}
+	var errorsList []error
+	var errorsMux sync.Mutex
+
+	queue := make(chan *paths.Path)
+	job := func(source *paths.Path) {
+		recipe := fmt.Sprintf("recipe%s.o.pattern", source.Ext())
+		if !b.buildProperties.ContainsKey(recipe) {
+			recipe = fmt.Sprintf("recipe%s.o.pattern", globals.SourceFilesValidExtensions[source.Ext()])
+		}
+		objectFile, err := b.compileFileWithRecipe(sourceDir, source, buildPath, includes, recipe)
+		if err != nil {
+			errorsMux.Lock()
+			errorsList = append(errorsList, err)
+			errorsMux.Unlock()
+		} else {
+			objectFilesMux.Lock()
+			objectFiles.Add(objectFile)
+			objectFilesMux.Unlock()
+		}
+	}
+
+	// Spawn jobs runners
+	var wg sync.WaitGroup
+	if b.jobs == 0 {
+		b.jobs = runtime.NumCPU()
+	}
+	for i := 0; i < b.jobs; i++ {
+		wg.Add(1)
+		go func() {
+			for source := range queue {
+				job(source)
+			}
+			wg.Done()
+		}()
+	}
+
+	// Feed jobs until error or done
+	for _, source := range sources {
+		errorsMux.Lock()
+		gotError := len(errorsList) > 0
+		errorsMux.Unlock()
+		if gotError {
+			break
+		}
+		queue <- source
+
+		b.Progress.CompleteStep()
+	}
+	close(queue)
+	wg.Wait()
+	if len(errorsList) > 0 {
+		// output the first error
+		return nil, errors.WithStack(errorsList[0])
+	}
+	objectFiles.Sort()
+	return objectFiles, nil
+}
+
+// CompileFilesRecursive fixdoc
+func (b *Builder) compileFileWithRecipe(
+	sourcePath *paths.Path,
+	source *paths.Path,
+	buildPath *paths.Path,
+	includes []string,
+	recipe string,
+) (*paths.Path, error) {
+	properties := b.buildProperties.Clone()
+	properties.Set("compiler.warning_flags", properties.Get("compiler.warning_flags."+b.logger.WarningsLevel()))
+	properties.Set("includes", strings.Join(includes, " "))
+	properties.SetPath("source_file", source)
+	relativeSource, err := sourcePath.RelTo(source)
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+	depsFile := buildPath.Join(relativeSource.String() + ".d")
+	objectFile := buildPath.Join(relativeSource.String() + ".o")
+
+	properties.SetPath("object_file", objectFile)
+	err = objectFile.Parent().MkdirAll()
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+
+	objIsUpToDate, err := utils.ObjFileIsUpToDate(source, objectFile, depsFile)
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+
+	command, err := b.prepareCommandForRecipe(properties, recipe, false)
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+	if b.compilationDatabase != nil {
+		b.compilationDatabase.Add(source, command)
+	}
+	if !objIsUpToDate && !b.onlyUpdateCompilationDatabase {
+		commandStdout, commandStderr := &bytes.Buffer{}, &bytes.Buffer{}
+		command.RedirectStdoutTo(commandStdout)
+		command.RedirectStderrTo(commandStderr)
+
+		if b.logger.Verbose() {
+			b.logger.Info(utils.PrintableCommand(command.GetArgs()))
+		}
+		// Since this compile could be multithreaded, we first capture the command output
+		if err := command.Start(); err != nil {
+			return nil, err
+		}
+		err := command.Wait()
+		// and transfer all at once at the end...
+		if b.logger.Verbose() {
+			b.logger.WriteStdout(commandStdout.Bytes())
+		}
+		b.logger.WriteStderr(commandStderr.Bytes())
+
+		// ...and then return the error
+		if err != nil {
+			return nil, errors.WithStack(err)
+		}
+	} else if b.logger.Verbose() {
+		if objIsUpToDate {
+			b.logger.Info(tr("Using previously compiled file: %[1]s", objectFile))
+		} else {
+			b.logger.Info(tr("Skipping compile of: %[1]s", objectFile))
+		}
+	}
+
+	return objectFile, nil
+}

arduino/builder/core.go

@@ -71,13 +71,10 @@ func (b *Builder) compileCore() (*paths.Path, paths.PathList, error) {
 	var err error
 	variantObjectFiles := paths.NewPathList()
 	if variantFolder != nil && variantFolder.IsDir() {
-		variantObjectFiles, err = utils.CompileFilesRecursive(
-			variantFolder, b.coreBuildPath, b.buildProperties, includes,
-			b.onlyUpdateCompilationDatabase,
-			b.compilationDatabase,
-			b.jobs,
-			b.logger,
-			b.Progress,
+		variantObjectFiles, err = b.compileFiles(
+			variantFolder, b.coreBuildPath,
+			true, /** recursive **/
+			includes,
 		)
 		if err != nil {
 			return nil, nil, errors.WithStack(err)
@@ -122,25 +119,16 @@ func (b *Builder) compileCore() (*paths.Path, paths.PathList, error) {
 		}
 	}
 
-	coreObjectFiles, err := utils.CompileFilesRecursive(
-		coreFolder, b.coreBuildPath, b.buildProperties, includes,
-		b.onlyUpdateCompilationDatabase,
-		b.compilationDatabase,
-		b.jobs,
-		b.logger,
-		b.Progress,
+	coreObjectFiles, err := b.compileFiles(
+		coreFolder, b.coreBuildPath,
+		true, /** recursive **/
+		includes,
 	)
 	if err != nil {
 		return nil, nil, errors.WithStack(err)
 	}
 
-	archiveFile, verboseInfo, err := utils.ArchiveCompiledFiles(
-		b.coreBuildPath, paths.New("core.a"), coreObjectFiles, b.buildProperties,
-		b.onlyUpdateCompilationDatabase, b.logger.Verbose(), b.logger.Stdout(), b.logger.Stderr(),
-	)
-	if b.logger.Verbose() {
-		b.logger.Info(string(verboseInfo))
-	}
+	archiveFile, err := b.archiveCompiledFiles(b.coreBuildPath, paths.New("core.a"), coreObjectFiles)
 	if err != nil {
 		return nil, nil, errors.WithStack(err)
 	}

arduino/builder/export_cmake.go

@@ -275,9 +275,9 @@ func (b *Builder) exportProjectCMake(importedLibraries libraries.List, includeFo
 	var dynamicLibsFromGccMinusL []string
 	var linkDirectories []string
 
-	extractCompileFlags(b.buildProperties, "recipe.c.combine.pattern", &defines, &dynamicLibsFromGccMinusL, &linkerflags, &linkDirectories)
-	extractCompileFlags(b.buildProperties, "recipe.c.o.pattern", &defines, &dynamicLibsFromGccMinusL, &linkerflags, &linkDirectories)
-	extractCompileFlags(b.buildProperties, "recipe.cpp.o.pattern", &defines, &dynamicLibsFromGccMinusL, &linkerflags, &linkDirectories)
+	b.extractCompileFlags(b.buildProperties, "recipe.c.combine.pattern", &defines, &dynamicLibsFromGccMinusL, &linkerflags, &linkDirectories)
+	b.extractCompileFlags(b.buildProperties, "recipe.c.o.pattern", &defines, &dynamicLibsFromGccMinusL, &linkerflags, &linkDirectories)
+	b.extractCompileFlags(b.buildProperties, "recipe.cpp.o.pattern", &defines, &dynamicLibsFromGccMinusL, &linkerflags, &linkDirectories)
 
 	// Extract folders with .h in them for adding in include list
 	headerFiles, _ := utils.FindFilesInFolder(cmakeFolder, true, validHeaderExtensions...)
@@ -348,7 +348,7 @@ func (b *Builder) exportProjectCMake(importedLibraries libraries.List, includeFo
 	return nil
 }
 
-func extractCompileFlags(buildProperties *properties.Map, recipe string, defines, dynamicLibs, linkerflags, linkDirectories *[]string) {
+func (b *Builder) extractCompileFlags(buildProperties *properties.Map, recipe string, defines, dynamicLibs, linkerflags, linkDirectories *[]string) {
 	appendIfNotPresent := func(target []string, elements ...string) []string {
 		for _, element := range elements {
 			if !slices.Contains(target, element) {
@@ -358,7 +358,7 @@ func extractCompileFlags(buildProperties *properties.Map, recipe string, defines
 		return target
 	}
 
-	command, _ := utils.PrepareCommandForRecipe(buildProperties, recipe, true)
+	command, _ := b.prepareCommandForRecipe(buildProperties, recipe, true)
 
 	for _, arg := range command.GetArgs() {
 		if strings.HasPrefix(arg, "-D") {

arduino/builder/internal/progress/progress.go

@@ -53,10 +53,10 @@ func (p *Struct) RemoveSubSteps() {
 // CompleteStep fixdoc
 func (p *Struct) CompleteStep() {
 	p.Progress += p.StepAmount
+	p.pushProgress()
 }
 
-// PushProgress fixdoc
-func (p *Struct) PushProgress() {
+func (p *Struct) pushProgress() {
 	if p.callback != nil {
 		p.callback(&rpc.TaskProgress{
 			Percent:   p.Progress,

arduino/builder/internal/utils/utils.go

@@ -16,25 +16,13 @@
 package utils
 
 import (
-	"bytes"
-	"fmt"
-	"io"
 	"os"
-	"path/filepath"
-	"runtime"
 	"strings"
-	"sync"
 	"unicode"
 
-	"github.com/arduino/arduino-cli/arduino/builder/internal/compilation"
-	"github.com/arduino/arduino-cli/arduino/builder/internal/logger"
-	"github.com/arduino/arduino-cli/arduino/builder/internal/progress"
-	"github.com/arduino/arduino-cli/arduino/globals"
-	"github.com/arduino/arduino-cli/executils"
 	"github.com/arduino/arduino-cli/i18n"
 	f "github.com/arduino/arduino-cli/internal/algorithms"
 	"github.com/arduino/go-paths-helper"
-	"github.com/arduino/go-properties-orderedmap"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/text/runes"
@@ -210,14 +198,6 @@ func FindFilesInFolder(dir *paths.Path, recurse bool, extensions ...string) (pat
 	return dir.ReadDir(fileFilter)
 }
 
-// nolint
-const (
-	Ignore        = 0 // Redirect to null
-	Show          = 1 // Show on stdout/stderr as normal
-	ShowIfVerbose = 2 // Show if verbose is set, Ignore otherwise
-	Capture       = 3 // Capture into buffer
-)
-
 func printableArgument(arg string) string {
 	if strings.ContainsAny(arg, "\"\\ \t") {
 		arg = strings.ReplaceAll(arg, "\\", "\\\\")
@@ -227,56 +207,14 @@ func printableArgument(arg string) string {
 	return arg
 }
 
-// Convert a command and argument slice back to a printable string.
+// PrintableCommand Convert a command and argument slice back to a printable string.
 // This adds basic escaping which is sufficient for debug output, but
 // probably not for shell interpretation. This essentially reverses
 // ParseCommandLine.
-func printableCommand(parts []string) string {
+func PrintableCommand(parts []string) string {
 	return strings.Join(f.Map(parts, printableArgument), " ")
 }
 
-// ExecCommand fixdoc
-func ExecCommand(
-	verbose bool,
-	stdoutWriter, stderrWriter io.Writer,
-	command *executils.Process, stdout int, stderr int,
-) ([]byte, []byte, []byte, error) {
-	verboseInfoBuf := &bytes.Buffer{}
-	if verbose {
-		verboseInfoBuf.WriteString(printableCommand(command.GetArgs()))
-	}
-
-	stdoutBuffer := &bytes.Buffer{}
-	if stdout == Capture {
-		command.RedirectStdoutTo(stdoutBuffer)
-	} else if stdout == Show || (stdout == ShowIfVerbose && verbose) {
-		if stdoutWriter != nil {
-			command.RedirectStdoutTo(stdoutWriter)
-		} else {
-			command.RedirectStdoutTo(os.Stdout)
-		}
-	}
-
-	stderrBuffer := &bytes.Buffer{}
-	if stderr == Capture {
-		command.RedirectStderrTo(stderrBuffer)
-	} else if stderr == Show || (stderr == ShowIfVerbose && verbose) {
-		if stderrWriter != nil {
-			command.RedirectStderrTo(stderrWriter)
-		} else {
-			command.RedirectStderrTo(os.Stderr)
-		}
-	}
-
-	err := command.Start()
-	if err != nil {
-		return verboseInfoBuf.Bytes(), nil, nil, errors.WithStack(err)
-	}
-
-	err = command.Wait()
-	return verboseInfoBuf.Bytes(), stdoutBuffer.Bytes(), stderrBuffer.Bytes(), errors.WithStack(err)
-}
-
 // DirContentIsOlderThan returns true if the content of the given directory is
 // older than target file. If extensions are given, only the files with these
 // extensions are tested.
@@ -302,330 +240,3 @@ func DirContentIsOlderThan(dir *paths.Path, target *paths.Path, extensions ...st
 	}
 	return true, nil
 }
-
-// PrepareCommandForRecipe fixdoc
-func PrepareCommandForRecipe(buildProperties *properties.Map, recipe string, removeUnsetProperties bool) (*executils.Process, error) {
-	pattern := buildProperties.Get(recipe)
-	if pattern == "" {
-		return nil, errors.Errorf(tr("%[1]s pattern is missing"), recipe)
-	}
-
-	commandLine := buildProperties.ExpandPropsInString(pattern)
-	if removeUnsetProperties {
-		commandLine = properties.DeleteUnexpandedPropsFromString(commandLine)
-	}
-
-	parts, err := properties.SplitQuotedString(commandLine, `"'`, false)
-	if err != nil {
-		return nil, errors.WithStack(err)
-	}
-
-	// if the overall commandline is too long for the platform
-	// try reducing the length by making the filenames relative
-	// and changing working directory to build.path
-	var relativePath string
-	if len(commandLine) > 30000 {
-		relativePath = buildProperties.Get("build.path")
-		for i, arg := range parts {
-			if _, err := os.Stat(arg); os.IsNotExist(err) {
-				continue
-			}
-			rel, err := filepath.Rel(relativePath, arg)
-			if err == nil && !strings.Contains(rel, "..") && len(rel) < len(arg) {
-				parts[i] = rel
-			}
-		}
-	}
-
-	command, err := executils.NewProcess(nil, parts...)
-	if err != nil {
-		return nil, errors.WithStack(err)
-	}
-	if relativePath != "" {
-		command.SetDir(relativePath)
-	}
-
-	return command, nil
-}
-
-// CompileFiles fixdoc
-func CompileFiles(
-	sourceDir, buildPath *paths.Path,
-	buildProperties *properties.Map,
-	includes []string,
-	onlyUpdateCompilationDatabase bool,
-	compilationDatabase *compilation.Database,
-	jobs int,
-	builderLogger *logger.BuilderLogger,
-	progress *progress.Struct,
-) (paths.PathList, error) {
-	return compileFiles(
-		onlyUpdateCompilationDatabase,
-		compilationDatabase,
-		jobs,
-		sourceDir,
-		false,
-		buildPath, buildProperties, includes,
-		builderLogger,
-		progress,
-	)
-}
-
-// CompileFilesRecursive fixdoc
-func CompileFilesRecursive(
-	sourceDir, buildPath *paths.Path,
-	buildProperties *properties.Map,
-	includes []string,
-	onlyUpdateCompilationDatabase bool,
-	compilationDatabase *compilation.Database,
-	jobs int,
-	builderLogger *logger.BuilderLogger,
-	progress *progress.Struct,
-) (paths.PathList, error) {
-	return compileFiles(
-		onlyUpdateCompilationDatabase,
-		compilationDatabase,
-		jobs,
-		sourceDir,
-		true,
-		buildPath, buildProperties, includes,
-		builderLogger,
-		progress,
-	)
-}
-
-func compileFiles(
-	onlyUpdateCompilationDatabase bool,
-	compilationDatabase *compilation.Database,
-	jobs int,
-	sourceDir *paths.Path,
-	recurse bool,
-	buildPath *paths.Path,
-	buildProperties *properties.Map,
-	includes []string,
-	builderLogger *logger.BuilderLogger,
-	progress *progress.Struct,
-) (paths.PathList, error) {
-	validExtensions := []string{}
-	for ext := range globals.SourceFilesValidExtensions {
-		validExtensions = append(validExtensions, ext)
-	}
-
-	sources, err := FindFilesInFolder(sourceDir, recurse, validExtensions...)
-	if err != nil {
-		return nil, err
-	}
-
-	progress.AddSubSteps(len(sources))
-	defer progress.RemoveSubSteps()
-
-	objectFiles := paths.NewPathList()
-	var objectFilesMux sync.Mutex
-	if len(sources) == 0 {
-		return objectFiles, nil
-	}
-	var errorsList []error
-	var errorsMux sync.Mutex
-
-	queue := make(chan *paths.Path)
-	job := func(source *paths.Path) {
-		recipe := fmt.Sprintf("recipe%s.o.pattern", source.Ext())
-		if !buildProperties.ContainsKey(recipe) {
-			recipe = fmt.Sprintf("recipe%s.o.pattern", globals.SourceFilesValidExtensions[source.Ext()])
-		}
-		objectFile, verboseInfo, verboseStdout, stderr, err := compileFileWithRecipe(
-			compilationDatabase,
-			onlyUpdateCompilationDatabase,
-			sourceDir, source, buildPath, buildProperties, includes, recipe,
-			builderLogger,
-		)
-		if builderLogger.Verbose() {
-			builderLogger.WriteStdout(verboseStdout)
-			builderLogger.Info(string(verboseInfo))
-		}
-		builderLogger.WriteStderr(stderr)
-		if err != nil {
-			errorsMux.Lock()
-			errorsList = append(errorsList, err)
-			errorsMux.Unlock()
-		} else {
-			objectFilesMux.Lock()
-			objectFiles.Add(objectFile)
-			objectFilesMux.Unlock()
-		}
-	}
-
-	// Spawn jobs runners
-	var wg sync.WaitGroup
-	if jobs == 0 {
-		jobs = runtime.NumCPU()
-	}
-	for i := 0; i < jobs; i++ {
-		wg.Add(1)
-		go func() {
-			for source := range queue {
-				job(source)
-			}
-			wg.Done()
-		}()
-	}
-
-	// Feed jobs until error or done
-	for _, source := range sources {
-		errorsMux.Lock()
-		gotError := len(errorsList) > 0
-		errorsMux.Unlock()
-		if gotError {
-			break
-		}
-		queue <- source
-
-		progress.CompleteStep()
-		progress.PushProgress()
-	}
-	close(queue)
-	wg.Wait()
-	if len(errorsList) > 0 {
-		// output the first error
-		return nil, errors.WithStack(errorsList[0])
-	}
-	objectFiles.Sort()
-	return objectFiles, nil
-}
-
-func compileFileWithRecipe(
-	compilationDatabase *compilation.Database,
-	onlyUpdateCompilationDatabase bool,
-	sourcePath *paths.Path,
-	source *paths.Path,
-	buildPath *paths.Path,
-	buildProperties *properties.Map,
-	includes []string,
-	recipe string,
-	builderLogger *logger.BuilderLogger,
-) (*paths.Path, []byte, []byte, []byte, error) {
-	verboseStdout, verboseInfo, errOut := &bytes.Buffer{}, &bytes.Buffer{}, &bytes.Buffer{}
-
-	properties := buildProperties.Clone()
-	properties.Set("compiler.warning_flags", properties.Get("compiler.warning_flags."+builderLogger.WarningsLevel()))
-	properties.Set("includes", strings.Join(includes, " "))
-	properties.SetPath("source_file", source)
-	relativeSource, err := sourcePath.RelTo(source)
-	if err != nil {
-		return nil, nil, nil, nil, errors.WithStack(err)
-	}
-	depsFile := buildPath.Join(relativeSource.String() + ".d")
-	objectFile := buildPath.Join(relativeSource.String() + ".o")
-
-	properties.SetPath("object_file", objectFile)
-	err = objectFile.Parent().MkdirAll()
-	if err != nil {
-		return nil, nil, nil, nil, errors.WithStack(err)
-	}
-
-	objIsUpToDate, err := ObjFileIsUpToDate(source, objectFile, depsFile)
-	if err != nil {
-		return nil, nil, nil, nil, errors.WithStack(err)
-	}
-
-	command, err := PrepareCommandForRecipe(properties, recipe, false)
-	if err != nil {
-		return nil, nil, nil, nil, errors.WithStack(err)
-	}
-	if compilationDatabase != nil {
-		compilationDatabase.Add(source, command)
-	}
-	if !objIsUpToDate && !onlyUpdateCompilationDatabase {
-		// Since this compile could be multithreaded, we first capture the command output
-		info, stdout, stderr, err := ExecCommand(
-			builderLogger.Verbose(),
-			builderLogger.Stdout(),
-			builderLogger.Stderr(),
-			command,
-			Capture,
-			Capture,
-		)
-		// and transfer all at once at the end...
-		if builderLogger.Verbose() {
-			verboseInfo.Write(info)
-			verboseStdout.Write(stdout)
-		}
-		errOut.Write(stderr)
-
-		// ...and then return the error
-		if err != nil {
-			return nil, verboseInfo.Bytes(), verboseStdout.Bytes(), errOut.Bytes(), errors.WithStack(err)
-		}
-	} else if builderLogger.Verbose() {
-		if objIsUpToDate {
-			verboseInfo.WriteString(tr("Using previously compiled file: %[1]s", objectFile))
-		} else {
-			verboseInfo.WriteString(tr("Skipping compile of: %[1]s", objectFile))
-		}
-	}
-
-	return objectFile, verboseInfo.Bytes(), verboseStdout.Bytes(), errOut.Bytes(), nil
-}
-
-// ArchiveCompiledFiles fixdoc
-func ArchiveCompiledFiles(
-	buildPath *paths.Path, archiveFile *paths.Path, objectFilesToArchive paths.PathList, buildProperties *properties.Map,
-	onlyUpdateCompilationDatabase, verbose bool,
-	stdoutWriter, stderrWriter io.Writer,
-) (*paths.Path, []byte, error) {
-	verboseInfobuf := &bytes.Buffer{}
-	archiveFilePath := buildPath.JoinPath(archiveFile)
-
-	if onlyUpdateCompilationDatabase {
-		if verbose {
-			verboseInfobuf.WriteString(tr("Skipping archive creation of: %[1]s", archiveFilePath))
-		}
-		return archiveFilePath, verboseInfobuf.Bytes(), nil
-	}
-
-	if archiveFileStat, err := archiveFilePath.Stat(); err == nil {
-		rebuildArchive := false
-		for _, objectFile := range objectFilesToArchive {
-			objectFileStat, err := objectFile.Stat()
-			if err != nil || objectFileStat.ModTime().After(archiveFileStat.ModTime()) {
-				// need to rebuild the archive
-				rebuildArchive = true
-				break
-			}
-		}
-
-		// something changed, rebuild the core archive
-		if rebuildArchive {
-			if err := archiveFilePath.Remove(); err != nil {
-				return nil, nil, errors.WithStack(err)
-			}
-		} else {
-			if verbose {
-				verboseInfobuf.WriteString(tr("Using previously compiled file: %[1]s", archiveFilePath))
-			}
-			return archiveFilePath, verboseInfobuf.Bytes(), nil
-		}
-	}
-
-	for _, objectFile := range objectFilesToArchive {
-		properties := buildProperties.Clone()
-		properties.Set("archive_file", archiveFilePath.Base())
-		properties.SetPath("archive_file_path", archiveFilePath)
-		properties.SetPath("object_file", objectFile)
-
-		command, err := PrepareCommandForRecipe(properties, "recipe.ar.pattern", false)
-		if err != nil {
-			return nil, verboseInfobuf.Bytes(), errors.WithStack(err)
-		}
-
-		verboseInfo, _, _, err := ExecCommand(verbose, stdoutWriter, stderrWriter, command, ShowIfVerbose /* stdout */, Show /* stderr */)
-		if verbose {
-			verboseInfobuf.WriteString(string(verboseInfo))
-		}
-		if err != nil {
-			return nil, verboseInfobuf.Bytes(), errors.WithStack(err)
-		}
-	}
-
-	return archiveFilePath, verboseInfobuf.Bytes(), nil
-}

arduino/builder/internal/utils/utils_test.go

@@ -39,7 +39,7 @@ func TestPrintableCommand(t *testing.T) {
 		" \"specialchar-`~!@#$%^&*()-_=+[{]}\\\\|;:'\\\",<.>/?-argument\"" +
 		" \"arg   with spaces\" \"arg\twith\t\ttabs\"" +
 		" lastarg"
-	result := printableCommand(parts)
+	result := PrintableCommand(parts)
 	require.Equal(t, correct, result)
 }
 

arduino/builder/libraries.go

@@ -21,7 +21,6 @@ import (
 	"time"
 
 	"github.com/arduino/arduino-cli/arduino/builder/cpp"
-	"github.com/arduino/arduino-cli/arduino/builder/internal/utils"
 	"github.com/arduino/arduino-cli/arduino/libraries"
 	f "github.com/arduino/arduino-cli/internal/algorithms"
 	"github.com/arduino/go-paths-helper"
@@ -68,7 +67,7 @@ func (b *Builder) findExpectedPrecompiledLibFolder(
 	// Add fpu specifications if they exist
 	// To do so, resolve recipe.cpp.o.pattern,
 	// search for -mfpu=xxx -mfloat-abi=yyy and add to a subfolder
-	command, _ := utils.PrepareCommandForRecipe(buildProperties, "recipe.cpp.o.pattern", true)
+	command, _ := b.prepareCommandForRecipe(buildProperties, "recipe.cpp.o.pattern", true)
 	fpuSpecs := ""
 	for _, el := range command.GetArgs() {
 		if strings.Contains(el, FpuCflag) {
@@ -124,7 +123,6 @@ func (b *Builder) compileLibraries(libraries libraries.List, includes []string)
 		objectFiles.AddAll(libraryObjectFiles)
 
 		b.Progress.CompleteStep()
-		b.Progress.PushProgress()
 	}
 
 	return objectFiles, nil
@@ -190,26 +188,16 @@ func (b *Builder) compileLibrary(library *libraries.Library, includes []string)
 	}
 
 	if library.Layout == libraries.RecursiveLayout {
-		libObjectFiles, err := utils.CompileFilesRecursive(
-			library.SourceDir, libraryBuildPath, b.buildProperties, includes,
-			b.onlyUpdateCompilationDatabase,
-			b.compilationDatabase,
-			b.jobs,
-			b.logger,
-			b.Progress,
+		libObjectFiles, err := b.compileFiles(
+			library.SourceDir, libraryBuildPath,
+			true, /** recursive **/
+			includes,
 		)
 		if err != nil {
 			return nil, errors.WithStack(err)
 		}
 		if library.DotALinkage {
-			archiveFile, verboseInfo, err := utils.ArchiveCompiledFiles(
-				libraryBuildPath, paths.New(library.DirName+".a"), libObjectFiles, b.buildProperties,
-				b.onlyUpdateCompilationDatabase, b.logger.Verbose(),
-				b.logger.Stdout(), b.logger.Stderr(),
-			)
-			if b.logger.Verbose() {
-				b.logger.Info(string(verboseInfo))
-			}
+			archiveFile, err := b.archiveCompiledFiles(libraryBuildPath, paths.New(library.DirName+".a"), libObjectFiles)
 			if err != nil {
 				return nil, errors.WithStack(err)
 			}
@@ -221,13 +209,10 @@ func (b *Builder) compileLibrary(library *libraries.Library, includes []string)
 		if library.UtilityDir != nil {
 			includes = append(includes, cpp.WrapWithHyphenI(library.UtilityDir.String()))
 		}
-		libObjectFiles, err := utils.CompileFiles(
-			library.SourceDir, libraryBuildPath, b.buildProperties, includes,
-			b.onlyUpdateCompilationDatabase,
-			b.compilationDatabase,
-			b.jobs,
-			b.logger,
-			b.Progress,
+		libObjectFiles, err := b.compileFiles(
+			library.SourceDir, libraryBuildPath,
+			false, /** recursive **/
+			includes,
 		)
 		if err != nil {
 			return nil, errors.WithStack(err)
@@ -236,13 +221,10 @@ func (b *Builder) compileLibrary(library *libraries.Library, includes []string)
 
 		if library.UtilityDir != nil {
 			utilityBuildPath := libraryBuildPath.Join("utility")
-			utilityObjectFiles, err := utils.CompileFiles(
-				library.UtilityDir, utilityBuildPath, b.buildProperties, includes,
-				b.onlyUpdateCompilationDatabase,
-				b.compilationDatabase,
-				b.jobs,
-				b.logger,
-				b.Progress,
+			utilityObjectFiles, err := b.compileFiles(
+				library.UtilityDir, utilityBuildPath,
+				false, /** recursive **/
+				includes,
 			)
 			if err != nil {
 				return nil, errors.WithStack(err)

arduino/builder/linker.go

@@ -18,7 +18,6 @@ package builder
 import (
 	"strings"
 
-	"github.com/arduino/arduino-cli/arduino/builder/internal/utils"
 	f "github.com/arduino/arduino-cli/internal/algorithms"
 	"github.com/arduino/go-paths-helper"
 	"github.com/pkg/errors"
@@ -72,15 +71,12 @@ func (b *Builder) link() error {
 			properties.SetPath("archive_file_path", archive)
 			properties.SetPath("object_file", object)
 
-			command, err := utils.PrepareCommandForRecipe(properties, "recipe.ar.pattern", false)
+			command, err := b.prepareCommandForRecipe(properties, "recipe.ar.pattern", false)
 			if err != nil {
 				return errors.WithStack(err)
 			}
 
-			if verboseInfo, _, _, err := utils.ExecCommand(b.logger.Verbose(), b.logger.Stdout(), b.logger.Stderr(), command, utils.ShowIfVerbose /* stdout */, utils.Show /* stderr */); err != nil {
-				if b.logger.Verbose() {
-					b.logger.Info(string(verboseInfo))
-				}
+			if err := b.execCommand(command); err != nil {
 				return errors.WithStack(err)
 			}
 		}
@@ -96,17 +92,10 @@ func (b *Builder) link() error {
 	properties.Set("archive_file_path", b.buildArtifacts.coreArchiveFilePath.String())
 	properties.Set("object_files", objectFileList)
 
-	command, err := utils.PrepareCommandForRecipe(properties, "recipe.c.combine.pattern", false)
+	command, err := b.prepareCommandForRecipe(properties, "recipe.c.combine.pattern", false)
 	if err != nil {
 		return err
 	}
 
-	verboseInfo, _, _, err := utils.ExecCommand(b.logger.Verbose(), b.logger.Stdout(), b.logger.Stderr(), command, utils.ShowIfVerbose /* stdout */, utils.Show /* stderr */)
-	if b.logger.Verbose() {
-		b.logger.Info(string(verboseInfo))
-	}
-	if err != nil {
-		return err
-	}
-	return nil
+	return b.execCommand(command)
 }

arduino/builder/recipe.go

@@ -20,7 +20,6 @@ import (
 	"sort"
 	"strings"
 
-	"github.com/arduino/arduino-cli/arduino/builder/internal/utils"
 	properties "github.com/arduino/go-properties-orderedmap"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
@@ -39,7 +38,7 @@ func (b *Builder) RunRecipe(prefix, suffix string, skipIfOnlyUpdatingCompilation
 	for _, recipe := range recipes {
 		logrus.Debugf(fmt.Sprintf("Running recipe: %s", recipe))
 
-		command, err := utils.PrepareCommandForRecipe(properties, recipe, false)
+		command, err := b.prepareCommandForRecipe(properties, recipe, false)
 		if err != nil {
 			return errors.WithStack(err)
 		}
@@ -51,11 +50,7 @@ func (b *Builder) RunRecipe(prefix, suffix string, skipIfOnlyUpdatingCompilation
 			return nil
 		}
 
-		verboseInfo, _, _, err := utils.ExecCommand(b.logger.Verbose(), b.logger.Stdout(), b.logger.Stderr(), command, utils.ShowIfVerbose /* stdout */, utils.Show /* stderr */)
-		if b.logger.Verbose() {
-			b.logger.Info(string(verboseInfo))
-		}
-		if err != nil {
+		if err := b.execCommand(command); err != nil {
 			return errors.WithStack(err)
 		}
 	}

arduino/builder/sizer.go

@@ -16,6 +16,7 @@
 package builder
 
 import (
+	"bytes"
 	"encoding/json"
 	"fmt"
 	"regexp"
@@ -72,16 +73,20 @@ func (b *Builder) size() error {
 }
 
 func (b *Builder) checkSizeAdvanced() (ExecutablesFileSections, error) {
-	command, err := utils.PrepareCommandForRecipe(b.buildProperties, "recipe.advanced_size.pattern", false)
+	command, err := b.prepareCommandForRecipe(b.buildProperties, "recipe.advanced_size.pattern", false)
 	if err != nil {
 		return nil, errors.New(tr("Error while determining sketch size: %s", err))
 	}
-
-	verboseInfo, out, _, err := utils.ExecCommand(b.logger.Verbose(), b.logger.Stdout(), b.logger.Stderr(), command, utils.Capture /* stdout */, utils.Show /* stderr */)
 	if b.logger.Verbose() {
-		b.logger.Info(string(verboseInfo))
+		b.logger.Info(utils.PrintableCommand(command.GetArgs()))
 	}
-	if err != nil {
+	out := &bytes.Buffer{}
+	command.RedirectStdoutTo(out)
+	command.RedirectStderrTo(b.logger.Stderr())
+	if err := command.Start(); err != nil {
+		return nil, errors.New(tr("Error while determining sketch size: %s", err))
+	}
+	if err := command.Wait(); err != nil {
 		return nil, errors.New(tr("Error while determining sketch size: %s", err))
 	}
 
@@ -100,7 +105,7 @@ func (b *Builder) checkSizeAdvanced() (ExecutablesFileSections, error) {
 	}
 
 	var resp AdvancedSizerResponse
-	if err := json.Unmarshal(out, &resp); err != nil {
+	if err := json.Unmarshal(out.Bytes(), &resp); err != nil {
 		return nil, errors.New(tr("Error while determining sketch size: %s", err))
 	}
 
@@ -204,20 +209,27 @@ func (b *Builder) checkSize() (ExecutablesFileSections, error) {
 }
 
 func (b *Builder) execSizeRecipe(properties *properties.Map) (textSize int, dataSize int, eepromSize int, resErr error) {
-	command, err := utils.PrepareCommandForRecipe(properties, "recipe.size.pattern", false)
+	command, err := b.prepareCommandForRecipe(properties, "recipe.size.pattern", false)
 	if err != nil {
 		resErr = fmt.Errorf(tr("Error while determining sketch size: %s"), err)
 		return
 	}
-
-	verboseInfo, out, _, err := utils.ExecCommand(b.logger.Verbose(), b.logger.Stdout(), b.logger.Stderr(), command, utils.Capture /* stdout */, utils.Show /* stderr */)
 	if b.logger.Verbose() {
-		b.logger.Info(string(verboseInfo))
+		b.logger.Info(utils.PrintableCommand(command.GetArgs()))
 	}
-	if err != nil {
+	commandStdout := &bytes.Buffer{}
+	command.RedirectStdoutTo(commandStdout)
+	command.RedirectStderrTo(b.logger.Stderr())
+	if err := command.Start(); err != nil {
 		resErr = fmt.Errorf(tr("Error while determining sketch size: %s"), err)
 		return
 	}
+	if err := command.Wait(); err != nil {
+		resErr = fmt.Errorf(tr("Error while determining sketch size: %s"), err)
+		return
+	}
+
+	out := commandStdout.Bytes()
 
 	// force multiline match prepending "(?m)" to the actual regexp
 	// return an error if RECIPE_SIZE_REGEXP doesn't exist

arduino/builder/sketch.go

@@ -24,7 +24,6 @@ import (
 	"strings"
 
 	"github.com/arduino/arduino-cli/arduino/builder/cpp"
-	"github.com/arduino/arduino-cli/arduino/builder/internal/utils"
 	"github.com/arduino/arduino-cli/i18n"
 	f "github.com/arduino/arduino-cli/internal/algorithms"
 	"github.com/arduino/go-paths-helper"
@@ -174,21 +173,18 @@ func writeIfDifferent(source []byte, destPath *paths.Path) error {
 	return nil
 }
 
-// BuildSketch fixdoc
-func (b *Builder) BuildSketch(includesFolders paths.PathList) error {
+// buildSketch fixdoc
+func (b *Builder) buildSketch(includesFolders paths.PathList) error {
 	includes := f.Map(includesFolders.AsStrings(), cpp.WrapWithHyphenI)
 
 	if err := b.sketchBuildPath.MkdirAll(); err != nil {
 		return errors.WithStack(err)
 	}
 
-	sketchObjectFiles, err := utils.CompileFiles(
-		b.sketchBuildPath, b.sketchBuildPath, b.buildProperties, includes,
-		b.onlyUpdateCompilationDatabase,
-		b.compilationDatabase,
-		b.jobs,
-		b.logger,
-		b.Progress,
+	sketchObjectFiles, err := b.compileFiles(
+		b.sketchBuildPath, b.sketchBuildPath,
+		false, /** recursive **/
+		includes,
 	)
 	if err != nil {
 		return errors.WithStack(err)
@@ -197,13 +193,10 @@ func (b *Builder) BuildSketch(includesFolders paths.PathList) error {
 	// The "src/" subdirectory of a sketch is compiled recursively
 	sketchSrcPath := b.sketchBuildPath.Join("src")
 	if sketchSrcPath.IsDir() {
-		srcObjectFiles, err := utils.CompileFilesRecursive(
-			sketchSrcPath, sketchSrcPath, b.buildProperties, includes,
-			b.onlyUpdateCompilationDatabase,
-			b.compilationDatabase,
-			b.jobs,
-			b.logger,
-			b.Progress,
+		srcObjectFiles, err := b.compileFiles(
+			sketchSrcPath, sketchSrcPath,
+			true, /** recursive **/
+			includes,
 		)
 		if err != nil {
 			return errors.WithStack(err)
@@ -215,8 +208,8 @@ func (b *Builder) BuildSketch(includesFolders paths.PathList) error {
 	return nil
 }
 
-// MergeSketchWithBootloader fixdoc
-func (b *Builder) MergeSketchWithBootloader() error {
+// mergeSketchWithBootloader fixdoc
+func (b *Builder) mergeSketchWithBootloader() error {
 	if b.onlyUpdateCompilationDatabase {
 		return nil
 	}