[skip-changelog] legacy: Builder refactorization (part 2) (#2298)

* remove unused LibraryDir from legacy context

* remove unused WatchedLocation from legacy context

* remove unused IgnoreSketchFolderNameErrors from legacy context

* remove CanUseCachedTools from legacy context

* remove UseArduinoPreprocessor from legacy context

* make the CoreBuilder command a function

* remove the use of context from builder_utils

* mvoe types.ProgressStruct in a dedicated pkg

* move ExecCommand under arduino/utils

* move LogIfVerbose from utils to legacy builder

* move some legacy constans in builder package

* move builder_utils under arduino/builder/utils pkg

* appease golint

* move coreBuildCachePath in the arduino Builder

* refactor Linker command in a function

* refactor SketchBuilder in a function

* refactor LibrariesBuilder in a function

* refactor Sizer in a function

* remove empty file

* remove unused struct FailIfBuildPathEqualsSketchPath

arduino/builder/builder.go

@@ -15,16 +15,34 @@
 
 package builder
 
-import "github.com/arduino/arduino-cli/arduino/sketch"
+import (
+	"github.com/arduino/arduino-cli/arduino/sketch"
+	"github.com/arduino/go-paths-helper"
+)
+
+// nolint
+const (
+	BuildPropertiesArchiveFile          = "archive_file"
+	BuildPropertiesArchiveFilePath      = "archive_file_path"
+	BuildPropertiesObjectFile           = "object_file"
+	RecipeARPattern                     = "recipe.ar.pattern"
+	BuildPropertiesIncludes             = "includes"
+	BuildPropertiesCompilerWarningFlags = "compiler.warning_flags"
+	Space                               = " "
+)
 
 // Builder is a Sketch builder.
 type Builder struct {
 	sketch *sketch.Sketch
+
+	// core related
+	coreBuildCachePath *paths.Path
 }
 
 // NewBuilder creates a sketch Builder.
-func NewBuilder(sk *sketch.Sketch) *Builder {
+func NewBuilder(sk *sketch.Sketch, coreBuildCachePath *paths.Path) *Builder {
 	return &Builder{
-		sketch: sk,
+		sketch:             sk,
+		coreBuildCachePath: coreBuildCachePath,
 	}
 }

arduino/builder/core.go

+package builder
+
+import "github.com/arduino/go-paths-helper"
+
+// CoreBuildCachePath fixdoc
+func (b *Builder) CoreBuildCachePath() *paths.Path {
+	return b.coreBuildCachePath
+}

arduino/builder/progress/progress.go

+package progress
+
+// Struct fixdoc
+type Struct struct {
+	Progress   float32
+	StepAmount float32
+	Parent     *Struct
+}
+
+// AddSubSteps fixdoc
+func (p *Struct) AddSubSteps(steps int) {
+	p.Parent = &Struct{
+		Progress:   p.Progress,
+		StepAmount: p.StepAmount,
+		Parent:     p.Parent,
+	}
+	if p.StepAmount == 0.0 {
+		p.StepAmount = 100.0
+	}
+	p.StepAmount /= float32(steps)
+}
+
+// RemoveSubSteps fixdoc
+func (p *Struct) RemoveSubSteps() {
+	p.Progress = p.Parent.Progress
+	p.StepAmount = p.Parent.StepAmount
+	p.Parent = p.Parent.Parent
+}
+
+// CompleteStep fixdoc
+func (p *Struct) CompleteStep() {
+	p.Progress += p.StepAmount
+}

legacy/builder/types/progress_test.go → arduino/builder/progress/progress_test.go

@@ -13,7 +13,7 @@
 // Arduino software without disclosing the source code of your own applications.
 // To purchase a commercial license, send an email to license@arduino.cc.
 
-package types
+package progress
 
 import (
 	"fmt"
@@ -23,7 +23,7 @@ import (
 )
 
 func TestProgress(t *testing.T) {
-	p := &ProgressStruct{}
+	p := &Struct{}
 	p.AddSubSteps(3)
 	require.Equal(t, float32(0.0), p.Progress)
 	require.InEpsilon(t, 33.33333, p.StepAmount, 0.00001)

arduino/builder/sizer.go

+package builder
+
+import rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1"
+
+// ExecutableSectionSize represents a section of the executable output file
+type ExecutableSectionSize struct {
+	Name    string `json:"name"`
+	Size    int    `json:"size"`
+	MaxSize int    `json:"max_size"`
+}
+
+// ExecutablesFileSections is an array of ExecutablesFileSection
+type ExecutablesFileSections []ExecutableSectionSize
+
+// ToRPCExecutableSectionSizeArray transforms this array into a []*rpc.ExecutableSectionSize
+func (s ExecutablesFileSections) ToRPCExecutableSectionSizeArray() []*rpc.ExecutableSectionSize {
+	res := []*rpc.ExecutableSectionSize{}
+	for _, section := range s {
+		res = append(res, &rpc.ExecutableSectionSize{
+			Name:    section.Name,
+			Size:    int64(section.Size),
+			MaxSize: int64(section.MaxSize),
+		})
+	}
+	return res
+}

arduino/builder/sketch_test.go

@@ -48,7 +48,7 @@ func TestMergeSketchSources(t *testing.T) {
 	}
 	mergedSources := strings.ReplaceAll(string(mergedBytes), "%s", pathToGoldenSource)
 
-	b := NewBuilder(sk)
+	b := NewBuilder(sk, nil)
 	offset, source, err := b.sketchMergeSources(nil)
 	require.Nil(t, err)
 	require.Equal(t, 2, offset)
@@ -61,7 +61,7 @@ func TestMergeSketchSourcesArduinoIncluded(t *testing.T) {
 	require.NotNil(t, sk)
 
 	// ensure not to include Arduino.h when it's already there
-	b := NewBuilder(sk)
+	b := NewBuilder(sk, nil)
 	_, source, err := b.sketchMergeSources(nil)
 	require.Nil(t, err)
 	require.Equal(t, 1, strings.Count(source, "<Arduino.h>"))
@@ -76,7 +76,7 @@ func TestCopyAdditionalFiles(t *testing.T) {
 	sk1, err := sketch.New(paths.New("testdata", t.Name()))
 	require.Nil(t, err)
 	require.Equal(t, sk1.AdditionalFiles.Len(), 1)
-	b1 := NewBuilder(sk1)
+	b1 := NewBuilder(sk1, nil)
 
 	// copy the sketch over, create a fake main file we don't care about it
 	// but we need it for `SketchLoad` to succeed later

legacy/builder/test/utils_test.go → arduino/builder/utils/utils_test.go

-// This file is part of arduino-cli.
-//
-// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
-//
-// This software is released under the GNU General Public License version 3,
-// which covers the main part of arduino-cli.
-// The terms of this license can be found at:
-// https://www.gnu.org/licenses/gpl-3.0.en.html
-//
-// You can be released from the requirements of the above licenses by purchasing
-// a commercial license. Buying such a license is mandatory if you want to
-// modify or otherwise use the software for commercial activities involving the
-// Arduino software without disclosing the source code of your own applications.
-// To purchase a commercial license, send an email to license@arduino.cc.
-
-package test
+package utils
 
 import (
 	"testing"
 
-	"github.com/arduino/arduino-cli/legacy/builder/utils"
 	"github.com/stretchr/testify/require"
 )
 
@@ -37,6 +21,6 @@ func TestPrintableCommand(t *testing.T) {
 		" \"specialchar-`~!@#$%^&*()-_=+[{]}\\\\|;:'\\\",<.>/?-argument\"" +
 		" \"arg   with spaces\" \"arg\twith\t\ttabs\"" +
 		" lastarg"
-	result := utils.PrintableCommand(parts)
+	result := printableCommand(parts)
 	require.Equal(t, correct, result)
 }

commands/compile/compile.go

@@ -155,7 +155,21 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream
 	// cache is purged after compilation to not remove entries that might be required
 	defer maybePurgeBuildCache()
 
-	sketchBuilder := bldr.NewBuilder(sk)
+	var coreBuildCachePath *paths.Path
+	if req.GetBuildCachePath() == "" {
+		coreBuildCachePath = paths.TempDir().Join("arduino", "cores")
+	} else {
+		buildCachePath, err := paths.New(req.GetBuildCachePath()).Abs()
+		if err != nil {
+			return nil, &arduino.PermissionDeniedError{Message: tr("Cannot create build cache directory"), Cause: err}
+		}
+		if err := buildCachePath.MkdirAll(); err != nil {
+			return nil, &arduino.PermissionDeniedError{Message: tr("Cannot create build cache directory"), Cause: err}
+		}
+		coreBuildCachePath = buildCachePath.Join("core")
+	}
+
+	sketchBuilder := bldr.NewBuilder(sk, coreBuildCachePath)
 
 	// Add build properites related to sketch data
 	buildProperties = sketchBuilder.SetupBuildProperties(buildProperties, buildPath, req.GetOptimizeForDebug())
@@ -193,7 +207,6 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream
 	builderCtx.BuiltInToolsDirs = configuration.BuiltinToolsDirectories(configuration.Settings)
 	builderCtx.OtherLibrariesDirs = paths.NewPathList(req.GetLibraries()...)
 	builderCtx.OtherLibrariesDirs.Add(configuration.LibrariesDir(configuration.Settings))
-	builderCtx.LibraryDirs = paths.NewPathList(req.Library...)
 
 	builderCtx.CompilationDatabase = bldr.NewCompilationDatabase(
 		builderCtx.BuildPath.Join("compile_commands.json"),
@@ -207,19 +220,6 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream
 		builderCtx.WarningsLevel = builder.DEFAULT_WARNINGS_LEVEL
 	}
 
-	if req.GetBuildCachePath() == "" {
-		builderCtx.CoreBuildCachePath = paths.TempDir().Join("arduino", "cores")
-	} else {
-		buildCachePath, err := paths.New(req.GetBuildCachePath()).Abs()
-		if err != nil {
-			return nil, &arduino.PermissionDeniedError{Message: tr("Cannot create build cache directory"), Cause: err}
-		}
-		if err := buildCachePath.MkdirAll(); err != nil {
-			return nil, &arduino.PermissionDeniedError{Message: tr("Cannot create build cache directory"), Cause: err}
-		}
-		builderCtx.CoreBuildCachePath = buildCachePath.Join("core")
-	}
-
 	builderCtx.BuiltInLibrariesDirs = configuration.IDEBuiltinLibrariesDir(configuration.Settings)
 
 	builderCtx.Stdout = outStream
@@ -255,9 +255,10 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream
 		libsManager = lm
 	}
 	useCachedLibrariesResolution := req.GetSkipLibrariesDiscovery()
+	libraryDir := paths.NewPathList(req.Library...)
 	libsManager, libsResolver, verboseOut, err := detector.LibrariesLoader(
 		useCachedLibrariesResolution, libsManager,
-		builderCtx.BuiltInLibrariesDirs, builderCtx.LibraryDirs, builderCtx.OtherLibrariesDirs,
+		builderCtx.BuiltInLibrariesDirs, libraryDir, builderCtx.OtherLibrariesDirs,
 		builderCtx.ActualPlatform, builderCtx.TargetPlatform,
 	)
 	if err != nil {

legacy/builder/builder.go

@@ -23,7 +23,6 @@ import (
 	"github.com/arduino/arduino-cli/i18n"
 	"github.com/arduino/arduino-cli/legacy/builder/phases"
 	"github.com/arduino/arduino-cli/legacy/builder/types"
-	"github.com/arduino/arduino-cli/legacy/builder/utils"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 )
@@ -51,33 +50,120 @@ func (s *Builder) Run(ctx *types.Context) error {
 			return _err
 		}),
 
-		utils.LogIfVerbose(false, tr("Detecting libraries used...")),
+		logIfVerbose(false, tr("Detecting libraries used...")),
 		findIncludes(ctx),
 
 		&WarnAboutArchIncompatibleLibraries{},
 
-		utils.LogIfVerbose(false, tr("Generating function prototypes...")),
+		logIfVerbose(false, tr("Generating function prototypes...")),
 		types.BareCommand(PreprocessSketch),
 
-		utils.LogIfVerbose(false, tr("Compiling sketch...")),
+		logIfVerbose(false, tr("Compiling sketch...")),
 		&RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.sketch.prebuild", Suffix: ".pattern"},
-		&phases.SketchBuilder{},
+		types.BareCommand(func(ctx *types.Context) error {
+			sketchObjectFiles, err := phases.SketchBuilder(
+				ctx.SketchBuildPath,
+				ctx.BuildProperties,
+				ctx.SketchLibrariesDetector.IncludeFolders(),
+				ctx.OnlyUpdateCompilationDatabase,
+				ctx.Verbose,
+				ctx.CompilationDatabase,
+				ctx.Jobs,
+				ctx.WarningsLevel,
+				ctx.Stdout, ctx.Stderr,
+				func(msg string) { ctx.Info(msg) },
+				func(data []byte) { ctx.WriteStdout(data) },
+				func(data []byte) { ctx.WriteStderr(data) },
+				&ctx.Progress, ctx.ProgressCB,
+			)
+			if err != nil {
+				return err
+			}
+			ctx.SketchObjectFiles = sketchObjectFiles
+			return nil
+		}),
 		&RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.sketch.postbuild", Suffix: ".pattern", SkipIfOnlyUpdatingCompilationDatabase: true},
 
-		utils.LogIfVerbose(false, tr("Compiling libraries...")),
+		logIfVerbose(false, tr("Compiling libraries...")),
 		&RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.libraries.prebuild", Suffix: ".pattern"},
 		&UnusedCompiledLibrariesRemover{},
-		&phases.LibrariesBuilder{},
+		types.BareCommand(func(ctx *types.Context) error {
+			librariesObjectFiles, err := phases.LibrariesBuilder(
+				ctx.LibrariesBuildPath,
+				ctx.BuildProperties,
+				ctx.SketchLibrariesDetector.IncludeFolders(),
+				ctx.SketchLibrariesDetector.ImportedLibraries(),
+				ctx.Verbose,
+				ctx.OnlyUpdateCompilationDatabase,
+				ctx.CompilationDatabase,
+				ctx.Jobs,
+				ctx.WarningsLevel,
+				ctx.Stdout,
+				ctx.Stderr,
+				func(msg string) { ctx.Info(msg) },
+				func(data []byte) { ctx.WriteStdout(data) },
+				func(data []byte) { ctx.WriteStderr(data) },
+				&ctx.Progress, ctx.ProgressCB,
+			)
+			if err != nil {
+				return err
+			}
+			ctx.LibrariesObjectFiles = librariesObjectFiles
+
+			return nil
+		}),
 		&RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.libraries.postbuild", Suffix: ".pattern", SkipIfOnlyUpdatingCompilationDatabase: true},
 
-		utils.LogIfVerbose(false, tr("Compiling core...")),
+		logIfVerbose(false, tr("Compiling core...")),
 		&RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.core.prebuild", Suffix: ".pattern"},
-		&phases.CoreBuilder{},
+
+		types.BareCommand(func(ctx *types.Context) error {
+			objectFiles, archiveFile, err := phases.CoreBuilder(
+				ctx.BuildPath, ctx.CoreBuildPath, ctx.Builder.CoreBuildCachePath(),
+				ctx.BuildProperties,
+				ctx.ActualPlatform,
+				ctx.Verbose, ctx.OnlyUpdateCompilationDatabase, ctx.Clean,
+				ctx.CompilationDatabase,
+				ctx.Jobs,
+				ctx.WarningsLevel,
+				ctx.Stdout, ctx.Stderr,
+				func(msg string) { ctx.Info(msg) },
+				func(data []byte) { ctx.WriteStdout(data) },
+				func(data []byte) { ctx.WriteStderr(data) },
+				&ctx.Progress, ctx.ProgressCB,
+			)
+
+			ctx.CoreObjectsFiles = objectFiles
+			ctx.CoreArchiveFilePath = archiveFile
+
+			return err
+		}),
+
 		&RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.core.postbuild", Suffix: ".pattern", SkipIfOnlyUpdatingCompilationDatabase: true},
 
-		utils.LogIfVerbose(false, tr("Linking everything together...")),
+		logIfVerbose(false, tr("Linking everything together...")),
 		&RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.linking.prelink", Suffix: ".pattern"},
-		&phases.Linker{},
+
+		types.BareCommand(func(ctx *types.Context) error {
+			verboseInfoOut, err := phases.Linker(
+				ctx.OnlyUpdateCompilationDatabase,
+				ctx.Verbose,
+				ctx.SketchObjectFiles,
+				ctx.LibrariesObjectFiles,
+				ctx.CoreObjectsFiles,
+				ctx.CoreArchiveFilePath,
+				ctx.BuildPath,
+				ctx.BuildProperties,
+				ctx.Stdout,
+				ctx.Stderr,
+				ctx.WarningsLevel,
+			)
+			if ctx.Verbose {
+				ctx.Info(string(verboseInfoOut))
+			}
+			return err
+		}),
+
 		&RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.linking.postlink", Suffix: ".pattern", SkipIfOnlyUpdatingCompilationDatabase: true},
 
 		&RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.objcopy.preobjcopy", Suffix: ".pattern"},
@@ -118,7 +204,18 @@ func (s *Builder) Run(ctx *types.Context) error {
 
 		&ExportProjectCMake{SketchError: mainErr != nil},
 
-		&phases.Sizer{SketchError: mainErr != nil},
+		types.BareCommand(func(ctx *types.Context) error {
+			executableSectionsSize, err := phases.Sizer(
+				ctx.OnlyUpdateCompilationDatabase, mainErr != nil, ctx.Verbose,
+				ctx.BuildProperties,
+				ctx.Stdout, ctx.Stderr,
+				func(msg string) { ctx.Info(msg) },
+				func(msg string) { ctx.Warn(msg) },
+				ctx.WarningsLevel,
+			)
+			ctx.ExecutableSectionsSize = executableSectionsSize
+			return err
+		}),
 	}
 	for _, command := range commands {
 		PrintRingNameIfDebug(ctx, command)
@@ -140,9 +237,6 @@ func (s *Builder) Run(ctx *types.Context) error {
 
 func PreprocessSketch(ctx *types.Context) error {
 	preprocessorImpl := preprocessor.PreprocessSketchWithCtags
-	if ctx.UseArduinoPreprocessor {
-		preprocessorImpl = preprocessor.PreprocessSketchWithArduinoPreprocessor
-	}
 	normalOutput, verboseOutput, err := preprocessorImpl(
 		ctx.Sketch, ctx.BuildPath, ctx.SketchLibrariesDetector.IncludeFolders(), ctx.LineOffset,
 		ctx.BuildProperties, ctx.OnlyUpdateCompilationDatabase)
@@ -235,3 +329,17 @@ func findIncludes(ctx *types.Context) types.BareCommand {
 		)
 	})
 }
+
+func logIfVerbose(warn bool, msg string) types.BareCommand {
+	return types.BareCommand(func(ctx *types.Context) error {
+		if !ctx.Verbose {
+			return nil
+		}
+		if warn {
+			ctx.Warn(msg)
+		} else {
+			ctx.Info(msg)
+		}
+		return nil
+	})
+}

legacy/builder/builder_utils/utils.go

-// This file is part of arduino-cli.
-//
-// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
-//
-// This software is released under the GNU General Public License version 3,
-// which covers the main part of arduino-cli.
-// The terms of this license can be found at:
-// https://www.gnu.org/licenses/gpl-3.0.en.html
-//
-// You can be released from the requirements of the above licenses by purchasing
-// a commercial license. Buying such a license is mandatory if you want to
-// modify or otherwise use the software for commercial activities involving the
-// Arduino software without disclosing the source code of your own applications.
-// To purchase a commercial license, send an email to license@arduino.cc.
-
-package builder_utils
-
-import (
-	"fmt"
-	"os"
-	"path/filepath"
-	"runtime"
-	"strings"
-	"sync"
-
-	bUtils "github.com/arduino/arduino-cli/arduino/builder/utils"
-	"github.com/arduino/arduino-cli/arduino/globals"
-	"github.com/arduino/arduino-cli/executils"
-	"github.com/arduino/arduino-cli/i18n"
-	"github.com/arduino/arduino-cli/legacy/builder/constants"
-	"github.com/arduino/arduino-cli/legacy/builder/types"
-	"github.com/arduino/arduino-cli/legacy/builder/utils"
-	"github.com/arduino/go-paths-helper"
-	"github.com/arduino/go-properties-orderedmap"
-	"github.com/pkg/errors"
-)
-
-var tr = i18n.Tr
-
-// DirContentIsOlderThan returns true if the content of the given directory is
-// older than target file. If extensions are given, only the files with these
-// extensions are tested.
-func DirContentIsOlderThan(dir *paths.Path, target *paths.Path, extensions ...string) (bool, error) {
-	targetStat, err := target.Stat()
-	if err != nil {
-		return false, err
-	}
-	targetModTime := targetStat.ModTime()
-
-	files, err := bUtils.FindFilesInFolder(dir, true, extensions...)
-	if err != nil {
-		return false, err
-	}
-	for _, file := range files {
-		file, err := file.Stat()
-		if err != nil {
-			return false, err
-		}
-		if file.ModTime().After(targetModTime) {
-			return false, nil
-		}
-	}
-	return true, nil
-}
-
-func CompileFiles(ctx *types.Context, sourceDir *paths.Path, buildPath *paths.Path, buildProperties *properties.Map, includes []string) (paths.PathList, error) {
-	return compileFiles(ctx, sourceDir, false, buildPath, buildProperties, includes)
-}
-
-func CompileFilesRecursive(ctx *types.Context, sourceDir *paths.Path, buildPath *paths.Path, buildProperties *properties.Map, includes []string) (paths.PathList, error) {
-	return compileFiles(ctx, sourceDir, true, buildPath, buildProperties, includes)
-}
-
-func compileFiles(ctx *types.Context, sourceDir *paths.Path, recurse bool, buildPath *paths.Path, buildProperties *properties.Map, includes []string) (paths.PathList, error) {
-	validExtensions := []string{}
-	for ext := range globals.SourceFilesValidExtensions {
-		validExtensions = append(validExtensions, ext)
-	}
-
-	sources, err := bUtils.FindFilesInFolder(sourceDir, recurse, validExtensions...)
-	if err != nil {
-		return nil, err
-	}
-
-	ctx.Progress.AddSubSteps(len(sources))
-	defer ctx.Progress.RemoveSubSteps()
-
-	objectFiles := paths.NewPathList()
-	var objectFilesMux sync.Mutex
-	if len(sources) == 0 {
-		return objectFiles, nil
-	}
-	var errorsList []error
-	var errorsMux sync.Mutex
-
-	queue := make(chan *paths.Path)
-	job := func(source *paths.Path) {
-		recipe := fmt.Sprintf("recipe%s.o.pattern", source.Ext())
-		if !buildProperties.ContainsKey(recipe) {
-			recipe = fmt.Sprintf("recipe%s.o.pattern", globals.SourceFilesValidExtensions[source.Ext()])
-		}
-		objectFile, err := compileFileWithRecipe(ctx, sourceDir, source, buildPath, buildProperties, includes, recipe)
-		if err != nil {
-			errorsMux.Lock()
-			errorsList = append(errorsList, err)
-			errorsMux.Unlock()
-		} else {
-			objectFilesMux.Lock()
-			objectFiles.Add(objectFile)
-			objectFilesMux.Unlock()
-		}
-	}
-
-	// Spawn jobs runners
-	var wg sync.WaitGroup
-	jobs := ctx.Jobs
-	if jobs == 0 {
-		jobs = runtime.NumCPU()
-	}
-	for i := 0; i < jobs; i++ {
-		wg.Add(1)
-		go func() {
-			for source := range queue {
-				job(source)
-			}
-			wg.Done()
-		}()
-	}
-
-	// Feed jobs until error or done
-	for _, source := range sources {
-		errorsMux.Lock()
-		gotError := len(errorsList) > 0
-		errorsMux.Unlock()
-		if gotError {
-			break
-		}
-		queue <- source
-
-		ctx.Progress.CompleteStep()
-		ctx.PushProgress()
-	}
-	close(queue)
-	wg.Wait()
-	if len(errorsList) > 0 {
-		// output the first error
-		return nil, errors.WithStack(errorsList[0])
-	}
-	objectFiles.Sort()
-	return objectFiles, nil
-}
-
-func compileFileWithRecipe(ctx *types.Context, sourcePath *paths.Path, source *paths.Path, buildPath *paths.Path, buildProperties *properties.Map, includes []string, recipe string) (*paths.Path, error) {
-	properties := buildProperties.Clone()
-	properties.Set(constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS, properties.Get(constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS+"."+ctx.WarningsLevel))
-	properties.Set(constants.BUILD_PROPERTIES_INCLUDES, strings.Join(includes, constants.SPACE))
-	properties.SetPath("source_file", source)
-	relativeSource, err := sourcePath.RelTo(source)
-	if err != nil {
-		return nil, errors.WithStack(err)
-	}
-	depsFile := buildPath.Join(relativeSource.String() + ".d")
-	objectFile := buildPath.Join(relativeSource.String() + ".o")
-
-	properties.SetPath(constants.BUILD_PROPERTIES_OBJECT_FILE, objectFile)
-	err = objectFile.Parent().MkdirAll()
-	if err != nil {
-		return nil, errors.WithStack(err)
-	}
-
-	objIsUpToDate, err := bUtils.ObjFileIsUpToDate(source, objectFile, depsFile)
-	if err != nil {
-		return nil, errors.WithStack(err)
-	}
-
-	command, err := PrepareCommandForRecipe(properties, recipe, false)
-	if err != nil {
-		return nil, errors.WithStack(err)
-	}
-	if ctx.CompilationDatabase != nil {
-		ctx.CompilationDatabase.Add(source, command)
-	}
-	if !objIsUpToDate && !ctx.OnlyUpdateCompilationDatabase {
-		// Since this compile could be multithreaded, we first capture the command output
-		stdout, stderr, err := utils.ExecCommand(ctx, command, utils.Capture, utils.Capture)
-		// and transfer all at once at the end...
-		if ctx.Verbose {
-			ctx.WriteStdout(stdout)
-		}
-		ctx.WriteStderr(stderr)
-
-		// ...and then return the error
-		if err != nil {
-			return nil, errors.WithStack(err)
-		}
-	} else if ctx.Verbose {
-		if objIsUpToDate {
-			ctx.Info(tr("Using previously compiled file: %[1]s", objectFile))
-		} else {
-			ctx.Info(tr("Skipping compile of: %[1]s", objectFile))
-		}
-	}
-
-	return objectFile, nil
-}
-
-func ArchiveCompiledFiles(ctx *types.Context, buildPath *paths.Path, archiveFile *paths.Path, objectFilesToArchive paths.PathList, buildProperties *properties.Map) (*paths.Path, error) {
-	archiveFilePath := buildPath.JoinPath(archiveFile)
-
-	if ctx.OnlyUpdateCompilationDatabase {
-		if ctx.Verbose {
-			ctx.Info(tr("Skipping archive creation of: %[1]s", archiveFilePath))
-		}
-		return archiveFilePath, nil
-	}
-
-	if archiveFileStat, err := archiveFilePath.Stat(); err == nil {
-		rebuildArchive := false
-		for _, objectFile := range objectFilesToArchive {
-			objectFileStat, err := objectFile.Stat()
-			if err != nil || objectFileStat.ModTime().After(archiveFileStat.ModTime()) {
-				// need to rebuild the archive
-				rebuildArchive = true
-				break
-			}
-		}
-
-		// something changed, rebuild the core archive
-		if rebuildArchive {
-			if err := archiveFilePath.Remove(); err != nil {
-				return nil, errors.WithStack(err)
-			}
-		} else {
-			if ctx.Verbose {
-				ctx.Info(tr("Using previously compiled file: %[1]s", archiveFilePath))
-			}
-			return archiveFilePath, nil
-		}
-	}
-
-	for _, objectFile := range objectFilesToArchive {
-		properties := buildProperties.Clone()
-		properties.Set(constants.BUILD_PROPERTIES_ARCHIVE_FILE, archiveFilePath.Base())
-		properties.SetPath(constants.BUILD_PROPERTIES_ARCHIVE_FILE_PATH, archiveFilePath)
-		properties.SetPath(constants.BUILD_PROPERTIES_OBJECT_FILE, objectFile)
-
-		command, err := PrepareCommandForRecipe(properties, constants.RECIPE_AR_PATTERN, false)
-		if err != nil {
-			return nil, errors.WithStack(err)
-		}
-
-		_, _, err = utils.ExecCommand(ctx, command, utils.ShowIfVerbose /* stdout */, utils.Show /* stderr */)
-		if err != nil {
-			return nil, errors.WithStack(err)
-		}
-	}
-
-	return archiveFilePath, nil
-}
-
-const COMMANDLINE_LIMIT = 30000
-
-func PrepareCommandForRecipe(buildProperties *properties.Map, recipe string, removeUnsetProperties bool) (*executils.Process, error) {
-	pattern := buildProperties.Get(recipe)
-	if pattern == "" {
-		return nil, errors.Errorf(tr("%[1]s pattern is missing"), recipe)
-	}
-
-	commandLine := buildProperties.ExpandPropsInString(pattern)
-	if removeUnsetProperties {
-		commandLine = properties.DeleteUnexpandedPropsFromString(commandLine)
-	}
-
-	parts, err := properties.SplitQuotedString(commandLine, `"'`, false)
-	if err != nil {
-		return nil, errors.WithStack(err)
-	}
-
-	// if the overall commandline is too long for the platform
-	// try reducing the length by making the filenames relative
-	// and changing working directory to build.path
-	var relativePath string
-	if len(commandLine) > COMMANDLINE_LIMIT {
-		relativePath = buildProperties.Get("build.path")
-		for i, arg := range parts {
-			if _, err := os.Stat(arg); os.IsNotExist(err) {
-				continue
-			}
-			rel, err := filepath.Rel(relativePath, arg)
-			if err == nil && !strings.Contains(rel, "..") && len(rel) < len(arg) {
-				parts[i] = rel
-			}
-		}
-	}
-
-	command, err := executils.NewProcess(nil, parts...)
-	if err != nil {
-		return nil, errors.WithStack(err)
-	}
-	if relativePath != "" {
-		command.SetDir(relativePath)
-	}
-
-	return command, nil
-}

legacy/builder/constants/constants.go

@@ -17,19 +17,13 @@
 package constants
 
 const BUILD_OPTIONS_FILE = "build.options.json"
-const BUILD_PROPERTIES_ARCHIVE_FILE = "archive_file"
-const BUILD_PROPERTIES_ARCHIVE_FILE_PATH = "archive_file_path"
 const BUILD_PROPERTIES_ARCH_OVERRIDE_CHECK = "architecture.override_check"
 const BUILD_PROPERTIES_BOOTLOADER_FILE = "bootloader.file"
 const BUILD_PROPERTIES_BOOTLOADER_NOBLINK = "bootloader.noblink"
 const BUILD_PROPERTIES_BUILD_BOARD = "build.board"
 const BUILD_PROPERTIES_BUILD_MCU = "build.mcu"
-const BUILD_PROPERTIES_COMPILER_C_ELF_FLAGS = "compiler.c.elf.flags"
 const BUILD_PROPERTIES_COMPILER_LDFLAGS = "compiler.ldflags"
 const BUILD_PROPERTIES_COMPILER_CPP_FLAGS = "compiler.cpp.flags"
-const BUILD_PROPERTIES_COMPILER_WARNING_FLAGS = "compiler.warning_flags"
-const BUILD_PROPERTIES_INCLUDES = "includes"
-const BUILD_PROPERTIES_OBJECT_FILE = "object_file"
 const BUILD_PROPERTIES_RUNTIME_PLATFORM_PATH = "runtime.platform.path"
 const EMPTY_STRING = ""
 const FOLDER_BOOTLOADERS = "bootloaders"
@@ -50,9 +44,7 @@ const PACKAGE_TOOLS = "tools"
 const PLATFORM_ARCHITECTURE = "architecture"
 const PLATFORM_URL = "url"
 const PLATFORM_VERSION = "version"
-const RECIPE_AR_PATTERN = "recipe.ar.pattern"
 const RECIPE_C_COMBINE_PATTERN = "recipe.c.combine.pattern"
-const SPACE = " "
 const TOOL_NAME = "name"
 const TOOL_URL = "url"
 const TOOL_VERSION = "version"

legacy/builder/create_cmake_rule.go

@@ -28,7 +28,6 @@ import (
 
 	"github.com/arduino/arduino-cli/arduino/builder/utils"
 	"github.com/arduino/arduino-cli/arduino/globals"
-	"github.com/arduino/arduino-cli/legacy/builder/builder_utils"
 	"github.com/arduino/arduino-cli/legacy/builder/constants"
 	"github.com/arduino/arduino-cli/legacy/builder/types"
 )
@@ -367,7 +366,7 @@ func extractCompileFlags(ctx *types.Context, recipe string, defines, dynamicLibs
 		return target
 	}
 
-	command, _ := builder_utils.PrepareCommandForRecipe(ctx.BuildProperties, recipe, true)
+	command, _ := utils.PrepareCommandForRecipe(ctx.BuildProperties, recipe, true)
 
 	for _, arg := range command.GetArgs() {
 		if strings.HasPrefix(arg, "-D") {

legacy/builder/ctags_target_file_saver.go

-// This file is part of arduino-cli.
-//
-// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
-//
-// This software is released under the GNU General Public License version 3,
-// which covers the main part of arduino-cli.
-// The terms of this license can be found at:
-// https://www.gnu.org/licenses/gpl-3.0.en.html
-//
-// You can be released from the requirements of the above licenses by purchasing
-// a commercial license. Buying such a license is mandatory if you want to
-// modify or otherwise use the software for commercial activities involving the
-// Arduino software without disclosing the source code of your own applications.
-// To purchase a commercial license, send an email to license@arduino.cc.
-
-package builder

legacy/builder/fail_if_buildpath_equals_sketchpath.go

-// This file is part of arduino-cli.
-//
-// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
-//
-// This software is released under the GNU General Public License version 3,
-// which covers the main part of arduino-cli.
-// The terms of this license can be found at:
-// https://www.gnu.org/licenses/gpl-3.0.en.html
-//
-// You can be released from the requirements of the above licenses by purchasing
-// a commercial license. Buying such a license is mandatory if you want to
-// modify or otherwise use the software for commercial activities involving the
-// Arduino software without disclosing the source code of your own applications.
-// To purchase a commercial license, send an email to license@arduino.cc.
-
-package builder
-
-import (
-	"github.com/arduino/arduino-cli/legacy/builder/types"
-	"github.com/pkg/errors"
-)
-
-type FailIfBuildPathEqualsSketchPath struct{}
-
-func (s *FailIfBuildPathEqualsSketchPath) Run(ctx *types.Context) error {
-	buildPath := ctx.BuildPath.Canonical()
-	sketchPath := ctx.Sketch.FullPath.Canonical()
-	if buildPath.EqualsTo(sketchPath) {
-		return errors.New(tr("Sketch cannot be located in build path. Please specify a different build path"))
-	}
-
-	return nil
-}

legacy/builder/phases/core_builder.go

@@ -19,74 +19,111 @@ import (
 	"crypto/md5"
 	"encoding/hex"
 	"fmt"
+	"io"
 	"os"
 	"strings"
 
+	"github.com/arduino/arduino-cli/arduino/builder"
 	"github.com/arduino/arduino-cli/arduino/builder/cpp"
+	"github.com/arduino/arduino-cli/arduino/builder/progress"
+	"github.com/arduino/arduino-cli/arduino/builder/utils"
+	"github.com/arduino/arduino-cli/arduino/cores"
 	"github.com/arduino/arduino-cli/buildcache"
 	"github.com/arduino/arduino-cli/i18n"
 	f "github.com/arduino/arduino-cli/internal/algorithms"
-	"github.com/arduino/arduino-cli/legacy/builder/builder_utils"
 	"github.com/arduino/arduino-cli/legacy/builder/constants"
-	"github.com/arduino/arduino-cli/legacy/builder/types"
+	rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1"
 	"github.com/arduino/go-paths-helper"
 	"github.com/arduino/go-properties-orderedmap"
 	"github.com/pkg/errors"
 )
 
-type CoreBuilder struct{}
-
 var tr = i18n.Tr
 
-func (s *CoreBuilder) Run(ctx *types.Context) error {
-	coreBuildPath := ctx.CoreBuildPath
-	coreBuildCachePath := ctx.CoreBuildCachePath
-	buildProperties := ctx.BuildProperties
-
+func CoreBuilder(
+	buildPath, coreBuildPath, coreBuildCachePath *paths.Path,
+	buildProperties *properties.Map,
+	actualPlatform *cores.PlatformRelease,
+	verbose, onlyUpdateCompilationDatabase, clean bool,
+	compilationDatabase *builder.CompilationDatabase,
+	jobs int,
+	warningsLevel string,
+	stdoutWriter, stderrWriter io.Writer,
+	verboseInfoFn func(msg string),
+	verboseStdoutFn, verboseStderrFn func(data []byte),
+	progress *progress.Struct, progressCB rpc.TaskProgressCB,
+) (paths.PathList, *paths.Path, error) {
 	if err := coreBuildPath.MkdirAll(); err != nil {
-		return errors.WithStack(err)
+		return nil, nil, errors.WithStack(err)
 	}
 
 	if coreBuildCachePath != nil {
-		if _, err := coreBuildCachePath.RelTo(ctx.BuildPath); err != nil {
-			ctx.Info(tr("Couldn't deeply cache core build: %[1]s", err))
-			ctx.Info(tr("Running normal build of the core..."))
+		if _, err := coreBuildCachePath.RelTo(buildPath); err != nil {
+			verboseInfoFn(tr("Couldn't deeply cache core build: %[1]s", err))
+			verboseInfoFn(tr("Running normal build of the core..."))
 			coreBuildCachePath = nil
-			ctx.CoreBuildCachePath = nil
 		} else if err := coreBuildCachePath.MkdirAll(); err != nil {
-			return errors.WithStack(err)
+			return nil, nil, errors.WithStack(err)
 		}
 	}
 
-	archiveFile, objectFiles, err := compileCore(ctx, coreBuildPath, coreBuildCachePath, buildProperties)
+	archiveFile, objectFiles, err := compileCore(
+		verbose, onlyUpdateCompilationDatabase, clean,
+		actualPlatform,
+		coreBuildPath, coreBuildCachePath,
+		buildProperties,
+		compilationDatabase,
+		jobs,
+		warningsLevel,
+		stdoutWriter, stderrWriter,
+		verboseInfoFn,
+		verboseStdoutFn, verboseStderrFn,
+		progress, progressCB,
+	)
 	if err != nil {
-		return errors.WithStack(err)
+		return nil, nil, errors.WithStack(err)
 	}
 
-	ctx.CoreArchiveFilePath = archiveFile
-	ctx.CoreObjectsFiles = objectFiles
-
-	return nil
+	return objectFiles, archiveFile, nil
 }
 
-func compileCore(ctx *types.Context, buildPath *paths.Path, buildCachePath *paths.Path, buildProperties *properties.Map) (*paths.Path, paths.PathList, error) {
+func compileCore(
+	verbose, onlyUpdateCompilationDatabase, clean bool,
+	actualPlatform *cores.PlatformRelease,
+	buildPath, buildCachePath *paths.Path,
+	buildProperties *properties.Map,
+	compilationDatabase *builder.CompilationDatabase,
+	jobs int,
+	warningsLevel string,
+	stdoutWriter, stderrWriter io.Writer,
+	verboseInfoFn func(msg string),
+	verboseStdoutFn, verboseStderrFn func(data []byte),
+	progress *progress.Struct, progressCB rpc.TaskProgressCB,
+) (*paths.Path, paths.PathList, error) {
 	coreFolder := buildProperties.GetPath("build.core.path")
 	variantFolder := buildProperties.GetPath("build.variant.path")
-
 	targetCoreFolder := buildProperties.GetPath(constants.BUILD_PROPERTIES_RUNTIME_PLATFORM_PATH)
 
-	includes := []string{}
-	includes = append(includes, coreFolder.String())
+	includes := []string{coreFolder.String()}
 	if variantFolder != nil && variantFolder.IsDir() {
 		includes = append(includes, variantFolder.String())
 	}
 	includes = f.Map(includes, cpp.WrapWithHyphenI)
 
 	var err error
-
 	variantObjectFiles := paths.NewPathList()
 	if variantFolder != nil && variantFolder.IsDir() {
-		variantObjectFiles, err = builder_utils.CompileFilesRecursive(ctx, variantFolder, buildPath, buildProperties, includes)
+		variantObjectFiles, err = utils.CompileFilesRecursive(
+			variantFolder, buildPath, buildProperties, includes,
+			onlyUpdateCompilationDatabase,
+			compilationDatabase,
+			jobs,
+			verbose,
+			warningsLevel,
+			stdoutWriter, stderrWriter,
+			verboseInfoFn, verboseStdoutFn, verboseStderrFn,
+			progress, progressCB,
+		)
 		if err != nil {
 			return nil, nil, errors.WithStack(err)
 		}
@@ -98,7 +135,8 @@ func compileCore(ctx *types.Context, buildPath *paths.Path, buildCachePath *path
 		archivedCoreName := GetCachedCoreArchiveDirName(
 			buildProperties.Get("build.fqbn"),
 			buildProperties.Get("compiler.optimization_flags"),
-			realCoreFolder)
+			realCoreFolder,
+		)
 		targetArchivedCore = buildCachePath.Join(archivedCoreName, "core.a")
 
 		if _, err := buildcache.New(buildCachePath).GetOrCreate(archivedCoreName); errors.Is(err, buildcache.CreateDirErr) {
@@ -106,14 +144,14 @@ func compileCore(ctx *types.Context, buildPath *paths.Path, buildCachePath *path
 		}
 
 		var canUseArchivedCore bool
-		if ctx.OnlyUpdateCompilationDatabase || ctx.Clean {
+		if onlyUpdateCompilationDatabase || clean {
 			canUseArchivedCore = false
-		} else if isOlder, err := builder_utils.DirContentIsOlderThan(realCoreFolder, targetArchivedCore); err != nil || !isOlder {
+		} else if isOlder, err := utils.DirContentIsOlderThan(realCoreFolder, targetArchivedCore); err != nil || !isOlder {
 			// Recreate the archive if ANY of the core files (including platform.txt) has changed
 			canUseArchivedCore = false
 		} else if targetCoreFolder == nil || realCoreFolder.EquivalentTo(targetCoreFolder) {
 			canUseArchivedCore = true
-		} else if isOlder, err := builder_utils.DirContentIsOlderThan(targetCoreFolder, targetArchivedCore); err != nil || !isOlder {
+		} else if isOlder, err := utils.DirContentIsOlderThan(targetCoreFolder, targetArchivedCore); err != nil || !isOlder {
 			// Recreate the archive if ANY of the build core files (including platform.txt) has changed
 			canUseArchivedCore = false
 		} else {
@@ -122,35 +160,51 @@ func compileCore(ctx *types.Context, buildPath *paths.Path, buildCachePath *path
 
 		if canUseArchivedCore {
 			// use archived core
-			if ctx.Verbose {
-				ctx.Info(tr("Using precompiled core: %[1]s", targetArchivedCore))
+			if verbose {
+				verboseInfoFn(tr("Using precompiled core: %[1]s", targetArchivedCore))
 			}
 			return targetArchivedCore, variantObjectFiles, nil
 		}
 	}
 
-	coreObjectFiles, err := builder_utils.CompileFilesRecursive(ctx, coreFolder, buildPath, buildProperties, includes)
+	coreObjectFiles, err := utils.CompileFilesRecursive(
+		coreFolder, buildPath, buildProperties, includes,
+		onlyUpdateCompilationDatabase,
+		compilationDatabase,
+		jobs,
+		verbose,
+		warningsLevel,
+		stdoutWriter, stderrWriter,
+		verboseInfoFn, verboseStdoutFn, verboseStderrFn,
+		progress, progressCB,
+	)
 	if err != nil {
 		return nil, nil, errors.WithStack(err)
 	}
 
-	archiveFile, err := builder_utils.ArchiveCompiledFiles(ctx, buildPath, paths.New("core.a"), coreObjectFiles, buildProperties)
+	archiveFile, verboseInfo, err := utils.ArchiveCompiledFiles(
+		buildPath, paths.New("core.a"), coreObjectFiles, buildProperties,
+		onlyUpdateCompilationDatabase, verbose, stdoutWriter, stderrWriter,
+	)
+	if verbose {
+		verboseInfoFn(string(verboseInfo))
+	}
 	if err != nil {
 		return nil, nil, errors.WithStack(err)
 	}
 
 	// archive core.a
-	if targetArchivedCore != nil && !ctx.OnlyUpdateCompilationDatabase {
+	if targetArchivedCore != nil && !onlyUpdateCompilationDatabase {
 		err := archiveFile.CopyTo(targetArchivedCore)
-		if ctx.Verbose {
+		if verbose {
 			if err == nil {
-				ctx.Info(tr("Archiving built core (caching) in: %[1]s", targetArchivedCore))
+				verboseInfoFn(tr("Archiving built core (caching) in: %[1]s", targetArchivedCore))
 			} else if os.IsNotExist(err) {
-				ctx.Info(tr("Unable to cache built core, please tell %[1]s maintainers to follow %[2]s",
-					ctx.ActualPlatform,
+				verboseInfoFn(tr("Unable to cache built core, please tell %[1]s maintainers to follow %[2]s",
+					actualPlatform,
 					"https://arduino.github.io/arduino-cli/latest/platform-specification/#recipes-to-build-the-corea-archive-file"))
 			} else {
-				ctx.Info(tr("Error archiving built core (caching) in %[1]s: %[2]s", targetArchivedCore, err))
+				verboseInfoFn(tr("Error archiving built core (caching) in %[1]s: %[2]s", targetArchivedCore, err))
 			}
 		}
 	}
@@ -161,8 +215,8 @@ func compileCore(ctx *types.Context, buildPath *paths.Path, buildCachePath *path
 // GetCachedCoreArchiveDirName returns the directory name to be used to store
 // the global cached core.a.
 func GetCachedCoreArchiveDirName(fqbn string, optimizationFlags string, coreFolder *paths.Path) string {
-	fqbnToUnderscore := strings.Replace(fqbn, ":", "_", -1)
-	fqbnToUnderscore = strings.Replace(fqbnToUnderscore, "=", "_", -1)
+	fqbnToUnderscore := strings.ReplaceAll(fqbn, ":", "_")
+	fqbnToUnderscore = strings.ReplaceAll(fqbnToUnderscore, "=", "_")
 	if absCoreFolder, err := coreFolder.Abs(); err == nil {
 		coreFolder = absCoreFolder
 	} // silently continue if absolute path can't be detected

legacy/builder/phases/linker.go

@@ -16,55 +16,68 @@
 package phases
 
 import (
+	"bytes"
+	"io"
 	"strings"
 
+	"github.com/arduino/arduino-cli/arduino/builder"
+	"github.com/arduino/arduino-cli/arduino/builder/utils"
 	f "github.com/arduino/arduino-cli/internal/algorithms"
-	"github.com/arduino/arduino-cli/legacy/builder/builder_utils"
-	"github.com/arduino/arduino-cli/legacy/builder/constants"
-	"github.com/arduino/arduino-cli/legacy/builder/types"
-	"github.com/arduino/arduino-cli/legacy/builder/utils"
 	"github.com/arduino/go-paths-helper"
 	"github.com/arduino/go-properties-orderedmap"
 	"github.com/pkg/errors"
 )
 
-type Linker struct{}
-
-func (s *Linker) Run(ctx *types.Context) error {
-	if ctx.OnlyUpdateCompilationDatabase {
-		if ctx.Verbose {
-			ctx.Info(tr("Skip linking of final executable."))
+func Linker(
+	onlyUpdateCompilationDatabase, verbose bool,
+	sketchObjectFiles, librariesObjectFiles, coreObjectsFiles paths.PathList,
+	coreArchiveFilePath, buildPath *paths.Path,
+	buildProperties *properties.Map,
+	stdoutWriter, stderrWriter io.Writer,
+	warningsLevel string,
+) ([]byte, error) {
+	verboseInfo := &bytes.Buffer{}
+	if onlyUpdateCompilationDatabase {
+		if verbose {
+			verboseInfo.WriteString(tr("Skip linking of final executable."))
 		}
-		return nil
+		return verboseInfo.Bytes(), nil
 	}
 
-	objectFilesSketch := ctx.SketchObjectFiles
-	objectFilesLibraries := ctx.LibrariesObjectFiles
-	objectFilesCore := ctx.CoreObjectsFiles
+	objectFilesSketch := sketchObjectFiles
+	objectFilesLibraries := librariesObjectFiles
+	objectFilesCore := coreObjectsFiles
 
 	objectFiles := paths.NewPathList()
 	objectFiles.AddAll(objectFilesSketch)
 	objectFiles.AddAll(objectFilesLibraries)
 	objectFiles.AddAll(objectFilesCore)
 
-	coreArchiveFilePath := ctx.CoreArchiveFilePath
-	buildPath := ctx.BuildPath
 	coreDotARelPath, err := buildPath.RelTo(coreArchiveFilePath)
 	if err != nil {
-		return errors.WithStack(err)
+		return nil, errors.WithStack(err)
 	}
 
-	buildProperties := ctx.BuildProperties
-
-	err = link(ctx, objectFiles, coreDotARelPath, coreArchiveFilePath, buildProperties)
+	verboseInfoOut, err := link(
+		objectFiles, coreDotARelPath, coreArchiveFilePath, buildProperties,
+		verbose, stdoutWriter, stderrWriter, warningsLevel,
+	)
+	verboseInfo.Write(verboseInfoOut)
 	if err != nil {
-		return errors.WithStack(err)
+		return verboseInfo.Bytes(), errors.WithStack(err)
 	}
 
-	return nil
+	return verboseInfo.Bytes(), nil
 }
 
-func link(ctx *types.Context, objectFiles paths.PathList, coreDotARelPath *paths.Path, coreArchiveFilePath *paths.Path, buildProperties *properties.Map) error {
+func link(
+	objectFiles paths.PathList, coreDotARelPath *paths.Path, coreArchiveFilePath *paths.Path, buildProperties *properties.Map,
+	verbose bool,
+	stdoutWriter, stderrWriter io.Writer,
+	warningsLevel string,
+) ([]byte, error) {
+	verboseBuffer := &bytes.Buffer{}
+	wrapWithDoubleQuotes := func(value string) string { return "\"" + value + "\"" }
 	objectFileList := strings.Join(f.Map(objectFiles.AsStrings(), wrapWithDoubleQuotes), " ")
 
 	// If command line length is too big (> 30000 chars), try to collect the object files into archives
@@ -93,13 +106,16 @@ func link(ctx *types.Context, objectFiles paths.PathList, coreDotARelPath *paths
 			properties.SetPath("archive_file_path", archive)
 			properties.SetPath("object_file", object)
 
-			command, err := builder_utils.PrepareCommandForRecipe(properties, constants.RECIPE_AR_PATTERN, false)
+			command, err := utils.PrepareCommandForRecipe(properties, builder.RecipeARPattern, false)
 			if err != nil {
-				return errors.WithStack(err)
+				return nil, errors.WithStack(err)
 			}
 
-			if _, _, err := utils.ExecCommand(ctx, command, utils.ShowIfVerbose /* stdout */, utils.Show /* stderr */); err != nil {
-				return errors.WithStack(err)
+			if verboseInfo, _, _, err := utils.ExecCommand(verbose, stdoutWriter, stderrWriter, command, utils.ShowIfVerbose /* stdout */, utils.Show /* stderr */); err != nil {
+				if verbose {
+					verboseBuffer.WriteString(string(verboseInfo))
+				}
+				return verboseBuffer.Bytes(), errors.WithStack(err)
 			}
 		}
 
@@ -108,21 +124,20 @@ func link(ctx *types.Context, objectFiles paths.PathList, coreDotARelPath *paths
 	}
 
 	properties := buildProperties.Clone()
-	properties.Set(constants.BUILD_PROPERTIES_COMPILER_C_ELF_FLAGS, properties.Get(constants.BUILD_PROPERTIES_COMPILER_C_ELF_FLAGS))
-	properties.Set(constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS, properties.Get(constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS+"."+ctx.WarningsLevel))
-	properties.Set(constants.BUILD_PROPERTIES_ARCHIVE_FILE, coreDotARelPath.String())
-	properties.Set(constants.BUILD_PROPERTIES_ARCHIVE_FILE_PATH, coreArchiveFilePath.String())
+	properties.Set("compiler.c.elf.flags", properties.Get("compiler.c.elf.flags"))
+	properties.Set(builder.BuildPropertiesCompilerWarningFlags, properties.Get(builder.BuildPropertiesCompilerWarningFlags+"."+warningsLevel))
+	properties.Set(builder.BuildPropertiesArchiveFile, coreDotARelPath.String())
+	properties.Set(builder.BuildPropertiesArchiveFilePath, coreArchiveFilePath.String())
 	properties.Set("object_files", objectFileList)
 
-	command, err := builder_utils.PrepareCommandForRecipe(properties, constants.RECIPE_C_COMBINE_PATTERN, false)
+	command, err := utils.PrepareCommandForRecipe(properties, "recipe.c.combine.pattern", false)
 	if err != nil {
-		return err
-}
-
-	_, _, err = utils.ExecCommand(ctx, command, utils.ShowIfVerbose /* stdout */, utils.Show /* stderr */)
-	return err
-}
+		return verboseBuffer.Bytes(), err
+	}
 
-func wrapWithDoubleQuotes(value string) string {
-	return "\"" + value + "\""
+	verboseInfo, _, _, err := utils.ExecCommand(verbose, stdoutWriter, stderrWriter, command, utils.ShowIfVerbose /* stdout */, utils.Show /* stderr */)
+	if verbose {
+		verboseBuffer.WriteString(string(verboseInfo))
+	}
+	return verboseBuffer.Bytes(), err
 }

legacy/builder/phases/sizer.go

@@ -18,46 +18,50 @@ package phases
 import (
 	"encoding/json"
 	"fmt"
+	"io"
 	"regexp"
 	"strconv"
 
-	"github.com/arduino/arduino-cli/legacy/builder/builder_utils"
-	"github.com/arduino/arduino-cli/legacy/builder/types"
-	"github.com/arduino/arduino-cli/legacy/builder/utils"
+	"github.com/arduino/arduino-cli/arduino/builder"
+	"github.com/arduino/arduino-cli/arduino/builder/utils"
 	"github.com/arduino/go-properties-orderedmap"
 	"github.com/pkg/errors"
 )
 
-type Sizer struct {
-	SketchError bool
-}
-
-func (s *Sizer) Run(ctx *types.Context) error {
-	if ctx.OnlyUpdateCompilationDatabase {
-		return nil
-	}
-	if s.SketchError {
-		return nil
+func Sizer(
+	onlyUpdateCompilationDatabase, sketchError, verbose bool,
+	buildProperties *properties.Map,
+	stdoutWriter, stderrWriter io.Writer,
+	printInfoFn, printWarnFn func(msg string),
+	warningsLevel string,
+) (builder.ExecutablesFileSections, error) {
+	if onlyUpdateCompilationDatabase || sketchError {
+		return nil, nil
 	}
 
-	buildProperties := ctx.BuildProperties
-
 	if buildProperties.ContainsKey("recipe.advanced_size.pattern") {
-		return checkSizeAdvanced(ctx, buildProperties)
+		return checkSizeAdvanced(buildProperties, verbose, stdoutWriter, stderrWriter, printInfoFn, printWarnFn)
 	}
 
-	return checkSize(ctx, buildProperties)
+	return checkSize(buildProperties, verbose, stdoutWriter, stderrWriter, printInfoFn, printWarnFn, warningsLevel)
 }
 
-func checkSizeAdvanced(ctx *types.Context, properties *properties.Map) error {
-	command, err := builder_utils.PrepareCommandForRecipe(properties, "recipe.advanced_size.pattern", false)
+func checkSizeAdvanced(buildProperties *properties.Map,
+	verbose bool,
+	stdoutWriter, stderrWriter io.Writer,
+	printInfoFn, printWarnFn func(msg string),
+) (builder.ExecutablesFileSections, error) {
+	command, err := utils.PrepareCommandForRecipe(buildProperties, "recipe.advanced_size.pattern", false)
 	if err != nil {
-		return errors.New(tr("Error while determining sketch size: %s", err))
+		return nil, errors.New(tr("Error while determining sketch size: %s", err))
 	}
 
-	out, _, err := utils.ExecCommand(ctx, command, utils.Capture /* stdout */, utils.Show /* stderr */)
+	verboseInfo, out, _, err := utils.ExecCommand(verbose, stdoutWriter, stderrWriter, command, utils.Capture /* stdout */, utils.Show /* stderr */)
+	if verbose {
+		printInfoFn(string(verboseInfo))
+	}
 	if err != nil {
-		return errors.New(tr("Error while determining sketch size: %s", err))
+		return nil, errors.New(tr("Error while determining sketch size: %s", err))
 	}
 
 	type AdvancedSizerResponse struct {
@@ -67,7 +71,7 @@ func checkSizeAdvanced(ctx *types.Context, properties *properties.Map) error {
 		// likely be printed in red. Errors will stop build/upload.
 		Severity string `json:"severity"`
 		// Sections are the sections sizes for machine readable use
-		Sections []types.ExecutableSectionSize `json:"sections"`
+		Sections []builder.ExecutableSectionSize `json:"sections"`
 		// ErrorMessage is a one line error message like:
 		// "text section exceeds available space in board"
 		// it must be set when Severity is "error"
@@ -76,71 +80,76 @@ func checkSizeAdvanced(ctx *types.Context, properties *properties.Map) error {
 
 	var resp AdvancedSizerResponse
 	if err := json.Unmarshal(out, &resp); err != nil {
-		return errors.New(tr("Error while determining sketch size: %s", err))
+		return nil, errors.New(tr("Error while determining sketch size: %s", err))
 	}
 
-	ctx.ExecutableSectionsSize = resp.Sections
+	executableSectionsSize := resp.Sections
 	switch resp.Severity {
 	case "error":
-		ctx.Warn(resp.Output)
-		return errors.New(resp.ErrorMessage)
+		printWarnFn(resp.Output)
+		return executableSectionsSize, errors.New(resp.ErrorMessage)
 	case "warning":
-		ctx.Warn(resp.Output)
+		printWarnFn(resp.Output)
 	case "info":
-		ctx.Info(resp.Output)
+		printInfoFn(resp.Output)
 	default:
-		return fmt.Errorf("invalid '%s' severity from sketch sizer: it must be 'error', 'warning' or 'info'", resp.Severity)
+		return executableSectionsSize, fmt.Errorf("invalid '%s' severity from sketch sizer: it must be 'error', 'warning' or 'info'", resp.Severity)
 	}
-	return nil
+	return executableSectionsSize, nil
 }
 
-func checkSize(ctx *types.Context, buildProperties *properties.Map) error {
+func checkSize(buildProperties *properties.Map,
+	verbose bool,
+	stdoutWriter, stderrWriter io.Writer,
+	printInfoFn, printWarnFn func(msg string),
+	warningsLevel string,
+) (builder.ExecutablesFileSections, error) {
 	properties := buildProperties.Clone()
-	properties.Set("compiler.warning_flags", properties.Get("compiler.warning_flags."+ctx.WarningsLevel))
+	properties.Set("compiler.warning_flags", properties.Get("compiler.warning_flags."+warningsLevel))
 
 	maxTextSizeString := properties.Get("upload.maximum_size")
 	maxDataSizeString := properties.Get("upload.maximum_data_size")
 
 	if maxTextSizeString == "" {
-		return nil
+		return nil, nil
 	}
 
 	maxTextSize, err := strconv.Atoi(maxTextSizeString)
 	if err != nil {
-		return err
+		return nil, err
 	}
 
 	maxDataSize := -1
 	if maxDataSizeString != "" {
 		maxDataSize, err = strconv.Atoi(maxDataSizeString)
 		if err != nil {
-			return err
+			return nil, err
 		}
 	}
 
-	textSize, dataSize, _, err := execSizeRecipe(ctx, properties)
+	textSize, dataSize, _, err := execSizeRecipe(properties, verbose, stdoutWriter, stderrWriter, printInfoFn)
 	if err != nil {
-		ctx.Warn(tr("Couldn't determine program size"))
-		return nil
+		printWarnFn(tr("Couldn't determine program size"))
+		return nil, nil
 	}
 
-	ctx.Info(tr("Sketch uses %[1]s bytes (%[3]s%%) of program storage space. Maximum is %[2]s bytes.",
+	printInfoFn(tr("Sketch uses %[1]s bytes (%[3]s%%) of program storage space. Maximum is %[2]s bytes.",
 		strconv.Itoa(textSize),
 		strconv.Itoa(maxTextSize),
 		strconv.Itoa(textSize*100/maxTextSize)))
 	if dataSize >= 0 {
 		if maxDataSize > 0 {
-			ctx.Info(tr("Global variables use %[1]s bytes (%[3]s%%) of dynamic memory, leaving %[4]s bytes for local variables. Maximum is %[2]s bytes.",
+			printInfoFn(tr("Global variables use %[1]s bytes (%[3]s%%) of dynamic memory, leaving %[4]s bytes for local variables. Maximum is %[2]s bytes.",
 				strconv.Itoa(dataSize),
 				strconv.Itoa(maxDataSize),
 				strconv.Itoa(dataSize*100/maxDataSize),
 				strconv.Itoa(maxDataSize-dataSize)))
 		} else {
-			ctx.Info(tr("Global variables use %[1]s bytes of dynamic memory.", strconv.Itoa(dataSize)))
+			printInfoFn(tr("Global variables use %[1]s bytes of dynamic memory.", strconv.Itoa(dataSize)))
 		}
 	}
 
-	ctx.ExecutableSectionsSize = []types.ExecutableSectionSize{
+	executableSectionsSize := []builder.ExecutableSectionSize{
 		{
 			Name:    "text",
 			Size:    textSize,
@@ -148,7 +157,7 @@ func checkSize(ctx *types.Context, buildProperties *properties.Map) error {
 		},
 	}
 	if maxDataSize > 0 {
-		ctx.ExecutableSectionsSize = append(ctx.ExecutableSectionsSize, types.ExecutableSectionSize{
+		executableSectionsSize = append(executableSectionsSize, builder.ExecutableSectionSize{
 			Name:    "data",
 			Size:    dataSize,
 			MaxSize: maxDataSize,
@@ -156,36 +165,43 @@ func checkSize(ctx *types.Context, buildProperties *properties.Map) error {
 	}
 
 	if textSize > maxTextSize {
-		ctx.Warn(tr("Sketch too big; see %[1]s for tips on reducing it.", "https://support.arduino.cc/hc/en-us/articles/360013825179"))
-		return errors.New(tr("text section exceeds available space in board"))
+		printWarnFn(tr("Sketch too big; see %[1]s for tips on reducing it.", "https://support.arduino.cc/hc/en-us/articles/360013825179"))
+		return executableSectionsSize, errors.New(tr("text section exceeds available space in board"))
 	}
 
 	if maxDataSize > 0 && dataSize > maxDataSize {
-		ctx.Warn(tr("Not enough memory; see %[1]s for tips on reducing your footprint.", "https://support.arduino.cc/hc/en-us/articles/360013825179"))
-		return errors.New(tr("data section exceeds available space in board"))
+		printWarnFn(tr("Not enough memory; see %[1]s for tips on reducing your footprint.", "https://support.arduino.cc/hc/en-us/articles/360013825179"))
+		return executableSectionsSize, errors.New(tr("data section exceeds available space in board"))
 	}
 
 	if w := properties.Get("build.warn_data_percentage"); w != "" {
 		warnDataPercentage, err := strconv.Atoi(w)
 		if err != nil {
-			return err
+			return executableSectionsSize, err
 		}
 		if maxDataSize > 0 && dataSize > maxDataSize*warnDataPercentage/100 {
-			ctx.Warn(tr("Low memory available, stability problems may occur."))
+			printWarnFn(tr("Low memory available, stability problems may occur."))
 		}
 	}
 
-	return nil
+	return executableSectionsSize, nil
 }
 
-func execSizeRecipe(ctx *types.Context, properties *properties.Map) (textSize int, dataSize int, eepromSize int, resErr error) {
-	command, err := builder_utils.PrepareCommandForRecipe(properties, "recipe.size.pattern", false)
+func execSizeRecipe(properties *properties.Map,
+	verbose bool,
+	stdoutWriter, stderrWriter io.Writer,
+	printInfoFn func(msg string),
+) (textSize int, dataSize int, eepromSize int, resErr error) {
+	command, err := utils.PrepareCommandForRecipe(properties, "recipe.size.pattern", false)
 	if err != nil {
 		resErr = fmt.Errorf(tr("Error while determining sketch size: %s"), err)
 		return
 	}
 
-	out, _, err := utils.ExecCommand(ctx, command, utils.Capture /* stdout */, utils.Show /* stderr */)
+	verboseInfo, out, _, err := utils.ExecCommand(verbose, stdoutWriter, stderrWriter, command, utils.Capture /* stdout */, utils.Show /* stderr */)
+	if verbose {
+		printInfoFn(string(verboseInfo))
+	}
 	if err != nil {
 		resErr = fmt.Errorf(tr("Error while determining sketch size: %s"), err)
 		return

legacy/builder/phases/sketch_builder.go

@@ -16,41 +16,76 @@
 package phases
 
 import (
+	"io"
+
+	"github.com/arduino/arduino-cli/arduino/builder"
 	"github.com/arduino/arduino-cli/arduino/builder/cpp"
+	"github.com/arduino/arduino-cli/arduino/builder/progress"
+	"github.com/arduino/arduino-cli/arduino/builder/utils"
 	f "github.com/arduino/arduino-cli/internal/algorithms"
-	"github.com/arduino/arduino-cli/legacy/builder/builder_utils"
-	"github.com/arduino/arduino-cli/legacy/builder/types"
+	rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1"
+	"github.com/arduino/go-paths-helper"
+	"github.com/arduino/go-properties-orderedmap"
 	"github.com/pkg/errors"
 )
 
-type SketchBuilder struct{}
-
-func (s *SketchBuilder) Run(ctx *types.Context) error {
-	sketchBuildPath := ctx.SketchBuildPath
-	buildProperties := ctx.BuildProperties
-	includesFolders := ctx.SketchLibrariesDetector.IncludeFolders()
+func SketchBuilder(
+	sketchBuildPath *paths.Path,
+	buildProperties *properties.Map,
+	includesFolders paths.PathList,
+	onlyUpdateCompilationDatabase, verbose bool,
+	compilationDatabase *builder.CompilationDatabase,
+	jobs int,
+	warningsLevel string,
+	stdoutWriter, stderrWriter io.Writer,
+	verboseInfoFn func(msg string),
+	verboseStdoutFn, verboseStderrFn func(data []byte),
+	progress *progress.Struct, progressCB rpc.TaskProgressCB,
+) (paths.PathList, error) {
 	includes := f.Map(includesFolders.AsStrings(), cpp.WrapWithHyphenI)
 
 	if err := sketchBuildPath.MkdirAll(); err != nil {
-		return errors.WithStack(err)
+		return nil, errors.WithStack(err)
 	}
 
-	objectFiles, err := builder_utils.CompileFiles(ctx, sketchBuildPath, sketchBuildPath, buildProperties, includes)
+	sketchObjectFiles, err := utils.CompileFiles(
+		sketchBuildPath, sketchBuildPath, buildProperties, includes,
+		onlyUpdateCompilationDatabase,
+		compilationDatabase,
+		jobs,
+		verbose,
+		warningsLevel,
+		stdoutWriter, stderrWriter,
+		verboseInfoFn,
+		verboseStdoutFn,
+		verboseStderrFn,
+		progress, progressCB,
+	)
 	if err != nil {
-		return errors.WithStack(err)
+		return nil, errors.WithStack(err)
 	}
 
 	// The "src/" subdirectory of a sketch is compiled recursively
 	sketchSrcPath := sketchBuildPath.Join("src")
 	if sketchSrcPath.IsDir() {
-		srcObjectFiles, err := builder_utils.CompileFilesRecursive(ctx, sketchSrcPath, sketchSrcPath, buildProperties, includes)
+		srcObjectFiles, err := utils.CompileFilesRecursive(
+			sketchSrcPath, sketchSrcPath, buildProperties, includes,
+			onlyUpdateCompilationDatabase,
+			compilationDatabase,
+			jobs,
+			verbose,
+			warningsLevel,
+			stdoutWriter, stderrWriter,
+			verboseInfoFn,
+			verboseStdoutFn,
+			verboseStderrFn,
+			progress, progressCB,
+		)
 		if err != nil {
-			return errors.WithStack(err)
+			return nil, errors.WithStack(err)
 		}
-		objectFiles.AddAll(srcObjectFiles)
+		sketchObjectFiles.AddAll(srcObjectFiles)
 	}
 
-	ctx.SketchObjectFiles = objectFiles
-
-	return nil
+	return sketchObjectFiles, nil
 }

legacy/builder/recipe_runner.go

@@ -20,9 +20,8 @@ import (
 	"sort"
 	"strings"
 
-	"github.com/arduino/arduino-cli/legacy/builder/builder_utils"
 	"github.com/arduino/arduino-cli/legacy/builder/types"
-	"github.com/arduino/arduino-cli/legacy/builder/utils"
+	"github.com/arduino/arduino-cli/arduino/builder/utils"
 	properties "github.com/arduino/go-properties-orderedmap"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
@@ -44,7 +43,7 @@ func (s *RecipeByPrefixSuffixRunner) Run(ctx *types.Context) error {
 	for _, recipe := range recipes {
 		logrus.Debugf(fmt.Sprintf("Running recipe: %s", recipe))
 
-		command, err := builder_utils.PrepareCommandForRecipe(properties, recipe, false)
+		command, err := utils.PrepareCommandForRecipe(properties, recipe, false)
 		if err != nil {
 			return errors.WithStack(err)
 		}
@@ -56,7 +55,10 @@ func (s *RecipeByPrefixSuffixRunner) Run(ctx *types.Context) error {
 			return nil
 		}
 
-		_, _, err = utils.ExecCommand(ctx, command, utils.ShowIfVerbose /* stdout */, utils.Show /* stderr */)
+		verboseInfo, _, _, err := utils.ExecCommand(ctx.Verbose, ctx.Stdout, ctx.Stderr, command, utils.ShowIfVerbose /* stdout */, utils.Show /* stderr */)
+		if ctx.Verbose {
+			ctx.Info(string(verboseInfo))
+		}
 		if err != nil {
 			return errors.WithStack(err)
 		}

legacy/builder/test/builder_test.go

@@ -84,11 +84,8 @@ func prepareBuilderTestContext(t *testing.T, ctx *types.Context, sketchPath *pat
 		// NoError(t, err)
 		fmt.Println(err)
 	}
-	if !ctx.CanUseCachedTools {
-		if ctx.BuiltInToolsDirs != nil {
-			pmb.LoadToolsFromBundleDirectories(ctx.BuiltInToolsDirs)
-		}
-		ctx.CanUseCachedTools = true
+	if ctx.BuiltInToolsDirs != nil {
+		pmb.LoadToolsFromBundleDirectories(ctx.BuiltInToolsDirs)
 	}
 	pm := pmb.Build()
 	pme, _ /* never release... */ := pm.NewExplorer()
@@ -100,7 +97,7 @@ func prepareBuilderTestContext(t *testing.T, ctx *types.Context, sketchPath *pat
 		ctx.Sketch = sk
 	}
 
-	ctx.Builder = bldr.NewBuilder(ctx.Sketch)
+	ctx.Builder = bldr.NewBuilder(ctx.Sketch, nil)
 	if fqbn != "" {
 		ctx.FQBN = parseFQBN(t, fqbn)
 		targetPackage, targetPlatform, targetBoard, buildProperties, buildPlatform, err := pme.ResolveFQBN(ctx.FQBN)
@@ -125,7 +122,7 @@ func prepareBuilderTestContext(t *testing.T, ctx *types.Context, sketchPath *pat
 	if !stepToSkip[skipLibraries] {
 		lm, libsResolver, _, err := detector.LibrariesLoader(
 			false, nil,
-			ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs,
+			ctx.BuiltInLibrariesDirs, nil, ctx.OtherLibrariesDirs,
 			ctx.ActualPlatform, ctx.TargetPlatform,
 		)
 		require.NoError(t, err)

legacy/builder/test/helper.go

@@ -21,7 +21,6 @@ import (
 
 	"github.com/arduino/arduino-cli/arduino/cores"
 	"github.com/arduino/arduino-cli/arduino/libraries"
-	"github.com/arduino/arduino-cli/legacy/builder/constants"
 	"github.com/arduino/arduino-cli/legacy/builder/types"
 	paths "github.com/arduino/go-paths-helper"
 	"github.com/stretchr/testify/require"
@@ -40,13 +39,6 @@ func SetupBuildPath(t *testing.T, ctx *types.Context) *paths.Path {
 	return buildPath
 }
 
-func SetupBuildCachePath(t *testing.T, ctx *types.Context) *paths.Path {
-	buildCachePath, err := paths.MkTempDir(constants.EMPTY_STRING, "test_build_cache")
-	require.NoError(t, err)
-	ctx.CoreBuildCachePath = buildCachePath
-	return buildCachePath
-}
-
 func parseFQBN(t *testing.T, fqbnIn string) *cores.FQBN {
 	fqbn, err := cores.ParseFQBN(fqbnIn)
 	require.NoError(t, err)

legacy/builder/test/libraries_loader_test.go

@@ -49,7 +49,7 @@ func TestLoadLibrariesAVR(t *testing.T) {
 
 	lm, libsResolver, _, err := detector.LibrariesLoader(
 		false, nil,
-		ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs,
+		ctx.BuiltInLibrariesDirs, nil, ctx.OtherLibrariesDirs,
 		ctx.ActualPlatform, ctx.TargetPlatform,
 	)
 	require.NoError(t, err)
@@ -153,7 +153,7 @@ func TestLoadLibrariesSAM(t *testing.T) {
 
 	lm, libsResolver, _, err := detector.LibrariesLoader(
 		false, nil,
-		ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs,
+		ctx.BuiltInLibrariesDirs, nil, ctx.OtherLibrariesDirs,
 		ctx.ActualPlatform, ctx.TargetPlatform,
 	)
 	require.NoError(t, err)
@@ -230,7 +230,7 @@ func TestLoadLibrariesAVRNoDuplicateLibrariesFolders(t *testing.T) {
 
 	lm, _, _, err := detector.LibrariesLoader(
 		false, nil,
-		ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs,
+		ctx.BuiltInLibrariesDirs, nil, ctx.OtherLibrariesDirs,
 		ctx.ActualPlatform, ctx.TargetPlatform,
 	)
 	require.NoError(t, err)
@@ -253,7 +253,7 @@ func TestLoadLibrariesMyAVRPlatform(t *testing.T) {
 
 	lm, _, _, err := detector.LibrariesLoader(
 		false, nil,
-		ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs,
+		ctx.BuiltInLibrariesDirs, nil, ctx.OtherLibrariesDirs,
 		ctx.ActualPlatform, ctx.TargetPlatform,
 	)
 	require.NoError(t, err)

legacy/builder/types/context.go

@@ -24,6 +24,7 @@ import (
 
 	"github.com/arduino/arduino-cli/arduino/builder"
 	"github.com/arduino/arduino-cli/arduino/builder/detector"
+	"github.com/arduino/arduino-cli/arduino/builder/progress"
 	"github.com/arduino/arduino-cli/arduino/cores"
 	"github.com/arduino/arduino-cli/arduino/cores/packagemanager"
 	"github.com/arduino/arduino-cli/arduino/sketch"
@@ -32,34 +33,6 @@ import (
 	properties "github.com/arduino/go-properties-orderedmap"
 )
 
-type ProgressStruct struct {
-	Progress   float32
-	StepAmount float32
-	Parent     *ProgressStruct
-}
-
-func (p *ProgressStruct) AddSubSteps(steps int) {
-	p.Parent = &ProgressStruct{
-		Progress:   p.Progress,
-		StepAmount: p.StepAmount,
-		Parent:     p.Parent,
-	}
-	if p.StepAmount == 0.0 {
-		p.StepAmount = 100.0
-	}
-	p.StepAmount /= float32(steps)
-}
-
-func (p *ProgressStruct) RemoveSubSteps() {
-	p.Progress = p.Parent.Progress
-	p.StepAmount = p.Parent.StepAmount
-	p.Parent = p.Parent.Parent
-}
-
-func (p *ProgressStruct) CompleteStep() {
-	p.Progress += p.StepAmount
-}
-
 // Context structure
 type Context struct {
 	Builder                 *builder.Builder
@@ -70,8 +43,6 @@ type Context struct {
 	BuiltInToolsDirs     paths.PathList
 	BuiltInLibrariesDirs *paths.Path
 	OtherLibrariesDirs   paths.PathList
-	LibraryDirs          paths.PathList // List of paths pointing to individual library root folders
-	WatchedLocations     paths.PathList
 	FQBN                 *cores.FQBN
 	Clean                bool
 
@@ -86,17 +57,15 @@ type Context struct {
 	TargetPlatform *cores.PlatformRelease
 	ActualPlatform *cores.PlatformRelease
 
-	BuildProperties              *properties.Map
-	BuildPath                    *paths.Path
-	SketchBuildPath              *paths.Path
-	CoreBuildPath                *paths.Path
-	CoreBuildCachePath           *paths.Path
-	CoreArchiveFilePath          *paths.Path
-	CoreObjectsFiles             paths.PathList
-	LibrariesBuildPath           *paths.Path
-	LibrariesObjectFiles         paths.PathList
-	SketchObjectFiles            paths.PathList
-	IgnoreSketchFolderNameErrors bool
+	BuildProperties      *properties.Map
+	BuildPath            *paths.Path
+	SketchBuildPath      *paths.Path
+	CoreBuildPath        *paths.Path
+	CoreArchiveFilePath  *paths.Path
+	CoreObjectsFiles     paths.PathList
+	LibrariesBuildPath   *paths.Path
+	LibrariesObjectFiles paths.PathList
+	SketchObjectFiles    paths.PathList
 
 	Sketch        *sketch.Sketch
 	WarningsLevel string
@@ -108,19 +77,13 @@ type Context struct {
 	Verbose bool
 
 	// Dry run, only create progress map
-	Progress ProgressStruct
+	Progress progress.Struct
 	// Send progress events to this callback
 	ProgressCB rpc.TaskProgressCB
 
 	// Custom build properties defined by user (line by line as "key=value" pairs)
 	CustomBuildProperties []string
 
-	// Reuse old tools since the backing storage didn't change
-	CanUseCachedTools bool
-
-	// Experimental: use arduino-preprocessor to create prototypes
-	UseArduinoPreprocessor bool
-
 	// Parallel processes
 	Jobs int
 
@@ -130,7 +93,7 @@ type Context struct {
 	stdLock sync.Mutex
 
 	// Sizer results
-	ExecutableSectionsSize ExecutablesFileSections
+	ExecutableSectionsSize builder.ExecutablesFileSections
 
 	// Compilation Database to build/update
 	CompilationDatabase *builder.CompilationDatabase
@@ -143,29 +106,6 @@ type Context struct {
 	SourceOverride map[string]string
 }
 
-// ExecutableSectionSize represents a section of the executable output file
-type ExecutableSectionSize struct {
-	Name    string `json:"name"`
-	Size    int    `json:"size"`
-	MaxSize int    `json:"max_size"`
-}
-
-// ExecutablesFileSections is an array of ExecutablesFileSection
-type ExecutablesFileSections []ExecutableSectionSize
-
-// ToRPCExecutableSectionSizeArray transforms this array into a []*rpc.ExecutableSectionSize
-func (s ExecutablesFileSections) ToRPCExecutableSectionSizeArray() []*rpc.ExecutableSectionSize {
-	res := []*rpc.ExecutableSectionSize{}
-	for _, section := range s {
-		res = append(res, &rpc.ExecutableSectionSize{
-			Name:    section.Name,
-			Size:    int64(section.Size),
-			MaxSize: int64(section.MaxSize),
-		})
-	}
-	return res
-}
-
 func (ctx *Context) ExtractBuildOptions() *properties.Map {
 	opts := properties.NewMap()
 	opts.Set("hardwareFolders", strings.Join(ctx.HardwareDirs.AsStrings(), ","))

legacy/builder/utils/utils.go

-// This file is part of arduino-cli.
-//
-// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
-//
-// This software is released under the GNU General Public License version 3,
-// which covers the main part of arduino-cli.
-// The terms of this license can be found at:
-// https://www.gnu.org/licenses/gpl-3.0.en.html
-//
-// You can be released from the requirements of the above licenses by purchasing
-// a commercial license. Buying such a license is mandatory if you want to
-// modify or otherwise use the software for commercial activities involving the
-// Arduino software without disclosing the source code of your own applications.
-// To purchase a commercial license, send an email to license@arduino.cc.
-
-package utils
-
-import (
-	"bytes"
-	"os"
-	"strings"
-
-	"github.com/arduino/arduino-cli/executils"
-	f "github.com/arduino/arduino-cli/internal/algorithms"
-	"github.com/arduino/arduino-cli/legacy/builder/types"
-	"github.com/pkg/errors"
-)
-
-func printableArgument(arg string) string {
-	if strings.ContainsAny(arg, "\"\\ \t") {
-		arg = strings.Replace(arg, "\\", "\\\\", -1)
-		arg = strings.Replace(arg, "\"", "\\\"", -1)
-		return "\"" + arg + "\""
-	} else {
-		return arg
-	}
-}
-
-// Convert a command and argument slice back to a printable string.
-// This adds basic escaping which is sufficient for debug output, but
-// probably not for shell interpretation. This essentially reverses
-// ParseCommandLine.
-func PrintableCommand(parts []string) string {
-	return strings.Join(f.Map(parts, printableArgument), " ")
-}
-
-const (
-	Ignore        = 0 // Redirect to null
-	Show          = 1 // Show on stdout/stderr as normal
-	ShowIfVerbose = 2 // Show if verbose is set, Ignore otherwise
-	Capture       = 3 // Capture into buffer
-)
-
-func ExecCommand(ctx *types.Context, command *executils.Process, stdout int, stderr int) ([]byte, []byte, error) {
-	if ctx.Verbose {
-		ctx.Info(PrintableCommand(command.GetArgs()))
-	}
-
-	stdoutBuffer := &bytes.Buffer{}
-	if stdout == Capture {
-		command.RedirectStdoutTo(stdoutBuffer)
-	} else if stdout == Show || (stdout == ShowIfVerbose && ctx.Verbose) {
-		if ctx.Stdout != nil {
-			command.RedirectStdoutTo(ctx.Stdout)
-		} else {
-			command.RedirectStdoutTo(os.Stdout)
-		}
-	}
-
-	stderrBuffer := &bytes.Buffer{}
-	if stderr == Capture {
-		command.RedirectStderrTo(stderrBuffer)
-	} else if stderr == Show || (stderr == ShowIfVerbose && ctx.Verbose) {
-		if ctx.Stderr != nil {
-			command.RedirectStderrTo(ctx.Stderr)
-		} else {
-			command.RedirectStderrTo(os.Stderr)
-		}
-	}
-
-	err := command.Start()
-	if err != nil {
-		return nil, nil, errors.WithStack(err)
-	}
-
-	err = command.Wait()
-	return stdoutBuffer.Bytes(), stderrBuffer.Bytes(), errors.WithStack(err)
-}
-
-type loggerAction struct {
-	onlyIfVerbose bool
-	warn          bool
-	msg           string
-}
-
-func (l *loggerAction) Run(ctx *types.Context) error {
-	if !l.onlyIfVerbose || ctx.Verbose {
-		if l.warn {
-			ctx.Warn(l.msg)
-		} else {
-			ctx.Info(l.msg)
-		}
-	}
-	return nil
-}
-
-func LogIfVerbose(warn bool, msg string) types.Command {
-	return &loggerAction{onlyIfVerbose: true, warn: warn, msg: msg}
-}

legacy/builder/wipeout_build_path_if_build_options_changed.go

@@ -19,7 +19,7 @@ import (
 	"encoding/json"
 	"path/filepath"
 
-	"github.com/arduino/arduino-cli/legacy/builder/builder_utils"
+	"github.com/arduino/arduino-cli/arduino/builder/utils"
 	"github.com/arduino/arduino-cli/legacy/builder/constants"
 	"github.com/arduino/arduino-cli/legacy/builder/types"
 	"github.com/arduino/go-paths-helper"
@@ -66,9 +66,9 @@ func (s *WipeoutBuildPathIfBuildOptionsChanged) Run(ctx *types.Context) error {
 		coreFolder := buildProperties.GetPath("build.core.path")
 		realCoreFolder := coreFolder.Parent().Parent()
 		jsonPath := ctx.BuildPath.Join(constants.BUILD_OPTIONS_FILE)
-		coreUnchanged, _ := builder_utils.DirContentIsOlderThan(realCoreFolder, jsonPath, ".txt")
+		coreUnchanged, _ := utils.DirContentIsOlderThan(realCoreFolder, jsonPath, ".txt")
 		if coreUnchanged && targetCoreFolder != nil && !realCoreFolder.EqualsTo(targetCoreFolder) {
-			coreUnchanged, _ = builder_utils.DirContentIsOlderThan(targetCoreFolder, jsonPath, ".txt")
+			coreUnchanged, _ = utils.DirContentIsOlderThan(targetCoreFolder, jsonPath, ".txt")
 		}
 		if coreUnchanged {
 			return nil