[skip-changelog] legacy: Arduino preprocess subroutine refactorization (part 2) (#2191)

* Movec ctags-related structs in ctags package

* Unified GCCPreprocRunner* functions

There is no need to duplicate a function that basically does the same
thing.

* Added implementation of functional algorithms

(until they are available in the golang standard library...)

* legacy: use generic algos where possibile

* Removed useless call to FilterSketchSource

* Converted AddPrototypes into a function

internal/algorithms/slices.go

+// This file is part of arduino-cli.
+//
+// Copyright 2023 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to license@arduino.cc.
+
+package f
+
+// Matcher is a function that tests if a given value match a certain criteria.
+type Matcher[T any] func(T) bool
+
+// Reducer is a function that combines two values of the same type and return
+// the combined value.
+type Reducer[T any] func(T, T) T
+
+// Mapper is a function that converts a value of one type to another type.
+type Mapper[T, U any] func(T) U
+
+// Filter takes a slice of type []T and a Matcher[T]. It returns a newly
+// allocated slice containing only those elements of the input slice that
+// satisfy the matcher.
+func Filter[T any](values []T, matcher Matcher[T]) []T {
+	res := []T{}
+	for _, x := range values {
+		if matcher(x) {
+			res = append(res, x)
+		}
+	}
+	return res
+}
+
+// Map applies the Mapper function to each element of the slice and returns
+// a new slice with the results in the same order.
+func Map[T, U any](values []T, mapper Mapper[T, U]) []U {
+	res := []U{}
+	for _, x := range values {
+		res = append(res, mapper(x))
+	}
+	return res
+}
+
+// Reduce applies the Reducer function to all elements of the input values
+// and returns the result.
+func Reduce[T any](values []T, reducer Reducer[T]) T {
+	var result T
+	for _, v := range values {
+		result = reducer(result, v)
+	}
+	return result
+}
+
+// Equals return a Matcher that matches the given value
+func Equals[T comparable](value T) Matcher[T] {
+	return func(x T) bool {
+		return x == value
+	}
+}
+
+// NotEquals return a Matcher that does not match the given value
+func NotEquals[T comparable](value T) Matcher[T] {
+	return func(x T) bool {
+		return x != value
+	}
+}

internal/algorithms/slices_test.go

+// This file is part of arduino-cli.
+//
+// Copyright 2023 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to license@arduino.cc.
+
+package f_test
+
+import (
+	"strings"
+	"testing"
+
+	f "github.com/arduino/arduino-cli/internal/algorithms"
+	"github.com/stretchr/testify/require"
+)
+
+func TestFilter(t *testing.T) {
+	a := []string{"aaa", "bbb", "ccc"}
+	require.Equal(t, []string{"bbb", "ccc"}, f.Filter(a, func(x string) bool { return x > "b" }))
+	b := []int{5, 9, 15, 2, 4, -2}
+	require.Equal(t, []int{5, 9, 15}, f.Filter(b, func(x int) bool { return x > 4 }))
+}
+
+func TestIsEmpty(t *testing.T) {
+	require.True(t, f.Equals(int(0))(0))
+	require.False(t, f.Equals(int(1))(0))
+	require.True(t, f.Equals("")(""))
+	require.False(t, f.Equals("abc")(""))
+	require.False(t, f.NotEquals(int(0))(0))
+	require.True(t, f.NotEquals(int(1))(0))
+	require.False(t, f.NotEquals("")(""))
+	require.True(t, f.NotEquals("abc")(""))
+}
+
+func TestMap(t *testing.T) {
+	value := "hello, world , how are,you? "
+	parts := f.Map(strings.Split(value, ","), strings.TrimSpace)
+
+	require.Equal(t, 4, len(parts))
+	require.Equal(t, "hello", parts[0])
+	require.Equal(t, "world", parts[1])
+	require.Equal(t, "how are", parts[2])
+	require.Equal(t, "you?", parts[3])
+}

legacy/builder/add_additional_entries_to_context.go

@@ -53,8 +53,6 @@ func (*AddAdditionalEntriesToContext) Run(ctx *types.Context) error {
 		ctx.WarningsLevel = DEFAULT_WARNINGS_LEVEL
 	}
 
-	ctx.CollectedSourceFiles = &types.UniqueSourceFileQueue{}
-
 	ctx.LibrariesResolutionResults = map[string]types.LibraryResolutionResult{}
 
 	return nil

legacy/builder/builder_utils/utils.go

@@ -26,6 +26,7 @@ import (
 
 	"github.com/arduino/arduino-cli/arduino/globals"
 	"github.com/arduino/arduino-cli/i18n"
+	f "github.com/arduino/arduino-cli/internal/algorithms"
 	"github.com/arduino/arduino-cli/legacy/builder/constants"
 	"github.com/arduino/arduino-cli/legacy/builder/types"
 	"github.com/arduino/arduino-cli/legacy/builder/utils"
@@ -262,10 +263,10 @@ func ObjFileIsUpToDate(sourceFile, objectFile, dependencyFile *paths.Path) (bool
 		return false, errors.WithStack(err)
 	}
 
-	rows = utils.Map(rows, removeEndingBackSlash)
-	rows = utils.Map(rows, strings.TrimSpace)
-	rows = utils.Map(rows, unescapeDep)
-	rows = utils.Filter(rows, nonEmptyString)
+	rows = f.Map(rows, removeEndingBackSlash)
+	rows = f.Map(rows, strings.TrimSpace)
+	rows = f.Map(rows, unescapeDep)
+	rows = f.Filter(rows, f.NotEquals(""))
 
 	if len(rows) == 0 {
 		return true, nil
@@ -327,10 +328,6 @@ func removeEndingBackSlash(s string) string {
 	return strings.TrimSuffix(s, "\\")
 }
 
-func nonEmptyString(s string) bool {
-	return s != constants.EMPTY_STRING
-}
-
 func CoreOrReferencedCoreHasChanged(corePath, targetCorePath, targetFile *paths.Path) bool {
 
 	targetFileStat, err := targetFile.Stat()

legacy/builder/container_add_prototypes.go

@@ -39,7 +39,9 @@ func PreprocessSketchWithCtags(ctx *types.Context) error {
 
 	// Run preprocessor
 	sourceFile := ctx.SketchBuildPath.Join(ctx.Sketch.MainFile.Base() + ".cpp")
-	if err := GCCPreprocRunner(ctx, sourceFile, targetFilePath, ctx.IncludeFolders); err != nil {
+	stderr, err := GCCPreprocRunner(ctx, sourceFile, targetFilePath, ctx.IncludeFolders)
+	ctx.WriteStderr(stderr)
+	if err != nil {
 		if !ctx.OnlyUpdateCompilationDatabase {
 			return errors.WithStack(err)
 		}
@@ -60,18 +62,10 @@ func PreprocessSketchWithCtags(ctx *types.Context) error {
 		ctx.SketchSourceAfterCppPreprocessing = filterSketchSource(ctx.Sketch, bytes.NewReader(src), false)
 	}
 
-	commands := []types.Command{
-		&CTagsRunner{Source: &ctx.SketchSourceAfterCppPreprocessing, TargetFileName: "sketch_merged.cpp"},
-		&PrototypesAdder{},
-	}
-
-	for _, command := range commands {
-		PrintRingNameIfDebug(ctx, command)
-		err := command.Run(ctx)
-		if err != nil {
-			return errors.WithStack(err)
-		}
+	if err := (&CTagsRunner{Source: &ctx.SketchSourceAfterCppPreprocessing, TargetFileName: "sketch_merged.cpp"}).Run(ctx); err != nil {
+		return errors.WithStack(err)
 	}
+	ctx.SketchSourceAfterArduinoPreprocessing, ctx.PrototypesSection = PrototypesAdder(ctx.SketchSourceMerged, ctx.PrototypesLineWhereToInsert, ctx.LineOffset, ctx.Prototypes, ctx.DebugPreprocessor)
 
 	if err := bldr.SketchSaveItemCpp(ctx.Sketch.MainFile, []byte(ctx.SketchSourceAfterArduinoPreprocessing), ctx.SketchBuildPath); err != nil {
 		return errors.WithStack(err)

legacy/builder/container_find_includes.go

@@ -143,23 +143,24 @@ func (s *ContainerFindIncludes) findIncludes(ctx *types.Context) error {
 		appendIncludeFolder(ctx, cache, nil, "", ctx.BuildProperties.GetPath("build.variant.path"))
 	}
 
+	sourceFileQueue := &types.UniqueSourceFileQueue{}
+
 	if !ctx.UseCachedLibrariesResolution {
 		sketch := ctx.Sketch
 		mergedfile, err := types.MakeSourceFile(ctx, sketch, paths.New(sketch.MainFile.Base()+".cpp"))
 		if err != nil {
 			return errors.WithStack(err)
 		}
-		ctx.CollectedSourceFiles.Push(mergedfile)
+		sourceFileQueue.Push(mergedfile)
 
-		sourceFilePaths := ctx.CollectedSourceFiles
-		queueSourceFilesFromFolder(ctx, sourceFilePaths, sketch, ctx.SketchBuildPath, false /* recurse */)
+		queueSourceFilesFromFolder(ctx, sourceFileQueue, sketch, ctx.SketchBuildPath, false /* recurse */)
 		srcSubfolderPath := ctx.SketchBuildPath.Join("src")
 		if srcSubfolderPath.IsDir() {
-			queueSourceFilesFromFolder(ctx, sourceFilePaths, sketch, srcSubfolderPath, true /* recurse */)
+			queueSourceFilesFromFolder(ctx, sourceFileQueue, sketch, srcSubfolderPath, true /* recurse */)
 		}
 
-		for !sourceFilePaths.Empty() {
-			err := findIncludesUntilDone(ctx, cache, sourceFilePaths.Pop())
+		for !sourceFileQueue.Empty() {
+			err := findIncludesUntilDone(ctx, cache, sourceFileQueue)
 			if err != nil {
 				cachePath.Remove()
 				return errors.WithStack(err)
@@ -314,7 +315,8 @@ func writeCache(cache *includeCache, path *paths.Path) error {
 	return nil
 }
 
-func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFile types.SourceFile) error {
+func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFileQueue *types.UniqueSourceFileQueue) error {
+	sourceFile := sourceFileQueue.Pop()
 	sourcePath := sourceFile.SourcePath(ctx)
 	targetFilePath := paths.NullPath()
 	depPath := sourceFile.DepfilePath(ctx)
@@ -367,7 +369,7 @@ func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFile t
 				ctx.Info(tr("Using cached library dependencies for file: %[1]s", sourcePath))
 			}
 		} else {
-			preproc_stderr, preproc_err = GCCPreprocRunnerForDiscoveringIncludes(ctx, sourcePath, targetFilePath, includes)
+			preproc_stderr, preproc_err = GCCPreprocRunner(ctx, sourcePath, targetFilePath, includes)
 			// Unwrap error and see if it is an ExitError.
 			_, is_exit_error := errors.Cause(preproc_err).(*exec.ExitError)
 			if preproc_err == nil {
@@ -399,7 +401,7 @@ func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFile t
 			// return errors.WithStack(err)
 			if preproc_err == nil || preproc_stderr == nil {
 				// Filename came from cache, so run preprocessor to obtain error to show
-				preproc_stderr, preproc_err = GCCPreprocRunnerForDiscoveringIncludes(ctx, sourcePath, targetFilePath, includes)
+				preproc_stderr, preproc_err = GCCPreprocRunner(ctx, sourcePath, targetFilePath, includes)
 				if preproc_err == nil {
 					// If there is a missing #include in the cache, but running
 					// gcc does not reproduce that, there is something wrong.
@@ -419,13 +421,13 @@ func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFile t
 		appendIncludeFolder(ctx, cache, sourcePath, include, library.SourceDir)
 		sourceDirs := library.SourceDirs()
 		for _, sourceDir := range sourceDirs {
-			queueSourceFilesFromFolder(ctx, ctx.CollectedSourceFiles, library, sourceDir.Dir, sourceDir.Recurse)
+			queueSourceFilesFromFolder(ctx, sourceFileQueue, library, sourceDir.Dir, sourceDir.Recurse)
 		}
 		first = false
 	}
 }
 
-func queueSourceFilesFromFolder(ctx *types.Context, queue *types.UniqueSourceFileQueue, origin interface{}, folder *paths.Path, recurse bool) error {
+func queueSourceFilesFromFolder(ctx *types.Context, sourceFileQueue *types.UniqueSourceFileQueue, origin interface{}, folder *paths.Path, recurse bool) error {
 	sourceFileExtensions := []string{}
 	for k := range globals.SourceFilesValidExtensions {
 		sourceFileExtensions = append(sourceFileExtensions, k)
@@ -440,7 +442,7 @@ func queueSourceFilesFromFolder(ctx *types.Context, queue *types.UniqueSourceFil
 		if err != nil {
 			return errors.WithStack(err)
 		}
-		queue.Push(sourceFile)
+		sourceFileQueue.Push(sourceFile)
 	}
 
 	return nil

legacy/builder/create_cmake_rule.go

@@ -119,9 +119,6 @@ func (s *ExportProjectCMake) Run(ctx *types.Context) error {
 		return err
 	}
 
-	// Use old ctags method to generate export file
-	ctx.SketchSourceMerged = filterSketchSource(ctx.Sketch, strings.NewReader(ctx.SketchSourceMerged), true)
-
 	err = utils.CopyDir(ctx.SketchBuildPath.String(), cmakeFolder.Join("sketch").String(), validExportExtensions)
 	if err != nil {
 		fmt.Println(err)

legacy/builder/ctags/ctags_has_issues.go

@@ -19,8 +19,6 @@ import (
 	"bufio"
 	"os"
 	"strings"
-
-	"github.com/arduino/arduino-cli/legacy/builder/types"
 )
 
 func (p *CTagsParser) FixCLinkageTagsDeclarations() {
@@ -42,7 +40,7 @@ func sliceContainsInt(s []int, e int) bool {
 	return false
 }
 
-func (p *CTagsParser) prototypeAndCodeDontMatch(tag *types.CTag) bool {
+func (p *CTagsParser) prototypeAndCodeDontMatch(tag *CTag) bool {
 	if tag.SkipMe {
 		return true
 	}
@@ -107,7 +105,7 @@ func (p *CTagsParser) prototypeAndCodeDontMatch(tag *types.CTag) bool {
 	return ret == -1
 }
 
-func findTemplateMultiline(tag *types.CTag) string {
+func findTemplateMultiline(tag *CTag) string {
 	code, _ := getFunctionProtoUntilTemplateToken(tag, tag.Code)
 	return removeEverythingAfterClosingRoundBracket(code)
 }
@@ -117,7 +115,7 @@ func removeEverythingAfterClosingRoundBracket(s string) string {
 	return s[0 : n+1]
 }
 
-func getFunctionProtoUntilTemplateToken(tag *types.CTag, code string) (string, int) {
+func getFunctionProtoUntilTemplateToken(tag *CTag, code string) (string, int) {
 
 	/* FIXME I'm ugly */
 	line := 0
@@ -150,7 +148,7 @@ func getFunctionProtoUntilTemplateToken(tag *types.CTag, code string) (string, i
 	return code, line
 }
 
-func getFunctionProtoWithNPreviousCharacters(tag *types.CTag, code string, n int) (string, int) {
+func getFunctionProtoWithNPreviousCharacters(tag *CTag, code string, n int) (string, int) {
 
 	/* FIXME I'm ugly */
 	expectedPrototypeLen := len(code) + n
@@ -216,7 +214,7 @@ func removeComments(text string, multilinecomment bool) (string, bool) {
 /* This function scans the source files searching for "extern C" context
  * It save the line numbers in a map filename -> {lines...}
  */
-func (p *CTagsParser) FindCLinkageLines(tags []*types.CTag) map[string][]int {
+func (p *CTagsParser) FindCLinkageLines(tags []*CTag) map[string][]int {
 
 	lines := make(map[string][]int)
 

legacy/builder/ctags/ctags_parser.go

@@ -20,8 +20,6 @@ import (
 	"strings"
 
 	"github.com/arduino/go-paths-helper"
-
-	"github.com/arduino/arduino-cli/legacy/builder/types"
 )
 
 const KIND_PROTOTYPE = "prototype"
@@ -39,11 +37,28 @@ var KNOWN_TAG_KINDS = map[string]bool{
 }
 
 type CTagsParser struct {
-	tags     []*types.CTag
+	tags     []*CTag
 	mainFile *paths.Path
 }
 
-func (p *CTagsParser) Parse(ctagsOutput []byte, mainFile *paths.Path) []*types.CTag {
+type CTag struct {
+	FunctionName string
+	Kind         string
+	Line         int
+	Code         string
+	Class        string
+	Struct       string
+	Namespace    string
+	Filename     string
+	Typeref      string
+	SkipMe       bool
+	Signature    string
+
+	Prototype          string
+	PrototypeModifiers string
+}
+
+func (p *CTagsParser) Parse(ctagsOutput []byte, mainFile *paths.Path) []*CTag {
 	rows := strings.Split(string(ctagsOutput), "\n")
 	rows = removeEmpty(rows)
 
@@ -71,7 +86,7 @@ func (p *CTagsParser) addPrototypes() {
 	}
 }
 
-func addPrototype(tag *types.CTag) {
+func addPrototype(tag *CTag) {
 	if strings.Index(tag.Prototype, TEMPLATE) == 0 {
 		if strings.Index(tag.Code, TEMPLATE) == 0 {
 			code := tag.Code
@@ -129,7 +144,7 @@ func (p *CTagsParser) skipDuplicates() {
 	}
 }
 
-type skipFuncType func(tag *types.CTag) bool
+type skipFuncType func(tag *CTag) bool
 
 func (p *CTagsParser) skipTagsWhere(skipFunc skipFuncType) {
 	for _, tag := range p.tags {
@@ -153,11 +168,11 @@ func removeSpacesAndTabs(s string) string {
 	return s
 }
 
-func tagIsUnhandled(tag *types.CTag) bool {
+func tagIsUnhandled(tag *CTag) bool {
 	return !isHandled(tag)
 }
 
-func isHandled(tag *types.CTag) bool {
+func isHandled(tag *CTag) bool {
 	if tag.Class != "" {
 		return false
 	}
@@ -170,12 +185,12 @@ func isHandled(tag *types.CTag) bool {
 	return true
 }
 
-func tagIsUnknown(tag *types.CTag) bool {
+func tagIsUnknown(tag *CTag) bool {
 	return !KNOWN_TAG_KINDS[tag.Kind]
 }
 
-func parseTag(row string) *types.CTag {
-	tag := &types.CTag{}
+func parseTag(row string) *CTag {
+	tag := &CTag{}
 	parts := strings.Split(row, "\t")
 
 	tag.FunctionName = parts[0]

legacy/builder/ctags/ctags_parser_test.go

@@ -20,12 +20,10 @@ import (
 	"path/filepath"
 	"testing"
 
-	"github.com/arduino/arduino-cli/legacy/builder/types"
-
 	"github.com/stretchr/testify/require"
 )
 
-func produceTags(t *testing.T, filename string) []*types.CTag {
+func produceTags(t *testing.T, filename string) []*CTag {
 	bytes, err := os.ReadFile(filepath.Join("test_data", filename))
 	require.NoError(t, err)
 

legacy/builder/ctags/ctags_to_prototypes.go

@@ -16,12 +16,23 @@
 package ctags
 
 import (
+	"strconv"
 	"strings"
-
-	"github.com/arduino/arduino-cli/legacy/builder/types"
 )
 
-func (p *CTagsParser) GeneratePrototypes() ([]*types.Prototype, int) {
+type Prototype struct {
+	FunctionName string
+	File         string
+	Prototype    string
+	Modifiers    string
+	Line         int
+}
+
+func (proto *Prototype) String() string {
+	return proto.Modifiers + " " + proto.Prototype + " @ " + strconv.Itoa(proto.Line)
+}
+
+func (p *CTagsParser) GeneratePrototypes() ([]*Prototype, int) {
 	return p.toPrototypes(), p.findLineWhereToInsertPrototypes()
 }
 
@@ -53,7 +64,7 @@ func (p *CTagsParser) firstFunctionPointerUsedAsArgument() int {
 	return -1
 }
 
-func functionNameUsedAsFunctionPointerIn(tag *types.CTag, functionTags []*types.CTag) bool {
+func functionNameUsedAsFunctionPointerIn(tag *CTag, functionTags []*CTag) bool {
 	for _, functionTag := range functionTags {
 		if tag.Line != functionTag.Line && strings.Contains(tag.Code, "&"+functionTag.FunctionName) {
 			return true
@@ -65,8 +76,8 @@ func functionNameUsedAsFunctionPointerIn(tag *types.CTag, functionTags []*types.
 	return false
 }
 
-func (p *CTagsParser) collectFunctions() []*types.CTag {
-	functionTags := []*types.CTag{}
+func (p *CTagsParser) collectFunctions() []*CTag {
+	functionTags := []*CTag{}
 	for _, tag := range p.tags {
 		if tag.Kind == KIND_FUNCTION && !tag.SkipMe {
 			functionTags = append(functionTags, tag)
@@ -84,14 +95,14 @@ func (p *CTagsParser) firstFunctionAtLine() int {
 	return -1
 }
 
-func (p *CTagsParser) toPrototypes() []*types.Prototype {
-	prototypes := []*types.Prototype{}
+func (p *CTagsParser) toPrototypes() []*Prototype {
+	prototypes := []*Prototype{}
 	for _, tag := range p.tags {
 		if strings.TrimSpace(tag.Prototype) == "" {
 			continue
 		}
 		if !tag.SkipMe {
-			prototype := &types.Prototype{
+			prototype := &Prototype{
 				FunctionName: tag.FunctionName,
 				File:         tag.Filename,
 				Prototype:    tag.Prototype,

legacy/builder/ctags/ctags_to_prototypes_test.go

@@ -20,12 +20,11 @@ import (
 	"path/filepath"
 	"testing"
 
-	"github.com/arduino/arduino-cli/legacy/builder/types"
 	paths "github.com/arduino/go-paths-helper"
 	"github.com/stretchr/testify/require"
 )
 
-func producePrototypes(t *testing.T, filename string, mainFile string) ([]*types.Prototype, int) {
+func producePrototypes(t *testing.T, filename string, mainFile string) ([]*Prototype, int) {
 	bytes, err := os.ReadFile(filepath.Join("test_data", filename))
 	require.NoError(t, err)
 

legacy/builder/gcc_preproc_runner.go

@@ -19,6 +19,7 @@ import (
 	"os/exec"
 	"strings"
 
+	f "github.com/arduino/arduino-cli/internal/algorithms"
 	"github.com/arduino/arduino-cli/legacy/builder/builder_utils"
 	"github.com/arduino/arduino-cli/legacy/builder/types"
 	"github.com/arduino/arduino-cli/legacy/builder/utils"
@@ -27,32 +28,13 @@ import (
 	"github.com/pkg/errors"
 )
 
-func GCCPreprocRunner(ctx *types.Context, sourceFilePath *paths.Path, targetFilePath *paths.Path, includes paths.PathList) error {
+func GCCPreprocRunner(ctx *types.Context, sourceFilePath *paths.Path, targetFilePath *paths.Path, includes paths.PathList) ([]byte, error) {
 	cmd, err := prepareGCCPreprocRecipeProperties(ctx, sourceFilePath, targetFilePath, includes)
 	if err != nil {
-		return errors.WithStack(err)
+		return nil, err
 	}
-
-	_, _, err = utils.ExecCommand(ctx, cmd /* stdout */, utils.ShowIfVerbose /* stderr */, utils.Show)
-	if err != nil {
-		return errors.WithStack(err)
-	}
-
-	return nil
-}
-
-func GCCPreprocRunnerForDiscoveringIncludes(ctx *types.Context, sourceFilePath *paths.Path, targetFilePath *paths.Path, includes paths.PathList) ([]byte, error) {
-	cmd, err := prepareGCCPreprocRecipeProperties(ctx, sourceFilePath, targetFilePath, includes)
-	if err != nil {
-		return nil, errors.WithStack(err)
-	}
-
-	_, stderr, err := utils.ExecCommand(ctx, cmd /* stdout */, utils.ShowIfVerbose /* stderr */, utils.Capture)
-	if err != nil {
-		return stderr, errors.WithStack(err)
-	}
-
-	return stderr, nil
+	_, stderr, err := utils.ExecCommand(ctx, cmd, utils.ShowIfVerbose /* stdout */, utils.Capture /* stderr */)
+	return stderr, err
 }
 
 func prepareGCCPreprocRecipeProperties(ctx *types.Context, sourceFilePath *paths.Path, targetFilePath *paths.Path, includes paths.PathList) (*exec.Cmd, error) {
@@ -63,7 +45,7 @@ func prepareGCCPreprocRecipeProperties(ctx *types.Context, sourceFilePath *paths
 	buildProperties.SetPath("source_file", sourceFilePath)
 	buildProperties.SetPath("preprocessed_file_path", targetFilePath)
 
-	includesStrings := utils.Map(includes.AsStrings(), utils.WrapWithHyphenI)
+	includesStrings := f.Map(includes.AsStrings(), utils.WrapWithHyphenI)
 	buildProperties.Set("includes", strings.Join(includesStrings, " "))
 
 	if buildProperties.Get("recipe.preproc.macros") == "" {
@@ -84,7 +66,7 @@ func prepareGCCPreprocRecipeProperties(ctx *types.Context, sourceFilePath *paths
 
 	// Remove -MMD argument if present. Leaving it will make gcc try
 	// to create a /dev/null.d dependency file, which won't work.
-	cmd.Args = utils.Filter(cmd.Args, func(a string) bool { return a != "-MMD" })
+	cmd.Args = f.Filter(cmd.Args, f.NotEquals("-MMD"))
 
 	return cmd, nil
 }

legacy/builder/phases/core_builder.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/arduino/arduino-cli/buildcache"
 	"github.com/arduino/arduino-cli/i18n"
+	f "github.com/arduino/arduino-cli/internal/algorithms"
 	"github.com/arduino/arduino-cli/legacy/builder/builder_utils"
 	"github.com/arduino/arduino-cli/legacy/builder/constants"
 	"github.com/arduino/arduino-cli/legacy/builder/types"
@@ -77,7 +78,7 @@ func compileCore(ctx *types.Context, buildPath *paths.Path, buildCachePath *path
 	if variantFolder != nil && variantFolder.IsDir() {
 		includes = append(includes, variantFolder.String())
 	}
-	includes = utils.Map(includes, utils.WrapWithHyphenI)
+	includes = f.Map(includes, utils.WrapWithHyphenI)
 
 	var err error
 

legacy/builder/phases/libraries_builder.go

@@ -19,6 +19,7 @@ import (
 	"strings"
 
 	"github.com/arduino/arduino-cli/arduino/libraries"
+	f "github.com/arduino/arduino-cli/internal/algorithms"
 	"github.com/arduino/arduino-cli/legacy/builder/builder_utils"
 	"github.com/arduino/arduino-cli/legacy/builder/constants"
 	"github.com/arduino/arduino-cli/legacy/builder/types"
@@ -36,7 +37,7 @@ type LibrariesBuilder struct{}
 func (s *LibrariesBuilder) Run(ctx *types.Context) error {
 	librariesBuildPath := ctx.LibrariesBuildPath
 	buildProperties := ctx.BuildProperties
-	includes := utils.Map(ctx.IncludeFolders.AsStrings(), utils.WrapWithHyphenI)
+	includes := f.Map(ctx.IncludeFolders.AsStrings(), utils.WrapWithHyphenI)
 	libs := ctx.ImportedLibraries
 
 	if err := librariesBuildPath.MkdirAll(); err != nil {

legacy/builder/phases/linker.go

@@ -18,6 +18,7 @@ package phases
 import (
 	"strings"
 
+	f "github.com/arduino/arduino-cli/internal/algorithms"
 	"github.com/arduino/arduino-cli/legacy/builder/builder_utils"
 	"github.com/arduino/arduino-cli/legacy/builder/constants"
 	"github.com/arduino/arduino-cli/legacy/builder/types"
@@ -64,7 +65,7 @@ func (s *Linker) Run(ctx *types.Context) error {
 }
 
 func link(ctx *types.Context, objectFiles paths.PathList, coreDotARelPath *paths.Path, coreArchiveFilePath *paths.Path, buildProperties *properties.Map) error {
-	objectFileList := strings.Join(utils.Map(objectFiles.AsStrings(), wrapWithDoubleQuotes), " ")
+	objectFileList := strings.Join(f.Map(objectFiles.AsStrings(), wrapWithDoubleQuotes), " ")
 
 	// If command line length is too big (> 30000 chars), try to collect the object files into archives
 	// and use that archives to complete the build.
@@ -102,7 +103,7 @@ func link(ctx *types.Context, objectFiles paths.PathList, coreDotARelPath *paths
 			}
 		}
 
-		objectFileList = strings.Join(utils.Map(archives.AsStrings(), wrapWithDoubleQuotes), " ")
+		objectFileList = strings.Join(f.Map(archives.AsStrings(), wrapWithDoubleQuotes), " ")
 		objectFileList = "-Wl,--whole-archive " + objectFileList + " -Wl,--no-whole-archive"
 	}
 

legacy/builder/phases/sketch_builder.go

@@ -16,6 +16,7 @@
 package phases
 
 import (
+	f "github.com/arduino/arduino-cli/internal/algorithms"
 	"github.com/arduino/arduino-cli/legacy/builder/builder_utils"
 	"github.com/arduino/arduino-cli/legacy/builder/types"
 	"github.com/arduino/arduino-cli/legacy/builder/utils"
@@ -27,7 +28,7 @@ type SketchBuilder struct{}
 func (s *SketchBuilder) Run(ctx *types.Context) error {
 	sketchBuildPath := ctx.SketchBuildPath
 	buildProperties := ctx.BuildProperties
-	includes := utils.Map(ctx.IncludeFolders.AsStrings(), utils.WrapWithHyphenI)
+	includes := f.Map(ctx.IncludeFolders.AsStrings(), utils.WrapWithHyphenI)
 
 	if err := sketchBuildPath.MkdirAll(); err != nil {
 		return errors.WithStack(err)

legacy/builder/preprocess_sketch.go

@@ -36,8 +36,11 @@ func PreprocessSketchWithArduinoPreprocessor(ctx *types.Context) error {
 
 	sourceFile := ctx.SketchBuildPath.Join(ctx.Sketch.MainFile.Base() + ".cpp")
 	targetFile := ctx.PreprocPath.Join("sketch_merged.cpp")
-	GCCPreprocRunner(ctx, sourceFile, targetFile, ctx.IncludeFolders)
-
+	stderr, err := GCCPreprocRunner(ctx, sourceFile, targetFile, ctx.IncludeFolders)
+	ctx.WriteStderr(stderr)
+	if err != nil {
+		return err
+	}
 	buildProperties := properties.NewMap()
 	buildProperties.Set("tools.arduino-preprocessor.path", "{runtime.tools.arduino-preprocessor.path}")
 	buildProperties.Set("tools.arduino-preprocessor.cmd.path", "{path}/arduino-preprocessor")

legacy/builder/prototypes_adder.go

@@ -21,31 +21,22 @@ import (
 	"strings"
 
 	"github.com/arduino/arduino-cli/legacy/builder/constants"
-	"github.com/arduino/arduino-cli/legacy/builder/types"
+	"github.com/arduino/arduino-cli/legacy/builder/ctags"
 	"github.com/arduino/arduino-cli/legacy/builder/utils"
 )
 
-type PrototypesAdder struct{}
-
-func (s *PrototypesAdder) Run(ctx *types.Context) error {
-	debugOutput := ctx.DebugPreprocessor
-
-	source := ctx.SketchSourceMerged
+func PrototypesAdder(source string, firstFunctionLine, lineOffset int, prototypes []*ctags.Prototype, debugOutput bool) (preprocessedSource, prototypeSection string) {
 	source = strings.Replace(source, "\r\n", "\n", -1)
 	source = strings.Replace(source, "\r", "\n", -1)
-
 	sourceRows := strings.Split(source, "\n")
-
-	firstFunctionLine := ctx.PrototypesLineWhereToInsert
 	if isFirstFunctionOutsideOfSource(firstFunctionLine, sourceRows) {
-		return nil
+		return
 	}
 
-	insertionLine := firstFunctionLine + ctx.LineOffset - 1
+	insertionLine := firstFunctionLine + lineOffset - 1
 	firstFunctionChar := len(strings.Join(sourceRows[:insertionLine], "\n")) + 1
-	prototypeSection := composePrototypeSection(firstFunctionLine, ctx.Prototypes)
-	ctx.PrototypesSection = prototypeSection
-	source = source[:firstFunctionChar] + prototypeSection + source[firstFunctionChar:]
+	prototypeSection = composePrototypeSection(firstFunctionLine, prototypes)
+	preprocessedSource = source[:firstFunctionChar] + prototypeSection + source[firstFunctionChar:]
 
 	if debugOutput {
 		fmt.Println("#PREPROCESSED SOURCE")
@@ -62,12 +53,10 @@ func (s *PrototypesAdder) Run(ctx *types.Context) error {
 		}
 		fmt.Println("#END OF PREPROCESSED SOURCE")
 	}
-	ctx.SketchSourceAfterArduinoPreprocessing = source
-
-	return nil
+	return
 }
 
-func composePrototypeSection(line int, prototypes []*types.Prototype) string {
+func composePrototypeSection(line int, prototypes []*ctags.Prototype) string {
 	if len(prototypes) == 0 {
 		return constants.EMPTY_STRING
 	}
@@ -81,7 +70,7 @@ func composePrototypeSection(line int, prototypes []*types.Prototype) string {
 	return str
 }
 
-func joinPrototypes(prototypes []*types.Prototype) string {
+func joinPrototypes(prototypes []*ctags.Prototype) string {
 	prototypesSlice := []string{}
 	for _, proto := range prototypes {
 		if signatureContainsaDefaultArg(proto) {
@@ -98,7 +87,7 @@ func joinPrototypes(prototypes []*types.Prototype) string {
 	return strings.Join(prototypesSlice, "\n")
 }
 
-func signatureContainsaDefaultArg(proto *types.Prototype) bool {
+func signatureContainsaDefaultArg(proto *ctags.Prototype) bool {
 	return strings.Contains(proto.Prototype, "=")
 }
 

legacy/builder/test/add_additional_entries_to_context_test.go

@@ -38,8 +38,6 @@ func TestAddAdditionalEntriesToContextNoBuildPath(t *testing.T) {
 
 	require.NotNil(t, ctx.WarningsLevel)
 
-	require.True(t, ctx.CollectedSourceFiles.Empty())
-
 	require.Equal(t, 0, len(ctx.LibrariesResolutionResults))
 }
 
@@ -57,7 +55,5 @@ func TestAddAdditionalEntriesToContextWithBuildPath(t *testing.T) {
 
 	require.NotNil(t, ctx.WarningsLevel)
 
-	require.True(t, ctx.CollectedSourceFiles.Empty())
-
 	require.Equal(t, 0, len(ctx.LibrariesResolutionResults))
 }

legacy/builder/test/utils_test.go

@@ -16,7 +16,6 @@
 package test
 
 import (
-	"strings"
 	"testing"
 
 	"github.com/arduino/arduino-cli/legacy/builder/utils"
@@ -42,17 +41,6 @@ func TestPrintableCommand(t *testing.T) {
 	require.Equal(t, correct, result)
 }
 
-func TestMapTrimSpace(t *testing.T) {
-	value := "hello, world , how are,you? "
-	parts := utils.Map(strings.Split(value, ","), utils.TrimSpace)
-
-	require.Equal(t, 4, len(parts))
-	require.Equal(t, "hello", parts[0])
-	require.Equal(t, "world", parts[1])
-	require.Equal(t, "how are", parts[2])
-	require.Equal(t, "you?", parts[3])
-}
-
 func TestQuoteCppString(t *testing.T) {
 	cases := map[string]string{
 		`foo`:                                  `"foo"`,

legacy/builder/types/context.go

@@ -29,6 +29,7 @@ import (
 	"github.com/arduino/arduino-cli/arduino/libraries/librariesmanager"
 	"github.com/arduino/arduino-cli/arduino/libraries/librariesresolver"
 	"github.com/arduino/arduino-cli/arduino/sketch"
+	"github.com/arduino/arduino-cli/legacy/builder/ctags"
 	rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1"
 	paths "github.com/arduino/go-paths-helper"
 	properties "github.com/arduino/go-properties-orderedmap"
@@ -98,8 +99,6 @@ type Context struct {
 	SketchObjectFiles            paths.PathList
 	IgnoreSketchFolderNameErrors bool
 
-	CollectedSourceFiles *UniqueSourceFileQueue
-
 	Sketch        *sketch.Sketch
 	WarningsLevel string
 
@@ -123,7 +122,7 @@ type Context struct {
 	LineOffset                  int
 	PrototypesSection           string
 	PrototypesLineWhereToInsert int
-	Prototypes                  []*Prototype
+	Prototypes                  []*ctags.Prototype
 
 	// Verbosity settings
 	Verbose           bool

legacy/builder/types/types.go

@@ -17,7 +17,6 @@ package types
 
 import (
 	"fmt"
-	"strconv"
 
 	"github.com/arduino/arduino-cli/arduino/libraries"
 	"github.com/arduino/arduino-cli/arduino/sketch"
@@ -85,40 +84,11 @@ func (f *SourceFile) DepfilePath(ctx *Context) *paths.Path {
 	return buildRoot(ctx, f.Origin).Join(f.RelativePath.String() + ".d")
 }
 
-type Prototype struct {
-	FunctionName string
-	File         string
-	Prototype    string
-	Modifiers    string
-	Line         int
-}
-
-func (proto *Prototype) String() string {
-	return proto.Modifiers + " " + proto.Prototype + " @ " + strconv.Itoa(proto.Line)
-}
-
 type LibraryResolutionResult struct {
 	Library          *libraries.Library
 	NotUsedLibraries []*libraries.Library
 }
 
-type CTag struct {
-	FunctionName string
-	Kind         string
-	Line         int
-	Code         string
-	Class        string
-	Struct       string
-	Namespace    string
-	Filename     string
-	Typeref      string
-	SkipMe       bool
-	Signature    string
-
-	Prototype          string
-	PrototypeModifiers string
-}
-
 type Command interface {
 	Run(ctx *Context) error
 }

legacy/builder/utils/utils.go

@@ -28,6 +28,7 @@ import (
 	"unicode"
 
 	"github.com/arduino/arduino-cli/i18n"
+	f "github.com/arduino/arduino-cli/internal/algorithms"
 	"github.com/arduino/arduino-cli/legacy/builder/gohasissues"
 	"github.com/arduino/arduino-cli/legacy/builder/types"
 	paths "github.com/arduino/go-paths-helper"
@@ -119,36 +120,10 @@ func IsSCCSFile(file os.FileInfo) bool {
 	return SOURCE_CONTROL_FOLDERS[name]
 }
 
-type mapFunc func(string) string
-
-func Map(slice []string, fn mapFunc) []string {
-	newSlice := []string{}
-	for _, elem := range slice {
-		newSlice = append(newSlice, fn(elem))
-	}
-	return newSlice
-}
-
-type filterFunc func(string) bool
-
-func Filter(slice []string, fn filterFunc) []string {
-	newSlice := []string{}
-	for _, elem := range slice {
-		if fn(elem) {
-			newSlice = append(newSlice, elem)
-		}
-	}
-	return newSlice
-}
-
 func WrapWithHyphenI(value string) string {
 	return "\"-I" + value + "\""
 }
 
-func TrimSpace(value string) string {
-	return strings.TrimSpace(value)
-}
-
 func printableArgument(arg string) string {
 	if strings.ContainsAny(arg, "\"\\ \t") {
 		arg = strings.Replace(arg, "\\", "\\\\", -1)
@@ -164,7 +139,7 @@ func printableArgument(arg string) string {
 // probably not for shell interpretation. This essentially reverses
 // ParseCommandLine.
 func PrintableCommand(parts []string) string {
-	return strings.Join(Map(parts, printableArgument), " ")
+	return strings.Join(f.Map(parts, printableArgument), " ")
 }
 
 const (