[skip-changelog] legacy: Arduino preprocess subroutine refactorization (part 1) (#2186)

* legacy: Removed ReadFileAndStoreInContext command

* ctags parser.Parse signature change

* removing ctags-related data from builder ctx (part 1)

* removing ctags-related data from builder ctx (part 2)

* removing ctags-related data from builder ctx (part 3)

* Clearly separate Source code processing phases

* Removed Command wrapper from ContainerMergeCopySketchFiles

* Moved builder.CopySketchFilesToBuildPath into proper location

* Converted FilterSketchSource into a function

* Moved a couple of functions in the proper builder package

* Fixed lint error

arduino/builder/cpp.go

+// This file is part of arduino-cli.
+//
+// Copyright 2023 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to license@arduino.cc.
+
+package builder
+
+import (
+	"strings"
+	"unicode/utf8"
+)
+
+// QuoteCppString returns the given string as a quoted string for use with the C
+// preprocessor. This adds double quotes around it and escapes any
+// double quotes and backslashes in the string.
+func QuoteCppString(str string) string {
+	str = strings.Replace(str, "\\", "\\\\", -1)
+	str = strings.Replace(str, "\"", "\\\"", -1)
+	return "\"" + str + "\""
+}
+
+// ParseCppString parse a C-preprocessor string as emitted by the preprocessor. This
+// is a string contained in double quotes, with any backslashes or
+// quotes escaped with a backslash. If a valid string was present at the
+// start of the given line, returns the unquoted string contents, the
+// remainder of the line (everything after the closing "), and true.
+// Otherwise, returns the empty string, the entire line and false.
+func ParseCppString(line string) (string, string, bool) {
+	// For details about how these strings are output by gcc, see:
+	// https://github.com/gcc-mirror/gcc/blob/a588355ab948cf551bc9d2b89f18e5ae5140f52c/libcpp/macro.c#L491-L511
+	// Note that the documentation suggests all non-printable
+	// characters are also escaped, but the implementation does not
+	// actually do this. See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=51259
+	if len(line) < 1 || line[0] != '"' {
+		return "", line, false
+	}
+
+	i := 1
+	res := ""
+	for {
+		if i >= len(line) {
+			return "", line, false
+		}
+
+		c, width := utf8.DecodeRuneInString(line[i:])
+
+		switch c {
+		case '\\':
+			// Backslash, next character is used unmodified
+			i += width
+			if i >= len(line) {
+				return "", line, false
+			}
+			res += string(line[i])
+		case '"':
+			// Quote, end of string
+			return res, line[i+width:], true
+		default:
+			res += string(c)
+		}
+
+		i += width
+	}
+}

arduino/builder/cpp_test.go

+package builder_test
+
+import (
+	"testing"
+
+	"github.com/arduino/arduino-cli/arduino/builder"
+	"github.com/stretchr/testify/require"
+)
+
+func TestParseCppString(t *testing.T) {
+	_, _, ok := builder.ParseCppString(`foo`)
+	require.Equal(t, false, ok)
+
+	_, _, ok = builder.ParseCppString(`"foo`)
+	require.Equal(t, false, ok)
+
+	str, rest, ok := builder.ParseCppString(`"foo"`)
+	require.Equal(t, true, ok)
+	require.Equal(t, `foo`, str)
+	require.Equal(t, ``, rest)
+
+	str, rest, ok = builder.ParseCppString(`"foo\\bar"`)
+	require.Equal(t, true, ok)
+	require.Equal(t, `foo\bar`, str)
+	require.Equal(t, ``, rest)
+
+	str, rest, ok = builder.ParseCppString(`"foo \"is\" quoted and \\\\bar\"\" escaped\\" and "then" some`)
+	require.Equal(t, true, ok)
+	require.Equal(t, `foo "is" quoted and \\bar"" escaped\`, str)
+	require.Equal(t, ` and "then" some`, rest)
+
+	str, rest, ok = builder.ParseCppString(`" !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_abcdefghijklmnopqrstuvwxyz{|}~"`)
+	require.Equal(t, true, ok)
+	require.Equal(t, ` !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_abcdefghijklmnopqrstuvwxyz{|}~`, str)
+	require.Equal(t, ``, rest)
+
+	str, rest, ok = builder.ParseCppString(`"/home/ççç/"`)
+	require.Equal(t, true, ok)
+	require.Equal(t, `/home/ççç/`, str)
+	require.Equal(t, ``, rest)
+
+	str, rest, ok = builder.ParseCppString(`"/home/ççç/ /$sdsdd\\"`)
+	require.Equal(t, true, ok)
+	require.Equal(t, `/home/ççç/ /$sdsdd\`, str)
+	require.Equal(t, ``, rest)
+}

arduino/builder/sketch.go

@@ -19,7 +19,6 @@ import (
 	"bytes"
 	"fmt"
 	"regexp"
-	"strings"
 
 	"github.com/arduino/arduino-cli/arduino/sketch"
 	"github.com/arduino/arduino-cli/i18n"
@@ -34,13 +33,20 @@ var (
 	tr               = i18n.Tr
 )
 
-// QuoteCppString returns the given string as a quoted string for use with the C
-// preprocessor. This adds double quotes around it and escapes any
-// double quotes and backslashes in the string.
-func QuoteCppString(str string) string {
-	str = strings.Replace(str, "\\", "\\\\", -1)
-	str = strings.Replace(str, "\"", "\\\"", -1)
-	return "\"" + str + "\""
+// PrepareSketchBuildPath copies the sketch source files in the build path.
+// The .ino files are merged together to create a .cpp file (by the way, the
+// .cpp file still needs to be Arduino-preprocessed to compile).
+func PrepareSketchBuildPath(sketch *sketch.Sketch, sourceOverrides map[string]string, buildPath *paths.Path) (offset int, mergedSource string, err error) {
+	if offset, mergedSource, err = sketchMergeSources(sketch, sourceOverrides); err != nil {
+		return
+	}
+	if err = SketchSaveItemCpp(sketch.MainFile, []byte(mergedSource), buildPath); err != nil {
+		return
+	}
+	if err = sketchCopyAdditionalFiles(sketch, buildPath, sourceOverrides); err != nil {
+		return
+	}
+	return
 }
 
 // SketchSaveItemCpp saves a preprocessed .cpp sketch file on disk
@@ -59,8 +65,9 @@ func SketchSaveItemCpp(path *paths.Path, contents []byte, destPath *paths.Path)
 	return nil
 }
 
-// SketchMergeSources merges all the source files included in a sketch
-func SketchMergeSources(sk *sketch.Sketch, overrides map[string]string) (int, string, error) {
+// sketchMergeSources merges all the .ino source files included in a sketch to produce
+// a single .cpp file.
+func sketchMergeSources(sk *sketch.Sketch, overrides map[string]string) (int, string, error) {
 	lineOffset := 0
 	mergedSource := ""
 
@@ -105,9 +112,9 @@ func SketchMergeSources(sk *sketch.Sketch, overrides map[string]string) (int, st
 	return lineOffset, mergedSource, nil
 }
 
-// SketchCopyAdditionalFiles copies the additional files for a sketch to the
+// sketchCopyAdditionalFiles copies the additional files for a sketch to the
 // specified destination directory.
-func SketchCopyAdditionalFiles(sketch *sketch.Sketch, destPath *paths.Path, overrides map[string]string) error {
+func sketchCopyAdditionalFiles(sketch *sketch.Sketch, destPath *paths.Path, overrides map[string]string) error {
 	if err := destPath.MkdirAll(); err != nil {
 		return errors.Wrap(err, tr("unable to create a folder to save the sketch files"))
 	}

arduino/builder/sketch_test.go

@@ -13,7 +13,7 @@
 // Arduino software without disclosing the source code of your own applications.
 // To purchase a commercial license, send an email to license@arduino.cc.
 
-package builder_test
+package builder
 
 import (
 	"fmt"
@@ -23,7 +23,6 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/arduino/arduino-cli/arduino/builder"
 	"github.com/arduino/arduino-cli/arduino/sketch"
 	"github.com/arduino/go-paths-helper"
 	"github.com/stretchr/testify/require"
@@ -48,7 +47,7 @@ func TestSaveSketch(t *testing.T) {
 		t.Fatalf("unable to read golden file %s: %v", sketchFile, err)
 	}
 
-	builder.SketchSaveItemCpp(paths.New(sketchName), source, tmp)
+	SketchSaveItemCpp(paths.New(sketchName), source, tmp)
 
 	out, err := tmp.Join(outName).ReadFile()
 	if err != nil {
@@ -82,7 +81,7 @@ func TestMergeSketchSources(t *testing.T) {
 	}
 	mergedSources := strings.ReplaceAll(string(mergedBytes), "%s", pathToGoldenSource)
 
-	offset, source, err := builder.SketchMergeSources(s, nil)
+	offset, source, err := sketchMergeSources(s, nil)
 	require.Nil(t, err)
 	require.Equal(t, 2, offset)
 	require.Equal(t, mergedSources, source)
@@ -94,7 +93,7 @@ func TestMergeSketchSourcesArduinoIncluded(t *testing.T) {
 	require.NotNil(t, s)
 
 	// ensure not to include Arduino.h when it's already there
-	_, source, err := builder.SketchMergeSources(s, nil)
+	_, source, err := sketchMergeSources(s, nil)
 	require.Nil(t, err)
 	require.Equal(t, 1, strings.Count(source, "<Arduino.h>"))
 }
@@ -110,7 +109,7 @@ func TestCopyAdditionalFiles(t *testing.T) {
 
 	// copy the sketch over, create a fake main file we don't care about it
 	// but we need it for `SketchLoad` to succeed later
-	err = builder.SketchCopyAdditionalFiles(s1, tmp, nil)
+	err = sketchCopyAdditionalFiles(s1, tmp, nil)
 	require.Nil(t, err)
 	fakeIno := tmp.Join(fmt.Sprintf("%s.ino", tmp.Base()))
 	require.Nil(t, fakeIno.WriteFile([]byte{}))
@@ -125,7 +124,7 @@ func TestCopyAdditionalFiles(t *testing.T) {
 	require.Nil(t, err)
 
 	// copy again
-	err = builder.SketchCopyAdditionalFiles(s1, tmp, nil)
+	err = sketchCopyAdditionalFiles(s1, tmp, nil)
 	require.Nil(t, err)
 
 	// verify file hasn't changed

legacy/builder/builder.go

@@ -19,6 +19,7 @@ import (
 	"reflect"
 	"time"
 
+	"github.com/arduino/arduino-cli/arduino/builder"
 	"github.com/arduino/arduino-cli/i18n"
 	"github.com/arduino/arduino-cli/legacy/builder/phases"
 	"github.com/arduino/arduino-cli/legacy/builder/types"
@@ -39,6 +40,7 @@ func (s *Builder) Run(ctx *types.Context) error {
 		return err
 	}
 
+	var _err error
 	commands := []types.Command{
 		&ContainerSetupHardwareToolsLibsSketchAndProps{},
 
@@ -46,7 +48,10 @@ func (s *Builder) Run(ctx *types.Context) error {
 
 		&RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.prebuild", Suffix: ".pattern"},
 
-		&ContainerMergeCopySketchFiles{},
+		types.BareCommand(func(ctx *types.Context) error {
+			ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath)
+			return _err
+		}),
 
 		utils.LogIfVerbose(false, tr("Detecting libraries used...")),
 		&ContainerFindIncludes{},
@@ -127,6 +132,7 @@ func (s *Preprocess) Run(ctx *types.Context) error {
 		return err
 	}
 
+	var _err error
 	commands := []types.Command{
 		&ContainerSetupHardwareToolsLibsSketchAndProps{},
 
@@ -134,7 +140,10 @@ func (s *Preprocess) Run(ctx *types.Context) error {
 
 		&RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.prebuild", Suffix: ".pattern"},
 
-		&ContainerMergeCopySketchFiles{},
+		types.BareCommand(func(ctx *types.Context) error {
+			ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath)
+			return _err
+		}),
 
 		&ContainerFindIncludes{},
 
@@ -148,7 +157,7 @@ func (s *Preprocess) Run(ctx *types.Context) error {
 	}
 
 	// Output arduino-preprocessed source
-	ctx.WriteStdout([]byte(ctx.Source))
+	ctx.WriteStdout([]byte(ctx.SketchSourceAfterArduinoPreprocessing))
 	return nil
 }
 

legacy/builder/container_add_prototypes.go

@@ -16,10 +16,17 @@
 package builder
 
 import (
+	"bufio"
+	"bytes"
 	"fmt"
+	"io"
+	"strconv"
+	"strings"
 
 	bldr "github.com/arduino/arduino-cli/arduino/builder"
+	"github.com/arduino/arduino-cli/arduino/sketch"
 	"github.com/arduino/arduino-cli/legacy/builder/types"
+	"github.com/arduino/go-paths-helper"
 	"github.com/pkg/errors"
 )
 
@@ -28,7 +35,7 @@ func PreprocessSketchWithCtags(ctx *types.Context) error {
 	if err := ctx.PreprocPath.MkdirAll(); err != nil {
 		return errors.WithStack(err)
 	}
-	targetFilePath := ctx.PreprocPath.Join("ctags_target_for_gcc_minus_e.cpp")
+	targetFilePath := ctx.PreprocPath.Join("sketch_merged.cpp")
 
 	// Run preprocessor
 	sourceFile := ctx.SketchBuildPath.Join(ctx.Sketch.MainFile.Base() + ".cpp")
@@ -47,11 +54,14 @@ func PreprocessSketchWithCtags(ctx *types.Context) error {
 		}
 	}
 
+	if src, err := targetFilePath.ReadFile(); err != nil {
+		return err
+	} else {
+		ctx.SketchSourceAfterCppPreprocessing = filterSketchSource(ctx.Sketch, bytes.NewReader(src), false)
+	}
+
 	commands := []types.Command{
-		&ReadFileAndStoreInContext{FileToRead: targetFilePath, Target: &ctx.SourceGccMinusE},
-		&FilterSketchSource{Source: &ctx.SourceGccMinusE},
-		&CTagsTargetFileSaver{Source: &ctx.SourceGccMinusE, TargetFileName: "ctags_target_for_gcc_minus_e.cpp"},
-		&CTagsRunner{},
+		&CTagsRunner{Source: &ctx.SketchSourceAfterCppPreprocessing, TargetFileName: "sketch_merged.cpp"},
 		&PrototypesAdder{},
 	}
 
@@ -63,9 +73,66 @@ func PreprocessSketchWithCtags(ctx *types.Context) error {
 		}
 	}
 
-	if err := bldr.SketchSaveItemCpp(ctx.Sketch.MainFile, []byte(ctx.Source), ctx.SketchBuildPath); err != nil {
+	if err := bldr.SketchSaveItemCpp(ctx.Sketch.MainFile, []byte(ctx.SketchSourceAfterArduinoPreprocessing), ctx.SketchBuildPath); err != nil {
 		return errors.WithStack(err)
 	}
 
 	return nil
 }
+
+func filterSketchSource(sketch *sketch.Sketch, source io.Reader, removeLineMarkers bool) string {
+	fileNames := paths.NewPathList()
+	fileNames.Add(sketch.MainFile)
+	fileNames.AddAll(sketch.OtherSketchFiles)
+
+	inSketch := false
+	filtered := ""
+
+	scanner := bufio.NewScanner(source)
+	for scanner.Scan() {
+		line := scanner.Text()
+		if filename := parseLineMarker(line); filename != nil {
+			inSketch = fileNames.Contains(filename)
+			if inSketch && removeLineMarkers {
+				continue
+			}
+		}
+
+		if inSketch {
+			filtered += line + "\n"
+		}
+	}
+
+	return filtered
+}
+
+// Parses the given line as a gcc line marker and returns the contained
+// filename.
+func parseLineMarker(line string) *paths.Path {
+	// A line marker contains the line number and filename and looks like:
+	// # 123 /path/to/file.cpp
+	// It can be followed by zero or more flag number that indicate the
+	// preprocessor state and can be ignored.
+	// For exact details on this format, see:
+	// https://github.com/gcc-mirror/gcc/blob/edd716b6b1caa1a5cb320a8cd7f626f30198e098/gcc/c-family/c-ppoutput.c#L413-L415
+
+	split := strings.SplitN(line, " ", 3)
+	if len(split) < 3 || len(split[0]) == 0 || split[0][0] != '#' {
+		return nil
+	}
+
+	_, err := strconv.Atoi(split[1])
+	if err != nil {
+		return nil
+	}
+
+	// If we get here, we found a # followed by a line number, so
+	// assume this is a line marker and see if the rest of the line
+	// starts with a string containing the filename
+	str, rest, ok := bldr.ParseCppString(split[2])
+
+	if ok && (rest == "" || rest[0] == ' ') {
+		return paths.New(str)
+	}
+	return nil
+}

legacy/builder/container_merge_copy_sketch_files.go

-// This file is part of arduino-cli.
-//
-// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
-//
-// This software is released under the GNU General Public License version 3,
-// which covers the main part of arduino-cli.
-// The terms of this license can be found at:
-// https://www.gnu.org/licenses/gpl-3.0.en.html
-//
-// You can be released from the requirements of the above licenses by purchasing
-// a commercial license. Buying such a license is mandatory if you want to
-// modify or otherwise use the software for commercial activities involving the
-// Arduino software without disclosing the source code of your own applications.
-// To purchase a commercial license, send an email to license@arduino.cc.
-
-package builder
-
-import (
-	bldr "github.com/arduino/arduino-cli/arduino/builder"
-	"github.com/arduino/arduino-cli/legacy/builder/types"
-	"github.com/pkg/errors"
-)
-
-type ContainerMergeCopySketchFiles struct{}
-
-func (s *ContainerMergeCopySketchFiles) Run(ctx *types.Context) error {
-	offset, source, err := bldr.SketchMergeSources(ctx.Sketch, ctx.SourceOverride)
-	if err != nil {
-		return err
-	}
-	ctx.LineOffset = offset
-	ctx.Source = source
-
-	if err := bldr.SketchSaveItemCpp(ctx.Sketch.MainFile, []byte(ctx.Source), ctx.SketchBuildPath); err != nil {
-		return errors.WithStack(err)
-	}
-
-	if err := bldr.SketchCopyAdditionalFiles(ctx.Sketch, ctx.SketchBuildPath, ctx.SourceOverride); err != nil {
-		return errors.WithStack(err)
-	}
-
-	return nil
-}

legacy/builder/create_cmake_rule.go

@@ -120,12 +120,7 @@ func (s *ExportProjectCMake) Run(ctx *types.Context) error {
 	}
 
 	// Use old ctags method to generate export file
-	commands := []types.Command{
-		&FilterSketchSource{Source: &ctx.Source, RemoveLineMarkers: true},
-	}
-	for _, command := range commands {
-		command.Run(ctx)
-	}
+	ctx.SketchSourceMerged = filterSketchSource(ctx.Sketch, strings.NewReader(ctx.SketchSourceMerged), true)
 
 	err = utils.CopyDir(ctx.SketchBuildPath.String(), cmakeFolder.Join("sketch").String(), validExportExtensions)
 	if err != nil {

legacy/builder/ctags/ctags_has_issues.go

@@ -23,14 +23,12 @@ import (
 	"github.com/arduino/arduino-cli/legacy/builder/types"
 )
 
-func (p *CTagsParser) FixCLinkageTagsDeclarations(tags []*types.CTag) {
-
-	linesMap := p.FindCLinkageLines(tags)
-	for i := range tags {
-
-		if sliceContainsInt(linesMap[tags[i].Filename], tags[i].Line) &&
-			!strings.Contains(tags[i].PrototypeModifiers, EXTERN) {
-			tags[i].PrototypeModifiers = tags[i].PrototypeModifiers + " " + EXTERN
+func (p *CTagsParser) FixCLinkageTagsDeclarations() {
+	linesMap := p.FindCLinkageLines(p.tags)
+	for i := range p.tags {
+		if sliceContainsInt(linesMap[p.tags[i].Filename], p.tags[i].Line) &&
+			!strings.Contains(p.tags[i].PrototypeModifiers, EXTERN) {
+			p.tags[i].PrototypeModifiers = p.tags[i].PrototypeModifiers + " " + EXTERN
 		}
 	}
 }

legacy/builder/ctags/ctags_parser.go

@@ -43,8 +43,8 @@ type CTagsParser struct {
 	mainFile *paths.Path
 }
 
-func (p *CTagsParser) Parse(ctagsOutput string, mainFile *paths.Path) []*types.CTag {
-	rows := strings.Split(ctagsOutput, "\n")
+func (p *CTagsParser) Parse(ctagsOutput []byte, mainFile *paths.Path) []*types.CTag {
+	rows := strings.Split(string(ctagsOutput), "\n")
 	rows = removeEmpty(rows)
 
 	p.mainFile = mainFile

legacy/builder/ctags/ctags_parser_test.go

@@ -30,7 +30,7 @@ func produceTags(t *testing.T, filename string) []*types.CTag {
 	require.NoError(t, err)
 
 	parser := CTagsParser{}
-	return parser.Parse(string(bytes), nil)
+	return parser.Parse(bytes, nil)
 }
 
 func TestCTagsParserShouldListPrototypes(t *testing.T) {

legacy/builder/ctags/ctags_to_prototypes_test.go

@@ -30,7 +30,7 @@ func producePrototypes(t *testing.T, filename string, mainFile string) ([]*types
 	require.NoError(t, err)
 
 	parser := &CTagsParser{}
-	parser.Parse(string(bytes), paths.New(mainFile))
+	parser.Parse(bytes, paths.New(mainFile))
 	return parser.GeneratePrototypes()
 }
 

legacy/builder/ctags_runner.go

@@ -26,10 +26,26 @@ import (
 	"github.com/pkg/errors"
 )
 
-type CTagsRunner struct{}
+type CTagsRunner struct {
+	Source         *string
+	TargetFileName string
 
-func (s *CTagsRunner) Run(ctx *types.Context) error {
-	ctagsTargetFilePath := ctx.CTagsTargetFile
+	// Needed for unit-testing
+	CtagsOutput []byte
+}
+
+func (r *CTagsRunner) Run(ctx *types.Context) error {
+	source := *r.Source
+
+	preprocPath := ctx.PreprocPath
+	if err := preprocPath.MkdirAll(); err != nil {
+		return errors.WithStack(err)
+	}
+
+	ctagsTargetFilePath := preprocPath.Join(r.TargetFileName)
+	if err := ctagsTargetFilePath.WriteFile([]byte(source)); err != nil {
+		return errors.WithStack(err)
+	}
 
 	buildProperties := properties.NewMap()
 	buildProperties.Set("tools.ctags.path", "{runtime.tools.ctags.path}")
@@ -52,17 +68,14 @@ func (s *CTagsRunner) Run(ctx *types.Context) error {
 	command := exec.Command(parts[0], parts[1:]...)
 	command.Env = append(os.Environ(), ctx.PackageManager.GetEnvVarsForSpawnedProcess()...)
 
-	sourceBytes, _, err := utils.ExecCommand(ctx, command, utils.Capture /* stdout */, utils.ShowIfVerbose /* stderr */)
+	ctagsOutput, _, err := utils.ExecCommand(ctx, command, utils.Capture /* stdout */, utils.ShowIfVerbose /* stderr */)
 	if err != nil {
 		return errors.WithStack(err)
 	}
 
-	ctx.CTagsOutput = string(sourceBytes)
-
 	parser := &ctags.CTagsParser{}
-
-	ctx.CTagsOfPreprocessedSource = parser.Parse(ctx.CTagsOutput, ctx.Sketch.MainFile)
-	parser.FixCLinkageTagsDeclarations(ctx.CTagsOfPreprocessedSource)
+	parser.Parse(ctagsOutput, ctx.Sketch.MainFile)
+	parser.FixCLinkageTagsDeclarations()
 
 	protos, line := parser.GeneratePrototypes()
 	if line != -1 {
@@ -70,5 +83,7 @@ func (s *CTagsRunner) Run(ctx *types.Context) error {
 	}
 	ctx.Prototypes = protos
 
+	// Needed for unit-testing
+	r.CtagsOutput = ctagsOutput
 	return nil
 }

legacy/builder/ctags_target_file_saver.go

@@ -14,30 +14,3 @@
 // To purchase a commercial license, send an email to license@arduino.cc.
 
 package builder
-
-import (
-	"github.com/arduino/arduino-cli/legacy/builder/types"
-	"github.com/pkg/errors"
-)
-
-type CTagsTargetFileSaver struct {
-	Source         *string
-	TargetFileName string
-}
-
-func (s *CTagsTargetFileSaver) Run(ctx *types.Context) error {
-	source := *s.Source
-
-	preprocPath := ctx.PreprocPath
-	if err := preprocPath.MkdirAll(); err != nil {
-		return errors.WithStack(err)
-	}
-
-	ctagsTargetFilePath := preprocPath.Join(s.TargetFileName)
-	if err := ctagsTargetFilePath.WriteFile([]byte(source)); err != nil {
-		return errors.WithStack(err)
-	}
-
-	ctx.CTagsTargetFile = ctagsTargetFilePath
-	return nil
-}

legacy/builder/filter_sketch_source.go

-// This file is part of arduino-cli.
-//
-// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
-//
-// This software is released under the GNU General Public License version 3,
-// which covers the main part of arduino-cli.
-// The terms of this license can be found at:
-// https://www.gnu.org/licenses/gpl-3.0.en.html
-//
-// You can be released from the requirements of the above licenses by purchasing
-// a commercial license. Buying such a license is mandatory if you want to
-// modify or otherwise use the software for commercial activities involving the
-// Arduino software without disclosing the source code of your own applications.
-// To purchase a commercial license, send an email to license@arduino.cc.
-
-package builder
-
-import (
-	"bufio"
-	"strconv"
-	"strings"
-
-	"github.com/arduino/go-paths-helper"
-
-	"github.com/arduino/arduino-cli/legacy/builder/types"
-	"github.com/arduino/arduino-cli/legacy/builder/utils"
-)
-
-type FilterSketchSource struct {
-	Source            *string
-	RemoveLineMarkers bool
-}
-
-func (s *FilterSketchSource) Run(ctx *types.Context) error {
-	fileNames := paths.NewPathList()
-	fileNames.Add(ctx.Sketch.MainFile)
-	fileNames.AddAll(ctx.Sketch.OtherSketchFiles)
-
-	inSketch := false
-	filtered := ""
-
-	scanner := bufio.NewScanner(strings.NewReader(*s.Source))
-	for scanner.Scan() {
-		line := scanner.Text()
-		if filename := parseLineMarker(line); filename != nil {
-			inSketch = fileNames.Contains(filename)
-			if inSketch && s.RemoveLineMarkers {
-				continue
-			}
-		}
-
-		if inSketch {
-			filtered += line + "\n"
-		}
-	}
-
-	*s.Source = filtered
-	return nil
-}
-
-// Parses the given line as a gcc line marker and returns the contained
-// filename.
-func parseLineMarker(line string) *paths.Path {
-	// A line marker contains the line number and filename and looks like:
-	// # 123 /path/to/file.cpp
-	// It can be followed by zero or more flag number that indicate the
-	// preprocessor state and can be ignored.
-	// For exact details on this format, see:
-	// https://github.com/gcc-mirror/gcc/blob/edd716b6b1caa1a5cb320a8cd7f626f30198e098/gcc/c-family/c-ppoutput.c#L413-L415
-
-	split := strings.SplitN(line, " ", 3)
-	if len(split) < 3 || len(split[0]) == 0 || split[0][0] != '#' {
-		return nil
-	}
-
-	_, err := strconv.Atoi(split[1])
-	if err != nil {
-		return nil
-	}
-
-	// If we get here, we found a # followed by a line number, so
-	// assume this is a line marker and see if the rest of the line
-	// starts with a string containing the filename
-	str, rest, ok := utils.ParseCppString(split[2])
-
-	if ok && (rest == "" || rest[0] == ' ') {
-		return paths.New(str)
-	}
-	return nil
-}

legacy/builder/preprocess_sketch.go

@@ -35,9 +35,9 @@ func PreprocessSketchWithArduinoPreprocessor(ctx *types.Context) error {
 	}
 
 	sourceFile := ctx.SketchBuildPath.Join(ctx.Sketch.MainFile.Base() + ".cpp")
-	GCCPreprocRunner(ctx, sourceFile, ctx.PreprocPath.Join("ctags_target_for_gcc_minus_e.cpp"), ctx.IncludeFolders)
+	targetFile := ctx.PreprocPath.Join("sketch_merged.cpp")
+	GCCPreprocRunner(ctx, sourceFile, targetFile, ctx.IncludeFolders)
 
-	targetFilePath := ctx.PreprocPath.Join("ctags_target_for_gcc_minus_e.cpp")
 	buildProperties := properties.NewMap()
 	buildProperties.Set("tools.arduino-preprocessor.path", "{runtime.tools.arduino-preprocessor.path}")
 	buildProperties.Set("tools.arduino-preprocessor.cmd.path", "{path}/arduino-preprocessor")
@@ -45,7 +45,7 @@ func PreprocessSketchWithArduinoPreprocessor(ctx *types.Context) error {
 	buildProperties.Set("preproc.macros.flags", "-w -x c++ -E -CC")
 	buildProperties.Merge(ctx.BuildProperties)
 	buildProperties.Merge(buildProperties.SubTree("tools").SubTree("arduino-preprocessor"))
-	buildProperties.SetPath("source_file", targetFilePath)
+	buildProperties.SetPath("source_file", targetFile)
 
 	pattern := buildProperties.Get("pattern")
 	if pattern == "" {
@@ -79,7 +79,6 @@ func PreprocessSketchWithArduinoPreprocessor(ctx *types.Context) error {
 	result := utils.NormalizeUTF8(buf)
 
 	//fmt.Printf("PREPROCESSOR OUTPUT:\n%s\n", output)
-	ctx.Source = string(result)
-
-	return bldr.SketchSaveItemCpp(ctx.Sketch.MainFile, []byte(ctx.Source), ctx.SketchBuildPath)
+	ctx.SketchSourceAfterArduinoPreprocessing = string(result)
+	return bldr.SketchSaveItemCpp(ctx.Sketch.MainFile, []byte(ctx.SketchSourceAfterArduinoPreprocessing), ctx.SketchBuildPath)
 }

legacy/builder/prototypes_adder.go

@@ -17,19 +17,20 @@ package builder
 
 import (
 	"fmt"
+	"strconv"
+	"strings"
+
 	"github.com/arduino/arduino-cli/legacy/builder/constants"
 	"github.com/arduino/arduino-cli/legacy/builder/types"
 	"github.com/arduino/arduino-cli/legacy/builder/utils"
-	"strconv"
-	"strings"
 )
 
 type PrototypesAdder struct{}
 
 func (s *PrototypesAdder) Run(ctx *types.Context) error {
 	debugOutput := ctx.DebugPreprocessor
-	source := ctx.Source
 
+	source := ctx.SketchSourceMerged
 	source = strings.Replace(source, "\r\n", "\n", -1)
 	source = strings.Replace(source, "\r", "\n", -1)
 
@@ -61,7 +62,7 @@ func (s *PrototypesAdder) Run(ctx *types.Context) error {
 		}
 		fmt.Println("#END OF PREPROCESSED SOURCE")
 	}
-	ctx.Source = source
+	ctx.SketchSourceAfterArduinoPreprocessing = source
 
 	return nil
 }

legacy/builder/read_file_and_store_in_context.go

-// This file is part of arduino-cli.
-//
-// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
-//
-// This software is released under the GNU General Public License version 3,
-// which covers the main part of arduino-cli.
-// The terms of this license can be found at:
-// https://www.gnu.org/licenses/gpl-3.0.en.html
-//
-// You can be released from the requirements of the above licenses by purchasing
-// a commercial license. Buying such a license is mandatory if you want to
-// modify or otherwise use the software for commercial activities involving the
-// Arduino software without disclosing the source code of your own applications.
-// To purchase a commercial license, send an email to license@arduino.cc.
-
-package builder
-
-import (
-	"github.com/arduino/arduino-cli/legacy/builder/types"
-	"github.com/arduino/go-paths-helper"
-	"github.com/pkg/errors"
-)
-
-type ReadFileAndStoreInContext struct {
-	FileToRead *paths.Path
-	Target     *string
-}
-
-func (s *ReadFileAndStoreInContext) Run(ctx *types.Context) error {
-	bytes, err := s.FileToRead.ReadFile()
-	if err != nil {
-		return errors.WithStack(err)
-	}
-
-	*s.Target = string(bytes)
-
-	return nil
-}

legacy/builder/test/builder_test.go

@@ -116,7 +116,7 @@ func TestBuilderEmptySketch(t *testing.T) {
 	exist, err := buildPath.Join(constants.FOLDER_CORE, "HardwareSerial.cpp.o").ExistCheck()
 	NoError(t, err)
 	require.True(t, exist)
-	exist, err = buildPath.Join(constants.FOLDER_PREPROC, "ctags_target_for_gcc_minus_e.cpp").ExistCheck()
+	exist, err = buildPath.Join(constants.FOLDER_PREPROC, "sketch_merged.cpp").ExistCheck()
 	NoError(t, err)
 	require.True(t, exist)
 	exist, err = buildPath.Join(constants.FOLDER_SKETCH, "sketch1.ino.cpp.o").ExistCheck()
@@ -143,7 +143,7 @@ func TestBuilderBridge(t *testing.T) {
 	exist, err := buildPath.Join(constants.FOLDER_CORE, "HardwareSerial.cpp.o").ExistCheck()
 	NoError(t, err)
 	require.True(t, exist)
-	exist, err = buildPath.Join(constants.FOLDER_PREPROC, "ctags_target_for_gcc_minus_e.cpp").ExistCheck()
+	exist, err = buildPath.Join(constants.FOLDER_PREPROC, "sketch_merged.cpp").ExistCheck()
 	NoError(t, err)
 	require.True(t, exist)
 	exist, err = buildPath.Join(constants.FOLDER_SKETCH, "Bridge.ino.cpp.o").ExistCheck()
@@ -173,7 +173,7 @@ func TestBuilderSketchWithConfig(t *testing.T) {
 	exist, err := buildPath.Join(constants.FOLDER_CORE, "HardwareSerial.cpp.o").ExistCheck()
 	NoError(t, err)
 	require.True(t, exist)
-	exist, err = buildPath.Join(constants.FOLDER_PREPROC, "ctags_target_for_gcc_minus_e.cpp").ExistCheck()
+	exist, err = buildPath.Join(constants.FOLDER_PREPROC, "sketch_merged.cpp").ExistCheck()
 	NoError(t, err)
 	require.True(t, exist)
 	exist, err = buildPath.Join(constants.FOLDER_SKETCH, "sketch_with_config.ino.cpp.o").ExistCheck()
@@ -208,7 +208,7 @@ func TestBuilderBridgeTwice(t *testing.T) {
 	exist, err := buildPath.Join(constants.FOLDER_CORE, "HardwareSerial.cpp.o").ExistCheck()
 	NoError(t, err)
 	require.True(t, exist)
-	exist, err = buildPath.Join(constants.FOLDER_PREPROC, "ctags_target_for_gcc_minus_e.cpp").ExistCheck()
+	exist, err = buildPath.Join(constants.FOLDER_PREPROC, "sketch_merged.cpp").ExistCheck()
 	NoError(t, err)
 	require.True(t, exist)
 	exist, err = buildPath.Join(constants.FOLDER_SKETCH, "Bridge.ino.cpp.o").ExistCheck()
@@ -245,7 +245,7 @@ func TestBuilderBridgeSAM(t *testing.T) {
 	exist, err = buildPath.Join(constants.FOLDER_CORE, "avr", "dtostrf.c.d").ExistCheck()
 	NoError(t, err)
 	require.True(t, exist)
-	exist, err = buildPath.Join(constants.FOLDER_PREPROC, "ctags_target_for_gcc_minus_e.cpp").ExistCheck()
+	exist, err = buildPath.Join(constants.FOLDER_PREPROC, "sketch_merged.cpp").ExistCheck()
 	NoError(t, err)
 	require.True(t, exist)
 	exist, err = buildPath.Join(constants.FOLDER_SKETCH, "Bridge.ino.cpp.o").ExistCheck()
@@ -286,7 +286,7 @@ func TestBuilderBridgeRedBearLab(t *testing.T) {
 	exist, err := buildPath.Join(constants.FOLDER_CORE, "HardwareSerial.cpp.o").ExistCheck()
 	NoError(t, err)
 	require.True(t, exist)
-	exist, err = buildPath.Join(constants.FOLDER_PREPROC, "ctags_target_for_gcc_minus_e.cpp").ExistCheck()
+	exist, err = buildPath.Join(constants.FOLDER_PREPROC, "sketch_merged.cpp").ExistCheck()
 	NoError(t, err)
 	require.True(t, exist)
 	exist, err = buildPath.Join(constants.FOLDER_SKETCH, "Bridge.ino.cpp.o").ExistCheck()

legacy/builder/test/ctags_runner_test.go

@@ -19,6 +19,7 @@ import (
 	"strings"
 	"testing"
 
+	bldr "github.com/arduino/arduino-cli/arduino/builder"
 	"github.com/arduino/arduino-cli/legacy/builder"
 	"github.com/arduino/arduino-cli/legacy/builder/types"
 	paths "github.com/arduino/go-paths-helper"
@@ -31,14 +32,18 @@ func TestCTagsRunner(t *testing.T) {
 	defer cleanUpBuilderTestContext(t, ctx)
 	ctx.Verbose = true
 
+	ctagsRunner := &builder.CTagsRunner{Source: &ctx.SketchSourceMerged, TargetFileName: "ctags_target.cpp"}
+	var _err error
 	commands := []types.Command{
 		&builder.ContainerSetupHardwareToolsLibsSketchAndProps{},
-		&builder.ContainerMergeCopySketchFiles{},
+		types.BareCommand(func(ctx *types.Context) error {
+			ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath)
+			return _err
+		}),
 		&builder.ContainerFindIncludes{},
 		&builder.PrintUsedLibrariesIfVerbose{},
 		&builder.WarnAboutArchIncompatibleLibraries{},
-		&builder.CTagsTargetFileSaver{Source: &ctx.Source, TargetFileName: "ctags_target.cpp"},
-		&builder.CTagsRunner{},
+		ctagsRunner,
 	}
 	for _, command := range commands {
 		err := command.Run(ctx)
@@ -53,8 +58,7 @@ func TestCTagsRunner(t *testing.T) {
 		"digitalCommand	" + quotedSketchLocation + "	/^void digitalCommand(BridgeClient client) {$/;\"	kind:function	line:82	signature:(BridgeClient client)	returntype:void\n" +
 		"analogCommand	" + quotedSketchLocation + "	/^void analogCommand(BridgeClient client) {$/;\"	kind:function	line:109	signature:(BridgeClient client)	returntype:void\n" +
 		"modeCommand	" + quotedSketchLocation + "	/^void modeCommand(BridgeClient client) {$/;\"	kind:function	line:149	signature:(BridgeClient client)	returntype:void\n"
-
-	require.Equal(t, expectedOutput, strings.Replace(ctx.CTagsOutput, "\r\n", "\n", -1))
+	require.Equal(t, expectedOutput, strings.Replace(string(ctagsRunner.CtagsOutput), "\r\n", "\n", -1))
 }
 
 func TestCTagsRunnerSketchWithClass(t *testing.T) {
@@ -63,14 +67,18 @@ func TestCTagsRunnerSketchWithClass(t *testing.T) {
 	defer cleanUpBuilderTestContext(t, ctx)
 	ctx.Verbose = true
 
+	ctagsRunner := &builder.CTagsRunner{Source: &ctx.SketchSourceMerged, TargetFileName: "ctags_target.cpp"}
+	var _err error
 	commands := []types.Command{
 		&builder.ContainerSetupHardwareToolsLibsSketchAndProps{},
-		&builder.ContainerMergeCopySketchFiles{},
+		types.BareCommand(func(ctx *types.Context) error {
+			ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath)
+			return _err
+		}),
 		&builder.ContainerFindIncludes{},
 		&builder.PrintUsedLibrariesIfVerbose{},
 		&builder.WarnAboutArchIncompatibleLibraries{},
-		&builder.CTagsTargetFileSaver{Source: &ctx.Source, TargetFileName: "ctags_target.cpp"},
-		&builder.CTagsRunner{},
+		ctagsRunner,
 	}
 	for _, command := range commands {
 		err := command.Run(ctx)
@@ -83,7 +91,7 @@ func TestCTagsRunnerSketchWithClass(t *testing.T) {
 		"set_values\t" + quotedSketchLocation + "\t/^void Rectangle::set_values (int x, int y) {$/;\"\tkind:function\tline:8\tclass:Rectangle\tsignature:(int x, int y)\treturntype:void\n" +
 		"setup\t" + quotedSketchLocation + "\t/^void setup() {$/;\"\tkind:function\tline:13\tsignature:()\treturntype:void\n" +
 		"loop\t" + quotedSketchLocation + "\t/^void loop() {$/;\"\tkind:function\tline:17\tsignature:()\treturntype:void\n"
-	require.Equal(t, expectedOutput, strings.Replace(ctx.CTagsOutput, "\r\n", "\n", -1))
+	require.Equal(t, expectedOutput, strings.Replace(string(ctagsRunner.CtagsOutput), "\r\n", "\n", -1))
 }
 
 func TestCTagsRunnerSketchWithTypename(t *testing.T) {
@@ -92,14 +100,18 @@ func TestCTagsRunnerSketchWithTypename(t *testing.T) {
 	defer cleanUpBuilderTestContext(t, ctx)
 	ctx.Verbose = true
 
+	ctagsRunner := &builder.CTagsRunner{Source: &ctx.SketchSourceMerged, TargetFileName: "ctags_target.cpp"}
+	var _err error
 	commands := []types.Command{
 		&builder.ContainerSetupHardwareToolsLibsSketchAndProps{},
-		&builder.ContainerMergeCopySketchFiles{},
+		types.BareCommand(func(ctx *types.Context) error {
+			ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath)
+			return _err
+		}),
 		&builder.ContainerFindIncludes{},
 		&builder.PrintUsedLibrariesIfVerbose{},
 		&builder.WarnAboutArchIncompatibleLibraries{},
-		&builder.CTagsTargetFileSaver{Source: &ctx.Source, TargetFileName: "ctags_target.cpp"},
-		&builder.CTagsRunner{},
+		ctagsRunner,
 	}
 	for _, command := range commands {
 		err := command.Run(ctx)
@@ -111,7 +123,7 @@ func TestCTagsRunnerSketchWithTypename(t *testing.T) {
 		"setup\t" + quotedSketchLocation + "\t/^void setup() {$/;\"\tkind:function\tline:6\tsignature:()\treturntype:void\n" +
 		"loop\t" + quotedSketchLocation + "\t/^void loop() {}$/;\"\tkind:function\tline:10\tsignature:()\treturntype:void\n" +
 		"func\t" + quotedSketchLocation + "\t/^typename Foo<char>::Bar func(){$/;\"\tkind:function\tline:12\tsignature:()\treturntype:Foo::Bar\n"
-	require.Equal(t, expectedOutput, strings.Replace(ctx.CTagsOutput, "\r\n", "\n", -1))
+	require.Equal(t, expectedOutput, strings.Replace(string(ctagsRunner.CtagsOutput), "\r\n", "\n", -1))
 }
 
 func TestCTagsRunnerSketchWithNamespace(t *testing.T) {
@@ -120,14 +132,18 @@ func TestCTagsRunnerSketchWithNamespace(t *testing.T) {
 	defer cleanUpBuilderTestContext(t, ctx)
 	ctx.Verbose = true
 
+	ctagsRunner := &builder.CTagsRunner{Source: &ctx.SketchSourceMerged, TargetFileName: "ctags_target.cpp"}
+	var _err error
 	commands := []types.Command{
 		&builder.ContainerSetupHardwareToolsLibsSketchAndProps{},
-		&builder.ContainerMergeCopySketchFiles{},
+		types.BareCommand(func(ctx *types.Context) error {
+			ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath)
+			return _err
+		}),
 		&builder.ContainerFindIncludes{},
 		&builder.PrintUsedLibrariesIfVerbose{},
 		&builder.WarnAboutArchIncompatibleLibraries{},
-		&builder.CTagsTargetFileSaver{Source: &ctx.Source, TargetFileName: "ctags_target.cpp"},
-		&builder.CTagsRunner{},
+		ctagsRunner,
 	}
 	for _, command := range commands {
 		err := command.Run(ctx)
@@ -138,7 +154,7 @@ func TestCTagsRunnerSketchWithNamespace(t *testing.T) {
 	expectedOutput := "value\t" + quotedSketchLocation + "\t/^\tint value() {$/;\"\tkind:function\tline:2\tnamespace:Test\tsignature:()\treturntype:int\n" +
 		"setup\t" + quotedSketchLocation + "\t/^void setup() {}$/;\"\tkind:function\tline:7\tsignature:()\treturntype:void\n" +
 		"loop\t" + quotedSketchLocation + "\t/^void loop() {}$/;\"\tkind:function\tline:8\tsignature:()\treturntype:void\n"
-	require.Equal(t, expectedOutput, strings.Replace(ctx.CTagsOutput, "\r\n", "\n", -1))
+	require.Equal(t, expectedOutput, strings.Replace(string(ctagsRunner.CtagsOutput), "\r\n", "\n", -1))
 }
 
 func TestCTagsRunnerSketchWithTemplates(t *testing.T) {
@@ -147,14 +163,18 @@ func TestCTagsRunnerSketchWithTemplates(t *testing.T) {
 	defer cleanUpBuilderTestContext(t, ctx)
 	ctx.Verbose = true
 
+	ctagsRunner := &builder.CTagsRunner{Source: &ctx.SketchSourceMerged, TargetFileName: "ctags_target.cpp"}
+	var _err error
 	commands := []types.Command{
 		&builder.ContainerSetupHardwareToolsLibsSketchAndProps{},
-		&builder.ContainerMergeCopySketchFiles{},
+		types.BareCommand(func(ctx *types.Context) error {
+			ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath)
+			return _err
+		}),
 		&builder.ContainerFindIncludes{},
 		&builder.PrintUsedLibrariesIfVerbose{},
 		&builder.WarnAboutArchIncompatibleLibraries{},
-		&builder.CTagsTargetFileSaver{Source: &ctx.Source, TargetFileName: "ctags_target.cpp"},
-		&builder.CTagsRunner{},
+		ctagsRunner,
 	}
 	for _, command := range commands {
 		err := command.Run(ctx)
@@ -166,5 +186,5 @@ func TestCTagsRunnerSketchWithTemplates(t *testing.T) {
 		"bVar\t" + quotedSketchLocation + "\t/^c< 8 > bVar;$/;\"\tkind:variable\tline:15\n" +
 		"aVar\t" + quotedSketchLocation + "\t/^c< 1<<8 > aVar;$/;\"\tkind:variable\tline:16\n" +
 		"func\t" + quotedSketchLocation + "\t/^template<int X> func( c< 1<<X> & aParam) {$/;\"\tkind:function\tline:18\tsignature:( c< 1<<X> & aParam)\treturntype:template\n"
-	require.Equal(t, expectedOutput, strings.Replace(ctx.CTagsOutput, "\r\n", "\n", -1))
+	require.Equal(t, expectedOutput, strings.Replace(string(ctagsRunner.CtagsOutput), "\r\n", "\n", -1))
 }

legacy/builder/test/includes_to_include_folders_test.go

@@ -20,6 +20,7 @@ import (
 	"sort"
 	"testing"
 
+	bldr "github.com/arduino/arduino-cli/arduino/builder"
 	"github.com/arduino/arduino-cli/legacy/builder"
 	"github.com/arduino/arduino-cli/legacy/builder/types"
 	paths "github.com/arduino/go-paths-helper"
@@ -31,9 +32,13 @@ func TestIncludesToIncludeFolders(t *testing.T) {
 	defer cleanUpBuilderTestContext(t, ctx)
 	ctx.Verbose = true
 
+	var _err error
 	commands := []types.Command{
 		&builder.ContainerSetupHardwareToolsLibsSketchAndProps{},
-		&builder.ContainerMergeCopySketchFiles{},
+		types.BareCommand(func(ctx *types.Context) error {
+			ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath)
+			return _err
+		}),
 		&builder.ContainerFindIncludes{},
 	}
 	for _, command := range commands {
@@ -51,9 +56,13 @@ func TestIncludesToIncludeFoldersSketchWithIfDef(t *testing.T) {
 	defer cleanUpBuilderTestContext(t, ctx)
 	ctx.Verbose = true
 
+	var _err error
 	commands := []types.Command{
 		&builder.ContainerSetupHardwareToolsLibsSketchAndProps{},
-		&builder.ContainerMergeCopySketchFiles{},
+		types.BareCommand(func(ctx *types.Context) error {
+			ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath)
+			return _err
+		}),
 		&builder.ContainerFindIncludes{},
 	}
 	for _, command := range commands {
@@ -70,9 +79,13 @@ func TestIncludesToIncludeFoldersIRremoteLibrary(t *testing.T) {
 	defer cleanUpBuilderTestContext(t, ctx)
 	ctx.Verbose = true
 
+	var _err error
 	commands := []types.Command{
 		&builder.ContainerSetupHardwareToolsLibsSketchAndProps{},
-		&builder.ContainerMergeCopySketchFiles{},
+		types.BareCommand(func(ctx *types.Context) error {
+			ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath)
+			return _err
+		}),
 		&builder.ContainerFindIncludes{},
 	}
 	for _, command := range commands {
@@ -92,9 +105,13 @@ func TestIncludesToIncludeFoldersANewLibrary(t *testing.T) {
 	defer cleanUpBuilderTestContext(t, ctx)
 	ctx.Verbose = true
 
+	var _err error
 	commands := []types.Command{
 		&builder.ContainerSetupHardwareToolsLibsSketchAndProps{},
-		&builder.ContainerMergeCopySketchFiles{},
+		types.BareCommand(func(ctx *types.Context) error {
+			ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath)
+			return _err
+		}),
 		&builder.ContainerFindIncludes{},
 	}
 	for _, command := range commands {
@@ -119,9 +136,13 @@ func TestIncludesToIncludeFoldersDuplicateLibs(t *testing.T) {
 	ctx = prepareBuilderTestContext(t, ctx, paths.New("user_hardware", "my_avr_platform", "avr", "libraries", "SPI", "examples", "BarometricPressureSensor", "BarometricPressureSensor.ino"), "my_avr_platform:avr:custom_yun")
 	defer cleanUpBuilderTestContext(t, ctx)
 
+	var _err error
 	commands := []types.Command{
 		&builder.ContainerSetupHardwareToolsLibsSketchAndProps{},
-		&builder.ContainerMergeCopySketchFiles{},
+		types.BareCommand(func(ctx *types.Context) error {
+			ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath)
+			return _err
+		}),
 		&builder.ContainerFindIncludes{},
 	}
 	for _, command := range commands {
@@ -147,9 +168,13 @@ func TestIncludesToIncludeFoldersDuplicateLibsWithConflictingLibsOutsideOfPlatfo
 	ctx = prepareBuilderTestContext(t, ctx, paths.New("user_hardware", "my_avr_platform", "avr", "libraries", "SPI", "examples", "BarometricPressureSensor", "BarometricPressureSensor.ino"), "my_avr_platform:avr:custom_yun")
 	defer cleanUpBuilderTestContext(t, ctx)
 
+	var _err error
 	commands := []types.Command{
 		&builder.ContainerSetupHardwareToolsLibsSketchAndProps{},
-		&builder.ContainerMergeCopySketchFiles{},
+		types.BareCommand(func(ctx *types.Context) error {
+			ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath)
+			return _err
+		}),
 		&builder.ContainerFindIncludes{},
 	}
 	for _, command := range commands {
@@ -175,9 +200,13 @@ func TestIncludesToIncludeFoldersDuplicateLibs2(t *testing.T) {
 	ctx = prepareBuilderTestContext(t, ctx, paths.New("sketch_usbhost", "sketch_usbhost.ino"), "arduino:samd:arduino_zero_native")
 	defer cleanUpBuilderTestContext(t, ctx)
 
+	var _err error
 	commands := []types.Command{
 		&builder.ContainerSetupHardwareToolsLibsSketchAndProps{},
-		&builder.ContainerMergeCopySketchFiles{},
+		types.BareCommand(func(ctx *types.Context) error {
+			ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath)
+			return _err
+		}),
 		&builder.ContainerFindIncludes{},
 	}
 	for _, command := range commands {
@@ -197,9 +226,13 @@ func TestIncludesToIncludeFoldersSubfolders(t *testing.T) {
 	defer cleanUpBuilderTestContext(t, ctx)
 	ctx.Verbose = true
 
+	var _err error
 	commands := []types.Command{
 		&builder.ContainerSetupHardwareToolsLibsSketchAndProps{},
-		&builder.ContainerMergeCopySketchFiles{},
+		types.BareCommand(func(ctx *types.Context) error {
+			ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath)
+			return _err
+		}),
 		&builder.ContainerFindIncludes{},
 	}
 	for _, command := range commands {

legacy/builder/test/read_file_and_store_in_context_test.go

-// This file is part of arduino-cli.
-//
-// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
-//
-// This software is released under the GNU General Public License version 3,
-// which covers the main part of arduino-cli.
-// The terms of this license can be found at:
-// https://www.gnu.org/licenses/gpl-3.0.en.html
-//
-// You can be released from the requirements of the above licenses by purchasing
-// a commercial license. Buying such a license is mandatory if you want to
-// modify or otherwise use the software for commercial activities involving the
-// Arduino software without disclosing the source code of your own applications.
-// To purchase a commercial license, send an email to license@arduino.cc.
-
-package test
-
-import (
-	"os"
-	"testing"
-
-	"github.com/arduino/arduino-cli/legacy/builder"
-	"github.com/arduino/arduino-cli/legacy/builder/types"
-	paths "github.com/arduino/go-paths-helper"
-	"github.com/stretchr/testify/require"
-)
-
-func TestReadFileAndStoreInContext(t *testing.T) {
-	filePath, err := os.CreateTemp("", "test")
-	NoError(t, err)
-
-	file := paths.New(filePath.Name())
-	defer file.RemoveAll()
-
-	file.WriteFile([]byte("test test\nciao"))
-
-	ctx := &types.Context{}
-
-	command := &builder.ReadFileAndStoreInContext{FileToRead: file, Target: &ctx.SourceGccMinusE}
-	err = command.Run(ctx)
-	NoError(t, err)
-
-	require.Equal(t, "test test\nciao", ctx.SourceGccMinusE)
-}

legacy/builder/test/utils_test.go

@@ -65,41 +65,3 @@ func TestQuoteCppString(t *testing.T) {
 		require.Equal(t, expected, utils.QuoteCppString(input))
 	}
 }
-
-func TestParseCppString(t *testing.T) {
-	_, _, ok := utils.ParseCppString(`foo`)
-	require.Equal(t, false, ok)
-
-	_, _, ok = utils.ParseCppString(`"foo`)
-	require.Equal(t, false, ok)
-
-	str, rest, ok := utils.ParseCppString(`"foo"`)
-	require.Equal(t, true, ok)
-	require.Equal(t, `foo`, str)
-	require.Equal(t, ``, rest)
-
-	str, rest, ok = utils.ParseCppString(`"foo\\bar"`)
-	require.Equal(t, true, ok)
-	require.Equal(t, `foo\bar`, str)
-	require.Equal(t, ``, rest)
-
-	str, rest, ok = utils.ParseCppString(`"foo \"is\" quoted and \\\\bar\"\" escaped\\" and "then" some`)
-	require.Equal(t, true, ok)
-	require.Equal(t, `foo "is" quoted and \\bar"" escaped\`, str)
-	require.Equal(t, ` and "then" some`, rest)
-
-	str, rest, ok = utils.ParseCppString(`" !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_abcdefghijklmnopqrstuvwxyz{|}~"`)
-	require.Equal(t, true, ok)
-	require.Equal(t, ` !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_abcdefghijklmnopqrstuvwxyz{|}~`, str)
-	require.Equal(t, ``, rest)
-
-	str, rest, ok = utils.ParseCppString(`"/home/ççç/"`)
-	require.Equal(t, true, ok)
-	require.Equal(t, `/home/ççç/`, str)
-	require.Equal(t, ``, rest)
-
-	str, rest, ok = utils.ParseCppString(`"/home/ççç/ /$sdsdd\\"`)
-	require.Equal(t, true, ok)
-	require.Equal(t, `/home/ççç/ /$sdsdd\`, str)
-	require.Equal(t, ``, rest)
-}

legacy/builder/types/context.go

@@ -100,12 +100,17 @@ type Context struct {
 
 	CollectedSourceFiles *UniqueSourceFileQueue
 
-	Sketch          *sketch.Sketch
-	Source          string
-	SourceGccMinusE string
-
+	Sketch        *sketch.Sketch
 	WarningsLevel string
 
+	// Arduino sketch (.ino) to C++ (.cpp) conversion steps:
+	// 1. Concatenate *.ino files into a single merged source file -> SketchSourceMerged
+	SketchSourceMerged string
+	// 2. Run a pass of C++ preprocessor to remove macro definitions and ifdef-ed code -> SketchSourceAfterCppPreprocessing
+	SketchSourceAfterCppPreprocessing string
+	// 3. Do the Arduino preprocessing of the sketch (add missing prototypes) -> SketchSourceAfterArduinoPreprocessing
+	SketchSourceAfterArduinoPreprocessing string
+
 	// Libraries handling
 	LibrariesManager             *librariesmanager.LibrariesManager
 	LibrariesResolver            *librariesresolver.Cpp
@@ -115,9 +120,6 @@ type Context struct {
 	UseCachedLibrariesResolution bool
 
 	// C++ Parsing
-	CTagsOutput                 string
-	CTagsTargetFile             *paths.Path
-	CTagsOfPreprocessedSource   []*CTag
 	LineOffset                  int
 	PrototypesSection           string
 	PrototypesLineWhereToInsert int

legacy/builder/types/types.go

@@ -122,3 +122,9 @@ type CTag struct {
 type Command interface {
 	Run(ctx *Context) error
 }
+
+type BareCommand func(ctx *Context) error
+
+func (cmd BareCommand) Run(ctx *Context) error {
+	return cmd(ctx)
+}

legacy/builder/utils/utils.go

@@ -26,7 +26,6 @@ import (
 	"path/filepath"
 	"strings"
 	"unicode"
-	"unicode/utf8"
 
 	"github.com/arduino/arduino-cli/i18n"
 	"github.com/arduino/arduino-cli/legacy/builder/gohasissues"
@@ -301,50 +300,6 @@ func QuoteCppPath(path *paths.Path) string {
 	return QuoteCppString(path.String())
 }
 
-// Parse a C-preprocessor string as emitted by the preprocessor. This
-// is a string contained in double quotes, with any backslashes or
-// quotes escaped with a backslash. If a valid string was present at the
-// start of the given line, returns the unquoted string contents, the
-// remainder of the line (everything after the closing "), and true.
-// Otherwise, returns the empty string, the entire line and false.
-func ParseCppString(line string) (string, string, bool) {
-	// For details about how these strings are output by gcc, see:
-	// https://github.com/gcc-mirror/gcc/blob/a588355ab948cf551bc9d2b89f18e5ae5140f52c/libcpp/macro.c#L491-L511
-	// Note that the documentation suggests all non-printable
-	// characters are also escaped, but the implementation does not
-	// actually do this. See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=51259
-	if len(line) < 1 || line[0] != '"' {
-		return "", line, false
-	}
-
-	i := 1
-	res := ""
-	for {
-		if i >= len(line) {
-			return "", line, false
-		}
-
-		c, width := utf8.DecodeRuneInString(line[i:])
-
-		switch c {
-		case '\\':
-			// Backslash, next character is used unmodified
-			i += width
-			if i >= len(line) {
-				return "", line, false
-			}
-			res += string(line[i])
-		case '"':
-			// Quote, end of string
-			return res, line[i+width:], true
-		default:
-			res += string(c)
-		}
-
-		i += width
-	}
-}
-
 // Normalizes an UTF8 byte slice
 // TODO: use it more often troughout all the project (maybe on logger interface?)
 func NormalizeUTF8(buf []byte) []byte {