[skip-changelog] legacy: Arduino preprocess subroutine refactorization (part 3) (#2193)

* Converted CTagsRunner into a function

* Removed useless tasks from ctags_runner test

* Simplified ctags_runner test

* Removed some ctags related fields from builder context

The last piece in RunCTags:

 	parser := &ctags.CTagsParser{}
 	parser.Parse(ctagsStdout, sketch.MainFile)
 	parser.FixCLinkageTagsDeclarations()

 	prototypes, line := parser.GeneratePrototypes()
 	if line != -1 {
 		prototypesLineWhereToInsert = line
 	}

has been moved at the beginning of PrototypesAdder.
RunCTags now returns the output of ctags instead of `prototypes` and `line`.
This also allows to remove the context variables that keeps those
information.

* Simplified RunCTags / factored test subroutine

* Removed DebugPreprocessor from builder ctx

* Added executils.RunAndCaptureOutput

* Moved RunCTags out of legacy package

arduino/builder/preprocessor/ctags.go

+// This file is part of arduino-cli.
+//
+// Copyright 2023 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to license@arduino.cc.
+
+package preprocessor
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/arduino/arduino-cli/executils"
+	"github.com/arduino/arduino-cli/i18n"
+	"github.com/arduino/go-paths-helper"
+	"github.com/arduino/go-properties-orderedmap"
+	"github.com/pkg/errors"
+)
+
+var tr = i18n.Tr
+
+// RunCTags performs a run of ctags on the given source file. Returns the ctags output and the stderr contents.
+func RunCTags(sourceFile *paths.Path, buildProperties *properties.Map) ([]byte, []byte, error) {
+	ctagsBuildProperties := properties.NewMap()
+	ctagsBuildProperties.Set("tools.ctags.path", "{runtime.tools.ctags.path}")
+	ctagsBuildProperties.Set("tools.ctags.cmd.path", "{path}/ctags")
+	ctagsBuildProperties.Set("tools.ctags.pattern", `"{cmd.path}" -u --language-force=c++ -f - --c++-kinds=svpf --fields=KSTtzns --line-directives "{source_file}"`)
+	ctagsBuildProperties.Merge(buildProperties)
+	ctagsBuildProperties.Merge(ctagsBuildProperties.SubTree("tools").SubTree("ctags"))
+	ctagsBuildProperties.SetPath("source_file", sourceFile)
+
+	pattern := ctagsBuildProperties.Get("pattern")
+	if pattern == "" {
+		return nil, nil, errors.Errorf(tr("%s pattern is missing"), "ctags")
+	}
+
+	commandLine := ctagsBuildProperties.ExpandPropsInString(pattern)
+	parts, err := properties.SplitQuotedString(commandLine, `"'`, false)
+	if err != nil {
+		return nil, nil, err
+	}
+	proc, err := executils.NewProcess(nil, parts...)
+	if err != nil {
+		return nil, nil, err
+	}
+	stdout, stderr, err := proc.RunAndCaptureOutput(context.Background())
+
+	// Append ctags arguments to stderr
+	args := fmt.Sprintln(strings.Join(parts, " "))
+	stderr = append([]byte(args), stderr...)
+	return stdout, stderr, err
+}

executils/process.go

@@ -16,6 +16,7 @@
 package executils
 
 import (
+	"bytes"
 	"context"
 	"io"
 	"os"
@@ -174,3 +175,15 @@ func (p *Process) RunWithinContext(ctx context.Context) error {
 	}()
 	return p.Wait()
 }
+
+// RunAndCaptureOutput starts the specified command and waits for it to complete. If the given context
+// is canceled before the normal process termination, the process is killed. The standard output and
+// standard error of the process are captured and returned at process termination.
+func (p *Process) RunAndCaptureOutput(ctx context.Context) ([]byte, []byte, error) {
+	stdout := &bytes.Buffer{}
+	stderr := &bytes.Buffer{}
+	p.RedirectStdoutTo(stdout)
+	p.RedirectStderrTo(stderr)
+	err := p.RunWithinContext(ctx)
+	return stdout.Bytes(), stderr.Bytes(), err
+}

legacy/builder/container_add_prototypes.go

@@ -24,6 +24,7 @@ import (
 	"strings"
 
 	bldr "github.com/arduino/arduino-cli/arduino/builder"
+	"github.com/arduino/arduino-cli/arduino/builder/preprocessor"
 	"github.com/arduino/arduino-cli/arduino/sketch"
 	"github.com/arduino/arduino-cli/legacy/builder/types"
 	"github.com/arduino/go-paths-helper"
@@ -62,10 +63,18 @@ func PreprocessSketchWithCtags(ctx *types.Context) error {
 		ctx.SketchSourceAfterCppPreprocessing = filterSketchSource(ctx.Sketch, bytes.NewReader(src), false)
 	}
 
-	if err := (&CTagsRunner{Source: &ctx.SketchSourceAfterCppPreprocessing, TargetFileName: "sketch_merged.cpp"}).Run(ctx); err != nil {
-		return errors.WithStack(err)
+	if err := targetFilePath.WriteFile([]byte(ctx.SketchSourceAfterCppPreprocessing)); err != nil {
+		return err
+	}
+
+	ctagsStdout, ctagsStderr, err := preprocessor.RunCTags(targetFilePath, ctx.BuildProperties)
+	if ctx.Verbose {
+		ctx.WriteStderr(ctagsStderr)
+	}
+	if err != nil {
+		return err
 	}
-	ctx.SketchSourceAfterArduinoPreprocessing, ctx.PrototypesSection = PrototypesAdder(ctx.SketchSourceMerged, ctx.PrototypesLineWhereToInsert, ctx.LineOffset, ctx.Prototypes, ctx.DebugPreprocessor)
+	ctx.SketchSourceAfterArduinoPreprocessing = PrototypesAdder(ctx.Sketch, ctx.SketchSourceMerged, ctagsStdout, ctx.LineOffset)
 
 	if err := bldr.SketchSaveItemCpp(ctx.Sketch.MainFile, []byte(ctx.SketchSourceAfterArduinoPreprocessing), ctx.SketchBuildPath); err != nil {
 		return errors.WithStack(err)

legacy/builder/ctags_runner.go

-// This file is part of arduino-cli.
-//
-// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
-//
-// This software is released under the GNU General Public License version 3,
-// which covers the main part of arduino-cli.
-// The terms of this license can be found at:
-// https://www.gnu.org/licenses/gpl-3.0.en.html
-//
-// You can be released from the requirements of the above licenses by purchasing
-// a commercial license. Buying such a license is mandatory if you want to
-// modify or otherwise use the software for commercial activities involving the
-// Arduino software without disclosing the source code of your own applications.
-// To purchase a commercial license, send an email to license@arduino.cc.
-
-package builder
-
-import (
-	"os"
-	"os/exec"
-
-	"github.com/arduino/arduino-cli/legacy/builder/ctags"
-	"github.com/arduino/arduino-cli/legacy/builder/types"
-	"github.com/arduino/arduino-cli/legacy/builder/utils"
-	properties "github.com/arduino/go-properties-orderedmap"
-	"github.com/pkg/errors"
-)
-
-type CTagsRunner struct {
-	Source         *string
-	TargetFileName string
-
-	// Needed for unit-testing
-	CtagsOutput []byte
-}
-
-func (r *CTagsRunner) Run(ctx *types.Context) error {
-	source := *r.Source
-
-	preprocPath := ctx.PreprocPath
-	if err := preprocPath.MkdirAll(); err != nil {
-		return errors.WithStack(err)
-	}
-
-	ctagsTargetFilePath := preprocPath.Join(r.TargetFileName)
-	if err := ctagsTargetFilePath.WriteFile([]byte(source)); err != nil {
-		return errors.WithStack(err)
-	}
-
-	buildProperties := properties.NewMap()
-	buildProperties.Set("tools.ctags.path", "{runtime.tools.ctags.path}")
-	buildProperties.Set("tools.ctags.cmd.path", "{path}/ctags")
-	buildProperties.Set("tools.ctags.pattern", `"{cmd.path}" -u --language-force=c++ -f - --c++-kinds=svpf --fields=KSTtzns --line-directives "{source_file}"`)
-	buildProperties.Merge(ctx.BuildProperties)
-	buildProperties.Merge(buildProperties.SubTree("tools").SubTree("ctags"))
-	buildProperties.SetPath("source_file", ctagsTargetFilePath)
-
-	pattern := buildProperties.Get("pattern")
-	if pattern == "" {
-		return errors.Errorf(tr("%s pattern is missing"), "ctags")
-	}
-
-	commandLine := buildProperties.ExpandPropsInString(pattern)
-	parts, err := properties.SplitQuotedString(commandLine, `"'`, false)
-	if err != nil {
-		return errors.WithStack(err)
-	}
-	command := exec.Command(parts[0], parts[1:]...)
-	command.Env = append(os.Environ(), ctx.PackageManager.GetEnvVarsForSpawnedProcess()...)
-
-	ctagsOutput, _, err := utils.ExecCommand(ctx, command, utils.Capture /* stdout */, utils.ShowIfVerbose /* stderr */)
-	if err != nil {
-		return errors.WithStack(err)
-	}
-
-	parser := &ctags.CTagsParser{}
-	parser.Parse(ctagsOutput, ctx.Sketch.MainFile)
-	parser.FixCLinkageTagsDeclarations()
-
-	protos, line := parser.GeneratePrototypes()
-	if line != -1 {
-		ctx.PrototypesLineWhereToInsert = line
-	}
-	ctx.Prototypes = protos
-
-	// Needed for unit-testing
-	r.CtagsOutput = ctagsOutput
-	return nil
-}

legacy/builder/prototypes_adder.go

@@ -20,25 +20,37 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/arduino/arduino-cli/arduino/sketch"
 	"github.com/arduino/arduino-cli/legacy/builder/constants"
 	"github.com/arduino/arduino-cli/legacy/builder/ctags"
 	"github.com/arduino/arduino-cli/legacy/builder/utils"
 )
 
-func PrototypesAdder(source string, firstFunctionLine, lineOffset int, prototypes []*ctags.Prototype, debugOutput bool) (preprocessedSource, prototypeSection string) {
+var DebugPreprocessor bool
+
+func PrototypesAdder(sketch *sketch.Sketch, source string, ctagsStdout []byte, lineOffset int) string {
+	parser := &ctags.CTagsParser{}
+	parser.Parse(ctagsStdout, sketch.MainFile)
+	parser.FixCLinkageTagsDeclarations()
+
+	prototypes, firstFunctionLine := parser.GeneratePrototypes()
+	if firstFunctionLine == -1 {
+		firstFunctionLine = 0
+	}
+
 	source = strings.Replace(source, "\r\n", "\n", -1)
 	source = strings.Replace(source, "\r", "\n", -1)
 	sourceRows := strings.Split(source, "\n")
 	if isFirstFunctionOutsideOfSource(firstFunctionLine, sourceRows) {
-		return
+		return ""
 	}
 
 	insertionLine := firstFunctionLine + lineOffset - 1
 	firstFunctionChar := len(strings.Join(sourceRows[:insertionLine], "\n")) + 1
-	prototypeSection = composePrototypeSection(firstFunctionLine, prototypes)
-	preprocessedSource = source[:firstFunctionChar] + prototypeSection + source[firstFunctionChar:]
+	prototypeSection := composePrototypeSection(firstFunctionLine, prototypes)
+	preprocessedSource := source[:firstFunctionChar] + prototypeSection + source[firstFunctionChar:]
 
-	if debugOutput {
+	if DebugPreprocessor {
 		fmt.Println("#PREPROCESSED SOURCE")
 		prototypesRows := strings.Split(prototypeSection, "\n")
 		prototypesRows = prototypesRows[:len(prototypesRows)-1]
@@ -53,7 +65,7 @@ func PrototypesAdder(source string, firstFunctionLine, lineOffset int, prototype
 		}
 		fmt.Println("#END OF PREPROCESSED SOURCE")
 	}
-	return
+	return preprocessedSource
 }
 
 func composePrototypeSection(line int, prototypes []*ctags.Prototype) string {

legacy/builder/test/ctags_runner_test.go

@@ -20,35 +20,35 @@ import (
 	"testing"
 
 	bldr "github.com/arduino/arduino-cli/arduino/builder"
+	"github.com/arduino/arduino-cli/arduino/builder/preprocessor"
 	"github.com/arduino/arduino-cli/legacy/builder"
-	"github.com/arduino/arduino-cli/legacy/builder/types"
 	paths "github.com/arduino/go-paths-helper"
 	"github.com/stretchr/testify/require"
 )
 
-func TestCTagsRunner(t *testing.T) {
-	sketchLocation := Abs(t, paths.New("downloaded_libraries", "Bridge", "examples", "Bridge", "Bridge.ino"))
+func ctagsRunnerTestTemplate(t *testing.T, sketchLocation *paths.Path) []byte {
 	ctx := prepareBuilderTestContext(t, nil, sketchLocation, "arduino:avr:leonardo")
 	defer cleanUpBuilderTestContext(t, ctx)
 	ctx.Verbose = true
 
-	ctagsRunner := &builder.CTagsRunner{Source: &ctx.SketchSourceMerged, TargetFileName: "ctags_target.cpp"}
-	var _err error
-	commands := []types.Command{
-		&builder.ContainerSetupHardwareToolsLibsSketchAndProps{},
-		types.BareCommand(func(ctx *types.Context) error {
-			ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath)
-			return _err
-		}),
-		&builder.ContainerFindIncludes{},
-		&builder.PrintUsedLibrariesIfVerbose{},
-		&builder.WarnAboutArchIncompatibleLibraries{},
-		ctagsRunner,
-	}
-	for _, command := range commands {
-		err := command.Run(ctx)
-		NoError(t, err)
-	}
+	err := (&builder.ContainerSetupHardwareToolsLibsSketchAndProps{}).Run(ctx)
+	NoError(t, err)
+
+	_, source, err := bldr.PrepareSketchBuildPath(ctx.Sketch, nil, ctx.SketchBuildPath)
+	NoError(t, err)
+
+	target := ctx.BuildPath.Join("ctags_target.cpp")
+	NoError(t, target.WriteFile([]byte(source)))
+
+	ctagsOutput, _, err := preprocessor.RunCTags(target, ctx.BuildProperties)
+	NoError(t, err)
+
+	return ctagsOutput
+}
+
+func TestCTagsRunner(t *testing.T) {
+	sketchLocation := Abs(t, paths.New("downloaded_libraries", "Bridge", "examples", "Bridge", "Bridge.ino"))
+	ctagsOutput := ctagsRunnerTestTemplate(t, sketchLocation)
 
 	quotedSketchLocation := strings.Replace(sketchLocation.String(), "\\", "\\\\", -1)
 	expectedOutput := "server	" + quotedSketchLocation + "	/^BridgeServer server;$/;\"	kind:variable	line:31\n" +
@@ -58,32 +58,12 @@ func TestCTagsRunner(t *testing.T) {
 		"digitalCommand	" + quotedSketchLocation + "	/^void digitalCommand(BridgeClient client) {$/;\"	kind:function	line:82	signature:(BridgeClient client)	returntype:void\n" +
 		"analogCommand	" + quotedSketchLocation + "	/^void analogCommand(BridgeClient client) {$/;\"	kind:function	line:109	signature:(BridgeClient client)	returntype:void\n" +
 		"modeCommand	" + quotedSketchLocation + "	/^void modeCommand(BridgeClient client) {$/;\"	kind:function	line:149	signature:(BridgeClient client)	returntype:void\n"
-	require.Equal(t, expectedOutput, strings.Replace(string(ctagsRunner.CtagsOutput), "\r\n", "\n", -1))
+	require.Equal(t, expectedOutput, strings.Replace(string(ctagsOutput), "\r\n", "\n", -1))
 }
 
 func TestCTagsRunnerSketchWithClass(t *testing.T) {
 	sketchLocation := Abs(t, paths.New("sketch_with_class", "sketch_with_class.ino"))
-	ctx := prepareBuilderTestContext(t, nil, sketchLocation, "arduino:avr:leonardo")
-	defer cleanUpBuilderTestContext(t, ctx)
-	ctx.Verbose = true
-
-	ctagsRunner := &builder.CTagsRunner{Source: &ctx.SketchSourceMerged, TargetFileName: "ctags_target.cpp"}
-	var _err error
-	commands := []types.Command{
-		&builder.ContainerSetupHardwareToolsLibsSketchAndProps{},
-		types.BareCommand(func(ctx *types.Context) error {
-			ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath)
-			return _err
-		}),
-		&builder.ContainerFindIncludes{},
-		&builder.PrintUsedLibrariesIfVerbose{},
-		&builder.WarnAboutArchIncompatibleLibraries{},
-		ctagsRunner,
-	}
-	for _, command := range commands {
-		err := command.Run(ctx)
-		NoError(t, err)
-	}
+	ctagsOutput := ctagsRunnerTestTemplate(t, sketchLocation)
 
 	quotedSketchLocation := strings.Replace(sketchLocation.String(), "\\", "\\\\", -1)
 	expectedOutput := "set_values\t" + quotedSketchLocation + "\t/^    void set_values (int,int);$/;\"\tkind:prototype\tline:4\tclass:Rectangle\tsignature:(int,int)\treturntype:void\n" +
@@ -91,100 +71,40 @@ func TestCTagsRunnerSketchWithClass(t *testing.T) {
 		"set_values\t" + quotedSketchLocation + "\t/^void Rectangle::set_values (int x, int y) {$/;\"\tkind:function\tline:8\tclass:Rectangle\tsignature:(int x, int y)\treturntype:void\n" +
 		"setup\t" + quotedSketchLocation + "\t/^void setup() {$/;\"\tkind:function\tline:13\tsignature:()\treturntype:void\n" +
 		"loop\t" + quotedSketchLocation + "\t/^void loop() {$/;\"\tkind:function\tline:17\tsignature:()\treturntype:void\n"
-	require.Equal(t, expectedOutput, strings.Replace(string(ctagsRunner.CtagsOutput), "\r\n", "\n", -1))
+	require.Equal(t, expectedOutput, strings.Replace(string(ctagsOutput), "\r\n", "\n", -1))
 }
 
 func TestCTagsRunnerSketchWithTypename(t *testing.T) {
 	sketchLocation := Abs(t, paths.New("sketch_with_typename", "sketch_with_typename.ino"))
-	ctx := prepareBuilderTestContext(t, nil, sketchLocation, "arduino:avr:leonardo")
-	defer cleanUpBuilderTestContext(t, ctx)
-	ctx.Verbose = true
-
-	ctagsRunner := &builder.CTagsRunner{Source: &ctx.SketchSourceMerged, TargetFileName: "ctags_target.cpp"}
-	var _err error
-	commands := []types.Command{
-		&builder.ContainerSetupHardwareToolsLibsSketchAndProps{},
-		types.BareCommand(func(ctx *types.Context) error {
-			ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath)
-			return _err
-		}),
-		&builder.ContainerFindIncludes{},
-		&builder.PrintUsedLibrariesIfVerbose{},
-		&builder.WarnAboutArchIncompatibleLibraries{},
-		ctagsRunner,
-	}
-	for _, command := range commands {
-		err := command.Run(ctx)
-		NoError(t, err)
-	}
+	ctagsOutput := ctagsRunnerTestTemplate(t, sketchLocation)
 
 	quotedSketchLocation := strings.Replace(sketchLocation.String(), "\\", "\\\\", -1)
 	expectedOutput := "Foo\t" + quotedSketchLocation + "\t/^  struct Foo{$/;\"\tkind:struct\tline:2\n" +
 		"setup\t" + quotedSketchLocation + "\t/^void setup() {$/;\"\tkind:function\tline:6\tsignature:()\treturntype:void\n" +
 		"loop\t" + quotedSketchLocation + "\t/^void loop() {}$/;\"\tkind:function\tline:10\tsignature:()\treturntype:void\n" +
 		"func\t" + quotedSketchLocation + "\t/^typename Foo<char>::Bar func(){$/;\"\tkind:function\tline:12\tsignature:()\treturntype:Foo::Bar\n"
-	require.Equal(t, expectedOutput, strings.Replace(string(ctagsRunner.CtagsOutput), "\r\n", "\n", -1))
+	require.Equal(t, expectedOutput, strings.Replace(string(ctagsOutput), "\r\n", "\n", -1))
 }
 
 func TestCTagsRunnerSketchWithNamespace(t *testing.T) {
 	sketchLocation := Abs(t, paths.New("sketch_with_namespace", "sketch_with_namespace.ino"))
-	ctx := prepareBuilderTestContext(t, nil, sketchLocation, "arduino:avr:leonardo")
-	defer cleanUpBuilderTestContext(t, ctx)
-	ctx.Verbose = true
-
-	ctagsRunner := &builder.CTagsRunner{Source: &ctx.SketchSourceMerged, TargetFileName: "ctags_target.cpp"}
-	var _err error
-	commands := []types.Command{
-		&builder.ContainerSetupHardwareToolsLibsSketchAndProps{},
-		types.BareCommand(func(ctx *types.Context) error {
-			ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath)
-			return _err
-		}),
-		&builder.ContainerFindIncludes{},
-		&builder.PrintUsedLibrariesIfVerbose{},
-		&builder.WarnAboutArchIncompatibleLibraries{},
-		ctagsRunner,
-	}
-	for _, command := range commands {
-		err := command.Run(ctx)
-		NoError(t, err)
-	}
+	ctagsOutput := ctagsRunnerTestTemplate(t, sketchLocation)
 
 	quotedSketchLocation := strings.Replace(sketchLocation.String(), "\\", "\\\\", -1)
 	expectedOutput := "value\t" + quotedSketchLocation + "\t/^\tint value() {$/;\"\tkind:function\tline:2\tnamespace:Test\tsignature:()\treturntype:int\n" +
 		"setup\t" + quotedSketchLocation + "\t/^void setup() {}$/;\"\tkind:function\tline:7\tsignature:()\treturntype:void\n" +
 		"loop\t" + quotedSketchLocation + "\t/^void loop() {}$/;\"\tkind:function\tline:8\tsignature:()\treturntype:void\n"
-	require.Equal(t, expectedOutput, strings.Replace(string(ctagsRunner.CtagsOutput), "\r\n", "\n", -1))
+	require.Equal(t, expectedOutput, strings.Replace(string(ctagsOutput), "\r\n", "\n", -1))
 }
 
 func TestCTagsRunnerSketchWithTemplates(t *testing.T) {
 	sketchLocation := Abs(t, paths.New("sketch_with_templates_and_shift", "sketch_with_templates_and_shift.ino"))
-	ctx := prepareBuilderTestContext(t, nil, sketchLocation, "arduino:avr:leonardo")
-	defer cleanUpBuilderTestContext(t, ctx)
-	ctx.Verbose = true
-
-	ctagsRunner := &builder.CTagsRunner{Source: &ctx.SketchSourceMerged, TargetFileName: "ctags_target.cpp"}
-	var _err error
-	commands := []types.Command{
-		&builder.ContainerSetupHardwareToolsLibsSketchAndProps{},
-		types.BareCommand(func(ctx *types.Context) error {
-			ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath)
-			return _err
-		}),
-		&builder.ContainerFindIncludes{},
-		&builder.PrintUsedLibrariesIfVerbose{},
-		&builder.WarnAboutArchIncompatibleLibraries{},
-		ctagsRunner,
-	}
-	for _, command := range commands {
-		err := command.Run(ctx)
-		NoError(t, err)
-	}
+	ctagsOutput := ctagsRunnerTestTemplate(t, sketchLocation)
 
 	quotedSketchLocation := strings.Replace(sketchLocation.String(), "\\", "\\\\", -1)
 	expectedOutput := "printGyro\t" + quotedSketchLocation + "\t/^void printGyro()$/;\"\tkind:function\tline:10\tsignature:()\treturntype:void\n" +
 		"bVar\t" + quotedSketchLocation + "\t/^c< 8 > bVar;$/;\"\tkind:variable\tline:15\n" +
 		"aVar\t" + quotedSketchLocation + "\t/^c< 1<<8 > aVar;$/;\"\tkind:variable\tline:16\n" +
 		"func\t" + quotedSketchLocation + "\t/^template<int X> func( c< 1<<X> & aParam) {$/;\"\tkind:function\tline:18\tsignature:( c< 1<<X> & aParam)\treturntype:template\n"
-	require.Equal(t, expectedOutput, strings.Replace(string(ctagsRunner.CtagsOutput), "\r\n", "\n", -1))
+	require.Equal(t, expectedOutput, strings.Replace(string(ctagsOutput), "\r\n", "\n", -1))
 }

legacy/builder/test/prototypes_adder_test.go

@@ -54,7 +54,7 @@ func TestPrototypesAdderBridgeExample(t *testing.T) {
 	NoError(t, builder.PreprocessSketchWithCtags(ctx))
 
 	require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#include <Arduino.h>\n#line 1 "+quotedSketchLocation+"\n")
-	require.Equal(t, "#line 33 "+quotedSketchLocation+"\nvoid setup();\n#line 46 "+quotedSketchLocation+"\nvoid loop();\n#line 62 "+quotedSketchLocation+"\nvoid process(BridgeClient client);\n#line 82 "+quotedSketchLocation+"\nvoid digitalCommand(BridgeClient client);\n#line 109 "+quotedSketchLocation+"\nvoid analogCommand(BridgeClient client);\n#line 149 "+quotedSketchLocation+"\nvoid modeCommand(BridgeClient client);\n#line 33 "+quotedSketchLocation+"\n", ctx.PrototypesSection)
+	require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#line 33 "+quotedSketchLocation+"\nvoid setup();\n#line 46 "+quotedSketchLocation+"\nvoid loop();\n#line 62 "+quotedSketchLocation+"\nvoid process(BridgeClient client);\n#line 82 "+quotedSketchLocation+"\nvoid digitalCommand(BridgeClient client);\n#line 109 "+quotedSketchLocation+"\nvoid analogCommand(BridgeClient client);\n#line 149 "+quotedSketchLocation+"\nvoid modeCommand(BridgeClient client);\n#line 33 "+quotedSketchLocation+"\n")
 }
 
 func TestPrototypesAdderSketchWithIfDef(t *testing.T) {
@@ -262,7 +262,7 @@ func TestPrototypesAdderSketchWithConfig(t *testing.T) {
 	NoError(t, builder.PreprocessSketchWithCtags(ctx))
 
 	require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#include <Arduino.h>\n#line 1 "+quotedSketchLocation+"\n")
-	require.Equal(t, "#line 13 "+quotedSketchLocation+"\nvoid setup();\n#line 17 "+quotedSketchLocation+"\nvoid loop();\n#line 13 "+quotedSketchLocation+"\n", ctx.PrototypesSection)
+	require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#line 13 "+quotedSketchLocation+"\nvoid setup();\n#line 17 "+quotedSketchLocation+"\nvoid loop();\n#line 13 "+quotedSketchLocation+"\n")
 
 	preprocessed := LoadAndInterpolate(t, filepath.Join("sketch_with_config", "sketch_with_config.preprocessed.txt"), ctx)
 	require.Equal(t, preprocessed, strings.Replace(ctx.SketchSourceAfterArduinoPreprocessing, "\r\n", "\n", -1))
@@ -293,7 +293,7 @@ func TestPrototypesAdderSketchNoFunctionsTwoFiles(t *testing.T) {
 	NoError(t, builder.PreprocessSketchWithCtags(ctx))
 
 	require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#include <Arduino.h>\n#line 1 "+quotedSketchLocation+"\n")
-	require.Equal(t, "", ctx.PrototypesSection)
+	require.Equal(t, ctx.SketchSourceMerged, ctx.SketchSourceAfterArduinoPreprocessing) // No prototypes added
 }
 
 func TestPrototypesAdderSketchNoFunctions(t *testing.T) {
@@ -320,7 +320,7 @@ func TestPrototypesAdderSketchNoFunctions(t *testing.T) {
 	NoError(t, builder.PreprocessSketchWithCtags(ctx))
 
 	require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#include <Arduino.h>\n#line 1 "+quotedSketchLocation+"\n")
-	require.Equal(t, "", ctx.PrototypesSection)
+	require.Equal(t, ctx.SketchSourceMerged, ctx.SketchSourceAfterArduinoPreprocessing) // No prototypes added
 }
 
 func TestPrototypesAdderSketchWithDefaultArgs(t *testing.T) {
@@ -348,7 +348,7 @@ func TestPrototypesAdderSketchWithDefaultArgs(t *testing.T) {
 	NoError(t, builder.PreprocessSketchWithCtags(ctx))
 
 	require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#include <Arduino.h>\n#line 1 "+quotedSketchLocation+"\n")
-	require.Equal(t, "#line 4 "+quotedSketchLocation+"\nvoid setup();\n#line 7 "+quotedSketchLocation+"\nvoid loop();\n#line 1 "+quotedSketchLocation+"\n", ctx.PrototypesSection)
+	require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#line 4 "+quotedSketchLocation+"\nvoid setup();\n#line 7 "+quotedSketchLocation+"\nvoid loop();\n#line 1 "+quotedSketchLocation+"\n")
 }
 
 func TestPrototypesAdderSketchWithInlineFunction(t *testing.T) {
@@ -378,7 +378,7 @@ func TestPrototypesAdderSketchWithInlineFunction(t *testing.T) {
 	require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#include <Arduino.h>\n#line 1 "+quotedSketchLocation+"\n")
 
 	expected := "#line 1 " + quotedSketchLocation + "\nvoid setup();\n#line 2 " + quotedSketchLocation + "\nvoid loop();\n#line 4 " + quotedSketchLocation + "\nshort unsigned int testInt();\n#line 8 " + quotedSketchLocation + "\nstatic int8_t testInline();\n#line 12 " + quotedSketchLocation + "\n__attribute__((always_inline)) uint8_t testAttribute();\n#line 1 " + quotedSketchLocation + "\n"
-	obtained := ctx.PrototypesSection
+	obtained := ctx.SketchSourceAfterArduinoPreprocessing
 	// ctags based preprocessing removes "inline" but this is still OK
 	// TODO: remove this exception when moving to a more powerful parser
 	expected = strings.Replace(expected, "static inline int8_t testInline();", "static int8_t testInline();", -1)
@@ -387,7 +387,7 @@ func TestPrototypesAdderSketchWithInlineFunction(t *testing.T) {
 	// TODO: remove this exception when moving to a more powerful parser
 	expected = strings.Replace(expected, "__attribute__((always_inline)) uint8_t testAttribute();", "uint8_t testAttribute();", -1)
 	obtained = strings.Replace(obtained, "__attribute__((always_inline)) uint8_t testAttribute();", "uint8_t testAttribute();", -1)
-	require.Equal(t, expected, obtained)
+	require.Contains(t, obtained, expected)
 }
 
 func TestPrototypesAdderSketchWithFunctionSignatureInsideIFDEF(t *testing.T) {
@@ -415,7 +415,7 @@ func TestPrototypesAdderSketchWithFunctionSignatureInsideIFDEF(t *testing.T) {
 	NoError(t, builder.PreprocessSketchWithCtags(ctx))
 
 	require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#include <Arduino.h>\n#line 1 "+quotedSketchLocation+"\n")
-	require.Equal(t, "#line 1 "+quotedSketchLocation+"\nvoid setup();\n#line 3 "+quotedSketchLocation+"\nvoid loop();\n#line 15 "+quotedSketchLocation+"\nint8_t adalight();\n#line 1 "+quotedSketchLocation+"\n", ctx.PrototypesSection)
+	require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#line 1 "+quotedSketchLocation+"\nvoid setup();\n#line 3 "+quotedSketchLocation+"\nvoid loop();\n#line 15 "+quotedSketchLocation+"\nint8_t adalight();\n#line 1 "+quotedSketchLocation+"\n")
 }
 
 func TestPrototypesAdderSketchWithUSBCON(t *testing.T) {
@@ -448,7 +448,7 @@ func TestPrototypesAdderSketchWithUSBCON(t *testing.T) {
 	NoError(t, builder.PreprocessSketchWithCtags(ctx))
 
 	require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#include <Arduino.h>\n#line 1 "+quotedSketchLocation+"\n")
-	require.Equal(t, "#line 5 "+quotedSketchLocation+"\nvoid ciao();\n#line 10 "+quotedSketchLocation+"\nvoid setup();\n#line 15 "+quotedSketchLocation+"\nvoid loop();\n#line 5 "+quotedSketchLocation+"\n", ctx.PrototypesSection)
+	require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#line 5 "+quotedSketchLocation+"\nvoid ciao();\n#line 10 "+quotedSketchLocation+"\nvoid setup();\n#line 15 "+quotedSketchLocation+"\nvoid loop();\n#line 5 "+quotedSketchLocation+"\n")
 }
 
 func TestPrototypesAdderSketchWithTypename(t *testing.T) {
@@ -481,12 +481,12 @@ func TestPrototypesAdderSketchWithTypename(t *testing.T) {
 
 	require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#include <Arduino.h>\n#line 1 "+quotedSketchLocation+"\n")
 	expected := "#line 6 " + quotedSketchLocation + "\nvoid setup();\n#line 10 " + quotedSketchLocation + "\nvoid loop();\n#line 12 " + quotedSketchLocation + "\ntypename Foo<char>::Bar func();\n#line 6 " + quotedSketchLocation + "\n"
-	obtained := ctx.PrototypesSection
+	obtained := ctx.SketchSourceAfterArduinoPreprocessing
 	// ctags based preprocessing ignores line with typename
 	// TODO: remove this exception when moving to a more powerful parser
 	expected = strings.Replace(expected, "#line 12 "+quotedSketchLocation+"\ntypename Foo<char>::Bar func();\n", "", -1)
 	obtained = strings.Replace(obtained, "#line 12 "+quotedSketchLocation+"\ntypename Foo<char>::Bar func();\n", "", -1)
-	require.Equal(t, expected, obtained)
+	require.Contains(t, obtained, expected)
 }
 
 func TestPrototypesAdderSketchWithIfDef2(t *testing.T) {
@@ -514,7 +514,7 @@ func TestPrototypesAdderSketchWithIfDef2(t *testing.T) {
 	NoError(t, builder.PreprocessSketchWithCtags(ctx))
 
 	require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#include <Arduino.h>\n#line 1 "+quotedSketchLocation+"\n")
-	require.Equal(t, "#line 5 "+quotedSketchLocation+"\nvoid elseBranch();\n#line 9 "+quotedSketchLocation+"\nvoid f1();\n#line 10 "+quotedSketchLocation+"\nvoid f2();\n#line 12 "+quotedSketchLocation+"\nvoid setup();\n#line 14 "+quotedSketchLocation+"\nvoid loop();\n#line 5 "+quotedSketchLocation+"\n", ctx.PrototypesSection)
+	require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#line 5 "+quotedSketchLocation+"\nvoid elseBranch();\n#line 9 "+quotedSketchLocation+"\nvoid f1();\n#line 10 "+quotedSketchLocation+"\nvoid f2();\n#line 12 "+quotedSketchLocation+"\nvoid setup();\n#line 14 "+quotedSketchLocation+"\nvoid loop();\n#line 5 "+quotedSketchLocation+"\n")
 
 	expectedSource := LoadAndInterpolate(t, filepath.Join("sketch_with_ifdef", "sketch.preprocessed.txt"), ctx)
 	require.Equal(t, expectedSource, strings.Replace(ctx.SketchSourceAfterArduinoPreprocessing, "\r\n", "\n", -1))
@@ -545,7 +545,7 @@ func TestPrototypesAdderSketchWithIfDef2SAM(t *testing.T) {
 	NoError(t, builder.PreprocessSketchWithCtags(ctx))
 
 	require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#include <Arduino.h>\n#line 1 "+quotedSketchLocation+"\n")
-	require.Equal(t, "#line 2 "+quotedSketchLocation+"\nvoid ifBranch();\n#line 9 "+quotedSketchLocation+"\nvoid f1();\n#line 10 "+quotedSketchLocation+"\nvoid f2();\n#line 12 "+quotedSketchLocation+"\nvoid setup();\n#line 14 "+quotedSketchLocation+"\nvoid loop();\n#line 2 "+quotedSketchLocation+"\n", ctx.PrototypesSection)
+	require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#line 2 "+quotedSketchLocation+"\nvoid ifBranch();\n#line 9 "+quotedSketchLocation+"\nvoid f1();\n#line 10 "+quotedSketchLocation+"\nvoid f2();\n#line 12 "+quotedSketchLocation+"\nvoid setup();\n#line 14 "+quotedSketchLocation+"\nvoid loop();\n#line 2 "+quotedSketchLocation+"\n")
 
 	expectedSource := LoadAndInterpolate(t, filepath.Join("sketch_with_ifdef", "sketch.preprocessed.SAM.txt"), ctx)
 	require.Equal(t, expectedSource, strings.Replace(ctx.SketchSourceAfterArduinoPreprocessing, "\r\n", "\n", -1))
@@ -576,7 +576,7 @@ func TestPrototypesAdderSketchWithConst(t *testing.T) {
 	NoError(t, builder.PreprocessSketchWithCtags(ctx))
 
 	require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#include <Arduino.h>\n#line 1 "+quotedSketchLocation+"\n")
-	require.Equal(t, "#line 1 "+quotedSketchLocation+"\nvoid setup();\n#line 2 "+quotedSketchLocation+"\nvoid loop();\n#line 4 "+quotedSketchLocation+"\nconst __FlashStringHelper* test();\n#line 6 "+quotedSketchLocation+"\nconst int test3();\n#line 8 "+quotedSketchLocation+"\nvolatile __FlashStringHelper* test2();\n#line 10 "+quotedSketchLocation+"\nvolatile int test4();\n#line 1 "+quotedSketchLocation+"\n", ctx.PrototypesSection)
+	require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#line 1 "+quotedSketchLocation+"\nvoid setup();\n#line 2 "+quotedSketchLocation+"\nvoid loop();\n#line 4 "+quotedSketchLocation+"\nconst __FlashStringHelper* test();\n#line 6 "+quotedSketchLocation+"\nconst int test3();\n#line 8 "+quotedSketchLocation+"\nvolatile __FlashStringHelper* test2();\n#line 10 "+quotedSketchLocation+"\nvolatile int test4();\n#line 1 "+quotedSketchLocation+"\n")
 }
 
 func TestPrototypesAdderSketchWithDosEol(t *testing.T) {

legacy/builder/test/try_build_of_problematic_sketch_test.go

@@ -202,13 +202,13 @@ func TestTryBuild042(t *testing.T) {
 }
 
 func makeDefaultContext() *types.Context {
+	builder.DebugPreprocessor = true
 	return &types.Context{
 		HardwareDirs:         paths.NewPathList(filepath.Join("..", "hardware"), "downloaded_hardware", "downloaded_board_manager_stuff"),
 		BuiltInToolsDirs:     paths.NewPathList("downloaded_tools"),
 		BuiltInLibrariesDirs: paths.New("downloaded_libraries"),
 		OtherLibrariesDirs:   paths.NewPathList("libraries"),
 		Verbose:              true,
-		DebugPreprocessor:    true,
 	}
 }
 

legacy/builder/types/context.go

@@ -29,7 +29,6 @@ import (
 	"github.com/arduino/arduino-cli/arduino/libraries/librariesmanager"
 	"github.com/arduino/arduino-cli/arduino/libraries/librariesresolver"
 	"github.com/arduino/arduino-cli/arduino/sketch"
-	"github.com/arduino/arduino-cli/legacy/builder/ctags"
 	rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1"
 	paths "github.com/arduino/go-paths-helper"
 	properties "github.com/arduino/go-properties-orderedmap"
@@ -119,14 +118,10 @@ type Context struct {
 	UseCachedLibrariesResolution bool
 
 	// C++ Parsing
-	LineOffset                  int
-	PrototypesSection           string
-	PrototypesLineWhereToInsert int
-	Prototypes                  []*ctags.Prototype
+	LineOffset int
 
 	// Verbosity settings
-	Verbose           bool
-	DebugPreprocessor bool
+	Verbose bool
 
 	// Dry run, only create progress map
 	Progress ProgressStruct