[skip-changelog] legacy: Arduino preprocess subroutine refactorization (part 3) (#2193)

* Converted CTagsRunner into a function

* Removed useless tasks from ctags_runner test

* Simplified ctags_runner test

* Removed some ctags related fields from builder context

The last piece in RunCTags:

 	parser := &ctags.CTagsParser{}
 	parser.Parse(ctagsStdout, sketch.MainFile)
 	parser.FixCLinkageTagsDeclarations()

 	prototypes, line := parser.GeneratePrototypes()
 	if line != -1 {
 		prototypesLineWhereToInsert = line
 	}

has been moved at the beginning of PrototypesAdder.
RunCTags now returns the output of ctags instead of `prototypes` and `line`.
This also allows to remove the context variables that keeps those
information.

* Simplified RunCTags / factored test subroutine

* Removed DebugPreprocessor from builder ctx

* Added executils.RunAndCaptureOutput

* Moved RunCTags out of legacy package

arduino/builder/preprocessor/ctags.go

+// This file is part of arduino-cli.
+//
+// Copyright 2023 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to license@arduino.cc.
+
+package preprocessor
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/arduino/arduino-cli/executils"
+	"github.com/arduino/arduino-cli/i18n"
+	"github.com/arduino/go-paths-helper"
+	"github.com/arduino/go-properties-orderedmap"
+	"github.com/pkg/errors"
+)
+
+var tr = i18n.Tr
+
+// RunCTags performs a run of ctags on the given source file. Returns the ctags output and the stderr contents.
+func RunCTags(sourceFile *paths.Path, buildProperties *properties.Map) ([]byte, []byte, error) {
+	ctagsBuildProperties := properties.NewMap()
+	ctagsBuildProperties.Set("tools.ctags.path", "{runtime.tools.ctags.path}")
+	ctagsBuildProperties.Set("tools.ctags.cmd.path", "{path}/ctags")
+	ctagsBuildProperties.Set("tools.ctags.pattern", `"{cmd.path}" -u --language-force=c++ -f - --c++-kinds=svpf --fields=KSTtzns --line-directives "{source_file}"`)
+	ctagsBuildProperties.Merge(buildProperties)
+	ctagsBuildProperties.Merge(ctagsBuildProperties.SubTree("tools").SubTree("ctags"))
+	ctagsBuildProperties.SetPath("source_file", sourceFile)
+
+	pattern := ctagsBuildProperties.Get("pattern")
+	if pattern == "" {
+		return nil, nil, errors.Errorf(tr("%s pattern is missing"), "ctags")
+	}
+
+	commandLine := ctagsBuildProperties.ExpandPropsInString(pattern)
+	parts, err := properties.SplitQuotedString(commandLine, `"'`, false)
+	if err != nil {
+		return nil, nil, err
+	}
+	proc, err := executils.NewProcess(nil, parts...)
+	if err != nil {
+		return nil, nil, err
+	}
+	stdout, stderr, err := proc.RunAndCaptureOutput(context.Background())
+
+	// Append ctags arguments to stderr
+	args := fmt.Sprintln(strings.Join(parts, " "))
+	stderr = append([]byte(args), stderr...)
+	return stdout, stderr, err
+}

executils/process.go

@@ -16,6 +16,7 @@
 package executils
 
 import (
+	"bytes"
 	"context"
 	"io"
 	"os"
@@ -174,3 +175,15 @@ func (p *Process) RunWithinContext(ctx context.Context) error {
 	}()
 	return p.Wait()
 }
+
+// RunAndCaptureOutput starts the specified command and waits for it to complete. If the given context
+// is canceled before the normal process termination, the process is killed. The standard output and
+// standard error of the process are captured and returned at process termination.
+func (p *Process) RunAndCaptureOutput(ctx context.Context) ([]byte, []byte, error) {
+	stdout := &bytes.Buffer{}
+	stderr := &bytes.Buffer{}
+	p.RedirectStdoutTo(stdout)
+	p.RedirectStderrTo(stderr)
+	err := p.RunWithinContext(ctx)
+	return stdout.Bytes(), stderr.Bytes(), err
+}

legacy/builder/container_add_prototypes.go

@@ -24,6 +24,7 @@ import (
 	"strings"
 
 	bldr "github.com/arduino/arduino-cli/arduino/builder"
+	"github.com/arduino/arduino-cli/arduino/builder/preprocessor"
 	"github.com/arduino/arduino-cli/arduino/sketch"
 	"github.com/arduino/arduino-cli/legacy/builder/types"
 	"github.com/arduino/go-paths-helper"
@@ -62,10 +63,18 @@ func PreprocessSketchWithCtags(ctx *types.Context) error {
 		ctx.SketchSourceAfterCppPreprocessing = filterSketchSource(ctx.Sketch, bytes.NewReader(src), false)
 	}
 
-	if err := (&CTagsRunner{Source: &ctx.SketchSourceAfterCppPreprocessing, TargetFileName: "sketch_merged.cpp"}).Run(ctx); err != nil {
-		return errors.WithStack(err)
+	if err := targetFilePath.WriteFile([]byte(ctx.SketchSourceAfterCppPreprocessing)); err != nil {
+		return err
+	}
+
+	ctagsStdout, ctagsStderr, err := preprocessor.RunCTags(targetFilePath, ctx.BuildProperties)
+	if ctx.Verbose {
+		ctx.WriteStderr(ctagsStderr)
+	}
+	if err != nil {
+		return err
 	}
-	ctx.SketchSourceAfterArduinoPreprocessing, ctx.PrototypesSection = PrototypesAdder(ctx.SketchSourceMerged, ctx.PrototypesLineWhereToInsert, ctx.LineOffset, ctx.Prototypes, ctx.DebugPreprocessor)
+	ctx.SketchSourceAfterArduinoPreprocessing = PrototypesAdder(ctx.Sketch, ctx.SketchSourceMerged, ctagsStdout, ctx.LineOffset)
 
 	if err := bldr.SketchSaveItemCpp(ctx.Sketch.MainFile, []byte(ctx.SketchSourceAfterArduinoPreprocessing), ctx.SketchBuildPath); err != nil {
 		return errors.WithStack(err)

legacy/builder/ctags_runner.go

-// This file is part of arduino-cli.
-//
-// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
-//
-// This software is released under the GNU General Public License version 3,
-// which covers the main part of arduino-cli.
-// The terms of this license can be found at:
-// https://www.gnu.org/licenses/gpl-3.0.en.html
-//
-// You can be released from the requirements of the above licenses by purchasing
-// a commercial license. Buying such a license is mandatory if you want to
-// modify or otherwise use the software for commercial activities involving the
-// Arduino software without disclosing the source code of your own applications.
-// To purchase a commercial license, send an email to license@arduino.cc.
-
-package builder
-
-import (
-	"os"
-	"os/exec"
-
-	"github.com/arduino/arduino-cli/legacy/builder/ctags"
-	"github.com/arduino/arduino-cli/legacy/builder/types"
-	"github.com/arduino/arduino-cli/legacy/builder/utils"
-	properties "github.com/arduino/go-properties-orderedmap"
-	"github.com/pkg/errors"
-)
-
-type CTagsRunner struct {
-	Source         *string
-	TargetFileName string
-
-	// Needed for unit-testing
-	CtagsOutput []byte
-}
-
-func (r *CTagsRunner) Run(ctx *types.Context) error {
-	source := *r.Source
-
-	preprocPath := ctx.PreprocPath
-	if err := preprocPath.MkdirAll(); err != nil {
-		return errors.WithStack(err)
-	}
-
-	ctagsTargetFilePath := preprocPath.Join(r.TargetFileName)
-	if err := ctagsTargetFilePath.WriteFile([]byte(source)); err != nil {
-		return errors.WithStack(err)
-	}
-
-	buildProperties := properties.NewMap()
-	buildProperties.Set("tools.ctags.path", "{runtime.tools.ctags.path}")
-	buildProperties.Set("tools.ctags.cmd.path", "{path}/ctags")
-	buildProperties.Set("tools.ctags.pattern", `"{cmd.path}" -u --language-force=c++ -f - --c++-kinds=svpf --fields=KSTtzns --line-directives "{source_file}"`)
-	buildProperties.Merge(ctx.BuildProperties)
-	buildProperties.Merge(buildProperties.SubTree("tools").SubTree("ctags"))
-	buildProperties.SetPath("source_file", ctagsTargetFilePath)
-
-	pattern := buildProperties.Get("pattern")
-	if pattern == "" {
-		return errors.Errorf(tr("%s pattern is missing"), "ctags")
-	}
-
-	commandLine := buildProperties.ExpandPropsInString(pattern)
-	parts, err := properties.SplitQuotedString(commandLine, `"'`, false)
-	if err != nil {
-		return errors.WithStack(err)
-	}
-	command := exec.Command(parts[0], parts[1:]...)
-	command.Env = append(os.Environ(), ctx.PackageManager.GetEnvVarsForSpawnedProcess()...)
-
-	ctagsOutput, _, err := utils.ExecCommand(ctx, command, utils.Capture /* stdout */, utils.ShowIfVerbose /* stderr */)
-	if err != nil {
-		return errors.WithStack(err)
-	}
-
-	parser := &ctags.CTagsParser{}
-	parser.Parse(ctagsOutput, ctx.Sketch.MainFile)
-	parser.FixCLinkageTagsDeclarations()
-
-	protos, line := parser.GeneratePrototypes()
-	if line != -1 {
-		ctx.PrototypesLineWhereToInsert = line
-	}
-	ctx.Prototypes = protos
-
-	// Needed for unit-testing
-	r.CtagsOutput = ctagsOutput
-	return nil
-}

legacy/builder/prototypes_adder.go

@@ -20,25 +20,37 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/arduino/arduino-cli/arduino/sketch"
 	"github.com/arduino/arduino-cli/legacy/builder/constants"
 	"github.com/arduino/arduino-cli/legacy/builder/ctags"
 	"github.com/arduino/arduino-cli/legacy/builder/utils"
 )
 
-func PrototypesAdder(source string, firstFunctionLine, lineOffset int, prototypes []*ctags.Prototype, debugOutput bool) (preprocessedSource, prototypeSection string) {
+var DebugPreprocessor bool
+
+func PrototypesAdder(sketch *sketch.Sketch, source string, ctagsStdout []byte, lineOffset int) string {
+	parser := &ctags.CTagsParser{}
+	parser.Parse(ctagsStdout, sketch.MainFile)
+	parser.FixCLinkageTagsDeclarations()
+
+	prototypes, firstFunctionLine := parser.GeneratePrototypes()
+	if firstFunctionLine == -1 {
+		firstFunctionLine = 0
+	}
+
 	source = strings.Replace(source, "\r\n", "\n", -1)
 	source = strings.Replace(source, "\r", "\n", -1)
 	sourceRows := strings.Split(source, "\n")
 	if isFirstFunctionOutsideOfSource(firstFunctionLine, sourceRows) {
-		return
+		return ""
 	}
 
 	insertionLine := firstFunctionLine + lineOffset - 1
 	firstFunctionChar := len(strings.Join(sourceRows[:insertionLine], "\n")) + 1
-	prototypeSection = composePrototypeSection(firstFunctionLine, prototypes)
-	preprocessedSource = source[:firstFunctionChar] + prototypeSection + source[firstFunctionChar:]
+	prototypeSection := composePrototypeSection(firstFunctionLine, prototypes)
+	preprocessedSource := source[:firstFunctionChar] + prototypeSection + source[firstFunctionChar:]
 
-	if debugOutput {
+	if DebugPreprocessor {
 		fmt.Println("#PREPROCESSED SOURCE")
 		prototypesRows := strings.Split(prototypeSection, "\n")
 		prototypesRows = prototypesRows[:len(prototypesRows)-1]
@@ -53,7 +65,7 @@ func PrototypesAdder(source string, firstFunctionLine, lineOffset int, prototype
 		}
 		fmt.Println("#END OF PREPROCESSED SOURCE")
 	}
-	return
+	return preprocessedSource
 }
 
 func composePrototypeSection(line int, prototypes []*ctags.Prototype) string {

legacy/builder/test/try_build_of_problematic_sketch_test.go

@@ -202,13 +202,13 @@ func TestTryBuild042(t *testing.T) {
 }
 
 func makeDefaultContext() *types.Context {
+	builder.DebugPreprocessor = true
 	return &types.Context{
 		HardwareDirs:         paths.NewPathList(filepath.Join("..", "hardware"), "downloaded_hardware", "downloaded_board_manager_stuff"),
 		BuiltInToolsDirs:     paths.NewPathList("downloaded_tools"),
 		BuiltInLibrariesDirs: paths.New("downloaded_libraries"),
 		OtherLibrariesDirs:   paths.NewPathList("libraries"),
 		Verbose:              true,
-		DebugPreprocessor:    true,
 	}
 }
 

legacy/builder/types/context.go

@@ -29,7 +29,6 @@ import (
 	"github.com/arduino/arduino-cli/arduino/libraries/librariesmanager"
 	"github.com/arduino/arduino-cli/arduino/libraries/librariesresolver"
 	"github.com/arduino/arduino-cli/arduino/sketch"
-	"github.com/arduino/arduino-cli/legacy/builder/ctags"
 	rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1"
 	paths "github.com/arduino/go-paths-helper"
 	properties "github.com/arduino/go-properties-orderedmap"
@@ -119,14 +118,10 @@ type Context struct {
 	UseCachedLibrariesResolution bool
 
 	// C++ Parsing
-	LineOffset                  int
-	PrototypesSection           string
-	PrototypesLineWhereToInsert int
-	Prototypes                  []*ctags.Prototype
+	LineOffset int
 
 	// Verbosity settings
-	Verbose           bool
-	DebugPreprocessor bool
+	Verbose bool
 
 	// Dry run, only create progress map
 	Progress ProgressStruct