Added 'source override' feature to compile command. (#1099)

This feature allows to selectively "override" the content of a sketch.
Overridden files will not be read from disk as usual but fetched directly
from gRPC paramaters (if the cli is running as daemon) or from a .json
file containing the new source code (if running from command line).

arduino/builder/sketch.go

@@ -216,12 +216,23 @@ func SketchLoad(sketchPath, buildPath string) (*sketch.Sketch, error) {
 }
 
 // SketchMergeSources merges all the source files included in a sketch
-func SketchMergeSources(sketch *sketch.Sketch) (int, string, error) {
+func SketchMergeSources(sk *sketch.Sketch, overrides map[string]string) (int, string, error) {
 	lineOffset := 0
 	mergedSource := ""
 
+	getSource := func(i *sketch.Item) (string, error) {
+		path, err := filepath.Rel(sk.LocationPath, i.Path)
+		if err != nil {
+			return "", errors.Wrap(err, "unable to compute relative path to the sketch for the item")
+		}
+		if override, ok := overrides[path]; ok {
+			return override, nil
+		}
+		return i.GetSourceStr()
+	}
+
 	// add Arduino.h inclusion directive if missing
-	mainSrc, err := sketch.MainFile.GetSourceStr()
+	mainSrc, err := getSource(sk.MainFile)
 	if err != nil {
 		return 0, "", err
 	}
@@ -230,12 +241,12 @@ func SketchMergeSources(sketch *sketch.Sketch) (int, string, error) {
 		lineOffset++
 	}
 
-	mergedSource += "#line 1 " + QuoteCppString(sketch.MainFile.Path) + "\n"
+	mergedSource += "#line 1 " + QuoteCppString(sk.MainFile.Path) + "\n"
 	mergedSource += mainSrc + "\n"
 	lineOffset++
 
-	for _, item := range sketch.OtherSketchFiles {
-		src, err := item.GetSourceStr()
+	for _, item := range sk.OtherSketchFiles {
+		src, err := getSource(item)
 		if err != nil {
 			return 0, "", err
 		}
@@ -248,7 +259,7 @@ func SketchMergeSources(sketch *sketch.Sketch) (int, string, error) {
 
 // SketchCopyAdditionalFiles copies the additional files for a sketch to the
 // specified destination directory.
-func SketchCopyAdditionalFiles(sketch *sketch.Sketch, destPath string) error {
+func SketchCopyAdditionalFiles(sketch *sketch.Sketch, destPath string, overrides map[string]string) error {
 	if err := os.MkdirAll(destPath, os.FileMode(0755)); err != nil {
 		return errors.Wrap(err, "unable to create a folder to save the sketch files")
 	}
@@ -265,7 +276,20 @@ func SketchCopyAdditionalFiles(sketch *sketch.Sketch, destPath string) error {
 			return errors.Wrap(err, "unable to create the folder containing the item")
 		}
 
-		err = writeIfDifferent(item.Path, targetPath)
+		var sourceBytes []byte
+		if override, ok := overrides[relpath]; ok {
+			// use override source
+			sourceBytes = []byte(override)
+		} else {
+			// read the source file
+			s, err := item.GetSourceBytes()
+			if err != nil {
+				return errors.Wrap(err, "unable to read contents of the source item")
+			}
+			sourceBytes = s
+		}
+
+		err = writeIfDifferent(sourceBytes, targetPath)
 		if err != nil {
 			return errors.Wrap(err, "unable to write to destination file")
 		}
@@ -274,18 +298,12 @@ func SketchCopyAdditionalFiles(sketch *sketch.Sketch, destPath string) error {
 	return nil
 }
 
-func writeIfDifferent(sourcePath, destPath string) error {
-	// read the source file
-	newbytes, err := ioutil.ReadFile(sourcePath)
-	if err != nil {
-		return errors.Wrap(err, "unable to read contents of the source item")
-	}
-
+func writeIfDifferent(source []byte, destPath string) error {
 	// check whether the destination file exists
-	_, err = os.Stat(destPath)
+	_, err := os.Stat(destPath)
 	if os.IsNotExist(err) {
 		// write directly
-		return ioutil.WriteFile(destPath, newbytes, os.FileMode(0644))
+		return ioutil.WriteFile(destPath, source, os.FileMode(0644))
 	}
 
 	// read the destination file if it ex
@@ -295,8 +313,8 @@ func writeIfDifferent(sourcePath, destPath string) error {
 	}
 
 	// overwrite if contents are different
-	if bytes.Compare(existingBytes, newbytes) != 0 {
-		return ioutil.WriteFile(destPath, newbytes, os.FileMode(0644))
+	if bytes.Compare(existingBytes, source) != 0 {
+		return ioutil.WriteFile(destPath, source, os.FileMode(0644))
 	}
 
 	// source and destination are the same, don't write anything

arduino/builder/sketch_test.go

@@ -180,7 +180,7 @@ func TestMergeSketchSources(t *testing.T) {
 		t.Fatalf("unable to read golden file %s: %v", mergedPath, err)
 	}
 
-	offset, source, err := builder.SketchMergeSources(s)
+	offset, source, err := builder.SketchMergeSources(s, nil)
 	require.Nil(t, err)
 	require.Equal(t, 2, offset)
 	require.Equal(t, string(mergedBytes), source)
@@ -192,7 +192,7 @@ func TestMergeSketchSourcesArduinoIncluded(t *testing.T) {
 	require.NotNil(t, s)
 
 	// ensure not to include Arduino.h when it's already there
-	_, source, err := builder.SketchMergeSources(s)
+	_, source, err := builder.SketchMergeSources(s, nil)
 	require.Nil(t, err)
 	require.Equal(t, 1, strings.Count(source, "<Arduino.h>"))
 }
@@ -208,7 +208,7 @@ func TestCopyAdditionalFiles(t *testing.T) {
 
 	// copy the sketch over, create a fake main file we don't care about it
 	// but we need it for `SketchLoad` to succeed later
-	err = builder.SketchCopyAdditionalFiles(s1, tmp)
+	err = builder.SketchCopyAdditionalFiles(s1, tmp, nil)
 	require.Nil(t, err)
 	fakeIno := filepath.Join(tmp, fmt.Sprintf("%s.ino", filepath.Base(tmp)))
 	require.Nil(t, ioutil.WriteFile(fakeIno, []byte{}, os.FileMode(0644)))
@@ -223,7 +223,7 @@ func TestCopyAdditionalFiles(t *testing.T) {
 	require.Nil(t, err)
 
 	// copy again
-	err = builder.SketchCopyAdditionalFiles(s1, tmp)
+	err = builder.SketchCopyAdditionalFiles(s1, tmp, nil)
 	require.Nil(t, err)
 
 	// verify file hasn't changed

cli/compile/compile.go

@@ -18,6 +18,7 @@ package compile
 import (
 	"bytes"
 	"context"
+	"encoding/json"
 	"os"
 
 	"github.com/arduino/arduino-cli/cli/feedback"
@@ -55,6 +56,7 @@ var (
 	clean                   bool     // Cleanup the build folder and do not use any cached build
 	exportBinaries          bool     // Copies compiled binaries to sketch folder when true
 	compilationDatabaseOnly bool     // Only create compilation database without actually compiling
+	sourceOverrides         string   // Path to a .json file that contains a set of replacements of the sketch source code.
 )
 
 // NewCommand created a new `compile` command
@@ -102,6 +104,8 @@ func NewCommand() *cobra.Command {
 	// This must be done because the value is set when the binding is accessed from viper. Accessing from cobra would only
 	// read the value if the flag is set explicitly by the user.
 	command.Flags().BoolP("export-binaries", "e", false, "If set built binaries will be exported to the sketch folder.")
+	command.Flags().StringVar(&sourceOverrides, "source-override", "", "Optional. Path to a .json file that contains a set of replacements of the sketch source code.")
+	command.Flag("source-override").Hidden = true
 
 	configuration.Settings.BindPFlag("sketch.always_export_binaries", command.Flags().Lookup("export-binaries"))
 
@@ -128,6 +132,23 @@ func run(cmd *cobra.Command, args []string) {
 	// the config file and the env vars.
 	exportBinaries = configuration.Settings.GetBool("sketch.always_export_binaries")
 
+	var overrides map[string]string
+	if sourceOverrides != "" {
+		data, err := paths.New(sourceOverrides).ReadFile()
+		if err != nil {
+			feedback.Errorf("Error opening source code overrides data file: %v", err)
+			os.Exit(errorcodes.ErrGeneric)
+		}
+		var o struct {
+			Overrides map[string]string `json:"overrides"`
+		}
+		if err := json.Unmarshal(data, &o); err != nil {
+			feedback.Errorf("Error: invalid source code overrides data file: %v", err)
+			os.Exit(errorcodes.ErrGeneric)
+		}
+		overrides = o.Overrides
+	}
+
 	compileReq := &rpc.CompileReq{
 		Instance:                      inst,
 		Fqbn:                          fqbn,
@@ -147,6 +168,7 @@ func run(cmd *cobra.Command, args []string) {
 		Clean:                         clean,
 		ExportBinaries:                exportBinaries,
 		CreateCompilationDatabaseOnly: compilationDatabaseOnly,
+		SourceOverride:                overrides,
 	}
 	compileOut := new(bytes.Buffer)
 	compileErr := new(bytes.Buffer)

commands/compile/compile.go

@@ -192,6 +192,8 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W
 	builderCtx.Clean = req.GetClean()
 	builderCtx.OnlyUpdateCompilationDatabase = req.GetCreateCompilationDatabaseOnly()
 
+	builderCtx.SourceOverride = req.GetSourceOverride()
+
 	// Use defer() to create an rpc.CompileResp with all the information available at the
 	// moment of return.
 	defer func() {

legacy/builder/container_merge_copy_sketch_files.go

@@ -28,7 +28,7 @@ func (s *ContainerMergeCopySketchFiles) Run(ctx *types.Context) error {
 	if sk == nil {
 		return errors.New("unable to convert legacy sketch to the new type")
 	}
-	offset, source, err := bldr.SketchMergeSources(sk)
+	offset, source, err := bldr.SketchMergeSources(sk, ctx.SourceOverride)
 	if err != nil {
 		return err
 	}
@@ -39,7 +39,7 @@ func (s *ContainerMergeCopySketchFiles) Run(ctx *types.Context) error {
 		return errors.WithStack(err)
 	}
 
-	if err := bldr.SketchCopyAdditionalFiles(sk, ctx.SketchBuildPath.String()); err != nil {
+	if err := bldr.SketchCopyAdditionalFiles(sk, ctx.SketchBuildPath.String(), ctx.SourceOverride); err != nil {
 		return errors.WithStack(err)
 	}
 

legacy/builder/types/context.go

@@ -169,6 +169,11 @@ type Context struct {
 	CompilationDatabase *builder.CompilationDatabase
 	// Set to true to skip build and produce only Compilation Database
 	OnlyUpdateCompilationDatabase bool
+
+	// Source code overrides (filename -> content map).
+	// The provided source data is used instead of reading it from disk.
+	// The keys of the map are paths relative to sketch folder.
+	SourceOverride map[string]string
 }
 
 // ExecutableSectionSize represents a section of the executable output file

rpc/commands/compile.proto

@@ -42,6 +42,7 @@ message CompileReq {
   bool clean = 19; // Optional: cleanup the build folder and do not use any previously cached build
   bool export_binaries = 20; // When set to `true` the compiled binary will be copied to the export directory.
   bool create_compilation_database_only = 21; // When set to `true` only the compilation database will be produced and no actual build will be performed.
+  map<string, string> source_override = 22; // This map (source file -> new content) let the builder use the provided content instead of reading the corresponding file on disk. This is useful for IDE that have unsaved changes in memory. The path must be relative to the sketch directory. Only files from the sketch are allowed.
 }
 
 message CompileResp {