Merge pull request #265 from arduino/massi/sketch

Turn Sketch commands into builder API functions

arduino/builder/sketch.go

+// This file is part of arduino-cli.
+//
+// Copyright 2019 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to modify or
+// otherwise use the software for commercial activities involving the Arduino
+// software without disclosing the source code of your own applications. To purchase
+// a commercial license, send an email to license@arduino.cc.
+
+package builder
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+
+	"github.com/arduino/arduino-cli/arduino/globals"
+	"github.com/arduino/arduino-cli/arduino/sketch"
+
+	"github.com/pkg/errors"
+)
+
+var includesArduinoH = regexp.MustCompile(`(?m)^\s*#\s*include\s*[<\"]Arduino\.h[>\"]`)
+
+// QuoteCppString returns the given string as a quoted string for use with the C
+// preprocessor. This adds double quotes around it and escapes any
+// double quotes and backslashes in the string.
+func QuoteCppString(str string) string {
+	str = strings.Replace(str, "\\", "\\\\", -1)
+	str = strings.Replace(str, "\"", "\\\"", -1)
+	return "\"" + str + "\""
+}
+
+// SaveSketchItemCpp saves a preprocessed .cpp sketch file on disk
+func SaveSketchItemCpp(item *sketch.Item, buildPath string) error {
+
+	sketchName := filepath.Base(item.Path)
+
+	if err := os.MkdirAll(buildPath, os.FileMode(0755)); err != nil {
+		return errors.Wrap(err, "unable to create a folder to save the sketch")
+	}
+
+	destFile := filepath.Join(buildPath, sketchName+".cpp")
+
+	if err := ioutil.WriteFile(destFile, item.Source, os.FileMode(0644)); err != nil {
+		return errors.Wrap(err, "unable to save the sketch on disk")
+	}
+
+	return nil
+}
+
+// LoadSketch collects all the files composing a sketch.
+// The parameter `sketchPath` holds a path pointing to a single sketch file or a sketch folder,
+// the path must be absolute.
+func LoadSketch(sketchPath, buildPath string) (*sketch.Sketch, error) {
+	stat, err := os.Stat(sketchPath)
+	if err != nil {
+		return nil, errors.Wrap(err, "unable to stat Sketch location")
+	}
+
+	var sketchFolder, mainSketchFile string
+
+	// if a sketch folder was passed, save the parent and point sketchPath to the main .ino file
+	if stat.IsDir() {
+		sketchFolder = sketchPath
+		mainSketchFile = filepath.Join(sketchPath, stat.Name()+".ino")
+		// in the case a dir was passed, ensure the main file exists and is readable
+		f, err := os.Open(mainSketchFile)
+		if err != nil {
+			return nil, errors.Wrap(err, "unable to find the main sketch file")
+		}
+		f.Close()
+	} else {
+		sketchFolder = filepath.Dir(sketchPath)
+		mainSketchFile = sketchPath
+	}
+
+	// collect all the sketch files
+	var files []string
+	err = filepath.Walk(sketchFolder, func(path string, info os.FileInfo, err error) error {
+		// ignore hidden files and skip hidden directories
+		if strings.HasPrefix(info.Name(), ".") {
+			if info.IsDir() {
+				return filepath.SkipDir
+			}
+			return nil
+		}
+
+		// skip legacy SCM directories
+		if info.IsDir() && strings.HasPrefix(info.Name(), "CVS") || strings.HasPrefix(info.Name(), "RCS") {
+			return filepath.SkipDir
+		}
+
+		// ignore directory entries
+		if info.IsDir() {
+			return nil
+		}
+
+		// ignore if file extension doesn't match
+		ext := strings.ToLower(filepath.Ext(path))
+		_, isMain := globals.MainFileValidExtensions[ext]
+		_, isAdditional := globals.AdditionalFileValidExtensions[ext]
+		if !(isMain || isAdditional) {
+			return nil
+		}
+
+		// check if file is readable
+		f, err := os.Open(path)
+		if err != nil {
+			return nil
+		}
+		f.Close()
+
+		// collect the file
+		files = append(files, path)
+
+		// done
+		return nil
+	})
+
+	if err != nil {
+		return nil, errors.Wrap(err, "there was an error while collecting the sketch files")
+	}
+
+	return sketch.New(sketchFolder, mainSketchFile, buildPath, files)
+}
+
+// MergeSketchSources merges all the source files included in a sketch
+func MergeSketchSources(sketch *sketch.Sketch) (int, string) {
+	lineOffset := 0
+	mergedSource := ""
+
+	// add Arduino.h inclusion directive if missing
+	if !includesArduinoH.MatchString(sketch.MainFile.GetSourceStr()) {
+		mergedSource += "#include <Arduino.h>\n"
+		lineOffset++
+	}
+
+	mergedSource += "#line 1 " + QuoteCppString(sketch.MainFile.Path) + "\n"
+	mergedSource += sketch.MainFile.GetSourceStr() + "\n"
+	lineOffset++
+
+	for _, item := range sketch.OtherSketchFiles {
+		mergedSource += "#line 1 " + QuoteCppString(item.Path) + "\n"
+		mergedSource += item.GetSourceStr() + "\n"
+	}
+
+	return lineOffset, mergedSource
+}

arduino/builder/sketch_test.go

+// This file is part of arduino-cli.
+//
+// Copyright 2019 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to modify or
+// otherwise use the software for commercial activities involving the Arduino
+// software without disclosing the source code of your own applications. To purchase
+// a commercial license, send an email to license@arduino.cc.
+
+package builder_test
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/arduino/arduino-cli/arduino/builder"
+	"github.com/arduino/arduino-cli/arduino/sketch"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSaveSketch(t *testing.T) {
+	sketchName := t.Name() + ".ino"
+	outName := sketchName + ".cpp"
+	sketchFile := filepath.Join("testdata", sketchName)
+	tmp := tmpDirOrDie()
+	defer os.RemoveAll(tmp)
+	source, err := ioutil.ReadFile(sketchFile)
+	if err != nil {
+		t.Fatalf("unable to read golden file %s: %v", sketchFile, err)
+	}
+
+	builder.SaveSketchItemCpp(&sketch.Item{Path: sketchName, Source: source}, tmp)
+
+	out, err := ioutil.ReadFile(filepath.Join(tmp, outName))
+	if err != nil {
+		t.Fatalf("unable to read output file %s: %v", outName, err)
+	}
+
+	assert.Equal(t, source, out)
+}
+
+func TestLoadSketchFolder(t *testing.T) {
+	// pass the path to the sketch folder
+	sketchPath := filepath.Join("testdata", t.Name())
+	mainFilePath := filepath.Join(sketchPath, t.Name()+".ino")
+	s, err := builder.LoadSketch(sketchPath, "")
+	assert.Nil(t, err)
+	assert.NotNil(t, s)
+	assert.Equal(t, mainFilePath, s.MainFile.Path)
+	assert.Equal(t, sketchPath, s.LocationPath)
+	assert.Len(t, s.OtherSketchFiles, 2)
+	assert.Equal(t, "old.pde", filepath.Base(s.OtherSketchFiles[0].Path))
+	assert.Equal(t, "other.ino", filepath.Base(s.OtherSketchFiles[1].Path))
+	assert.Len(t, s.AdditionalFiles, 3)
+	assert.Equal(t, "header.h", filepath.Base(s.AdditionalFiles[0].Path))
+	assert.Equal(t, "s_file.S", filepath.Base(s.AdditionalFiles[1].Path))
+	assert.Equal(t, "helper.h", filepath.Base(s.AdditionalFiles[2].Path))
+
+	// pass the path to the main file
+	sketchPath = mainFilePath
+	s, err = builder.LoadSketch(sketchPath, "")
+	assert.Nil(t, err)
+	assert.NotNil(t, s)
+	assert.Equal(t, mainFilePath, s.MainFile.Path)
+	assert.Len(t, s.OtherSketchFiles, 2)
+	assert.Equal(t, "old.pde", filepath.Base(s.OtherSketchFiles[0].Path))
+	assert.Equal(t, "other.ino", filepath.Base(s.OtherSketchFiles[1].Path))
+	assert.Len(t, s.AdditionalFiles, 3)
+	assert.Equal(t, "header.h", filepath.Base(s.AdditionalFiles[0].Path))
+	assert.Equal(t, "s_file.S", filepath.Base(s.AdditionalFiles[1].Path))
+	assert.Equal(t, "helper.h", filepath.Base(s.AdditionalFiles[2].Path))
+}
+
+func TestLoadSketchFolderWrongMain(t *testing.T) {
+	sketchPath := filepath.Join("testdata", t.Name())
+	_, err := builder.LoadSketch(sketchPath, "")
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "unable to find the main sketch file")
+
+	_, err = builder.LoadSketch("does/not/exist", "")
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "no such file or directory")
+}
+
+func TestMergeSketchSources(t *testing.T) {
+	// borrow the sketch from TestLoadSketchFolder to avoid boilerplate
+	s, err := builder.LoadSketch(filepath.Join("testdata", "TestLoadSketchFolder"), "")
+	assert.Nil(t, err)
+	assert.NotNil(t, s)
+
+	// load expected result
+	mergedPath := filepath.Join("testdata", t.Name()+".txt")
+	mergedBytes, err := ioutil.ReadFile(mergedPath)
+	if err != nil {
+		t.Fatalf("unable to read golden file %s: %v", mergedPath, err)
+	}
+
+	offset, source := builder.MergeSketchSources(s)
+	assert.Equal(t, 2, offset)
+	assert.Equal(t, string(mergedBytes), source)
+}
+
+func TestMergeSketchSourcesArduinoIncluded(t *testing.T) {
+	s, err := builder.LoadSketch(filepath.Join("testdata", t.Name()), "")
+	assert.Nil(t, err)
+	assert.NotNil(t, s)
+
+	// ensure not to include Arduino.h when it's already there
+	_, source := builder.MergeSketchSources(s)
+	assert.Equal(t, 1, strings.Count(source, "<Arduino.h>"))
+}

arduino/builder/testdata/TestLoadSketchFolder/.#sketch.ino

+void setup()
+void loop) }
\ No newline at end of file

arduino/builder/testdata/TestLoadSketchFolder/TestLoadSketchFolder.ino

+void setup() {
+
+}
+
+void loop() {
+
+}
\ No newline at end of file

arduino/builder/testdata/TestLoadSketchFolder/header.h

+#define FOO "BAR"
\ No newline at end of file

arduino/builder/testdata/TestLoadSketchFolder/other.ino

+String hello() {
+  return "world";
+}
\ No newline at end of file

arduino/builder/testdata/TestLoadSketchFolder/src/dont_load_me.ino

+#include <testlib4.h>
+#error "Whattya looking at?"

arduino/builder/testdata/TestMergeSketchSources.txt

+#include <Arduino.h>
+#line 1 "testdata/TestLoadSketchFolder/TestLoadSketchFolder.ino"
+void setup() {
+
+}
+
+void loop() {
+
+}
+#line 1 "testdata/TestLoadSketchFolder/old.pde"
+
+#line 1 "testdata/TestLoadSketchFolder/other.ino"
+String hello() {
+  return "world";
+}

arduino/builder/testdata/TestMergeSketchSourcesArduinoIncluded/TestMergeSketchSourcesArduinoIncluded.ino

+// a comment
+
+   #    include		<Arduino.h>
+
+void setup() {
+}
+
+void loop() {
+}

arduino/builder/testdata/TestSaveSketch.ino

+#include <Bridge.h>
+#include <IRremote.h>
+#include <IRremoteInt.h>
+
+void setup() {}
+void loop() {}
\ No newline at end of file

arduino/globals/globals.go

+// This file is part of arduino-cli.
+//
+// Copyright 2019 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to modify or
+// otherwise use the software for commercial activities involving the Arduino
+// software without disclosing the source code of your own applications. To purchase
+// a commercial license, send an email to license@arduino.cc.
+
+package globals
+
+var (
+	empty struct{}
+
+	// MainFileValidExtensions lists valid extensions for a sketch file
+	MainFileValidExtensions = map[string]struct{}{
+		".ino": empty,
+		".pde": empty,
+	}
+
+	// AdditionalFileValidExtensions lists any file extension the builder considers as valid
+	AdditionalFileValidExtensions = map[string]struct{}{
+		".h":   empty,
+		".c":   empty,
+		".hpp": empty,
+		".hh":  empty,
+		".cpp": empty,
+		".s":   empty,
+	}
+
+	// SourceFilesValidExtensions lists valid extensions for source files (no headers)
+	SourceFilesValidExtensions = map[string]struct{}{
+		".c":   empty,
+		".cpp": empty,
+		".s":   empty,
+	}
+)

arduino/sketch/sketch.go

+// This file is part of arduino-cli.
+//
+// Copyright 2019 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to modify or
+// otherwise use the software for commercial activities involving the Arduino
+// software without disclosing the source code of your own applications. To purchase
+// a commercial license, send an email to license@arduino.cc.
+
+package sketch
+
+import (
+	"io/ioutil"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"github.com/arduino/arduino-cli/arduino/globals"
+	"github.com/pkg/errors"
+)
+
+// Item holds the source and the path for a single sketch file
+type Item struct {
+	Path   string
+	Source []byte
+}
+
+// NewItem reads the source code for a sketch item and returns an
+// Item instance
+func NewItem(itemPath string) (*Item, error) {
+	// read the file
+	source, err := ioutil.ReadFile(itemPath)
+	if err != nil {
+		return nil, errors.Wrap(err, "error reading source file")
+	}
+
+	return &Item{itemPath, source}, nil
+}
+
+// GetSourceStr returns the Source contents in string format
+func (i *Item) GetSourceStr() string {
+	return string(i.Source)
+}
+
+// ItemByPath implements sort.Interface for []Item based on
+// lexicographic order of the path string.
+type ItemByPath []*Item
+
+func (ibn ItemByPath) Len() int           { return len(ibn) }
+func (ibn ItemByPath) Swap(i, j int)      { ibn[i], ibn[j] = ibn[j], ibn[i] }
+func (ibn ItemByPath) Less(i, j int) bool { return ibn[i].Path < ibn[j].Path }
+
+// Sketch holds all the files composing a sketch
+type Sketch struct {
+	MainFile         *Item
+	LocationPath     string
+	OtherSketchFiles []*Item
+	AdditionalFiles  []*Item
+}
+
+// New creates an Sketch instance by reading all the files composing a sketch and grouping them
+// by file type.
+func New(sketchFolderPath, mainFilePath, buildPath string, allFilesPaths []string) (*Sketch, error) {
+	var mainFile *Item
+
+	// read all the sketch contents and create sketch Items
+	pathToItem := make(map[string]*Item)
+	for _, p := range allFilesPaths {
+		// create an Item
+		item, err := NewItem(p)
+		if err != nil {
+			return nil, errors.Wrap(err, "error creating the sketch")
+		}
+
+		if p == mainFilePath {
+			// store the main sketch file
+			mainFile = item
+		} else {
+			// map the file path to sketch.Item
+			pathToItem[p] = item
+		}
+	}
+
+	// organize the Items
+	additionalFiles := []*Item{}
+	otherSketchFiles := []*Item{}
+	for p, item := range pathToItem {
+		ext := strings.ToLower(filepath.Ext(p))
+		if _, found := globals.MainFileValidExtensions[ext]; found {
+			// item is a valid main file, see if it's stored at the
+			// sketch root and ignore if it's not.
+			if filepath.Dir(p) == sketchFolderPath {
+				otherSketchFiles = append(otherSketchFiles, item)
+			}
+		} else if _, found := globals.AdditionalFileValidExtensions[ext]; found {
+			// item is a valid sketch file, grab it only if the buildPath is empty
+			// or the file is within the buildPath
+			if buildPath == "" || !strings.Contains(filepath.Dir(p), buildPath) {
+				additionalFiles = append(additionalFiles, item)
+			}
+		} else {
+			return nil, errors.Errorf("unknown sketch file extension '%s'", ext)
+		}
+	}
+
+	sort.Sort(ItemByPath(additionalFiles))
+	sort.Sort(ItemByPath(otherSketchFiles))
+
+	return &Sketch{
+		MainFile:         mainFile,
+		LocationPath:     sketchFolderPath,
+		OtherSketchFiles: otherSketchFiles,
+		AdditionalFiles:  additionalFiles,
+	}, nil
+}

arduino/sketch/sketch_test.go

+// This file is part of arduino-cli.
+//
+// Copyright 2019 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to modify or
+// otherwise use the software for commercial activities involving the Arduino
+// software without disclosing the source code of your own applications. To purchase
+// a commercial license, send an email to license@arduino.cc.
+
+package sketch_test
+
+import (
+	"path/filepath"
+	"sort"
+	"testing"
+
+	"github.com/arduino/arduino-cli/arduino/sketch"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNewItem(t *testing.T) {
+	sketchItem := filepath.Join("testdata", t.Name()+".ino")
+	item, err := sketch.NewItem(sketchItem)
+	assert.Nil(t, err)
+	assert.Equal(t, sketchItem, item.Path)
+	assert.Equal(t, []byte(`#include <testlib.h>`), item.Source)
+	assert.Equal(t, "#include <testlib.h>", item.GetSourceStr())
+
+	item, err = sketch.NewItem("doesnt/exist")
+	assert.Nil(t, item)
+	assert.NotNil(t, err)
+}
+
+func TestSort(t *testing.T) {
+	items := []*sketch.Item{
+		&sketch.Item{"foo", nil},
+		&sketch.Item{"baz", nil},
+		&sketch.Item{"bar", nil},
+	}
+
+	sort.Sort(sketch.ItemByPath(items))
+
+	assert.Equal(t, "bar", items[0].Path)
+	assert.Equal(t, "baz", items[1].Path)
+	assert.Equal(t, "foo", items[2].Path)
+}
+
+func TestNew(t *testing.T) {
+	sketchFolderPath := filepath.Join("testdata", t.Name())
+	mainFilePath := filepath.Join(sketchFolderPath, t.Name()+".ino")
+	otherFile := filepath.Join(sketchFolderPath, "other.cpp")
+	allFilesPaths := []string{
+		mainFilePath,
+		otherFile,
+	}
+
+	sketch, err := sketch.New(sketchFolderPath, mainFilePath, "", allFilesPaths)
+	assert.Nil(t, err)
+	assert.Equal(t, mainFilePath, sketch.MainFile.Path)
+	assert.Equal(t, sketchFolderPath, sketch.LocationPath)
+	assert.Len(t, sketch.OtherSketchFiles, 0)
+	assert.Len(t, sketch.AdditionalFiles, 1)
+}

arduino/sketch/testdata/TestNew/TestNew.ino

+#include <testlib1.h>
+#include "subfolder/other.h"
+#include "src/subfolder/other.h"
+
+MyClass myClass;
+
+void setup() {
+    myClass.init ( &Serial );
+}
+
+void loop() {
+}

arduino/sketch/testdata/TestNew/other.cpp

+#include <Arduino.h> // Arduino 1.0
+#include <testlib2.h>
+
+#include "other.h"
+
+MyClass::MyClass() {
+}
+
+void MyClass::init ( Stream *stream ) {
+    controllerStream = stream;
+}

arduino/sketch/testdata/TestNewItem.ino

+#include <testlib.h>
\ No newline at end of file

go.sum

@@ -2,6 +2,7 @@ bou.ke/monkey v1.0.1 h1:zEMLInw9xvNakzUUPjfS4Ds6jYPqCFx3m7bRmG5NH2U=
 bou.ke/monkey v1.0.1/go.mod h1:FgHuK96Rv2Nlf+0u1OOVDpCMdsWyOFmeeketDHE7LIg=
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/arduino/arduino v0.0.0-20190521072917-00a7546fb43e h1:5TJufBg+nZAFruNEALBVHDPRMtMDbRg7Hox/WA4ZI40=
 github.com/arduino/board-discovery v0.0.0-20180823133458-1ba29327fb0c h1:agh2JT96G8egU7FEb13L4dq3fnCN7lxXhJ86t69+W7s=
 github.com/arduino/board-discovery v0.0.0-20180823133458-1ba29327fb0c/go.mod h1:HK7SpkEax/3P+0w78iRQx1sz1vCDYYw9RXwHjQTB5i8=
 github.com/arduino/go-paths-helper v0.0.0-20190214132331-c3c98d1bf2e1 h1:S0NpDSqjlkNA510vmRCP5Cq9mPgu3rWDSdeN4SI1Mwc=

legacy/builder/container_add_prototypes.go

@@ -30,6 +30,8 @@
 package builder
 
 import (
+	bldr "github.com/arduino/arduino-cli/arduino/builder"
+	"github.com/arduino/arduino-cli/arduino/sketch"
 	"github.com/arduino/arduino-cli/legacy/builder/constants"
 	"github.com/arduino/arduino-cli/legacy/builder/i18n"
 	"github.com/arduino/arduino-cli/legacy/builder/types"
@@ -56,7 +58,6 @@ func (s *ContainerAddPrototypes) Run(ctx *types.Context) error {
 		&CTagsTargetFileSaver{Source: &ctx.SourceGccMinusE, TargetFileName: constants.FILE_CTAGS_TARGET_FOR_GCC_MINUS_E},
 		&CTagsRunner{},
 		&PrototypesAdder{},
-		&SketchSaver{},
 	}
 
 	for _, command := range commands {
@@ -67,5 +68,9 @@ func (s *ContainerAddPrototypes) Run(ctx *types.Context) error {
 		}
 	}
 
+	if err := bldr.SaveSketchItemCpp(&sketch.Item{ctx.Sketch.MainFile.Name.String(), []byte(ctx.Source)}, ctx.SketchBuildPath.String()); err != nil {
+		return i18n.WrapError(err)
+	}
+
 	return nil
 }

legacy/builder/container_merge_copy_sketch_files.go

@@ -30,27 +30,31 @@
 package builder
 
 import (
+	bldr "github.com/arduino/arduino-cli/arduino/builder"
+	"github.com/arduino/arduino-cli/arduino/sketch"
 	"github.com/arduino/arduino-cli/legacy/builder/i18n"
 	"github.com/arduino/arduino-cli/legacy/builder/types"
+	"github.com/go-errors/errors"
 )
 
 type ContainerMergeCopySketchFiles struct{}
 
 func (s *ContainerMergeCopySketchFiles) Run(ctx *types.Context) error {
-	commands := []types.Command{
-		&SketchSourceMerger{},
-		&SketchSaver{},
-		&AdditionalSketchFilesCopier{},
+	sk := types.SketchFromLegacy(ctx.Sketch)
+	if sk == nil {
+		return i18n.WrapError(errors.New("unable to convert legacy sketch to the new type"))
 	}
+	offset, source := bldr.MergeSketchSources(sk)
+	ctx.LineOffset = offset
+	ctx.Source = source
 
-	for _, command := range commands {
-		PrintRingNameIfDebug(ctx, command)
-		err := command.Run(ctx)
-		if err != nil {
-			return i18n.WrapError(err)
-		}
+	if err := bldr.SaveSketchItemCpp(&sketch.Item{ctx.Sketch.MainFile.Name.String(), []byte(ctx.Source)}, ctx.SketchBuildPath.String()); err != nil {
+		return i18n.WrapError(err)
 	}
 
-	return nil
+	if err := new(AdditionalSketchFilesCopier).Run(ctx); err != nil {
+		return i18n.WrapError(err)
+	}
 
+	return nil
 }

legacy/builder/container_setup.go

@@ -30,9 +30,11 @@
 package builder
 
 import (
+	bldr "github.com/arduino/arduino-cli/arduino/builder"
 	"github.com/arduino/arduino-cli/legacy/builder/builder_utils"
 	"github.com/arduino/arduino-cli/legacy/builder/i18n"
 	"github.com/arduino/arduino-cli/legacy/builder/types"
+	"github.com/arduino/go-paths-helper"
 )
 
 type ContainerSetupHardwareToolsLibsSketchAndProps struct{}
@@ -48,7 +50,34 @@ func (s *ContainerSetupHardwareToolsLibsSketchAndProps) Run(ctx *types.Context)
 		&ToolsLoader{},
 		&AddBuildBoardPropertyIfMissing{},
 		&LibrariesLoader{},
-		&SketchLoader{},
+	}
+
+	ctx.Progress.Steps = ctx.Progress.Steps / float64(len(commands))
+
+	for _, command := range commands {
+		builder_utils.PrintProgressIfProgressEnabledAndMachineLogger(ctx)
+		PrintRingNameIfDebug(ctx, command)
+		err := command.Run(ctx)
+		if err != nil {
+			return i18n.WrapError(err)
+		}
+	}
+
+	// get abs path to sketch
+	sketchLocation, err := ctx.SketchLocation.Abs()
+	if err != nil {
+		return i18n.WrapError(err)
+	}
+
+	// load sketch
+	sketch, err := bldr.LoadSketch(sketchLocation.String(), ctx.BuildPath.String())
+	if err != nil {
+		return i18n.WrapError(err)
+	}
+	ctx.SketchLocation = paths.New(sketch.MainFile.Path)
+	ctx.Sketch = types.SketchToLegacy(sketch)
+
+	commands = []types.Command{
 		&SetupBuildProperties{},
 		&LoadVIDPIDSpecificProperties{},
 		&SetCustomBuildProperties{},

legacy/builder/create_cmake_rule.go

@@ -130,7 +130,6 @@ func (s *ExportProjectCMake) Run(ctx *types.Context) error {
 		//&ContainerMergeCopySketchFiles{},
 		&ContainerAddPrototypes{},
 		//&FilterSketchSource{Source: &ctx.Source, RemoveLineMarkers: true},
-		//&SketchSaver{},
 	}
 
 	for _, command := range commands {

legacy/builder/preprocess_sketch.go

@@ -37,6 +37,8 @@ import (
 	"runtime"
 	"strings"
 
+	bldr "github.com/arduino/arduino-cli/arduino/builder"
+	"github.com/arduino/arduino-cli/arduino/sketch"
 	"github.com/arduino/arduino-cli/legacy/builder/constants"
 	"github.com/arduino/arduino-cli/legacy/builder/i18n"
 	"github.com/arduino/arduino-cli/legacy/builder/types"
@@ -66,12 +68,6 @@ func (s *PreprocessSketchArduino) Run(ctx *types.Context) error {
 		return i18n.WrapError(err)
 	}
 
-	if ctx.CodeCompleteAt != "" {
-		commands = append(commands, &OutputCodeCompletions{})
-	} else {
-		commands = append(commands, &SketchSaver{})
-	}
-
 	GCCPreprocRunner(ctx, sourceFile, ctx.PreprocPath.Join(constants.FILE_CTAGS_TARGET_FOR_GCC_MINUS_E), ctx.IncludeFolders)
 
 	for _, command := range commands {
@@ -82,7 +78,14 @@ func (s *PreprocessSketchArduino) Run(ctx *types.Context) error {
 		}
 	}
 
-	return nil
+	var err error
+	if ctx.CodeCompleteAt != "" {
+		err = new(OutputCodeCompletions).Run(ctx)
+	} else {
+		err = bldr.SaveSketchItemCpp(&sketch.Item{ctx.Sketch.MainFile.Name.String(), []byte(ctx.Source)}, ctx.SketchBuildPath.String())
+	}
+
+	return err
 }
 
 type ArduinoPreprocessorRunner struct{}

legacy/builder/sketch_saver.go

-/*
- * This file is part of Arduino Builder.
- *
- * Arduino Builder is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
- *
- * As a special exception, you may use this file as part of a free software
- * library without restriction.  Specifically, if other files instantiate
- * templates or use macros or inline functions from this file, or you compile
- * this file and link it with other files to produce an executable, this
- * file does not by itself cause the resulting executable to be covered by
- * the GNU General Public License.  This exception does not however
- * invalidate any other reasons why the executable file might be covered by
- * the GNU General Public License.
- *
- * Copyright 2015 Arduino LLC (http://www.arduino.cc/)
- */
-
-package builder
-
-import (
-	"github.com/arduino/arduino-cli/legacy/builder/i18n"
-	"github.com/arduino/arduino-cli/legacy/builder/types"
-)
-
-type SketchSaver struct{}
-
-func (s *SketchSaver) Run(ctx *types.Context) error {
-	sketch := ctx.Sketch
-	sketchBuildPath := ctx.SketchBuildPath
-
-	if err := sketchBuildPath.MkdirAll(); err != nil {
-		return i18n.WrapError(err)
-	}
-
-	err := sketchBuildPath.Join(sketch.MainFile.Name.Base() + ".cpp").WriteFile([]byte(ctx.Source))
-	return i18n.WrapError(err)
-}

legacy/builder/sketch_source_merger.go

-/*
- * This file is part of Arduino Builder.
- *
- * Arduino Builder is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
- *
- * As a special exception, you may use this file as part of a free software
- * library without restriction.  Specifically, if other files instantiate
- * templates or use macros or inline functions from this file, or you compile
- * this file and link it with other files to produce an executable, this
- * file does not by itself cause the resulting executable to be covered by
- * the GNU General Public License.  This exception does not however
- * invalidate any other reasons why the executable file might be covered by
- * the GNU General Public License.
- *
- * Copyright 2015 Arduino LLC (http://www.arduino.cc/)
- * Copyright 2015 Matthijs Kooijman
- */
-
-package builder
-
-import (
-	"regexp"
-
-	"github.com/arduino/arduino-cli/legacy/builder/types"
-	"github.com/arduino/arduino-cli/legacy/builder/utils"
-)
-
-type SketchSourceMerger struct{}
-
-func (s *SketchSourceMerger) Run(ctx *types.Context) error {
-	sketch := ctx.Sketch
-
-	lineOffset := 0
-	includeSection := ""
-	if !sketchIncludesArduinoH(&sketch.MainFile) {
-		includeSection += "#include <Arduino.h>\n"
-		lineOffset++
-	}
-	includeSection += "#line 1 " + utils.QuoteCppString(sketch.MainFile.Name.String()) + "\n"
-	lineOffset++
-	ctx.IncludeSection = includeSection
-
-	source := includeSection
-	source += addSourceWrappedWithLineDirective(&sketch.MainFile)
-	lineOffset += 1
-	for _, file := range sketch.OtherSketchFiles {
-		source += addSourceWrappedWithLineDirective(&file)
-	}
-
-	ctx.LineOffset = lineOffset
-	ctx.Source = source
-
-	return nil
-}
-
-func sketchIncludesArduinoH(sketch *types.SketchFile) bool {
-	if matched, err := regexp.MatchString("(?m)^\\s*#\\s*include\\s*[<\"]Arduino\\.h[>\"]", sketch.Source); err != nil {
-		panic(err)
-	} else {
-		return matched
-	}
-}
-
-func addSourceWrappedWithLineDirective(sketch *types.SketchFile) string {
-	source := "#line 1 " + utils.QuoteCppString(sketch.Name.String()) + "\n"
-	source += sketch.Source
-	source += "\n"
-
-	return source
-}

legacy/builder/test/prototypes_adder_test.go

@@ -83,7 +83,7 @@ func TestPrototypesAdderBridgeExample(t *testing.T) {
 		NoError(t, err)
 	}
 
-	require.Equal(t, "#include <Arduino.h>\n#line 1 "+quotedSketchLocation+"\n", ctx.IncludeSection)
+	require.Contains(t, ctx.Source, "#include <Arduino.h>\n#line 1 "+quotedSketchLocation+"\n")
 	require.Equal(t, "#line 33 "+quotedSketchLocation+"\nvoid setup();\n#line 46 "+quotedSketchLocation+"\nvoid loop();\n#line 62 "+quotedSketchLocation+"\nvoid process(BridgeClient client);\n#line 82 "+quotedSketchLocation+"\nvoid digitalCommand(BridgeClient client);\n#line 109 "+quotedSketchLocation+"\nvoid analogCommand(BridgeClient client);\n#line 149 "+quotedSketchLocation+"\nvoid modeCommand(BridgeClient client);\n#line 33 "+quotedSketchLocation+"\n", ctx.PrototypesSection)
 }
 
@@ -411,7 +411,7 @@ func TestPrototypesAdderSketchWithConfig(t *testing.T) {
 		NoError(t, err)
 	}
 
-	require.Equal(t, "#include <Arduino.h>\n#line 1 "+quotedSketchLocation+"\n", ctx.IncludeSection)
+	require.Contains(t, ctx.Source, "#include <Arduino.h>\n#line 1 "+quotedSketchLocation+"\n")
 	require.Equal(t, "#line 13 "+quotedSketchLocation+"\nvoid setup();\n#line 17 "+quotedSketchLocation+"\nvoid loop();\n#line 13 "+quotedSketchLocation+"\n", ctx.PrototypesSection)
 
 	preprocessed := LoadAndInterpolate(t, filepath.Join("sketch_with_config", "sketch_with_config.preprocessed.txt"), ctx)
@@ -457,7 +457,7 @@ func TestPrototypesAdderSketchNoFunctionsTwoFiles(t *testing.T) {
 		NoError(t, err)
 	}
 
-	require.Equal(t, "#include <Arduino.h>\n#line 1 "+quotedSketchLocation+"\n", ctx.IncludeSection)
+	require.Contains(t, ctx.Source, "#include <Arduino.h>\n#line 1 "+quotedSketchLocation+"\n")
 	require.Equal(t, "", ctx.PrototypesSection)
 }
 
@@ -500,7 +500,7 @@ func TestPrototypesAdderSketchNoFunctions(t *testing.T) {
 		NoError(t, err)
 	}
 
-	require.Equal(t, "#include <Arduino.h>\n#line 1 "+quotedSketchLocation+"\n", ctx.IncludeSection)
+	require.Contains(t, ctx.Source, "#include <Arduino.h>\n#line 1 "+quotedSketchLocation+"\n")
 	require.Equal(t, "", ctx.PrototypesSection)
 }
 
@@ -543,7 +543,7 @@ func TestPrototypesAdderSketchWithDefaultArgs(t *testing.T) {
 		NoError(t, err)
 	}
 
-	require.Equal(t, "#include <Arduino.h>\n#line 1 "+quotedSketchLocation+"\n", ctx.IncludeSection)
+	require.Contains(t, ctx.Source, "#include <Arduino.h>\n#line 1 "+quotedSketchLocation+"\n")
 	require.Equal(t, "#line 4 "+quotedSketchLocation+"\nvoid setup();\n#line 7 "+quotedSketchLocation+"\nvoid loop();\n#line 1 "+quotedSketchLocation+"\n", ctx.PrototypesSection)
 }
 
@@ -586,7 +586,7 @@ func TestPrototypesAdderSketchWithInlineFunction(t *testing.T) {
 		NoError(t, err)
 	}
 
-	require.Equal(t, "#include <Arduino.h>\n#line 1 "+quotedSketchLocation+"\n", ctx.IncludeSection)
+	require.Contains(t, ctx.Source, "#include <Arduino.h>\n#line 1 "+quotedSketchLocation+"\n")
 
 	expected := "#line 1 " + quotedSketchLocation + "\nvoid setup();\n#line 2 " + quotedSketchLocation + "\nvoid loop();\n#line 4 " + quotedSketchLocation + "\nshort unsigned int testInt();\n#line 8 " + quotedSketchLocation + "\nstatic int8_t testInline();\n#line 12 " + quotedSketchLocation + "\n__attribute__((always_inline)) uint8_t testAttribute();\n#line 1 " + quotedSketchLocation + "\n"
 	obtained := ctx.PrototypesSection
@@ -640,7 +640,7 @@ func TestPrototypesAdderSketchWithFunctionSignatureInsideIFDEF(t *testing.T) {
 		NoError(t, err)
 	}
 
-	require.Equal(t, "#include <Arduino.h>\n#line 1 "+quotedSketchLocation+"\n", ctx.IncludeSection)
+	require.Contains(t, ctx.Source, "#include <Arduino.h>\n#line 1 "+quotedSketchLocation+"\n")
 	require.Equal(t, "#line 1 "+quotedSketchLocation+"\nvoid setup();\n#line 3 "+quotedSketchLocation+"\nvoid loop();\n#line 15 "+quotedSketchLocation+"\nint8_t adalight();\n#line 1 "+quotedSketchLocation+"\n", ctx.PrototypesSection)
 }
 
@@ -683,7 +683,7 @@ func TestPrototypesAdderSketchWithUSBCON(t *testing.T) {
 		NoError(t, err)
 	}
 
-	require.Equal(t, "#include <Arduino.h>\n#line 1 "+quotedSketchLocation+"\n", ctx.IncludeSection)
+	require.Contains(t, ctx.Source, "#include <Arduino.h>\n#line 1 "+quotedSketchLocation+"\n")
 	require.Equal(t, "#line 5 "+quotedSketchLocation+"\nvoid ciao();\n#line 10 "+quotedSketchLocation+"\nvoid setup();\n#line 15 "+quotedSketchLocation+"\nvoid loop();\n#line 5 "+quotedSketchLocation+"\n", ctx.PrototypesSection)
 }
 
@@ -725,7 +725,7 @@ func TestPrototypesAdderSketchWithTypename(t *testing.T) {
 		NoError(t, err)
 	}
 
-	require.Equal(t, "#include <Arduino.h>\n#line 1 "+quotedSketchLocation+"\n", ctx.IncludeSection)
+	require.Contains(t, ctx.Source, "#include <Arduino.h>\n#line 1 "+quotedSketchLocation+"\n")
 	expected := "#line 6 " + quotedSketchLocation + "\nvoid setup();\n#line 10 " + quotedSketchLocation + "\nvoid loop();\n#line 12 " + quotedSketchLocation + "\ntypename Foo<char>::Bar func();\n#line 6 " + quotedSketchLocation + "\n"
 	obtained := ctx.PrototypesSection
 	// ctags based preprocessing ignores line with typename
@@ -774,7 +774,7 @@ func TestPrototypesAdderSketchWithIfDef2(t *testing.T) {
 		NoError(t, err)
 	}
 
-	require.Equal(t, "#include <Arduino.h>\n#line 1 "+quotedSketchLocation+"\n", ctx.IncludeSection)
+	require.Contains(t, ctx.Source, "#include <Arduino.h>\n#line 1 "+quotedSketchLocation+"\n")
 	require.Equal(t, "#line 5 "+quotedSketchLocation+"\nvoid elseBranch();\n#line 9 "+quotedSketchLocation+"\nvoid f1();\n#line 10 "+quotedSketchLocation+"\nvoid f2();\n#line 12 "+quotedSketchLocation+"\nvoid setup();\n#line 14 "+quotedSketchLocation+"\nvoid loop();\n#line 5 "+quotedSketchLocation+"\n", ctx.PrototypesSection)
 
 	expectedSource := LoadAndInterpolate(t, filepath.Join("sketch_with_ifdef", "sketch.preprocessed.txt"), ctx)
@@ -820,7 +820,7 @@ func TestPrototypesAdderSketchWithIfDef2SAM(t *testing.T) {
 		NoError(t, err)
 	}
 
-	require.Equal(t, "#include <Arduino.h>\n#line 1 "+quotedSketchLocation+"\n", ctx.IncludeSection)
+	require.Contains(t, ctx.Source, "#include <Arduino.h>\n#line 1 "+quotedSketchLocation+"\n")
 	require.Equal(t, "#line 2 "+quotedSketchLocation+"\nvoid ifBranch();\n#line 9 "+quotedSketchLocation+"\nvoid f1();\n#line 10 "+quotedSketchLocation+"\nvoid f2();\n#line 12 "+quotedSketchLocation+"\nvoid setup();\n#line 14 "+quotedSketchLocation+"\nvoid loop();\n#line 2 "+quotedSketchLocation+"\n", ctx.PrototypesSection)
 
 	expectedSource := LoadAndInterpolate(t, filepath.Join("sketch_with_ifdef", "sketch.preprocessed.SAM.txt"), ctx)
@@ -866,7 +866,7 @@ func TestPrototypesAdderSketchWithConst(t *testing.T) {
 		NoError(t, err)
 	}
 
-	require.Equal(t, "#include <Arduino.h>\n#line 1 "+quotedSketchLocation+"\n", ctx.IncludeSection)
+	require.Contains(t, ctx.Source, "#include <Arduino.h>\n#line 1 "+quotedSketchLocation+"\n")
 	require.Equal(t, "#line 1 "+quotedSketchLocation+"\nvoid setup();\n#line 2 "+quotedSketchLocation+"\nvoid loop();\n#line 4 "+quotedSketchLocation+"\nconst __FlashStringHelper* test();\n#line 6 "+quotedSketchLocation+"\nconst int test3();\n#line 8 "+quotedSketchLocation+"\nvolatile __FlashStringHelper* test2();\n#line 10 "+quotedSketchLocation+"\nvolatile int test4();\n#line 1 "+quotedSketchLocation+"\n", ctx.PrototypesSection)
 }
 

legacy/builder/test/sketch1/merged_sketch.txt

 #include <Arduino.h>
 #line 1 {{QuoteCppString .sketch.MainFile.Name}}
-#line 1 {{QuoteCppString .sketch.MainFile.Name}}
 void setup() {
 
 }

legacy/builder/test/sketch2/SketchWithIfDef.preprocessed.txt

 #include <Arduino.h>
 #line 1 {{QuoteCppString .sketch.MainFile.Name}}
-#line 1 {{QuoteCppString .sketch.MainFile.Name}}
 #define DEBUG 1
 #define DISABLED 0
 

legacy/builder/test/sketch2/SketchWithIfDef.resolved.directives.txt

 #include <Arduino.h>
 #line 1 {{QuoteCppString .sketch.MainFile.Name}}
-#line 1 {{QuoteCppString .sketch.MainFile.Name}}
 #define DEBUG 1
 #define DISABLED 0
 

legacy/builder/test/sketch3/Baladuino.preprocessed.txt

 #include <Arduino.h>
 #line 1 {{QuoteCppString .sketch.MainFile.Name}}
-#line 1 {{QuoteCppString .sketch.MainFile.Name}}
 /*
  * The code is released under the GNU General Public License.
  * Developed by Kristian Lauszus, TKJ Electronics 2013

legacy/builder/test/sketch4/CharWithEscapedDoubleQuote.preprocessed.txt

 #include <Arduino.h>
 #line 1 {{QuoteCppString .sketch.MainFile.Name}}
-#line 1 {{QuoteCppString .sketch.MainFile.Name}}
 #include <SoftwareSerial.h> // required to send and receive AT commands from the GPRS Shield
 #include <Wire.h> // required for I2C communication with the RTC
 

legacy/builder/test/sketch5/IncludeBetweenMultilineComment.preprocessed.txt

 #include <Arduino.h>
 #line 1 {{QuoteCppString .sketch.MainFile.Name}}
-#line 1 {{QuoteCppString .sketch.MainFile.Name}}
 #include <CapacitiveSensor.h>
 /*
 #include <WiFi.h>

legacy/builder/test/sketch6/LineContinuations.preprocessed.txt

 #include <Arduino.h>
 #line 1 {{QuoteCppString .sketch.MainFile.Name}}
-#line 1 {{QuoteCppString .sketch.MainFile.Name}}
 const char *foo = "\
 hello \
 world\n";

legacy/builder/test/sketch7/StringWithComment.preprocessed.txt

 #include <Arduino.h>
 #line 1 {{QuoteCppString .sketch.MainFile.Name}}
 #line 1 {{QuoteCppString .sketch.MainFile.Name}}
-#line 1 {{QuoteCppString .sketch.MainFile.Name}}
 void setup();
 #line 10 {{QuoteCppString .sketch.MainFile.Name}}
 void loop();

legacy/builder/test/sketch8/SketchWithStruct.preprocessed.txt

 #include <Arduino.h>
 #line 1 {{QuoteCppString .sketch.MainFile.Name}}
-#line 1 {{QuoteCppString .sketch.MainFile.Name}}
 /* START CODE */
 
 struct A_NEW_TYPE {

legacy/builder/test/sketch_loader_test.go

-/*
- * This file is part of Arduino Builder.
- *
- * Arduino Builder is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
- *
- * As a special exception, you may use this file as part of a free software
- * library without restriction.  Specifically, if other files instantiate
- * templates or use macros or inline functions from this file, or you compile
- * this file and link it with other files to produce an executable, this
- * file does not by itself cause the resulting executable to be covered by
- * the GNU General Public License.  This exception does not however
- * invalidate any other reasons why the executable file might be covered by
- * the GNU General Public License.
- *
- * Copyright 2015 Arduino LLC (http://www.arduino.cc/)
- */
-
-package test
-
-import (
-	"path/filepath"
-	"testing"
-
-	"github.com/arduino/arduino-cli/legacy/builder"
-	"github.com/arduino/arduino-cli/legacy/builder/types"
-	paths "github.com/arduino/go-paths-helper"
-	"github.com/stretchr/testify/require"
-)
-
-func TestLoadSketchWithFolder(t *testing.T) {
-	ctx := &types.Context{
-		SketchLocation: paths.New("sketch1"),
-	}
-
-	loader := builder.SketchLoader{}
-	err := loader.Run(ctx)
-
-	require.Error(t, err)
-	require.Nil(t, ctx.Sketch)
-}
-
-func TestLoadSketchNonExistentPath(t *testing.T) {
-	ctx := &types.Context{
-		SketchLocation: paths.New("asdasd78128123981723981273asdasd"),
-	}
-
-	loader := builder.SketchLoader{}
-	err := loader.Run(ctx)
-
-	require.Error(t, err)
-	require.Nil(t, ctx.Sketch)
-}
-
-func TestLoadSketch(t *testing.T) {
-	ctx := &types.Context{
-		SketchLocation: paths.New("sketch1", "sketch.ino"),
-	}
-
-	commands := []types.Command{
-		&builder.SketchLoader{},
-	}
-
-	for _, command := range commands {
-		err := command.Run(ctx)
-		NoError(t, err)
-	}
-
-	sketch := ctx.Sketch
-	require.NotNil(t, sketch)
-
-	require.Contains(t, sketch.MainFile.Name.String(), "sketch.ino")
-
-	require.Equal(t, 2, len(sketch.OtherSketchFiles))
-	require.Contains(t, sketch.OtherSketchFiles[0].Name.String(), "old.pde")
-	require.Contains(t, sketch.OtherSketchFiles[1].Name.String(), "other.ino")
-
-	require.Equal(t, 3, len(sketch.AdditionalFiles))
-	require.Contains(t, sketch.AdditionalFiles[0].Name.String(), "header.h")
-	require.Contains(t, sketch.AdditionalFiles[1].Name.String(), "s_file.S")
-	require.Contains(t, sketch.AdditionalFiles[2].Name.String(), "helper.h")
-}
-
-func TestFailToLoadSketchFromFolder(t *testing.T) {
-	ctx := &types.Context{
-		SketchLocation: paths.New("./sketch1"),
-	}
-
-	loader := builder.SketchLoader{}
-	err := loader.Run(ctx)
-	require.Error(t, err)
-	require.Nil(t, ctx.Sketch)
-}
-
-func TestLoadSketchFromFolder(t *testing.T) {
-	ctx := &types.Context{
-		SketchLocation: paths.New("sketch_with_subfolders"),
-	}
-
-	commands := []types.Command{
-		&builder.SketchLoader{},
-	}
-
-	for _, command := range commands {
-		err := command.Run(ctx)
-		NoError(t, err)
-	}
-
-	sketch := ctx.Sketch
-	require.NotNil(t, sketch)
-
-	require.Contains(t, sketch.MainFile.Name.String(), "sketch_with_subfolders.ino")
-
-	require.Equal(t, 0, len(sketch.OtherSketchFiles))
-
-	require.Equal(t, 4, len(sketch.AdditionalFiles))
-	require.Contains(t, filepath.ToSlash(sketch.AdditionalFiles[0].Name.String()), "sketch_with_subfolders/src/subfolder/other.cpp")
-	require.Contains(t, filepath.ToSlash(sketch.AdditionalFiles[1].Name.String()), "sketch_with_subfolders/src/subfolder/other.h")
-	require.Contains(t, filepath.ToSlash(sketch.AdditionalFiles[2].Name.String()), "sketch_with_subfolders/subfolder/dont_load_me.cpp")
-	require.Contains(t, filepath.ToSlash(sketch.AdditionalFiles[3].Name.String()), "sketch_with_subfolders/subfolder/other.h")
-}
-
-func TestLoadSketchWithBackup(t *testing.T) {
-	ctx := &types.Context{
-		SketchLocation: paths.New("sketch_with_backup_files", "sketch.ino"),
-	}
-
-	commands := []types.Command{
-		&builder.SketchLoader{},
-	}
-
-	for _, command := range commands {
-		err := command.Run(ctx)
-		NoError(t, err)
-	}
-
-	sketch := ctx.Sketch
-	require.NotNil(t, sketch)
-
-	require.Contains(t, sketch.MainFile.Name.String(), "sketch.ino")
-
-	require.Equal(t, 0, len(sketch.AdditionalFiles))
-	require.Equal(t, 0, len(sketch.OtherSketchFiles))
-}
-
-func TestLoadSketchWithMacOSXGarbage(t *testing.T) {
-	ctx := &types.Context{
-		SketchLocation: paths.New("sketch_with_macosx_garbage", "sketch.ino"),
-	}
-
-	commands := []types.Command{
-		&builder.SketchLoader{},
-	}
-
-	for _, command := range commands {
-		err := command.Run(ctx)
-		NoError(t, err)
-	}
-
-	sketch := ctx.Sketch
-	require.NotNil(t, sketch)
-
-	require.Contains(t, sketch.MainFile.Name.String(), "sketch.ino")
-
-	require.Equal(t, 0, len(sketch.AdditionalFiles))
-	require.Equal(t, 0, len(sketch.OtherSketchFiles))
-}

legacy/builder/test/sketch_source_merger_test.go

-/*
- * This file is part of Arduino Builder.
- *
- * Arduino Builder is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
- *
- * As a special exception, you may use this file as part of a free software
- * library without restriction.  Specifically, if other files instantiate
- * templates or use macros or inline functions from this file, or you compile
- * this file and link it with other files to produce an executable, this
- * file does not by itself cause the resulting executable to be covered by
- * the GNU General Public License.  This exception does not however
- * invalidate any other reasons why the executable file might be covered by
- * the GNU General Public License.
- *
- * Copyright 2015 Arduino LLC (http://www.arduino.cc/)
- * Copyright 2015 Matthijs Kooijman
- */
-
-package test
-
-import (
-	"path/filepath"
-	"strings"
-	"testing"
-
-	"github.com/arduino/arduino-cli/legacy/builder"
-	"github.com/arduino/arduino-cli/legacy/builder/types"
-	"github.com/arduino/go-paths-helper"
-	"github.com/stretchr/testify/require"
-)
-
-func TestMergeSketch(t *testing.T) {
-	ctx := &types.Context{
-		SketchLocation: paths.New("sketch1", "sketch.ino"),
-	}
-
-	commands := []types.Command{
-		&builder.SketchLoader{},
-		&builder.SketchSourceMerger{},
-	}
-
-	for _, command := range commands {
-		err := command.Run(ctx)
-		NoError(t, err)
-	}
-
-	source := ctx.Source
-
-	expected_source := LoadAndInterpolate(t, filepath.Join("sketch1", "merged_sketch.txt"), ctx)
-	require.Equal(t, expected_source, strings.Replace(source, "\r\n", "\n", -1))
-}

legacy/builder/test/sketch_with_config/sketch_with_config.preprocessed.txt

 #include <Arduino.h>
 #line 1 {{QuoteCppString .sketch.MainFile.Name}}
-#line 1 {{QuoteCppString .sketch.MainFile.Name}}
 #include "config.h"
 
 #ifdef DEBUG

legacy/builder/test/sketch_with_ifdef/sketch.preprocessed.SAM.txt

 #include <Arduino.h>
 #line 1 {{QuoteCppString .sketch.MainFile.Name}}
-#line 1 {{QuoteCppString .sketch.MainFile.Name}}
 #if __SAM3X8E__
 #line 2 {{QuoteCppString .sketch.MainFile.Name}}
 void ifBranch();

legacy/builder/test/sketch_with_ifdef/sketch.preprocessed.txt

 #include <Arduino.h>
 #line 1 {{QuoteCppString .sketch.MainFile.Name}}
-#line 1 {{QuoteCppString .sketch.MainFile.Name}}
 #if __SAM3X8E__
 void ifBranch() {
 }

legacy/builder/types/context.go

@@ -86,7 +86,6 @@ type Context struct {
 	CTagsOutput                 string
 	CTagsTargetFile             *paths.Path
 	CTagsOfPreprocessedSource   []*CTag
-	IncludeSection              string
 	LineOffset                  int
 	PrototypesSection           string
 	PrototypesLineWhereToInsert int

legacy/builder/types/types.go

@@ -34,6 +34,7 @@ import (
 	"strconv"
 
 	"github.com/arduino/arduino-cli/arduino/libraries"
+	"github.com/arduino/arduino-cli/arduino/sketch"
 	paths "github.com/arduino/go-paths-helper"
 )
 
@@ -123,6 +124,56 @@ type Sketch struct {
 	AdditionalFiles  []SketchFile
 }
 
+func SketchToLegacy(sketch *sketch.Sketch) *Sketch {
+	s := &Sketch{}
+	s.MainFile = SketchFile{
+		paths.New(sketch.MainFile.Path),
+		string(sketch.MainFile.Source),
+	}
+
+	for _, item := range sketch.OtherSketchFiles {
+		s.OtherSketchFiles = append(s.OtherSketchFiles, SketchFile{
+			paths.New(item.Path),
+			string(item.Source),
+		})
+	}
+
+	for _, item := range sketch.AdditionalFiles {
+		s.AdditionalFiles = append(s.AdditionalFiles, SketchFile{
+			paths.New(item.Path),
+			string(item.Source),
+		})
+	}
+
+	return s
+}
+
+func SketchFromLegacy(s *Sketch) *sketch.Sketch {
+	others := []*sketch.Item{}
+	for _, f := range s.OtherSketchFiles {
+		if i, err := sketch.NewItem(f.Name.String()); err == nil {
+			others = append(others, i)
+		}
+	}
+
+	additional := []*sketch.Item{}
+	for _, f := range s.AdditionalFiles {
+		if i, err := sketch.NewItem(f.Name.String()); err == nil {
+			additional = append(additional, i)
+		}
+	}
+
+	return &sketch.Sketch{
+		MainFile: &sketch.Item{
+			Path:   s.MainFile.Name.String(),
+			Source: []byte(s.MainFile.Source),
+		},
+		LocationPath:     s.MainFile.Name.Parent().String(),
+		OtherSketchFiles: others,
+		AdditionalFiles:  additional,
+	}
+}
+
 type PlatforKeysRewrite struct {
 	Rewrites []PlatforKeyRewrite
 }