Add archive command to zip a sketch and its files (#931)

cli/sketch/archive.go

+// This file is part of arduino-cli.
+//
+// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to license@arduino.cc.
+
+package sketch
+
+import (
+	"context"
+	"os"
+
+	"github.com/arduino/arduino-cli/cli/errorcodes"
+	"github.com/arduino/arduino-cli/cli/feedback"
+	"github.com/arduino/arduino-cli/commands/sketch"
+	rpc "github.com/arduino/arduino-cli/rpc/commands"
+	"github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+)
+
+var includeBuildDir bool
+
+// initArchiveCommand creates a new `archive` command
+func initArchiveCommand() *cobra.Command {
+	command := &cobra.Command{
+		Use:   "archive <sketchPath> <archivePath>",
+		Short: "Creates a zip file containing all sketch files.",
+		Long:  "Creates a zip file containing all sketch files.",
+		Example: "" +
+			"  " + os.Args[0] + " archive\n" +
+			"  " + os.Args[0] + " archive .\n" +
+			"  " + os.Args[0] + " archive . MySketchArchive.zip\n" +
+			"  " + os.Args[0] + " archive /home/user/Arduino/MySketch\n" +
+			"  " + os.Args[0] + " archive /home/user/Arduino/MySketch /home/user/MySketchArchive.zip",
+		Args: cobra.MaximumNArgs(2),
+		Run:  runArchiveCommand,
+	}
+
+	command.Flags().BoolVar(&includeBuildDir, "include-build-dir", false, "Includes build directory in the archive.")
+
+	return command
+}
+
+func runArchiveCommand(cmd *cobra.Command, args []string) {
+	logrus.Info("Executing `arduino sketch archive`")
+
+	sketchPath := ""
+	if len(args) >= 1 {
+		sketchPath = args[0]
+	}
+
+	archivePath := ""
+	if len(args) == 2 {
+		archivePath = args[1]
+	}
+
+	_, err := sketch.ArchiveSketch(context.Background(),
+		&rpc.ArchiveSketchReq{
+			SketchPath:      sketchPath,
+			ArchivePath:     archivePath,
+			IncludeBuildDir: includeBuildDir,
+		})
+
+	if err != nil {
+		feedback.Errorf("Error archiving: %v", err)
+		os.Exit(errorcodes.ErrGeneric)
+	}
+}

cli/sketch/sketch.go

@@ -31,6 +31,7 @@ func NewCommand() *cobra.Command {
 	}
 
 	cmd.AddCommand(initNewCommand())
+	cmd.AddCommand(initArchiveCommand())
 
 	return cmd
 }

commands/daemon/daemon.go

@@ -26,6 +26,7 @@ import (
 	"github.com/arduino/arduino-cli/commands/compile"
 	"github.com/arduino/arduino-cli/commands/core"
 	"github.com/arduino/arduino-cli/commands/lib"
+	"github.com/arduino/arduino-cli/commands/sketch"
 	"github.com/arduino/arduino-cli/commands/upload"
 	rpc "github.com/arduino/arduino-cli/rpc/commands"
 )
@@ -337,3 +338,8 @@ func (s *ArduinoCoreServerImpl) LibrarySearch(ctx context.Context, req *rpc.Libr
 func (s *ArduinoCoreServerImpl) LibraryList(ctx context.Context, req *rpc.LibraryListReq) (*rpc.LibraryListResp, error) {
 	return lib.LibraryList(ctx, req)
 }
+
+// ArchiveSketch FIXMEDOC
+func (s *ArduinoCoreServerImpl) ArchiveSketch(ctx context.Context, req *rpc.ArchiveSketchReq) (*rpc.ArchiveSketchResp, error) {
+	return sketch.ArchiveSketch(ctx, req)
+}

commands/sketch/archive.go

+// This file is part of arduino-cli.
+//
+// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to license@arduino.cc.
+
+package sketch
+
+import (
+	"archive/zip"
+	"context"
+	"fmt"
+	"io"
+	"path/filepath"
+	"strings"
+
+	rpc "github.com/arduino/arduino-cli/rpc/commands"
+	paths "github.com/arduino/go-paths-helper"
+)
+
+// ArchiveSketch FIXMEDOC
+func ArchiveSketch(ctx context.Context, req *rpc.ArchiveSketchReq) (*rpc.ArchiveSketchResp, error) {
+	// sketchName is the name of the sketch without extension, for example "MySketch"
+	var sketchName string
+
+	sketchPath := paths.New(req.SketchPath)
+	if sketchPath == nil {
+		sketchPath = paths.New(".")
+	}
+
+	sketchPath, err := sketchPath.Clean().Abs()
+	if err != nil {
+		return nil, fmt.Errorf("Error getting absolute sketch path %v", err)
+	}
+
+	// Get the sketch name and make sketchPath point to the ino file
+	if sketchPath.IsDir() {
+		sketchName = sketchPath.Base()
+		sketchPath = sketchPath.Join(sketchName + ".ino")
+	} else if sketchPath.Ext() == ".ino" {
+		sketchName = strings.TrimSuffix(sketchPath.Base(), ".ino")
+	}
+
+	// Checks if it's really a sketch
+	if sketchPath.NotExist() {
+		return nil, fmt.Errorf("specified path is not a sketch: %v", sketchPath.String())
+	}
+
+	archivePath := paths.New(req.ArchivePath)
+	if archivePath == nil {
+		archivePath = sketchPath.Parent().Parent()
+	}
+
+	archivePath, err = archivePath.Clean().Abs()
+	if err != nil {
+		return nil, fmt.Errorf("Error getting absolute archive path %v", err)
+	}
+
+	// Makes archivePath point to a zip file
+	if archivePath.IsDir() {
+		archivePath = archivePath.Join(sketchName + ".zip")
+	} else if archivePath.Ext() == "" {
+		archivePath = paths.New(archivePath.String() + ".zip")
+	}
+
+	if archivePath.Exist() {
+		return nil, fmt.Errorf("archive already exists")
+	}
+
+	filesToZip, err := sketchPath.Parent().ReadDirRecursive()
+	if err != nil {
+		return nil, fmt.Errorf("Error retrieving sketch files: %v", err)
+	}
+	filesToZip.FilterOutDirs()
+
+	archive, err := archivePath.Create()
+	if err != nil {
+		return nil, fmt.Errorf("Error creating archive: %v", err)
+	}
+	defer archive.Close()
+
+	zipWriter := zip.NewWriter(archive)
+	defer zipWriter.Close()
+
+	for _, f := range filesToZip {
+
+		if !req.IncludeBuildDir {
+			filePath, err := sketchPath.Parent().Parent().RelTo(f)
+			if err != nil {
+				return nil, fmt.Errorf("Error calculating relative file path: %v", err)
+			}
+
+			// Skips build folder
+			if strings.HasPrefix(filePath.String(), sketchName+string(filepath.Separator)+"build") {
+				continue
+			}
+		}
+
+		// We get the parent path since we want the archive to unpack as a folder.
+		// If we don't do this the archive would contain all the sketch files as top level.
+		err = addFileToSketchArchive(zipWriter, f, sketchPath.Parent().Parent())
+		if err != nil {
+			return nil, fmt.Errorf("Error adding file to archive: %v", err)
+		}
+	}
+
+	return &rpc.ArchiveSketchResp{}, nil
+}
+
+// Adds a single file to an existing zip file
+func addFileToSketchArchive(zipWriter *zip.Writer, filePath, sketchPath *paths.Path) error {
+	f, err := filePath.Open()
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	info, err := f.Stat()
+	if err != nil {
+		return err
+	}
+
+	header, err := zip.FileInfoHeader(info)
+	if err != nil {
+		return err
+	}
+
+	filePath, err = sketchPath.RelTo(filePath)
+	if err != nil {
+		return err
+	}
+
+	header.Name = filePath.String()
+	header.Method = zip.Deflate
+
+	writer, err := zipWriter.CreateHeader(header)
+	if err != nil {
+		return err
+	}
+
+	_, err = io.Copy(writer, f)
+	return err
+}

go.mod

@@ -9,7 +9,7 @@ require (
 	bou.ke/monkey v1.0.1
 	github.com/GeertJohan/go.rice v1.0.0
 	github.com/arduino/board-discovery v0.0.0-20180823133458-1ba29327fb0c
-	github.com/arduino/go-paths-helper v1.2.0
+	github.com/arduino/go-paths-helper v1.3.1
 	github.com/arduino/go-properties-orderedmap v1.3.0
 	github.com/arduino/go-timeutils v0.0.0-20171220113728-d1dd9e313b1b
 	github.com/arduino/go-win32-utils v0.0.0-20180330194947-ed041402e83b

go.sum

@@ -16,6 +16,8 @@ github.com/arduino/go-paths-helper v1.0.1 h1:utYXLM2RfFlc9qp/MJTIYp3t6ux/xM6mWje
 github.com/arduino/go-paths-helper v1.0.1/go.mod h1:HpxtKph+g238EJHq4geEPv9p+gl3v5YYu35Yb+w31Ck=
 github.com/arduino/go-paths-helper v1.2.0 h1:qDW93PR5IZUN/jzO4rCtexiwF8P4OIcOmcSgAYLZfY4=
 github.com/arduino/go-paths-helper v1.2.0/go.mod h1:HpxtKph+g238EJHq4geEPv9p+gl3v5YYu35Yb+w31Ck=
+github.com/arduino/go-paths-helper v1.3.1 h1:Gz+PVt0luQyH4nffDePd8WBs/O5P05jADtJsY8NqvCM=
+github.com/arduino/go-paths-helper v1.3.1/go.mod h1:HpxtKph+g238EJHq4geEPv9p+gl3v5YYu35Yb+w31Ck=
 github.com/arduino/go-properties-orderedmap v1.3.0 h1:4No/vQopB36e7WUIk6H6TxiSEJPiMrVOCZylYmua39o=
 github.com/arduino/go-properties-orderedmap v1.3.0/go.mod h1:DKjD2VXY/NZmlingh4lSFMEYCVubfeArCsGPGDwb2yk=
 github.com/arduino/go-timeutils v0.0.0-20171220113728-d1dd9e313b1b h1:9hDi4F2st6dbLC3y4i02zFT5quS4X6iioWifGlVwfy4=

rpc/commands/commands.proto

@@ -61,6 +61,9 @@ service ArduinoCore {
   // Returns all files composing a Sketch
   rpc LoadSketch(LoadSketchReq) returns (LoadSketchResp) {}
 
+  // Creates a zip file containing all files of specified Sketch
+  rpc ArchiveSketch(ArchiveSketchReq) returns (ArchiveSketchResp) {}
+
   // BOARD COMMANDS
   // --------------
 
@@ -251,3 +254,14 @@ message LoadSketchResp {
   // List of absolute paths to additional sketch files
   repeated string additional_files = 4;
 }
+
+message ArchiveSketchReq{
+  // Absolute path to Sketch file or folder containing Sketch file
+  string sketch_path = 1;
+  // Absolute path to archive that will be created or folder that will contain it
+  string archive_path = 2;
+  // Specifies if build directory should be included in the archive
+  bool include_build_dir = 3;
+}
+
+message ArchiveSketchResp { }

test/conftest.py

@@ -15,6 +15,8 @@
 import os
 import platform
 import signal
+import shutil
+from pathlib import Path
 
 import pytest
 import simplejson as json
@@ -84,18 +86,29 @@ def run_command(pytestconfig, data_dir, downloads_dir, working_dir):
     Useful reference:
         http://docs.pyinvoke.org/en/1.4/api/runners.html#invoke.runners.Result
     """
-    cli_path = os.path.join(str(pytestconfig.rootdir), "..", "arduino-cli")
+
+    cli_path = Path(pytestconfig.rootdir).parent / "arduino-cli"
     env = {
         "ARDUINO_DATA_DIR": data_dir,
         "ARDUINO_DOWNLOADS_DIR": downloads_dir,
         "ARDUINO_SKETCHBOOK_DIR": data_dir,
     }
-    os.makedirs(os.path.join(data_dir, "packages"))
+    (Path(data_dir) / "packages").mkdir()
 
-    def _run(cmd_string):
-        cli_full_line = "{} {}".format(cli_path, cmd_string)
+    def _run(cmd_string, custom_working_dir=None):
+        if not custom_working_dir:
+            custom_working_dir = working_dir
+        cli_full_line = '"{}" {}'.format(cli_path, cmd_string)
         run_context = Context()
-        with run_context.cd(working_dir):
+        # It might happen that we need to change directories between drives on Windows,
+        # in that case the "/d" flag must be used otherwise directory wouldn't change
+        cd_command = "cd"
+        if platform.system() == "Windows":
+            cd_command += " /d"
+        # Context.cd() is not used since it doesn't work correctly on Windows.
+        # It escapes spaces in the path using "\ " but it doesn't always work,
+        # wrapping the path in quotation marks is the safest approach
+        with run_context.prefix(f'{cd_command} "{custom_working_dir}"'):
             return run_context.run(cli_full_line, echo=False, hide=True, warn=True, env=env)
 
     return _run
@@ -112,15 +125,23 @@ def daemon_runner(pytestconfig, data_dir, downloads_dir, working_dir):
         http://docs.pyinvoke.org/en/1.4/api/runners.html#invoke.runners.Local
         http://docs.pyinvoke.org/en/1.4/api/runners.html
     """
-    cli_full_line = os.path.join(str(pytestconfig.rootdir), "..", "arduino-cli daemon")
+    cli_full_line = str(Path(pytestconfig.rootdir).parent / "arduino-cli daemon")
     env = {
         "ARDUINO_DATA_DIR": data_dir,
         "ARDUINO_DOWNLOADS_DIR": downloads_dir,
         "ARDUINO_SKETCHBOOK_DIR": data_dir,
     }
-    os.makedirs(os.path.join(data_dir, "packages"))
+    (Path(data_dir) / "packages").mkdir()
     run_context = Context()
-    run_context.cd(working_dir)
+    # It might happen that we need to change directories between drives on Windows,
+    # in that case the "/d" flag must be used otherwise directory wouldn't change
+    cd_command = "cd"
+    if platform.system() == "Windows":
+        cd_command += " /d"
+    # Context.cd() is not used since it doesn't work correctly on Windows.
+    # It escapes spaces in the path using "\ " but it doesn't always work,
+    # wrapping the path in quotation marks is the safest approach
+    run_context.prefix(f'{cd_command} "{working_dir}"')
     # Local Class is the implementation of a Runner abstract class
     runner = Local(run_context)
     runner.run(cli_full_line, echo=False, hide=True, warn=True, env=env, asynchronous=True)
@@ -165,3 +186,12 @@ def detected_boards(run_command):
             )
 
     return detected_boards
+
+
+@pytest.fixture(scope="function")
+def copy_sketch(working_dir):
+    # Copies sketch for testing
+    sketch_path = Path(__file__).parent / "testdata" / "sketch_simple"
+    test_sketch_path = Path(working_dir) / "sketch_simple"
+    shutil.copytree(sketch_path, test_sketch_path)
+    yield str(test_sketch_path)

test/test_config.py

@@ -12,7 +12,7 @@
 # otherwise use the software for commercial activities involving the Arduino
 # software without disclosing the source code of your own applications. To purchase
 # a commercial license, send an email to license@arduino.cc.
-import os
+from pathlib import Path
 
 
 def test_init(run_command, data_dir, working_dir):
@@ -22,7 +22,7 @@ def test_init(run_command, data_dir, working_dir):
 
 
 def test_init_dest(run_command, working_dir):
-    dest = os.path.join(working_dir, "config", "test")
-    result = run_command("config init --dest-dir " + dest)
+    dest = str(Path(working_dir) / "config" / "test")
+    result = run_command(f'config init --dest-dir "{dest}"')
     assert result.ok
     assert dest in result.stdout

test/testdata/sketch_simple/build/arduino.avr.uno/sketch_simple.ino.eep

+:00000001FF

test/testdata/sketch_simple/build/arduino.avr.uno/sketch_simple.ino.hex

+:100000000C9434000C9446000C9446000C9446006A
+:100010000C9446000C9446000C9446000C94460048
+:100020000C9446000C9446000C9446000C94460038
+:100030000C9446000C9446000C9446000C94460028
+:100040000C9448000C9446000C9446000C94460016
+:100050000C9446000C9446000C9446000C94460008
+:100060000C9446000C94460011241FBECFEFD8E03C
+:10007000DEBFCDBF21E0A0E0B1E001C01D92A930FC
+:10008000B207E1F70E9492000C94DC000C9400008F
+:100090001F920F920FB60F9211242F933F938F93BD
+:1000A0009F93AF93BF938091050190910601A0911A
+:1000B0000701B09108013091040123E0230F2D378F
+:1000C00058F50196A11DB11D2093040180930501EF
+:1000D00090930601A0930701B0930801809100015D
+:1000E00090910101A0910201B09103010196A11D1F
+:1000F000B11D8093000190930101A0930201B09380
+:100100000301BF91AF919F918F913F912F910F90DC
+:100110000FBE0F901F90189526E8230F0296A11D81
+:10012000B11DD2CF789484B5826084BD84B58160DE
+:1001300084BD85B5826085BD85B5816085BD8091B2
+:100140006E00816080936E0010928100809181002A
+:100150008260809381008091810081608093810022
+:10016000809180008160809380008091B1008460E4
+:100170008093B1008091B00081608093B000809145
+:100180007A00846080937A0080917A008260809304
+:100190007A0080917A00816080937A0080917A0061
+:1001A000806880937A001092C100C0E0D0E0209770
+:0C01B000F1F30E940000FBCFF894FFCF99
+:00000001FF

test/testdata/sketch_simple/build/arduino.avr.uno/sketch_simple.ino.with_bootloader.hex

+:020000040000FA
+:100000000C9434000C9446000C9446000C9446006A
+:100010000C9446000C9446000C9446000C94460048
+:100020000C9446000C9446000C9446000C94460038
+:100030000C9446000C9446000C9446000C94460028
+:100040000C9448000C9446000C9446000C94460016
+:100050000C9446000C9446000C9446000C94460008
+:100060000C9446000C94460011241FBECFEFD8E03C
+:10007000DEBFCDBF21E0A0E0B1E001C01D92A930FC
+:10008000B207E1F70E9492000C94DC000C9400008F
+:100090001F920F920FB60F9211242F933F938F93BD
+:1000A0009F93AF93BF938091050190910601A0911A
+:1000B0000701B09108013091040123E0230F2D378F
+:1000C00058F50196A11DB11D2093040180930501EF
+:1000D00090930601A0930701B0930801809100015D
+:1000E00090910101A0910201B09103010196A11D1F
+:1000F000B11D8093000190930101A0930201B09380
+:100100000301BF91AF919F918F913F912F910F90DC
+:100110000FBE0F901F90189526E8230F0296A11D81
+:10012000B11DD2CF789484B5826084BD84B58160DE
+:1001300084BD85B5826085BD85B5816085BD8091B2
+:100140006E00816080936E0010928100809181002A
+:100150008260809381008091810081608093810022
+:10016000809180008160809380008091B1008460E4
+:100170008093B1008091B00081608093B000809145
+:100180007A00846080937A0080917A008260809304
+:100190007A0080917A00816080937A0080917A0061
+:1001A000806880937A001092C100C0E0D0E0209770
+:0C01B000F1F30E940000FBCFF894FFCF99
+:107E0000112484B714BE81FFF0D085E080938100F7
+:107E100082E08093C00088E18093C10086E0809377
+:107E2000C20080E18093C4008EE0C9D0259A86E02C
+:107E300020E33CEF91E0309385002093840096BBD3
+:107E4000B09BFECF1D9AA8958150A9F7CC24DD24C4
+:107E500088248394B5E0AB2EA1E19A2EF3E0BF2EE7
+:107E6000A2D0813461F49FD0082FAFD0023811F036
+:107E7000013811F484E001C083E08DD089C08234E0
+:107E800011F484E103C0853419F485E0A6D080C0E4
+:107E9000853579F488D0E82EFF2485D0082F10E0AE
+:107EA000102F00270E291F29000F111F8ED06801E7
+:107EB0006FC0863521F484E090D080E0DECF843638
+:107EC00009F040C070D06FD0082F6DD080E0C81688
+:107ED00080E7D80618F4F601B7BEE895C0E0D1E017
+:107EE00062D089930C17E1F7F0E0CF16F0E7DF06D8
+:107EF00018F0F601B7BEE89568D007B600FCFDCFD4
+:107F0000A601A0E0B1E02C9130E011968C91119780
+:107F100090E0982F8827822B932B1296FA010C0160
+:107F200087BEE89511244E5F5F4FF1E0A038BF0790
+:107F300051F7F601A7BEE89507B600FCFDCF97BE46
+:107F4000E89526C08437B1F42ED02DD0F82E2BD052
+:107F50003CD0F601EF2C8F010F5F1F4F84911BD097
+:107F6000EA94F801C1F70894C11CD11CFA94CF0C13
+:107F7000D11C0EC0853739F428D08EE10CD085E9AC
+:107F80000AD08FE07ACF813511F488E018D01DD067
+:107F900080E101D065CF982F8091C00085FFFCCF94
+:107FA0009093C60008958091C00087FFFCCF809118
+:107FB000C00084FD01C0A8958091C6000895E0E648
+:107FC000F0E098E1908380830895EDDF803219F02E
+:107FD00088E0F5DFFFCF84E1DECF1F93182FE3DFCA
+:107FE0001150E9F7F2DF1F91089580E0E8DFEE27F6
+:047FF000FF270994CA
+:027FFE00040479
+:00000001FF

test/testdata/sketch_simple/header.h

+#define TRUE FALSE
\ No newline at end of file

test/testdata/sketch_simple/merged_sketch.txt

+#include <Arduino.h>
+#line 1 {{QuoteCppString .sketch.MainFile.Name}}
+void setup() {
+
+}
+
+void loop() {
+
+}
+#line 1 {{QuoteCppString (index .sketch.OtherSketchFiles 0).Name}}
+
+#line 1 {{QuoteCppString (index .sketch.OtherSketchFiles 1).Name}}
+String hello() {
+  return "world";
+}

test/testdata/sketch_simple/other.ino

+String hello() {
+  return "world";
+}
\ No newline at end of file

test/testdata/sketch_simple/sketch_simple.ino

+void setup() {
+
+}
+
+void loop() {
+
+}
\ No newline at end of file