[skip-changelog]Migrate tests from `upload_test.py` to `test_upload.go` (#2001)

* Migrate TestUpload from test_upload.py to upload_test.go

* Migrate TestUploadWithInputDirFlag from test_upload.py to upload_test.go

* Migrate TestUploadWithInputFileFlag from test_upload.py to upload_test.go

* Migrate TestCompileAndUploadCombo from test_upload.py to upload_test.go

* Migrate TestCompileAndUploadComboWithCustomBuildPath from test_upload.py to upload_test.go

* Migrate TestCompileAndUploadComboWithPdeExtension from test_upload.py to upload_test.go

* Migrate TestUploadSketchWithPdeExtension from test_upload.py to upload_test.go

* Migrate TestUploadWithInputDirContainingMultipleBinaries from test_upload.py to upload_test.go

* Migrate TestCompileAndUploadComboSketchWithMismatchedCasing from test_upload.py to upload_test.go

* Migrate TestUploadSketchWithMismatchedCasing from test_upload.py to upload_test.go

* Migrate TestUploadToPortWithBoardAutodetect from test_upload.py to upload_test.go

* Migrate TestCompileAndUploadToPortWithBoardAutodetect to upload_test.go and delete test_upload.py

internal/integrationtest/upload/upload_test.go

+// This file is part of arduino-cli.
+//
+// Copyright 2022 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to license@arduino.cc.
+
+package upload_test
+
+import (
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/arduino/arduino-cli/internal/integrationtest"
+	"github.com/arduino/go-paths-helper"
+	"github.com/stretchr/testify/require"
+	"go.bug.st/testifyjson/requirejson"
+)
+
+type board struct {
+	address      string
+	fqbn         string
+	pack         string
+	architecture string
+	id           string
+	core         string
+}
+
+func detectedBoards(t *testing.T, cli *integrationtest.ArduinoCLI) []board {
+	// This fixture provides a list of all the boards attached to the host.
+	// This fixture will parse the JSON output of `arduino-cli board list --format json`
+	//to extract all the connected boards data.
+
+	// :returns a list `Board` objects.
+
+	var boards []board
+	stdout, _, err := cli.Run("board", "list", "--format", "json")
+	require.NoError(t, err)
+	len, err := strconv.Atoi(requirejson.Parse(t, stdout).Query(".[] | .matching_boards | length").String())
+	require.NoError(t, err)
+	for i := 0; i < len; i++ {
+		fqbn := strings.Trim(requirejson.Parse(t, stdout).Query(".[] | .matching_boards | .["+fmt.Sprint(i)+"] | .fqbn").String(), "\"")
+		boards = append(boards, board{
+			address:      strings.Trim(requirejson.Parse(t, stdout).Query(".[] | .port | .address").String(), "\""),
+			fqbn:         fqbn,
+			pack:         strings.Split(fqbn, ":")[0],
+			architecture: strings.Split(fqbn, ":")[1],
+			id:           strings.Split(fqbn, ":")[2],
+			core:         strings.Split(fqbn, ":")[0] + ":" + strings.Split(fqbn, ":")[1],
+		})
+	}
+	return boards
+}
+
+func waitForBoard(t *testing.T, cli *integrationtest.ArduinoCLI) {
+	timeEnd := time.Now().Unix() + 10
+	for time.Now().Unix() < timeEnd {
+		stdout, _, err := cli.Run("board", "list", "--format", "json")
+		require.NoError(t, err)
+		len, err := strconv.Atoi(requirejson.Parse(t, stdout).Query("length").String())
+		require.NoError(t, err)
+		numBoards := 0
+		for i := 0; i < len; i++ {
+			numBoards, err = strconv.Atoi(requirejson.Parse(t, stdout).Query(".[] | .matching_boards | length").String())
+			require.NoError(t, err)
+			if numBoards > 0 {
+				break
+			}
+		}
+		if numBoards > 0 {
+			break
+		}
+	}
+}
+
+func TestUpload(t *testing.T) {
+	if os.Getenv("CI") != "" {
+		t.Skip("VMs have no serial ports")
+	}
+
+	env, cli := integrationtest.CreateArduinoCLIWithEnvironment(t)
+	defer env.CleanUp()
+
+	// Init the environment explicitly
+	_, _, err := cli.Run("core", "update-index")
+	require.NoError(t, err)
+
+	for _, board := range detectedBoards(t, cli) {
+		// Download platform
+		_, _, err = cli.Run("core", "install", board.core)
+		require.NoError(t, err)
+		// Create a sketch
+		sketchName := "TestUploadSketch" + board.id
+		sketchPath := cli.SketchbookDir().Join(sketchName)
+		fqbn := board.fqbn
+		address := board.address
+		_, _, err = cli.Run("sketch", "new", sketchPath.String())
+		require.NoError(t, err)
+		// Build sketch
+		_, _, err = cli.Run("compile", "-b", fqbn, sketchPath.String())
+		require.NoError(t, err)
+
+		// Verifies binaries are not exported
+		require.NoFileExists(t, sketchPath.Join("build").String())
+
+		// Upload without port must fail
+		_, _, err = cli.Run("upload", "-b", fqbn, sketchPath.String())
+		require.Error(t, err)
+
+		// Upload
+		_, _, err = cli.Run("upload", "-b", fqbn, "-p", address, sketchPath.String())
+		require.NoError(t, err)
+	}
+}
+
+func TestUploadWithInputDirFlag(t *testing.T) {
+	if os.Getenv("CI") != "" {
+		t.Skip("VMs have no serial ports")
+	}
+
+	env, cli := integrationtest.CreateArduinoCLIWithEnvironment(t)
+	defer env.CleanUp()
+
+	// Init the environment explicitly
+	_, _, err := cli.Run("core", "update-index")
+	require.NoError(t, err)
+
+	for _, board := range detectedBoards(t, cli) {
+		// Download board platform
+		_, _, err = cli.Run("core", "install", board.core)
+		require.NoError(t, err)
+
+		// Create a sketch
+		sketchName := "TestUploadSketch" + board.id
+		sketchPath := cli.SketchbookDir().Join(sketchName)
+		fqbn := board.fqbn
+		address := board.address
+		_, _, err = cli.Run("sketch", "new", sketchPath.String())
+		require.NoError(t, err)
+
+		// Build sketch and export binaries to custom directory
+		outputDir := cli.SketchbookDir().Join("test_dir", sketchName, "build")
+		_, _, err = cli.Run("compile", "-b", fqbn, sketchPath.String(), "--output-dir", outputDir.String())
+		require.NoError(t, err)
+
+		// Upload with --input-dir flag
+		_, _, err = cli.Run("upload", "-b", fqbn, "-p", address, "--input-dir", outputDir.String())
+		require.NoError(t, err)
+	}
+}
+
+func TestUploadWithInputFileFlag(t *testing.T) {
+	if os.Getenv("CI") != "" {
+		t.Skip("VMs have no serial ports")
+	}
+
+	env, cli := integrationtest.CreateArduinoCLIWithEnvironment(t)
+	defer env.CleanUp()
+
+	// Init the environment explicitly
+	_, _, err := cli.Run("core", "update-index")
+	require.NoError(t, err)
+
+	for _, board := range detectedBoards(t, cli) {
+		// Download board platform
+		_, _, err = cli.Run("core", "install", board.core)
+		require.NoError(t, err)
+
+		// Create a sketch
+		sketchName := "TestUploadSketch" + board.id
+		sketchPath := cli.SketchbookDir().Join(sketchName)
+		fqbn := board.fqbn
+		address := board.address
+		_, _, err = cli.Run("sketch", "new", sketchPath.String())
+		require.NoError(t, err)
+
+		// Build sketch and export binaries to custom directory
+		outputDir := cli.SketchbookDir().Join("test_dir", sketchName, "build")
+		_, _, err = cli.Run("compile", "-b", fqbn, sketchPath.String(), "--output-dir", outputDir.String())
+		require.NoError(t, err)
+
+		// We don't need a specific file when using the --input-file flag to upload since
+		// it's just used to calculate the directory, so it's enough to get a random file
+		// that's inside that directory
+		inputFile := outputDir.Join(sketchName + ".ino.bin")
+		// Upload using --input-file
+		_, _, err = cli.Run("upload", "-b", fqbn, "-p", address, "--input-file", inputFile.String())
+		require.NoError(t, err)
+	}
+}
+
+func TestCompileAndUploadCombo(t *testing.T) {
+	if os.Getenv("CI") != "" {
+		t.Skip("VMs have no serial ports")
+	}
+
+	env, cli := integrationtest.CreateArduinoCLIWithEnvironment(t)
+	defer env.CleanUp()
+
+	// Init the environment explicitly
+	_, _, err := cli.Run("core", "update-index")
+	require.NoError(t, err)
+
+	// Create a test sketch
+	sketchName := "CompileAndUploadIntegrationTest"
+	sketchPath := cli.SketchbookDir().Join(sketchName)
+	sketchMainFile := sketchPath.Join(sketchName + ".ino")
+	stdout, _, err := cli.Run("sketch", "new", sketchPath.String())
+	require.NoError(t, err)
+	require.Contains(t, string(stdout), "Sketch created in: "+sketchPath.String())
+
+	// Build sketch for each detected board
+	for _, board := range detectedBoards(t, cli) {
+		logFileName := strings.ReplaceAll(board.fqbn, ":", "-") + "-compile.log"
+		logFilePath := cli.SketchbookDir().Join(logFileName)
+
+		_, _, err = cli.Run("core", "install", board.core)
+		require.NoError(t, err)
+
+		runTest := func(s string) {
+			waitForBoard(t, cli)
+			_, _, err := cli.Run("compile", "-b", board.fqbn, "--upload", "-p", board.address, s,
+				"--log-format", "json", "--log-file", logFilePath.String(), "--log-level", "trace")
+			require.NoError(t, err)
+			logJson, err := logFilePath.ReadFile()
+			require.NoError(t, err)
+
+			// check from the logs if the bin file were uploaded on the current board
+			logJson = []byte("[" + strings.ReplaceAll(strings.TrimSuffix(string(logJson), "\n"), "\n", ",") + "]")
+			traces := requirejson.Parse(t, logJson).Query("[ .[] | select(.level==\"trace\") | .msg ]").String()
+			traces = strings.ReplaceAll(traces, "\\\\", "\\")
+			require.Contains(t, traces, "Compile "+sketchPath.String()+" for "+board.fqbn+" started")
+			require.Contains(t, traces, "Compile "+sketchName+" for "+board.fqbn+" successful")
+			require.Contains(t, traces, "Upload "+sketchPath.String()+" on "+board.fqbn+" started")
+			require.Contains(t, traces, "Upload successful")
+		}
+
+		runTest(sketchPath.String())
+		runTest(sketchMainFile.String())
+	}
+}
+
+func TestCompileAndUploadComboWithCustomBuildPath(t *testing.T) {
+	if os.Getenv("CI") != "" {
+		t.Skip("VMs have no serial ports")
+	}
+
+	env, cli := integrationtest.CreateArduinoCLIWithEnvironment(t)
+	defer env.CleanUp()
+
+	// Init the environment explicitly
+	_, _, err := cli.Run("core", "update-index")
+	require.NoError(t, err)
+
+	// Create a test sketch
+	sketchName := "CompileAndUploadCustomBuildPathIntegrationTest"
+	sketchPath := cli.SketchbookDir().Join(sketchName)
+	_, _, err = cli.Run("sketch", "new", sketchPath.String())
+	require.NoError(t, err)
+
+	// Build sketch for each detected board
+	for _, board := range detectedBoards(t, cli) {
+		fqbnNormalized := strings.ReplaceAll(board.fqbn, ":", "-")
+		logFileName := fqbnNormalized + "-compile.log"
+		logFilePath := cli.SketchbookDir().Join(logFileName)
+
+		_, _, err = cli.Run("core", "install", board.core)
+		require.NoError(t, err)
+
+		waitForBoard(t, cli)
+
+		buildPath := cli.SketchbookDir().Join("test_dir", fqbnNormalized, "build_dir")
+		_, _, err := cli.Run("compile", "-b", board.fqbn, "--upload", "-p", board.address, "--build-path", buildPath.String(),
+			sketchPath.String(), "--log-format", "json", "--log-file", logFilePath.String(), "--log-level", "trace")
+		require.NoError(t, err)
+		logJson, err := logFilePath.ReadFile()
+		require.NoError(t, err)
+
+		// check from the logs if the bin file were uploaded on the current board
+		logJson = []byte("[" + strings.ReplaceAll(strings.TrimSuffix(string(logJson), "\n"), "\n", ",") + "]")
+		traces := requirejson.Parse(t, logJson).Query("[ .[] | select(.level==\"trace\") | .msg ]").String()
+		traces = strings.ReplaceAll(traces, "\\\\", "\\")
+		require.Contains(t, traces, "Compile "+sketchPath.String()+" for "+board.fqbn+" started")
+		require.Contains(t, traces, "Compile "+sketchName+" for "+board.fqbn+" successful")
+		require.Contains(t, traces, "Upload "+sketchPath.String()+" on "+board.fqbn+" started")
+		require.Contains(t, traces, "Upload successful")
+	}
+}
+
+func TestCompileAndUploadComboSketchWithPdeExtension(t *testing.T) {
+	if os.Getenv("CI") != "" {
+		t.Skip("VMs have no serial ports")
+	}
+
+	env, cli := integrationtest.CreateArduinoCLIWithEnvironment(t)
+	defer env.CleanUp()
+
+	_, _, err := cli.Run("update")
+	require.NoError(t, err)
+
+	sketchName := "CompileAndUploadPdeSketch"
+	sketchPath := cli.SketchbookDir().Join(sketchName)
+
+	// Create a test sketch
+	_, _, err = cli.Run("sketch", "new", sketchPath.String())
+	require.NoError(t, err)
+
+	// Renames sketch file to pde
+	sketchFile := sketchPath.Join(sketchName + ".pde")
+	require.NoError(t, sketchPath.Join(sketchName+".ino").Rename(sketchFile))
+
+	for _, board := range detectedBoards(t, cli) {
+		// Install core
+		_, _, err = cli.Run("core", "install", board.core)
+		require.NoError(t, err)
+
+		// Build sketch and upload from folder
+		waitForBoard(t, cli)
+		_, stderr, err := cli.Run("compile", "--clean", "-b", board.fqbn, "-u", "-p", board.address, sketchPath.String())
+		require.NoError(t, err)
+		require.Contains(t, string(stderr), "Sketches with .pde extension are deprecated, please rename the following files to .ino")
+		require.Contains(t, string(stderr), sketchFile.String())
+
+		// Build sketch and upload from file
+		waitForBoard(t, cli)
+		_, stderr, err = cli.Run("compile", "--clean", "-b", board.fqbn, "-u", "-p", board.address, sketchFile.String())
+		require.NoError(t, err)
+		require.Contains(t, string(stderr), "Sketches with .pde extension are deprecated, please rename the following files to .ino")
+		require.Contains(t, string(stderr), sketchFile.String())
+	}
+}
+
+func TestUploadSketchWithPdeExtension(t *testing.T) {
+	if os.Getenv("CI") != "" {
+		t.Skip("VMs have no serial ports")
+	}
+
+	env, cli := integrationtest.CreateArduinoCLIWithEnvironment(t)
+	defer env.CleanUp()
+
+	_, _, err := cli.Run("update")
+	require.NoError(t, err)
+
+	sketchName := "UploadPdeSketch"
+	sketchPath := cli.SketchbookDir().Join(sketchName)
+
+	// Create a test sketch
+	_, _, err = cli.Run("sketch", "new", sketchPath.String())
+	require.NoError(t, err)
+
+	// Renames sketch file to pde
+	sketchFile := sketchPath.Join(sketchName + ".pde")
+	require.NoError(t, sketchPath.Join(sketchName+".ino").Rename(sketchFile))
+
+	for _, board := range detectedBoards(t, cli) {
+		// Install core
+		_, _, err = cli.Run("core", "install", board.core)
+		require.NoError(t, err)
+
+		// Compile sketch first
+		stdout, _, err := cli.Run("compile", "--clean", "-b", board.fqbn, sketchPath.String(), "--format", "json")
+		require.NoError(t, err)
+		buildDir := requirejson.Parse(t, stdout).Query(".builder_result | .build_path").String()
+		buildDir = strings.Trim(strings.ReplaceAll(buildDir, "\\\\", "\\"), "\"")
+
+		// Upload from sketch folder
+		waitForBoard(t, cli)
+		_, _, err = cli.Run("upload", "-b", board.fqbn, "-p", board.address, sketchPath.String())
+		require.NoError(t, err)
+
+		// Upload from sketch file
+		waitForBoard(t, cli)
+		_, _, err = cli.Run("upload", "-b", board.fqbn, "-p", board.address, sketchFile.String())
+		require.NoError(t, err)
+
+		waitForBoard(t, cli)
+		_, stderr, err := cli.Run("upload", "-b", board.fqbn, "-p", board.address, "--input-dir", buildDir)
+		require.NoError(t, err)
+		require.Contains(t, string(stderr), "Sketches with .pde extension are deprecated, please rename the following files to .ino:")
+
+		// Upload from binary file
+		waitForBoard(t, cli)
+		// We don't need a specific file when using the --input-file flag to upload since
+		// it's just used to calculate the directory, so it's enough to get a random file
+		// that's inside that directory
+		binaryFile := paths.New(buildDir, sketchName+".pde.bin")
+		_, stderr, err = cli.Run("upload", "-b", board.fqbn, "-p", board.address, "--input-file", binaryFile.String())
+		require.NoError(t, err)
+		require.Contains(t, string(stderr), "Sketches with .pde extension are deprecated, please rename the following files to .ino:")
+	}
+}
+
+func TestUploadWithInputDirContainingMultipleBinaries(t *testing.T) {
+	if os.Getenv("CI") != "" {
+		t.Skip("VMs have no serial ports")
+	}
+
+	env, cli := integrationtest.CreateArduinoCLIWithEnvironment(t)
+	defer env.CleanUp()
+
+	// This tests verifies the behaviour outlined in this issue:
+	// https://github.com/arduino/arduino-cli/issues/765#issuecomment-699678646
+	_, _, err := cli.Run("update")
+	require.NoError(t, err)
+
+	// Create two different sketches
+	sketchOneName := "UploadMultipleBinariesSketchOne"
+	sketchOnePath := cli.SketchbookDir().Join(sketchOneName)
+	_, _, err = cli.Run("sketch", "new", sketchOnePath.String())
+	require.NoError(t, err)
+
+	sketchTwoName := "UploadMultipleBinariesSketchTwo"
+	sketchTwoPath := cli.SketchbookDir().Join(sketchTwoName)
+	_, _, err = cli.Run("sketch", "new", sketchTwoPath.String())
+	require.NoError(t, err)
+
+	for _, board := range detectedBoards(t, cli) {
+		// Install core
+		_, _, err = cli.Run("core", "install", board.core)
+		require.NoError(t, err)
+
+		// Compile both sketches and copy binaries in the same build directory
+		binariesDir := cli.SketchbookDir().Join("build", "BuiltBinaries")
+		_, _, err = cli.Run("compile", "--clean", "-b", board.fqbn, sketchOnePath.String(), "--build-path", binariesDir.String())
+		require.NoError(t, err)
+		stdout, _, err := cli.Run("compile", "--clean", "-b", board.fqbn, sketchTwoPath.String(), "--format", "json")
+		require.NoError(t, err)
+		buildDirTwo := requirejson.Parse(t, stdout).Query(".builder_result | .build_path").String()
+		buildDirTwo = strings.Trim(strings.ReplaceAll(buildDirTwo, "\\\\", "\\"), "\"")
+		require.NoError(t, paths.New(buildDirTwo).Join(sketchTwoName+".ino.bin").CopyTo(binariesDir.Join(sketchTwoName+".ino.bin")))
+
+		waitForBoard(t, cli)
+		// Verifies upload fails because multiple binaries are found
+		_, stderr, err := cli.Run("upload", "-b", board.fqbn, "-p", board.address, "--input-dir", binariesDir.String())
+		require.Error(t, err)
+		require.Contains(t, string(stderr), "Error during Upload: ")
+		require.Contains(t, string(stderr), "Error finding build artifacts: ")
+		require.Contains(t, string(stderr), "autodetect build artifact: ")
+		require.Contains(t, string(stderr), "multiple build artifacts found:")
+
+		// Copy binaries to folder with same name of a sketch
+		binariesDirSketch := cli.SketchbookDir().Join("build", "UploadMultipleBinariesSketchOne")
+		require.NoError(t, binariesDir.CopyDirTo(binariesDirSketch))
+
+		waitForBoard(t, cli)
+		// Verifies upload is successful using the binaries with the same name of the containing folder
+		_, _, err = cli.Run("upload", "-b", board.fqbn, "-p", board.address, "--input-dir", binariesDirSketch.String())
+		require.NoError(t, err)
+	}
+}
+
+func TestCompileAndUploadComboSketchWithMismatchedCasing(t *testing.T) {
+	if os.Getenv("CI") != "" {
+		t.Skip("VMs have no serial ports")
+	}
+
+	env, cli := integrationtest.CreateArduinoCLIWithEnvironment(t)
+	defer env.CleanUp()
+
+	_, _, err := cli.Run("update")
+	require.NoError(t, err)
+
+	// Create a sketch
+	sketchName := "CompileUploadComboMismatchCasing"
+	sketchPath := cli.SketchbookDir().Join(sketchName)
+	_, _, err = cli.Run("sketch", "new", sketchPath.String())
+	require.NoError(t, err)
+
+	// Rename main .ino file so casing is different from sketch name
+	require.NoError(t, sketchPath.Join(sketchName+".ino").Rename(sketchPath.Join(strings.ToLower(sketchName)+".ino")))
+
+	for _, board := range detectedBoards(t, cli) {
+		// Install core
+		_, _, err = cli.Run("core", "install", board.core)
+		require.NoError(t, err)
+
+		// Try to compile
+		_, stderr, err := cli.Run("compile", "--clean", "-b", board.fqbn, "-u", "-p", board.address, sketchPath.String())
+		require.Error(t, err)
+		require.Contains(t, string(stderr), "Error opening sketch:")
+	}
+}
+
+func TestUploadSketchWithMismatchedCasing(t *testing.T) {
+	if os.Getenv("CI") != "" {
+		t.Skip("VMs have no serial ports")
+	}
+
+	env, cli := integrationtest.CreateArduinoCLIWithEnvironment(t)
+	defer env.CleanUp()
+
+	_, _, err := cli.Run("update")
+	require.NoError(t, err)
+
+	// Create a sketch
+	sketchName := "UploadMismatchCasing"
+	sketchPath := cli.SketchbookDir().Join(sketchName)
+	_, _, err = cli.Run("sketch", "new", sketchPath.String())
+	require.NoError(t, err)
+
+	// Rename main .ino file so casing is different from sketch name
+	require.NoError(t, sketchPath.Join(sketchName+".ino").Rename(sketchPath.Join(strings.ToLower(sketchName)+".ino")))
+
+	for _, board := range detectedBoards(t, cli) {
+		// Install core
+		_, _, err = cli.Run("core", "install", board.core)
+		require.NoError(t, err)
+
+		// Tries to upload given sketch, it has not been compiled but it fails even before
+		// searching for binaries since the sketch is not valid
+		_, stderr, err := cli.Run("upload", "-b", board.fqbn, "-p", board.address, sketchPath.String())
+		require.Error(t, err)
+		require.Contains(t, string(stderr), "Error during Upload:")
+	}
+}
+
+func TestUploadToPortWithBoardAutodetect(t *testing.T) {
+	if os.Getenv("CI") != "" {
+		t.Skip("VMs have no serial ports")
+	}
+
+	env, cli := integrationtest.CreateArduinoCLIWithEnvironment(t)
+	defer env.CleanUp()
+
+	_, _, err := cli.Run("update")
+	require.NoError(t, err)
+
+	// Create a sketch
+	sketchPath := cli.SketchbookDir().Join("SketchSimple")
+	_, _, err = cli.Run("sketch", "new", sketchPath.String())
+	require.NoError(t, err)
+
+	for _, board := range detectedBoards(t, cli) {
+		// Install core
+		_, _, err = cli.Run("core", "install", board.core)
+		require.NoError(t, err)
+
+		_, _, err = cli.Run("compile", "-b", board.fqbn, sketchPath.String())
+		require.NoError(t, err)
+
+		_, _, err = cli.Run("upload", "-p", board.address, sketchPath.String())
+		require.NoError(t, err)
+	}
+}
+
+func TestCompileAndUploadToPortWithBoardAutodetect(t *testing.T) {
+	if os.Getenv("CI") != "" {
+		t.Skip("VMs have no serial ports")
+	}
+
+	env, cli := integrationtest.CreateArduinoCLIWithEnvironment(t)
+	defer env.CleanUp()
+
+	_, _, err := cli.Run("update")
+	require.NoError(t, err)
+
+	// Create a sketch
+	sketchPath := cli.SketchbookDir().Join("SketchSimple")
+	_, _, err = cli.Run("sketch", "new", sketchPath.String())
+	require.NoError(t, err)
+
+	for _, board := range detectedBoards(t, cli) {
+		// Install core
+		_, _, err = cli.Run("core", "install", board.core)
+		require.NoError(t, err)
+
+		_, _, err = cli.Run("compile", "-u", "-p", board.address, sketchPath.String())
+		require.NoError(t, err)
+	}
+}

test/test_upload.py

-# This file is part of arduino-cli.
-#
-# Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
-#
-# This software is released under the GNU General Public License version 3,
-# which covers the main part of arduino-cli.
-# The terms of this license can be found at:
-# https://www.gnu.org/licenses/gpl-3.0.en.html
-#
-# You can be released from the requirements of the above licenses by purchasing
-# a commercial license. Buying such a license is mandatory if you want to modify or
-# otherwise use the software for commercial activities involving the Arduino
-# software without disclosing the source code of your own applications. To purchase
-# a commercial license, send an email to license@arduino.cc.
-import os
-import shutil
-import json
-from pathlib import Path
-
-import pytest
-
-from .common import running_on_ci, parse_json_traces
-
-# Skip this module when running in CI environments
-pytestmark = pytest.mark.skipif(running_on_ci(), reason="VMs have no serial ports")
-
-
-def test_upload(run_command, data_dir, detected_boards):
-    # Init the environment explicitly
-    run_command(["core", "update-index"])
-
-    for board in detected_boards:
-        # Download platform
-        run_command(["core", "install", board.core])
-        # Create a sketch
-        sketch_name = f"TestUploadSketch{board.id}"
-        sketch_path = Path(data_dir, sketch_name)
-        fqbn = board.fqbn
-        address = board.address
-        assert run_command(["sketch", "new", sketch_path])
-        # Build sketch
-        assert run_command(["compile", "-b", fqbn, sketch_path])
-
-        # Verifies binaries are not exported
-        assert not (sketch_path / "build").exists()
-
-        # Upload without port must fail
-        assert not run_command(["upload", "-b", fqbn, sketch_path])
-
-        # Upload
-        assert run_command(["upload", "-b", fqbn, "-p", address, sketch_path])
-
-
-def test_upload_with_input_dir_flag(run_command, data_dir, detected_boards):
-    # Init the environment explicitly
-    run_command(["core", "update-index"])
-
-    for board in detected_boards:
-        # Download board platform
-        run_command(["core", "install", board.core])
-
-        # Create sketch
-        sketch_name = f"TestUploadInputDirSketch{board.id}"
-        sketch_path = Path(data_dir, sketch_name)
-        fqbn = board.fqbn
-        address = board.address
-        assert run_command(["sketch", "new", sketch_path])
-
-        # Build sketch and export binaries to custom directory
-        output_dir = Path(data_dir, "test_dir", sketch_name, "build")
-        assert run_command(["compile", "-b", fqbn, sketch_path, "--output-dir", output_dir])
-
-        # Upload with --input-dir flag
-        assert run_command(["upload", "-b", fqbn, "-p", address, "--input-dir", output_dir])
-
-
-def test_upload_with_input_file_flag(run_command, data_dir, detected_boards):
-    # Init the environment explicitly
-    run_command(["core", "update-index"])
-
-    for board in detected_boards:
-        # Download board platform
-        run_command(["core", "install", board.core])
-
-        # Create sketch
-        sketch_name = f"TestUploadInputFileSketch{board.id}"
-        sketch_path = Path(data_dir, sketch_name)
-        fqbn = board.fqbn
-        address = board.address
-        assert run_command(["sketch", "new", sketch_path])
-
-        # Build sketch and export binaries to custom directory
-        output_dir = Path(data_dir, "test_dir", sketch_name, "build")
-        assert run_command(["compile", "-b", fqbn, sketch_path, "--output-dir", output_dir])
-
-        # We don't need a specific file when using the --input-file flag to upload since
-        # it's just used to calculate the directory, so it's enough to get a random file
-        # that's inside that directory
-        input_file = next(output_dir.glob(f"{sketch_name}.ino.*"))
-        # Upload using --input-file
-        assert run_command(["upload", "-b", fqbn, "-p", address, "--input-file", input_file])
-
-
-def test_upload_after_attach(run_command, data_dir, detected_boards):
-    # Init the environment explicitly
-    run_command(["core", "update-index"])
-
-    for board in detected_boards:
-        # Download core
-        run_command(["core", "install", board.core])
-        # Create a sketch
-        sketch_path = os.path.join(data_dir, "foo")
-        assert run_command(["sketch", "new", sketch_path])
-        assert run_command(["board", "attach", "-p", board.address, sketch_path])
-        # Build sketch
-        assert run_command(["compile", sketch_path])
-        # Upload
-        assert run_command(["upload", sketch_path])
-
-
-def test_compile_and_upload_combo(run_command, data_dir, detected_boards, wait_for_board):
-    # Init the environment explicitly
-    run_command(["core", "update-index"])
-
-    # Install required core(s)
-    run_command(["core", "install", "arduino:avr@1.8.3"])
-    run_command(["core", "install", "arduino:samd@1.8.6"])
-
-    # Create a test sketch
-    sketch_name = "CompileAndUploadIntegrationTest"
-    sketch_path = os.path.join(data_dir, sketch_name)
-    sketch_main_file = os.path.join(sketch_path, sketch_name + ".ino")
-    result = run_command(["sketch", "new", sketch_path])
-    assert result.ok
-    assert "Sketch created in: {}".format(sketch_path) in result.stdout
-
-    # Build sketch for each detected board
-    for board in detected_boards:
-        log_file_name = "{fqbn}-compile.log".format(fqbn=board.fqbn.replace(":", "-"))
-        log_file_path = os.path.join(data_dir, log_file_name)
-        command_log_flags = ["--log-format", "json", "--log-file", log_file_path, "--log-level", "trace"]
-
-        def run_test(s):
-            wait_for_board()
-            result = run_command(["compile", "-b", board.fqbn, "--upload", "-p", board.address, s] + command_log_flags)
-            print(result.stderr)
-            assert result.ok
-
-            # check from the logs if the bin file were uploaded on the current board
-            log_json = open(log_file_path, "r")
-            traces = parse_json_traces(log_json.readlines())
-            assert f"Compile {sketch_path} for {board.fqbn} started" in traces
-            assert f"Compile {sketch_name} for {board.fqbn} successful" in traces
-            assert f"Upload {sketch_path} on {board.fqbn} started" in traces
-            assert "Upload successful" in traces
-
-        run_test(sketch_path)
-        run_test(sketch_main_file)
-
-
-def test_compile_and_upload_combo_with_custom_build_path(run_command, data_dir, detected_boards, wait_for_board):
-    # Init the environment explicitly
-    run_command(["core", "update-index"])
-
-    # Install required core(s)
-    run_command(["core", "install", "arduino:avr@1.8.3"])
-    run_command(["core", "install", "arduino:samd@1.8.6"])
-
-    sketch_name = "CompileAndUploadCustomBuildPathIntegrationTest"
-    sketch_path = Path(data_dir, sketch_name)
-    assert run_command(["sketch", "new", sketch_path])
-
-    for board in detected_boards:
-        fqbn_normalized = board.fqbn.replace(":", "-")
-        log_file_name = f"{fqbn_normalized}-compile.log"
-        log_file = Path(data_dir, log_file_name)
-        command_log_flags = ["--log-format", "json", "--log-file", log_file, "--log-level", "trace"]
-
-        wait_for_board()
-
-        build_path = Path(data_dir, "test_dir", fqbn_normalized, "build_dir")
-        result = run_command(
-            [
-                "compile",
-                "-b",
-                board.fqbn,
-                "--upload",
-                "-p",
-                board.address,
-                "--build-path",
-                build_path,
-                sketch_path,
-            ]
-            + command_log_flags
-        )
-        print(result.stderr)
-        assert result.ok
-
-        # check from the logs if the bin file were uploaded on the current board
-        log_json = open(log_file, "r")
-        traces = parse_json_traces(log_json.readlines())
-        assert f"Compile {sketch_path} for {board.fqbn} started" in traces
-        assert f"Compile {sketch_name} for {board.fqbn} successful" in traces
-        assert f"Upload {sketch_path} on {board.fqbn} started" in traces
-        assert "Upload successful" in traces
-
-
-def test_compile_and_upload_combo_sketch_with_pde_extension(run_command, data_dir, detected_boards, wait_for_board):
-    assert run_command(["update"])
-
-    sketch_name = "CompileAndUploadPdeSketch"
-    sketch_path = Path(data_dir, sketch_name)
-
-    # Create a test sketch
-    assert run_command(["sketch", "new", sketch_path])
-
-    # Renames sketch file to pde
-    sketch_file = Path(sketch_path, f"{sketch_name}.ino").rename(sketch_path / f"{sketch_name}.pde")
-
-    for board in detected_boards:
-        # Install core
-        core = ":".join(board.fqbn.split(":")[:2])
-        assert run_command(["core", "install", core])
-
-        # Build sketch and upload from folder
-        wait_for_board()
-        res = run_command(["compile", "--clean", "-b", board.fqbn, "-u", "-p", board.address, sketch_path])
-        assert res.ok
-        assert "Sketches with .pde extension are deprecated, please rename the following files to .ino" in res.stderr
-        assert str(sketch_file) in res.stderr
-
-        # Build sketch and upload from file
-        wait_for_board()
-        res = run_command(["compile", "--clean", "-b", board.fqbn, "-u", "-p", board.address, sketch_file])
-        assert res.ok
-        assert "Sketches with .pde extension are deprecated, please rename the following files to .ino" in res.stderr
-        assert str(sketch_file) in res.stderr
-
-
-def test_upload_sketch_with_pde_extension(run_command, data_dir, detected_boards, wait_for_board):
-    assert run_command(["update"])
-
-    sketch_name = "UploadPdeSketch"
-    sketch_path = Path(data_dir, sketch_name)
-
-    # Create a test sketch
-    assert run_command(["sketch", "new", sketch_path])
-
-    # Renames sketch file to pde
-    sketch_file = Path(sketch_path, f"{sketch_name}.ino").rename(sketch_path / f"{sketch_name}.pde")
-
-    for board in detected_boards:
-        # Install core
-        core = ":".join(board.fqbn.split(":")[:2])
-        assert run_command(["core", "install", core])
-
-        # Compile sketch first
-        res = run_command(["compile", "--clean", "-b", board.fqbn, sketch_path, "--format", "json"])
-        assert res.ok
-        data = json.loads(res.stdout)
-        build_dir = Path(data["builder_result"]["build_path"])
-
-        # Upload from sketch folder
-        wait_for_board()
-        assert run_command(["upload", "-b", board.fqbn, "-p", board.address, sketch_path])
-
-        # Upload from sketch file
-        wait_for_board()
-        assert run_command(["upload", "-b", board.fqbn, "-p", board.address, sketch_file])
-
-        wait_for_board()
-        res = run_command(["upload", "-b", board.fqbn, "-p", board.address, "--input-dir", build_dir])
-        assert (
-            "Sketches with .pde extension are deprecated, please rename the following files to .ino:" not in res.stderr
-        )
-
-        # Upload from binary file
-        wait_for_board()
-        # We don't need a specific file when using the --input-file flag to upload since
-        # it's just used to calculate the directory, so it's enough to get a random file
-        # that's inside that directory
-        binary_file = next(build_dir.glob(f"{sketch_name}.pde.*"))
-        res = run_command(["upload", "-b", board.fqbn, "-p", board.address, "--input-file", binary_file])
-        assert (
-            "Sketches with .pde extension are deprecated, please rename the following files to .ino:" not in res.stderr
-        )
-
-
-def test_upload_with_input_dir_containing_multiple_binaries(run_command, data_dir, detected_boards, wait_for_board):
-    # This tests verifies the behaviour outlined in this issue:
-    # https://github.com/arduino/arduino-cli/issues/765#issuecomment-699678646
-    assert run_command(["update"])
-
-    # Create a two different sketches
-    sketch_one_name = "UploadMultipleBinariesSketchOne"
-    sketch_one_path = Path(data_dir, sketch_one_name)
-    assert run_command(["sketch", "new", sketch_one_path])
-
-    sketch_two_name = "UploadMultipleBinariesSketchTwo"
-    sketch_two_path = Path(data_dir, sketch_two_name)
-    assert run_command(["sketch", "new", sketch_two_path])
-
-    for board in detected_boards:
-        # Install core
-        core = ":".join(board.fqbn.split(":")[:2])
-        assert run_command(["core", "install", core])
-
-        # Compile both sketches and copy binaries in the same directory same build directory
-        res = run_command(["compile", "--clean", "-b", board.fqbn, sketch_one_path, "--format", "json"])
-        assert res.ok
-        data = json.loads(res.stdout)
-        build_dir_one = Path(data["builder_result"]["build_path"])
-        res = run_command(["compile", "--clean", "-b", board.fqbn, sketch_two_path, "--format", "json"])
-        assert res.ok
-        data = json.loads(res.stdout)
-        build_dir_two = Path(data["builder_result"]["build_path"])
-
-        # Copy binaries to same folder
-        binaries_dir = Path(data_dir, "build", "BuiltBinaries")
-        shutil.copytree(build_dir_one, binaries_dir, dirs_exist_ok=True)
-        shutil.copytree(build_dir_two, binaries_dir, dirs_exist_ok=True)
-
-        wait_for_board()
-        # Verifies upload fails because multiple binaries are found
-        res = run_command(["upload", "-b", board.fqbn, "-p", board.address, "--input-dir", binaries_dir])
-        assert res.failed
-        assert (
-            "Error during Upload: "
-            + "Error finding build artifacts: "
-            + "autodetect build artifact: "
-            + "multiple build artifacts found:"
-            in res.stderr
-        )
-
-        # Copy binaries to folder with same name of a sketch
-        binaries_dir = Path(data_dir, "build", "UploadMultipleBinariesSketchOne")
-        shutil.copytree(build_dir_one, binaries_dir, dirs_exist_ok=True)
-        shutil.copytree(build_dir_two, binaries_dir, dirs_exist_ok=True)
-
-        wait_for_board()
-        # Verifies upload is successful using the binaries with the same name of the containing folder
-        res = run_command(["upload", "-b", board.fqbn, "-p", board.address, "--input-dir", binaries_dir])
-        assert (
-            "Sketches with .pde extension are deprecated, please rename the following files to .ino:" not in res.stderr
-        )
-
-
-def test_compile_and_upload_combo_sketch_with_mismatched_casing(run_command, data_dir, detected_boards, wait_for_board):
-    assert run_command(["update"])
-
-    # Create a sketch
-    sketch_name = "CompileUploadComboMismatchCasing"
-    sketch_path = Path(data_dir, sketch_name)
-    assert run_command(["sketch", "new", sketch_path])
-
-    # Rename main .ino file so casing is different from sketch name
-    Path(sketch_path, f"{sketch_name}.ino").rename(sketch_path / f"{sketch_name.lower()}.ino")
-
-    for board in detected_boards:
-        # Install core
-        core = ":".join(board.fqbn.split(":")[:2])
-        assert run_command(["core", "install", core])
-
-        # Try to compile
-        res = run_command(["compile", "--clean", "-b", board.fqbn, "-u", "-p", board.address, sketch_path])
-        assert res.failed
-        assert "Error opening sketch:" in res.stderr
-
-
-def test_upload_sketch_with_mismatched_casing(run_command, data_dir, detected_boards, wait_for_board):
-    assert run_command(["update"])
-
-    # Create a sketch
-    sketch_name = "UploadMismatchCasing"
-    sketch_path = Path(data_dir, sketch_name)
-    assert run_command(["sketch", "new", sketch_path])
-
-    # Rename main .ino file so casing is different from sketch name
-    Path(sketch_path, f"{sketch_name}.ino").rename(sketch_path / f"{sketch_name.lower()}.ino")
-
-    for board in detected_boards:
-        # Install core
-        core = ":".join(board.fqbn.split(":")[:2])
-        assert run_command(["core", "install", core])
-
-        # Tries to upload given sketch, it has not been compiled but it fails even before
-        # searching for binaries since the sketch is not valid
-        res = run_command(["upload", "-b", board.fqbn, "-p", board.address, sketch_path])
-        assert res.failed
-        assert "Error during Upload:" in res.stderr
-
-
-def test_upload_to_port_with_board_autodetect(run_command, data_dir, detected_boards):
-    assert run_command(["update"])
-
-    # Create a sketch
-    sketch_name = "SketchSimple"
-    sketch_path = Path(data_dir, sketch_name)
-    assert run_command(["sketch", "new", sketch_path])
-
-    for board in detected_boards:
-        # Install core
-        core = ":".join(board.fqbn.split(":")[:2])
-        assert run_command(["core", "install", core])
-
-        assert run_command(["compile", "-b", board.fqbn, sketch_path])
-
-        res = run_command(["upload", "-p", board.address, sketch_path])
-        assert res.ok
-
-
-def test_compile_and_upload_to_port_with_board_autodetect(run_command, data_dir, detected_boards):
-    assert run_command(["update"])
-
-    # Create a sketch
-    sketch_name = "SketchSimple"
-    sketch_path = Path(data_dir, sketch_name)
-    assert run_command(["sketch", "new", sketch_path])
-
-    for board in detected_boards:
-        # Install core
-        core = ":".join(board.fqbn.split(":")[:2])
-        assert run_command(["core", "install", core])
-
-        res = run_command(["compile", "-u", "-p", board.address, sketch_path])
-        assert res.ok