[skip changelog] Move integration tests to pytest (#564)

* move compilation tests to pytest

* moved core tests to pytest

* moved core install/upgrade/uninstall to pytest

* Added detected_boards fixture
Co-authored-by: Maurizio Branca <m.branca@arduino.cc>

* add upload tests, remove mocked ones

* use fixture

* docs + dont run go tests anymore

* remove debug prints, bump pytest to latest

* skip problematic test on win
Co-authored-by: Maurizio Branca <m.branca@arduino.cc>

CONTRIBUTING.md

@@ -60,6 +60,7 @@ see following paragraph):
 task test-integration
 ```
 ### Running only some tests
+
 By default, all tests from all go packages are run. To run only unit
 tests from one or more specific packages, you can set the TARGETS
 environment variable, e.g.:
@@ -75,11 +76,6 @@ Both can be combined as well, typically to run only a specific test:
 
     TEST_REGEX='^TestFindBoardWithFQBN$' TARGETS=./arduino/cores/packagemanager task test-unit
 
-For integration test, the same options are supported. Note that when not
-specified, `TEST_REGEX` defaults to "Integration" to select only
-integration tests, so if you specify a broader regex, this could cause
-non-integration tests to be run as well.
-
 ### Integration tests
 
 Being a command line interface, Arduino CLI is heavily interactive and it has to
@@ -95,7 +91,8 @@ assess the options are correctly understood and the output is what we expect.
 To run the full suite of integration tests you need an Arduino device attached
 to a serial port and a working Python environment. Chances are that you already
 have Python installed in your system, if this is not the case you can
-[download][3] the official distribution or use the package manager provided by your Operating System.
+[download][3] the official distribution or use the package manager provided by
+your Operating System.
 
 Some dependencies need to be installed before running the tests and to avoid
 polluting your global Python enviroment with dependencies that might be only
@@ -151,7 +148,8 @@ a list of items you can check before submitting a PR:
 * Maintain **clean commit history** and use **meaningful commit messages**.
   PRs with messy commit history are difficult to review and require a lot of
   work to be merged.
-* Your PR must pass all CI tests before we will merge it. If you're seeing an      error and don't think
+* Your PR must pass all CI tests before we will merge it. If you're seeing an
+  error and don't think
   it's your fault, it may not be! The reviewer will help you if there are test
   failures that seem
   not related to the change you are making.

Taskfile.yml

@@ -27,7 +27,6 @@ tasks:
   test-integration:
     desc: Run integration tests only
     cmds:
-      - go test -run '{{ default "Integration" .TEST_REGEX }}' {{ default "-v" .GOFLAGS }} -coverprofile=coverage_integ.txt {{ default .DEFAULT_TARGETS .TARGETS }} {{.TEST_LDFLAGS}}
       - pytest test
 
   test-legacy:

cli/testdata/Blink/Blink.ino

-
-void setup() {}
-void loop() {}

cli/testdata/Test3rdPartyCoreIntegration/arduino-cli.yaml

-board_manager:
-  additional_urls:
-    - https://raw.githubusercontent.com/espressif/arduino-esp32/gh-pages/package_esp32_index.json
\ No newline at end of file

cli/testdata/TestInvalidCoreURLIntegration/arduino-cli.yaml

-board_manager:
-  additional_urls:
-    - http://www.invalid-domain-asjkdakdhadjkh.com/package_example_index.json
\ No newline at end of file

test/README.md

 # Integration tests
 
-This dir contains integration tests, the aim is to test the Command Line Interface and its output
-from a pure user point of view.
+This dir contains integration tests, aimed to test the Command Line Interface
+and its output from a pure user point of view.
 
 ## Installation
 

test/common.py

@@ -13,6 +13,10 @@
 # software without disclosing the source code of your own applications. To purchase
 # a commercial license, send an email to license@arduino.cc.
 import os
+import collections
+
+
+Board = collections.namedtuple("Board", "address fqbn package architecture id core")
 
 
 def running_on_ci():

test/conftest.py

@@ -16,6 +16,9 @@ import os
 
 import pytest
 from invoke.context import Context
+import simplejson as json
+
+from .common import Board
 
 
 @pytest.fixture(scope="function")
@@ -73,3 +76,35 @@ def run_command(pytestconfig, data_dir, downloads_dir, working_dir):
             )
 
     return _run
+
+
+@pytest.fixture(scope="function")
+def detected_boards(run_command):
+    """
+    This fixture provides a list of all the boards attached to the host.
+    This fixture will parse the JSON output of `arduino-cli board list --format json`
+    to extract all the connected boards data.
+
+    :returns a list `Board` objects.
+    """
+    assert run_command("core update-index")
+    result = run_command("board list --format json")
+    assert result.ok
+
+    detected_boards = []
+    for port in json.loads(result.stdout):
+        for board in port.get("boards", []):
+            fqbn = board.get("FQBN")
+            package, architecture, _id = fqbn.split(":")
+            detected_boards.append(
+                Board(
+                    address=port.get("address"),
+                    fqbn=fqbn,
+                    package=package,
+                    architecture=architecture,
+                    id=_id,
+                    core="{}:{}".format(package, architecture),
+                )
+            )
+
+    return detected_boards

test/requirements.in

-pytest==5.3.1
+pytest==5.3.4
 simplejson==3.17.0
 semver==2.9.0
 pyserial==3.4

test/requirements.txt

@@ -12,7 +12,7 @@ pluggy==0.13.1            # via pytest
 py==1.8.0                 # via pytest
 pyparsing==2.4.0          # via packaging
 pyserial==3.4
-pytest==5.3.1
+pytest==5.3.4
 semver==2.9.0
 simplejson==3.17.0
 six==1.12.0               # via packaging

test/test_compile.py

@@ -14,6 +14,7 @@
 # a commercial license, send an email to license@arduino.cc.
 import json
 import os
+import platform
 
 import pytest
 
@@ -34,7 +35,7 @@ def test_compile_without_fqbn(run_command):
     assert result.failed
 
 
-def test_compile_with_simple_sketch(run_command, data_dir):
+def test_compile_with_simple_sketch(run_command, data_dir, working_dir):
     # Init the environment explicitly
     result = run_command("core update-index")
     assert result.ok
@@ -73,6 +74,54 @@ def test_compile_with_simple_sketch(run_command, data_dir):
         expected_trace_sequence, json_log_lines
     )
 
+    # Test the --output flag with absolute path
+    target = os.path.join(data_dir, "test.hex")
+    result = run_command(
+        "compile -b {fqbn} {sketch_path} -o {target}".format(
+            fqbn=fqbn, sketch_path=sketch_path, target=target
+        )
+    )
+    assert result.ok
+    assert os.path.exists(target)
+
+
+@pytest.mark.skipif(
+    running_on_ci() and platform.system() == "Windows",
+    reason="Test disabled on Github Actions Win VM until tmpdir inconsistent behavior bug is fixed",
+)
+def test_output_flag_default_path(run_command, data_dir, working_dir):
+    # Init the environment explicitly
+    result = run_command("core update-index")
+    assert result.ok
+
+    # Download latest AVR
+    result = run_command("core install arduino:avr")
+    assert result.ok
+
+    # Create a test sketch
+    sketch_path = os.path.join(data_dir, "test_output_flag_default_path")
+    fqbn = "arduino:avr:uno"
+    result = run_command("sketch new {}".format(sketch_path))
+    assert result.ok
+
+    # Test the --output flag defaulting to current working dir
+    result = run_command(
+        "compile -b {fqbn} {sketch_path} -o test".format(
+            fqbn=fqbn, sketch_path=sketch_path
+        )
+    )
+    assert result.ok
+    assert os.path.exists(os.path.join(working_dir, "test.hex"))
+
+    # Test extension won't be added if already present
+    result = run_command(
+        "compile -b {fqbn} {sketch_path} -o test2.hex".format(
+            fqbn=fqbn, sketch_path=sketch_path
+        )
+    )
+    assert result.ok
+    assert os.path.exists(os.path.join(working_dir, "test2.hex"))
+
 
 def test_compile_with_sketch_with_symlink_selfloop(run_command, data_dir):
     # Init the environment explicitly
@@ -131,7 +180,7 @@ def test_compile_with_sketch_with_symlink_selfloop(run_command, data_dir):
 
 
 @pytest.mark.skipif(running_on_ci(), reason="VMs have no serial ports")
-def test_compile_and_compile_combo(run_command, data_dir):
+def test_compile_and_compile_combo(run_command, data_dir, detected_boards):
     # Init the environment explicitly
     result = run_command("core update-index")
     assert result.ok
@@ -148,58 +197,17 @@ def test_compile_and_compile_combo(run_command, data_dir):
     assert result.ok
     assert "Sketch created in: {}".format(sketch_path) in result.stdout
 
-    #
-    # Build a list of detected boards to test, if any.
-    #
-    result = run_command("board list --format json")
-    assert result.ok
-
-    #
-    # The `board list --format json` returns a JSON that looks like to the following:
-    #
-    # [
-    #     {
-    #       "address": "/dev/cu.usbmodem14201",
-    #       "protocol": "serial",
-    #       "protocol_label": "Serial Port (USB)",
-    #       "boards": [
-    #         {
-    #           "name": "Arduino NANO 33 IoT",
-    #           "FQBN": "arduino:samd:nano_33_iot"
-    #         }
-    #       ]
-    #     }
-    #   ]
-
-    detected_boards = []
-
-    ports = json.loads(result.stdout)
-    assert isinstance(ports, list)
-    for port in ports:
-        boards = port.get('boards')
-        if boards is None:
-            continue
-        assert isinstance(boards, list)
-        for board in boards:
-            detected_boards.append(
-                dict(address=port.get("address"), fqbn=board.get("FQBN"))
-            )
-
-    assert len(detected_boards) >= 1, "There are no boards available for testing"
-
     # Build sketch for each detected board
     for board in detected_boards:
-        log_file_name = "{fqbn}-compile.log".format(
-            fqbn=board.get("fqbn").replace(":", "-")
-        )
+        log_file_name = "{fqbn}-compile.log".format(fqbn=board.fqbn.replace(":", "-"))
         log_file_path = os.path.join(data_dir, log_file_name)
         command_log_flags = "--log-format json --log-file {} --log-level trace".format(
             log_file_path
         )
         result = run_command(
             "compile -b {fqbn} --upload -p {address} {sketch_path} {log_flags}".format(
-                fqbn=board.get("fqbn"),
-                address=board.get("address"),
+                fqbn=board.fqbn,
+                address=board.address,
                 sketch_path=sketch_path,
                 log_flags=command_log_flags,
             )
@@ -210,16 +218,16 @@ def test_compile_and_compile_combo(run_command, data_dir):
         json_log_lines = log_json.readlines()
         expected_trace_sequence = [
             "Compile {sketch} for {fqbn} started".format(
-                sketch=sketch_path, fqbn=board.get("fqbn")
+                sketch=sketch_path, fqbn=board.fqbn
             ),
             "Compile {sketch} for {fqbn} successful".format(
-                sketch=sketch_name, fqbn=board.get("fqbn")
+                sketch=sketch_name, fqbn=board.fqbn
             ),
             "Upload {sketch} on {fqbn} started".format(
-                sketch=sketch_path, fqbn=board.get("fqbn")
+                sketch=sketch_path, fqbn=board.fqbn
             ),
             "Upload {sketch} on {fqbn} successful".format(
-                sketch=sketch_name, fqbn=board.get("fqbn")
+                sketch=sketch_name, fqbn=board.fqbn
             ),
         ]
         assert is_message_sequence_in_json_log_traces(

test/test_core.py

@@ -12,6 +12,8 @@
 # otherwise use the software for commercial activities involving the Arduino
 # software without disclosing the source code of your own applications. To purchase
 # a commercial license, send an email to license@arduino.cc.
+import os
+import platform
 import pytest
 import simplejson as json
 
@@ -95,3 +97,101 @@ def test_core_search_no_args(run_command):
             break
     assert found
     assert len(platforms) == num_platforms
+
+
+def test_core_updateindex_invalid_url(run_command):
+    url = "http://www.invalid-domain-asjkdakdhadjkh.com/package_example_index.json"
+    result = run_command("core update-index --additional-urls={}".format(url))
+    assert result.failed
+
+
+@pytest.mark.skipif(
+    platform.system() == "Windows",
+    reason="core fails with fatal error: bits/c++config.h: No such file or directory",
+)
+def test_core_install_esp32(run_command, data_dir):
+    # update index
+    url = "https://dl.espressif.com/dl/package_esp32_index.json"
+    assert run_command("core update-index --additional-urls={}".format(url))
+    # install 3rd-party core
+    assert run_command("core install esp32:esp32 --additional-urls={}".format(url))
+    # create a sketch and compile to double check the core was successfully installed
+    sketch_path = os.path.join(data_dir, "test_core_install_esp32")
+    assert run_command("sketch new {}".format(sketch_path))
+    assert run_command("compile -b esp32:esp32:esp32 {}".format(sketch_path))
+    # prevent regressions for https://github.com/arduino/arduino-cli/issues/163
+    assert os.path.exists(
+        os.path.join(
+            sketch_path, "test_core_install_esp32.esp32.esp32.esp32.partitions.bin"
+        )
+    )
+
+
+def test_core_download(run_command, downloads_dir):
+    assert run_command("core update-index")
+
+    # Download a specific core version
+    assert run_command("core download arduino:avr@1.6.16")
+    assert os.path.exists(os.path.join(downloads_dir, "packages", "avr-1.6.16.tar.bz2"))
+
+    # Wrong core version
+    result = run_command("core download arduino:avr@69.42.0")
+    assert result.failed
+
+    # Wrong core
+    result = run_command("core download bananas:avr")
+    assert result.failed
+
+
+def _in(jsondata, name, version=None):
+    installed_cores = json.loads(jsondata)
+    for c in installed_cores:
+        if name == c.get("ID"):
+            if version is None:
+                return True
+            elif version == c.get("Installed"):
+                return True
+    return False
+
+
+def test_core_install(run_command):
+    assert run_command("core update-index")
+
+    # Install a specific core version
+    assert run_command("core install arduino:avr@1.6.16")
+    result = run_command("core list --format json")
+    assert result.ok
+    assert _in(result.stdout, "arduino:avr", "1.6.16")
+
+    # Replace it with a more recent one
+    assert run_command("core install arduino:avr@1.6.17")
+    result = run_command("core list --format json")
+    assert result.ok
+    assert _in(result.stdout, "arduino:avr", "1.6.17")
+
+    # Confirm core is listed as "updatable"
+    result = run_command("core list --updatable --format json")
+    assert result.ok
+    assert _in(result.stdout, "arduino:avr", "1.6.17")
+
+    # Upgrade the core to latest version
+    assert run_command("core upgrade arduino:avr")
+    result = run_command("core list --format json")
+    assert result.ok
+    assert not _in(result.stdout, "arduino:avr", "1.6.17")
+    # double check the code isn't updatable anymore
+    result = run_command("core list --updatable --format json")
+    assert result.ok
+    assert not _in(result.stdout, "arduino:avr")
+
+
+def test_core_uninstall(run_command):
+    assert run_command("core update-index")
+    assert run_command("core install arduino:avr")
+    result = run_command("core list --format json")
+    assert result.ok
+    assert _in(result.stdout, "arduino:avr")
+    assert run_command("core uninstall arduino:avr")
+    result = run_command("core list --format json")
+    assert result.ok
+    assert not _in(result.stdout, "arduino:avr")

test/test_upload.py

+# This file is part of arduino-cli.
+#
+# Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
+#
+# This software is released under the GNU General Public License version 3,
+# which covers the main part of arduino-cli.
+# The terms of this license can be found at:
+# https://www.gnu.org/licenses/gpl-3.0.en.html
+#
+# You can be released from the requirements of the above licenses by purchasing
+# a commercial license. Buying such a license is mandatory if you want to modify or
+# otherwise use the software for commercial activities involving the Arduino
+# software without disclosing the source code of your own applications. To purchase
+# a commercial license, send an email to license@arduino.cc.
+import os
+
+import pytest
+
+from .common import running_on_ci
+
+# Skip this module when running in CI environments
+pytestmark = pytest.mark.skipif(running_on_ci(), reason="VMs have no serial ports")
+
+
+def test_upload(run_command, data_dir, detected_boards):
+    # Init the environment explicitly
+    assert run_command("core update-index")
+
+    for board in detected_boards:
+        # Download core
+        assert run_command("core install {}".format(board.core))
+        # Create a sketch
+        sketch_path = os.path.join(data_dir, "foo")
+        assert run_command("sketch new {}".format(sketch_path))
+        # Build sketch
+        assert run_command(
+            "compile -b {fqbn} {sketch_path}".format(
+                fqbn=board.fqbn, sketch_path=sketch_path
+            )
+        )
+        # Upload
+        assert run_command(
+            "upload -b {fqbn} -p {port} {sketch_path}".format(
+                sketch_path=sketch_path, fqbn=board.fqbn, port=board.address
+            )
+        )