Enhances tracking of installed platforms (#998)

arduino/cores/cores.go

@@ -35,12 +35,18 @@ type Platform struct {
 	Package      *Package                    `json:"-"`
 }
 
+// PlatformReleaseHelp represents the help URL for this Platform release
+type PlatformReleaseHelp struct {
+	Online string `json:"-"`
+}
+
 // PlatformRelease represents a release of a plaform package.
 type PlatformRelease struct {
 	Resource       *resources.DownloadResource
 	Version        *semver.Version
 	BoardsManifest []*BoardManifest
 	Dependencies   ToolDependencies       // The Dependency entries to load tools.
+	Help           PlatformReleaseHelp    `json:"-"`
 	Platform       *Platform              `json:"-"`
 	Properties     *properties.Map        `json:"-"`
 	Boards         map[string]*Board      `json:"-"`

arduino/cores/packageindex/index.go

@@ -86,13 +86,15 @@ type indexToolReleaseFlavour struct {
 // indexBoard represents a single Board as written in package_index.json file.
 type indexBoard struct {
 	Name string         `json:"name"`
-	ID   []indexBoardID `json:"id"`
+	ID   []indexBoardID `json:"id,omitempty"`
 }
 
+// indexBoardID represents the ID of a single board. i.e. uno, yun, diecimila, micro and the likes
 type indexBoardID struct {
 	USB string `json:"usb"`
 }
 
+// indexHelp represents the help URL
 type indexHelp struct {
 	Online string `json:"online,omitempty"`
 }
@@ -105,6 +107,82 @@ func (index Index) MergeIntoPackages(outPackages cores.Packages) {
 	}
 }
 
+// IndexFromPlatformRelease creates an Index that contains a single indexPackage
+// which in turn contains a single indexPlatformRelease converted from the one
+// passed as argument
+func IndexFromPlatformRelease(pr *cores.PlatformRelease) Index {
+	boards := []indexBoard{}
+	for _, manifest := range pr.BoardsManifest {
+		board := indexBoard{
+			Name: manifest.Name,
+		}
+		for _, id := range manifest.ID {
+			if id.USB != "" {
+				board.ID = []indexBoardID{{USB: id.USB}}
+			}
+		}
+		boards = append(boards, board)
+	}
+
+	tools := []indexToolDependency{}
+	for _, t := range pr.Dependencies {
+		tools = append(tools, indexToolDependency{
+			Packager: t.ToolPackager,
+			Name:     t.ToolName,
+			Version:  t.ToolVersion,
+		})
+	}
+
+	packageTools := []*indexToolRelease{}
+	for name, tool := range pr.Platform.Package.Tools {
+		for _, toolRelease := range tool.Releases {
+			flavours := []indexToolReleaseFlavour{}
+			for _, flavour := range toolRelease.Flavors {
+				flavours = append(flavours, indexToolReleaseFlavour{
+					OS:              flavour.OS,
+					URL:             flavour.Resource.URL,
+					ArchiveFileName: flavour.Resource.ArchiveFileName,
+					Size:            json.Number(fmt.Sprintf("%d", flavour.Resource.Size)),
+					Checksum:        flavour.Resource.Checksum,
+				})
+			}
+			packageTools = append(packageTools, &indexToolRelease{
+				Name:    name,
+				Version: toolRelease.Version,
+				Systems: flavours,
+			})
+		}
+	}
+
+	return Index{
+		IsTrusted: pr.IsTrusted,
+		Packages: []*indexPackage{
+			{
+				Name:       pr.Platform.Package.Name,
+				Maintainer: pr.Platform.Package.Maintainer,
+				WebsiteURL: pr.Platform.Package.WebsiteURL,
+				URL:        pr.Platform.Package.URL,
+				Email:      pr.Platform.Package.Email,
+				Platforms: []*indexPlatformRelease{{
+					Name:             pr.Platform.Name,
+					Architecture:     pr.Platform.Architecture,
+					Version:          pr.Version,
+					Category:         pr.Platform.Category,
+					URL:              pr.Resource.URL,
+					ArchiveFileName:  pr.Resource.ArchiveFileName,
+					Checksum:         pr.Resource.Checksum,
+					Size:             json.Number(fmt.Sprintf("%d", pr.Resource.Size)),
+					Boards:           boards,
+					Help:             indexHelp{Online: pr.Help.Online},
+					ToolDependencies: tools,
+				}},
+				Tools: packageTools,
+				Help:  indexHelp{Online: pr.Platform.Package.Help.Online},
+			},
+		},
+	}
+}
+
 func (inPackage indexPackage) extractPackageIn(outPackages cores.Packages, trusted bool) {
 	outPackage := outPackages.GetOrCreatePackage(inPackage.Name)
 	outPackage.Maintainer = inPackage.Maintainer
@@ -144,6 +222,7 @@ func (inPlatformRelease indexPlatformRelease) extractPlatformIn(outPackage *core
 		URL:             inPlatformRelease.URL,
 		CachePath:       "packages",
 	}
+	outPlatformRelease.Help = cores.PlatformReleaseHelp{Online: inPlatformRelease.Help.Online}
 	outPlatformRelease.BoardsManifest = inPlatformRelease.extractBoardsManifest()
 	if deps, err := inPlatformRelease.extractDeps(); err == nil {
 		outPlatformRelease.Dependencies = deps

arduino/cores/packagemanager/install_uninstall.go

@@ -16,10 +16,12 @@
 package packagemanager
 
 import (
+	"encoding/json"
 	"fmt"
 	"runtime"
 
 	"github.com/arduino/arduino-cli/arduino/cores"
+	"github.com/arduino/arduino-cli/arduino/cores/packageindex"
 	"github.com/arduino/arduino-cli/executils"
 	"github.com/pkg/errors"
 )
@@ -39,6 +41,20 @@ func (pm *PackageManager) InstallPlatform(platformRelease *cores.PlatformRelease
 	} else {
 		return err
 	}
+	if err := pm.cacheInstalledJSON(platformRelease); err != nil {
+		return errors.Errorf("creating installed.json in %s: %s", platformRelease.InstallDir, err)
+	}
+	return nil
+}
+
+func (pm *PackageManager) cacheInstalledJSON(platformRelease *cores.PlatformRelease) error {
+	index := packageindex.IndexFromPlatformRelease(platformRelease)
+	platformJSON, err := json.MarshalIndent(index, "", "  ")
+	if err != nil {
+		return err
+	}
+	installedJSON := platformRelease.InstallDir.Join("installed.json")
+	installedJSON.WriteFile(platformJSON)
 	return nil
 }
 

arduino/cores/packagemanager/loader.go

@@ -265,10 +265,20 @@ func (pm *PackageManager) loadPlatformRelease(platform *cores.PlatformRelease, p
 	platform.InstallDir = path
 
 	// Some useful paths
+	installedJSONPath := path.Join("installed.json")
 	platformTxtPath := path.Join("platform.txt")
 	platformTxtLocalPath := path.Join("platform.local.txt")
 	programmersTxtPath := path.Join("programmers.txt")
 
+	// If the installed.json file is found load it, this is done to handle the
+	// case in which the platform's index and its url have been deleted locally,
+	// if we don't load it some information about the platform is lost
+	if installedJSONPath.Exist() {
+		if _, err := pm.LoadPackageIndexFromFile(installedJSONPath); err != nil {
+			return fmt.Errorf("loading %s: %s", installedJSONPath, err)
+		}
+	}
+
 	// Create platform properties
 	platform.Properties = platform.Properties.Clone() // TODO: why CLONE?
 	if p, err := properties.SafeLoad(platformTxtPath.String()); err == nil {

test/conftest.py

@@ -131,7 +131,7 @@ def run_command(pytestconfig, data_dir, downloads_dir, working_dir):
         # It escapes spaces in the path using "\ " but it doesn't always work,
         # wrapping the path in quotation marks is the safest approach
         with run_context.prefix(f'{cd_command} "{custom_working_dir}"'):
-            return run_context.run(cli_full_line, echo=False, hide=True, warn=True, env=custom_env)
+            return run_context.run(cli_full_line, echo=False, hide=True, warn=True, env=custom_env, encoding="utf-8")
 
     return _run
 

test/test_board.py

@@ -12,11 +12,8 @@
 # otherwise use the software for commercial activities involving the Arduino
 # software without disclosing the source code of your own applications. To purchase
 # a commercial license, send an email to license@arduino.cc.
-import pytest
 import simplejson as json
 
-from .common import running_on_ci
-
 
 gold_board = """
     {
@@ -401,12 +398,40 @@ def test_board_list(run_command):
         assert "protocol_label" in port
 
 
-@pytest.mark.skipif(running_on_ci(), reason="VMs have no serial ports")
 def test_board_listall(run_command):
     run_command("core update-index")
-    result = run_command("board listall")
-    assert result.ok
-    assert ["Board", "Name", "FQBN"] == result.stdout.splitlines()[0].strip().split()
+    run_command("core install arduino:avr@1.8.3")
+    res = run_command("board listall")
+    assert res.ok
+    lines = [l.rsplit(maxsplit=1) for l in res.stdout.strip().splitlines()]
+    assert len(lines) == 27
+    assert ["Board Name", "FQBN"] in lines
+    assert ["Arduino Yún", "arduino:avr:yun"] in lines
+    assert ["Arduino Uno", "arduino:avr:uno"] in lines
+    assert ["Arduino Duemilanove or Diecimila", "arduino:avr:diecimila"] in lines
+    assert ["Arduino Nano", "arduino:avr:nano"] in lines
+    assert ["Arduino Mega or Mega 2560", "arduino:avr:mega"] in lines
+    assert ["Arduino Mega ADK", "arduino:avr:megaADK"] in lines
+    assert ["Arduino Leonardo", "arduino:avr:leonardo"] in lines
+    assert ["Arduino Leonardo ETH", "arduino:avr:leonardoeth"] in lines
+    assert ["Arduino Micro", "arduino:avr:micro"] in lines
+    assert ["Arduino Esplora", "arduino:avr:esplora"] in lines
+    assert ["Arduino Mini", "arduino:avr:mini"] in lines
+    assert ["Arduino Ethernet", "arduino:avr:ethernet"] in lines
+    assert ["Arduino Fio", "arduino:avr:fio"] in lines
+    assert ["Arduino BT", "arduino:avr:bt"] in lines
+    assert ["LilyPad Arduino USB", "arduino:avr:LilyPadUSB"] in lines
+    assert ["LilyPad Arduino", "arduino:avr:lilypad"] in lines
+    assert ["Arduino Pro or Pro Mini", "arduino:avr:pro"] in lines
+    assert ["Arduino NG or older", "arduino:avr:atmegang"] in lines
+    assert ["Arduino Robot Control", "arduino:avr:robotControl"] in lines
+    assert ["Arduino Robot Motor", "arduino:avr:robotMotor"] in lines
+    assert ["Arduino Gemma", "arduino:avr:gemma"] in lines
+    assert ["Adafruit Circuit Playground", "arduino:avr:circuitplay32u4cat"] in lines
+    assert ["Arduino Yún Mini", "arduino:avr:yunmini"] in lines
+    assert ["Arduino Industrial 101", "arduino:avr:chiwawa"] in lines
+    assert ["Linino One", "arduino:avr:one"] in lines
+    assert ["Arduino Uno WiFi", "arduino:avr:unowifi"] in lines
 
 
 def test_board_details(run_command):

test/test_core.py

@@ -95,17 +95,24 @@ def test_core_search_no_args(run_command, httpserver):
     result = run_command("core search")
     assert result.ok
     num_platforms = 0
+    found = False
     for l in result.stdout.splitlines()[1:]:  # ignore the header on first line
         if l:  # ignore empty lines
-            assert not l.startswith("test:x86")
+            if l.startswith("test:x86"):
+                found = True
             num_platforms += 1
 
     # same thing in JSON format, also check the number of platforms found is the same
     result = run_command("core search --format json")
     assert result.ok
     platforms = json.loads(result.stdout)
+    found = False
+    platforms = json.loads(result.stdout)
     for elem in platforms:
-        assert elem.get("Name") != "test_core"
+        if elem.get("Name") == "test_core":
+            found = True
+            break
+    assert found
     assert len(platforms) == num_platforms
 
     # list all with additional urls, check the test core is there
@@ -270,3 +277,24 @@ def test_core_broken_install(run_command):
     url = "https://raw.githubusercontent.com/arduino/arduino-cli/master/test/testdata/test_index.json"
     assert run_command("core update-index --additional-urls={}".format(url))
     assert not run_command("core install brokenchecksum:x86 --additional-urls={}".format(url))
+
+
+def test_core_install_creates_installed_json(run_command, data_dir):
+    assert run_command("core update-index")
+    assert run_command("core install arduino:avr@1.6.23")
+
+    installed_json_file = Path(data_dir, "packages", "arduino", "hardware", "avr", "1.6.23", "installed.json")
+    assert installed_json_file.exists()
+    installed_json = json.load(installed_json_file.open("r"))
+
+    expected_installed_json = json.load((Path(__file__).parent / "testdata" / "installed.json").open("r"))
+
+    def ordered(obj):
+        if isinstance(obj, dict):
+            return sorted({k: ordered(v) for k, v in obj.items()})
+        if isinstance(obj, list):
+            return sorted(ordered(x) for x in obj)
+        else:
+            return obj
+
+    assert ordered(installed_json) == ordered(expected_installed_json)