Improved pluggable discovery conversion for platforms not supporting it (#1629)

* Factored property composition in convertUploadToolsToPluggableDiscovery

* Auto-generated properties are cached and added after cycling on the original map

* Autogenerated upload tool properties are searched in sub-configs too

Fix #1444

* Added STMicroelectronics:stm32:Nucleo_32:pnum=NUCLEO_F031K6 to upload mock tests

* Use composed board properties to detect user fields

Some platforms may add information through the optional config part of
the FQBN (platforms menu items).

arduino/cores/packagemanager/loader.go

@@ -574,18 +574,35 @@ func convertVidPidIdentificationPropertiesToPluggableDiscovery(boardProperties *
 
 func convertUploadToolsToPluggableDiscovery(props *properties.Map) {
 	actions := []string{"upload", "bootloader", "program"}
+	propsToAdd := properties.NewMap()
 	for _, action := range actions {
-		if !props.ContainsKey(fmt.Sprintf("%s.tool.default", action)) {
-			tool, found := props.GetOk(fmt.Sprintf("%s.tool", action))
+		action += ".tool"
+		defaultAction := action + ".default"
+		if !props.ContainsKey(defaultAction) {
+			// Search for "menu.MENU-ID.MENU-ITEM.ACTION.tool" (some platforms sets ACTION.tool on
+			// submenu config entries). See https://github.com/arduino/arduino-cli/issues/1444
+			for key, value := range props.AsMap() {
+				if !strings.HasPrefix(key, "menu.") {
+					continue
+				}
+				split := strings.Split(key, ".")
+				if len(split) != 5 || split[3]+"."+split[4] != action {
+					continue
+				}
+				prefix := split[0] + "." + split[1] + "." + split[2]
+				propsToAdd.Set(prefix+"."+defaultAction, value)
+			}
+			tool, found := props.GetOk(action)
 			if !found {
 				// Just skip it, ideally this must never happen but if a platform
 				// doesn't define an expected upload.tool, bootloader.tool or program.tool
 				// there will be other issues further down the road after this conversion
 				continue
 			}
-			props.Set(fmt.Sprintf("%s.tool.default", action), tool)
+			propsToAdd.Set(defaultAction, tool)
 		}
 	}
+	props.Merge(propsToAdd)
 }
 
 func (pm *PackageManager) loadToolsFromPackage(targetPackage *cores.Package, toolsPath *paths.Path) []*status.Status {

arduino/cores/packagemanager/loader_test.go

@@ -254,3 +254,71 @@ program.extra_params=-P{serial.port}
 
 	require.Equal(t, expectedProps.AsMap(), props.AsMap())
 }
+
+func TestConvertUploadToolsToPluggableDiscoveryWithMenus(t *testing.T) {
+	props, err := properties.LoadFromBytes([]byte(`
+name=Nucleo-64
+
+build.core=arduino
+build.board=Nucleo_64
+build.variant_h=variant_{build.board}.h
+build.extra_flags=-D{build.product_line} {build.enable_usb} {build.xSerial}
+
+# Upload menu
+menu.upload_method.MassStorage=Mass Storage
+menu.upload_method.MassStorage.upload.protocol=
+menu.upload_method.MassStorage.upload.tool=massStorageCopy
+
+menu.upload_method.swdMethod=STM32CubeProgrammer (SWD)
+menu.upload_method.swdMethod.upload.protocol=0
+menu.upload_method.swdMethod.upload.options=-g
+menu.upload_method.swdMethod.upload.tool=stm32CubeProg
+
+menu.upload_method.serialMethod=STM32CubeProgrammer (Serial)
+menu.upload_method.serialMethod.upload.protocol=1
+menu.upload_method.serialMethod.upload.options={serial.port.file} -s
+menu.upload_method.serialMethod.upload.tool=stm32CubeProg
+
+menu.upload_method.dfuMethod=STM32CubeProgrammer (DFU)
+menu.upload_method.dfuMethod.upload.protocol=2
+menu.upload_method.dfuMethod.upload.options=-g
+menu.upload_method.dfuMethod.upload.tool=stm32CubeProg
+`))
+	require.NoError(t, err)
+	convertUploadToolsToPluggableDiscovery(props)
+
+	expectedProps, err := properties.LoadFromBytes([]byte(`
+name=Nucleo-64
+
+build.core=arduino
+build.board=Nucleo_64
+build.variant_h=variant_{build.board}.h
+build.extra_flags=-D{build.product_line} {build.enable_usb} {build.xSerial}
+
+# Upload menu
+menu.upload_method.MassStorage=Mass Storage
+menu.upload_method.MassStorage.upload.protocol=
+menu.upload_method.MassStorage.upload.tool=massStorageCopy
+menu.upload_method.MassStorage.upload.tool.default=massStorageCopy
+
+menu.upload_method.swdMethod=STM32CubeProgrammer (SWD)
+menu.upload_method.swdMethod.upload.protocol=0
+menu.upload_method.swdMethod.upload.options=-g
+menu.upload_method.swdMethod.upload.tool=stm32CubeProg
+menu.upload_method.swdMethod.upload.tool.default=stm32CubeProg
+
+menu.upload_method.serialMethod=STM32CubeProgrammer (Serial)
+menu.upload_method.serialMethod.upload.protocol=1
+menu.upload_method.serialMethod.upload.options={serial.port.file} -s
+menu.upload_method.serialMethod.upload.tool=stm32CubeProg
+menu.upload_method.serialMethod.upload.tool.default=stm32CubeProg
+
+menu.upload_method.dfuMethod=STM32CubeProgrammer (DFU)
+menu.upload_method.dfuMethod.upload.protocol=2
+menu.upload_method.dfuMethod.upload.options=-g
+menu.upload_method.dfuMethod.upload.tool=stm32CubeProg
+menu.upload_method.dfuMethod.upload.tool.default=stm32CubeProg
+`))
+	require.NoError(t, err)
+	require.Equal(t, expectedProps.AsMap(), props.AsMap())
+}

commands/upload/upload.go

@@ -66,12 +66,12 @@ func SupportedUserFields(ctx context.Context, req *rpc.SupportedUserFieldsReques
 		return nil, &arduino.InvalidFQBNError{Cause: err}
 	}
 
-	_, platformRelease, board, _, _, err := pm.ResolveFQBN(fqbn)
+	_, platformRelease, _, boardProperties, _, err := pm.ResolveFQBN(fqbn)
 	if err != nil {
 		return nil, &arduino.UnknownFQBNError{Cause: err}
 	}
 
-	toolID, err := getToolID(board.Properties, "upload", req.Protocol)
+	toolID, err := getToolID(boardProperties, "upload", req.Protocol)
 	if err != nil {
 		return nil, err
 	}

test/test_upload_mock.py

@@ -28,6 +28,7 @@ def generate_build_dir(sketch_path):
 
 
 indexes = [
+    "https://github.com/stm32duino/BoardManagerFiles/raw/main/package_stmicroelectronics_index.json",
     "https://adafruit.github.io/arduino-board-index/package_adafruit_index.json",
     "https://raw.githubusercontent.com/espressif/arduino-esp32/gh-pages/package_esp32_index.json",
     "http://arduino.esp8266.com/stable/package_esp8266com_index.json",
@@ -35,6 +36,7 @@ indexes = [
 ]
 
 cores_to_install = [
+    "STMicroelectronics:stm32@2.2.0",
     "arduino:avr@1.8.3",
     "adafruit:avr@1.4.13",
     "arduino:samd@1.8.11",
@@ -44,6 +46,23 @@ cores_to_install = [
 ]
 
 testdata = [
+    (
+        "STMicroelectronics:stm32:Nucleo_32:pnum=NUCLEO_F031K6,upload_method=serialMethod",
+        "/dev/ttyACM0",
+        "",
+        {
+            "darwin": '"" sh '
+            '"{data_dir}/packages/STMicroelectronics/tools/STM32Tools/2.1.1/stm32CubeProg.sh" '
+            '1 "{build_dir}/{sketch_name}.ino.bin" ttyACM0 -s\n',
+            "linux": '"" sh '
+            '"{data_dir}/packages/STMicroelectronics/tools/STM32Tools/2.1.1/stm32CubeProg.sh" '
+            '1 "{build_dir}/{sketch_name}.ino.bin" ttyACM0 -s\n',
+            "win32": '"{data_dir}/packages/STMicroelectronics/tools/STM32Tools/2.1.1/win/busybox.exe" '
+            "sh "
+            '"{data_dir}/packages/STMicroelectronics/tools/STM32Tools/2.1.1/stm32CubeProg.sh" '
+            '1 "{build_dir}/{sketch_name}.ino.bin" ttyACM0 -s\n',
+        },
+    ),
     (
         "arduino:avr:uno",
         "/dev/ttyACM0",