Added search using qualifier[:=]value syntax (#2373)

commands/lib/search.go

@@ -23,7 +23,6 @@ import (
 	"github.com/arduino/arduino-cli/arduino"
 	"github.com/arduino/arduino-cli/arduino/libraries/librariesindex"
 	"github.com/arduino/arduino-cli/arduino/libraries/librariesmanager"
-	"github.com/arduino/arduino-cli/arduino/utils"
 	"github.com/arduino/arduino-cli/commands/internal/instances"
 	rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1"
 	semver "go.bug.st/relaxed-semver"
@@ -44,18 +43,11 @@ func searchLibrary(req *rpc.LibrarySearchRequest, lm *librariesmanager.Libraries
 	if query == "" {
 		query = req.GetQuery()
 	}
-	queryTerms := utils.SearchTermsFromQueryString(query)
 
-	for _, lib := range lm.Index.Libraries {
-		toTest := lib.Name + " " +
-			lib.Latest.Paragraph + " " +
-			lib.Latest.Sentence + " " +
-			lib.Latest.Author + " "
-		for _, include := range lib.Latest.ProvidesIncludes {
-			toTest += include + " "
-		}
+	matcher := MatcherFromQueryString(query)
 
-		if utils.Match(toTest, queryTerms) {
+	for _, lib := range lm.Index.Libraries {
+		if matcher(lib) {
 			res = append(res, indexLibraryToRPCSearchLibrary(lib, req.GetOmitReleasesDetails()))
 		}
 	}

commands/lib/search_matcher.go

+// This file is part of arduino-cli.
+//
+// Copyright 2023 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to license@arduino.cc.
+
+package lib
+
+import (
+	"strings"
+
+	"github.com/arduino/arduino-cli/arduino/libraries/librariesindex"
+	"github.com/arduino/arduino-cli/arduino/utils"
+)
+
+// matcherTokensFromQueryString parses the query string into tokens of interest
+// for the qualifier-value pattern matching.
+func matcherTokensFromQueryString(query string) []string {
+	escaped := false
+	quoted := false
+	tokens := []string{}
+	sb := &strings.Builder{}
+
+	for _, r := range query {
+		// Short circuit the loop on backslash so that all other paths can clear
+		// the escaped flag.
+		if !escaped && r == '\\' {
+			escaped = true
+			continue
+		}
+
+		if r == '"' {
+			if !escaped {
+				quoted = !quoted
+			} else {
+				sb.WriteRune(r)
+			}
+		} else if !quoted && r == ' ' {
+			tokens = append(tokens, strings.ToLower(sb.String()))
+			sb.Reset()
+		} else {
+			sb.WriteRune(r)
+		}
+		escaped = false
+	}
+	if sb.Len() > 0 {
+		tokens = append(tokens, strings.ToLower(sb.String()))
+	}
+
+	return tokens
+}
+
+// defaulLibraryMatchExtractor returns a string describing the library that
+// is used for the simple search.
+func defaultLibraryMatchExtractor(lib *librariesindex.Library) string {
+	res := lib.Name + " " +
+		lib.Latest.Paragraph + " " +
+		lib.Latest.Sentence + " " +
+		lib.Latest.Author + " "
+	for _, include := range lib.Latest.ProvidesIncludes {
+		res += include + " "
+	}
+	return res
+}
+
+var qualifiers map[string]func(lib *librariesindex.Library) string = map[string]func(lib *librariesindex.Library) string{
+	"name":          func(lib *librariesindex.Library) string { return lib.Name },
+	"architectures": func(lib *librariesindex.Library) string { return strings.Join(lib.Latest.Architectures, " ") },
+	"author":        func(lib *librariesindex.Library) string { return lib.Latest.Author },
+	"category":      func(lib *librariesindex.Library) string { return lib.Latest.Category },
+	"dependencies": func(lib *librariesindex.Library) string {
+		names := make([]string, len(lib.Latest.Dependencies))
+		for i, dep := range lib.Latest.Dependencies {
+			names[i] = dep.GetName()
+		}
+		return strings.Join(names, " ")
+	},
+	"license":    func(lib *librariesindex.Library) string { return lib.Latest.License },
+	"maintainer": func(lib *librariesindex.Library) string { return lib.Latest.Maintainer },
+	"paragraph":  func(lib *librariesindex.Library) string { return lib.Latest.Paragraph },
+	"provides":   func(lib *librariesindex.Library) string { return strings.Join(lib.Latest.ProvidesIncludes, " ") },
+	"sentence":   func(lib *librariesindex.Library) string { return lib.Latest.Sentence },
+	"types":      func(lib *librariesindex.Library) string { return strings.Join(lib.Latest.Types, " ") },
+	"version":    func(lib *librariesindex.Library) string { return lib.Latest.Version.String() },
+	"website":    func(lib *librariesindex.Library) string { return lib.Latest.Website },
+}
+
+// MatcherFromQueryString returns a closure that takes a library as a
+// parameter and returns true if the library matches the query.
+func MatcherFromQueryString(query string) func(*librariesindex.Library) bool {
+	// A qv-query is one using <qualifier>[:=]<value> syntax.
+	qvQuery := strings.Contains(query, ":") || strings.Contains(query, "=")
+
+	if !qvQuery {
+		queryTerms := utils.SearchTermsFromQueryString(query)
+		return func(lib *librariesindex.Library) bool {
+			return utils.Match(defaultLibraryMatchExtractor(lib), queryTerms)
+		}
+	}
+
+	queryTerms := matcherTokensFromQueryString(query)
+
+	return func(lib *librariesindex.Library) bool {
+		matched := true
+		for _, term := range queryTerms {
+			if sepIdx := strings.IndexAny(term, ":="); sepIdx != -1 {
+				qualifier, separator, target := term[:sepIdx], term[sepIdx], term[sepIdx+1:]
+				if extractor, ok := qualifiers[qualifier]; ok {
+					switch separator {
+					case ':':
+						matched = (matched && utils.Match(extractor(lib), []string{target}))
+						continue
+					case '=':
+						matched = (matched && strings.ToLower(extractor(lib)) == target)
+						continue
+					}
+				}
+			}
+			// We perform the usual match in the following cases:
+			// 1. Unknown qualifier names revert to basic search terms.
+			// 2. Terms that do not use qv-syntax.
+			matched = (matched && utils.Match(defaultLibraryMatchExtractor(lib), []string{term}))
+		}
+		return matched
+	}
+}

commands/lib/search_test.go

@@ -28,6 +28,7 @@ import (
 
 var customIndexPath = paths.New("testdata", "test1")
 var fullIndexPath = paths.New("testdata", "full")
+var qualifiedSearchIndexPath = paths.New("testdata", "qualified_search")
 
 func TestSearchLibrary(t *testing.T) {
 	lm := librariesmanager.NewLibraryManager(customIndexPath, nil)
@@ -94,3 +95,112 @@ func TestSearchLibraryFields(t *testing.T) {
 	require.Len(t, res, 19)
 	require.Equal(t, "FlashStorage", res[0])
 }
+
+func TestSearchLibraryWithQualifiers(t *testing.T) {
+	lm := librariesmanager.NewLibraryManager(qualifiedSearchIndexPath, nil)
+	lm.LoadIndex()
+
+	query := func(q string) []string {
+		libs := []string{}
+		for _, lib := range searchLibrary(&rpc.LibrarySearchRequest{SearchArgs: q}, lm).Libraries {
+			libs = append(libs, lib.Name)
+		}
+		return libs
+	}
+
+	res := query("mesh")
+	require.Len(t, res, 4)
+
+	res = query("name:Mesh")
+	require.Len(t, res, 3)
+
+	res = query("name=Mesh")
+	require.Len(t, res, 0)
+
+	// Space not in double-quoted string
+	res = query("name=Painless Mesh")
+	require.Len(t, res, 0)
+
+	// Embedded space in double-quoted string
+	res = query("name=\"Painless Mesh\"")
+	require.Len(t, res, 1)
+	require.Equal(t, "Painless Mesh", res[0])
+
+	// No closing double-quote - still tokenizes with embedded space
+	res = query("name:\"Painless Mesh")
+	require.Len(t, res, 1)
+
+	// Malformed double-quoted string with escaped first double-quote
+	res = query("name:\\\"Painless Mesh\"")
+	require.Len(t, res, 0)
+
+	res = query("name:mesh author:TMRh20")
+	require.Len(t, res, 1)
+	require.Equal(t, "RF24Mesh", res[0])
+
+	res = query("mesh dependencies:ArduinoJson")
+	require.Len(t, res, 1)
+	require.Equal(t, "Painless Mesh", res[0])
+
+	res = query("architectures:esp author=\"Suraj I.\"")
+	require.Len(t, res, 1)
+	require.Equal(t, "esp8266-framework", res[0])
+
+	res = query("mesh esp")
+	require.Len(t, res, 2)
+
+	res = query("mesh esp paragraph:wifi")
+	require.Len(t, res, 1)
+	require.Equal(t, "esp8266-framework", res[0])
+
+	// Unknown qualifier should revert to original matching
+	res = query("std::array")
+	require.Len(t, res, 1)
+	require.Equal(t, "Array", res[0])
+
+	res = query("data storage")
+	require.Len(t, res, 1)
+	require.Equal(t, "Pushdata_ESP8266_SSL", res[0])
+
+	res = query("category:\"data storage\"")
+	require.Len(t, res, 1)
+	require.Equal(t, "Array", res[0])
+
+	res = query("maintainer:@")
+	require.Len(t, res, 4)
+
+	res = query("sentence:\"A library for NRF24L01(+) devices mesh.\"")
+	require.Len(t, res, 1)
+	require.Equal(t, "RF24Mesh", res[0])
+
+	res = query("types=contributed")
+	require.Len(t, res, 7)
+
+	res = query("version:1.0")
+	require.Len(t, res, 3)
+
+	res = query("version=1.2.1")
+	require.Len(t, res, 1)
+	require.Equal(t, "Array", res[0])
+
+	// Non-SSL URLs
+	res = query("website:http://")
+	require.Len(t, res, 1)
+	require.Equal(t, "RF24Mesh", res[0])
+
+	// Literal double-quote
+	res = query("sentence:\\\"")
+	require.Len(t, res, 1)
+	require.Equal(t, "RTCtime", res[0])
+
+	res = query("license=MIT")
+	require.Len(t, res, 2)
+
+	// Empty string
+	res = query("license=\"\"")
+	require.Len(t, res, 5)
+
+	res = query("provides:painlessmesh.h")
+	require.Len(t, res, 1)
+	require.Equal(t, "Painless Mesh", res[0])
+}

commands/lib/testdata/qualified_search/library_index.json

+{
+  "libraries": [
+    {
+      "name": "Array",
+      "version": "1.2.1",
+      "author": "Peter Polidoro \u003cpeterpolidoro@gmail.com\u003e",
+      "maintainer": "Peter Polidoro \u003cpeterpolidoro@gmail.com\u003e",
+      "sentence": "An array container similar to the C++ std::array",
+      "paragraph": "Like this project? Please star it on GitHub!",
+      "website": "https://github.com/janelia-arduino/Array.git",
+      "category": "Data Storage",
+      "architectures": ["*"],
+      "types": ["Contributed"],
+      "repository": "https://github.com/janelia-arduino/Array.git",
+      "url": "https://downloads.arduino.cc/libraries/github.com/janelia-arduino/Array-1.2.1.zip",
+      "archiveFileName": "Array-1.2.1.zip",
+      "size": 7859,
+      "checksum": "SHA-256:dc69e0b4d1390c08253120a80e6e07e5cc6185ec24cbe3cb96dec2d8173e6495"
+    },
+    {
+      "name": "esp8266-framework",
+      "version": "1.1.5",
+      "author": "Suraj I.",
+      "maintainer": "Suraj I. \u003csurajinamdar151@gmail.com\u003e",
+      "sentence": "esp8266 framework stack for easy configurable applications",
+      "paragraph": "esp8266 framework includes all services like gpio, wifi, http, mqtt, ntp, ota, napt, espnow, mesh, server etc. which are ready to use in all applications",
+      "website": "https://github.com/Suraj151/esp8266-framework",
+      "category": "Communication",
+      "architectures": ["esp8266"],
+      "types": ["Contributed"],
+      "repository": "https://github.com/Suraj151/esp8266-framework.git",
+      "url": "https://downloads.arduino.cc/libraries/github.com/Suraj151/esp8266_framework-1.1.5.zip",
+      "archiveFileName": "esp8266_framework-1.1.5.zip",
+      "size": 1918535,
+      "checksum": "SHA-256:81731d4ccc80846c317a2d4e2086d32caa695ed97d3e4765a59c5651b4be30b5"
+    },
+    {
+      "name": "Painless Mesh",
+      "version": "1.5.0",
+      "author": "Coopdis,Scotty Franzyshen,Edwin van Leeuwen,Germán Martín,Maximilian Schwarz,Doanh Doanh",
+      "maintainer": "Edwin van Leeuwen",
+      "sentence": "A painless way to setup a mesh with ESP8266 and ESP32 devices",
+      "paragraph": "A painless way to setup a mesh with ESP8266 and ESP32 devices",
+      "website": "https://gitlab.com/painlessMesh/painlessMesh",
+      "category": "Communication",
+      "architectures": ["esp8266", "esp32"],
+      "types": ["Contributed"],
+      "repository": "https://gitlab.com/painlessMesh/painlessMesh.git",
+      "providesIncludes": ["painlessMesh.h"],
+      "dependencies": [
+        {
+          "name": "ArduinoJson"
+        },
+        {
+          "name": "TaskScheduler"
+        }
+      ],
+      "url": "https://downloads.arduino.cc/libraries/gitlab.com/painlessMesh/Painless_Mesh-1.5.0.zip",
+      "archiveFileName": "Painless_Mesh-1.5.0.zip",
+      "size": 293531,
+      "checksum": "SHA-256:9d965064fc704e8ba19c0452cc50e619145f7869b9b135dbf7e521f6ec0a4b33"
+    },
+    {
+      "name": "Pushdata_ESP8266_SSL",
+      "version": "0.0.6",
+      "author": "Ragnar Lonn",
+      "maintainer": "Ragnar Lonn \u003chello@pushdata.io\u003e",
+      "license": "MIT",
+      "sentence": "Free, ultra-simple time series data storage for your IoT sensors",
+      "paragraph": "Pushdata.io client library that makes it very simple to store your time series data online",
+      "website": "https://pushdata.io",
+      "category": "Communication",
+      "architectures": ["*"],
+      "types": ["Contributed"],
+      "repository": "https://github.com/pushdata-io/Arduino_ESP8266_SSL.git",
+      "providesIncludes": ["Pushdata_ESP8266_SSL.h"],
+      "url": "https://downloads.arduino.cc/libraries/github.com/pushdata-io/Pushdata_ESP8266_SSL-0.0.6.zip",
+      "archiveFileName": "Pushdata_ESP8266_SSL-0.0.6.zip",
+      "size": 12160,
+      "checksum": "SHA-256:5d592eb7900782f681b86f5fd77c5d9f25c78555e3b5f0880c52197031206df0"
+    },
+    {
+      "name": "RF24Mesh",
+      "version": "1.0.0",
+      "author": "TMRh20",
+      "maintainer": "TMRh20",
+      "sentence": "A library for NRF24L01(+) devices mesh.",
+      "paragraph": "Provides a simple and seamless 'mesh' layer for sensor networks, allowing automatic and dynamic configuration that can be customized to suit many scenarios. It is currently designed to interface directly with with the RF24Network Development library, an OSI Network Layer using nRF24L01(+) radios driven by the newly optimized RF24 library fork.",
+      "website": "http://tmrh20.github.io/RF24Mesh/",
+      "category": "Communication",
+      "architectures": ["avr"],
+      "types": ["Contributed"],
+      "repository": "https://github.com/TMRh20/RF24Mesh.git",
+      "url": "https://downloads.arduino.cc/libraries/github.com/TMRh20/RF24Mesh-1.0.0.zip",
+      "archiveFileName": "RF24Mesh-1.0.0.zip",
+      "size": 31419,
+      "checksum": "SHA-256:1b122a6412bc06a33a7fbcef34e2210d0990c25839fd7bc547604103f28194b5"
+    },
+    {
+      "name": "RTCtime",
+      "version": "1.0.5",
+      "author": "smz \u003ctinker@smz.it\u003e",
+      "maintainer": "smz (https://github.com/smz)",
+      "sentence": "A \"Standard C Runtime\" compatible library for interfacing the DS1307 and DS3231 Real Time Clock modules.",
+      "paragraph": "This library is for getting/setting time from hardware RTC modules. It uses an API compatible with the AVR implementation of the Standard C runtime time library as available in the Arduino IDE since version 1.6.10 (AVR C Runtime Library 2.0.0)",
+      "website": "https://github.com/smz/Arduino-RTCtime",
+      "category": "Timing",
+      "architectures": ["*"],
+      "types": ["Contributed"],
+      "repository": "https://github.com/smz/Arduino-RTCtime.git",
+      "url": "https://downloads.arduino.cc/libraries/github.com/smz/RTCtime-1.0.5.zip",
+      "archiveFileName": "RTCtime-1.0.5.zip",
+      "size": 18870,
+      "checksum": "SHA-256:89493bb6d1f834426e82330fdf55a249ff43eb61707831d75deed8644a7ebce8"
+    },
+    {
+      "name": "DLLN3X ZigBee Mesh Module Library",
+      "version": "1.0.1",
+      "author": "Duke Liu \u003cmentalflow@ourdocs.cn\u003e",
+      "maintainer": "Duke Liu \u003cmentalflow@ourdocs.cn\u003e",
+      "license": "MIT",
+      "sentence": "This library allows you to use DLLN3X ZigBee mesh module very easily.",
+      "paragraph": "This library currently allows basic send and receive operations using the DLLN3X module, with more features to come.",
+      "website": "https://github.com/mentalfl0w/DLLN3X_zigbee_mesh_module_library",
+      "category": "Communication",
+      "architectures": ["*"],
+      "types": ["Contributed"],
+      "repository": "https://github.com/mentalfl0w/DLLN3X_zigbee_mesh_module_library.git",
+      "providesIncludes": ["DLLN3X.h"],
+      "url": "https://downloads.arduino.cc/libraries/github.com/mentalfl0w/DLLN3X_ZigBee_Mesh_Module_Library-1.0.1.zip",
+      "archiveFileName": "DLLN3X_ZigBee_Mesh_Module_Library-1.0.1.zip",
+      "size": 6122,
+      "checksum": "SHA-256:a28833bbd575ef8deab744a1f0e1175dad9e5329bf5c620fc2fe53e1de1d32ba"
+    }
+  ]
+}

internal/cli/lib/search.go

@@ -37,11 +37,60 @@ func initSearchCommand() *cobra.Command {
 	var namesOnly bool
 	var omitReleasesDetails bool
 	searchCommand := &cobra.Command{
-		Use:     fmt.Sprintf("search [%s]", tr("LIBRARY_NAME")),
-		Short:   tr("Searches for one or more libraries data."),
-		Long:    tr("Search for one or more libraries data (case insensitive search)."),
-		Example: "  " + os.Args[0] + " lib search audio",
-		Args:    cobra.ArbitraryArgs,
+		Use:   fmt.Sprintf("search [%s ...]", tr("SEARCH_TERM")),
+		Short: tr("Searches for one or more libraries matching a query."),
+		Long: tr(`Search for libraries matching zero or more search terms.
+
+All searches are performed in a case-insensitive fashion. Queries containing
+multiple search terms will return only libraries that match all of the terms.
+
+Search terms that do not match the QV syntax described below are basic search
+terms, and will match libraries that include the term anywhere in any of the
+following fields:
+ - Author
+ - Name
+ - Paragraph
+ - Provides
+ - Sentence
+
+A special syntax, called qualifier-value (QV), indicates that a search term
+should be compared against only one field of each library index entry. This
+syntax uses the name of an index field (case-insensitive), an equals sign (=)
+or a colon (:), and a value, e.g. 'name=ArduinoJson' or 'provides:tinyusb.h'.
+
+QV search terms that use a colon separator will match all libraries with the
+value anywhere in the named field, and QV search terms that use an equals
+separator will match only libraries with exactly the provided value in the
+named field.
+
+QV search terms can include embedded spaces using double-quote (") characters
+around the value or the entire term, e.g. 'category="Data Processing"' and
+'"category=Data Processing"' are equivalent. A QV term can include a literal
+double-quote character by preceding it with a backslash (\) character.
+
+NOTE: QV search terms using double-quote or backslash characters that are
+passed as command-line arguments may require quoting or escaping to prevent
+the shell from interpreting those characters.
+
+In addition to the fields listed above, QV terms can use these qualifiers:
+ - Architectures
+ - Category
+ - Dependencies
+ - License
+ - Maintainer
+ - Types
+ - Version
+ - Website
+		`),
+		Example: "  " + os.Args[0] + " lib search audio                               # " + tr("basic search for \"audio\"") + "\n" +
+			"  " + os.Args[0] + " lib search name:buzzer                         # " + tr("libraries with \"buzzer\" in the Name field") + "\n" +
+			"  " + os.Args[0] + " lib search name=pcf8523                        # " + tr("libraries with a Name exactly matching \"pcf8523\"") + "\n" +
+			"  " + os.Args[0] + " lib search \"author:\\\"Daniel Garcia\\\"\"          # " + tr("libraries authored by Daniel Garcia") + "\n" +
+			"  " + os.Args[0] + " lib search author=Adafruit name:gfx            # " + tr("libraries authored only by Adafruit with \"gfx\" in their Name") + "\n" +
+			"  " + os.Args[0] + " lib search esp32 display maintainer=espressif  # " + tr("basic search for \"esp32\" and \"display\" limited to official Maintainer") + "\n" +
+			"  " + os.Args[0] + " lib search dependencies:IRremote               # " + tr("libraries that depend on at least \"IRremote\"") + "\n" +
+			"  " + os.Args[0] + " lib search dependencies=IRremote               # " + tr("libraries that depend only on \"IRremote\"") + "\n",
+		Args: cobra.ArbitraryArgs,
 		Run: func(cmd *cobra.Command, args []string) {
 			runSearchCommand(args, namesOnly, omitReleasesDetails)
 		},

internal/integrationtest/lib/lib_test.go

@@ -788,6 +788,29 @@ func TestSearch(t *testing.T) {
 	runSearch("json", []string{"ArduinoJson", "Arduino_JSON"})
 }
 
+func TestQualifiedSearch(t *testing.T) {
+	env, cli := integrationtest.CreateArduinoCLIWithEnvironment(t)
+	defer env.CleanUp()
+
+	runSearch := func(args string, expectedLibs []string) {
+		stdout, _, err := cli.Run("lib", "search", "--names", "--format", "json", args)
+		require.NoError(t, err)
+		libraries := requirejson.Parse(t, stdout).Query("[ .libraries | .[] | .name ]").String()
+		for _, l := range expectedLibs {
+			require.Contains(t, libraries, l)
+		}
+	}
+	runSearch("name:MKRIoTCarrier", []string{"Arduino_MKRIoTCarrier"})
+	runSearch("name=Arduino_MKRIoTCarrier", []string{"Arduino_MKRIoTCarrier"})
+	// Embedded space in double-quoted string
+	runSearch("name=\"dht sensor library\"", []string{"DHT sensor library"})
+	// No closing double-quote
+	runSearch("name=\"dht sensor library", []string{"DHT sensor library"})
+	runSearch("name:\"sensor dht\"", []string{})
+	// Literal double-quote
+	runSearch("sentence:\\\"", []string{"RTCtime"})
+}
+
 func TestSearchParagraph(t *testing.T) {
 	env, cli := integrationtest.CreateArduinoCLIWithEnvironment(t)
 	defer env.CleanUp()