Added configuration option for proxy (#609)

* Use downloader.SetDefaultConfig() to set user-agent for the arduino-cli

This change allows to not pass-trough the downloader configuration from
function to function everywhere (and sometime we forget to pass it for
example in the "core update-index" command).

* Add support for network proxy configuration

* Cosmetic: added blank space in configuration/defaults.go

* Update go-downloader to 1.2.0

* Refactored function to generate user-agent

* Added a TODO for missing proxy usage in board list API query

* Partially revert global network configuration.

It's better to read the network configuration before each http request so
in case it is changed using "Settings" functions in daemon mode the changes
are immediately applied.

arduino/cores/packagemanager/download.go

@@ -17,7 +17,6 @@ package packagemanager
 
 import (
 	"fmt"
-	"net/http"
 
 	"github.com/arduino/arduino-cli/arduino/cores"
 	"go.bug.st/downloader"
@@ -101,16 +100,16 @@ func (pm *PackageManager) FindPlatformReleaseDependencies(item *PlatformReferenc
 
 // DownloadToolRelease downloads a ToolRelease. If the tool is already downloaded a nil Downloader
 // is returned.
-func (pm *PackageManager) DownloadToolRelease(tool *cores.ToolRelease, downloaderHeaders http.Header) (*downloader.Downloader, error) {
+func (pm *PackageManager) DownloadToolRelease(tool *cores.ToolRelease, config *downloader.Config) (*downloader.Downloader, error) {
 	resource := tool.GetCompatibleFlavour()
 	if resource == nil {
 		return nil, fmt.Errorf("tool not available for your OS")
 	}
-	return resource.Download(pm.DownloadDir, downloaderHeaders)
+	return resource.Download(pm.DownloadDir, config)
 }
 
 // DownloadPlatformRelease downloads a PlatformRelease. If the platform is already downloaded a
 // nil Downloader is returned.
-func (pm *PackageManager) DownloadPlatformRelease(platform *cores.PlatformRelease, downloaderHeaders http.Header) (*downloader.Downloader, error) {
-	return platform.Resource.Download(pm.DownloadDir, downloaderHeaders)
+func (pm *PackageManager) DownloadPlatformRelease(platform *cores.PlatformRelease, config *downloader.Config) (*downloader.Downloader, error) {
+	return platform.Resource.Download(pm.DownloadDir, config)
 }

arduino/libraries/librariesmanager/download.go

@@ -25,8 +25,8 @@ import (
 var LibraryIndexURL, _ = url.Parse("https://downloads.arduino.cc/libraries/library_index.json")
 
 // UpdateIndex downloads the libraries index file from Arduino repository.
-func (lm *LibrariesManager) UpdateIndex() (*downloader.Downloader, error) {
+func (lm *LibrariesManager) UpdateIndex(config *downloader.Config) (*downloader.Downloader, error) {
 	lm.IndexFile.Parent().MkdirAll()
 	// TODO: Download from gzipped URL index
-	return downloader.Download(lm.IndexFile.String(), LibraryIndexURL.String(), downloader.NoResume)
+	return downloader.DownloadWithConfig(lm.IndexFile.String(), LibraryIndexURL.String(), *config, downloader.NoResume)
 }

arduino/resources/helpers.go

@@ -17,7 +17,6 @@ package resources
 
 import (
 	"fmt"
-	"net/http"
 	"os"
 
 	"github.com/arduino/go-paths-helper"
@@ -44,7 +43,7 @@ func (r *DownloadResource) IsCached(downloadDir *paths.Path) (bool, error) {
 }
 
 // Download a DownloadResource.
-func (r *DownloadResource) Download(downloadDir *paths.Path, downloaderHeaders http.Header) (*downloader.Downloader, error) {
+func (r *DownloadResource) Download(downloadDir *paths.Path, config *downloader.Config) (*downloader.Downloader, error) {
 	cached, err := r.TestLocalArchiveIntegrity(downloadDir)
 	if err != nil {
 		return nil, fmt.Errorf("testing local archive integrity: %s", err)
@@ -72,7 +71,5 @@ func (r *DownloadResource) Download(downloadDir *paths.Path, downloaderHeaders h
 		return nil, fmt.Errorf("getting archive file info: %s", err)
 	}
 
-	downloadConfig := downloader.Config{
-		RequestHeaders: downloaderHeaders}
-	return downloader.DownloadWithConfig(path.String(), r.URL, downloadConfig)
+	return downloader.DownloadWithConfig(path.String(), r.URL, *config)
 }

arduino/resources/helpers_test.go

@@ -25,6 +25,7 @@ import (
 
 	"github.com/arduino/go-paths-helper"
 	"github.com/stretchr/testify/require"
+	"go.bug.st/downloader"
 )
 
 type EchoHandler struct{}
@@ -52,7 +53,7 @@ func TestDownloadApplyUserAgentHeaderUsingConfig(t *testing.T) {
 		URL:             srv.URL,
 	}
 
-	d, err := r.Download(tmp, http.Header{"User-Agent": []string{goldUserAgentValue}})
+	d, err := r.Download(tmp, &downloader.Config{RequestHeaders: http.Header{"User-Agent": []string{goldUserAgentValue}}})
 	require.NoError(t, err)
 	err = d.Run()
 	require.NoError(t, err)

arduino/resources/resources_test.go

@@ -18,11 +18,11 @@ package resources
 import (
 	"crypto"
 	"encoding/hex"
-	"net/http"
 	"testing"
 
 	"github.com/arduino/go-paths-helper"
 	"github.com/stretchr/testify/require"
+	"go.bug.st/downloader"
 )
 
 func TestDownloadAndChecksums(t *testing.T) {
@@ -42,7 +42,7 @@ func TestDownloadAndChecksums(t *testing.T) {
 	require.NoError(t, err)
 
 	downloadAndTestChecksum := func() {
-		d, err := r.Download(tmp, http.Header{})
+		d, err := r.Download(tmp, &downloader.Config{})
 		require.NoError(t, err)
 		err = d.Run()
 		require.NoError(t, err)
@@ -58,7 +58,7 @@ func TestDownloadAndChecksums(t *testing.T) {
 	downloadAndTestChecksum()
 
 	// Download with cached file
-	d, err := r.Download(tmp, http.Header{})
+	d, err := r.Download(tmp, &downloader.Config{})
 	require.NoError(t, err)
 	require.Nil(t, d)
 

cli/core/download.go

@@ -66,8 +66,7 @@ func runDownloadCommand(cmd *cobra.Command, args []string) {
 			Architecture:    platformRef.Architecture,
 			Version:         platformRef.Version,
 		}
-		_, err := core.PlatformDownload(context.Background(), platformDownloadreq, output.ProgressBar(),
-			globals.NewHTTPClientHeader())
+		_, err := core.PlatformDownload(context.Background(), platformDownloadreq, output.ProgressBar())
 		if err != nil {
 			feedback.Errorf("Error downloading %s: %v", args[i], err)
 			os.Exit(errorcodes.ErrNetwork)

cli/core/install.go

@@ -67,8 +67,7 @@ func runInstallCommand(cmd *cobra.Command, args []string) {
 			Architecture:    platformRef.Architecture,
 			Version:         platformRef.Version,
 		}
-		_, err := core.PlatformInstall(context.Background(), plattformInstallReq, output.ProgressBar(),
-			output.TaskProgress(), globals.NewHTTPClientHeader())
+		_, err := core.PlatformInstall(context.Background(), plattformInstallReq, output.ProgressBar(), output.TaskProgress())
 		if err != nil {
 			feedback.Errorf("Error during install: %v", err)
 			os.Exit(errorcodes.ErrGeneric)

cli/core/upgrade.go

@@ -93,7 +93,7 @@ func runUpgradeCommand(cmd *cobra.Command, args []string) {
 			Architecture:    platformRef.Architecture,
 		}
 
-		_, err := core.PlatformUpgrade(context.Background(), r, output.ProgressBar(), output.TaskProgress(), globals.NewHTTPClientHeader())
+		_, err := core.PlatformUpgrade(context.Background(), r, output.ProgressBar(), output.TaskProgress())
 		if err == core.ErrAlreadyLatest {
 			feedback.Printf("Platform %s is already at the latest version", platformRef)
 		} else if err != nil {

cli/daemon/daemon.go

@@ -21,9 +21,7 @@ import (
 	"io"
 	"io/ioutil"
 	"net"
-	"net/http"
 	"os"
-	"runtime"
 	"syscall"
 
 	"github.com/arduino/arduino-cli/cli/errorcodes"
@@ -67,26 +65,15 @@ func runDaemonCommand(cmd *cobra.Command, args []string) {
 		stats.Incr("daemon", stats.T("success", "true"))
 		defer stats.Flush()
 	}
-
 	port := viper.GetString("daemon.port")
 	s := grpc.NewServer()
 
-	// Compose user agent header
-	headers := http.Header{
-		"User-Agent": []string{
-			fmt.Sprintf("%s/%s daemon (%s; %s; %s) Commit:%s",
-				globals.VersionInfo.Application,
-				globals.VersionInfo.VersionString,
-				runtime.GOARCH,
-				runtime.GOOS,
-				runtime.Version(),
-				globals.VersionInfo.Commit),
-		},
-	}
-	// Register the commands service
+	// Set specific user-agent for the daemon
+	viper.Set("network.user_agent_ext", "daemon")
+
+	// register the commands service
 	srv_commands.RegisterArduinoCoreServer(s, &daemon.ArduinoCoreServerImpl{
-		DownloaderHeaders: headers,
-		VersionString:     globals.VersionInfo.VersionString,
+		VersionString: globals.VersionInfo.VersionString,
 	})
 
 	// Register the monitors service

cli/globals/globals.go

@@ -33,8 +33,16 @@ var (
 )
 
 // NewHTTPClientHeader returns the http.Header object that must be used by the clients inside the downloaders
-func NewHTTPClientHeader() http.Header {
-	userAgentValue := fmt.Sprintf("%s/%s (%s; %s; %s) Commit:%s", VersionInfo.Application,
-		VersionInfo.VersionString, runtime.GOARCH, runtime.GOOS, runtime.Version(), VersionInfo.Commit)
+// and adds the subComponent if specified
+func NewHTTPClientHeader(subComponent string) http.Header {
+	if subComponent != "" {
+		subComponent = " " + subComponent
+	}
+	userAgentValue := fmt.Sprintf("%s/%s%s (%s; %s; %s) Commit:%s",
+		VersionInfo.Application,
+		VersionInfo.VersionString,
+		subComponent,
+		runtime.GOARCH, runtime.GOOS, runtime.Version(),
+		VersionInfo.Commit)
 	return http.Header{"User-Agent": []string{userAgentValue}}
 }

cli/instance/instance.go

@@ -18,7 +18,6 @@ package instance
 import (
 	"context"
 
-	"github.com/arduino/arduino-cli/cli/globals"
 	"github.com/arduino/arduino-cli/cli/output"
 	"github.com/arduino/arduino-cli/commands"
 	rpc "github.com/arduino/arduino-cli/rpc/commands"
@@ -45,8 +44,7 @@ func CreateInstance() (*rpc.Instance, error) {
 
 func getInitResponse() (*rpc.InitResp, error) {
 	// invoke Init()
-	resp, err := commands.Init(context.Background(), &rpc.InitReq{},
-		output.ProgressBar(), output.TaskProgress(), globals.NewHTTPClientHeader())
+	resp, err := commands.Init(context.Background(), &rpc.InitReq{}, output.ProgressBar(), output.TaskProgress())
 
 	// Init() failed
 	if err != nil {

cli/lib/download.go

@@ -21,7 +21,6 @@ import (
 
 	"github.com/arduino/arduino-cli/cli/errorcodes"
 	"github.com/arduino/arduino-cli/cli/feedback"
-	"github.com/arduino/arduino-cli/cli/globals"
 	"github.com/arduino/arduino-cli/cli/instance"
 	"github.com/arduino/arduino-cli/cli/output"
 	"github.com/arduino/arduino-cli/commands/lib"
@@ -57,8 +56,7 @@ func runDownloadCommand(cmd *cobra.Command, args []string) {
 			Name:     library.Name,
 			Version:  library.Version,
 		}
-		_, err := lib.LibraryDownload(context.Background(), libraryDownloadReq, output.ProgressBar(),
-			globals.NewHTTPClientHeader())
+		_, err := lib.LibraryDownload(context.Background(), libraryDownloadReq, output.ProgressBar())
 		if err != nil {
 			feedback.Errorf("Error downloading %s: %v", library, err)
 			os.Exit(errorcodes.ErrNetwork)

cli/lib/install.go

@@ -21,7 +21,6 @@ import (
 
 	"github.com/arduino/arduino-cli/cli/errorcodes"
 	"github.com/arduino/arduino-cli/cli/feedback"
-	"github.com/arduino/arduino-cli/cli/globals"
 	"github.com/arduino/arduino-cli/cli/instance"
 	"github.com/arduino/arduino-cli/cli/output"
 	"github.com/arduino/arduino-cli/commands/lib"
@@ -95,8 +94,7 @@ func runInstallCommand(cmd *cobra.Command, args []string) {
 			Name:     library.Name,
 			Version:  library.VersionRequired,
 		}
-		err := lib.LibraryInstall(context.Background(), libraryInstallReq, output.ProgressBar(),
-			output.TaskProgress(), globals.NewHTTPClientHeader())
+		err := lib.LibraryInstall(context.Background(), libraryInstallReq, output.ProgressBar(), output.TaskProgress())
 		if err != nil {
 			feedback.Errorf("Error installing %s: %v", library, err)
 			os.Exit(errorcodes.ErrGeneric)

cli/lib/upgrade.go

@@ -20,7 +20,6 @@ import (
 
 	"github.com/arduino/arduino-cli/cli/errorcodes"
 	"github.com/arduino/arduino-cli/cli/feedback"
-	"github.com/arduino/arduino-cli/cli/globals"
 	"github.com/arduino/arduino-cli/cli/instance"
 	"github.com/arduino/arduino-cli/cli/output"
 	"github.com/arduino/arduino-cli/commands/lib"
@@ -48,13 +47,13 @@ func runUpgradeCommand(cmd *cobra.Command, args []string) {
 	instance := instance.CreateInstanceIgnorePlatformIndexErrors()
 
 	if len(args) == 0 {
-		err := lib.LibraryUpgradeAll(instance.Id, output.ProgressBar(), output.TaskProgress(), globals.NewHTTPClientHeader())
+		err := lib.LibraryUpgradeAll(instance.Id, output.ProgressBar(), output.TaskProgress())
 		if err != nil {
 			feedback.Errorf("Error upgrading libraries: %v", err)
 			os.Exit(errorcodes.ErrGeneric)
 		}
 	} else {
-		err := lib.LibraryUpgrade(instance.Id, args, output.ProgressBar(), output.TaskProgress(), globals.NewHTTPClientHeader())
+		err := lib.LibraryUpgrade(instance.Id, args, output.ProgressBar(), output.TaskProgress())
 		if err != nil {
 			feedback.Errorf("Error upgrading libraries: %v", err)
 			os.Exit(errorcodes.ErrGeneric)

commands/board/list.go

@@ -51,9 +51,11 @@ func apiByVidPid(vid, pid string) ([]*rpc.BoardListItem, error) {
 	url := fmt.Sprintf("%s/%s/%s", vidPidURL, vid, pid)
 	retVal := []*rpc.BoardListItem{}
 	req, _ := http.NewRequest("GET", url, nil)
-	req.Header = globals.NewHTTPClientHeader()
+	req.Header = globals.NewHTTPClientHeader("")
 	req.Header.Set("Content-Type", "application/json")
 
+	// TODO: use proxy if set
+
 	if res, err := http.DefaultClient.Do(req); err == nil {
 		if res.StatusCode >= 400 {
 			if res.StatusCode == 404 {

commands/bundled_tools.go

@@ -17,7 +17,6 @@ package commands
 
 import (
 	"fmt"
-	"net/http"
 
 	"github.com/arduino/arduino-cli/arduino/cores"
 	"github.com/arduino/arduino-cli/arduino/cores/packagemanager"
@@ -25,9 +24,12 @@ import (
 )
 
 // DownloadToolRelease downloads a ToolRelease
-func DownloadToolRelease(pm *packagemanager.PackageManager, toolRelease *cores.ToolRelease,
-	downloadCB DownloadProgressCB, downloaderHeaders http.Header) error {
-	resp, err := pm.DownloadToolRelease(toolRelease, downloaderHeaders)
+func DownloadToolRelease(pm *packagemanager.PackageManager, toolRelease *cores.ToolRelease, downloadCB DownloadProgressCB) error {
+	config, err := GetDownloaderConfig()
+	if err != nil {
+		return err
+	}
+	resp, err := pm.DownloadToolRelease(toolRelease, config)
 	if err != nil {
 		return err
 	}

commands/core/download.go

@@ -19,7 +19,6 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"net/http"
 
 	"github.com/arduino/arduino-cli/arduino/cores"
 	"github.com/arduino/arduino-cli/arduino/cores/packagemanager"
@@ -28,8 +27,7 @@ import (
 )
 
 // PlatformDownload FIXMEDOC
-func PlatformDownload(ctx context.Context, req *rpc.PlatformDownloadReq, downloadCB commands.DownloadProgressCB,
-	downloaderHeaders http.Header) (*rpc.PlatformDownloadResp, error) {
+func PlatformDownload(ctx context.Context, req *rpc.PlatformDownloadReq, downloadCB commands.DownloadProgressCB) (*rpc.PlatformDownloadResp, error) {
 	pm := commands.GetPackageManager(req.GetInstance().GetId())
 	if pm == nil {
 		return nil, errors.New("invalid instance")
@@ -49,13 +47,13 @@ func PlatformDownload(ctx context.Context, req *rpc.PlatformDownloadReq, downloa
 		return nil, fmt.Errorf("find platform dependencies: %s", err)
 	}
 
-	err = downloadPlatform(pm, platform, downloadCB, downloaderHeaders)
+	err = downloadPlatform(pm, platform, downloadCB)
 	if err != nil {
 		return nil, err
 	}
 
 	for _, tool := range tools {
-		err := downloadTool(pm, tool, downloadCB, downloaderHeaders)
+		err := downloadTool(pm, tool, downloadCB)
 		if err != nil {
 			return nil, fmt.Errorf("downloading tool %s: %s", tool, err)
 		}
@@ -64,22 +62,24 @@ func PlatformDownload(ctx context.Context, req *rpc.PlatformDownloadReq, downloa
 	return &rpc.PlatformDownloadResp{}, nil
 }
 
-func downloadPlatform(pm *packagemanager.PackageManager, platformRelease *cores.PlatformRelease,
-	downloadCB commands.DownloadProgressCB, downloaderHeaders http.Header) error {
+func downloadPlatform(pm *packagemanager.PackageManager, platformRelease *cores.PlatformRelease, downloadCB commands.DownloadProgressCB) error {
 	// Download platform
-	resp, err := pm.DownloadPlatformRelease(platformRelease, downloaderHeaders)
+	config, err := commands.GetDownloaderConfig()
+	if err != nil {
+		return err
+	}
+	resp, err := pm.DownloadPlatformRelease(platformRelease, config)
 	if err != nil {
 		return err
 	}
 	return commands.Download(resp, platformRelease.String(), downloadCB)
 }
 
-func downloadTool(pm *packagemanager.PackageManager, tool *cores.ToolRelease, downloadCB commands.DownloadProgressCB,
-	downloaderHeaders http.Header) error {
+func downloadTool(pm *packagemanager.PackageManager, tool *cores.ToolRelease, downloadCB commands.DownloadProgressCB) error {
 	// Check if tool has a flavor available for the current OS
 	if tool.GetCompatibleFlavour() == nil {
 		return fmt.Errorf("tool %s not available for the current OS", tool)
 	}
 
-	return commands.DownloadToolRelease(pm, tool, downloadCB, downloaderHeaders)
+	return commands.DownloadToolRelease(pm, tool, downloadCB)
 }

commands/core/install.go

@@ -19,7 +19,6 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"net/http"
 
 	"github.com/arduino/arduino-cli/arduino/cores"
 	"github.com/arduino/arduino-cli/arduino/cores/packagemanager"
@@ -29,7 +28,7 @@ import (
 
 // PlatformInstall FIXMEDOC
 func PlatformInstall(ctx context.Context, req *rpc.PlatformInstallReq,
-	downloadCB commands.DownloadProgressCB, taskCB commands.TaskProgressCB, downloaderHeaders http.Header) (*rpc.PlatformInstallResp, error) {
+	downloadCB commands.DownloadProgressCB, taskCB commands.TaskProgressCB) (*rpc.PlatformInstallResp, error) {
 
 	pm := commands.GetPackageManager(req.GetInstance().GetId())
 	if pm == nil {
@@ -50,7 +49,7 @@ func PlatformInstall(ctx context.Context, req *rpc.PlatformInstallReq,
 		return nil, fmt.Errorf("finding platform dependencies: %s", err)
 	}
 
-	err = installPlatform(pm, platform, tools, downloadCB, taskCB, downloaderHeaders)
+	err = installPlatform(pm, platform, tools, downloadCB, taskCB)
 	if err != nil {
 		return nil, err
 	}
@@ -65,7 +64,7 @@ func PlatformInstall(ctx context.Context, req *rpc.PlatformInstallReq,
 
 func installPlatform(pm *packagemanager.PackageManager,
 	platformRelease *cores.PlatformRelease, requiredTools []*cores.ToolRelease,
-	downloadCB commands.DownloadProgressCB, taskCB commands.TaskProgressCB, downloaderHeaders http.Header) error {
+	downloadCB commands.DownloadProgressCB, taskCB commands.TaskProgressCB) error {
 	log := pm.Log.WithField("platform", platformRelease)
 
 	// Prerequisite checks before install
@@ -87,9 +86,9 @@ func installPlatform(pm *packagemanager.PackageManager,
 	// Package download
 	taskCB(&rpc.TaskProgress{Name: "Downloading packages"})
 	for _, tool := range toolsToInstall {
-		downloadTool(pm, tool, downloadCB, downloaderHeaders)
+		downloadTool(pm, tool, downloadCB)
 	}
-	downloadPlatform(pm, platformRelease, downloadCB, downloaderHeaders)
+	downloadPlatform(pm, platformRelease, downloadCB)
 	taskCB(&rpc.TaskProgress{Completed: true})
 
 	// Install tools first

commands/core/upgrade.go

@@ -19,7 +19,6 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"net/http"
 
 	"github.com/arduino/arduino-cli/arduino/cores/packagemanager"
 	"github.com/arduino/arduino-cli/commands"
@@ -34,7 +33,7 @@ var (
 
 // PlatformUpgrade FIXMEDOC
 func PlatformUpgrade(ctx context.Context, req *rpc.PlatformUpgradeReq,
-	downloadCB commands.DownloadProgressCB, taskCB commands.TaskProgressCB, downloaderHeaders http.Header) (*rpc.PlatformUpgradeResp, error) {
+	downloadCB commands.DownloadProgressCB, taskCB commands.TaskProgressCB) (*rpc.PlatformUpgradeResp, error) {
 
 	pm := commands.GetPackageManager(req.GetInstance().GetId())
 	if pm == nil {
@@ -46,7 +45,7 @@ func PlatformUpgrade(ctx context.Context, req *rpc.PlatformUpgradeReq,
 		Package:              req.PlatformPackage,
 		PlatformArchitecture: req.Architecture,
 	}
-	if err := upgradePlatform(pm, ref, downloadCB, taskCB, downloaderHeaders); err != nil {
+	if err := upgradePlatform(pm, ref, downloadCB, taskCB); err != nil {
 		return nil, err
 	}
 
@@ -58,7 +57,7 @@ func PlatformUpgrade(ctx context.Context, req *rpc.PlatformUpgradeReq,
 }
 
 func upgradePlatform(pm *packagemanager.PackageManager, platformRef *packagemanager.PlatformReference,
-	downloadCB commands.DownloadProgressCB, taskCB commands.TaskProgressCB, downloaderHeaders http.Header) error {
+	downloadCB commands.DownloadProgressCB, taskCB commands.TaskProgressCB) error {
 	if platformRef.PlatformVersion != nil {
 		return fmt.Errorf("upgrade doesn't accept parameters with version")
 	}
@@ -85,7 +84,7 @@ func upgradePlatform(pm *packagemanager.PackageManager, platformRef *packagemana
 		if err != nil {
 			return fmt.Errorf("platform %s is not installed", platformRef)
 		}
-		err = installPlatform(pm, platform, tools, downloadCB, taskCB, downloaderHeaders)
+		err = installPlatform(pm, platform, tools, downloadCB, taskCB)
 		if err != nil {
 			return err
 		}

commands/daemon/daemon.go

@@ -19,7 +19,6 @@ package daemon
 
 import (
 	"context"
-	"net/http"
 
 	"github.com/arduino/arduino-cli/arduino/utils"
 	"github.com/arduino/arduino-cli/commands"
@@ -33,8 +32,7 @@ import (
 
 // ArduinoCoreServerImpl FIXMEDOC
 type ArduinoCoreServerImpl struct {
-	DownloaderHeaders http.Header
-	VersionString     string
+	VersionString string
 }
 
 // BoardDetails FIXMEDOC
@@ -108,7 +106,6 @@ func (s *ArduinoCoreServerImpl) Init(req *rpc.InitReq, stream rpc.ArduinoCore_In
 	resp, err := commands.Init(stream.Context(), req,
 		func(p *rpc.DownloadProgress) { stream.Send(&rpc.InitResp{DownloadProgress: p}) },
 		func(p *rpc.TaskProgress) { stream.Send(&rpc.InitResp{TaskProgress: p}) },
-		s.DownloaderHeaders,
 	)
 	if err != nil {
 		return err
@@ -140,7 +137,6 @@ func (s *ArduinoCoreServerImpl) PlatformInstall(req *rpc.PlatformInstallReq, str
 		stream.Context(), req,
 		func(p *rpc.DownloadProgress) { stream.Send(&rpc.PlatformInstallResp{Progress: p}) },
 		func(p *rpc.TaskProgress) { stream.Send(&rpc.PlatformInstallResp{TaskProgress: p}) },
-		s.DownloaderHeaders,
 	)
 	if err != nil {
 		return err
@@ -153,7 +149,6 @@ func (s *ArduinoCoreServerImpl) PlatformDownload(req *rpc.PlatformDownloadReq, s
 	resp, err := core.PlatformDownload(
 		stream.Context(), req,
 		func(p *rpc.DownloadProgress) { stream.Send(&rpc.PlatformDownloadResp{Progress: p}) },
-		s.DownloaderHeaders,
 	)
 	if err != nil {
 		return err
@@ -179,7 +174,6 @@ func (s *ArduinoCoreServerImpl) PlatformUpgrade(req *rpc.PlatformUpgradeReq, str
 		stream.Context(), req,
 		func(p *rpc.DownloadProgress) { stream.Send(&rpc.PlatformUpgradeResp{Progress: p}) },
 		func(p *rpc.TaskProgress) { stream.Send(&rpc.PlatformUpgradeResp{TaskProgress: p}) },
-		s.DownloaderHeaders,
 	)
 	if err != nil {
 		return err
@@ -227,7 +221,6 @@ func (s *ArduinoCoreServerImpl) LibraryDownload(req *rpc.LibraryDownloadReq, str
 	resp, err := lib.LibraryDownload(
 		stream.Context(), req,
 		func(p *rpc.DownloadProgress) { stream.Send(&rpc.LibraryDownloadResp{Progress: p}) },
-		s.DownloaderHeaders,
 	)
 	if err != nil {
 		return err
@@ -241,7 +234,6 @@ func (s *ArduinoCoreServerImpl) LibraryInstall(req *rpc.LibraryInstallReq, strea
 		stream.Context(), req,
 		func(p *rpc.DownloadProgress) { stream.Send(&rpc.LibraryInstallResp{Progress: p}) },
 		func(p *rpc.TaskProgress) { stream.Send(&rpc.LibraryInstallResp{TaskProgress: p}) },
-		s.DownloaderHeaders,
 	)
 	if err != nil {
 		return err
@@ -265,7 +257,6 @@ func (s *ArduinoCoreServerImpl) LibraryUpgradeAll(req *rpc.LibraryUpgradeAllReq,
 	err := lib.LibraryUpgradeAll(req.GetInstance().GetId(),
 		func(p *rpc.DownloadProgress) { stream.Send(&rpc.LibraryUpgradeAllResp{Progress: p}) },
 		func(p *rpc.TaskProgress) { stream.Send(&rpc.LibraryUpgradeAllResp{TaskProgress: p}) },
-		s.DownloaderHeaders,
 	)
 	if err != nil {
 		return err

commands/download.go

+// This file is part of arduino-cli.
+//
+// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to license@arduino.cc.
+
+package commands
+
+import (
+	"net/url"
+	"time"
+
+	"github.com/arduino/arduino-cli/cli/globals"
+	rpc "github.com/arduino/arduino-cli/rpc/commands"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"github.com/spf13/viper"
+	"go.bug.st/downloader"
+)
+
+// GetDownloaderConfig returns the downloader configuration based on
+// current settings.
+func GetDownloaderConfig() (*downloader.Config, error) {
+	res := &downloader.Config{
+		RequestHeaders: globals.NewHTTPClientHeader(viper.GetString("network.user_agent_ext")),
+	}
+	if viper.IsSet("network.proxy") {
+		proxy := viper.GetString("network.proxy")
+		if _, err := url.Parse(proxy); err != nil {
+			return nil, errors.New("Invalid network.proxy '" + proxy + "': " + err.Error())
+		}
+		res.ProxyURL = proxy
+		logrus.Infof("Using proxy %s", proxy)
+	}
+	return res, nil
+}
+
+// Download performs a download loop using the provided downloader.Downloader.
+// Messages are passed back to the DownloadProgressCB using label as text for the File field.
+func Download(d *downloader.Downloader, label string, downloadCB DownloadProgressCB) error {
+	if d == nil {
+		// This signal means that the file is already downloaded
+		downloadCB(&rpc.DownloadProgress{
+			File:      label,
+			Completed: true,
+		})
+		return nil
+	}
+	downloadCB(&rpc.DownloadProgress{
+		File:      label,
+		Url:       d.URL,
+		TotalSize: d.Size(),
+	})
+	d.RunAndPoll(func(downloaded int64) {
+		downloadCB(&rpc.DownloadProgress{Downloaded: downloaded})
+	}, 250*time.Millisecond)
+	if d.Error() != nil {
+		return d.Error()
+	}
+	downloadCB(&rpc.DownloadProgress{Completed: true})
+	return nil
+}

commands/instances.go

@@ -19,10 +19,8 @@ import (
 	"context"
 	"fmt"
 	"io/ioutil"
-	"net/http"
 	"net/url"
 	"path"
-	"time"
 
 	"github.com/arduino/arduino-cli/arduino/cores"
 	"github.com/arduino/arduino-cli/arduino/cores/packageindex"
@@ -89,13 +87,12 @@ func GetLibraryManager(instanceID int32) *librariesmanager.LibrariesManager {
 	return i.lm
 }
 
-func (instance *CoreInstance) installToolIfMissing(tool *cores.ToolRelease, downloadCB DownloadProgressCB,
-	taskCB TaskProgressCB, downloaderHeaders http.Header) (bool, error) {
+func (instance *CoreInstance) installToolIfMissing(tool *cores.ToolRelease, downloadCB DownloadProgressCB, taskCB TaskProgressCB) (bool, error) {
 	if tool.IsInstalled() {
 		return false, nil
 	}
 	taskCB(&rpc.TaskProgress{Name: "Downloading missing tool " + tool.String()})
-	if err := DownloadToolRelease(instance.PackageManager, tool, downloadCB, downloaderHeaders); err != nil {
+	if err := DownloadToolRelease(instance.PackageManager, tool, downloadCB); err != nil {
 		return false, fmt.Errorf("downloading %s tool: %s", tool, err)
 	}
 	taskCB(&rpc.TaskProgress{Completed: true})
@@ -105,18 +102,17 @@ func (instance *CoreInstance) installToolIfMissing(tool *cores.ToolRelease, down
 	return true, nil
 }
 
-func (instance *CoreInstance) checkForBuiltinTools(downloadCB DownloadProgressCB, taskCB TaskProgressCB,
-	downloaderHeaders http.Header) error {
+func (instance *CoreInstance) checkForBuiltinTools(downloadCB DownloadProgressCB, taskCB TaskProgressCB) error {
 	// Check for ctags tool
 	ctags, _ := getBuiltinCtagsTool(instance.PackageManager)
-	ctagsInstalled, err := instance.installToolIfMissing(ctags, downloadCB, taskCB, downloaderHeaders)
+	ctagsInstalled, err := instance.installToolIfMissing(ctags, downloadCB, taskCB)
 	if err != nil {
 		return err
 	}
 
 	// Check for bultin serial-discovery tool
 	serialDiscoveryTool, _ := getBuiltinSerialDiscoveryTool(instance.PackageManager)
-	serialDiscoveryInstalled, err := instance.installToolIfMissing(serialDiscoveryTool, downloadCB, taskCB, downloaderHeaders)
+	serialDiscoveryInstalled, err := instance.installToolIfMissing(serialDiscoveryTool, downloadCB, taskCB)
 	if err != nil {
 		return err
 	}
@@ -130,9 +126,7 @@ func (instance *CoreInstance) checkForBuiltinTools(downloadCB DownloadProgressCB
 }
 
 // Init FIXMEDOC
-func Init(ctx context.Context, req *rpc.InitReq, downloadCB DownloadProgressCB, taskCB TaskProgressCB,
-	downloaderHeaders http.Header) (*rpc.InitResp, error) {
-
+func Init(ctx context.Context, req *rpc.InitReq, downloadCB DownloadProgressCB, taskCB TaskProgressCB) (*rpc.InitResp, error) {
 	res, err := createInstance(ctx, req.GetLibraryManagerOnly())
 	if err != nil {
 		return nil, fmt.Errorf("cannot initialize package manager: %s", err)
@@ -147,7 +141,7 @@ func Init(ctx context.Context, req *rpc.InitReq, downloadCB DownloadProgressCB,
 	instancesCount++
 	instances[handle] = instance
 
-	if err := instance.checkForBuiltinTools(downloadCB, taskCB, downloaderHeaders); err != nil {
+	if err := instance.checkForBuiltinTools(downloadCB, taskCB); err != nil {
 		return nil, err
 	}
 
@@ -176,7 +170,11 @@ func UpdateLibrariesIndex(ctx context.Context, req *rpc.UpdateLibrariesIndexReq,
 	if lm == nil {
 		return fmt.Errorf("invalid handle")
 	}
-	d, err := lm.UpdateIndex()
+	config, err := GetDownloaderConfig()
+	if err != nil {
+		return err
+	}
+	d, err := lm.UpdateIndex(config)
 	if err != nil {
 		return err
 	}
@@ -220,7 +218,11 @@ func UpdateIndex(ctx context.Context, req *rpc.UpdateIndexReq, downloadCB Downlo
 		tmp := paths.New(tmpFile.Name())
 		defer tmp.Remove()
 
-		d, err := downloader.Download(tmp.String(), URL.String())
+		config, err := GetDownloaderConfig()
+		if err != nil {
+			return nil, fmt.Errorf("downloading index %s: %s", URL, err)
+		}
+		d, err := downloader.DownloadWithConfig(tmp.String(), URL.String(), *config)
 		if err != nil {
 			return nil, fmt.Errorf("downloading index %s: %s", URL, err)
 		}
@@ -353,28 +355,3 @@ func createInstance(ctx context.Context, getLibOnly bool) (*createInstanceResult
 
 	return res, nil
 }
-
-// Download FIXMEDOC
-func Download(d *downloader.Downloader, label string, downloadCB DownloadProgressCB) error {
-	if d == nil {
-		// This signal means that the file is already downloaded
-		downloadCB(&rpc.DownloadProgress{
-			File:      label,
-			Completed: true,
-		})
-		return nil
-	}
-	downloadCB(&rpc.DownloadProgress{
-		File:      label,
-		Url:       d.URL,
-		TotalSize: d.Size(),
-	})
-	d.RunAndPoll(func(downloaded int64) {
-		downloadCB(&rpc.DownloadProgress{Downloaded: downloaded})
-	}, 250*time.Millisecond)
-	if d.Error() != nil {
-		return d.Error()
-	}
-	downloadCB(&rpc.DownloadProgress{Completed: true})
-	return nil
-}

commands/lib/download.go

@@ -18,7 +18,6 @@ package lib
 import (
 	"context"
 	"fmt"
-	"net/http"
 
 	"github.com/arduino/arduino-cli/arduino/libraries/librariesindex"
 	"github.com/arduino/arduino-cli/arduino/libraries/librariesmanager"
@@ -28,8 +27,7 @@ import (
 )
 
 // LibraryDownload FIXMEDOC
-func LibraryDownload(ctx context.Context, req *rpc.LibraryDownloadReq, downloadCB commands.DownloadProgressCB,
-	downloaderHeaders http.Header) (*rpc.LibraryDownloadResp, error) {
+func LibraryDownload(ctx context.Context, req *rpc.LibraryDownloadReq, downloadCB commands.DownloadProgressCB) (*rpc.LibraryDownloadResp, error) {
 	logrus.Info("Executing `arduino lib download`")
 
 	lm := commands.GetLibraryManager(req.GetInstance().GetId())
@@ -41,7 +39,7 @@ func LibraryDownload(ctx context.Context, req *rpc.LibraryDownloadReq, downloadC
 		return nil, fmt.Errorf("looking for library: %s", err)
 	}
 
-	if err := downloadLibrary(lm, lib, downloadCB, func(*rpc.TaskProgress) {}, downloaderHeaders); err != nil {
+	if err := downloadLibrary(lm, lib, downloadCB, func(*rpc.TaskProgress) {}); err != nil {
 		return nil, err
 	}
 
@@ -49,10 +47,14 @@ func LibraryDownload(ctx context.Context, req *rpc.LibraryDownloadReq, downloadC
 }
 
 func downloadLibrary(lm *librariesmanager.LibrariesManager, libRelease *librariesindex.Release,
-	downloadCB commands.DownloadProgressCB, taskCB commands.TaskProgressCB, downloaderHeaders http.Header) error {
+	downloadCB commands.DownloadProgressCB, taskCB commands.TaskProgressCB) error {
 
 	taskCB(&rpc.TaskProgress{Name: "Downloading " + libRelease.String()})
-	if d, err := libRelease.Resource.Download(lm.DownloadsDir, downloaderHeaders); err != nil {
+	config, err := commands.GetDownloaderConfig()
+	if err != nil {
+		return err
+	}
+	if d, err := libRelease.Resource.Download(lm.DownloadsDir, config); err != nil {
 		return err
 	} else if err := commands.Download(d, libRelease.String(), downloadCB); err != nil {
 		return err

commands/lib/install.go

@@ -18,7 +18,6 @@ package lib
 import (
 	"context"
 	"fmt"
-	"net/http"
 
 	"github.com/arduino/arduino-cli/arduino/libraries/librariesindex"
 	"github.com/arduino/arduino-cli/arduino/libraries/librariesmanager"
@@ -29,7 +28,7 @@ import (
 
 // LibraryInstall FIXMEDOC
 func LibraryInstall(ctx context.Context, req *rpc.LibraryInstallReq,
-	downloadCB commands.DownloadProgressCB, taskCB commands.TaskProgressCB, downloaderHeaders http.Header) error {
+	downloadCB commands.DownloadProgressCB, taskCB commands.TaskProgressCB) error {
 
 	lm := commands.GetLibraryManager(req.GetInstance().GetId())
 
@@ -38,7 +37,7 @@ func LibraryInstall(ctx context.Context, req *rpc.LibraryInstallReq,
 		return fmt.Errorf("looking for library: %s", err)
 	}
 
-	if err := downloadLibrary(lm, libRelease, downloadCB, taskCB, downloaderHeaders); err != nil {
+	if err := downloadLibrary(lm, libRelease, downloadCB, taskCB); err != nil {
 		return fmt.Errorf("downloading library: %s", err)
 	}
 

commands/lib/upgrade.go

@@ -17,7 +17,6 @@ package lib
 
 import (
 	"fmt"
-	"net/http"
 
 	"github.com/arduino/arduino-cli/arduino/libraries/librariesmanager"
 	"github.com/arduino/arduino-cli/commands"
@@ -25,11 +24,11 @@ import (
 
 // LibraryUpgradeAll upgrades all the available libraries
 func LibraryUpgradeAll(instanceID int32, downloadCB commands.DownloadProgressCB,
-	taskCB commands.TaskProgressCB, headers http.Header) error {
+	taskCB commands.TaskProgressCB) error {
 	// get the library manager
 	lm := commands.GetLibraryManager(instanceID)
 
-	if err := upgrade(lm, listLibraries(lm, true, true), downloadCB, taskCB, headers); err != nil {
+	if err := upgrade(lm, listLibraries(lm, true, true), downloadCB, taskCB); err != nil {
 		return err
 	}
 
@@ -42,7 +41,7 @@ func LibraryUpgradeAll(instanceID int32, downloadCB commands.DownloadProgressCB,
 
 // LibraryUpgrade upgrades only the given libraries
 func LibraryUpgrade(instanceID int32, libraryNames []string, downloadCB commands.DownloadProgressCB,
-	taskCB commands.TaskProgressCB, headers http.Header) error {
+	taskCB commands.TaskProgressCB) error {
 	// get the library manager
 	lm := commands.GetLibraryManager(instanceID)
 
@@ -50,16 +49,16 @@ func LibraryUpgrade(instanceID int32, libraryNames []string, downloadCB commands
 	libs := filterByName(listLibraries(lm, true, true), libraryNames)
 
 	// do it
-	return upgrade(lm, libs, downloadCB, taskCB, headers)
+	return upgrade(lm, libs, downloadCB, taskCB)
 }
 
 func upgrade(lm *librariesmanager.LibrariesManager, libs []*installedLib, downloadCB commands.DownloadProgressCB,
-	taskCB commands.TaskProgressCB, downloaderHeaders http.Header) error {
+	taskCB commands.TaskProgressCB) error {
 
 	// Go through the list and download them
 
 	for _, lib := range libs {
-		if err := downloadLibrary(lm, lib.Available, downloadCB, taskCB, downloaderHeaders); err != nil {
+		if err := downloadLibrary(lm, lib.Available, downloadCB, taskCB); err != nil {
 			return err
 		}
 	}

configuration/defaults.go

@@ -25,6 +25,7 @@ func setDefaults(dataDir, userDir string) {
 	// logging
 	viper.SetDefault("logging.level", "info")
 	viper.SetDefault("logging.format", "text")
+
 	// board manager
 	viper.SetDefault("board_manager.additional_urls", []string{})
 

go.mod

@@ -38,7 +38,7 @@ require (
 	github.com/spf13/viper v1.6.2
 	github.com/stretchr/testify v1.4.0
 	go.bug.st/cleanup v1.0.0
-	go.bug.st/downloader v1.1.0
+	go.bug.st/downloader v1.2.0
 	go.bug.st/relaxed-semver v0.0.0-20190922224835-391e10178d18
 	go.bug.st/serial v1.0.0
 	go.bug.st/serial.v1 v0.0.0-20180827123349-5f7892a7bb45 // indirect

go.sum

@@ -189,6 +189,7 @@ github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DM
 github.com/spf13/viper v1.6.2 h1:7aKfF+e8/k68gda3LOjo5RxiUqddoFxVq4BKBPrxk5E=
 github.com/spf13/viper v1.6.2/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfDi5k=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
@@ -204,8 +205,8 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 go.bug.st/cleanup v1.0.0 h1:XVj1HZxkBXeq3gMT7ijWUpHyIC1j8XAoNSyQ06CskgA=
 go.bug.st/cleanup v1.0.0/go.mod h1:EqVmTg2IBk4znLbPD28xne3abjsJftMdqqJEjhn70bk=
-go.bug.st/downloader v1.1.0 h1:LipC9rqRCe8kwa+ah3ZDfCqneVaf34cB/TKjXZiZt54=
-go.bug.st/downloader v1.1.0/go.mod h1:l+RPbNbrTB+MoAIp8nrZsP22nRPDy26XJZQqmm4gNT4=
+go.bug.st/downloader v1.2.0 h1:YmXFTcTnm0v8WzAWHn2DyV46c/Izlc/gReXubc2oBho=
+go.bug.st/downloader v1.2.0/go.mod h1:l+RPbNbrTB+MoAIp8nrZsP22nRPDy26XJZQqmm4gNT4=
 go.bug.st/relaxed-semver v0.0.0-20190922224835-391e10178d18 h1:F1qxtaFuewctYc/SsHRn+Q7Dtwi+yJGPgVq8YLtQz98=
 go.bug.st/relaxed-semver v0.0.0-20190922224835-391e10178d18/go.mod h1:Cx1VqMtEhE9pIkEyUj3LVVVPkv89dgW8aCKrRPDR/uE=
 go.bug.st/serial v1.0.0 h1:ogEPzrllCsnG00EqKRjeYvPRsO7NJW6DqykzkdD6E/k=