Use jobs flag to really control number of concurrent compilations

The issue was due to the peculiar way concurrency and parallelism are handled in go.
We used to set GOMAXPROC to 1 to avoid parallelizing the WaitGroup execution.
This would work, in theory, unless the goroutines sleep.
In that case, another goroutine is allowed to start concurrently (only 1 goroutine running in parallel, so GOMAXPROC is happy).
Since our goroutines sleep (wait) after calling gcc, another task is started, without any hard limit, till the WaitGroup is completely spawned.
On systems with limited resources (as RaspberryPi) and cores with lots of files this can trigger an out of memory condition.

commands/compile/compile.go

@@ -23,6 +23,7 @@ import (
 	"fmt"
 	"io"
 	"path/filepath"
+	"runtime"
 	"sort"
 	"strings"
 
@@ -115,6 +116,12 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W
 
 	builderCtx.CoreBuildCachePath = paths.TempDir().Join("arduino-core-cache")
 
+	jobs := runtime.NumCPU()
+	if req.GetJobs() > 0 {
+		jobs = int(req.GetJobs())
+	}
+	builderCtx.Jobs = jobs
+
 	builderCtx.USBVidPid = req.GetVidPid()
 	builderCtx.WarningsLevel = req.GetWarnings()
 

legacy/builder/builder_utils/utils.go

@@ -175,20 +175,30 @@ func compileFilesWithRecipe(ctx *types.Context, sourcePath *paths.Path, sources
 
 	ctx.Progress.Steps = ctx.Progress.Steps / float64(len(sources))
 	var wg sync.WaitGroup
-	wg.Add(len(sources))
 
-	for _, source := range sources {
-		go func(source *paths.Path) {
-			defer wg.Done()
-			PrintProgressIfProgressEnabledAndMachineLogger(ctx)
-			objectFile, err := compileFileWithRecipe(ctx, sourcePath, source, buildPath, buildProperties, includes, recipe)
-			if err != nil {
-				errorsChan <- err
-			} else {
-				objectFilesChan <- objectFile
+	// Split jobs into batches of N jobs each; wait for the completion of a batch to start the next
+	par := ctx.Jobs
+
+	go func() {
+		for total := 0; total < len(sources); total += par {
+			for i := total; i < total+par && i < len(sources); i++ {
+				wg.Add(1)
+				go func(source *paths.Path) {
+					defer wg.Done()
+					PrintProgressIfProgressEnabledAndMachineLogger(ctx)
+					objectFile, err := compileFileWithRecipe(ctx, sourcePath, source, buildPath, buildProperties, includes, recipe)
+					if err != nil {
+						errorsChan <- err
+					} else {
+						objectFilesChan <- objectFile
+					}
+				}(sources[i])
 			}
-		}(source)
-	}
+			wg.Wait()
+		}
+
+		doneChan <- struct{}{}
+	}()
 
 	go func() {
 		wg.Wait()

legacy/builder/types/context.go

@@ -111,6 +111,9 @@ type Context struct {
 	// Experimental: use arduino-preprocessor to create prototypes
 	UseArduinoPreprocessor bool
 
+	// Parallel processes
+	Jobs int
+
 	// Out and Err stream to redirect all Exec commands
 	ExecStdout io.Writer
 	ExecStderr io.Writer