Commit c2678cff authored by Martino Facchin's avatar Martino Facchin Committed by Cristian Maglie

Reuse (cache) object archives in large sketch projects (#2464)

* Reuse archiveCompiledFiles helper for long commandline shrink

Since archiveCompiledFiles already handles hot cache correctly, this avoids objs.a being rebuilt even if files don't change.

Would be ideal if PathList could expose a generic Filter API (to get rid of the "duplicated" filter)

* Upgrade go-paths / remove duplicate filter function

* Consider existing archives during the build

* Simplified archiveCompiledFiles function signature

It doesn't make sense anymore to keep path and filename separated.

* Added integration test

---------
Co-authored-by: default avatarCristian Maglie <c.maglie@arduino.cc>
parent 77222ecd
---
name: github.com/arduino/go-paths-helper
version: v1.9.2
version: v1.11.0
type: go
summary:
homepage: https://pkg.go.dev/github.com/arduino/go-paths-helper
......
......@@ -21,9 +21,7 @@ import (
)
// ArchiveCompiledFiles fixdoc
func (b *Builder) archiveCompiledFiles(buildPath *paths.Path, archiveFile *paths.Path, objectFilesToArchive paths.PathList) (*paths.Path, error) {
archiveFilePath := buildPath.JoinPath(archiveFile)
func (b *Builder) archiveCompiledFiles(archiveFilePath *paths.Path, objectFilesToArchive paths.PathList) (*paths.Path, error) {
if b.onlyUpdateCompilationDatabase {
if b.logger.Verbose() {
b.logger.Info(tr("Skipping archive creation of: %[1]s", archiveFilePath))
......
......@@ -128,7 +128,7 @@ func (b *Builder) compileCore() (*paths.Path, paths.PathList, error) {
return nil, nil, errors.WithStack(err)
}
archiveFile, err := b.archiveCompiledFiles(b.coreBuildPath, paths.New("core.a"), coreObjectFiles)
archiveFile, err := b.archiveCompiledFiles(b.coreBuildPath.Join("core.a"), coreObjectFiles)
if err != nil {
return nil, nil, errors.WithStack(err)
}
......
......@@ -197,7 +197,7 @@ func (b *Builder) compileLibrary(library *libraries.Library, includes []string)
return nil, errors.WithStack(err)
}
if library.DotALinkage {
archiveFile, err := b.archiveCompiledFiles(libraryBuildPath, paths.New(library.DirName+".a"), libObjectFiles)
archiveFile, err := b.archiveCompiledFiles(libraryBuildPath.Join(library.DirName+".a"), libObjectFiles)
if err != nil {
return nil, errors.WithStack(err)
}
......
......@@ -54,34 +54,31 @@ func (b *Builder) link() error {
// it may happen that a subdir/spi.o inside the archive may be overwritten by a anotherdir/spi.o
// because thery are both named spi.o.
properties := b.buildProperties.Clone()
archives := paths.NewPathList()
// Put all the existing archives apart from the other object files
existingArchives := objectFiles.Clone()
existingArchives.FilterSuffix(".a")
objectFiles.FilterOutSuffix(".a")
// Generate an archive for each directory from the remaining object files
newArchives := paths.NewPathList()
for _, object := range objectFiles {
if object.HasSuffix(".a") {
archives.Add(object)
continue
}
archive := object.Parent().Join("objs.a")
if !archives.Contains(archive) {
archives.Add(archive)
// Cleanup old archives
_ = archive.Remove()
}
properties.Set("archive_file", archive.Base())
properties.SetPath("archive_file_path", archive)
properties.SetPath("object_file", object)
command, err := b.prepareCommandForRecipe(properties, "recipe.ar.pattern", false)
if err != nil {
return errors.WithStack(err)
}
if err := b.execCommand(command); err != nil {
return errors.WithStack(err)
}
newArchives.AddIfMissing(archive)
}
for _, archive := range newArchives {
archiveDir := archive.Parent()
relatedObjectFiles := objectFiles.Clone()
relatedObjectFiles.Filter(func(object *paths.Path) bool {
// extract all the object files that are in the same directory of the archive
return object.Parent().EquivalentTo(archiveDir)
})
b.archiveCompiledFiles(archive, relatedObjectFiles)
}
objectFileList = strings.Join(f.Map(archives.AsStrings(), wrapWithDoubleQuotes), " ")
// Put everything together
allArchives := existingArchives.Clone()
allArchives.AddAll(newArchives)
objectFileList = strings.Join(f.Map(allArchives.AsStrings(), wrapWithDoubleQuotes), " ")
objectFileList = "-Wl,--whole-archive " + objectFileList + " -Wl,--no-whole-archive"
}
......
......@@ -7,7 +7,7 @@ replace github.com/mailru/easyjson => github.com/cmaglie/easyjson v0.8.1
require (
github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371
github.com/arduino/go-paths-helper v1.9.2
github.com/arduino/go-paths-helper v1.11.0
github.com/arduino/go-properties-orderedmap v1.8.0
github.com/arduino/go-timeutils v0.0.0-20171220113728-d1dd9e313b1b
github.com/arduino/go-win32-utils v1.0.0
......
......@@ -21,6 +21,7 @@ import (
"encoding/json"
"fmt"
"os"
"regexp"
"sort"
"strings"
"testing"
......@@ -830,8 +831,17 @@ func TestCompileWithArchivesAndLongPaths(t *testing.T) {
sketchPath := paths.New(libOutput[0]["library"].(map[string]interface{})["install_dir"].(string))
sketchPath = sketchPath.Join("examples", "ArduinoIoTCloud-Advanced")
_, _, err = cli.Run("compile", "-b", "esp8266:esp8266:huzzah", sketchPath.String(), "--config-file", "arduino-cli.yaml")
require.NoError(t, err)
t.Run("Compile", func(t *testing.T) {
_, _, err = cli.Run("compile", "-b", "esp8266:esp8266:huzzah", sketchPath.String(), "--config-file", "arduino-cli.yaml")
require.NoError(t, err)
})
t.Run("CheckCachingOfFolderArchives", func(t *testing.T) {
// Run compile again and check if the archive is re-used (cached)
out, _, err := cli.Run("compile", "-b", "esp8266:esp8266:huzzah", sketchPath.String(), "--config-file", "arduino-cli.yaml", "-v")
require.NoError(t, err)
require.True(t, regexp.MustCompile(`(?m)^Using previously compiled file:.*libraries.ArduinoIoTCloud.objs\.a$`).Match(out))
})
}
func TestCompileWithPrecompileLibrary(t *testing.T) {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment