Unverified Commit 88468f21 authored by Roberto Sora's avatar Roberto Sora Committed by GitHub

Add support for compile with debug optimizations (#593)

* add debug symbols option in CLI and gRPC insterface

* Calculate precompiled core path usign also build optimization flags

* Align failing lecacy tests with new hashing behaviour
parent fc130473
......@@ -48,6 +48,7 @@ var (
verify bool // Upload, verify uploaded binary after the upload.
exportFile string // The compiled binary is written to this file
libraries []string // List of custom libraries paths separated by commas. Or can be used multiple times for multiple libraries paths.
optimizeForDebug bool // Optimize compile output for debug, not for release
)
// NewCommand created a new `compile` command
......@@ -80,6 +81,7 @@ func NewCommand() *cobra.Command {
command.Flags().StringVar(&vidPid, "vid-pid", "", "When specified, VID/PID specific build properties are used, if boards supports them.")
command.Flags().StringSliceVar(&libraries, "libraries", []string{},
"List of custom libraries paths separated by commas. Or can be used multiple times for multiple libraries paths.")
command.Flags().BoolVar(&optimizeForDebug, "optimize-for-debug", false, "Optional, optimize compile output for debug, not for release.")
return command
}
......@@ -99,20 +101,21 @@ func run(cmd *cobra.Command, args []string) {
sketchPath := initSketchPath(path)
_, err = compile.Compile(context.Background(), &rpc.CompileReq{
Instance: inst,
Fqbn: fqbn,
SketchPath: sketchPath.String(),
ShowProperties: showProperties,
Preprocess: preprocess,
BuildCachePath: buildCachePath,
BuildPath: buildPath,
BuildProperties: buildProperties,
Warnings: warnings,
Verbose: verbose,
Quiet: quiet,
VidPid: vidPid,
ExportFile: exportFile,
Libraries: libraries,
Instance: inst,
Fqbn: fqbn,
SketchPath: sketchPath.String(),
ShowProperties: showProperties,
Preprocess: preprocess,
BuildCachePath: buildCachePath,
BuildPath: buildPath,
BuildProperties: buildProperties,
Warnings: warnings,
Verbose: verbose,
Quiet: quiet,
VidPid: vidPid,
ExportFile: exportFile,
Libraries: libraries,
OptimizeForDebug: optimizeForDebug,
}, os.Stdout, os.Stderr, viper.GetString("logging.level") == "debug")
if err != nil {
......
......@@ -103,6 +103,9 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W
builderCtx.Verbose = req.GetVerbose()
// Optimize for debug
builderCtx.OptimizeForDebug = req.GetOptimizeForDebug()
builderCtx.CoreBuildCachePath = paths.TempDir().Join("arduino-core-cache")
builderCtx.Jobs = int(req.GetJobs())
......
......@@ -519,20 +519,3 @@ func PrepareCommandForRecipe(ctx *types.Context, buildProperties *properties.Map
return command, nil
}
// GetCachedCoreArchiveFileName returns the filename to be used to store
// the global cached core.a.
func GetCachedCoreArchiveFileName(fqbn string, coreFolder *paths.Path) string {
fqbnToUnderscore := strings.Replace(fqbn, ":", "_", -1)
fqbnToUnderscore = strings.Replace(fqbnToUnderscore, "=", "_", -1)
if absCoreFolder, err := coreFolder.Abs(); err == nil {
coreFolder = absCoreFolder
} // silently continue if absolute path can't be detected
hash := utils.MD5Sum([]byte(coreFolder.String()))
realName := "core_" + fqbnToUnderscore + "_" + hash + ".a"
if len(realName) > 100 {
// avoid really long names, simply hash the final part
realName = "core_" + utils.MD5Sum([]byte(fqbnToUnderscore+"_"+hash)) + ".a"
}
return realName
}
......@@ -17,7 +17,6 @@ package builder
import (
"encoding/json"
"github.com/arduino/arduino-cli/legacy/builder/i18n"
"github.com/arduino/arduino-cli/legacy/builder/types"
)
......@@ -30,7 +29,6 @@ func (s *CreateBuildOptionsMap) Run(ctx *types.Context) error {
if err != nil {
return i18n.WrapError(err)
}
ctx.BuildOptionsJson = string(bytes)
return nil
......
......@@ -17,6 +17,7 @@ package phases
import (
"os"
"strings"
"github.com/arduino/arduino-cli/legacy/builder/builder_utils"
"github.com/arduino/arduino-cli/legacy/builder/constants"
......@@ -90,7 +91,8 @@ func compileCore(ctx *types.Context, buildPath *paths.Path, buildCachePath *path
var targetArchivedCore *paths.Path
if buildCachePath != nil {
archivedCoreName := builder_utils.GetCachedCoreArchiveFileName(buildProperties.Get(constants.BUILD_PROPERTIES_FQBN), realCoreFolder)
archivedCoreName := GetCachedCoreArchiveFileName(buildProperties.Get(constants.BUILD_PROPERTIES_FQBN),
buildProperties.Get("compiler.optimization_flags"), realCoreFolder)
targetArchivedCore = buildCachePath.Join(archivedCoreName)
canUseArchivedCore := !builder_utils.CoreOrReferencedCoreHasChanged(realCoreFolder, targetCoreFolder, targetArchivedCore)
......@@ -129,3 +131,20 @@ func compileCore(ctx *types.Context, buildPath *paths.Path, buildCachePath *path
return archiveFile, variantObjectFiles, nil
}
// GetCachedCoreArchiveFileName returns the filename to be used to store
// the global cached core.a.
func GetCachedCoreArchiveFileName(fqbn string, optimizationFlags string, coreFolder *paths.Path) string {
fqbnToUnderscore := strings.Replace(fqbn, ":", "_", -1)
fqbnToUnderscore = strings.Replace(fqbnToUnderscore, "=", "_", -1)
if absCoreFolder, err := coreFolder.Abs(); err == nil {
coreFolder = absCoreFolder
} // silently continue if absolute path can't be detected
hash := utils.MD5Sum([]byte(coreFolder.String() + optimizationFlags))
realName := "core_" + fqbnToUnderscore + "_" + hash + ".a"
if len(realName) > 100 {
// avoid really long names, simply hash the final part
realName = "core_" + utils.MD5Sum([]byte(fqbnToUnderscore+"_"+hash)) + ".a"
}
return realName
}
......@@ -69,6 +69,17 @@ func (s *SetupBuildProperties) Run(ctx *types.Context) error {
buildProperties.Set("ide_version", ctx.ArduinoAPIVersion)
buildProperties.Set("runtime.os", utils.PrettyOSName())
if ctx.OptimizeForDebug {
if buildProperties.ContainsKey("compiler.optimization_flags.debug") {
buildProperties.Set("compiler.optimization_flags", buildProperties.Get("compiler.optimization_flags.debug"))
}
} else {
if buildProperties.ContainsKey("compiler.optimization_flags.release") {
buildProperties.Set("compiler.optimization_flags", buildProperties.Get("compiler.optimization_flags.release"))
}
}
ctx.OptimizationFlags = buildProperties.Get("compiler.optimization_flags")
variant := buildProperties.Get("build.variant")
if variant == "" {
buildProperties.Set("build.variant.path", "")
......
......@@ -24,8 +24,8 @@ import (
"github.com/arduino/go-paths-helper"
"github.com/arduino/arduino-cli/legacy/builder"
"github.com/arduino/arduino-cli/legacy/builder/builder_utils"
"github.com/arduino/arduino-cli/legacy/builder/constants"
"github.com/arduino/arduino-cli/legacy/builder/phases"
"github.com/arduino/arduino-cli/legacy/builder/types"
"github.com/stretchr/testify/require"
)
......@@ -379,7 +379,7 @@ func TestBuilderCacheCoreAFile(t *testing.T) {
// Pick timestamp of cached core
coreFolder := paths.New("downloaded_hardware", "arduino", "avr")
coreFileName := builder_utils.GetCachedCoreArchiveFileName(ctx.FQBN.String(), coreFolder)
coreFileName := phases.GetCachedCoreArchiveFileName(ctx.FQBN.String(), ctx.OptimizationFlags, coreFolder)
cachedCoreFile := ctx.CoreBuildCachePath.Join(coreFileName)
coreStatBefore, err := cachedCoreFile.Stat()
require.NoError(t, err)
......
......@@ -35,6 +35,7 @@ func TestCreateBuildOptionsMap(t *testing.T) {
Verbose: true,
BuildPath: paths.New("buildPath"),
DebugLevel: 5,
OptimizationFlags: "-Os",
}
create := builder.CreateBuildOptionsMap{}
......@@ -45,6 +46,7 @@ func TestCreateBuildOptionsMap(t *testing.T) {
"additionalFiles": "",
"builtInLibrariesFolders": "",
"builtInToolsFolders": "tools",
"compiler.optimization_flags": "-Os",
"customBuildProperties": "",
"fqbn": "my:nice:fqbn",
"hardwareFolders": "hardware,hardware2",
......
......@@ -37,6 +37,7 @@ func TestStoreBuildOptionsMap(t *testing.T) {
CustomBuildProperties: []string{"custom=prop"},
Verbose: true,
DebugLevel: 5,
OptimizationFlags: "-Os",
}
buildPath := SetupBuildPath(t, ctx)
......@@ -63,6 +64,7 @@ func TestStoreBuildOptionsMap(t *testing.T) {
"additionalFiles": "",
"builtInLibrariesFolders": "built-in libraries",
"builtInToolsFolders": "tools",
"compiler.optimization_flags": "-Os",
"customBuildProperties": "custom=prop",
"fqbn": "my:nice:fqbn",
"hardwareFolders": "hardware",
......
......@@ -94,6 +94,10 @@ type Context struct {
Verbose bool
DebugPreprocessor bool
// Compile optimization settings
OptimizeForDebug bool
OptimizationFlags string
// Dry run, only create progress map
Progress ProgressStruct
......@@ -140,6 +144,7 @@ func (ctx *Context) ExtractBuildOptions() *properties.Map {
opts.Set("runtime.ide.version", ctx.ArduinoAPIVersion)
opts.Set("customBuildProperties", strings.Join(ctx.CustomBuildProperties, ","))
opts.Set("additionalFiles", strings.Join(additionalFilesRelative, ","))
opts.Set("compiler.optimization_flags", ctx.OptimizationFlags)
return opts
}
......@@ -156,6 +161,7 @@ func (ctx *Context) InjectBuildOptions(opts *properties.Map) {
ctx.FQBN = fqbn
ctx.ArduinoAPIVersion = opts.Get("runtime.ide.version")
ctx.CustomBuildProperties = strings.Split(opts.Get("customBuildProperties"), ",")
ctx.OptimizationFlags = opts.Get("compiler.optimization_flags")
}
func (ctx *Context) GetLogger() i18n.Logger {
......
......@@ -36,6 +36,7 @@ type CompileReq struct {
ExportFile string `protobuf:"bytes,13,opt,name=exportFile,proto3" json:"exportFile,omitempty"`
Jobs int32 `protobuf:"varint,14,opt,name=jobs,proto3" json:"jobs,omitempty"`
Libraries []string `protobuf:"bytes,15,rep,name=libraries,proto3" json:"libraries,omitempty"`
OptimizeForDebug bool `protobuf:"varint,16,opt,name=optimizeForDebug,proto3" json:"optimizeForDebug,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
......@@ -171,6 +172,13 @@ func (m *CompileReq) GetLibraries() []string {
return nil
}
func (m *CompileReq) GetOptimizeForDebug() bool {
if m != nil {
return m.OptimizeForDebug
}
return false
}
type CompileResp struct {
OutStream []byte `protobuf:"bytes,1,opt,name=out_stream,json=outStream,proto3" json:"out_stream,omitempty"`
ErrStream []byte `protobuf:"bytes,2,opt,name=err_stream,json=errStream,proto3" json:"err_stream,omitempty"`
......@@ -226,31 +234,32 @@ func init() {
func init() { proto.RegisterFile("commands/compile.proto", fileDescriptor_86bc582849c76c3d) }
var fileDescriptor_86bc582849c76c3d = []byte{
// 405 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x74, 0x92, 0x41, 0x6f, 0xd4, 0x30,
0x10, 0x85, 0x95, 0x6e, 0x77, 0xbb, 0x99, 0x2d, 0xad, 0x64, 0x41, 0xb1, 0x2a, 0x40, 0xa1, 0x07,
0x14, 0x09, 0x35, 0x2b, 0xc1, 0x99, 0x0b, 0x95, 0x90, 0x10, 0x97, 0x55, 0xb8, 0x71, 0x41, 0x89,
0x33, 0x34, 0x86, 0x24, 0xf6, 0x8e, 0x9d, 0x96, 0x3f, 0xcd, 0x7f, 0x40, 0x9e, 0x6c, 0x36, 0xab,
0x95, 0x7a, 0x4a, 0xe6, 0x7b, 0xcf, 0xcf, 0x63, 0x7b, 0xe0, 0x4a, 0x99, 0xb6, 0x2d, 0xba, 0xca,
0xad, 0x95, 0x69, 0xad, 0x6e, 0x30, 0xb3, 0x64, 0xbc, 0x11, 0x2f, 0x95, 0xca, 0x0a, 0xaa, 0x7a,
0xdd, 0x99, 0x4c, 0x35, 0x3a, 0x1b, 0x6d, 0xd7, 0x2f, 0x0e, 0x17, 0xb4, 0xa6, 0x1b, 0xfc, 0x37,
0xff, 0x66, 0x00, 0x77, 0x43, 0x42, 0x8e, 0x5b, 0xf1, 0x09, 0x96, 0xba, 0x73, 0xbe, 0xe8, 0x14,
0xca, 0x28, 0x89, 0xd2, 0xd5, 0x87, 0xb7, 0xd9, 0x13, 0x89, 0xd9, 0xd7, 0x9d, 0x31, 0xdf, 0x2f,
0x11, 0x02, 0x4e, 0x7f, 0x6d, 0xcb, 0x4e, 0x9e, 0x24, 0x51, 0x1a, 0xe7, 0xfc, 0x2f, 0xde, 0x00,
0xb8, 0x3f, 0xe8, 0x55, 0xbd, 0x29, 0x7c, 0x2d, 0x67, 0xac, 0x1c, 0x10, 0xf1, 0x0e, 0x2e, 0x5c,
0x6d, 0x1e, 0x37, 0x64, 0x2c, 0x92, 0xd7, 0xe8, 0xe4, 0x69, 0x12, 0xa5, 0xcb, 0xfc, 0x88, 0x86,
0x1c, 0x4b, 0x68, 0xc9, 0x28, 0x74, 0x4e, 0xce, 0xd9, 0x73, 0x40, 0x42, 0x4e, 0xd9, 0xeb, 0xa6,
0xba, 0x2b, 0x54, 0x8d, 0xbc, 0xd7, 0x82, 0xf7, 0x3a, 0xa2, 0xe2, 0x15, 0xc4, 0x4c, 0xd8, 0x72,
0xc6, 0x96, 0x09, 0x88, 0x14, 0x2e, 0x87, 0x62, 0x6a, 0x67, 0x99, 0xcc, 0xd2, 0x38, 0x3f, 0xc6,
0xe2, 0x1a, 0x96, 0x8f, 0x05, 0x75, 0xba, 0xbb, 0x77, 0x32, 0xe6, 0x98, 0x7d, 0x2d, 0x24, 0x9c,
0x3d, 0x20, 0x95, 0xc6, 0xa1, 0x04, 0x6e, 0x74, 0x2c, 0xc5, 0x73, 0x98, 0x6f, 0x7b, 0x8d, 0x5e,
0xae, 0x98, 0x0f, 0x85, 0xb8, 0x82, 0xc5, 0x83, 0xae, 0x36, 0xba, 0x92, 0xe7, 0x9c, 0xb4, 0xab,
0xc2, 0x99, 0xf1, 0xaf, 0x35, 0xe4, 0xbf, 0xe8, 0x06, 0xe5, 0xb3, 0xe1, 0xee, 0x26, 0x12, 0xee,
0xfb, 0xb7, 0x29, 0x9d, 0xbc, 0x48, 0xa2, 0x74, 0x9e, 0xf3, 0x7f, 0x38, 0x5f, 0xa3, 0x4b, 0x2a,
0x28, 0xf4, 0x7e, 0xc9, 0xbd, 0x4f, 0xe0, 0xe6, 0x1b, 0xac, 0xf6, 0xcf, 0xed, 0xac, 0x78, 0x0d,
0x60, 0x7a, 0xff, 0xd3, 0x79, 0xc2, 0xa2, 0xe5, 0x17, 0x3f, 0xcf, 0x63, 0xd3, 0xfb, 0xef, 0x0c,
0x82, 0x8c, 0x44, 0xa3, 0x7c, 0x32, 0xc8, 0x48, 0x34, 0xc8, 0x9f, 0x6f, 0x7f, 0xbc, 0xbf, 0xd7,
0xbe, 0xee, 0xcb, 0x30, 0x14, 0xeb, 0xdd, 0x90, 0x8c, 0xdf, 0x5b, 0xd5, 0xe8, 0x35, 0x59, 0xb5,
0x1e, 0x07, 0xa6, 0x5c, 0xf0, 0xc8, 0x7d, 0xfc, 0x1f, 0x00, 0x00, 0xff, 0xff, 0x89, 0x00, 0xe4,
0x1f, 0xbc, 0x02, 0x00, 0x00,
// 426 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x74, 0x92, 0xc1, 0x6f, 0xd3, 0x30,
0x14, 0xc6, 0x95, 0xad, 0xed, 0x9a, 0xd7, 0xb1, 0x4d, 0x16, 0x0c, 0x6b, 0x02, 0x14, 0x76, 0x40,
0x11, 0x68, 0xa9, 0x04, 0x67, 0x2e, 0x0c, 0x4d, 0x42, 0x5c, 0xaa, 0x70, 0xe3, 0x82, 0x12, 0xe7,
0xd1, 0x18, 0x92, 0xd8, 0x7d, 0x76, 0x36, 0xc4, 0x7f, 0xc2, 0x7f, 0x8b, 0xf2, 0xd2, 0x34, 0x55,
0x11, 0x27, 0xfb, 0xfd, 0xde, 0xe7, 0xcf, 0xcf, 0xd6, 0x07, 0x97, 0xca, 0xd4, 0x75, 0xd6, 0x14,
0x6e, 0xa9, 0x4c, 0x6d, 0x75, 0x85, 0x89, 0x25, 0xe3, 0x8d, 0x78, 0xaa, 0x54, 0x92, 0x51, 0xd1,
0xea, 0xc6, 0x24, 0xaa, 0xd2, 0xc9, 0x20, 0xbb, 0x7a, 0xb2, 0x7f, 0xa0, 0x36, 0x4d, 0xaf, 0xbf,
0xfe, 0x33, 0x01, 0xb8, 0xed, 0x1d, 0x52, 0xdc, 0x88, 0xf7, 0x30, 0xd7, 0x8d, 0xf3, 0x59, 0xa3,
0x50, 0x06, 0x51, 0x10, 0x2f, 0xde, 0xbe, 0x4c, 0xfe, 0xe3, 0x98, 0x7c, 0xda, 0x0a, 0xd3, 0xdd,
0x11, 0x21, 0x60, 0xf2, 0x7d, 0x93, 0x37, 0xf2, 0x28, 0x0a, 0xe2, 0x30, 0xe5, 0xbd, 0x78, 0x01,
0xe0, 0x7e, 0xa2, 0x57, 0xe5, 0x2a, 0xf3, 0xa5, 0x3c, 0xe6, 0xce, 0x1e, 0x11, 0xaf, 0xe0, 0xcc,
0x95, 0xe6, 0x61, 0x45, 0xc6, 0x22, 0x79, 0x8d, 0x4e, 0x4e, 0xa2, 0x20, 0x9e, 0xa7, 0x07, 0xb4,
0xf3, 0xb1, 0x84, 0x96, 0x8c, 0x42, 0xe7, 0xe4, 0x94, 0x35, 0x7b, 0xa4, 0xf3, 0xc9, 0x5b, 0x5d,
0x15, 0xb7, 0x99, 0x2a, 0x91, 0xef, 0x9a, 0xf1, 0x5d, 0x07, 0x54, 0x3c, 0x83, 0x90, 0x09, 0x4b,
0x4e, 0x58, 0x32, 0x02, 0x11, 0xc3, 0x79, 0x5f, 0x8c, 0xe3, 0xcc, 0xa3, 0xe3, 0x38, 0x4c, 0x0f,
0xb1, 0xb8, 0x82, 0xf9, 0x43, 0x46, 0x8d, 0x6e, 0xd6, 0x4e, 0x86, 0x6c, 0xb3, 0xab, 0x85, 0x84,
0x93, 0x7b, 0xa4, 0xdc, 0x38, 0x94, 0xc0, 0x83, 0x0e, 0xa5, 0x78, 0x0c, 0xd3, 0x4d, 0xab, 0xd1,
0xcb, 0x05, 0xf3, 0xbe, 0x10, 0x97, 0x30, 0xbb, 0xd7, 0xc5, 0x4a, 0x17, 0xf2, 0x94, 0x9d, 0xb6,
0x55, 0xf7, 0x66, 0xfc, 0x65, 0x0d, 0xf9, 0x3b, 0x5d, 0xa1, 0x7c, 0xd4, 0xff, 0xdd, 0x48, 0xba,
0xff, 0xfe, 0x61, 0x72, 0x27, 0xcf, 0xa2, 0x20, 0x9e, 0xa6, 0xbc, 0xef, 0xde, 0x57, 0xe9, 0x9c,
0x32, 0xea, 0x66, 0x3f, 0xe7, 0xd9, 0x47, 0x20, 0x5e, 0xc3, 0x85, 0xb1, 0x5e, 0xd7, 0xfa, 0x37,
0xde, 0x19, 0xfa, 0x88, 0x79, 0xbb, 0x96, 0x17, 0x3c, 0xca, 0x3f, 0xfc, 0xfa, 0x33, 0x2c, 0x76,
0xd1, 0x70, 0x56, 0x3c, 0x07, 0x30, 0xad, 0xff, 0xe6, 0x3c, 0x61, 0x56, 0x73, 0x3a, 0x4e, 0xd3,
0xd0, 0xb4, 0xfe, 0x0b, 0x83, 0xae, 0x8d, 0x44, 0x43, 0xfb, 0xa8, 0x6f, 0x23, 0x51, 0xdf, 0xfe,
0x70, 0xf3, 0xf5, 0xcd, 0x5a, 0xfb, 0xb2, 0xcd, 0xbb, 0x00, 0x2d, 0xb7, 0x81, 0x1a, 0xd6, 0x1b,
0x55, 0xe9, 0x25, 0x59, 0xb5, 0x1c, 0xc2, 0x95, 0xcf, 0x38, 0x9e, 0xef, 0xfe, 0x06, 0x00, 0x00,
0xff, 0xff, 0x89, 0xd3, 0x60, 0xf6, 0xe8, 0x02, 0x00, 0x00,
}
......@@ -37,6 +37,7 @@ message CompileReq {
string exportFile = 13; // The compiled binary is written to this file
int32 jobs = 14; // The max number of concurrent compiler instances to run (as make -jx)
repeated string libraries = 15; // List of custom libraries paths separated by commas. Or can be used multiple times for multiple libraries paths.
bool optimizeForDebug = 16; // Optimize compile output for debug, not for release
}
message CompileResp {
......
......@@ -205,8 +205,7 @@ func (m *DebugResp) GetError() string {
return ""
}
// duplicate from commands/common.proto
// as module imports seems not to work
// TODO remove this in next proto refactoring because is a duplicate from commands/common.proto
type Instance struct {
Id int32 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment