Draft: Esp insights library support (#7566)

* ESP Insights: Added library support

* ESP Insights: Added Examples

* ESP Insights: Added custom partitions file

* ESP Insights: Added API documentation.

* Added recipe and script to create Insights package

* Updated ESP Insights examples.

* Changed Insights Firmware package output directory

* Changed license to include SPDX license

* Fix Insights package for Windows

* Updated .exe of insights script

* Added coredump partition to all schemes

* Updated header files of Insights diagnostics

* hotfix: Added elf-sha256-offset flag in elf2bin
builder

* Update API to be more Arduino-like and partitions offsets
Co-authored-by: Me No Dev <me-no-dev@users.noreply.github.com>

CMakeLists.txt

@@ -88,6 +88,7 @@ set(LIBRARY_SRCS
   libraries/HTTPClient/src/HTTPClient.cpp
   libraries/HTTPUpdate/src/HTTPUpdate.cpp
   libraries/LittleFS/src/LittleFS.cpp
+  libraries/Insights/src/Insights.cpp
   libraries/I2S/src/I2S.cpp
   libraries/NetBIOS/src/NetBIOS.cpp
   libraries/Preferences/src/Preferences.cpp
@@ -184,6 +185,7 @@ set(includedirs
   libraries/HTTPClient/src
   libraries/HTTPUpdate/src
   libraries/LittleFS/src
+  libraries/Insights/src
   libraries/I2S/src
   libraries/NetBIOS/src
   libraries/Preferences/src

boards.txt

@@ -17110,7 +17110,7 @@ deneyapminiv2.menu.DebugLevel.verbose.build.code_debug=5
 deneyapminiv2.menu.EraseFlash.none=Disabled
 deneyapminiv2.menu.EraseFlash.none.upload.erase_cmd=
 deneyapminiv2.menu.EraseFlash.all=Enabled
-deneyapminiv2.menu.EraseFlash.all.upload.erase_cmd=-e
+deneyapminiv2.menu.EraseFlash.all.upload.erase_cmd=-
 
 ##############################################################
 

docs/source/api/insights.rst

+############
+ESP Insights
+############
+
+About
+-----
+
+ESP Insights is a remote diagnostics solution that allows users to remotely monitor the health of ESP devices in the field.
+
+Developers normally prefer debugging issues by physically probing them using gdb or observing the logs. This surely helps debug issues, but there are often cases wherein issues are seen only in specific environments under specific conditions. Even things like casings and placement of the product can affect the behaviour. A few examples are
+
+- Wi-Fi disconnections for a smart switch concealed in a wall.
+- Smart speakers crashing during some specific usage pattern.
+- Appliance frequently rebooting due to power supply issues.
+
+Additional information about ESP Insights can be found `here <https://insights.espressif.com/>`__.
+
+ESP Insights Agent API
+----------------------
+
+Insights.begin
+**************
+
+This initializes the ESP Insights agent.
+
+.. code-block:: arduino
+
+    bool begin(const char *auth_key, const char *node_id = NULL, uint32_t log_type = 0xFFFFFFFF, bool alloc_ext_ram = false);
+
+* ``auth_key`` Auth key generated using Insights dashboard
+* ``log_type`` Type of logs to be captured (value can be a mask of ESP_DIAG_LOG_TYPE_ERROR,  ESP_DIAG_LOG_TYPE_WARNING and ESP_DIAG_LOG_TYPE_EVENT)
+
+This function will return
+
+1. true : On success
+2. false in case of failure
+
+Insights.send
+*************
+
+Read insights data from buffers and send it to the cloud. Call to this function is asynchronous, it may take some time to send the data.
+
+.. code-block:: arduino
+
+    bool sendData()
+
+This function will return
+
+1. true : On success
+2. false in case of failure
+
+Insights.end
+************
+
+Deinitialize ESP Insights.
+
+.. code-block:: arduino
+
+    void end();
+
+Insights.disable
+****************
+
+Disable ESP Insights.
+
+.. code-block:: arduino
+
+    void disable();
+
+ESP Insights Metrics API
+------------------------
+
+`metrics` object of `Insights` class expose API's for using metrics.
+
+Insights.metrics.addX
+*********************
+
+Register a metric of type X, where X is one of: Bool, Int, Uint, Float, String, IPv4 or MAC
+
+.. code-block:: arduino
+
+    bool addX(const char *tag, const char *key, const char *label, const char *path);
+
+* ``tag`` :  Tag of metrics
+* ``key`` :  Unique key for the metrics
+* ``label`` : Label for the metrics
+* ``path`` : Hierarchical path for key, must be separated by '.' for more than one level
+
+This function will return
+
+1. true : On success
+2. false in case of failure
+
+Insights.metrics.remove
+***********************
+
+Unregister a diagnostics metrics
+
+.. code-block:: arduino
+
+    bool remove(const char *key);
+
+* ``key`` : Key for the metrics
+
+This function will return
+
+1. true : On success
+2. false in case of failure
+
+Insights.metrics.removeAll
+**************************
+
+Unregister all previously registered metrics
+
+.. code-block:: arduino
+
+    bool removeAll();
+
+This function will return
+
+1. true : On success
+2. false in case of failure
+
+Insights.metrics.setX
+*********************
+
+Add metrics of type X to storage, where X is one of: Bool, Int, Uint, Float, String, IPv4 or MAC
+
+.. code-block:: arduino
+
+    bool setX(const char *key, const void val);
+
+* ``key`` :       Key of metrics
+* ``val`` :       Value of metrics
+
+This function will return
+
+1. `ESP_OK` : On success
+2. Error in case of failure
+
+Insights.metrics.dumpHeap
+*************************
+
+Dumps the heap metrics and prints them to the console.
+This API collects and reports metrics value at any give point in time.
+
+.. code-block:: arduino
+
+    bool dumpHeap();
+
+This function will return
+
+1. true : On success
+2. false in case of failure
+
+Insights.metrics.dumpWiFi
+*************************
+
+Dumps the wifi metrics and prints them to the console.
+This API can be used to collect wifi metrics at any given point in time.
+
+.. code-block:: arduino
+
+    bool dumpWiFi();
+
+This function will return
+
+1. true : On success
+2. false in case of failure
+
+Insights.metrics.setHeapPeriod
+******************************
+
+Reset the periodic interval
+By default, heap metrics are collected every 30 seconds, this function can be used to change the interval.
+If the interval is set to 0, heap metrics collection disabled.
+
+.. code-block:: arduino
+
+    void setHeapPeriod(uint32_t period);
+
+* ``period`` : Period interval in seconds
+
+Insights.metrics.setWiFiPeriod
+******************************
+
+Reset the periodic interval
+By default, wifi metrics are collected every 30 seconds, this function can be used to change the interval.
+If the interval is set to 0, wifi metrics collection disabled.
+
+.. code-block:: arduino
+
+    void setHeapPeriod(uint32_t period);
+
+* ``period`` : Period interval in seconds
+
+ESP Insights Variables API
+--------------------------
+
+`variables` object of `Insights` class expose API's for using variables.
+
+Insights.variables.addX
+***********************
+
+Register a variable of type X, where X is one of: Bool, Int, Uint, Float, String, IPv4 or MAC
+
+.. code-block:: arduino
+
+    bool addX(const char *tag, const char *key, const char *label, const char *path);
+
+* ``tag`` :  Tag of variable
+* ``key`` :  Unique key for the variable
+* ``label`` : Label for the variable
+* ``path`` : Hierarchical path for key, must be separated by '.' for more than one level
+
+This function will return
+
+1. true : On success
+2. false in case of failure
+
+Insights.variables.remove
+*************************
+
+Unregister a diagnostics variable
+
+.. code-block:: arduino
+
+    bool remove(const char *key);
+
+* ``key`` : Key for the variable
+
+This function will return
+
+1. true : On success
+2. false in case of failure
+
+Insights.variables.removeAll
+****************************
+
+Unregister all previously registered variables
+
+.. code-block:: arduino
+
+    bool unregisterAll();
+
+This function will return
+
+1. true : On success
+2. false in case of failure
+
+Insights.variables.setX
+***********************
+
+Add variable of type X to storage, where X is one of: Bool, Int, Uint, Float, String, IPv4 or MAC
+
+.. code-block:: arduino
+
+    bool setX(const char *key, const void val);
+
+* ``key`` :       Key of metrics
+* ``val`` :       Value of metrics
+
+This function will return
+
+1. true : On success
+2. false in case of failure
+
+Example
+-------
+
+To get started with Insights, you can try:
+
+.. literalinclude:: ../../../libraries/Insights/examples/MinimalDiagnostics/MinimalDiagnostics.ino
+    :language: arduino

libraries/Insights/examples/DiagnosticsSmokeTest/DiagnosticsSmokeTest.ino

+#include "Insights.h"
+#include "WiFi.h"
+#include "inttypes.h"
+#include "esp_err.h"
+
+const char insights_auth_key[] = "<ENTER YOUR AUTH KEY>";
+
+#define WIFI_SSID       "<ENTER YOUR SSID>"
+#define WIFI_PASSPHRASE "<ENTER YOUR PASSWORD>"
+
+#define MAX_CRASHES     5
+#define MAX_PTRS        30
+#define TAG             "sketch"
+
+RTC_NOINIT_ATTR static uint32_t s_reset_count;
+static void *s_ptrs[MAX_PTRS];
+
+static void smoke_test()
+{
+    int dice;
+    int count = 0;
+    bool allocating = false;
+
+    while (1) {
+        dice = esp_random() % 500;
+        log_i("dice=%d", dice);
+        if (dice > 0 && dice < 150) {
+            log_e("[count][%d]", count);
+        } else if (dice > 150 && dice < 300) {
+            log_w("[count][%d]", count);
+        } else if (dice > 300 && dice < 470) {
+            Insights.event(TAG, "[count][%d]", count);
+        } else {
+            /* 30 in 500 probability to crash */
+            if (s_reset_count > MAX_CRASHES) {
+                Insights.event(TAG, "[count][%d]", count);
+            } else {
+               log_e("[count][%d] [crash_count][%" PRIu32 "] [excvaddr][0x0f] Crashing...", count, s_reset_count);
+               *(int *)0x0F = 0x10;
+           }
+        }
+
+        Insights.metrics.dumpHeap();
+        if (count % MAX_PTRS == 0) {
+            allocating = !allocating;
+            log_i("Allocating:%s\n", allocating ? "true" : "false");
+        }
+        if (allocating) {
+            uint32_t size = 1024 * (esp_random() % 8);
+            void *p = malloc(size);
+            if (p) {
+                memset(p, size, 'A' + (esp_random() % 26));
+                log_i("Allocated %" PRIu32 " bytes", size);
+            }
+            s_ptrs[count % MAX_PTRS] = p;
+        } else {
+            free(s_ptrs[count % MAX_PTRS]);
+            s_ptrs[count % MAX_PTRS] = NULL;
+            log_i("Freeing some memory...");
+        }
+
+        count++;
+        delay(1000);
+    }
+}
+
+void setup()
+{
+    Serial.begin(115200);
+    WiFi.mode(WIFI_STA);
+    WiFi.begin(WIFI_SSID, WIFI_PASSPHRASE);
+    while (WiFi.status() != WL_CONNECTED) {
+        delay(500);
+        Serial.print(".");
+    }
+    Serial.println("");
+    Serial.println("WiFi connected");
+    
+    if(!Insights.begin(insights_auth_key)){
+        return;
+    }
+    Serial.println("=========================================");
+    Serial.printf("ESP Insights enabled Node ID %s\n", Insights.nodeID());
+    Serial.println("=========================================");
+
+    if (esp_reset_reason() == ESP_RST_POWERON)  {
+        s_reset_count = 1;
+    } else {
+        s_reset_count++;
+    } 
+}
+
+void loop()
+{
+    smoke_test();
+    delay(100);
+}

libraries/Insights/examples/DiagnosticsSmokeTest/README.md

+# Smoke Test App
+
+## What to expect in this example?
+
+- This example is expected to exercise the various features of the ESP Insights framework
+- As a smoke test, this allows you to validate, by a quick perusal of the ESP Insights dashboard, the functioning of all the high-level features
+
+
+## End-to-End Tests
+
+### Lifecycle of the test (Hard reset resets the cycle)
+* Device boots up and logs errors/warnings/events in random order every 10 seconds
+* Every error/warning/event log with "diag_smoke" tag is associated with an incremental counter
+* There's a 30/500 probability that device will crash, this is done for verification of crash
+* Device will crash only five times and hard reset will reset the counter to 1
+* On sixth boot onwards device will not crash and logs errors/warnings/events and adds heap metrics
+
+### Facilitate the Auth Key
+In this example we will be using the auth key that we downloaded while [setting up ESP Insights account](https://github.com/espressif/esp-insights/tree/main/examples#set-up-esp-insights-account).
+
+Copy Auth Key to the example
+```
+const char insights_auth_key[] = "<ENTER YOUR AUTH KEY>";
+```
+
+### Enter WiFi Credentials
+Inside the example sketch, enter your WiFi credentials in `WIFI_SSID` and `WIFI_PASSPHRASE` macros.
+
+### Setup
+* Build and flash the sketch and monitor the console
+* Device will eventually crash after some time
+* Before every crash you will see below log print
+```
+E (75826) diag_smoke: [count][7] [crash_count][1] [excvaddr][0x0f] Crashing...
+// [count][7]: count associated with the log
+// [crash_count][1]: This is the first crash since device boot up, this number will increment as the crash count increases
+// [excvaddr][0x0f]: Exception vaddr, will see this in crash verification part below
+```
+* You'll see five crashes([crash_count][5]) and after that device will not crash and will keep on logging and adding metrics
+* Onwards this point keep device running for more than 30 minutes
+* Now we are all set to visit the [dashboard](https://dashboard.insights.espressif.com)
+* Select the node-id printed on the console, look for the below log. It is printed early when device boots up
+```
+ESP Insights enabled for Node ID ----- wx3vEoGgJPk7Rn5JvRUFs9
+```
\ No newline at end of file

libraries/Insights/examples/MinimalDiagnostics/MinimalDiagnostics.ino

+#include "Insights.h"
+#include "WiFi.h"
+
+const char insights_auth_key[] = "<ENTER YOUR AUTH KEY>";
+
+#define WIFI_SSID       "<ENTER YOUR SSID>"
+#define WIFI_PASSPHRASE "<ENTER YOUR PASSWORD>"
+
+void setup()
+{
+    Serial.begin(115200);
+    WiFi.mode(WIFI_STA);
+    WiFi.begin(WIFI_SSID, WIFI_PASSPHRASE);
+    while (WiFi.status() != WL_CONNECTED) {
+        delay(500);
+        Serial.print(".");
+    }
+    Serial.println("");
+    Serial.println("WiFi connected");
+    if(!Insights.begin(insights_auth_key)){
+        return;
+    }
+    Serial.println("=========================================");
+    Serial.printf("ESP Insights enabled Node ID %s\n", Insights.nodeID());
+    Serial.println("=========================================");
+}
+
+void loop()
+{
+    delay(1000);
+}

libraries/Insights/examples/MinimalDiagnostics/README.md

+# Minimal Diagnostics example
+- [What to expect in this example](#what-to-expect-in-this-example)
+- [Try out the example](#try-out-the-example)
+- [Insights Dashboard](#insights-dashboard)
+
+## What to expect in this example?
+- This example demonstrates the use of ESP Insights framework in minimal way
+- Device will try to connect with the configured WiFi network
+- ESP Insights is enabled in this example, so any error/warning logs, crashes will be reported to cloud
+- This example collects heap and wifi metrics every 10 minutes and network variables are collected when they change
+
+## Try out the example
+
+### Facilitate the Auth Key
+In this example we will be using the auth key that we downloaded while [setting up ESP Insights account](https://github.com/espressif/esp-insights/tree/main/examples#set-up-esp-insights-account).
+
+Copy Auth Key to the example
+```
+const char insights_auth_key[] = "<ENTER YOUR AUTH KEY>";
+```
+
+### Enter WiFi Credentials
+Inside the example sketch, enter your WiFi credentials in `WIFI_SSID` and `WIFI_PASSPHRASE` macros.
+
+### Get the Node ID
+- Start the Serial monitor
+
+- Once the device boots, it will connect to the Wi-Fi network, look for logs similar to below and make a note of Node ID.
+```
+I (4161) esp_insights: =========================================
+I (4171) esp_insights: Insights enabled for Node ID 246F2880371C
+I (4181) esp_insights: =========================================
+```
+
+
+## Insights Dashboard
+Once everything is set up, any diagnostics information reported will show up on the [Insights Dashboard](https://dashboard.insights.espressif.com). Sign in using the your credentials.
+
+
+### Monitor the device diagnostics
+Visit [Nodes](https://dashboard.insights.espressif.com/home/nodes) section on the dashboard and click on the Node ID to monitor device diagnostics information.
+
+> Note: Diagnostics data is reported dynamically or when the buffers are filled to configured threshold. So, it can take some time for the logs to reflect on the dashboard. Moreover, if a large number of logs are generated then data will be sent to cloud but, if it fails(eg reasons: Wi-Fi failure, No internet) then any newer logs will be dropped.

libraries/Insights/library.properties

+name=ESP Insights
+version=1.0.0
+author=Sanket Wadekar <sanket.wadekar@espressif.com>
+maintainer=Sanket Wadekar <sanket.wadekar@espressif.com>
+sentence=ESP Insights
+paragraph=With this library you can remotely monitor your device error logs, Network variables, WiFi/Heap Metrics, and also custom varibles / metrics.
+url=https://insights.espressif.com
+architectures=esp32

libraries/Insights/src/Insights.h

+/*
+ * SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+#pragma once
+#include "sdkconfig.h"
+#ifdef CONFIG_ESP_INSIGHTS_ENABLED
+#include "Arduino.h"
+
+#ifdef __cplusplus
+
+class ESPInsightsMetricsClass
+{
+    private:
+        bool initialized;
+    
+    public:
+        ESPInsightsMetricsClass():initialized(false){}
+    
+        bool addBool(const char *tag, const char *key, const char *label, const char *path);
+        bool addInt(const char *tag, const char *key, const char *label, const char *path);
+        bool addUint(const char *tag, const char *key, const char *label, const char *path);
+        bool addFloat(const char *tag, const char *key, const char *label, const char *path);
+        bool addString(const char *tag, const char *key, const char *label, const char *path);
+        bool addIPv4(const char *tag, const char *key, const char *label, const char *path);
+        bool addMAC(const char *tag, const char *key, const char *label, const char *path);
+
+        bool setBool(const char *key, bool b);
+        bool setInt(const char *key, int32_t i);
+        bool setUint(const char *key, uint32_t u);
+        bool setFloat(const char *key, float f);
+        bool setString(const char *key, const char *str);
+        bool setIPv4(const char *key, uint32_t ip);
+        bool setMAC(const char *key, uint8_t *mac);
+
+        bool remove(const char *key);
+        bool removeAll();
+
+        void setHeapPeriod(uint32_t seconds);
+        void setWiFiPeriod(uint32_t seconds);
+
+        bool dumpHeap();
+        bool dumpWiFi();
+
+        //internal use
+        void setInitialized(bool init){ initialized = init; }
+};
+
+class ESPInsightsVariablesClass
+{
+    private:
+        bool initialized;
+    
+    public:
+        ESPInsightsVariablesClass():initialized(false){}
+    
+        bool addBool(const char *tag, const char *key, const char *label, const char *path);
+        bool addInt(const char *tag, const char *key, const char *label, const char *path);
+        bool addUint(const char *tag, const char *key, const char *label, const char *path);
+        bool addFloat(const char *tag, const char *key, const char *label, const char *path);
+        bool addString(const char *tag, const char *key, const char *label, const char *path);
+        bool addIPv4(const char *tag, const char *key, const char *label, const char *path);
+        bool addMAC(const char *tag, const char *key, const char *label, const char *path);
+
+        bool setBool(const char *key, bool b);
+        bool setInt(const char *key, int32_t i);
+        bool setUint(const char *key, uint32_t u);
+        bool setFloat(const char *key, float f);
+        bool setString(const char *key, const char *str);
+        bool setIPv4(const char *key, uint32_t ip);
+        bool setMAC(const char *key, uint8_t *mac);
+
+        bool remove(const char *key);
+        bool removeAll();
+
+        //internal use
+        void setInitialized(bool init){ initialized = init; }
+};
+
+class ESPInsightsClass
+{
+    private:
+        bool initialized;
+    
+    public:
+        ESPInsightsMetricsClass metrics;
+        ESPInsightsVariablesClass variables;
+
+        ESPInsightsClass();
+        ~ESPInsightsClass();
+        
+        bool begin(const char *auth_key, const char *node_id = NULL, uint32_t log_type = 0xFFFFFFFF, bool alloc_ext_ram = false);
+        void end();
+        bool send();
+        const char * nodeID();
+        
+        void dumpTasksStatus();
+        
+        bool event(const char *tag, const char *format, ...);
+};
+
+extern ESPInsightsClass Insights;
+
+extern "C"
+{
+#endif
+
+#include "esp_err.h"
+#include "esp_log.h"
+esp_err_t esp_diag_log_event(const char *tag, const char *format, ...) __attribute__ ((format (printf, 2, 3)));
+#define insightsEvent(tag, format, ...) {esp_diag_log_event(tag, "EV (%" PRIu32 ") %s: " format, esp_log_timestamp(), tag, ##__VA_ARGS__);}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

platform.txt

@@ -24,6 +24,9 @@ tools.esp_ota.cmd.windows="{runtime.platform.path}/tools/espota.exe" -r
 tools.gen_esp32part.cmd=python3 "{runtime.platform.path}/tools/gen_esp32part.py"
 tools.gen_esp32part.cmd.windows="{runtime.platform.path}/tools/gen_esp32part.exe"
 
+tools.gen_insights_pkg.cmd=python3 "{runtime.platform.path}"/tools/gen_insights_package.py
+tools.gen_insights_pkg.cmd.windows="{runtime.platform.path}/tools/gen_insights_package.exe"
+
 compiler.path={runtime.tools.{build.tarch}-{build.target}-elf-gcc.path}/bin/
 compiler.sdk.path={runtime.platform.path}/tools/sdk/{build.mcu}
 compiler.prefix={build.tarch}-{build.target}-elf-
@@ -211,10 +214,15 @@ recipe.c.combine.pattern="{compiler.path}{compiler.c.elf.cmd}" "-Wl,--Map={build
 recipe.objcopy.partitions.bin.pattern={tools.gen_esp32part.cmd} -q "{build.path}/partitions.csv" "{build.path}/{build.project_name}.partitions.bin"
 
 ## Create bin
-recipe.objcopy.bin.pattern_args=--chip {build.mcu} elf2image --flash_mode "{build.flash_mode}" --flash_freq "{build.flash_freq}" --flash_size "{build.flash_size}" -o "{build.path}/{build.project_name}.bin" "{build.path}/{build.project_name}.elf"
+recipe.objcopy.bin.pattern_args=--chip {build.mcu} elf2image --flash_mode "{build.flash_mode}" --flash_freq "{build.flash_freq}" --flash_size "{build.flash_size}" --elf-sha256-offset 0xb0 -o "{build.path}/{build.project_name}.bin" "{build.path}/{build.project_name}.elf"
 recipe.objcopy.bin.pattern="{tools.esptool_py.path}/{tools.esptool_py.cmd}" {recipe.objcopy.bin.pattern_args}
 recipe.objcopy.bin.pattern.linux=python3 "{tools.esptool_py.path}/{tools.esptool_py.cmd}" {recipe.objcopy.bin.pattern_args}
 
+## Create Insights Firmware Package
+recipe.hooks.objcopy.postobjcopy.1.pattern_args={build.path} {build.project_name} "{build.source.path}"
+recipe.hooks.objcopy.postobjcopy.1.pattern=bash -c "[ ! -d "{build.path}"/libraries/Insights ] || {tools.gen_insights_pkg.cmd} {recipe.hooks.objcopy.postobjcopy.1.pattern_args}"
+recipe.hooks.objcopy.postobjcopy.1.pattern.windows=cmd /c if exist "{build.path}\libraries\Insights" {tools.gen_insights_pkg.cmd} {recipe.hooks.objcopy.postobjcopy.1.pattern_args}
+
 ## Save bin
 recipe.output.tmp_file={build.project_name}.bin
 recipe.output.save_file={build.project_name}.{build.variant}.bin

tools/gen_insights_package.py

+import os
+import sys
+import shutil
+import json
+
+APP_HEADER_SIZE = 32
+VERSION_NAME_OFFSET = APP_HEADER_SIZE + 16
+VERSION_NAME_SIZE = 32
+PROJECT_NAME_OFFSET = VERSION_NAME_OFFSET + VERSION_NAME_SIZE
+PROJECT_NAME_SIZE = 32
+
+# Input path of temporary build directory created by Arduino
+BUILD_DIR=sys.argv[1]
+# Input project name
+PROJ_NAME=sys.argv[2]
+# Input path to create output package
+TARGET_PATH=sys.argv[3]
+
+def main():
+    print("Creating ESP Insights Firmware Package.")
+    archive_path = os.path.join(BUILD_DIR, PROJ_NAME)
+    out_path = os.path.join(TARGET_PATH, PROJ_NAME)
+    
+    # Create target archive directories
+    os.makedirs(archive_path, exist_ok = True)
+    os.makedirs(os.path.join(archive_path, "partition_table"), exist_ok = True)
+    os.makedirs(os.path.join(archive_path, "bootloader"), exist_ok = True)
+    
+    # Copy files from build directory to archive directory
+    shutil.copy2(os.path.join(BUILD_DIR, PROJ_NAME + ".bin"), archive_path)
+    shutil.copy2(os.path.join(BUILD_DIR, PROJ_NAME + ".elf"), archive_path)
+    shutil.copy2(os.path.join(BUILD_DIR, PROJ_NAME + ".map"), archive_path)
+    shutil.copy2(os.path.join(BUILD_DIR, "partitions.csv"), archive_path)
+    shutil.copy2(os.path.join(BUILD_DIR, PROJ_NAME + ".bootloader.bin"), os.path.join(archive_path, "bootloader"))
+    shutil.copy2(os.path.join(BUILD_DIR, PROJ_NAME + ".partitions.bin"), os.path.join(archive_path, "partition_table"))
+    
+    with open(os.path.join(BUILD_DIR, PROJ_NAME + ".bin"), 'rb') as bin_file:
+        bin_file.seek(VERSION_NAME_OFFSET)
+        version_name = (bin_file.read(VERSION_NAME_SIZE).decode('utf-8')).split('\x00', 1)[0]
+        bin_file.seek(PROJECT_NAME_OFFSET)
+        project_name = (bin_file.read(PROJECT_NAME_SIZE).decode('utf-8')).split('\x00', 1)[0]
+        project_build_config_obj = {
+            "project" : {
+                "name" : project_name,
+                "version": version_name
+            }
+        }
+        with open(os.path.join(archive_path, "project_build_config.json"), "w") as json_file:
+            json_file.write(json.dumps(project_build_config_obj))
+    
+    shutil.make_archive(out_path, "zip", BUILD_DIR, PROJ_NAME)
+    print("Archive created at {}".format(out_path + ".zip"))
+    return
+
+if __name__ == '__main__':
+    main()
\ No newline at end of file

tools/partitions/app3M_fat9M_16MB.csv

@@ -3,5 +3,6 @@ nvs,      data, nvs,     0x9000,  0x5000,
 otadata,  data, ota,     0xe000,  0x2000,
 app0,     app,  ota_0,   0x10000, 0x300000,
 app1,     app,  ota_1,   0x310000,0x300000,
-ffat,     data, fat,     0x610000,0x9F0000,
-# to create/use ffat, see https://github.com/marcmerlin/esp32_fatfsimage
+ffat,     data, fat,     0x610000,0x9E0000,
+coredump, data, coredump,0xFF0000,0x10000,
+# to create/use ffat, see https://github.com/marcmerlin/esp32_fatfsimage
\ No newline at end of file

tools/partitions/bare_minimum_2MB.csv

 # Name,   Type, SubType, Offset,  Size, Flags
 nvs,      data, nvs,     36K,     20K,
-factory,  app,  factory, 64K,     1900K,
+factory,  app,  factory, 64K,     1900K,
\ No newline at end of file

tools/partitions/default.csv

@@ -3,4 +3,5 @@ nvs,      data, nvs,     0x9000,  0x5000,
 otadata,  data, ota,     0xe000,  0x2000,
 app0,     app,  ota_0,   0x10000, 0x140000,
 app1,     app,  ota_1,   0x150000,0x140000,
-spiffs,   data, spiffs,  0x290000,0x170000,
+spiffs,   data, spiffs,  0x290000,0x160000,
+coredump, data, coredump,0x3F0000,0x10000,
\ No newline at end of file

tools/partitions/default_16MB.csv

@@ -3,4 +3,5 @@ nvs,      data, nvs,     0x9000,  0x5000,
 otadata,  data, ota,     0xe000,  0x2000,
 app0,     app,  ota_0,   0x10000, 0x640000,
 app1,     app,  ota_1,   0x650000,0x640000,
-spiffs,   data, spiffs,  0xc90000,0x370000,
+spiffs,   data, spiffs,  0xc90000,0x360000,
+coredump, data, coredump,0xFF0000,0x10000,
\ No newline at end of file

tools/partitions/default_8MB.csv

@@ -3,4 +3,5 @@ nvs,      data, nvs,     0x9000,  0x5000,
 otadata,  data, ota,     0xe000,  0x2000,
 app0,     app,  ota_0,   0x10000, 0x330000,
 app1,     app,  ota_1,   0x340000,0x330000,
-spiffs,   data, spiffs,  0x670000,0x190000,
+spiffs,   data, spiffs,  0x670000,0x180000,
+coredump, data, coredump,0x7F0000,0x10000,
\ No newline at end of file

tools/partitions/default_ffat.csv

@@ -3,4 +3,5 @@ nvs,      data, nvs,     0x9000,  0x5000,
 otadata,  data, ota,     0xe000,  0x2000,
 app0,     app,  ota_0,   0x10000, 0x140000,
 app1,     app,  ota_1,   0x150000,0x140000,
-ffat,     data, fat,     0x290000,0x170000,
+ffat,     data, fat,     0x290000,0x160000,
+coredump, data, coredump,0x3F0000,0x10000,
\ No newline at end of file

tools/partitions/ffat.csv

@@ -3,5 +3,6 @@ nvs,      data, nvs,     0x9000,  0x5000,
 otadata,  data, ota,     0xe000,  0x2000,
 app0,     app,  ota_0,   0x10000, 0x200000,
 app1,     app,  ota_1,   0x210000,0x200000,
-ffat,     data, fat,     0x410000,0xBF0000,
+ffat,     data, fat,     0x410000,0xBE0000,
+coredump, data, coredump,0xFF0000,0x10000,
 # to create/use ffat, see https://github.com/marcmerlin/esp32_fatfsimage

tools/partitions/huge_app.csv

@@ -2,4 +2,5 @@
 nvs,      data, nvs,     0x9000,  0x5000,
 otadata,  data, ota,     0xe000,  0x2000,
 app0,     app,  ota_0,   0x10000, 0x300000,
-spiffs,   data, spiffs,  0x310000,0xF0000,
+spiffs,   data, spiffs,  0x310000,0xE0000,
+coredump, data, coredump,0x3F0000,0x10000,
\ No newline at end of file

tools/partitions/large_spiffs_16MB.csv

@@ -3,4 +3,5 @@ nvs,      data, nvs,     0x9000,  0x5000,
 otadata,  data, ota,     0xe000,  0x2000,
 app0,     app,  ota_0,   0x10000, 0x480000,
 app1,     app,  ota_1,   0x490000,0x480000,
-spiffs,   data, spiffs,  0x910000,0x6F0000,
+spiffs,   data, spiffs,  0x910000,0x6E0000,
+coredump, data, coredump,0xFF0000,0x10000,
\ No newline at end of file

tools/partitions/max_app_8MB.csv

 # Name,   Type, SubType, Offset,  Size, Flags
 nvs,      data, nvs,     0x9000,  0x5000,
 otadata,  data, ota,     0xe000,  0x2000,
-app0,     app,  factory,   0x10000, 0x7F0000,
+app0,     app,  factory, 0x10000, 0x7E0000,
+coredump, data, coredump,0x7F0000,0x10000,
\ No newline at end of file

tools/partitions/min_spiffs.csv

@@ -3,4 +3,5 @@ nvs,      data, nvs,     0x9000,  0x5000,
 otadata,  data, ota,     0xe000,  0x2000,
 app0,     app,  ota_0,   0x10000, 0x1E0000,
 app1,     app,  ota_1,   0x1F0000,0x1E0000,
-spiffs,   data, spiffs,  0x3D0000,0x30000,
+spiffs,   data, spiffs,  0x3D0000,0x20000,
+coredump, data, coredump,0x3F0000,0x10000,
\ No newline at end of file

tools/partitions/minimal.csv

@@ -2,4 +2,5 @@
 nvs,      data, nvs,     0x9000,   0x5000,
 otadata,  data, ota,     0xe000,   0x2000,
 app0,     app,  ota_0,   0x10000,  0x140000,
-spiffs,   data, spiffs,  0x150000, 0xB0000,
+spiffs,   data, spiffs,  0x150000, 0xA0000,
+coredump, data, coredump,0x1F0000, 0x10000,
\ No newline at end of file

tools/partitions/no_ota.csv

@@ -2,4 +2,5 @@
 nvs,      data, nvs,     0x9000,  0x5000,
 otadata,  data, ota,     0xe000,  0x2000,
 app0,     app,  ota_0,   0x10000, 0x200000,
-spiffs,   data, spiffs,  0x210000,0x1F0000,
+spiffs,   data, spiffs,  0x210000,0x1E0000,
+coredump, data, coredump,0x3F0000,0x10000,
\ No newline at end of file

tools/partitions/noota_3g.csv

@@ -2,4 +2,5 @@
 nvs,      data, nvs,     0x9000,  0x5000,
 otadata,  data, ota,     0xe000,  0x2000,
 app0,     app,  ota_0,   0x10000, 0x100000,
-spiffs,   data, spiffs,  0x110000,0x2F0000,
+spiffs,   data, spiffs,  0x110000,0x2E0000,
+coredump, data, coredump,0x3F0000,0x10000,
\ No newline at end of file

tools/partitions/noota_3gffat.csv

@@ -2,5 +2,6 @@
 nvs,      data, nvs,     0x9000,  0x5000,
 otadata,  data, ota,     0xe000,  0x2000,
 app0,     app,  ota_0,   0x10000, 0x100000,
-ffat,     data, fat,     0x110000,0x2F0000,
+ffat,     data, fat,     0x110000,0x2E0000,
+coredump, data, coredump,0x3F0000,0x10000,
 # to create/use ffat, see https://github.com/marcmerlin/esp32_fatfsimage

tools/partitions/noota_ffat.csv

@@ -2,5 +2,6 @@
 nvs,      data, nvs,     0x9000,  0x5000,
 otadata,  data, ota,     0xe000,  0x2000,
 app0,     app,  ota_0,   0x10000, 0x200000,
-ffat,     data, fat,     0x210000,0x1F0000,
+ffat,     data, fat,     0x210000,0x1E0000,
+coredump, data, coredump,0x3F0000,0x10000,
 # to create/use ffat, see https://github.com/marcmerlin/esp32_fatfsimage

tools/partitions/rainmaker.csv

@@ -4,3 +4,4 @@ otadata,   data,  ota,     0xe000,   0x2000,
 ota_0,     app,   ota_0,   0x10000,  0x1E0000,
 ota_1,     app,   ota_1,   0x1F0000, 0x1E0000,
 fctry,     data,  nvs,     0x3D0000, 0x6000,
+coredump, data, coredump,  0x3F0000, 0x10000,
\ No newline at end of file

tools/platformio-build.py

@@ -25,7 +25,7 @@ http://arduino.cc/en/Reference/HomePage
 # Extends: https://github.com/platformio/platform-espressif32/blob/develop/builder/main.py
 
 from os.path import abspath, basename, isdir, isfile, join
-
+from copy import deepcopy
 from SCons.Script import DefaultEnvironment, SConscript
 
 env = DefaultEnvironment()
@@ -237,3 +237,13 @@ partition_table = env.Command(
     ),
 )
 env.Depends("$BUILD_DIR/$PROGNAME$PROGSUFFIX", partition_table)
+
+#
+#  Adjust the `esptoolpy` command in the `ElfToBin` builder with firmware checksum offset
+#
+
+action = deepcopy(env["BUILDERS"]["ElfToBin"].action)
+action.cmd_list = env["BUILDERS"]["ElfToBin"].action.cmd_list.replace(
+    "-o", "--elf-sha256-offset 0xb0 -o"
+)
+env["BUILDERS"]["ElfToBin"].action = action

tools/sdk/esp32/include/esp_diagnostics/include/esp_diagnostics.h

@@ -238,7 +238,7 @@ esp_err_t esp_diag_log_event(const char *tag, const char *format, ...) __attribu
  */
 #define ESP_DIAG_EVENT(tag, format, ...) \
 { \
-    esp_diag_log_event(tag, "EV (%"PRIu32") %s: " format, esp_log_timestamp(), tag, ##__VA_ARGS__); \
+    esp_diag_log_event(tag, "EV (%" PRIu32 ") %s: " format, esp_log_timestamp(), tag, ##__VA_ARGS__); \
     ESP_LOGI(tag, format, ##__VA_ARGS__); \
 }
 

tools/sdk/esp32c3/include/esp_diagnostics/include/esp_diagnostics.h

@@ -238,7 +238,7 @@ esp_err_t esp_diag_log_event(const char *tag, const char *format, ...) __attribu
  */
 #define ESP_DIAG_EVENT(tag, format, ...) \
 { \
-    esp_diag_log_event(tag, "EV (%"PRIu32") %s: " format, esp_log_timestamp(), tag, ##__VA_ARGS__); \
+    esp_diag_log_event(tag, "EV (%" PRIu32 ") %s: " format, esp_log_timestamp(), tag, ##__VA_ARGS__); \
     ESP_LOGI(tag, format, ##__VA_ARGS__); \
 }
 

tools/sdk/esp32s2/include/esp_diagnostics/include/esp_diagnostics.h

@@ -238,7 +238,7 @@ esp_err_t esp_diag_log_event(const char *tag, const char *format, ...) __attribu
  */
 #define ESP_DIAG_EVENT(tag, format, ...) \
 { \
-    esp_diag_log_event(tag, "EV (%"PRIu32") %s: " format, esp_log_timestamp(), tag, ##__VA_ARGS__); \
+    esp_diag_log_event(tag, "EV (%" PRIu32 ") %s: " format, esp_log_timestamp(), tag, ##__VA_ARGS__); \
     ESP_LOGI(tag, format, ##__VA_ARGS__); \
 }
 

tools/sdk/esp32s3/include/esp_diagnostics/include/esp_diagnostics.h

@@ -238,7 +238,7 @@ esp_err_t esp_diag_log_event(const char *tag, const char *format, ...) __attribu
  */
 #define ESP_DIAG_EVENT(tag, format, ...) \
 { \
-    esp_diag_log_event(tag, "EV (%"PRIu32") %s: " format, esp_log_timestamp(), tag, ##__VA_ARGS__); \
+    esp_diag_log_event(tag, "EV (%" PRIu32 ") %s: " format, esp_log_timestamp(), tag, ##__VA_ARGS__); \
     ESP_LOGI(tag, format, ##__VA_ARGS__); \
 }