Feature/http update (#1979)

* Added HTTPUpdate class for downloading sketches from a server

* Added HTTPUpdate class for downloading sketches from a server

* Added HTTPUpdate to CMakeLists.txt

* Change ESP8266 class references to ESP32 for httpUpdate.ino example

* Change ESP8266 class references to ESP32 for httpUpdate.ino example. setLedPin() commented out because not all boards support LED_BUITLIN

* Added check to handle mixup of old and present api properly

* Correct HTTPClient::setTimeout() to convert milliseconds to seconds. Correct WiFiClient::setTimeout() to call Stream::setTimeout() with seconds converted back to milliseconds. Remove inproper checks for _insecure.

* Added small comment because it looked like the Travis build did not finish

CMakeLists.txt

@@ -46,6 +46,7 @@ set(LIBRARY_SRCS
   libraries/FS/src/FS.cpp
   libraries/FS/src/vfs_api.cpp
   libraries/HTTPClient/src/HTTPClient.cpp
+  libraries/HTTPUpdate/src/HTTPUpdate.cpp
   libraries/NetBIOS/src/NetBIOS.cpp
   libraries/Preferences/src/Preferences.cpp
   libraries/SD_MMC/src/SD_MMC.cpp
@@ -181,6 +182,7 @@ set(COMPONENT_ADD_INCLUDEDIRS
   libraries/FFat/src
   libraries/FS/src
   libraries/HTTPClient/src
+  libraries/HTTPUpdate/src
   libraries/NetBIOS/src
   libraries/Preferences/src
   libraries/SD_MMC/src

libraries/HTTPUpdate/examples/httpUpdate/httpUpdate.ino

+/**
+   httpUpdate.ino
+
+    Created on: 27.11.2015
+
+*/
+
+#include <Arduino.h>
+
+#include <WiFi.h>
+#include <WiFiMulti.h>
+
+#include <HTTPClient.h>
+#include <HTTPUpdate.h>
+
+WiFiMulti WiFiMulti;
+
+void setup() {
+
+  Serial.begin(115200);
+  // Serial.setDebugOutput(true);
+
+  Serial.println();
+  Serial.println();
+  Serial.println();
+
+  for (uint8_t t = 4; t > 0; t--) {
+    Serial.printf("[SETUP] WAIT %d...\n", t);
+    Serial.flush();
+    delay(1000);
+  }
+
+  WiFi.mode(WIFI_STA);
+  WiFiMulti.addAP("SSID", "PASSWORD");
+
+
+}
+
+void loop() {
+  // wait for WiFi connection
+  if ((WiFiMulti.run() == WL_CONNECTED)) {
+
+    WiFiClient client;
+
+    // The line below is optional. It can be used to blink the LED on the board during flashing
+    // The LED will be on during download of one buffer of data from the network. The LED will
+    // be off during writing that buffer to flash
+    // On a good connection the LED should flash regularly. On a bad connection the LED will be
+    // on much longer than it will be off. Other pins than LED_BUILTIN may be used. The second
+    // value is used to put the LED on. If the LED is on with HIGH, that value should be passed
+    // httpUpdate.setLedPin(LED_BUILTIN, LOW);
+
+    t_httpUpdate_return ret = httpUpdate.update(client, "http://server/file.bin");
+    // Or:
+    //t_httpUpdate_return ret = httpUpdate.update(client, "server", 80, "file.bin");
+
+    switch (ret) {
+      case HTTP_UPDATE_FAILED:
+        Serial.printf("HTTP_UPDATE_FAILD Error (%d): %s\n", httpUpdate.getLastError(), httpUpdate.getLastErrorString().c_str());
+        break;
+
+      case HTTP_UPDATE_NO_UPDATES:
+        Serial.println("HTTP_UPDATE_NO_UPDATES");
+        break;
+
+      case HTTP_UPDATE_OK:
+        Serial.println("HTTP_UPDATE_OK");
+        break;
+    }
+  }
+}
+

libraries/HTTPUpdate/examples/httpUpdateSPIFFS/httpUpdateSPIFFS.ino

+/**
+   httpUpdateSPIFFS.ino
+
+    Created on: 05.12.2015
+
+*/
+
+#include <Arduino.h>
+
+#include <WiFi.h>
+#include <WiFiMulti.h>
+
+#include <HTTPClient.h>
+#include <HTTPUpdate.h>
+
+WiFiMulti WiFiMulti;
+
+void setup() {
+
+  Serial.begin(115200);
+  // Serial.setDebugOutput(true);
+
+  Serial.println();
+  Serial.println();
+  Serial.println();
+
+  for (uint8_t t = 4; t > 0; t--) {
+    Serial.printf("[SETUP] WAIT %d...\n", t);
+    Serial.flush();
+    delay(1000);
+  }
+
+  WiFi.mode(WIFI_STA);
+  WiFiMulti.addAP("SSID", "PASSWORD");
+
+}
+
+void loop() {
+  // wait for WiFi connection
+  if ((WiFiMulti.run() == WL_CONNECTED)) {
+
+    Serial.println("Update SPIFFS...");
+
+    WiFiClient client;
+
+    // The line below is optional. It can be used to blink the LED on the board during flashing
+    // The LED will be on during download of one buffer of data from the network. The LED will
+    // be off during writing that buffer to flash
+    // On a good connection the LED should flash regularly. On a bad connection the LED will be
+    // on much longer than it will be off. Other pins than LED_BUILTIN may be used. The second
+    // value is used to put the LED on. If the LED is on with HIGH, that value should be passed
+    // httpUpdate.setLedPin(LED_BUILTIN, LOW);
+
+    t_httpUpdate_return ret = httpUpdate.updateSpiffs(client, "http://server/spiffs.bin");
+    if (ret == HTTP_UPDATE_OK) {
+      Serial.println("Update sketch...");
+      ret = httpUpdate.update(client, "http://server/file.bin");
+
+      switch (ret) {
+        case HTTP_UPDATE_FAILED:
+          Serial.printf("HTTP_UPDATE_FAILED Error (%d): %s", httpUpdate.getLastError(), httpUpdate.getLastErrorString().c_str());
+          break;
+
+        case HTTP_UPDATE_NO_UPDATES:
+          Serial.println("HTTP_UPDATE_NO_UPDATES");
+          break;
+
+        case HTTP_UPDATE_OK:
+          Serial.println("HTTP_UPDATE_OK");
+          break;
+      }
+    }
+  }
+}
+

libraries/HTTPUpdate/examples/httpUpdateSecure/httpUpdateSecure.ino

+/**
+   httpUpdateSecure.ino
+
+    Created on: 16.10.2018 as an adaptation of the ESP8266 version of httpUpdate.ino
+
+*/
+
+#include <WiFi.h>
+#include <WiFiMulti.h>
+
+#include <HTTPClient.h>
+#include <HTTPUpdate.h>
+
+#include <time.h>
+
+WiFiMulti WiFiMulti;
+
+// Set time via NTP, as required for x.509 validation
+void setClock() {
+  configTime(0, 0, "pool.ntp.org", "time.nist.gov");  // UTC
+
+  Serial.print(F("Waiting for NTP time sync: "));
+  time_t now = time(nullptr);
+  while (now < 8 * 3600 * 2) {
+    yield();
+    delay(500);
+    Serial.print(F("."));
+    now = time(nullptr);
+  }
+
+  Serial.println(F(""));
+  struct tm timeinfo;
+  gmtime_r(&now, &timeinfo);
+  Serial.print(F("Current time: "));
+  Serial.print(asctime(&timeinfo));
+}
+
+/**
+ * This is lets-encrypt-x3-cross-signed.pem
+ */
+const char* rootCACertificate = \
+"-----BEGIN CERTIFICATE-----\n" \
+"MIIEkjCCA3qgAwIBAgIQCgFBQgAAAVOFc2oLheynCDANBgkqhkiG9w0BAQsFADA/\n" \
+"MSQwIgYDVQQKExtEaWdpdGFsIFNpZ25hdHVyZSBUcnVzdCBDby4xFzAVBgNVBAMT\n" \
+"DkRTVCBSb290IENBIFgzMB4XDTE2MDMxNzE2NDA0NloXDTIxMDMxNzE2NDA0Nlow\n" \
+"SjELMAkGA1UEBhMCVVMxFjAUBgNVBAoTDUxldCdzIEVuY3J5cHQxIzAhBgNVBAMT\n" \
+"GkxldCdzIEVuY3J5cHQgQXV0aG9yaXR5IFgzMIIBIjANBgkqhkiG9w0BAQEFAAOC\n" \
+"AQ8AMIIBCgKCAQEAnNMM8FrlLke3cl03g7NoYzDq1zUmGSXhvb418XCSL7e4S0EF\n" \
+"q6meNQhY7LEqxGiHC6PjdeTm86dicbp5gWAf15Gan/PQeGdxyGkOlZHP/uaZ6WA8\n" \
+"SMx+yk13EiSdRxta67nsHjcAHJyse6cF6s5K671B5TaYucv9bTyWaN8jKkKQDIZ0\n" \
+"Z8h/pZq4UmEUEz9l6YKHy9v6Dlb2honzhT+Xhq+w3Brvaw2VFn3EK6BlspkENnWA\n" \
+"a6xK8xuQSXgvopZPKiAlKQTGdMDQMc2PMTiVFrqoM7hD8bEfwzB/onkxEz0tNvjj\n" \
+"/PIzark5McWvxI0NHWQWM6r6hCm21AvA2H3DkwIDAQABo4IBfTCCAXkwEgYDVR0T\n" \
+"AQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwfwYIKwYBBQUHAQEEczBxMDIG\n" \
+"CCsGAQUFBzABhiZodHRwOi8vaXNyZy50cnVzdGlkLm9jc3AuaWRlbnRydXN0LmNv\n" \
+"bTA7BggrBgEFBQcwAoYvaHR0cDovL2FwcHMuaWRlbnRydXN0LmNvbS9yb290cy9k\n" \
+"c3Ryb290Y2F4My5wN2MwHwYDVR0jBBgwFoAUxKexpHsscfrb4UuQdf/EFWCFiRAw\n" \
+"VAYDVR0gBE0wSzAIBgZngQwBAgEwPwYLKwYBBAGC3xMBAQEwMDAuBggrBgEFBQcC\n" \
+"ARYiaHR0cDovL2Nwcy5yb290LXgxLmxldHNlbmNyeXB0Lm9yZzA8BgNVHR8ENTAz\n" \
+"MDGgL6AthitodHRwOi8vY3JsLmlkZW50cnVzdC5jb20vRFNUUk9PVENBWDNDUkwu\n" \
+"Y3JsMB0GA1UdDgQWBBSoSmpjBH3duubRObemRWXv86jsoTANBgkqhkiG9w0BAQsF\n" \
+"AAOCAQEA3TPXEfNjWDjdGBX7CVW+dla5cEilaUcne8IkCJLxWh9KEik3JHRRHGJo\n" \
+"uM2VcGfl96S8TihRzZvoroed6ti6WqEBmtzw3Wodatg+VyOeph4EYpr/1wXKtx8/\n" \
+"wApIvJSwtmVi4MFU5aMqrSDE6ea73Mj2tcMyo5jMd6jmeWUHK8so/joWUoHOUgwu\n" \
+"X4Po1QYz+3dszkDqMp4fklxBwXRsW10KXzPMTZ+sOPAveyxindmjkW8lGy+QsRlG\n" \
+"PfZ+G6Z6h7mjem0Y+iWlkYcV4PIWL1iwBi8saCbGS5jN2p8M+X+Q7UNKEkROb3N6\n" \
+"KOqkqm57TH2H3eDJAkSnh6/DNFu0Qg==\n" \
+"-----END CERTIFICATE-----\n";
+
+void setup() {
+
+  Serial.begin(115200);
+  // Serial.setDebugOutput(true);
+
+  Serial.println();
+  Serial.println();
+  Serial.println();
+
+  for (uint8_t t = 4; t > 0; t--) {
+    Serial.printf("[SETUP] WAIT %d...\n", t);
+    Serial.flush();
+    delay(1000);
+  }
+
+  WiFi.mode(WIFI_STA);
+  WiFiMulti.addAP("SSID", "PASSWORD");
+}
+
+void loop() {
+  // wait for WiFi connection
+  if ((WiFiMulti.run() == WL_CONNECTED)) {
+
+    setClock();
+
+    WiFiClientSecure client;
+    client.setCACert(rootCACertificate);
+
+    // Reading data over SSL may be slow, use an adequate timeout
+    client.setTimeout(12000);
+
+    // The line below is optional. It can be used to blink the LED on the board during flashing
+    // The LED will be on during download of one buffer of data from the network. The LED will
+    // be off during writing that buffer to flash
+    // On a good connection the LED should flash regularly. On a bad connection the LED will be
+    // on much longer than it will be off. Other pins than LED_BUILTIN may be used. The second
+    // value is used to put the LED on. If the LED is on with HIGH, that value should be passed
+    // httpUpdate.setLedPin(LED_BUILTIN, HIGH);
+
+    t_httpUpdate_return ret = httpUpdate.update(client, "https://server/file.bin");
+    // Or:
+    //t_httpUpdate_return ret = httpUpdate.update(client, "server", 443, "file.bin");
+
+
+    switch (ret) {
+      case HTTP_UPDATE_FAILED:
+        Serial.printf("HTTP_UPDATE_FAILED Error (%d): %s\n", httpUpdate.getLastError(), httpUpdate.getLastErrorString().c_str());
+        break;
+
+      case HTTP_UPDATE_NO_UPDATES:
+        Serial.println("HTTP_UPDATE_NO_UPDATES");
+        break;
+
+      case HTTP_UPDATE_OK:
+        Serial.println("HTTP_UPDATE_OK");
+        break;
+    }
+  }
+}

libraries/HTTPUpdate/keywords.txt

+#######################################
+# Syntax Coloring Map For ESP8266httpUpdate
+#######################################
+
+#######################################
+# Library (KEYWORD3)
+#######################################
+
+ESP8266httpUpdate	KEYWORD3		RESERVED_WORD
+
+#######################################
+# Datatypes (KEYWORD1)
+#######################################
+
+HTTPUpdateResult	KEYWORD1		DATA_TYPE
+ESPhttpUpdate	KEYWORD1		DATA_TYPE
+
+#######################################
+# Methods and Functions (KEYWORD2)
+#######################################
+
+rebootOnUpdate	KEYWORD2
+update	KEYWORD2
+updateSpiffs	KEYWORD2
+getLastError	KEYWORD2
+getLastErrorString	KEYWORD2
+
+#######################################
+# Constants (LITERAL1)
+#######################################
+
+HTTP_UE_TOO_LESS_SPACE	LITERAL1		RESERVED_WORD_2
+HTTP_UE_SERVER_NOT_REPORT_SIZE	LITERAL1		RESERVED_WORD_2
+HTTP_UE_SERVER_FILE_NOT_FOUND	LITERAL1		RESERVED_WORD_2
+HTTP_UE_SERVER_FORBIDDEN	LITERAL1		RESERVED_WORD_2
+HTTP_UE_SERVER_WRONG_HTTP_CODE	LITERAL1		RESERVED_WORD_2
+HTTP_UE_SERVER_FAULTY_MD5	LITERAL1		RESERVED_WORD_2
+HTTP_UE_BIN_VERIFY_HEADER_FAILED	LITERAL1		RESERVED_WORD_2
+HTTP_UE_BIN_FOR_WRONG_FLASH	LITERAL1		RESERVED_WORD_2
+HTTP_UPDATE_FAILED	LITERAL1		RESERVED_WORD_2
+HTTP_UPDATE_NO_UPDATES	LITERAL1		RESERVED_WORD_2
+HTTP_UPDATE_OK	LITERAL1		RESERVED_WORD_2

libraries/HTTPUpdate/library.properties

+name=HTTPUpdate
+version=1.3
+author=Markus Sattler
+maintainer=Markus Sattler
+sentence=Http Update for ESP32
+paragraph=
+category=Data Processing
+url=https://github.com/Links2004/Arduino/tree/esp8266/hardware/esp8266com/esp8266/libraries/ESP8266httpUpdate
+architectures=esp32

libraries/HTTPUpdate/src/HTTPUpdate.h

+/**
+ *
+ * @file HTTPUpdate.h based on ESP8266HTTPUpdate.h
+ * @date 16.10.2018
+ * @author Markus Sattler
+ *
+ * Copyright (c) 2015 Markus Sattler. All rights reserved.
+ * This file is part of the ESP32 Http Updater.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ */
+
+#ifndef ___HTTP_UPDATE_H___
+#define ___HTTP_UPDATE_H___
+
+#include <Arduino.h>
+#include <WiFi.h>
+#include <WiFiClient.h>
+#include <WiFiUdp.h>
+#include <HTTPClient.h>
+#include <Update.h>
+
+/// note we use HTTP client errors too so we start at 100
+#define HTTP_UE_TOO_LESS_SPACE              (-100)
+#define HTTP_UE_SERVER_NOT_REPORT_SIZE      (-101)
+#define HTTP_UE_SERVER_FILE_NOT_FOUND       (-102)
+#define HTTP_UE_SERVER_FORBIDDEN            (-103)
+#define HTTP_UE_SERVER_WRONG_HTTP_CODE      (-104)
+#define HTTP_UE_SERVER_FAULTY_MD5           (-105)
+#define HTTP_UE_BIN_VERIFY_HEADER_FAILED    (-106)
+#define HTTP_UE_BIN_FOR_WRONG_FLASH         (-107)
+
+enum HTTPUpdateResult {
+    HTTP_UPDATE_FAILED,
+    HTTP_UPDATE_NO_UPDATES,
+    HTTP_UPDATE_OK
+};
+
+typedef HTTPUpdateResult t_httpUpdate_return; // backward compatibility
+
+class HTTPUpdate
+{
+public:
+    HTTPUpdate(void);
+    HTTPUpdate(int httpClientTimeout);
+    ~HTTPUpdate(void);
+
+    void rebootOnUpdate(bool reboot)
+    {
+        _rebootOnUpdate = reboot;
+    }
+
+    void setLedPin(int ledPin = -1, uint8_t ledOn = HIGH)
+    {
+        _ledPin = ledPin;
+        _ledOn = ledOn;
+    }
+
+    t_httpUpdate_return update(WiFiClient& client, const String& url, const String& currentVersion = "");
+
+    t_httpUpdate_return update(WiFiClient& client, const String& host, uint16_t port, const String& uri = "/",
+                               const String& currentVersion = "");
+
+    t_httpUpdate_return updateSpiffs(WiFiClient& client, const String& url, const String& currentVersion = "");
+
+
+    int getLastError(void);
+    String getLastErrorString(void);
+
+protected:
+    t_httpUpdate_return handleUpdate(HTTPClient& http, const String& currentVersion, bool spiffs = false);
+    bool runUpdate(Stream& in, uint32_t size, String md5, int command = U_FLASH);
+
+    int _lastError;
+    bool _rebootOnUpdate = true;
+private:
+    int _httpClientTimeout;
+
+    int _ledPin;
+    uint8_t _ledOn;
+};
+
+#if !defined(NO_GLOBAL_INSTANCES) && !defined(NO_GLOBAL_HTTPUPDATE)
+extern HTTPUpdate httpUpdate;
+#endif
+
+#endif /* ___HTTP_UPDATE_H___ */

libraries/Update/src/Update.h

@@ -41,7 +41,7 @@ class UpdateClass {
       Call this to check the space needed for the update
       Will return false if there is not enough space
     */
-    bool begin(size_t size=UPDATE_SIZE_UNKNOWN, int command = U_FLASH);
+    bool begin(size_t size=UPDATE_SIZE_UNKNOWN, int command = U_FLASH, int ledPin = -1, uint8_t ledOn = LOW);
 
     /*
       Writes a buffer to the flash and increments the address
@@ -174,6 +174,9 @@ class UpdateClass {
 
     String _target_md5;
     MD5Builder _md5;
+
+    int _ledPin;
+    uint8_t _ledOn;
 };
 
 extern UpdateClass Update;

libraries/Update/src/Updater.cpp

@@ -88,6 +88,10 @@ void UpdateClass::_reset() {
     _progress = 0;
     _size = 0;
     _command = U_FLASH;
+
+    if(_ledPin != -1) {
+      digitalWrite(_ledPin, !_ledOn); // off
+    }
 }
 
 bool UpdateClass::canRollBack(){
@@ -106,12 +110,15 @@ bool UpdateClass::rollBack(){
     return _partitionIsBootable(partition) && !esp_ota_set_boot_partition(partition);
 }
 
-bool UpdateClass::begin(size_t size, int command) {
+bool UpdateClass::begin(size_t size, int command, int ledPin, uint8_t ledOn) {
     if(_size > 0){
         log_w("already running");
         return false;
     }
 
+    _ledPin = ledPin;
+    _ledOn = !!ledOn; // 0(LOW) or 1(HIGH)
+
     _reset();
     _error = 0;
 
@@ -315,16 +322,32 @@ size_t UpdateClass::writeStream(Stream &data) {
     if (_progress_callback) {
         _progress_callback(0, _size);
     }
+
+    if(_ledPin != -1) {
+        pinMode(_ledPin, OUTPUT);
+    }
+
     while(remaining()) {
-        toRead = data.readBytes(_buffer + _bufferLen,  (SPI_FLASH_SEC_SIZE - _bufferLen));
+        if(_ledPin != -1) {
+            digitalWrite(_ledPin, _ledOn); // Switch LED on
+        }
+        size_t bytesToRead = SPI_FLASH_SEC_SIZE - _bufferLen;
+        if(bytesToRead > remaining()) {
+            bytesToRead = remaining();
+        }
+
+        toRead = data.readBytes(_buffer + _bufferLen,  bytesToRead);
         if(toRead == 0) { //Timeout
             delay(100);
-            toRead = data.readBytes(_buffer + _bufferLen, (SPI_FLASH_SEC_SIZE - _bufferLen));
+            toRead = data.readBytes(_buffer + _bufferLen, bytesToRead);
             if(toRead == 0) { //Timeout
                 _abort(UPDATE_ERROR_STREAM);
                 return written;
             }
         }
+        if(_ledPin != -1) {
+            digitalWrite(_ledPin, !_ledOn); // Switch LED off
+        }
         _bufferLen += toRead;
         if((_bufferLen == remaining() || _bufferLen == SPI_FLASH_SEC_SIZE) && !_writeBuffer())
             return written;