Add TFLite Micro examples (#8717)

* create TFLite library

* add TFLite hello_world example

* add TFLite micro_speech example

---------
Co-authored-by: Sanket Wadekar <67091512+sanketwadekar@users.noreply.github.com>

libraries/TFLiteMicro/examples/hello_world/README.md

+# Hello World Example
+
+This example is designed to demonstrate the absolute basics of using [TensorFlow
+Lite for Microcontrollers](https://www.tensorflow.org/lite/microcontrollers).
+It includes the full end-to-end workflow of training a model, converting it for
+use with TensorFlow Lite for Microcontrollers for running inference on a
+microcontroller.
+
+The model is trained to replicate a `sine` function and generates a pattern of
+data to either blink LEDs or control an animation, depending on the capabilities
+of the device.
+
+## Deploy to ESP32
+
+The sample has been tested on ESP-IDF version `release/v4.2` and `release/v4.4` with the following devices:
+- [ESP32-DevKitC](http://esp-idf.readthedocs.io/en/latest/get-started/get-started-devkitc.html)
+- [ESP32-S3-DevKitC](https://docs.espressif.com/projects/esp-idf/en/latest/esp32s3/hw-reference/esp32s3/user-guide-devkitc-1.html)
+- [ESP-EYE](https://github.com/espressif/esp-who/blob/master/docs/en/get-started/ESP-EYE_Getting_Started_Guide.md)

libraries/TFLiteMicro/examples/hello_world/constants.cpp

+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "constants.h"
+
+// This is a small number so that it's easy to read the logs
+const int kInferencesPerCycle = 20;

libraries/TFLiteMicro/examples/hello_world/constants.h

+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_LITE_MICRO_EXAMPLES_HELLO_WORLD_CONSTANTS_H_
+#define TENSORFLOW_LITE_MICRO_EXAMPLES_HELLO_WORLD_CONSTANTS_H_
+
+// This constant represents the range of x values our model was trained on,
+// which is from 0 to (2 * Pi). We approximate Pi to avoid requiring additional
+// libraries.
+const float kXrange = 2.f * 3.14159265359f;
+
+// This constant determines the number of inferences to perform across the range
+// of x values defined above. Since each inference takes time, the higher this
+// number, the more time it will take to run through the entire range. The value
+// of this constant can be tuned so that one full cycle takes a desired amount
+// of time. Since different devices take different amounts of time to perform
+// inference, this value should be defined per-device.
+extern const int kInferencesPerCycle;
+
+#endif  // TENSORFLOW_LITE_MICRO_EXAMPLES_HELLO_WORLD_CONSTANTS_H_

libraries/TFLiteMicro/examples/hello_world/hello_world.ino

+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+
+#include "tensorflow/lite/micro/micro_mutable_op_resolver.h"
+#include "tensorflow/lite/micro/micro_interpreter.h"
+#include "tensorflow/lite/micro/system_setup.h"
+#include "tensorflow/lite/schema/schema_generated.h"
+
+#include "model.h"
+#include "constants.h"
+#include "output_handler.h"
+
+// Globals, used for compatibility with Arduino-style sketches.
+namespace {
+const tflite::Model* model = nullptr;
+tflite::MicroInterpreter* interpreter = nullptr;
+TfLiteTensor* input = nullptr;
+TfLiteTensor* output = nullptr;
+int inference_count = 0;
+
+constexpr int kTensorArenaSize = 2000;
+uint8_t tensor_arena[kTensorArenaSize];
+}  // namespace
+
+// The name of this function is important for Arduino compatibility.
+void setup() {
+  // Map the model into a usable data structure. This doesn't involve any
+  // copying or parsing, it's a very lightweight operation.
+  model = tflite::GetModel(g_model);
+  if (model->version() != TFLITE_SCHEMA_VERSION) {
+    MicroPrintf("Model provided is schema version %d not equal to supported "
+                "version %d.", model->version(), TFLITE_SCHEMA_VERSION);
+    return;
+  }
+
+  // Pull in only the operation implementations we need.
+  static tflite::MicroMutableOpResolver<1> resolver;
+  if (resolver.AddFullyConnected() != kTfLiteOk) {
+    return;
+  }
+
+  // Build an interpreter to run the model with.
+  static tflite::MicroInterpreter static_interpreter(
+      model, resolver, tensor_arena, kTensorArenaSize);
+  interpreter = &static_interpreter;
+
+  // Allocate memory from the tensor_arena for the model's tensors.
+  TfLiteStatus allocate_status = interpreter->AllocateTensors();
+  if (allocate_status != kTfLiteOk) {
+    MicroPrintf("AllocateTensors() failed");
+    return;
+  }
+
+  // Obtain pointers to the model's input and output tensors.
+  input = interpreter->input(0);
+  output = interpreter->output(0);
+
+  // Keep track of how many inferences we have performed.
+  inference_count = 0;
+}
+
+// The name of this function is important for Arduino compatibility.
+void loop() {
+  // Calculate an x value to feed into the model. We compare the current
+  // inference_count to the number of inferences per cycle to determine
+  // our position within the range of possible x values the model was
+  // trained on, and use this to calculate a value.
+  float position = static_cast<float>(inference_count) /
+                   static_cast<float>(kInferencesPerCycle);
+  float x = position * kXrange;
+
+  // Quantize the input from floating-point to integer
+  int8_t x_quantized = x / input->params.scale + input->params.zero_point;
+  // Place the quantized input in the model's input tensor
+  input->data.int8[0] = x_quantized;
+
+  // Run inference, and report any error
+  TfLiteStatus invoke_status = interpreter->Invoke();
+  if (invoke_status != kTfLiteOk) {
+    MicroPrintf("Invoke failed on x: %f\n",
+                         static_cast<double>(x));
+    return;
+  }
+
+  // Obtain the quantized output from model's output tensor
+  int8_t y_quantized = output->data.int8[0];
+  // Dequantize the output from integer to floating-point
+  float y = (y_quantized - output->params.zero_point) * output->params.scale;
+
+  // Output the results. A custom HandleOutput function can be implemented
+  // for each supported hardware target.
+  HandleOutput(x, y);
+
+  // Increment the inference_counter, and reset it if we have reached
+  // the total number per cycle
+  inference_count += 1;
+  if (inference_count >= kInferencesPerCycle) inference_count = 0;
+}

libraries/TFLiteMicro/examples/hello_world/model.h

+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+// Automatically created from a TensorFlow Lite flatbuffer using the command:
+// xxd -i model.tflite > model.cc
+
+// This is a standard TensorFlow Lite model file that has been converted into a
+// C data array, so it can be easily compiled into a binary for devices that
+// don't have a file system.
+
+// See train/README.md for a full description of the creation process.
+
+#ifndef TENSORFLOW_LITE_MICRO_EXAMPLES_HELLO_WORLD_MODEL_H_
+#define TENSORFLOW_LITE_MICRO_EXAMPLES_HELLO_WORLD_MODEL_H_
+
+extern const unsigned char g_model[];
+extern const int g_model_len;
+
+#endif  // TENSORFLOW_LITE_MICRO_EXAMPLES_HELLO_WORLD_MODEL_H_

libraries/TFLiteMicro/examples/hello_world/output_handler.cpp

+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "output_handler.h"
+#include "tensorflow/lite/micro/micro_log.h"
+
+void HandleOutput(float x_value, float y_value) {
+  // Log the current X and Y values
+  MicroPrintf("x_value: %f, y_value: %f\n", static_cast<double>(x_value),
+              static_cast<double>(y_value));
+}

libraries/TFLiteMicro/examples/hello_world/output_handler.h

+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_LITE_MICRO_EXAMPLES_HELLO_WORLD_OUTPUT_HANDLER_H_
+#define TENSORFLOW_LITE_MICRO_EXAMPLES_HELLO_WORLD_OUTPUT_HANDLER_H_
+
+#include "tensorflow/lite/c/common.h"
+
+// Called by the main loop to produce some output based on the x and y values
+void HandleOutput(float x_value, float y_value);
+
+#endif  // TENSORFLOW_LITE_MICRO_EXAMPLES_HELLO_WORLD_OUTPUT_HANDLER_H_

libraries/TFLiteMicro/examples/micro_speech/README.md

+# Micro Speech Example
+
+This example shows how to run a 20 kB model that can recognize 2 keywords,
+"yes" and "no", from speech data.
+
+The application listens to its surroundings with a microphone and indicates
+when it has detected a word by displaying data on a screen.
+
+## Deploy to ESP32
+
+The sample has been tested on ESP-IDF version `release/v4.2` and `release/v4.4` with the following devices:
+- [ESP32-DevKitC](http://esp-idf.readthedocs.io/en/latest/get-started/get-started-devkitc.html)
+- [ESP32-S3-DevKitC](https://docs.espressif.com/projects/esp-idf/en/latest/esp32s3/hw-reference/esp32s3/user-guide-devkitc-1.html)
+- [ESP-EYE](https://github.com/espressif/esp-who/blob/master/docs/en/get-started/ESP-EYE_Getting_Started_Guide.md)
+
+### Sample output
+
+  * When a keyword is detected you will see following output sample output on the log screen:
+
+```
+Heard yes (<score>) at <time>
+```

libraries/TFLiteMicro/examples/micro_speech/audio_provider.cpp

+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "audio_provider.h"
+
+#include <cstdlib>
+#include <cstring>
+
+// FreeRTOS.h must be included before some of the following dependencies.
+// Solves b/150260343.
+// clang-format off
+#include "freertos/FreeRTOS.h"
+// clang-format on
+
+#include "driver/i2s.h"
+#include "esp_log.h"
+#include "esp_spi_flash.h"
+#include "esp_system.h"
+#include "esp_timer.h"
+#include "freertos/task.h"
+#include "ringbuf.h"
+#include "micro_model_settings.h"
+
+using namespace std;
+
+#define NO_I2S_SUPPORT CONFIG_IDF_TARGET_ESP32C2 || \
+                          (CONFIG_IDF_TARGET_ESP32C3 \
+                          && (ESP_IDF_VERSION < ESP_IDF_VERSION_VAL(4, 4, 0)))
+
+static const char* TAG = "TF_LITE_AUDIO_PROVIDER";
+/* ringbuffer to hold the incoming audio data */
+ringbuf_t* g_audio_capture_buffer;
+volatile int32_t g_latest_audio_timestamp = 0;
+/* model requires 20ms new data from g_audio_capture_buffer and 10ms old data
+ * each time , storing old data in the histrory buffer , {
+ * history_samples_to_keep = 10 * 16 } */
+constexpr int32_t history_samples_to_keep =
+    ((kFeatureSliceDurationMs - kFeatureSliceStrideMs) *
+     (kAudioSampleFrequency / 1000));
+/* new samples to get each time from ringbuffer, { new_samples_to_get =  20 * 16
+ * } */
+constexpr int32_t new_samples_to_get =
+    (kFeatureSliceStrideMs * (kAudioSampleFrequency / 1000));
+
+namespace {
+int16_t g_audio_output_buffer[kMaxAudioSampleSize];
+bool g_is_audio_initialized = false;
+int16_t g_history_buffer[history_samples_to_keep];
+}  // namespace
+
+const int32_t kAudioCaptureBufferSize = 80000;
+const int32_t i2s_bytes_to_read = 3200;
+
+#if NO_I2S_SUPPORT
+  // nothing to be done here
+#else
+static void i2s_init(void) {
+  // Start listening for audio: MONO @ 16KHz
+  i2s_config_t i2s_config = {
+      .mode = (i2s_mode_t)(I2S_MODE_MASTER | I2S_MODE_RX | I2S_MODE_TX),
+      .sample_rate = 16000,
+      .bits_per_sample = (i2s_bits_per_sample_t)16,
+      .channel_format = I2S_CHANNEL_FMT_ONLY_LEFT,
+      .communication_format = I2S_COMM_FORMAT_I2S,
+      .intr_alloc_flags = 0,
+      .dma_buf_count = 3,
+      .dma_buf_len = 300,
+      .use_apll = false,
+      .tx_desc_auto_clear = false,
+      .fixed_mclk = -1,
+  };
+  i2s_pin_config_t pin_config = {
+      .bck_io_num = 26,    // IIS_SCLK
+      .ws_io_num = 32,     // IIS_LCLK
+      .data_out_num = -1,  // IIS_DSIN
+      .data_in_num = 33,   // IIS_DOUT
+  };
+  esp_err_t ret = 0;
+  ret = i2s_driver_install((i2s_port_t)1, &i2s_config, 0, NULL);
+  if (ret != ESP_OK) {
+    ESP_LOGE(TAG, "Error in i2s_driver_install");
+  }
+  ret = i2s_set_pin((i2s_port_t)1, &pin_config);
+  if (ret != ESP_OK) {
+    ESP_LOGE(TAG, "Error in i2s_set_pin");
+  }
+
+  ret = i2s_zero_dma_buffer((i2s_port_t)1);
+  if (ret != ESP_OK) {
+    ESP_LOGE(TAG, "Error in initializing dma buffer with 0");
+  }
+}
+#endif
+
+static void CaptureSamples(void* arg) {
+#if NO_I2S_SUPPORT
+  ESP_LOGE(TAG, "i2s support not available on C3 chip for IDF < 4.4.0");
+  return;
+#else
+  size_t bytes_read = i2s_bytes_to_read;
+  uint8_t i2s_read_buffer[i2s_bytes_to_read] = {};
+  i2s_init();
+  while (1) {
+    /* read 100ms data at once from i2s */
+    i2s_read((i2s_port_t)1, (void*)i2s_read_buffer, i2s_bytes_to_read,
+             &bytes_read, pdMS_TO_TICKS(100));
+    if (bytes_read <= 0) {
+      ESP_LOGE(TAG, "Error in I2S read : %d", bytes_read);
+    } else {
+      if (bytes_read < i2s_bytes_to_read) {
+        ESP_LOGW(TAG, "Partial I2S read");
+      }
+      /* write bytes read by i2s into ring buffer */
+      int bytes_written = rb_write(g_audio_capture_buffer,
+                                   (uint8_t*)i2s_read_buffer, bytes_read, pdMS_TO_TICKS(100));
+      /* update the timestamp (in ms) to let the model know that new data has
+       * arrived */
+      g_latest_audio_timestamp = g_latest_audio_timestamp +
+          ((1000 * (bytes_written / 2)) / kAudioSampleFrequency);
+      if (bytes_written <= 0) {
+        ESP_LOGE(TAG, "Could Not Write in Ring Buffer: %d ", bytes_written);
+      } else if (bytes_written < bytes_read) {
+        ESP_LOGW(TAG, "Partial Write");
+      }
+    }
+  }
+#endif
+  vTaskDelete(NULL);
+}
+
+TfLiteStatus InitAudioRecording() {
+  g_audio_capture_buffer = rb_init("tf_ringbuffer", kAudioCaptureBufferSize);
+  if (!g_audio_capture_buffer) {
+    ESP_LOGE(TAG, "Error creating ring buffer");
+    return kTfLiteError;
+  }
+  /* create CaptureSamples Task which will get the i2s_data from mic and fill it
+   * in the ring buffer */
+  xTaskCreate(CaptureSamples, "CaptureSamples", 1024 * 32, NULL, 10, NULL);
+  while (!g_latest_audio_timestamp) {
+    vTaskDelay(1); // one tick delay to avoid watchdog
+  }
+  ESP_LOGI(TAG, "Audio Recording started");
+  return kTfLiteOk;
+}
+
+TfLiteStatus GetAudioSamples(int start_ms, int duration_ms,
+                             int* audio_samples_size, int16_t** audio_samples) {
+  if (!g_is_audio_initialized) {
+    TfLiteStatus init_status = InitAudioRecording();
+    if (init_status != kTfLiteOk) {
+      return init_status;
+    }
+    g_is_audio_initialized = true;
+  }
+  /* copy 160 samples (320 bytes) into output_buff from history */
+  memcpy((void*)(g_audio_output_buffer), (void*)(g_history_buffer),
+         history_samples_to_keep * sizeof(int16_t));
+
+  /* copy 320 samples (640 bytes) from rb at ( int16_t*(g_audio_output_buffer) +
+   * 160 ), first 160 samples (320 bytes) will be from history */
+  int bytes_read =
+      rb_read(g_audio_capture_buffer,
+              ((uint8_t*)(g_audio_output_buffer + history_samples_to_keep)),
+              new_samples_to_get * sizeof(int16_t), pdMS_TO_TICKS(100));
+  if (bytes_read < 0) {
+    ESP_LOGE(TAG, " Model Could not read data from Ring Buffer");
+  } else if (bytes_read < new_samples_to_get * sizeof(int16_t)) {
+    ESP_LOGD(TAG, "RB FILLED RIGHT NOW IS %d",
+             rb_filled(g_audio_capture_buffer));
+    ESP_LOGD(TAG, " Partial Read of Data by Model ");
+    ESP_LOGV(TAG, " Could only read %d bytes when required %d bytes ",
+             bytes_read, (int) (new_samples_to_get * sizeof(int16_t)));
+  }
+
+  /* copy 320 bytes from output_buff into history */
+  memcpy((void*)(g_history_buffer),
+         (void*)(g_audio_output_buffer + new_samples_to_get),
+         history_samples_to_keep * sizeof(int16_t));
+
+  *audio_samples_size = kMaxAudioSampleSize;
+  *audio_samples = g_audio_output_buffer;
+  return kTfLiteOk;
+}
+
+int32_t LatestAudioTimestamp() { return g_latest_audio_timestamp; }

libraries/TFLiteMicro/examples/micro_speech/audio_provider.h

+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_LITE_MICRO_EXAMPLES_MICRO_SPEECH_AUDIO_PROVIDER_H_
+#define TENSORFLOW_LITE_MICRO_EXAMPLES_MICRO_SPEECH_AUDIO_PROVIDER_H_
+
+#include "tensorflow/lite/c/common.h"
+
+// This is an abstraction around an audio source like a microphone, and is
+// expected to return 16-bit PCM sample data for a given point in time. The
+// sample data itself should be used as quickly as possible by the caller, since
+// to allow memory optimizations there are no guarantees that the samples won't
+// be overwritten by new data in the future. In practice, implementations should
+// ensure that there's a reasonable time allowed for clients to access the data
+// before any reuse.
+// The reference implementation can have no platform-specific dependencies, so
+// it just returns an array filled with zeros. For real applications, you should
+// ensure there's a specialized implementation that accesses hardware APIs.
+TfLiteStatus GetAudioSamples(int start_ms, int duration_ms,
+                             int* audio_samples_size, int16_t** audio_samples);
+
+// Returns the time that audio data was last captured in milliseconds. There's
+// no contract about what time zero represents, the accuracy, or the granularity
+// of the result. Subsequent calls will generally not return a lower value, but
+// even that's not guaranteed if there's an overflow wraparound.
+// The reference implementation of this function just returns a constantly
+// incrementing value for each call, since it would need a non-portable platform
+// call to access time information. For real applications, you'll need to write
+// your own platform-specific implementation.
+int32_t LatestAudioTimestamp();
+
+#endif  // TENSORFLOW_LITE_MICRO_EXAMPLES_MICRO_SPEECH_AUDIO_PROVIDER_H_

libraries/TFLiteMicro/examples/micro_speech/command_responder.cpp

+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "command_responder.h"
+#include "tensorflow/lite/micro/micro_log.h"
+
+// The default implementation writes out the name of the recognized command
+// to the error console. Real applications will want to take some custom
+// action instead, and should implement their own versions of this function.
+void RespondToCommand(int32_t current_time, const char* found_command,
+                      uint8_t score, bool is_new_command) {
+  if (is_new_command) {
+    MicroPrintf("Heard %s (%d) @%dms", found_command, score, current_time);
+  }
+}

libraries/TFLiteMicro/examples/micro_speech/command_responder.h

+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+// Provides an interface to take an action based on an audio command.
+
+#ifndef TENSORFLOW_LITE_MICRO_EXAMPLES_MICRO_SPEECH_COMMAND_RESPONDER_H_
+#define TENSORFLOW_LITE_MICRO_EXAMPLES_MICRO_SPEECH_COMMAND_RESPONDER_H_
+
+#include "tensorflow/lite/c/common.h"
+
+// Called every time the results of an audio recognition run are available. The
+// human-readable name of any recognized command is in the `found_command`
+// argument, `score` has the numerical confidence, and `is_new_command` is set
+// if the previous command was different to this one.
+void RespondToCommand(int32_t current_time, const char* found_command,
+                      uint8_t score, bool is_new_command);
+
+#endif  // TENSORFLOW_LITE_MICRO_EXAMPLES_MICRO_SPEECH_COMMAND_RESPONDER_H_

libraries/TFLiteMicro/examples/micro_speech/feature_provider.cpp

+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "feature_provider.h"
+
+#include "audio_provider.h"
+#include "micro_features_generator.h"
+#include "micro_model_settings.h"
+#include "tensorflow/lite/micro/micro_log.h"
+
+FeatureProvider::FeatureProvider(int feature_size, int8_t* feature_data)
+    : feature_size_(feature_size),
+      feature_data_(feature_data),
+      is_first_run_(true) {
+  // Initialize the feature data to default values.
+  for (int n = 0; n < feature_size_; ++n) {
+    feature_data_[n] = 0;
+  }
+}
+
+FeatureProvider::~FeatureProvider() {}
+
+TfLiteStatus FeatureProvider::PopulateFeatureData(
+    int32_t last_time_in_ms, int32_t time_in_ms, int* how_many_new_slices) {
+  if (feature_size_ != kFeatureElementCount) {
+    MicroPrintf("Requested feature_data_ size %d doesn't match %d",
+                feature_size_, kFeatureElementCount);
+    return kTfLiteError;
+  }
+
+  // Quantize the time into steps as long as each window stride, so we can
+  // figure out which audio data we need to fetch.
+  const int last_step = (last_time_in_ms / kFeatureSliceStrideMs);
+  const int current_step = (time_in_ms / kFeatureSliceStrideMs);
+
+  int slices_needed = current_step - last_step;
+  // If this is the first call, make sure we don't use any cached information.
+  if (is_first_run_) {
+    TfLiteStatus init_status = InitializeMicroFeatures();
+    if (init_status != kTfLiteOk) {
+      return init_status;
+    }
+    is_first_run_ = false;
+    slices_needed = kFeatureSliceCount;
+  }
+  if (slices_needed > kFeatureSliceCount) {
+    slices_needed = kFeatureSliceCount;
+  }
+  *how_many_new_slices = slices_needed;
+
+  const int slices_to_keep = kFeatureSliceCount - slices_needed;
+  const int slices_to_drop = kFeatureSliceCount - slices_to_keep;
+  // If we can avoid recalculating some slices, just move the existing data
+  // up in the spectrogram, to perform something like this:
+  // last time = 80ms          current time = 120ms
+  // +-----------+             +-----------+
+  // | data@20ms |         --> | data@60ms |
+  // +-----------+       --    +-----------+
+  // | data@40ms |     --  --> | data@80ms |
+  // +-----------+   --  --    +-----------+
+  // | data@60ms | --  --      |  <empty>  |
+  // +-----------+   --        +-----------+
+  // | data@80ms | --          |  <empty>  |
+  // +-----------+             +-----------+
+  if (slices_to_keep > 0) {
+    for (int dest_slice = 0; dest_slice < slices_to_keep; ++dest_slice) {
+      int8_t* dest_slice_data =
+          feature_data_ + (dest_slice * kFeatureSliceSize);
+      const int src_slice = dest_slice + slices_to_drop;
+      const int8_t* src_slice_data =
+          feature_data_ + (src_slice * kFeatureSliceSize);
+      for (int i = 0; i < kFeatureSliceSize; ++i) {
+        dest_slice_data[i] = src_slice_data[i];
+      }
+    }
+  }
+  // Any slices that need to be filled in with feature data have their
+  // appropriate audio data pulled, and features calculated for that slice.
+  if (slices_needed > 0) {
+    for (int new_slice = slices_to_keep; new_slice < kFeatureSliceCount;
+         ++new_slice) {
+      const int new_step = (current_step - kFeatureSliceCount + 1) + new_slice;
+      const int32_t slice_start_ms = (new_step * kFeatureSliceStrideMs);
+      int16_t* audio_samples = nullptr;
+      int audio_samples_size = 0;
+      // TODO(petewarden): Fix bug that leads to non-zero slice_start_ms
+      GetAudioSamples((slice_start_ms > 0 ? slice_start_ms : 0),
+                      kFeatureSliceDurationMs, &audio_samples_size,
+                      &audio_samples);
+      if (audio_samples_size < kMaxAudioSampleSize) {
+        MicroPrintf("Audio data size %d too small, want %d",
+                    audio_samples_size, kMaxAudioSampleSize);
+        return kTfLiteError;
+      }
+      int8_t* new_slice_data = feature_data_ + (new_slice * kFeatureSliceSize);
+      size_t num_samples_read;
+      TfLiteStatus generate_status = GenerateMicroFeatures(
+          audio_samples, audio_samples_size, kFeatureSliceSize,
+          new_slice_data, &num_samples_read);
+      if (generate_status != kTfLiteOk) {
+        return generate_status;
+      }
+    }
+  }
+  return kTfLiteOk;
+}

libraries/TFLiteMicro/examples/micro_speech/feature_provider.h

+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_LITE_MICRO_EXAMPLES_MICRO_SPEECH_FEATURE_PROVIDER_H_
+#define TENSORFLOW_LITE_MICRO_EXAMPLES_MICRO_SPEECH_FEATURE_PROVIDER_H_
+
+#include "tensorflow/lite/c/common.h"
+
+// Binds itself to an area of memory intended to hold the input features for an
+// audio-recognition neural network model, and fills that data area with the
+// features representing the current audio input, for example from a microphone.
+// The audio features themselves are a two-dimensional array, made up of
+// horizontal slices representing the frequencies at one point in time, stacked
+// on top of each other to form a spectrogram showing how those frequencies
+// changed over time.
+class FeatureProvider {
+ public:
+  // Create the provider, and bind it to an area of memory. This memory should
+  // remain accessible for the lifetime of the provider object, since subsequent
+  // calls will fill it with feature data. The provider does no memory
+  // management of this data.
+  FeatureProvider(int feature_size, int8_t* feature_data);
+  ~FeatureProvider();
+
+  // Fills the feature data with information from audio inputs, and returns how
+  // many feature slices were updated.
+  TfLiteStatus PopulateFeatureData(int32_t last_time_in_ms, int32_t time_in_ms,
+                                   int* how_many_new_slices);
+
+ private:
+  int feature_size_;
+  int8_t* feature_data_;
+  // Make sure we don't try to use cached information if this is the first call
+  // into the provider.
+  bool is_first_run_;
+};
+
+#endif  // TENSORFLOW_LITE_MICRO_EXAMPLES_MICRO_SPEECH_FEATURE_PROVIDER_H_

libraries/TFLiteMicro/examples/micro_speech/micro_features_generator.cpp

+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "micro_features_generator.h"
+
+#include <cmath>
+#include <cstring>
+
+#include "tensorflow/lite/experimental/microfrontend/lib/frontend.h"
+#include "tensorflow/lite/experimental/microfrontend/lib/frontend_util.h"
+#include "tensorflow/lite/micro/micro_log.h"
+#include "micro_model_settings.h"
+
+// Configure FFT to output 16 bit fixed point.
+#define FIXED_POINT 16
+
+namespace {
+
+FrontendState g_micro_features_state;
+bool g_is_first_time = true;
+
+}  // namespace
+
+TfLiteStatus InitializeMicroFeatures() {
+  FrontendConfig config;
+  config.window.size_ms = kFeatureSliceDurationMs;
+  config.window.step_size_ms = kFeatureSliceStrideMs;
+  config.noise_reduction.smoothing_bits = 10;
+  config.filterbank.num_channels = kFeatureSliceSize;
+  config.filterbank.lower_band_limit = 125.0;
+  config.filterbank.upper_band_limit = 7500.0;
+  config.noise_reduction.smoothing_bits = 10;
+  config.noise_reduction.even_smoothing = 0.025;
+  config.noise_reduction.odd_smoothing = 0.06;
+  config.noise_reduction.min_signal_remaining = 0.05;
+  config.pcan_gain_control.enable_pcan = 1;
+  config.pcan_gain_control.strength = 0.95;
+  config.pcan_gain_control.offset = 80.0;
+  config.pcan_gain_control.gain_bits = 21;
+  config.log_scale.enable_log = 1;
+  config.log_scale.scale_shift = 6;
+  if (!FrontendPopulateState(&config, &g_micro_features_state,
+                             kAudioSampleFrequency)) {
+    MicroPrintf("FrontendPopulateState() failed");
+    return kTfLiteError;
+  }
+  g_is_first_time = true;
+  return kTfLiteOk;
+}
+
+// This is not exposed in any header, and is only used for testing, to ensure
+// that the state is correctly set up before generating results.
+void SetMicroFeaturesNoiseEstimates(const uint32_t* estimate_presets) {
+  for (int i = 0; i < g_micro_features_state.filterbank.num_channels; ++i) {
+    g_micro_features_state.noise_reduction.estimate[i] = estimate_presets[i];
+  }
+}
+
+TfLiteStatus GenerateMicroFeatures(const int16_t* input, int input_size,
+                                   int output_size, int8_t* output,
+                                   size_t* num_samples_read) {
+  const int16_t* frontend_input;
+  if (g_is_first_time) {
+    frontend_input = input;
+    g_is_first_time = false;
+  } else {
+    frontend_input = input + 160;
+  }
+  FrontendOutput frontend_output = FrontendProcessSamples(
+      &g_micro_features_state, frontend_input, input_size, num_samples_read);
+
+  for (size_t i = 0; i < frontend_output.size; ++i) {
+    // These scaling values are derived from those used in input_data.py in the
+    // training pipeline.
+    // The feature pipeline outputs 16-bit signed integers in roughly a 0 to 670
+    // range. In training, these are then arbitrarily divided by 25.6 to get
+    // float values in the rough range of 0.0 to 26.0. This scaling is performed
+    // for historical reasons, to match up with the output of other feature
+    // generators.
+    // The process is then further complicated when we quantize the model. This
+    // means we have to scale the 0.0 to 26.0 real values to the -128 to 127
+    // signed integer numbers.
+    // All this means that to get matching values from our integer feature
+    // output into the tensor input, we have to perform:
+    // input = (((feature / 25.6) / 26.0) * 256) - 128
+    // To simplify this and perform it in 32-bit integer math, we rearrange to:
+    // input = (feature * 256) / (25.6 * 26.0) - 128
+    constexpr int32_t value_scale = 256;
+    constexpr int32_t value_div = static_cast<int32_t>((25.6f * 26.0f) + 0.5f);
+    int32_t value =
+        ((frontend_output.values[i] * value_scale) + (value_div / 2)) /
+        value_div;
+    value -= 128;
+    if (value < -128) {
+      value = -128;
+    }
+    if (value > 127) {
+      value = 127;
+    }
+    output[i] = value;
+  }
+
+  return kTfLiteOk;
+}

libraries/TFLiteMicro/examples/micro_speech/micro_features_generator.h

+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_LITE_MICRO_EXAMPLES_MICRO_SPEECH_MICRO_FEATURES_MICRO_FEATURES_GENERATOR_H_
+#define TENSORFLOW_LITE_MICRO_EXAMPLES_MICRO_SPEECH_MICRO_FEATURES_MICRO_FEATURES_GENERATOR_H_
+
+#include "tensorflow/lite/c/common.h"
+
+// Sets up any resources needed for the feature generation pipeline.
+TfLiteStatus InitializeMicroFeatures();
+
+// Converts audio sample data into a more compact form that's appropriate for
+// feeding into a neural network.
+TfLiteStatus GenerateMicroFeatures(const int16_t* input, int input_size,
+                                   int output_size, int8_t* output,
+                                   size_t* num_samples_read);
+
+#endif  // TENSORFLOW_LITE_MICRO_EXAMPLES_MICRO_SPEECH_MICRO_FEATURES_MICRO_FEATURES_GENERATOR_H_

libraries/TFLiteMicro/examples/micro_speech/micro_model_settings.cpp

+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "micro_model_settings.h"
+
+const char* kCategoryLabels[kCategoryCount] = {
+    "silence",
+    "unknown",
+    "yes",
+    "no",
+};

libraries/TFLiteMicro/examples/micro_speech/micro_model_settings.h

+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_LITE_MICRO_EXAMPLES_MICRO_SPEECH_MICRO_FEATURES_MICRO_MODEL_SETTINGS_H_
+#define TENSORFLOW_LITE_MICRO_EXAMPLES_MICRO_SPEECH_MICRO_FEATURES_MICRO_MODEL_SETTINGS_H_
+
+// Keeping these as constant expressions allow us to allocate fixed-sized arrays
+// on the stack for our working memory.
+
+// The size of the input time series data we pass to the FFT to produce the
+// frequency information. This has to be a power of two, and since we're dealing
+// with 30ms of 16KHz inputs, which means 480 samples, this is the next value.
+constexpr int kMaxAudioSampleSize = 512;
+constexpr int kAudioSampleFrequency = 16000;
+
+// The following values are derived from values used during model training.
+// If you change the way you preprocess the input, update all these constants.
+constexpr int kFeatureSliceSize = 40;
+constexpr int kFeatureSliceCount = 49;
+constexpr int kFeatureElementCount = (kFeatureSliceSize * kFeatureSliceCount);
+constexpr int kFeatureSliceStrideMs = 20;
+constexpr int kFeatureSliceDurationMs = 30;
+
+// Variables for the model's output categories.
+constexpr int kSilenceIndex = 0;
+constexpr int kUnknownIndex = 1;
+// If you modify the output categories, you need to update the following values.
+constexpr int kCategoryCount = 4;
+extern const char* kCategoryLabels[kCategoryCount];
+
+#endif  // TENSORFLOW_LITE_MICRO_EXAMPLES_MICRO_SPEECH_MICRO_FEATURES_MICRO_MODEL_SETTINGS_H_

libraries/TFLiteMicro/examples/micro_speech/micro_speech.ino

+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "audio_provider.h"
+#include "command_responder.h"
+#include "feature_provider.h"
+#include "micro_model_settings.h"
+#include "model.h"
+#include "recognize_commands.h"
+#include "tensorflow/lite/micro/micro_interpreter.h"
+#include "tensorflow/lite/micro/micro_log.h"
+#include "tensorflow/lite/micro/micro_mutable_op_resolver.h"
+#include "tensorflow/lite/micro/system_setup.h"
+#include "tensorflow/lite/schema/schema_generated.h"
+
+// Globals, used for compatibility with Arduino-style sketches.
+namespace {
+const tflite::Model* model = nullptr;
+tflite::MicroInterpreter* interpreter = nullptr;
+TfLiteTensor* model_input = nullptr;
+FeatureProvider* feature_provider = nullptr;
+RecognizeCommands* recognizer = nullptr;
+int32_t previous_time = 0;
+
+// Create an area of memory to use for input, output, and intermediate arrays.
+// The size of this will depend on the model you're using, and may need to be
+// determined by experimentation.
+constexpr int kTensorArenaSize = 30 * 1024;
+uint8_t tensor_arena[kTensorArenaSize];
+int8_t feature_buffer[kFeatureElementCount];
+int8_t* model_input_buffer = nullptr;
+}  // namespace
+
+// The name of this function is important for Arduino compatibility.
+void setup() {
+  // Map the model into a usable data structure. This doesn't involve any
+  // copying or parsing, it's a very lightweight operation.
+  model = tflite::GetModel(g_model);
+  if (model->version() != TFLITE_SCHEMA_VERSION) {
+    MicroPrintf("Model provided is schema version %d not equal to supported "
+                "version %d.", model->version(), TFLITE_SCHEMA_VERSION);
+    return;
+  }
+
+  // Pull in only the operation implementations we need.
+  // This relies on a complete list of all the ops needed by this graph.
+  // An easier approach is to just use the AllOpsResolver, but this will
+  // incur some penalty in code space for op implementations that are not
+  // needed by this graph.
+  //
+  // tflite::AllOpsResolver resolver;
+  // NOLINTNEXTLINE(runtime-global-variables)
+  static tflite::MicroMutableOpResolver<4> micro_op_resolver;
+  if (micro_op_resolver.AddDepthwiseConv2D() != kTfLiteOk) {
+    return;
+  }
+  if (micro_op_resolver.AddFullyConnected() != kTfLiteOk) {
+    return;
+  }
+  if (micro_op_resolver.AddSoftmax() != kTfLiteOk) {
+    return;
+  }
+  if (micro_op_resolver.AddReshape() != kTfLiteOk) {
+    return;
+  }
+
+  // Build an interpreter to run the model with.
+  static tflite::MicroInterpreter static_interpreter(
+      model, micro_op_resolver, tensor_arena, kTensorArenaSize);
+  interpreter = &static_interpreter;
+
+  // Allocate memory from the tensor_arena for the model's tensors.
+  TfLiteStatus allocate_status = interpreter->AllocateTensors();
+  if (allocate_status != kTfLiteOk) {
+    MicroPrintf("AllocateTensors() failed");
+    return;
+  }
+
+  // Get information about the memory area to use for the model's input.
+  model_input = interpreter->input(0);
+  if ((model_input->dims->size != 2) || (model_input->dims->data[0] != 1) ||
+      (model_input->dims->data[1] !=
+       (kFeatureSliceCount * kFeatureSliceSize)) ||
+      (model_input->type != kTfLiteInt8)) {
+    MicroPrintf("Bad input tensor parameters in model");
+    return;
+  }
+  model_input_buffer = model_input->data.int8;
+
+  // Prepare to access the audio spectrograms from a microphone or other source
+  // that will provide the inputs to the neural network.
+  // NOLINTNEXTLINE(runtime-global-variables)
+  static FeatureProvider static_feature_provider(kFeatureElementCount,
+                                                 feature_buffer);
+  feature_provider = &static_feature_provider;
+
+  static RecognizeCommands static_recognizer;
+  recognizer = &static_recognizer;
+
+  previous_time = 0;
+}
+
+// The name of this function is important for Arduino compatibility.
+void loop() {
+  // Fetch the spectrogram for the current time.
+  const int32_t current_time = LatestAudioTimestamp();
+  int how_many_new_slices = 0;
+  TfLiteStatus feature_status = feature_provider->PopulateFeatureData(
+      previous_time, current_time, &how_many_new_slices);
+  if (feature_status != kTfLiteOk) {
+    MicroPrintf( "Feature generation failed");
+    return;
+  }
+  previous_time = current_time;
+  // If no new audio samples have been received since last time, don't bother
+  // running the network model.
+  if (how_many_new_slices == 0) {
+    return;
+  }
+
+  // Copy feature buffer to input tensor
+  for (int i = 0; i < kFeatureElementCount; i++) {
+    model_input_buffer[i] = feature_buffer[i];
+  }
+
+  // Run the model on the spectrogram input and make sure it succeeds.
+  TfLiteStatus invoke_status = interpreter->Invoke();
+  if (invoke_status != kTfLiteOk) {
+    MicroPrintf( "Invoke failed");
+    return;
+  }
+
+  // Obtain a pointer to the output tensor
+  TfLiteTensor* output = interpreter->output(0);
+  // Determine whether a command was recognized based on the output of inference
+  const char* found_command = nullptr;
+  uint8_t score = 0;
+  bool is_new_command = false;
+  TfLiteStatus process_status = recognizer->ProcessLatestResults(
+      output, current_time, &found_command, &score, &is_new_command);
+  if (process_status != kTfLiteOk) {
+    MicroPrintf("RecognizeCommands::ProcessLatestResults() failed");
+    return;
+  }
+  // Do something based on the recognized command. The default implementation
+  // just prints to the error console, but you should replace this with your
+  // own function for a real application.
+  RespondToCommand(current_time, found_command, score, is_new_command);
+}

libraries/TFLiteMicro/examples/micro_speech/model.h

+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+// This is a standard TensorFlow Lite FlatBuffer model file that has been
+// converted into a C data array, so it can be easily compiled into a binary
+// for devices that don't have a file system. It was created using the command:
+// xxd -i model.tflite > model.cc
+
+#ifndef TENSORFLOW_LITE_MICRO_EXAMPLES_MICRO_SPEECH_MICRO_FEATURES_MODEL_H_
+#define TENSORFLOW_LITE_MICRO_EXAMPLES_MICRO_SPEECH_MICRO_FEATURES_MODEL_H_
+
+extern const unsigned char g_model[];
+extern const int g_model_len;
+
+#endif  // TENSORFLOW_LITE_MICRO_EXAMPLES_MICRO_SPEECH_MICRO_FEATURES_MODEL_H_

libraries/TFLiteMicro/examples/micro_speech/recognize_commands.cpp

+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "recognize_commands.h"
+
+#include <limits>
+
+RecognizeCommands::RecognizeCommands(int32_t average_window_duration_ms,
+                                     uint8_t detection_threshold,
+                                     int32_t suppression_ms,
+                                     int32_t minimum_count)
+    : average_window_duration_ms_(average_window_duration_ms),
+      detection_threshold_(detection_threshold),
+      suppression_ms_(suppression_ms),
+      minimum_count_(minimum_count),
+      previous_results_() {
+  previous_top_label_ = "silence";
+  previous_top_label_time_ = std::numeric_limits<int32_t>::min();
+}
+
+TfLiteStatus RecognizeCommands::ProcessLatestResults(
+    const TfLiteTensor* latest_results, const int32_t current_time_ms,
+    const char** found_command, uint8_t* score, bool* is_new_command) {
+  if ((latest_results->dims->size != 2) ||
+      (latest_results->dims->data[0] != 1) ||
+      (latest_results->dims->data[1] != kCategoryCount)) {
+    MicroPrintf(
+        "The results for recognition should contain %d elements, but there are "
+        "%d in an %d-dimensional shape",
+        kCategoryCount, latest_results->dims->data[1],
+        latest_results->dims->size);
+    return kTfLiteError;
+  }
+
+  if (latest_results->type != kTfLiteInt8) {
+    MicroPrintf(
+        "The results for recognition should be int8_t elements, but are %d",
+        latest_results->type);
+    return kTfLiteError;
+  }
+
+  if ((!previous_results_.empty()) &&
+      (current_time_ms < previous_results_.front().time_)) {
+    MicroPrintf(
+        "Results must be fed in increasing time order, but received a "
+        "timestamp of %d that was earlier than the previous one of %d",
+        current_time_ms, previous_results_.front().time_);
+    return kTfLiteError;
+  }
+
+  // Add the latest results to the head of the queue.
+  previous_results_.push_back({current_time_ms, latest_results->data.int8});
+
+  // Prune any earlier results that are too old for the averaging window.
+  const int64_t time_limit = current_time_ms - average_window_duration_ms_;
+  while ((!previous_results_.empty()) &&
+         previous_results_.front().time_ < time_limit) {
+    previous_results_.pop_front();
+  }
+
+  // If there are too few results, assume the result will be unreliable and
+  // bail.
+  const int64_t how_many_results = previous_results_.size();
+  const int64_t earliest_time = previous_results_.front().time_;
+  const int64_t samples_duration = current_time_ms - earliest_time;
+  if ((how_many_results < minimum_count_) ||
+      (samples_duration < (average_window_duration_ms_ / 4))) {
+    *found_command = previous_top_label_;
+    *score = 0;
+    *is_new_command = false;
+    return kTfLiteOk;
+  }
+
+  // Calculate the average score across all the results in the window.
+  int32_t average_scores[kCategoryCount];
+  for (int offset = 0; offset < previous_results_.size(); ++offset) {
+    PreviousResultsQueue::Result previous_result =
+        previous_results_.from_front(offset);
+    const int8_t* scores = previous_result.scores;
+    for (int i = 0; i < kCategoryCount; ++i) {
+      if (offset == 0) {
+        average_scores[i] = scores[i] + 128;
+      } else {
+        average_scores[i] += scores[i] + 128;
+      }
+    }
+  }
+  for (int i = 0; i < kCategoryCount; ++i) {
+    average_scores[i] /= how_many_results;
+  }
+
+  // Find the current highest scoring category.
+  int current_top_index = 0;
+  int32_t current_top_score = 0;
+  for (int i = 0; i < kCategoryCount; ++i) {
+    if (average_scores[i] > current_top_score) {
+      current_top_score = average_scores[i];
+      current_top_index = i;
+    }
+  }
+  const char* current_top_label = kCategoryLabels[current_top_index];
+
+  // If we've recently had another label trigger, assume one that occurs too
+  // soon afterwards is a bad result.
+  int64_t time_since_last_top;
+  if ((previous_top_label_ == kCategoryLabels[0]) ||
+      (previous_top_label_time_ == std::numeric_limits<int32_t>::min())) {
+    time_since_last_top = std::numeric_limits<int32_t>::max();
+  } else {
+    time_since_last_top = current_time_ms - previous_top_label_time_;
+  }
+  if ((current_top_score > detection_threshold_) &&
+      ((current_top_label != previous_top_label_) ||
+       (time_since_last_top > suppression_ms_))) {
+    previous_top_label_ = current_top_label;
+    previous_top_label_time_ = current_time_ms;
+    *is_new_command = true;
+  } else {
+    *is_new_command = false;
+  }
+  *found_command = current_top_label;
+  *score = current_top_score;
+
+  return kTfLiteOk;
+}

libraries/TFLiteMicro/examples/micro_speech/recognize_commands.h

+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_LITE_MICRO_EXAMPLES_MICRO_SPEECH_RECOGNIZE_COMMANDS_H_
+#define TENSORFLOW_LITE_MICRO_EXAMPLES_MICRO_SPEECH_RECOGNIZE_COMMANDS_H_
+
+#include <cstdint>
+
+#include "tensorflow/lite/c/common.h"
+#include "micro_model_settings.h"
+#include "tensorflow/lite/micro/micro_log.h"
+
+// Partial implementation of std::dequeue, just providing the functionality
+// that's needed to keep a record of previous neural network results over a
+// short time period, so they can be averaged together to produce a more
+// accurate overall prediction. This doesn't use any dynamic memory allocation
+// so it's a better fit for microcontroller applications, but this does mean
+// there are hard limits on the number of results it can store.
+class PreviousResultsQueue {
+ public:
+  PreviousResultsQueue() : front_index_(0), size_(0) {}
+
+  // Data structure that holds an inference result, and the time when it
+  // was recorded.
+  struct Result {
+    Result() : time_(0), scores() {}
+    Result(int32_t time, int8_t* input_scores) : time_(time) {
+      for (int i = 0; i < kCategoryCount; ++i) {
+        scores[i] = input_scores[i];
+      }
+    }
+    int32_t time_;
+    int8_t scores[kCategoryCount];
+  };
+
+  int size() { return size_; }
+  bool empty() { return size_ == 0; }
+  Result& front() { return results_[front_index_]; }
+  Result& back() {
+    int back_index = front_index_ + (size_ - 1);
+    if (back_index >= kMaxResults) {
+      back_index -= kMaxResults;
+    }
+    return results_[back_index];
+  }
+
+  void push_back(const Result& entry) {
+    if (size() >= kMaxResults) {
+      MicroPrintf("Couldn't push_back latest result, too many already!");
+      return;
+    }
+    size_ += 1;
+    back() = entry;
+  }
+
+  Result pop_front() {
+    if (size() <= 0) {
+      MicroPrintf("Couldn't pop_front result, none present!");
+      return Result();
+    }
+    Result result = front();
+    front_index_ += 1;
+    if (front_index_ >= kMaxResults) {
+      front_index_ = 0;
+    }
+    size_ -= 1;
+    return result;
+  }
+
+  // Most of the functions are duplicates of dequeue containers, but this
+  // is a helper that makes it easy to iterate through the contents of the
+  // queue.
+  Result& from_front(int offset) {
+    if ((offset < 0) || (offset >= size_)) {
+      MicroPrintf("Attempt to read beyond the end of the queue!");
+      offset = size_ - 1;
+    }
+    int index = front_index_ + offset;
+    if (index >= kMaxResults) {
+      index -= kMaxResults;
+    }
+    return results_[index];
+  }
+
+ private:
+  static constexpr int kMaxResults = 50;
+  Result results_[kMaxResults];
+
+  int front_index_;
+  int size_;
+};
+
+// This class is designed to apply a very primitive decoding model on top of the
+// instantaneous results from running an audio recognition model on a single
+// window of samples. It applies smoothing over time so that noisy individual
+// label scores are averaged, increasing the confidence that apparent matches
+// are real.
+// To use it, you should create a class object with the configuration you
+// want, and then feed results from running a TensorFlow model into the
+// processing method. The timestamp for each subsequent call should be
+// increasing from the previous, since the class is designed to process a stream
+// of data over time.
+class RecognizeCommands {
+ public:
+  // labels should be a list of the strings associated with each one-hot score.
+  // The window duration controls the smoothing. Longer durations will give a
+  // higher confidence that the results are correct, but may miss some commands.
+  // The detection threshold has a similar effect, with high values increasing
+  // the precision at the cost of recall. The minimum count controls how many
+  // results need to be in the averaging window before it's seen as a reliable
+  // average. This prevents erroneous results when the averaging window is
+  // initially being populated for example. The suppression argument disables
+  // further recognitions for a set time after one has been triggered, which can
+  // help reduce spurious recognitions.
+  explicit RecognizeCommands(int32_t average_window_duration_ms = 1000,
+                             uint8_t detection_threshold = 200,
+                             int32_t suppression_ms = 1500,
+                             int32_t minimum_count = 3);
+
+  // Call this with the results of running a model on sample data.
+  TfLiteStatus ProcessLatestResults(const TfLiteTensor* latest_results,
+                                    const int32_t current_time_ms,
+                                    const char** found_command, uint8_t* score,
+                                    bool* is_new_command);
+
+ private:
+  // Configuration
+  int32_t average_window_duration_ms_;
+  uint8_t detection_threshold_;
+  int32_t suppression_ms_;
+  int32_t minimum_count_;
+
+  // Working variables
+  PreviousResultsQueue previous_results_;
+  const char* previous_top_label_;
+  int32_t previous_top_label_time_;
+};
+
+#endif  // TENSORFLOW_LITE_MICRO_EXAMPLES_MICRO_SPEECH_RECOGNIZE_COMMANDS_H_

libraries/TFLiteMicro/examples/micro_speech/ringbuf.c

+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "ringbuf.h"
+
+#include <esp_heap_caps.h>
+#include <sdkconfig.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "esp_err.h"
+#include "esp_log.h"
+#include "freertos/FreeRTOS.h"
+#include "freertos/queue.h"
+#include "freertos/semphr.h"
+#include "freertos/task.h"
+
+#define RB_TAG "RINGBUF"
+
+ringbuf_t* rb_init(const char* name, uint32_t size) {
+  ringbuf_t* r;
+  unsigned char* buf;
+
+  if (size < 2 || !name) {
+    return NULL;
+  }
+
+  r = malloc(sizeof(ringbuf_t));
+  assert(r);
+#if (CONFIG_SPIRAM_SUPPORT && \
+     (CONFIG_SPIRAM_USE_CAPS_ALLOC || CONFIG_SPIRAM_USE_MALLOC))
+  buf = heap_caps_calloc(1, size, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT);
+#else
+  buf = calloc(1, size);
+#endif
+  assert(buf);
+
+  r->name = (char*)name;
+  r->base = r->readptr = r->writeptr = buf;
+  r->fill_cnt = 0;
+  r->size = size;
+
+  r->can_read = xSemaphoreCreateBinary();
+  assert(r->can_read);
+  r->can_write = xSemaphoreCreateBinary();
+  assert(r->can_write);
+  r->lock = xSemaphoreCreateMutex();
+  assert(r->lock);
+
+  r->abort_read = 0;
+  r->abort_write = 0;
+  r->writer_finished = 0;
+  r->reader_unblock = 0;
+
+  return r;
+}
+
+void rb_cleanup(ringbuf_t* rb) {
+  free(rb->base);
+  rb->base = NULL;
+  vSemaphoreDelete(rb->can_read);
+  rb->can_read = NULL;
+  vSemaphoreDelete(rb->can_write);
+  rb->can_write = NULL;
+  vSemaphoreDelete(rb->lock);
+  rb->lock = NULL;
+  free(rb);
+}
+
+/*
+ * @brief: get the number of filled bytes in the buffer
+ */
+ssize_t rb_filled(ringbuf_t* rb) { return rb->fill_cnt; }
+
+/*
+ * @brief: get the number of empty bytes available in the buffer
+ */
+ssize_t rb_available(ringbuf_t* rb) {
+  ESP_LOGD(RB_TAG, "rb leftover %d bytes", rb->size - rb->fill_cnt);
+  return (rb->size - rb->fill_cnt);
+}
+
+int rb_read(ringbuf_t* rb, uint8_t* buf, int buf_len, uint32_t ticks_to_wait) {
+  int read_size;
+  int total_read_size = 0;
+
+  /**
+   * In case where we are able to read buf_len in one go,
+   * we are not able to check for abort and keep returning buf_len as bytes
+   * read. Check for argument validity check and abort case before entering
+   * memcpy loop.
+   */
+
+  if (rb == NULL || rb->abort_read == 1) {
+    return ESP_FAIL;
+  }
+
+  xSemaphoreTake(rb->lock, portMAX_DELAY);
+
+  while (buf_len) {
+    if (rb->fill_cnt < buf_len) {
+      read_size = rb->fill_cnt;
+    } else {
+      read_size = buf_len;
+    }
+    if ((rb->readptr + read_size) > (rb->base + rb->size)) {
+      int rlen1 = rb->base + rb->size - rb->readptr;
+      int rlen2 = read_size - rlen1;
+      if (buf) {
+        memcpy(buf, rb->readptr, rlen1);
+        memcpy(buf + rlen1, rb->base, rlen2);
+      }
+      rb->readptr = rb->base + rlen2;
+    } else {
+      if (buf) {
+        memcpy(buf, rb->readptr, read_size);
+      }
+      rb->readptr = rb->readptr + read_size;
+    }
+
+    buf_len -= read_size;
+    rb->fill_cnt -= read_size;
+    total_read_size += read_size;
+    if (buf) {
+      buf += read_size;
+    }
+
+    xSemaphoreGive(rb->can_write);
+
+    if (buf_len == 0) {
+      break;
+    }
+
+    xSemaphoreGive(rb->lock);
+    if (!rb->writer_finished && !rb->abort_read && !rb->reader_unblock) {
+      if (xSemaphoreTake(rb->can_read, ticks_to_wait) != pdTRUE) {
+        goto out;
+      }
+    }
+    if (rb->abort_read == 1) {
+      total_read_size = RB_ABORT;
+      goto out;
+    }
+    if (rb->writer_finished == 1) {
+      goto out;
+    }
+    if (rb->reader_unblock == 1) {
+      if (total_read_size == 0) {
+        total_read_size = RB_READER_UNBLOCK;
+      }
+      goto out;
+    }
+
+    xSemaphoreTake(rb->lock, portMAX_DELAY);
+  }
+
+  xSemaphoreGive(rb->lock);
+out:
+  if (rb->writer_finished == 1 && total_read_size == 0) {
+    total_read_size = RB_WRITER_FINISHED;
+  }
+  rb->reader_unblock = 0; /* We are anyway unblocking reader */
+  return total_read_size;
+}
+
+int rb_write(ringbuf_t* rb, const uint8_t* buf, int buf_len,
+             uint32_t ticks_to_wait) {
+  int write_size;
+  int total_write_size = 0;
+
+  /**
+   * In case where we are able to write buf_len in one go,
+   * we are not able to check for abort and keep returning buf_len as bytes
+   * written. Check for arguments' validity and abort case before entering
+   * memcpy loop.
+   */
+
+  if (rb == NULL || buf == NULL || rb->abort_write == 1) {
+    return RB_FAIL;
+  }
+
+  xSemaphoreTake(rb->lock, portMAX_DELAY);
+
+  while (buf_len) {
+    if ((rb->size - rb->fill_cnt) < buf_len) {
+      write_size = rb->size - rb->fill_cnt;
+    } else {
+      write_size = buf_len;
+    }
+    if ((rb->writeptr + write_size) > (rb->base + rb->size)) {
+      int wlen1 = rb->base + rb->size - rb->writeptr;
+      int wlen2 = write_size - wlen1;
+      memcpy(rb->writeptr, buf, wlen1);
+      memcpy(rb->base, buf + wlen1, wlen2);
+      rb->writeptr = rb->base + wlen2;
+    } else {
+      memcpy(rb->writeptr, buf, write_size);
+      rb->writeptr = rb->writeptr + write_size;
+    }
+
+    buf_len -= write_size;
+    rb->fill_cnt += write_size;
+    total_write_size += write_size;
+    buf += write_size;
+
+    xSemaphoreGive(rb->can_read);
+
+    if (buf_len == 0) {
+      break;
+    }
+
+    xSemaphoreGive(rb->lock);
+    if (rb->writer_finished) {
+      return write_size > 0 ? write_size : RB_WRITER_FINISHED;
+    }
+    if (xSemaphoreTake(rb->can_write, ticks_to_wait) != pdTRUE) {
+      goto out;
+    }
+    if (rb->abort_write == 1) {
+      goto out;
+    }
+    xSemaphoreTake(rb->lock, portMAX_DELAY);
+  }
+
+  xSemaphoreGive(rb->lock);
+out:
+  return total_write_size;
+}
+
+/**
+ * abort and set abort_read and abort_write to asked values.
+ */
+static void _rb_reset(ringbuf_t* rb, int abort_read, int abort_write) {
+  if (rb == NULL) {
+    return;
+  }
+  xSemaphoreTake(rb->lock, portMAX_DELAY);
+  rb->readptr = rb->writeptr = rb->base;
+  rb->fill_cnt = 0;
+  rb->writer_finished = 0;
+  rb->reader_unblock = 0;
+  rb->abort_read = abort_read;
+  rb->abort_write = abort_write;
+  xSemaphoreGive(rb->lock);
+}
+
+void rb_reset(ringbuf_t* rb) { _rb_reset(rb, 0, 0); }
+
+void rb_abort_read(ringbuf_t* rb) {
+  if (rb == NULL) {
+    return;
+  }
+  rb->abort_read = 1;
+  xSemaphoreGive(rb->can_read);
+  xSemaphoreGive(rb->lock);
+}
+
+void rb_abort_write(ringbuf_t* rb) {
+  if (rb == NULL) {
+    return;
+  }
+  rb->abort_write = 1;
+  xSemaphoreGive(rb->can_write);
+  xSemaphoreGive(rb->lock);
+}
+
+void rb_abort(ringbuf_t* rb) {
+  if (rb == NULL) {
+    return;
+  }
+  rb->abort_read = 1;
+  rb->abort_write = 1;
+  xSemaphoreGive(rb->can_read);
+  xSemaphoreGive(rb->can_write);
+  xSemaphoreGive(rb->lock);
+}
+
+/**
+ * Reset the ringbuffer and keep rb_write aborted.
+ * Note that we are taking lock before even toggling `abort_write` variable.
+ * This serves a special purpose to not allow this abort to be mixed with
+ * rb_write.
+ */
+void rb_reset_and_abort_write(ringbuf_t* rb) {
+  _rb_reset(rb, 0, 1);
+  xSemaphoreGive(rb->can_write);
+}
+
+void rb_signal_writer_finished(ringbuf_t* rb) {
+  if (rb == NULL) {
+    return;
+  }
+  rb->writer_finished = 1;
+  xSemaphoreGive(rb->can_read);
+}
+
+int rb_is_writer_finished(ringbuf_t* rb) {
+  if (rb == NULL) {
+    return RB_FAIL;
+  }
+  return (rb->writer_finished);
+}
+
+void rb_wakeup_reader(ringbuf_t* rb) {
+  if (rb == NULL) {
+    return;
+  }
+  rb->reader_unblock = 1;
+  xSemaphoreGive(rb->can_read);
+}
+
+void rb_stat(ringbuf_t* rb) {
+  xSemaphoreTake(rb->lock, portMAX_DELAY);
+  ESP_LOGI(RB_TAG,
+           "filled: %d, base: %p, read_ptr: %p, write_ptr: %p, size: %d\n",
+           rb->fill_cnt, rb->base, rb->readptr, rb->writeptr, rb->size);
+  xSemaphoreGive(rb->lock);
+}

libraries/TFLiteMicro/examples/micro_speech/ringbuf.h

+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_LITE_MICRO_EXAMPLES_MICRO_SPEECH_ESP_RINGBUF_H_
+#define TENSORFLOW_LITE_MICRO_EXAMPLES_MICRO_SPEECH_ESP_RINGBUF_H_
+
+#include <freertos/FreeRTOS.h>
+#include <freertos/semphr.h>
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define RB_FAIL ESP_FAIL
+#define RB_ABORT -1
+#define RB_WRITER_FINISHED -2
+#define RB_READER_UNBLOCK -3
+
+#if __has_include("esp_idf_version.h")
+#include "esp_idf_version.h"
+#else
+#define ESP_IDF_VERSION_VAL(major, minor, patch) 0
+#endif
+
+#if (ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(5, 0, 0))
+#if !(configENABLE_BACKWARD_COMPATIBILITY == 1)
+#define xSemaphoreHandle SemaphoreHandle_t
+#endif
+#endif
+
+typedef struct ringbuf {
+  char* name;
+  uint8_t* base; /**< Original pointer */
+  /* XXX: these need to be volatile? */
+  uint8_t* volatile readptr;  /**< Read pointer */
+  uint8_t* volatile writeptr; /**< Write pointer */
+  volatile ssize_t fill_cnt;  /**< Number of filled slots */
+  ssize_t size;               /**< Buffer size */
+  xSemaphoreHandle can_read;
+  xSemaphoreHandle can_write;
+  xSemaphoreHandle lock;
+  int abort_read;
+  int abort_write;
+  int writer_finished;  // to prevent infinite blocking for buffer read
+  int reader_unblock;
+} ringbuf_t;
+
+ringbuf_t* rb_init(const char* rb_name, uint32_t size);
+void rb_abort_read(ringbuf_t* rb);
+void rb_abort_write(ringbuf_t* rb);
+void rb_abort(ringbuf_t* rb);
+void rb_reset(ringbuf_t* rb);
+/**
+ * @brief Special function to reset the buffer while keeping rb_write aborted.
+ *        This rb needs to be reset again before being useful.
+ */
+void rb_reset_and_abort_write(ringbuf_t* rb);
+void rb_stat(ringbuf_t* rb);
+ssize_t rb_filled(ringbuf_t* rb);
+ssize_t rb_available(ringbuf_t* rb);
+int rb_read(ringbuf_t* rb, uint8_t* buf, int len, uint32_t ticks_to_wait);
+int rb_write(ringbuf_t* rb, const uint8_t* buf, int len,
+             uint32_t ticks_to_wait);
+void rb_cleanup(ringbuf_t* rb);
+void rb_signal_writer_finished(ringbuf_t* rb);
+void rb_wakeup_reader(ringbuf_t* rb);
+int rb_is_writer_finished(ringbuf_t* rb);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TENSORFLOW_LITE_MICRO_EXAMPLES_MICRO_SPEECH_ESP_RINGBUF_H_

libraries/TFLiteMicro/library.properties

+name=TFLite Micro
+version=1.0.0
+author=Sanket Wadekar
+maintainer=Sanket Wadekar
+sentence=TensorFlow Lite for Microcontrollers
+paragraph=With this library you can train and deploy Machine Learning models on esp32 series of microcontrollers.
+url=https://www.tensorflow.org/lite/microcontrollers
+architectures=esp32,esp32s3

libraries/TFLiteMicro/src/TFLIteMicro.h

+/*
+ * SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+// This header file is kept to avoid Invalid libary error prints while compiling TFLite Micro examples.

libraries/TFLiteMicro/src/utility.h

+/*
+ * SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+#pragma once
+#include <utility>
+/*
+The example sketches include "tensorflow/lite/micro/micro_interpreter.h" which internally include "utility.h" header file when compiling examples for Arduino (when -DARDUINO flag is passed to the compiler). This header file does not exist in esp32-arduino core. Hence, keeping this file here as a workaround and including an alternate header file.
+*/