Skip to content

GitLab

A
arduino-esp32
Unverified Commit 0c4b35e0 authored by Me No Dev's avatar Me No Dev Committed by GitHub
Browse files
Options

IDF release/v5.1 (#9613) 

* IDF release/v5.1 01b912a9e5

* Fix USB OTG Init on new IDF

* Delete libraries/TFLiteMicro/examples/micro_speech directory

Done in order to fix a CI problem created by an entire folder that was removed in original Library Repository.

* IDF release/v5.1 442a798083

* Update esp32-hal-tinyusb.c

---------
Co-authored-by: default avatarRodrigo Garcia <rodrigo.garcia@espressif.com>
parent e10de73e
Hide whitespace changes
Inline Side-by-side
Showing with 99 additions and 2335 deletions
CMakeLists.txt
View file @ 0c4b35e0
......@@ -288,7 +288,7 @@ set(includedirs variants/${CONFIG_ARDUINO_VARIANT}/ cores/esp32/ ${ARDUINO_LIBRA
set(srcs ${CORE_SRCS} ${ARDUINO_LIBRARIES_SRCS})
set(priv_includes cores/esp32/libb64)
set(requires spi_flash esp_partition mbedtls wifi_provisioning wpa_supplicant esp_adc esp_eth http_parser)
set(priv_requires fatfs nvs_flash app_update spiffs bootloader_support bt esp_hid ${ARDUINO_LIBRARIES_REQUIRES})
set(priv_requires fatfs nvs_flash app_update spiffs bootloader_support bt esp_hid usb ${ARDUINO_LIBRARIES_REQUIRES})
idf_component_register(INCLUDE_DIRS ${includedirs} PRIV_INCLUDE_DIRS ${priv_includes} SRCS ${srcs} REQUIRES ${requires} PRIV_REQUIRES ${priv_requires})
......
cores/esp32/esp32-hal-tinyusb.c
View file @ 0c4b35e0
......@@ -22,7 +22,6 @@
#include "rom/gpio.h"
#include "hal/usb_hal.h"
#include "hal/gpio_ll.h"
#include "hal/clk_gate_ll.h"
......@@ -63,6 +62,10 @@ typedef struct {
bool external_phy;
} tinyusb_config_t;
#if __has_include("hal/usb_hal.h")
#include "hal/usb_hal.h"
static bool usb_otg_deinit(void *busptr) {
// Once USB OTG is initialized, its GPIOs are assigned and it shall never be deinited
// except when S3 swithicng usb from cdc to jtag while resetting to bootrom
......@@ -101,10 +104,67 @@ static void configure_pins(usb_hal_context_t *usb) {
}
}
esp_err_t tinyusb_driver_install(const tinyusb_config_t *config) {
usb_hal_context_t hal = {.use_external_phy = config->external_phy};
esp_err_t init_usb_hal(bool external_phy) {
usb_hal_context_t hal = {.use_external_phy = external_phy};
usb_hal_init(&hal);
configure_pins(&hal);
return ESP_OK;
}
esp_err_t deinit_usb_hal() {
return ESP_OK;
}
#elif __has_include("esp_private/usb_phy.h")
#include "esp_private/usb_phy.h"
static usb_phy_handle_t phy_handle = NULL;
esp_err_t init_usb_hal(bool external_phy) {
esp_err_t ret = ESP_OK;
usb_phy_config_t phy_config = {
.controller = USB_PHY_CTRL_OTG,
.target = USB_PHY_TARGET_INT,
.otg_mode = USB_OTG_MODE_DEVICE,
.otg_speed = USB_PHY_SPEED_FULL,
.ext_io_conf = NULL,
.otg_io_conf = NULL,
};
ret = usb_new_phy(&phy_config, &phy_handle);
if (ret != ESP_OK) {
log_e("Failed to init USB PHY");
}
return ret;
}
esp_err_t deinit_usb_hal() {
esp_err_t ret = ESP_OK;
if (phy_handle) {
ret = usb_del_phy(phy_handle);
if (ret != ESP_OK) {
log_e("Failed to deinit USB PHY");
}
}
return ret;
}
#else
#error No way to initialize USP PHY
void init_usb_hal(bool external_phy) {
return ESP_OK;
}
void deinit_usb_hal() {
return ESP_OK;
}
#endif
esp_err_t tinyusb_driver_install(const tinyusb_config_t *config) {
init_usb_hal(config->external_phy);
if (!tusb_init()) {
log_e("Can't initialize the TinyUSB stack.");
return ESP_FAIL;
......@@ -420,6 +480,7 @@ static void hw_cdc_reset_handler(void *arg) {
static void usb_switch_to_cdc_jtag() {
// Disable USB-OTG
deinit_usb_hal();
periph_ll_reset(PERIPH_USB_MODULE);
//periph_ll_enable_clk_clear_rst(PERIPH_USB_MODULE);
periph_ll_disable_clk_set_rst(PERIPH_USB_MODULE);
......
libraries/TFLiteMicro/examples/micro_speech/README.md deleted 100644 → 0
View file @ e10de73e
# Micro Speech Example
This example shows how to run a 20 kB model that can recognize 2 keywords,
"yes" and "no", from speech data.
The application listens to its surroundings with a microphone and indicates
when it has detected a word by displaying data on a screen.
## Deploy to ESP32
The sample has been tested on ESP-IDF version `release/v4.2` and `release/v4.4` with the following devices:
- [ESP32-DevKitC](http://esp-idf.readthedocs.io/en/latest/get-started/get-started-devkitc.html)
- [ESP32-S3-DevKitC](https://docs.espressif.com/projects/esp-idf/en/latest/esp32s3/hw-reference/esp32s3/user-guide-devkitc-1.html)
- [ESP-EYE](https://github.com/espressif/esp-who/blob/master/docs/en/get-started/ESP-EYE_Getting_Started_Guide.md)
### Sample output
* When a keyword is detected you will see following output sample output on the log screen:
```
Heard yes (<score>) at <time>
```
libraries/TFLiteMicro/examples/micro_speech/audio_provider.cpp deleted 100644 → 0
View file @ e10de73e
/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "audio_provider.h"
#include <cstdlib>
#include <cstring>
// FreeRTOS.h must be included before some of the following dependencies.
// Solves b/150260343.
// clang-format off
#include "freertos/FreeRTOS.h"
// clang-format on
#include "driver/i2s.h"
#include "esp_log.h"
#include "esp_spi_flash.h"
#include "esp_system.h"
#include "esp_timer.h"
#include "freertos/task.h"
#include "ringbuf.h"
#include "micro_model_settings.h"
using namespace std;
#define NO_I2S_SUPPORT CONFIG_IDF_TARGET_ESP32C2 || (CONFIG_IDF_TARGET_ESP32C3 && (ESP_IDF_VERSION < ESP_IDF_VERSION_VAL(4, 4, 0)))
static const char *TAG = "TF_LITE_AUDIO_PROVIDER";
/* ringbuffer to hold the incoming audio data */
ringbuf_t *g_audio_capture_buffer;
volatile int32_t g_latest_audio_timestamp = 0;
/* model requires 20ms new data from g_audio_capture_buffer and 10ms old data
* each time , storing old data in the histrory buffer , {
* history_samples_to_keep = 10 * 16 } */
constexpr int32_t history_samples_to_keep = ((kFeatureSliceDurationMs - kFeatureSliceStrideMs) * (kAudioSampleFrequency / 1000));
/* new samples to get each time from ringbuffer, { new_samples_to_get = 20 * 16
* } */
constexpr int32_t new_samples_to_get = (kFeatureSliceStrideMs * (kAudioSampleFrequency / 1000));
namespace {
int16_t g_audio_output_buffer[kMaxAudioSampleSize];
bool g_is_audio_initialized = false;
int16_t g_history_buffer[history_samples_to_keep];
} // namespace
const int32_t kAudioCaptureBufferSize = 80000;
const int32_t i2s_bytes_to_read = 3200;
#if NO_I2S_SUPPORT
// nothing to be done here
#else
static void i2s_init(void) {
// Start listening for audio: MONO @ 16KHz
i2s_config_t i2s_config = {
.mode = (i2s_mode_t)(I2S_MODE_MASTER | I2S_MODE_RX | I2S_MODE_TX),
.sample_rate = 16000,
.bits_per_sample = (i2s_bits_per_sample_t)16,
.channel_format = I2S_CHANNEL_FMT_ONLY_LEFT,
.communication_format = I2S_COMM_FORMAT_I2S,
.intr_alloc_flags = 0,
.dma_buf_count = 3,
.dma_buf_len = 300,
.use_apll = false,
.tx_desc_auto_clear = false,
.fixed_mclk = -1,
};
i2s_pin_config_t pin_config = {
.bck_io_num = 26, // IIS_SCLK
.ws_io_num = 32, // IIS_LCLK
.data_out_num = -1, // IIS_DSIN
.data_in_num = 33, // IIS_DOUT
};
esp_err_t ret = 0;
ret = i2s_driver_install((i2s_port_t)1, &i2s_config, 0, NULL);
if (ret != ESP_OK) {
ESP_LOGE(TAG, "Error in i2s_driver_install");
}
ret = i2s_set_pin((i2s_port_t)1, &pin_config);
if (ret != ESP_OK) {
ESP_LOGE(TAG, "Error in i2s_set_pin");
}
ret = i2s_zero_dma_buffer((i2s_port_t)1);
if (ret != ESP_OK) {
ESP_LOGE(TAG, "Error in initializing dma buffer with 0");
}
}
#endif
static void CaptureSamples(void *arg) {
#if NO_I2S_SUPPORT
ESP_LOGE(TAG, "i2s support not available on C3 chip for IDF < 4.4.0");
return;
#else
size_t bytes_read = i2s_bytes_to_read;
uint8_t i2s_read_buffer[i2s_bytes_to_read] = {};
i2s_init();
while (1) {
/* read 100ms data at once from i2s */
i2s_read((i2s_port_t)1, (void *)i2s_read_buffer, i2s_bytes_to_read, &bytes_read, pdMS_TO_TICKS(100));
if (bytes_read <= 0) {
ESP_LOGE(TAG, "Error in I2S read : %d", bytes_read);
} else {
if (bytes_read < i2s_bytes_to_read) {
ESP_LOGW(TAG, "Partial I2S read");
}
/* write bytes read by i2s into ring buffer */
int bytes_written = rb_write(g_audio_capture_buffer, (uint8_t *)i2s_read_buffer, bytes_read, pdMS_TO_TICKS(100));
/* update the timestamp (in ms) to let the model know that new data has
* arrived */
g_latest_audio_timestamp = g_latest_audio_timestamp + ((1000 * (bytes_written / 2)) / kAudioSampleFrequency);
if (bytes_written <= 0) {
ESP_LOGE(TAG, "Could Not Write in Ring Buffer: %d ", bytes_written);
} else if (bytes_written < bytes_read) {
ESP_LOGW(TAG, "Partial Write");
}
}
}
#endif
vTaskDelete(NULL);
}
TfLiteStatus InitAudioRecording() {
g_audio_capture_buffer = rb_init("tf_ringbuffer", kAudioCaptureBufferSize);
if (!g_audio_capture_buffer) {
ESP_LOGE(TAG, "Error creating ring buffer");
return kTfLiteError;
}
/* create CaptureSamples Task which will get the i2s_data from mic and fill it
* in the ring buffer */
xTaskCreate(CaptureSamples, "CaptureSamples", 1024 * 32, NULL, 10, NULL);
while (!g_latest_audio_timestamp) {
vTaskDelay(1); // one tick delay to avoid watchdog
}
ESP_LOGI(TAG, "Audio Recording started");
return kTfLiteOk;
}
TfLiteStatus GetAudioSamples(int start_ms, int duration_ms, int *audio_samples_size, int16_t **audio_samples) {
if (!g_is_audio_initialized) {
TfLiteStatus init_status = InitAudioRecording();
if (init_status != kTfLiteOk) {
return init_status;
}
g_is_audio_initialized = true;
}
/* copy 160 samples (320 bytes) into output_buff from history */
memcpy((void *)(g_audio_output_buffer), (void *)(g_history_buffer), history_samples_to_keep * sizeof(int16_t));
/* copy 320 samples (640 bytes) from rb at ( int16_t*(g_audio_output_buffer) +
* 160 ), first 160 samples (320 bytes) will be from history */
int bytes_read =
rb_read(g_audio_capture_buffer, ((uint8_t *)(g_audio_output_buffer + history_samples_to_keep)), new_samples_to_get * sizeof(int16_t), pdMS_TO_TICKS(100));
if (bytes_read < 0) {
ESP_LOGE(TAG, " Model Could not read data from Ring Buffer");
} else if (bytes_read < new_samples_to_get * sizeof(int16_t)) {
ESP_LOGD(TAG, "RB FILLED RIGHT NOW IS %d", rb_filled(g_audio_capture_buffer));
ESP_LOGD(TAG, " Partial Read of Data by Model ");
ESP_LOGV(TAG, " Could only read %d bytes when required %d bytes ", bytes_read, (int)(new_samples_to_get * sizeof(int16_t)));
}
/* copy 320 bytes from output_buff into history */
memcpy((void *)(g_history_buffer), (void *)(g_audio_output_buffer + new_samples_to_get), history_samples_to_keep * sizeof(int16_t));
*audio_samples_size = kMaxAudioSampleSize;
*audio_samples = g_audio_output_buffer;
return kTfLiteOk;
}
int32_t LatestAudioTimestamp() {
return g_latest_audio_timestamp;
}
libraries/TFLiteMicro/examples/micro_speech/audio_provider.h deleted 100644 → 0
View file @ e10de73e
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_LITE_MICRO_EXAMPLES_MICRO_SPEECH_AUDIO_PROVIDER_H_
#define TENSORFLOW_LITE_MICRO_EXAMPLES_MICRO_SPEECH_AUDIO_PROVIDER_H_
#include "tensorflow/lite/c/common.h"
// This is an abstraction around an audio source like a microphone, and is
// expected to return 16-bit PCM sample data for a given point in time. The
// sample data itself should be used as quickly as possible by the caller, since
// to allow memory optimizations there are no guarantees that the samples won't
// be overwritten by new data in the future. In practice, implementations should
// ensure that there's a reasonable time allowed for clients to access the data
// before any reuse.
// The reference implementation can have no platform-specific dependencies, so
// it just returns an array filled with zeros. For real applications, you should
// ensure there's a specialized implementation that accesses hardware APIs.
TfLiteStatus GetAudioSamples(int start_ms, int duration_ms, int *audio_samples_size, int16_t **audio_samples);
// Returns the time that audio data was last captured in milliseconds. There's
// no contract about what time zero represents, the accuracy, or the granularity
// of the result. Subsequent calls will generally not return a lower value, but
// even that's not guaranteed if there's an overflow wraparound.
// The reference implementation of this function just returns a constantly
// incrementing value for each call, since it would need a non-portable platform
// call to access time information. For real applications, you'll need to write
// your own platform-specific implementation.
int32_t LatestAudioTimestamp();
#endif // TENSORFLOW_LITE_MICRO_EXAMPLES_MICRO_SPEECH_AUDIO_PROVIDER_H_
libraries/TFLiteMicro/examples/micro_speech/command_responder.cpp deleted 100644 → 0
View file @ e10de73e
/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "command_responder.h"
#include "tensorflow/lite/micro/micro_log.h"
// The default implementation writes out the name of the recognized command
// to the error console. Real applications will want to take some custom
// action instead, and should implement their own versions of this function.
void RespondToCommand(int32_t current_time, const char *found_command, uint8_t score, bool is_new_command) {
if (is_new_command) {
MicroPrintf("Heard %s (%d) @%dms", found_command, score, current_time);
}
}
libraries/TFLiteMicro/examples/micro_speech/command_responder.h deleted 100644 → 0
View file @ e10de73e
/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// Provides an interface to take an action based on an audio command.
#ifndef TENSORFLOW_LITE_MICRO_EXAMPLES_MICRO_SPEECH_COMMAND_RESPONDER_H_
#define TENSORFLOW_LITE_MICRO_EXAMPLES_MICRO_SPEECH_COMMAND_RESPONDER_H_
#include "tensorflow/lite/c/common.h"
// Called every time the results of an audio recognition run are available. The
// human-readable name of any recognized command is in the `found_command`
// argument, `score` has the numerical confidence, and `is_new_command` is set
// if the previous command was different to this one.
void RespondToCommand(int32_t current_time, const char *found_command, uint8_t score, bool is_new_command);
#endif // TENSORFLOW_LITE_MICRO_EXAMPLES_MICRO_SPEECH_COMMAND_RESPONDER_H_
libraries/TFLiteMicro/examples/micro_speech/feature_provider.cpp deleted 100644 → 0
View file @ e10de73e
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "feature_provider.h"
#include "audio_provider.h"
#include "micro_features_generator.h"
#include "micro_model_settings.h"
#include "tensorflow/lite/micro/micro_log.h"
FeatureProvider::FeatureProvider(int feature_size, int8_t *feature_data) : feature_size_(feature_size), feature_data_(feature_data), is_first_run_(true) {
// Initialize the feature data to default values.
for (int n = 0; n < feature_size_; ++n) {
feature_data_[n] = 0;
}
}
FeatureProvider::~FeatureProvider() {}
TfLiteStatus FeatureProvider::PopulateFeatureData(int32_t last_time_in_ms, int32_t time_in_ms, int *how_many_new_slices) {
if (feature_size_ != kFeatureElementCount) {
MicroPrintf("Requested feature_data_ size %d doesn't match %d", feature_size_, kFeatureElementCount);
return kTfLiteError;
}
// Quantize the time into steps as long as each window stride, so we can
// figure out which audio data we need to fetch.
const int last_step = (last_time_in_ms / kFeatureSliceStrideMs);
const int current_step = (time_in_ms / kFeatureSliceStrideMs);
int slices_needed = current_step - last_step;
// If this is the first call, make sure we don't use any cached information.
if (is_first_run_) {
TfLiteStatus init_status = InitializeMicroFeatures();
if (init_status != kTfLiteOk) {
return init_status;
}
is_first_run_ = false;
slices_needed = kFeatureSliceCount;
}
if (slices_needed > kFeatureSliceCount) {
slices_needed = kFeatureSliceCount;
}
*how_many_new_slices = slices_needed;
const int slices_to_keep = kFeatureSliceCount - slices_needed;
const int slices_to_drop = kFeatureSliceCount - slices_to_keep;
// If we can avoid recalculating some slices, just move the existing data
// up in the spectrogram, to perform something like this:
// last time = 80ms current time = 120ms
// +-----------+ +-----------+
// | data@20ms | --> | data@60ms |
// +-----------+ -- +-----------+
// | data@40ms | -- --> | data@80ms |
// +-----------+ -- -- +-----------+
// | data@60ms | -- -- | <empty> |
// +-----------+ -- +-----------+
// | data@80ms | -- | <empty> |
// +-----------+ +-----------+
if (slices_to_keep > 0) {
for (int dest_slice = 0; dest_slice < slices_to_keep; ++dest_slice) {
int8_t *dest_slice_data = feature_data_ + (dest_slice * kFeatureSliceSize);
const int src_slice = dest_slice + slices_to_drop;
const int8_t *src_slice_data = feature_data_ + (src_slice * kFeatureSliceSize);
for (int i = 0; i < kFeatureSliceSize; ++i) {
dest_slice_data[i] = src_slice_data[i];
}
}
}
// Any slices that need to be filled in with feature data have their
// appropriate audio data pulled, and features calculated for that slice.
if (slices_needed > 0) {
for (int new_slice = slices_to_keep; new_slice < kFeatureSliceCount; ++new_slice) {
const int new_step = (current_step - kFeatureSliceCount + 1) + new_slice;
const int32_t slice_start_ms = (new_step * kFeatureSliceStrideMs);
int16_t *audio_samples = nullptr;
int audio_samples_size = 0;
// TODO(petewarden): Fix bug that leads to non-zero slice_start_ms
GetAudioSamples((slice_start_ms > 0 ? slice_start_ms : 0), kFeatureSliceDurationMs, &audio_samples_size, &audio_samples);
if (audio_samples_size < kMaxAudioSampleSize) {
MicroPrintf("Audio data size %d too small, want %d", audio_samples_size, kMaxAudioSampleSize);
return kTfLiteError;
}
int8_t *new_slice_data = feature_data_ + (new_slice * kFeatureSliceSize);
size_t num_samples_read;
TfLiteStatus generate_status = GenerateMicroFeatures(audio_samples, audio_samples_size, kFeatureSliceSize, new_slice_data, &num_samples_read);
if (generate_status != kTfLiteOk) {
return generate_status;
}
}
}
return kTfLiteOk;
}
libraries/TFLiteMicro/examples/micro_speech/feature_provider.h deleted 100644 → 0
View file @ e10de73e
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_LITE_MICRO_EXAMPLES_MICRO_SPEECH_FEATURE_PROVIDER_H_
#define TENSORFLOW_LITE_MICRO_EXAMPLES_MICRO_SPEECH_FEATURE_PROVIDER_H_
#include "tensorflow/lite/c/common.h"
// Binds itself to an area of memory intended to hold the input features for an
// audio-recognition neural network model, and fills that data area with the
// features representing the current audio input, for example from a microphone.
// The audio features themselves are a two-dimensional array, made up of
// horizontal slices representing the frequencies at one point in time, stacked
// on top of each other to form a spectrogram showing how those frequencies
// changed over time.
class FeatureProvider {
public:
// Create the provider, and bind it to an area of memory. This memory should
// remain accessible for the lifetime of the provider object, since subsequent
// calls will fill it with feature data. The provider does no memory
// management of this data.
FeatureProvider(int feature_size, int8_t *feature_data);
~FeatureProvider();
// Fills the feature data with information from audio inputs, and returns how
// many feature slices were updated.
TfLiteStatus PopulateFeatureData(int32_t last_time_in_ms, int32_t time_in_ms, int *how_many_new_slices);
private:
int feature_size_;
int8_t *feature_data_;
// Make sure we don't try to use cached information if this is the first call
// into the provider.
bool is_first_run_;
};
#endif // TENSORFLOW_LITE_MICRO_EXAMPLES_MICRO_SPEECH_FEATURE_PROVIDER_H_
libraries/TFLiteMicro/examples/micro_speech/micro_features_generator.cpp deleted 100644 → 0
View file @ e10de73e
/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "micro_features_generator.h"
#include <cmath>
#include <cstring>
#include "tensorflow/lite/experimental/microfrontend/lib/frontend.h"
#include "tensorflow/lite/experimental/microfrontend/lib/frontend_util.h"
#include "tensorflow/lite/micro/micro_log.h"
#include "micro_model_settings.h"
// Configure FFT to output 16 bit fixed point.
#define FIXED_POINT 16
namespace {
FrontendState g_micro_features_state;
bool g_is_first_time = true;
} // namespace
TfLiteStatus InitializeMicroFeatures() {
FrontendConfig config;
config.window.size_ms = kFeatureSliceDurationMs;
config.window.step_size_ms = kFeatureSliceStrideMs;
config.noise_reduction.smoothing_bits = 10;
config.filterbank.num_channels = kFeatureSliceSize;
config.filterbank.lower_band_limit = 125.0;
config.filterbank.upper_band_limit = 7500.0;
config.noise_reduction.smoothing_bits = 10;
config.noise_reduction.even_smoothing = 0.025;
config.noise_reduction.odd_smoothing = 0.06;
config.noise_reduction.min_signal_remaining = 0.05;
config.pcan_gain_control.enable_pcan = 1;
config.pcan_gain_control.strength = 0.95;
config.pcan_gain_control.offset = 80.0;
config.pcan_gain_control.gain_bits = 21;
config.log_scale.enable_log = 1;
config.log_scale.scale_shift = 6;
if (!FrontendPopulateState(&config, &g_micro_features_state, kAudioSampleFrequency)) {
MicroPrintf("FrontendPopulateState() failed");
return kTfLiteError;
}
g_is_first_time = true;
return kTfLiteOk;
}
// This is not exposed in any header, and is only used for testing, to ensure
// that the state is correctly set up before generating results.
void SetMicroFeaturesNoiseEstimates(const uint32_t *estimate_presets) {
for (int i = 0; i < g_micro_features_state.filterbank.num_channels; ++i) {
g_micro_features_state.noise_reduction.estimate[i] = estimate_presets[i];
}
}
TfLiteStatus GenerateMicroFeatures(const int16_t *input, int input_size, int output_size, int8_t *output, size_t *num_samples_read) {
const int16_t *frontend_input;
if (g_is_first_time) {
frontend_input = input;
g_is_first_time = false;
} else {
frontend_input = input + 160;
}
FrontendOutput frontend_output = FrontendProcessSamples(&g_micro_features_state, frontend_input, input_size, num_samples_read);
for (size_t i = 0; i < frontend_output.size; ++i) {
// These scaling values are derived from those used in input_data.py in the
// training pipeline.
// The feature pipeline outputs 16-bit signed integers in roughly a 0 to 670
// range. In training, these are then arbitrarily divided by 25.6 to get
// float values in the rough range of 0.0 to 26.0. This scaling is performed
// for historical reasons, to match up with the output of other feature
// generators.
// The process is then further complicated when we quantize the model. This
// means we have to scale the 0.0 to 26.0 real values to the -128 to 127
// signed integer numbers.
// All this means that to get matching values from our integer feature
// output into the tensor input, we have to perform:
// input = (((feature / 25.6) / 26.0) * 256) - 128
// To simplify this and perform it in 32-bit integer math, we rearrange to:
// input = (feature * 256) / (25.6 * 26.0) - 128
constexpr int32_t value_scale = 256;
constexpr int32_t value_div = static_cast<int32_t>((25.6f * 26.0f) + 0.5f);
int32_t value = ((frontend_output.values[i] * value_scale) + (value_div / 2)) / value_div;
value -= 128;
if (value < -128) {
value = -128;
}
if (value > 127) {
value = 127;
}
output[i] = value;
}
return kTfLiteOk;
}
libraries/TFLiteMicro/examples/micro_speech/micro_features_generator.h deleted 100644 → 0
View file @ e10de73e
/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_LITE_MICRO_EXAMPLES_MICRO_SPEECH_MICRO_FEATURES_MICRO_FEATURES_GENERATOR_H_
#define TENSORFLOW_LITE_MICRO_EXAMPLES_MICRO_SPEECH_MICRO_FEATURES_MICRO_FEATURES_GENERATOR_H_
#include "tensorflow/lite/c/common.h"
// Sets up any resources needed for the feature generation pipeline.
TfLiteStatus InitializeMicroFeatures();
// Converts audio sample data into a more compact form that's appropriate for
// feeding into a neural network.
TfLiteStatus GenerateMicroFeatures(const int16_t *input, int input_size, int output_size, int8_t *output, size_t *num_samples_read);
#endif // TENSORFLOW_LITE_MICRO_EXAMPLES_MICRO_SPEECH_MICRO_FEATURES_MICRO_FEATURES_GENERATOR_H_
libraries/TFLiteMicro/examples/micro_speech/micro_model_settings.cpp deleted 100644 → 0
View file @ e10de73e
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "micro_model_settings.h"
const char *kCategoryLabels[kCategoryCount] = {
"silence",
"unknown",
"yes",
"no",
};
libraries/TFLiteMicro/examples/micro_speech/micro_model_settings.h deleted 100644 → 0
View file @ e10de73e
/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_LITE_MICRO_EXAMPLES_MICRO_SPEECH_MICRO_FEATURES_MICRO_MODEL_SETTINGS_H_
#define TENSORFLOW_LITE_MICRO_EXAMPLES_MICRO_SPEECH_MICRO_FEATURES_MICRO_MODEL_SETTINGS_H_
// Keeping these as constant expressions allow us to allocate fixed-sized arrays
// on the stack for our working memory.
// The size of the input time series data we pass to the FFT to produce the
// frequency information. This has to be a power of two, and since we're dealing
// with 30ms of 16KHz inputs, which means 480 samples, this is the next value.
constexpr int kMaxAudioSampleSize = 512;
constexpr int kAudioSampleFrequency = 16000;
// The following values are derived from values used during model training.
// If you change the way you preprocess the input, update all these constants.
constexpr int kFeatureSliceSize = 40;
constexpr int kFeatureSliceCount = 49;
constexpr int kFeatureElementCount = (kFeatureSliceSize * kFeatureSliceCount);
constexpr int kFeatureSliceStrideMs = 20;
constexpr int kFeatureSliceDurationMs = 30;
// Variables for the model's output categories.
constexpr int kSilenceIndex = 0;
constexpr int kUnknownIndex = 1;
// If you modify the output categories, you need to update the following values.
constexpr int kCategoryCount = 4;
extern const char *kCategoryLabels[kCategoryCount];
#endif // TENSORFLOW_LITE_MICRO_EXAMPLES_MICRO_SPEECH_MICRO_FEATURES_MICRO_MODEL_SETTINGS_H_
libraries/TFLiteMicro/examples/micro_speech/micro_speech.ino deleted 100644 → 0
View file @ e10de73e
/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "audio_provider.h"
#include "command_responder.h"
#include "feature_provider.h"
#include "micro_model_settings.h"
#include "model.h"
#include "recognize_commands.h"
#include "tensorflow/lite/micro/micro_interpreter.h"
#include "tensorflow/lite/micro/micro_log.h"
#include "tensorflow/lite/micro/micro_mutable_op_resolver.h"
#include "tensorflow/lite/micro/system_setup.h"
#include "tensorflow/lite/schema/schema_generated.h"
// Globals, used for compatibility with Arduino-style sketches.
namespace {
const tflite::Model *model = nullptr;
tflite::MicroInterpreter *interpreter = nullptr;
TfLiteTensor *model_input = nullptr;
FeatureProvider *feature_provider = nullptr;
RecognizeCommands *recognizer = nullptr;
int32_t previous_time = 0;
// Create an area of memory to use for input, output, and intermediate arrays.
// The size of this will depend on the model you're using, and may need to be
// determined by experimentation.
constexpr int kTensorArenaSize = 30 * 1024;
uint8_t tensor_arena[kTensorArenaSize];
int8_t feature_buffer[kFeatureElementCount];
int8_t *model_input_buffer = nullptr;
} // namespace
// The name of this function is important for Arduino compatibility.
void setup() {
// Map the model into a usable data structure. This doesn't involve any
// copying or parsing, it's a very lightweight operation.
model = tflite::GetModel(g_model);
if (model->version() != TFLITE_SCHEMA_VERSION) {
MicroPrintf(
"Model provided is schema version %d not equal to supported "
"version %d.",
model->version(), TFLITE_SCHEMA_VERSION
);
return;
}
// Pull in only the operation implementations we need.
// This relies on a complete list of all the ops needed by this graph.
// An easier approach is to just use the AllOpsResolver, but this will
// incur some penalty in code space for op implementations that are not
// needed by this graph.
//
// tflite::AllOpsResolver resolver;
// NOLINTNEXTLINE(runtime-global-variables)
static tflite::MicroMutableOpResolver<4> micro_op_resolver;
if (micro_op_resolver.AddDepthwiseConv2D() != kTfLiteOk) {
return;
}
if (micro_op_resolver.AddFullyConnected() != kTfLiteOk) {
return;
}
if (micro_op_resolver.AddSoftmax() != kTfLiteOk) {
return;
}
if (micro_op_resolver.AddReshape() != kTfLiteOk) {
return;
}
// Build an interpreter to run the model with.
static tflite::MicroInterpreter static_interpreter(model, micro_op_resolver, tensor_arena, kTensorArenaSize);
interpreter = &static_interpreter;
// Allocate memory from the tensor_arena for the model's tensors.
TfLiteStatus allocate_status = interpreter->AllocateTensors();
if (allocate_status != kTfLiteOk) {
MicroPrintf("AllocateTensors() failed");
return;
}
// Get information about the memory area to use for the model's input.
model_input = interpreter->input(0);
if ((model_input->dims->size != 2) || (model_input->dims->data[0] != 1) || (model_input->dims->data[1] != (kFeatureSliceCount * kFeatureSliceSize))
|| (model_input->type != kTfLiteInt8)) {
MicroPrintf("Bad input tensor parameters in model");
return;
}
model_input_buffer = model_input->data.int8;
// Prepare to access the audio spectrograms from a microphone or other source
// that will provide the inputs to the neural network.
// NOLINTNEXTLINE(runtime-global-variables)
static FeatureProvider static_feature_provider(kFeatureElementCount, feature_buffer);
feature_provider = &static_feature_provider;
static RecognizeCommands static_recognizer;
recognizer = &static_recognizer;
previous_time = 0;
}
// The name of this function is important for Arduino compatibility.
void loop() {
// Fetch the spectrogram for the current time.
const int32_t current_time = LatestAudioTimestamp();
int how_many_new_slices = 0;
TfLiteStatus feature_status = feature_provider->PopulateFeatureData(previous_time, current_time, &how_many_new_slices);
if (feature_status != kTfLiteOk) {
MicroPrintf("Feature generation failed");
return;
}
previous_time = current_time;
// If no new audio samples have been received since last time, don't bother
// running the network model.
if (how_many_new_slices == 0) {
return;
}
// Copy feature buffer to input tensor
for (int i = 0; i < kFeatureElementCount; i++) {
model_input_buffer[i] = feature_buffer[i];
}
// Run the model on the spectrogram input and make sure it succeeds.
TfLiteStatus invoke_status = interpreter->Invoke();
if (invoke_status != kTfLiteOk) {
MicroPrintf("Invoke failed");
return;
}
// Obtain a pointer to the output tensor
TfLiteTensor *output = interpreter->output(0);
// Determine whether a command was recognized based on the output of inference
const char *found_command = nullptr;
uint8_t score = 0;
bool is_new_command = false;
TfLiteStatus process_status = recognizer->ProcessLatestResults(output, current_time, &found_command, &score, &is_new_command);
if (process_status != kTfLiteOk) {
MicroPrintf("RecognizeCommands::ProcessLatestResults() failed");
return;
}
// Do something based on the recognized command. The default implementation
// just prints to the error console, but you should replace this with your
// own function for a real application.
RespondToCommand(current_time, found_command, score, is_new_command);
}
libraries/TFLiteMicro/examples/micro_speech/model.cpp deleted 100644 → 0
View file @ e10de73e
This source diff could not be displayed because it is too large. You can view the blob instead.
libraries/TFLiteMicro/examples/micro_speech/model.h deleted 100644 → 0
View file @ e10de73e
/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// This is a standard TensorFlow Lite FlatBuffer model file that has been
// converted into a C data array, so it can be easily compiled into a binary
// for devices that don't have a file system. It was created using the command:
// xxd -i model.tflite > model.cc
#ifndef TENSORFLOW_LITE_MICRO_EXAMPLES_MICRO_SPEECH_MICRO_FEATURES_MODEL_H_
#define TENSORFLOW_LITE_MICRO_EXAMPLES_MICRO_SPEECH_MICRO_FEATURES_MODEL_H_
extern const unsigned char g_model[];
extern const int g_model_len;
#endif // TENSORFLOW_LITE_MICRO_EXAMPLES_MICRO_SPEECH_MICRO_FEATURES_MODEL_H_
libraries/TFLiteMicro/examples/micro_speech/recognize_commands.cpp deleted 100644 → 0
View file @ e10de73e
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "recognize_commands.h"
#include <limits>
RecognizeCommands::RecognizeCommands(int32_t average_window_duration_ms, uint8_t detection_threshold, int32_t suppression_ms, int32_t minimum_count)
: average_window_duration_ms_(average_window_duration_ms), detection_threshold_(detection_threshold), suppression_ms_(suppression_ms),
minimum_count_(minimum_count), previous_results_() {
previous_top_label_ = "silence";
previous_top_label_time_ = std::numeric_limits<int32_t>::min();
}
TfLiteStatus RecognizeCommands::ProcessLatestResults(
const TfLiteTensor *latest_results, const int32_t current_time_ms, const char **found_command, uint8_t *score, bool *is_new_command
) {
if ((latest_results->dims->size != 2) || (latest_results->dims->data[0] != 1) || (latest_results->dims->data[1] != kCategoryCount)) {
MicroPrintf(
"The results for recognition should contain %d elements, but there are "
"%d in an %d-dimensional shape",
kCategoryCount, latest_results->dims->data[1], latest_results->dims->size
);
return kTfLiteError;
}
if (latest_results->type != kTfLiteInt8) {
MicroPrintf("The results for recognition should be int8_t elements, but are %d", latest_results->type);
return kTfLiteError;
}
if ((!previous_results_.empty()) && (current_time_ms < previous_results_.front().time_)) {
MicroPrintf(
"Results must be fed in increasing time order, but received a "
"timestamp of %d that was earlier than the previous one of %d",
current_time_ms, previous_results_.front().time_
);
return kTfLiteError;
}
// Add the latest results to the head of the queue.
previous_results_.push_back({current_time_ms, latest_results->data.int8});
// Prune any earlier results that are too old for the averaging window.
const int64_t time_limit = current_time_ms - average_window_duration_ms_;
while ((!previous_results_.empty()) && previous_results_.front().time_ < time_limit) {
previous_results_.pop_front();
}
// If there are too few results, assume the result will be unreliable and
// bail.
const int64_t how_many_results = previous_results_.size();
const int64_t earliest_time = previous_results_.front().time_;
const int64_t samples_duration = current_time_ms - earliest_time;
if ((how_many_results < minimum_count_) || (samples_duration < (average_window_duration_ms_ / 4))) {
*found_command = previous_top_label_;
*score = 0;
*is_new_command = false;
return kTfLiteOk;
}
// Calculate the average score across all the results in the window.
int32_t average_scores[kCategoryCount];
for (int offset = 0; offset < previous_results_.size(); ++offset) {
PreviousResultsQueue::Result previous_result = previous_results_.from_front(offset);
const int8_t *scores = previous_result.scores;
for (int i = 0; i < kCategoryCount; ++i) {
if (offset == 0) {
average_scores[i] = scores[i] + 128;
} else {
average_scores[i] += scores[i] + 128;
}
}
}
for (int i = 0; i < kCategoryCount; ++i) {
average_scores[i] /= how_many_results;
}
// Find the current highest scoring category.
int current_top_index = 0;
int32_t current_top_score = 0;
for (int i = 0; i < kCategoryCount; ++i) {
if (average_scores[i] > current_top_score) {
current_top_score = average_scores[i];
current_top_index = i;
}
}
const char *current_top_label = kCategoryLabels[current_top_index];
// If we've recently had another label trigger, assume one that occurs too
// soon afterwards is a bad result.
int64_t time_since_last_top;
if ((previous_top_label_ == kCategoryLabels[0]) || (previous_top_label_time_ == std::numeric_limits<int32_t>::min())) {
time_since_last_top = std::numeric_limits<int32_t>::max();
} else {
time_since_last_top = current_time_ms - previous_top_label_time_;
}
if ((current_top_score > detection_threshold_) && ((current_top_label != previous_top_label_) || (time_since_last_top > suppression_ms_))) {
previous_top_label_ = current_top_label;
previous_top_label_time_ = current_time_ms;
*is_new_command = true;
} else {
*is_new_command = false;
}
*found_command = current_top_label;
*score = current_top_score;
return kTfLiteOk;
}
libraries/TFLiteMicro/examples/micro_speech/recognize_commands.h deleted 100644 → 0
View file @ e10de73e
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_LITE_MICRO_EXAMPLES_MICRO_SPEECH_RECOGNIZE_COMMANDS_H_
#define TENSORFLOW_LITE_MICRO_EXAMPLES_MICRO_SPEECH_RECOGNIZE_COMMANDS_H_
#include <cstdint>
#include "tensorflow/lite/c/common.h"
#include "micro_model_settings.h"
#include "tensorflow/lite/micro/micro_log.h"
// Partial implementation of std::dequeue, just providing the functionality
// that's needed to keep a record of previous neural network results over a
// short time period, so they can be averaged together to produce a more
// accurate overall prediction. This doesn't use any dynamic memory allocation
// so it's a better fit for microcontroller applications, but this does mean
// there are hard limits on the number of results it can store.
class PreviousResultsQueue {
public:
PreviousResultsQueue() : front_index_(0), size_(0) {}
// Data structure that holds an inference result, and the time when it
// was recorded.
struct Result {
Result() : time_(0), scores() {}
Result(int32_t time, int8_t *input_scores) : time_(time) {
for (int i = 0; i < kCategoryCount; ++i) {
scores[i] = input_scores[i];
}
}
int32_t time_;
int8_t scores[kCategoryCount];
};
int size() {
return size_;
}
bool empty() {
return size_ == 0;
}
Result &front() {
return results_[front_index_];
}
Result &back() {
int back_index = front_index_ + (size_ - 1);
if (back_index >= kMaxResults) {
back_index -= kMaxResults;
}
return results_[back_index];
}
void push_back(const Result &entry) {
if (size() >= kMaxResults) {
MicroPrintf("Couldn't push_back latest result, too many already!");
return;
}
size_ += 1;
back() = entry;
}
Result pop_front() {
if (size() <= 0) {
MicroPrintf("Couldn't pop_front result, none present!");
return Result();
}
Result result = front();
front_index_ += 1;
if (front_index_ >= kMaxResults) {
front_index_ = 0;
}
size_ -= 1;
return result;
}
// Most of the functions are duplicates of dequeue containers, but this
// is a helper that makes it easy to iterate through the contents of the
// queue.
Result &from_front(int offset) {
if ((offset < 0) || (offset >= size_)) {
MicroPrintf("Attempt to read beyond the end of the queue!");
offset = size_ - 1;
}
int index = front_index_ + offset;
if (index >= kMaxResults) {
index -= kMaxResults;
}
return results_[index];
}
private:
static constexpr int kMaxResults = 50;
Result results_[kMaxResults];
int front_index_;
int size_;
};
// This class is designed to apply a very primitive decoding model on top of the
// instantaneous results from running an audio recognition model on a single
// window of samples. It applies smoothing over time so that noisy individual
// label scores are averaged, increasing the confidence that apparent matches
// are real.
// To use it, you should create a class object with the configuration you
// want, and then feed results from running a TensorFlow model into the
// processing method. The timestamp for each subsequent call should be
// increasing from the previous, since the class is designed to process a stream
// of data over time.
class RecognizeCommands {
public:
// labels should be a list of the strings associated with each one-hot score.
// The window duration controls the smoothing. Longer durations will give a
// higher confidence that the results are correct, but may miss some commands.
// The detection threshold has a similar effect, with high values increasing
// the precision at the cost of recall. The minimum count controls how many
// results need to be in the averaging window before it's seen as a reliable
// average. This prevents erroneous results when the averaging window is
// initially being populated for example. The suppression argument disables
// further recognitions for a set time after one has been triggered, which can
// help reduce spurious recognitions.
explicit RecognizeCommands(
int32_t average_window_duration_ms = 1000, uint8_t detection_threshold = 200, int32_t suppression_ms = 1500, int32_t minimum_count = 3
);
// Call this with the results of running a model on sample data.
TfLiteStatus
ProcessLatestResults(const TfLiteTensor *latest_results, const int32_t current_time_ms, const char **found_command, uint8_t *score, bool *is_new_command);
private:
// Configuration
int32_t average_window_duration_ms_;
uint8_t detection_threshold_;
int32_t suppression_ms_;
int32_t minimum_count_;
// Working variables
PreviousResultsQueue previous_results_;
const char *previous_top_label_;
int32_t previous_top_label_time_;
};
#endif // TENSORFLOW_LITE_MICRO_EXAMPLES_MICRO_SPEECH_RECOGNIZE_COMMANDS_H_
libraries/TFLiteMicro/examples/micro_speech/ringbuf.c deleted 100644 → 0
View file @ e10de73e
/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "ringbuf.h"
#include <esp_heap_caps.h>
#include <sdkconfig.h>
#include <stdbool.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "esp_err.h"
#include "esp_log.h"
#include "freertos/FreeRTOS.h"
#include "freertos/queue.h"
#include "freertos/semphr.h"
#include "freertos/task.h"
#define RB_TAG "RINGBUF"
ringbuf_t *rb_init(const char *name, uint32_t size) {
ringbuf_t *r;
unsigned char *buf;
if (size < 2 || !name) {
return NULL;
}
r = malloc(sizeof(ringbuf_t));
assert(r);
#if (CONFIG_SPIRAM_SUPPORT && (CONFIG_SPIRAM_USE_CAPS_ALLOC || CONFIG_SPIRAM_USE_MALLOC))
buf = heap_caps_calloc(1, size, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT);
#else
buf = calloc(1, size);
#endif
assert(buf);
r->name = (char *)name;
r->base = r->readptr = r->writeptr = buf;
r->fill_cnt = 0;
r->size = size;
r->can_read = xSemaphoreCreateBinary();
assert(r->can_read);
r->can_write = xSemaphoreCreateBinary();
assert(r->can_write);
r->lock = xSemaphoreCreateMutex();
assert(r->lock);
r->abort_read = 0;
r->abort_write = 0;
r->writer_finished = 0;
r->reader_unblock = 0;
return r;
}
void rb_cleanup(ringbuf_t *rb) {
free(rb->base);
rb->base = NULL;
vSemaphoreDelete(rb->can_read);
rb->can_read = NULL;
vSemaphoreDelete(rb->can_write);
rb->can_write = NULL;
vSemaphoreDelete(rb->lock);
rb->lock = NULL;
free(rb);
}
/*
* @brief: get the number of filled bytes in the buffer
*/
ssize_t rb_filled(ringbuf_t *rb) {
return rb->fill_cnt;
}
/*
* @brief: get the number of empty bytes available in the buffer
*/
ssize_t rb_available(ringbuf_t *rb) {
ESP_LOGD(RB_TAG, "rb leftover %d bytes", rb->size - rb->fill_cnt);
return (rb->size - rb->fill_cnt);
}
int rb_read(ringbuf_t *rb, uint8_t *buf, int buf_len, uint32_t ticks_to_wait) {
int read_size;
int total_read_size = 0;
/**
* In case where we are able to read buf_len in one go,
* we are not able to check for abort and keep returning buf_len as bytes
* read. Check for argument validity check and abort case before entering
* memcpy loop.
*/
if (rb == NULL || rb->abort_read == 1) {
return ESP_FAIL;
}
xSemaphoreTake(rb->lock, portMAX_DELAY);
while (buf_len) {
if (rb->fill_cnt < buf_len) {
read_size = rb->fill_cnt;
} else {
read_size = buf_len;
}
if ((rb->readptr + read_size) > (rb->base + rb->size)) {
int rlen1 = rb->base + rb->size - rb->readptr;
int rlen2 = read_size - rlen1;
if (buf) {
memcpy(buf, rb->readptr, rlen1);
memcpy(buf + rlen1, rb->base, rlen2);
}
rb->readptr = rb->base + rlen2;
} else {
if (buf) {
memcpy(buf, rb->readptr, read_size);
}
rb->readptr = rb->readptr + read_size;
}
buf_len -= read_size;
rb->fill_cnt -= read_size;
total_read_size += read_size;
if (buf) {
buf += read_size;
}
xSemaphoreGive(rb->can_write);
if (buf_len == 0) {
break;
}
xSemaphoreGive(rb->lock);
if (!rb->writer_finished && !rb->abort_read && !rb->reader_unblock) {
if (xSemaphoreTake(rb->can_read, ticks_to_wait) != pdTRUE) {
goto out;
}
}
if (rb->abort_read == 1) {
total_read_size = RB_ABORT;
goto out;
}
if (rb->writer_finished == 1) {
goto out;
}
if (rb->reader_unblock == 1) {
if (total_read_size == 0) {
total_read_size = RB_READER_UNBLOCK;
}
goto out;
}
xSemaphoreTake(rb->lock, portMAX_DELAY);
}
xSemaphoreGive(rb->lock);
out:
if (rb->writer_finished == 1 && total_read_size == 0) {
total_read_size = RB_WRITER_FINISHED;
}
rb->reader_unblock = 0; /* We are anyway unblocking reader */
return total_read_size;
}
int rb_write(ringbuf_t *rb, const uint8_t *buf, int buf_len, uint32_t ticks_to_wait) {
int write_size;
int total_write_size = 0;
/**
* In case where we are able to write buf_len in one go,
* we are not able to check for abort and keep returning buf_len as bytes
* written. Check for arguments' validity and abort case before entering
* memcpy loop.
*/
if (rb == NULL || buf == NULL || rb->abort_write == 1) {
return RB_FAIL;
}
xSemaphoreTake(rb->lock, portMAX_DELAY);
while (buf_len) {
if ((rb->size - rb->fill_cnt) < buf_len) {
write_size = rb->size - rb->fill_cnt;
} else {
write_size = buf_len;
}
if ((rb->writeptr + write_size) > (rb->base + rb->size)) {
int wlen1 = rb->base + rb->size - rb->writeptr;
int wlen2 = write_size - wlen1;
memcpy(rb->writeptr, buf, wlen1);
memcpy(rb->base, buf + wlen1, wlen2);
rb->writeptr = rb->base + wlen2;
} else {
memcpy(rb->writeptr, buf, write_size);
rb->writeptr = rb->writeptr + write_size;
}
buf_len -= write_size;
rb->fill_cnt += write_size;
total_write_size += write_size;
buf += write_size;
xSemaphoreGive(rb->can_read);
if (buf_len == 0) {
break;
}
xSemaphoreGive(rb->lock);
if (rb->writer_finished) {
return write_size > 0 ? write_size : RB_WRITER_FINISHED;
}
if (xSemaphoreTake(rb->can_write, ticks_to_wait) != pdTRUE) {
goto out;
}
if (rb->abort_write == 1) {
goto out;
}
xSemaphoreTake(rb->lock, portMAX_DELAY);
}
xSemaphoreGive(rb->lock);
out:
return total_write_size;
}
/**
* abort and set abort_read and abort_write to asked values.
*/
static void _rb_reset(ringbuf_t *rb, int abort_read, int abort_write) {
if (rb == NULL) {
return;
}
xSemaphoreTake(rb->lock, portMAX_DELAY);
rb->readptr = rb->writeptr = rb->base;
rb->fill_cnt = 0;
rb->writer_finished = 0;
rb->reader_unblock = 0;
rb->abort_read = abort_read;
rb->abort_write = abort_write;
xSemaphoreGive(rb->lock);
}
void rb_reset(ringbuf_t *rb) {
_rb_reset(rb, 0, 0);
}
void rb_abort_read(ringbuf_t *rb) {
if (rb == NULL) {
return;
}
rb->abort_read = 1;
xSemaphoreGive(rb->can_read);
xSemaphoreGive(rb->lock);
}
void rb_abort_write(ringbuf_t *rb) {
if (rb == NULL) {
return;
}
rb->abort_write = 1;
xSemaphoreGive(rb->can_write);
xSemaphoreGive(rb->lock);
}
void rb_abort(ringbuf_t *rb) {
if (rb == NULL) {
return;
}
rb->abort_read = 1;
rb->abort_write = 1;
xSemaphoreGive(rb->can_read);
xSemaphoreGive(rb->can_write);
xSemaphoreGive(rb->lock);
}
/**
* Reset the ringbuffer and keep rb_write aborted.
* Note that we are taking lock before even toggling `abort_write` variable.
* This serves a special purpose to not allow this abort to be mixed with
* rb_write.
*/
void rb_reset_and_abort_write(ringbuf_t *rb) {
_rb_reset(rb, 0, 1);
xSemaphoreGive(rb->can_write);
}
void rb_signal_writer_finished(ringbuf_t *rb) {
if (rb == NULL) {
return;
}
rb->writer_finished = 1;
xSemaphoreGive(rb->can_read);
}
int rb_is_writer_finished(ringbuf_t *rb) {
if (rb == NULL) {
return RB_FAIL;
}
return (rb->writer_finished);
}
void rb_wakeup_reader(ringbuf_t *rb) {
if (rb == NULL) {
return;
}
rb->reader_unblock = 1;
xSemaphoreGive(rb->can_read);
}
void rb_stat(ringbuf_t *rb) {
xSemaphoreTake(rb->lock, portMAX_DELAY);
ESP_LOGI(RB_TAG, "filled: %d, base: %p, read_ptr: %p, write_ptr: %p, size: %d\n", rb->fill_cnt, rb->base, rb->readptr, rb->writeptr, rb->size);
xSemaphoreGive(rb->lock);
}
libraries/TFLiteMicro/examples/micro_speech/ringbuf.h deleted 100644 → 0
View file @ e10de73e
/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_LITE_MICRO_EXAMPLES_MICRO_SPEECH_ESP_RINGBUF_H_
#define TENSORFLOW_LITE_MICRO_EXAMPLES_MICRO_SPEECH_ESP_RINGBUF_H_
#include <freertos/FreeRTOS.h>
#include <freertos/semphr.h>
#include <stdint.h>
#ifdef __cplusplus
extern "C" {
#endif
#define RB_FAIL ESP_FAIL
#define RB_ABORT -1
#define RB_WRITER_FINISHED -2
#define RB_READER_UNBLOCK -3
#if __has_include("esp_idf_version.h")
#include "esp_idf_version.h"
#else
#define ESP_IDF_VERSION_VAL(major, minor, patch) 0
#endif
#if (ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(5, 0, 0))
#if !(configENABLE_BACKWARD_COMPATIBILITY == 1)
#define xSemaphoreHandle SemaphoreHandle_t
#endif
#endif
typedef struct ringbuf {
char *name;
uint8_t *base; /**< Original pointer */
/* XXX: these need to be volatile? */
uint8_t *volatile readptr; /**< Read pointer */
uint8_t *volatile writeptr; /**< Write pointer */
volatile ssize_t fill_cnt; /**< Number of filled slots */
ssize_t size; /**< Buffer size */
xSemaphoreHandle can_read;
xSemaphoreHandle can_write;
xSemaphoreHandle lock;
int abort_read;
int abort_write;
int writer_finished; // to prevent infinite blocking for buffer read
int reader_unblock;
} ringbuf_t;
ringbuf_t *rb_init(const char *rb_name, uint32_t size);
void rb_abort_read(ringbuf_t *rb);
void rb_abort_write(ringbuf_t *rb);
void rb_abort(ringbuf_t *rb);
void rb_reset(ringbuf_t *rb);
/**
* @brief Special function to reset the buffer while keeping rb_write aborted.
* This rb needs to be reset again before being useful.
*/
void rb_reset_and_abort_write(ringbuf_t *rb);
void rb_stat(ringbuf_t *rb);
ssize_t rb_filled(ringbuf_t *rb);
ssize_t rb_available(ringbuf_t *rb);
int rb_read(ringbuf_t *rb, uint8_t *buf, int len, uint32_t ticks_to_wait);
int rb_write(ringbuf_t *rb, const uint8_t *buf, int len, uint32_t ticks_to_wait);
void rb_cleanup(ringbuf_t *rb);
void rb_signal_writer_finished(ringbuf_t *rb);
void rb_wakeup_reader(ringbuf_t *rb);
int rb_is_writer_finished(ringbuf_t *rb);
#ifdef __cplusplus
}
#endif
#endif // TENSORFLOW_LITE_MICRO_EXAMPLES_MICRO_SPEECH_ESP_RINGBUF_H_
package/package_esp32_index.template.json
View file @ 0c4b35e0
......@@ -42,7 +42,7 @@
{
"packager": "esp32",
"name": "esp32-arduino-libs",
"version": "idf-release_v5.1-d06c758489"
"version": "idf-release_v5.1-442a798083"
},
{
"packager": "esp32",
......@@ -105,63 +105,63 @@
"tools": [
{
"name": "esp32-arduino-libs",
"version": "idf-release_v5.1-d06c758489",
"version": "idf-release_v5.1-442a798083",
"systems": [
{
"host": "i686-mingw32",
"url": "https://codeload.github.com/espressif/esp32-arduino-libs/zip/07d6415e1df493c23a3091761be59ece49527313",
"archiveFileName": "esp32-arduino-libs-07d6415e1df493c23a3091761be59ece49527313.zip",
"checksum": "SHA-256:9d47ec1df5b217ce4e163f116dbff7b718d1ca87914725731455eae786add447",
"size": "371892775"
"url": "https://codeload.github.com/espressif/esp32-arduino-libs/zip/951ade74d7886e1ce931ea46614c4ac47ae3a6c0",
"archiveFileName": "esp32-arduino-libs-951ade74d7886e1ce931ea46614c4ac47ae3a6c0.zip",
"checksum": "SHA-256:ac9e200eac443655c5661a4a9251032cb08a7d7a4e32c34ebeb8d340f0030aab",
"size": "375249835"
},
{
"host": "x86_64-mingw32",
"url": "https://codeload.github.com/espressif/esp32-arduino-libs/zip/07d6415e1df493c23a3091761be59ece49527313",
"archiveFileName": "esp32-arduino-libs-07d6415e1df493c23a3091761be59ece49527313.zip",
"checksum": "SHA-256:9d47ec1df5b217ce4e163f116dbff7b718d1ca87914725731455eae786add447",
"size": "371892775"
"url": "https://codeload.github.com/espressif/esp32-arduino-libs/zip/951ade74d7886e1ce931ea46614c4ac47ae3a6c0",
"archiveFileName": "esp32-arduino-libs-951ade74d7886e1ce931ea46614c4ac47ae3a6c0.zip",
"checksum": "SHA-256:ac9e200eac443655c5661a4a9251032cb08a7d7a4e32c34ebeb8d340f0030aab",
"size": "375249835"
},
{
"host": "arm64-apple-darwin",
"url": "https://codeload.github.com/espressif/esp32-arduino-libs/zip/07d6415e1df493c23a3091761be59ece49527313",
"archiveFileName": "esp32-arduino-libs-07d6415e1df493c23a3091761be59ece49527313.zip",
"checksum": "SHA-256:9d47ec1df5b217ce4e163f116dbff7b718d1ca87914725731455eae786add447",
"size": "371892775"
"url": "https://codeload.github.com/espressif/esp32-arduino-libs/zip/951ade74d7886e1ce931ea46614c4ac47ae3a6c0",
"archiveFileName": "esp32-arduino-libs-951ade74d7886e1ce931ea46614c4ac47ae3a6c0.zip",
"checksum": "SHA-256:ac9e200eac443655c5661a4a9251032cb08a7d7a4e32c34ebeb8d340f0030aab",
"size": "375249835"
},
{
"host": "x86_64-apple-darwin",
"url": "https://codeload.github.com/espressif/esp32-arduino-libs/zip/07d6415e1df493c23a3091761be59ece49527313",
"archiveFileName": "esp32-arduino-libs-07d6415e1df493c23a3091761be59ece49527313.zip",
"checksum": "SHA-256:9d47ec1df5b217ce4e163f116dbff7b718d1ca87914725731455eae786add447",
"size": "371892775"
"url": "https://codeload.github.com/espressif/esp32-arduino-libs/zip/951ade74d7886e1ce931ea46614c4ac47ae3a6c0",
"archiveFileName": "esp32-arduino-libs-951ade74d7886e1ce931ea46614c4ac47ae3a6c0.zip",
"checksum": "SHA-256:ac9e200eac443655c5661a4a9251032cb08a7d7a4e32c34ebeb8d340f0030aab",
"size": "375249835"
},
{
"host": "x86_64-pc-linux-gnu",
"url": "https://codeload.github.com/espressif/esp32-arduino-libs/zip/07d6415e1df493c23a3091761be59ece49527313",
"archiveFileName": "esp32-arduino-libs-07d6415e1df493c23a3091761be59ece49527313.zip",
"checksum": "SHA-256:9d47ec1df5b217ce4e163f116dbff7b718d1ca87914725731455eae786add447",
"size": "371892775"
"url": "https://codeload.github.com/espressif/esp32-arduino-libs/zip/951ade74d7886e1ce931ea46614c4ac47ae3a6c0",
"archiveFileName": "esp32-arduino-libs-951ade74d7886e1ce931ea46614c4ac47ae3a6c0.zip",
"checksum": "SHA-256:ac9e200eac443655c5661a4a9251032cb08a7d7a4e32c34ebeb8d340f0030aab",
"size": "375249835"
},
{
"host": "i686-pc-linux-gnu",
"url": "https://codeload.github.com/espressif/esp32-arduino-libs/zip/07d6415e1df493c23a3091761be59ece49527313",
"archiveFileName": "esp32-arduino-libs-07d6415e1df493c23a3091761be59ece49527313.zip",
"checksum": "SHA-256:9d47ec1df5b217ce4e163f116dbff7b718d1ca87914725731455eae786add447",
"size": "371892775"
"url": "https://codeload.github.com/espressif/esp32-arduino-libs/zip/951ade74d7886e1ce931ea46614c4ac47ae3a6c0",
"archiveFileName": "esp32-arduino-libs-951ade74d7886e1ce931ea46614c4ac47ae3a6c0.zip",
"checksum": "SHA-256:ac9e200eac443655c5661a4a9251032cb08a7d7a4e32c34ebeb8d340f0030aab",
"size": "375249835"
},
{
"host": "aarch64-linux-gnu",
"url": "https://codeload.github.com/espressif/esp32-arduino-libs/zip/07d6415e1df493c23a3091761be59ece49527313",
"archiveFileName": "esp32-arduino-libs-07d6415e1df493c23a3091761be59ece49527313.zip",
"checksum": "SHA-256:9d47ec1df5b217ce4e163f116dbff7b718d1ca87914725731455eae786add447",
"size": "371892775"
"url": "https://codeload.github.com/espressif/esp32-arduino-libs/zip/951ade74d7886e1ce931ea46614c4ac47ae3a6c0",
"archiveFileName": "esp32-arduino-libs-951ade74d7886e1ce931ea46614c4ac47ae3a6c0.zip",
"checksum": "SHA-256:ac9e200eac443655c5661a4a9251032cb08a7d7a4e32c34ebeb8d340f0030aab",
"size": "375249835"
},
{
"host": "arm-linux-gnueabihf",
"url": "https://codeload.github.com/espressif/esp32-arduino-libs/zip/07d6415e1df493c23a3091761be59ece49527313",
"archiveFileName": "esp32-arduino-libs-07d6415e1df493c23a3091761be59ece49527313.zip",
"checksum": "SHA-256:9d47ec1df5b217ce4e163f116dbff7b718d1ca87914725731455eae786add447",
"size": "371892775"
"url": "https://codeload.github.com/espressif/esp32-arduino-libs/zip/951ade74d7886e1ce931ea46614c4ac47ae3a6c0",
"archiveFileName": "esp32-arduino-libs-951ade74d7886e1ce931ea46614c4ac47ae3a6c0.zip",
"checksum": "SHA-256:ac9e200eac443655c5661a4a9251032cb08a7d7a4e32c34ebeb8d340f0030aab",
"size": "375249835"
}
]
},
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment