esphome 2025.4.1__py3-none-any.whl → 2025.5.0__py3-none-any.whl

esphome/components/micro_wake_word/micro_wake_word.h

@@ -5,33 +5,27 @@
 #include "preprocessor_settings.h"
 #include "streaming_model.h"
 
+#include "esphome/components/microphone/microphone_source.h"
+
 #include "esphome/core/automation.h"
 #include "esphome/core/component.h"
 #include "esphome/core/ring_buffer.h"
 
-#include "esphome/components/microphone/microphone.h"
+#include <freertos/event_groups.h>
 
+#include <frontend.h>
 #include <frontend_util.h>
 
-#include <tensorflow/lite/core/c/common.h>
-#include <tensorflow/lite/micro/micro_interpreter.h>
-#include <tensorflow/lite/micro/micro_mutable_op_resolver.h>
-
 namespace esphome {
 namespace micro_wake_word {
 
 enum State {
-  IDLE,
-  START_MICROPHONE,
-  STARTING_MICROPHONE,
+  STARTING,
   DETECTING_WAKE_WORD,
-  STOP_MICROPHONE,
-  STOPPING_MICROPHONE,
+  STOPPING,
+  STOPPED,
 };
 
-// The number of audio slices to process before accepting a positive detection
-static const uint8_t MIN_SLICES_BEFORE_DETECTION = 74;
-
 class MicroWakeWord : public Component {
  public:
   void setup() override;
@@ -42,132 +36,91 @@ class MicroWakeWord : public Component {
   void start();
   void stop();
 
-  bool is_running() const { return this->state_ != State::IDLE; }
+  bool is_running() const { return this->state_ != State::STOPPED; }
 
   void set_features_step_size(uint8_t step_size) { this->features_step_size_ = step_size; }
 
-  void set_microphone(microphone::Microphone *microphone) { this->microphone_ = microphone; }
+  void set_microphone_source(microphone::MicrophoneSource *microphone_source) {
+    this->microphone_source_ = microphone_source;
+  }
+
+  void set_stop_after_detection(bool stop_after_detection) { this->stop_after_detection_ = stop_after_detection; }
 
   Trigger<std::string> *get_wake_word_detected_trigger() const { return this->wake_word_detected_trigger_; }
 
-  void add_wake_word_model(const uint8_t *model_start, float probability_cutoff, size_t sliding_window_average_size,
-                           const std::string &wake_word, size_t tensor_arena_size);
+  void add_wake_word_model(WakeWordModel *model);
 
 #ifdef USE_MICRO_WAKE_WORD_VAD
-  void add_vad_model(const uint8_t *model_start, float probability_cutoff, size_t sliding_window_size,
+  void add_vad_model(const uint8_t *model_start, uint8_t probability_cutoff, size_t sliding_window_size,
                      size_t tensor_arena_size);
+
+  // Intended for the voice assistant component to fetch VAD status
+  bool get_vad_state() { return this->vad_state_; }
 #endif
 
+  // Intended for the voice assistant component to access which wake words are available
+  // Since these are pointers to the WakeWordModel objects, the voice assistant component can enable or disable them
+  std::vector<WakeWordModel *> get_wake_words();
+
  protected:
-  microphone::Microphone *microphone_{nullptr};
+  microphone::MicrophoneSource *microphone_source_{nullptr};
   Trigger<std::string> *wake_word_detected_trigger_ = new Trigger<std::string>();
-  State state_{State::IDLE};
-  HighFrequencyLoopRequester high_freq_;
+  State state_{State::STOPPED};
 
-  std::unique_ptr<RingBuffer> ring_buffer_;
-
-  std::vector<WakeWordModel> wake_word_models_;
+  std::weak_ptr<RingBuffer> ring_buffer_;
+  std::vector<WakeWordModel *> wake_word_models_;
 
 #ifdef USE_MICRO_WAKE_WORD_VAD
   std::unique_ptr<VADModel> vad_model_;
+  bool vad_state_{false};
 #endif
 
-  tflite::MicroMutableOpResolver<20> streaming_op_resolver_;
+  bool pending_start_{false};
+  bool pending_stop_{false};
+
+  bool stop_after_detection_;
+
+  uint8_t features_step_size_;
 
   // Audio frontend handles generating spectrogram features
   struct FrontendConfig frontend_config_;
   struct FrontendState frontend_state_;
 
-  // When the wake word detection first starts, we ignore this many audio
-  // feature slices before accepting a positive detection
-  int16_t ignore_windows_{-MIN_SLICES_BEFORE_DETECTION};
+  // Handles managing the stop/state of the inference task
+  EventGroupHandle_t event_group_;
 
-  uint8_t features_step_size_;
+  // Used to send messages about the models' states to the main loop
+  QueueHandle_t detection_queue_;
 
-  // Stores audio read from the microphone before being added to the ring buffer.
-  int16_t *input_buffer_{nullptr};
-  // Stores audio to be fed into the audio frontend for generating features.
-  int16_t *preprocessor_audio_buffer_{nullptr};
+  static void inference_task(void *params);
+  TaskHandle_t inference_task_handle_{nullptr};
 
-  bool detected_{false};
-  std::string detected_wake_word_{""};
+  /// @brief Suspends the inference task
+  void suspend_task_();
+  /// @brief Resumes the inference task
+  void resume_task_();
 
   void set_state_(State state);
 
-  /// @brief Tests if there are enough samples in the ring buffer to generate new features.
-  /// @return True if enough samples, false otherwise.
-  bool has_enough_samples_();
-
-  /** Reads audio from microphone into the ring buffer
-   *
-   * Audio data (16000 kHz with int16 samples) is read into the input_buffer_.
-   * Verifies the ring buffer has enough space for all audio data. If not, it logs
-   * a warning and resets the ring buffer entirely.
-   * @return Number of bytes written to the ring buffer
-   */
-  size_t read_microphone_();
+  /// @brief Generates spectrogram features from an input buffer of audio samples
+  /// @param audio_buffer (int16_t *) Buffer containing input audio samples
+  /// @param samples_available (size_t) Number of samples avaiable in the input buffer
+  /// @param features_buffer (int8_t *) Buffer to store generated features
+  /// @return (size_t) Number of samples processed from the input buffer
+  size_t generate_features_(int16_t *audio_buffer, size_t samples_available,
+                            int8_t features_buffer[PREPROCESSOR_FEATURE_SIZE]);
 
-  /// @brief Allocates memory for input_buffer_, preprocessor_audio_buffer_, and ring_buffer_
-  /// @return True if successful, false otherwise
-  bool allocate_buffers_();
+  /// @brief Processes any new probabilities for each model. If any wake word is detected, it will send a DetectionEvent
+  /// to the detection_queue_.
+  void process_probabilities_();
 
-  /// @brief Frees memory allocated for input_buffer_ and preprocessor_audio_buffer_
-  void deallocate_buffers_();
-
-  /// @brief Loads streaming models and prepares the feature generation frontend
-  /// @return True if successful, false otherwise
-  bool load_models_();
-
-  /// @brief Deletes each model's TFLite interpreters and frees tensor arena memory. Frees memory used by the feature
-  /// generation frontend.
+  /// @brief Deletes each model's TFLite interpreters and frees tensor arena memory.
   void unload_models_();
 
-  /** Performs inference with each configured model
-   *
-   * If enough audio samples are available, it will generate one slice of new features.
-   * It then loops through and performs inference with each of the loaded models.
-   */
-  void update_model_probabilities_();
-
-  /** Checks every model's recent probabilities to determine if the wake word has been predicted
-   *
-   * Verifies the models have processed enough new samples for accurate predictions.
-   * Sets detected_wake_word_ to the wake word, if one is detected.
-   * @return True if a wake word is predicted, false otherwise
-   */
-  bool detect_wake_words_();
-
-  /** Generates features for a window of audio samples
-   *
-   * Reads samples from the ring buffer and feeds them into the preprocessor frontend.
-   * Adapted from TFLite microspeech frontend.
-   * @param features int8_t array to store the audio features
-   * @return True if successful, false otherwise.
-   */
-  bool generate_features_for_window_(int8_t features[PREPROCESSOR_FEATURE_SIZE]);
-
-  /// @brief Resets the ring buffer, ignore_windows_, and sliding window probabilities
-  void reset_states_();
-
-  /// @brief Returns true if successfully registered the streaming model's TensorFlow operations
-  bool register_streaming_ops_(tflite::MicroMutableOpResolver<20> &op_resolver);
-
-  inline uint16_t new_samples_to_get_() { return (this->features_step_size_ * (AUDIO_SAMPLE_FREQUENCY / 1000)); }
-};
-
-template<typename... Ts> class StartAction : public Action<Ts...>, public Parented<MicroWakeWord> {
- public:
-  void play(Ts... x) override { this->parent_->start(); }
-};
-
-template<typename... Ts> class StopAction : public Action<Ts...>, public Parented<MicroWakeWord> {
- public:
-  void play(Ts... x) override { this->parent_->stop(); }
-};
-
-template<typename... Ts> class IsRunningCondition : public Condition<Ts...>, public Parented<MicroWakeWord> {
- public:
-  bool check(Ts... x) override { return this->parent_->is_running(); }
+  /// @brief Runs an inference with each model using the new spectrogram features
+  /// @param audio_features (int8_t *) Buffer containing new spectrogram features
+  /// @return True if successful, false if any errors were encountered
+  bool update_model_probabilities_(const int8_t audio_features[PREPROCESSOR_FEATURE_SIZE]);
 };
 
 }  // namespace micro_wake_word

esphome/components/micro_wake_word/preprocessor_settings.h

@@ -7,13 +7,30 @@
 namespace esphome {
 namespace micro_wake_word {
 
+// Settings for controlling the spectrogram feature generation by the preprocessor.
+// These must match the settings used when training a particular model.
+// All microWakeWord models have been trained with these specific paramters.
+
 // The number of features the audio preprocessor generates per slice
 static const uint8_t PREPROCESSOR_FEATURE_SIZE = 40;
 // Duration of each slice used as input into the preprocessor
 static const uint8_t FEATURE_DURATION_MS = 30;
-// Audio sample frequency in hertz
-static const uint16_t AUDIO_SAMPLE_FREQUENCY = 16000;
 
+static const float FILTERBANK_LOWER_BAND_LIMIT = 125.0;
+static const float FILTERBANK_UPPER_BAND_LIMIT = 7500.0;
+
+static const uint8_t NOISE_REDUCTION_SMOOTHING_BITS = 10;
+static const float NOISE_REDUCTION_EVEN_SMOOTHING = 0.025;
+static const float NOISE_REDUCTION_ODD_SMOOTHING = 0.06;
+static const float NOISE_REDUCTION_MIN_SIGNAL_REMAINING = 0.05;
+
+static const bool PCAN_GAIN_CONTROL_ENABLE_PCAN = true;
+static const float PCAN_GAIN_CONTROL_STRENGTH = 0.95;
+static const float PCAN_GAIN_CONTROL_OFFSET = 80.0;
+static const uint8_t PCAN_GAIN_CONTROL_GAIN_BITS = 21;
+
+static const bool LOG_SCALE_ENABLE_LOG = true;
+static const uint8_t LOG_SCALE_SCALE_SHIFT = 6;
 }  // namespace micro_wake_word
 }  // namespace esphome
 

esphome/components/micro_wake_word/streaming_model.cpp

@@ -1,8 +1,7 @@
-#ifdef USE_ESP_IDF
-
 #include "streaming_model.h"
 
-#include "esphome/core/hal.h"
+#ifdef USE_ESP_IDF
+
 #include "esphome/core/helpers.h"
 #include "esphome/core/log.h"
 
@@ -13,18 +12,18 @@ namespace micro_wake_word {
 
 void WakeWordModel::log_model_config() {
   ESP_LOGCONFIG(TAG, "    - Wake Word: %s", this->wake_word_.c_str());
-  ESP_LOGCONFIG(TAG, "      Probability cutoff: %.3f", this->probability_cutoff_);
+  ESP_LOGCONFIG(TAG, "      Probability cutoff: %.2f", this->probability_cutoff_ / 255.0f);
   ESP_LOGCONFIG(TAG, "      Sliding window size: %d", this->sliding_window_size_);
 }
 
 void VADModel::log_model_config() {
   ESP_LOGCONFIG(TAG, "    - VAD Model");
-  ESP_LOGCONFIG(TAG, "      Probability cutoff: %.3f", this->probability_cutoff_);
+  ESP_LOGCONFIG(TAG, "      Probability cutoff: %.2f", this->probability_cutoff_ / 255.0f);
   ESP_LOGCONFIG(TAG, "      Sliding window size: %d", this->sliding_window_size_);
 }
 
-bool StreamingModel::load_model(tflite::MicroMutableOpResolver<20> &op_resolver) {
-  ExternalRAMAllocator<uint8_t> arena_allocator(ExternalRAMAllocator<uint8_t>::ALLOW_FAILURE);
+bool StreamingModel::load_model_() {
+  RAMAllocator<uint8_t> arena_allocator(RAMAllocator<uint8_t>::ALLOW_FAILURE);
 
   if (this->tensor_arena_ == nullptr) {
     this->tensor_arena_ = arena_allocator.allocate(this->tensor_arena_size_);
@@ -51,8 +50,9 @@ bool StreamingModel::load_model(tflite::MicroMutableOpResolver<20> &op_resolver)
   }
 
   if (this->interpreter_ == nullptr) {
-    this->interpreter_ = make_unique<tflite::MicroInterpreter>(
-        tflite::GetModel(this->model_start_), op_resolver, this->tensor_arena_, this->tensor_arena_size_, this->mrv_);
+    this->interpreter_ =
+        make_unique<tflite::MicroInterpreter>(tflite::GetModel(this->model_start_), this->streaming_op_resolver_,
+                                              this->tensor_arena_, this->tensor_arena_size_, this->mrv_);
     if (this->interpreter_->AllocateTensors() != kTfLiteOk) {
       ESP_LOGE(TAG, "Failed to allocate tensors for the streaming model");
       return false;
@@ -84,34 +84,55 @@ bool StreamingModel::load_model(tflite::MicroMutableOpResolver<20> &op_resolver)
     }
   }
 
+  this->loaded_ = true;
+  this->reset_probabilities();
   return true;
 }
 
 void StreamingModel::unload_model() {
   this->interpreter_.reset();
 
-  ExternalRAMAllocator<uint8_t> arena_allocator(ExternalRAMAllocator<uint8_t>::ALLOW_FAILURE);
+  RAMAllocator<uint8_t> arena_allocator(RAMAllocator<uint8_t>::ALLOW_FAILURE);
+
+  if (this->tensor_arena_ != nullptr) {
+    arena_allocator.deallocate(this->tensor_arena_, this->tensor_arena_size_);
+    this->tensor_arena_ = nullptr;
+  }
+
+  if (this->var_arena_ != nullptr) {
+    arena_allocator.deallocate(this->var_arena_, STREAMING_MODEL_VARIABLE_ARENA_SIZE);
+    this->var_arena_ = nullptr;
+  }
 
-  arena_allocator.deallocate(this->tensor_arena_, this->tensor_arena_size_);
-  this->tensor_arena_ = nullptr;
-  arena_allocator.deallocate(this->var_arena_, STREAMING_MODEL_VARIABLE_ARENA_SIZE);
-  this->var_arena_ = nullptr;
+  this->loaded_ = false;
 }
 
 bool StreamingModel::perform_streaming_inference(const int8_t features[PREPROCESSOR_FEATURE_SIZE]) {
-  if (this->interpreter_ != nullptr) {
+  if (this->enabled_ && !this->loaded_) {
+    // Model is enabled but isn't loaded
+    if (!this->load_model_()) {
+      return false;
+    }
+  }
+
+  if (!this->enabled_ && this->loaded_) {
+    // Model is disabled but still loaded
+    this->unload_model();
+    return true;
+  }
+
+  if (this->loaded_) {
     TfLiteTensor *input = this->interpreter_->input(0);
 
+    uint8_t stride = this->interpreter_->input(0)->dims->data[1];
+    this->current_stride_step_ = this->current_stride_step_ % stride;
+
     std::memmove(
         (int8_t *) (tflite::GetTensorData<int8_t>(input)) + PREPROCESSOR_FEATURE_SIZE * this->current_stride_step_,
         features, PREPROCESSOR_FEATURE_SIZE);
     ++this->current_stride_step_;
 
-    uint8_t stride = this->interpreter_->input(0)->dims->data[1];
-
     if (this->current_stride_step_ >= stride) {
-      this->current_stride_step_ = 0;
-
       TfLiteStatus invoke_status = this->interpreter_->Invoke();
       if (invoke_status != kTfLiteOk) {
         ESP_LOGW(TAG, "Streaming interpreter invoke failed");
@@ -124,65 +145,161 @@ bool StreamingModel::perform_streaming_inference(const int8_t features[PREPROCES
       if (this->last_n_index_ == this->sliding_window_size_)
         this->last_n_index_ = 0;
       this->recent_streaming_probabilities_[this->last_n_index_] = output->data.uint8[0];  // probability;
+      this->unprocessed_probability_status_ = true;
     }
-    return true;
+    this->ignore_windows_ = std::min(this->ignore_windows_ + 1, 0);
   }
-  ESP_LOGE(TAG, "Streaming interpreter is not initialized.");
-  return false;
+  return true;
 }
 
 void StreamingModel::reset_probabilities() {
   for (auto &prob : this->recent_streaming_probabilities_) {
     prob = 0;
   }
+  this->ignore_windows_ = -MIN_SLICES_BEFORE_DETECTION;
 }
 
-WakeWordModel::WakeWordModel(const uint8_t *model_start, float probability_cutoff, size_t sliding_window_average_size,
-                             const std::string &wake_word, size_t tensor_arena_size) {
+WakeWordModel::WakeWordModel(const std::string &id, const uint8_t *model_start, uint8_t default_probability_cutoff,
+                             size_t sliding_window_average_size, const std::string &wake_word, size_t tensor_arena_size,
+                             bool default_enabled, bool internal_only) {
+  this->id_ = id;
   this->model_start_ = model_start;
-  this->probability_cutoff_ = probability_cutoff;
+  this->default_probability_cutoff_ = default_probability_cutoff;
+  this->probability_cutoff_ = default_probability_cutoff;
   this->sliding_window_size_ = sliding_window_average_size;
   this->recent_streaming_probabilities_.resize(sliding_window_average_size, 0);
   this->wake_word_ = wake_word;
   this->tensor_arena_size_ = tensor_arena_size;
+  this->register_streaming_ops_(this->streaming_op_resolver_);
+  this->current_stride_step_ = 0;
+  this->internal_only_ = internal_only;
+
+  this->pref_ = global_preferences->make_preference<bool>(fnv1_hash(id));
+  bool enabled;
+  if (this->pref_.load(&enabled)) {
+    // Use the enabled state loaded from flash
+    this->enabled_ = enabled;
+  } else {
+    // If no state saved, then use the default
+    this->enabled_ = default_enabled;
+  }
 };
 
-bool WakeWordModel::determine_detected() {
+void WakeWordModel::enable() {
+  this->enabled_ = true;
+  if (!this->internal_only_) {
+    this->pref_.save(&this->enabled_);
+  }
+}
+
+void WakeWordModel::disable() {
+  this->enabled_ = false;
+  if (!this->internal_only_) {
+    this->pref_.save(&this->enabled_);
+  }
+}
+
+DetectionEvent WakeWordModel::determine_detected() {
+  DetectionEvent detection_event;
+  detection_event.wake_word = &this->wake_word_;
+  detection_event.max_probability = 0;
+  detection_event.average_probability = 0;
+
+  if ((this->ignore_windows_ < 0) || !this->enabled_) {
+    detection_event.detected = false;
+    return detection_event;
+  }
+
   uint32_t sum = 0;
   for (auto &prob : this->recent_streaming_probabilities_) {
+    detection_event.max_probability = std::max(detection_event.max_probability, prob);
     sum += prob;
   }
 
-  float sliding_window_average = static_cast<float>(sum) / static_cast<float>(255 * this->sliding_window_size_);
+  detection_event.average_probability = sum / this->sliding_window_size_;
+  detection_event.detected = sum > this->probability_cutoff_ * this->sliding_window_size_;
 
-  // Detect the wake word if the sliding window average is above the cutoff
-  if (sliding_window_average > this->probability_cutoff_) {
-    ESP_LOGD(TAG, "The '%s' model sliding average probability is %.3f and most recent probability is %.3f",
-             this->wake_word_.c_str(), sliding_window_average,
-             this->recent_streaming_probabilities_[this->last_n_index_] / (255.0));
-    return true;
-  }
-  return false;
+  this->unprocessed_probability_status_ = false;
+  return detection_event;
 }
 
-VADModel::VADModel(const uint8_t *model_start, float probability_cutoff, size_t sliding_window_size,
+VADModel::VADModel(const uint8_t *model_start, uint8_t default_probability_cutoff, size_t sliding_window_size,
                    size_t tensor_arena_size) {
   this->model_start_ = model_start;
-  this->probability_cutoff_ = probability_cutoff;
+  this->default_probability_cutoff_ = default_probability_cutoff;
+  this->probability_cutoff_ = default_probability_cutoff;
   this->sliding_window_size_ = sliding_window_size;
   this->recent_streaming_probabilities_.resize(sliding_window_size, 0);
   this->tensor_arena_size_ = tensor_arena_size;
-};
+  this->register_streaming_ops_(this->streaming_op_resolver_);
+}
+
+DetectionEvent VADModel::determine_detected() {
+  DetectionEvent detection_event;
+  detection_event.max_probability = 0;
+  detection_event.average_probability = 0;
+
+  if (!this->enabled_) {
+    // We disabled the VAD model for some reason... so we shouldn't block wake words from being detected
+    detection_event.detected = true;
+    return detection_event;
+  }
 
-bool VADModel::determine_detected() {
   uint32_t sum = 0;
   for (auto &prob : this->recent_streaming_probabilities_) {
+    detection_event.max_probability = std::max(detection_event.max_probability, prob);
     sum += prob;
   }
 
-  float sliding_window_average = static_cast<float>(sum) / static_cast<float>(255 * this->sliding_window_size_);
+  detection_event.average_probability = sum / this->sliding_window_size_;
+  detection_event.detected = sum > (this->probability_cutoff_ * this->sliding_window_size_);
+
+  return detection_event;
+}
 
-  return sliding_window_average > this->probability_cutoff_;
+bool StreamingModel::register_streaming_ops_(tflite::MicroMutableOpResolver<20> &op_resolver) {
+  if (op_resolver.AddCallOnce() != kTfLiteOk)
+    return false;
+  if (op_resolver.AddVarHandle() != kTfLiteOk)
+    return false;
+  if (op_resolver.AddReshape() != kTfLiteOk)
+    return false;
+  if (op_resolver.AddReadVariable() != kTfLiteOk)
+    return false;
+  if (op_resolver.AddStridedSlice() != kTfLiteOk)
+    return false;
+  if (op_resolver.AddConcatenation() != kTfLiteOk)
+    return false;
+  if (op_resolver.AddAssignVariable() != kTfLiteOk)
+    return false;
+  if (op_resolver.AddConv2D() != kTfLiteOk)
+    return false;
+  if (op_resolver.AddMul() != kTfLiteOk)
+    return false;
+  if (op_resolver.AddAdd() != kTfLiteOk)
+    return false;
+  if (op_resolver.AddMean() != kTfLiteOk)
+    return false;
+  if (op_resolver.AddFullyConnected() != kTfLiteOk)
+    return false;
+  if (op_resolver.AddLogistic() != kTfLiteOk)
+    return false;
+  if (op_resolver.AddQuantize() != kTfLiteOk)
+    return false;
+  if (op_resolver.AddDepthwiseConv2D() != kTfLiteOk)
+    return false;
+  if (op_resolver.AddAveragePool2D() != kTfLiteOk)
+    return false;
+  if (op_resolver.AddMaxPool2D() != kTfLiteOk)
+    return false;
+  if (op_resolver.AddPad() != kTfLiteOk)
+    return false;
+  if (op_resolver.AddPack() != kTfLiteOk)
+    return false;
+  if (op_resolver.AddSplitV() != kTfLiteOk)
+    return false;
+
+  return true;
 }
 
 }  // namespace micro_wake_word