ESPHome  2024.9.2
voice_assistant.h
Go to the documentation of this file.
1 #pragma once
2 
3 #include "esphome/core/defines.h"
4 
5 #ifdef USE_VOICE_ASSISTANT
6 
9 #include "esphome/core/helpers.h"
11 
15 #ifdef USE_SPEAKER
17 #endif
18 #ifdef USE_MEDIA_PLAYER
20 #endif
22 
23 #ifdef USE_ESP_ADF
24 #include <esp_vad.h>
25 #endif
26 
27 #include <unordered_map>
28 #include <vector>
29 
30 namespace esphome {
31 namespace voice_assistant {
32 
33 // Version 1: Initial version
34 // Version 2: Adds raw speaker support
35 static const uint32_t LEGACY_INITIAL_VERSION = 1;
36 static const uint32_t LEGACY_SPEAKER_SUPPORT = 2;
37 
38 enum VoiceAssistantFeature : uint32_t {
40  FEATURE_SPEAKER = 1 << 1,
42  FEATURE_TIMERS = 1 << 3,
43 };
44 
45 enum class State {
46  IDLE,
59 };
60 
61 enum AudioMode : uint8_t {
64 };
65 
66 struct Timer {
67  std::string id;
68  std::string name;
69  uint32_t total_seconds;
70  uint32_t seconds_left;
71  bool is_active;
72 
73  std::string to_string() const {
74  return str_sprintf("Timer(id=%s, name=%s, total_seconds=%" PRIu32 ", seconds_left=%" PRIu32 ", is_active=%s)",
75  this->id.c_str(), this->name.c_str(), this->total_seconds, this->seconds_left,
76  YESNO(this->is_active));
77  }
78 };
79 
80 struct WakeWord {
81  std::string id;
82  std::string wake_word;
83  std::vector<std::string> trained_languages;
84 };
85 
86 struct Configuration {
87  std::vector<WakeWord> available_wake_words;
88  std::vector<std::string> active_wake_words;
90 };
91 
92 class VoiceAssistant : public Component {
93  public:
94  void setup() override;
95  void loop() override;
96  float get_setup_priority() const override;
97  void start_streaming();
98  void start_streaming(struct sockaddr_storage *addr, uint16_t port);
99  void failed_to_start();
100 
101  void set_microphone(microphone::Microphone *mic) { this->mic_ = mic; }
102 #ifdef USE_SPEAKER
103  void set_speaker(speaker::Speaker *speaker) {
104  this->speaker_ = speaker;
105  this->local_output_ = true;
106  }
107 #endif
108 #ifdef USE_MEDIA_PLAYER
110  this->media_player_ = media_player;
111  this->local_output_ = true;
112  }
113 #endif
114 
115  uint32_t get_legacy_version() const {
116 #ifdef USE_SPEAKER
117  if (this->speaker_ != nullptr) {
118  return LEGACY_SPEAKER_SUPPORT;
119  }
120 #endif
121  return LEGACY_INITIAL_VERSION;
122  }
123 
124  uint32_t get_feature_flags() const {
125  uint32_t flags = 0;
128 #ifdef USE_SPEAKER
129  if (this->speaker_ != nullptr) {
131  }
132 #endif
133 
134  if (this->has_timers_) {
136  }
137 
138  return flags;
139  }
140 
141  void request_start(bool continuous, bool silence_detection);
142  void request_stop();
143 
144  void on_event(const api::VoiceAssistantEventResponse &msg);
145  void on_audio(const api::VoiceAssistantAudio &msg);
146  void on_timer_event(const api::VoiceAssistantTimerEventResponse &msg);
147  void on_announce(const api::VoiceAssistantAnnounceRequest &msg);
148  void on_set_configuration(const std::vector<std::string> &active_wake_words){};
149  const Configuration &get_configuration() { return this->config_; };
150 
151  bool is_running() const { return this->state_ != State::IDLE; }
152  void set_continuous(bool continuous) { this->continuous_ = continuous; }
153  bool is_continuous() const { return this->continuous_; }
154 
155  void set_use_wake_word(bool use_wake_word) { this->use_wake_word_ = use_wake_word; }
156 #ifdef USE_ESP_ADF
157  void set_vad_threshold(uint8_t vad_threshold) { this->vad_threshold_ = vad_threshold; }
158 #endif
159 
160  void set_noise_suppression_level(uint8_t noise_suppression_level) {
161  this->noise_suppression_level_ = noise_suppression_level;
162  }
163  void set_auto_gain(uint8_t auto_gain) { this->auto_gain_ = auto_gain; }
164  void set_volume_multiplier(float volume_multiplier) { this->volume_multiplier_ = volume_multiplier; }
165  void set_conversation_timeout(uint32_t conversation_timeout) { this->conversation_timeout_ = conversation_timeout; }
166  void reset_conversation_id();
167 
168  Trigger<> *get_intent_end_trigger() const { return this->intent_end_trigger_; }
169  Trigger<> *get_intent_start_trigger() const { return this->intent_start_trigger_; }
170  Trigger<> *get_listening_trigger() const { return this->listening_trigger_; }
171  Trigger<> *get_end_trigger() const { return this->end_trigger_; }
172  Trigger<> *get_start_trigger() const { return this->start_trigger_; }
173  Trigger<> *get_stt_vad_end_trigger() const { return this->stt_vad_end_trigger_; }
174  Trigger<> *get_stt_vad_start_trigger() const { return this->stt_vad_start_trigger_; }
175 #ifdef USE_SPEAKER
176  Trigger<> *get_tts_stream_start_trigger() const { return this->tts_stream_start_trigger_; }
177  Trigger<> *get_tts_stream_end_trigger() const { return this->tts_stream_end_trigger_; }
178 #endif
179  Trigger<> *get_wake_word_detected_trigger() const { return this->wake_word_detected_trigger_; }
180  Trigger<std::string> *get_stt_end_trigger() const { return this->stt_end_trigger_; }
181  Trigger<std::string> *get_tts_end_trigger() const { return this->tts_end_trigger_; }
182  Trigger<std::string> *get_tts_start_trigger() const { return this->tts_start_trigger_; }
183  Trigger<std::string, std::string> *get_error_trigger() const { return this->error_trigger_; }
184  Trigger<> *get_idle_trigger() const { return this->idle_trigger_; }
185 
186  Trigger<> *get_client_connected_trigger() const { return this->client_connected_trigger_; }
187  Trigger<> *get_client_disconnected_trigger() const { return this->client_disconnected_trigger_; }
188 
189  void client_subscription(api::APIConnection *client, bool subscribe);
190  api::APIConnection *get_api_connection() const { return this->api_client_; }
191 
192  void set_wake_word(const std::string &wake_word) { this->wake_word_ = wake_word; }
193 
194  Trigger<Timer> *get_timer_started_trigger() const { return this->timer_started_trigger_; }
195  Trigger<Timer> *get_timer_updated_trigger() const { return this->timer_updated_trigger_; }
196  Trigger<Timer> *get_timer_cancelled_trigger() const { return this->timer_cancelled_trigger_; }
197  Trigger<Timer> *get_timer_finished_trigger() const { return this->timer_finished_trigger_; }
198  Trigger<std::vector<Timer>> *get_timer_tick_trigger() const { return this->timer_tick_trigger_; }
199  void set_has_timers(bool has_timers) { this->has_timers_ = has_timers; }
200  const std::unordered_map<std::string, Timer> &get_timers() const { return this->timers_; }
201 
202  protected:
203  bool allocate_buffers_();
204  void clear_buffers_();
205  void deallocate_buffers_();
206 
207  int read_microphone_();
208  void set_state_(State state);
209  void set_state_(State state, State desired_state);
210  void signal_stop_();
211 
212  std::unique_ptr<socket::Socket> socket_ = nullptr;
213  struct sockaddr_storage dest_addr_;
214 
215  Trigger<> *intent_end_trigger_ = new Trigger<>();
216  Trigger<> *intent_start_trigger_ = new Trigger<>();
217  Trigger<> *listening_trigger_ = new Trigger<>();
218  Trigger<> *end_trigger_ = new Trigger<>();
219  Trigger<> *start_trigger_ = new Trigger<>();
220  Trigger<> *stt_vad_start_trigger_ = new Trigger<>();
221  Trigger<> *stt_vad_end_trigger_ = new Trigger<>();
222 #ifdef USE_SPEAKER
223  Trigger<> *tts_stream_start_trigger_ = new Trigger<>();
224  Trigger<> *tts_stream_end_trigger_ = new Trigger<>();
225 #endif
226  Trigger<> *wake_word_detected_trigger_ = new Trigger<>();
227  Trigger<std::string> *stt_end_trigger_ = new Trigger<std::string>();
228  Trigger<std::string> *tts_end_trigger_ = new Trigger<std::string>();
229  Trigger<std::string> *tts_start_trigger_ = new Trigger<std::string>();
231  Trigger<> *idle_trigger_ = new Trigger<>();
232 
233  Trigger<> *client_connected_trigger_ = new Trigger<>();
234  Trigger<> *client_disconnected_trigger_ = new Trigger<>();
235 
236  api::APIConnection *api_client_{nullptr};
237 
238  std::unordered_map<std::string, Timer> timers_;
239  void timer_tick_();
240  Trigger<Timer> *timer_started_trigger_ = new Trigger<Timer>();
241  Trigger<Timer> *timer_finished_trigger_ = new Trigger<Timer>();
242  Trigger<Timer> *timer_updated_trigger_ = new Trigger<Timer>();
243  Trigger<Timer> *timer_cancelled_trigger_ = new Trigger<Timer>();
245  bool has_timers_{false};
246  bool timer_tick_running_{false};
247 
248  microphone::Microphone *mic_{nullptr};
249 #ifdef USE_SPEAKER
250  void write_speaker_();
251  speaker::Speaker *speaker_{nullptr};
252  uint8_t *speaker_buffer_;
253  size_t speaker_buffer_index_{0};
254  size_t speaker_buffer_size_{0};
255  size_t speaker_bytes_received_{0};
256  bool wait_for_stream_end_{false};
257  bool stream_ended_{false};
258 #endif
259 #ifdef USE_MEDIA_PLAYER
260  media_player::MediaPlayer *media_player_{nullptr};
261 #endif
262 
263  bool local_output_{false};
264 
265  std::string conversation_id_{""};
266 
267  std::string wake_word_{""};
268 
270 
271 #ifdef USE_ESP_ADF
272  vad_handle_t vad_instance_;
273  uint8_t vad_threshold_{5};
274  uint8_t vad_counter_{0};
275 #endif
276  std::unique_ptr<RingBuffer> ring_buffer_;
277 
280  uint8_t auto_gain_;
283 
284  uint8_t *send_buffer_;
285  int16_t *input_buffer_;
286 
287  bool continuous_{false};
289 
291  State desired_state_{State::IDLE};
292 
293  AudioMode audio_mode_{AUDIO_MODE_UDP};
294  bool udp_socket_running_{false};
295  bool start_udp_socket_();
296 
297  Configuration config_{};
298 };
299 
300 template<typename... Ts> class StartAction : public Action<Ts...>, public Parented<VoiceAssistant> {
301  TEMPLATABLE_VALUE(std::string, wake_word);
302 
303  public:
304  void play(Ts... x) override {
305  this->parent_->set_wake_word(this->wake_word_.value(x...));
306  this->parent_->request_start(false, this->silence_detection_);
307  }
308 
309  void set_silence_detection(bool silence_detection) { this->silence_detection_ = silence_detection; }
310 
311  protected:
313 };
314 
315 template<typename... Ts> class StartContinuousAction : public Action<Ts...>, public Parented<VoiceAssistant> {
316  public:
317  void play(Ts... x) override { this->parent_->request_start(true, true); }
318 };
319 
320 template<typename... Ts> class StopAction : public Action<Ts...>, public Parented<VoiceAssistant> {
321  public:
322  void play(Ts... x) override { this->parent_->request_stop(); }
323 };
324 
325 template<typename... Ts> class IsRunningCondition : public Condition<Ts...>, public Parented<VoiceAssistant> {
326  public:
327  bool check(Ts... x) override { return this->parent_->is_running() || this->parent_->is_continuous(); }
328 };
329 
330 template<typename... Ts> class ConnectedCondition : public Condition<Ts...>, public Parented<VoiceAssistant> {
331  public:
332  bool check(Ts... x) override { return this->parent_->get_api_connection() != nullptr; }
333 };
334 
335 extern VoiceAssistant *global_voice_assistant; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables)
336 
337 } // namespace voice_assistant
338 } // namespace esphome
339 
340 #endif // USE_VOICE_ASSISTANT
void setup()
Trigger< Timer > * get_timer_finished_trigger() const
void loop()
void set_microphone(microphone::Microphone *mic)
Trigger< Timer > * get_timer_started_trigger() const
void on_set_configuration(const std::vector< std::string > &active_wake_words)
std::unordered_map< std::string, Timer > timers_
HighFrequencyLoopRequester high_freq_
VoiceAssistant * global_voice_assistant
Trigger< Timer > * get_timer_cancelled_trigger() const
uint16_t x
Definition: tt21100.cpp:17
Helper class to request loop() to be called as fast as possible.
Definition: helpers.h:609
api::APIConnection * get_api_connection() const
const std::unordered_map< std::string, Timer > & get_timers() const
Trigger< std::vector< Timer > > * get_timer_tick_trigger() const
void set_wake_word(const std::string &wake_word)
void set_noise_suppression_level(uint8_t noise_suppression_level)
std::vector< std::string > trained_languages
void set_volume_multiplier(float volume_multiplier)
Base class for all automation conditions.
Definition: automation.h:74
Trigger< std::string > * get_tts_start_trigger() const
std::string str_sprintf(const char *fmt,...)
Definition: helpers.cpp:312
void set_conversation_timeout(uint32_t conversation_timeout)
std::vector< std::string > active_wake_words
const uint32_t flags
Definition: stm32flash.h:85
Trigger< std::string, std::string > * get_error_trigger() const
std::unique_ptr< RingBuffer > ring_buffer_
void set_speaker(speaker::Speaker *speaker)
void set_vad_threshold(uint8_t vad_threshold)
Trigger< std::string > * get_tts_end_trigger() const
void set_use_wake_word(bool use_wake_word)
void set_media_player(media_player::MediaPlayer *media_player)
std::vector< WakeWord > available_wake_words
void set_silence_detection(bool silence_detection)
Implementation of SPI Controller mode.
Definition: a01nyub.cpp:7
Trigger< std::string > * get_stt_end_trigger() const
Trigger< Timer > * get_timer_updated_trigger() const
Helper class to easily give an object a parent of type T.
Definition: helpers.h:521
bool state
Definition: fan.h:34