From bab9c7c70e3c4185675506dec4e6cae86f322bfd Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Thu, 31 Aug 2023 17:38:49 +1200 Subject: [PATCH] Remove afe and try to use algorithm stream --- .../esp_adf/microphone/esp_adf_microphone.cpp | 219 ++++++++---------- .../esp_adf/microphone/esp_adf_microphone.h | 21 +- .../esp_adf/speaker/esp_adf_speaker.cpp | 8 +- 3 files changed, 103 insertions(+), 145 deletions(-) diff --git a/esphome/components/esp_adf/microphone/esp_adf_microphone.cpp b/esphome/components/esp_adf/microphone/esp_adf_microphone.cpp index 219249f2ec..e8d5484c4b 100644 --- a/esphome/components/esp_adf/microphone/esp_adf_microphone.cpp +++ b/esphome/components/esp_adf/microphone/esp_adf_microphone.cpp @@ -7,58 +7,32 @@ #include "esphome/core/hal.h" #include "esphome/core/log.h" +#include +#include #include +#include #include #include #include #include +#include + namespace esphome { namespace esp_adf { static const char *const TAG = "esp_adf.microphone"; void ESPADFMicrophone::setup() { - this->ring_buffer_ = rb_create(BUFFER_SIZE, sizeof(int16_t)); + this->ring_buffer_ = rb_create(8000, sizeof(int16_t)); if (this->ring_buffer_ == nullptr) { ESP_LOGW(TAG, "Could not allocate ring buffer."); this->mark_failed(); return; } - this->feed_event_queue_ = xQueueCreate(20, sizeof(TaskEvent)); - this->feed_command_queue_ = xQueueCreate(20, sizeof(CommandEvent)); - this->fetch_command_queue_ = xQueueCreate(20, sizeof(CommandEvent)); - afe_config_t cfg_afe = { - .aec_init = false, - .se_init = true, - .vad_init = true, - .wakenet_init = false, - .voice_communication_init = false, - .voice_communication_agc_init = false, - .voice_communication_agc_gain = 15, - .vad_mode = VAD_MODE_3, - .wakenet_model_name = nullptr, - .wakenet_mode = DET_MODE_2CH_90, - .afe_mode = SR_MODE_HIGH_PERF, - .afe_perferred_core = 1, - .afe_perferred_priority = 5, - .afe_ringbuf_size = 50, - .memory_alloc_mode = AFE_MEMORY_ALLOC_MORE_PSRAM, - .agc_mode = AFE_MN_PEAK_AGC_MODE_3, - .pcm_config = - { - .total_ch_num = 1, - .mic_num = 1, - .ref_num = 0, - .sample_rate = 16000, - }, - .debug_init = false, - .debug_hook = {{AFE_DEBUG_HOOK_MASE_TASK_IN, NULL}, {AFE_DEBUG_HOOK_FETCH_TASK_IN, NULL}}, - }; - - this->afe_data_ = this->afe_handle_->create_from_config(&cfg_afe); - this->afe_chunk_size_ = this->afe_handle_->get_feed_chunksize(this->afe_data_); + this->read_event_queue_ = xQueueCreate(20, sizeof(TaskEvent)); + this->read_command_queue_ = xQueueCreate(20, sizeof(CommandEvent)); } void ESPADFMicrophone::start() { @@ -74,42 +48,48 @@ void ESPADFMicrophone::start_() { if (!this->parent_->try_lock()) { return; } - this->state_ = microphone::STATE_RUNNING; - xTaskCreate(ESPADFMicrophone::feed_task, "feed_task", 8192, (void *) this, 0, &this->feed_task_handle_); - xTaskCreate(ESPADFMicrophone::fetch_task, "fetch_task", 8192, (void *) this, 0, &this->fetch_task_handle_); + xTaskCreate(ESPADFMicrophone::read_task, "read_task", 8192, (void *) this, 0, &this->read_task_handle_); } -void ESPADFMicrophone::feed_task(void *params) { +void ESPADFMicrophone::read_task(void *params) { ESPADFMicrophone *this_mic = (ESPADFMicrophone *) params; TaskEvent event; - event.type = TaskEventType::STARTING; - xQueueSend(this_mic->feed_event_queue_, &event, portMAX_DELAY); - - size_t buffer_size = this_mic->afe_chunk_size_ * sizeof(int16_t); - ExternalRAMAllocator allocator(ExternalRAMAllocator::ALLOW_FAILURE); - int16_t *afe_buffer = allocator.allocate(this_mic->afe_chunk_size_); - if (afe_buffer == nullptr) { + int16_t *buffer = allocator.allocate(BUFFER_SIZE / sizeof(int16_t)); + if (buffer == nullptr) { + event.type = TaskEventType::WARNING; + event.err = ESP_ERR_NO_MEM; + xQueueSend(this_mic->read_event_queue_, &event, portMAX_DELAY); + event.type = TaskEventType::STOPPED; - xQueueSend(this_mic->feed_event_queue_, &event, portMAX_DELAY); + event.err = ESP_OK; + xQueueSend(this_mic->read_event_queue_, &event, portMAX_DELAY); while (true) { delay(10); - }; + } return; } + event.type = TaskEventType::STARTING; + xQueueSend(this_mic->read_event_queue_, &event, portMAX_DELAY); + + audio_pipeline_cfg_t pipeline_cfg = { + .rb_size = 8 * 1024, + }; + audio_pipeline_handle_t pipeline = audio_pipeline_init(&pipeline_cfg); + i2s_driver_config_t i2s_config = { .mode = (i2s_mode_t) (I2S_MODE_MASTER | I2S_MODE_RX), - .sample_rate = 44100, + .sample_rate = 16000, .bits_per_sample = I2S_BITS_PER_SAMPLE_16BIT, .channel_format = I2S_CHANNEL_FMT_RIGHT_LEFT, .communication_format = I2S_COMM_FORMAT_STAND_I2S, .intr_alloc_flags = ESP_INTR_FLAG_LEVEL2 | ESP_INTR_FLAG_IRAM, - .dma_buf_count = 3, - .dma_buf_len = 300, + .dma_buf_count = 8, + .dma_buf_len = 128, .use_apll = false, .tx_desc_auto_clear = true, .fixed_mclk = 0, @@ -117,11 +97,6 @@ void ESPADFMicrophone::feed_task(void *params) { .bits_per_chan = I2S_BITS_PER_CHAN_DEFAULT, }; - audio_pipeline_cfg_t pipeline_cfg = { - .rb_size = 8 * 1024, - }; - audio_pipeline_handle_t pipeline = audio_pipeline_init(&pipeline_cfg); - i2s_stream_cfg_t i2s_cfg = { .type = AUDIO_STREAM_READER, .i2s_config = i2s_config, @@ -141,7 +116,7 @@ void ESPADFMicrophone::feed_task(void *params) { audio_element_handle_t i2s_stream_reader = i2s_stream_init(&i2s_cfg); rsp_filter_cfg_t rsp_cfg = { - .src_rate = 44100, + .src_rate = 16000, .src_ch = 2, .dest_rate = 16000, .dest_bits = 16, @@ -162,6 +137,30 @@ void ESPADFMicrophone::feed_task(void *params) { }; audio_element_handle_t filter = rsp_filter_init(&rsp_cfg); + algorithm_stream_cfg_t algo_cfg = { + .input_type = ALGORITHM_STREAM_INPUT_TYPE1, + .task_stack = 10 * 1024, + .task_prio = ALGORITHM_STREAM_TASK_PERIOD, + .task_core = ALGORITHM_STREAM_PINNED_TO_CORE, + .out_rb_size = ALGORITHM_STREAM_RINGBUFFER_SIZE, + .stack_in_ext = true, + .rec_linear_factor = 1, + .ref_linear_factor = 1, + .debug_input = false, + .swap_ch = false, + // .algo_mask = ALGORITHM_STREAM_USE_AGC, + // .algo_mask = (ALGORITHM_STREAM_USE_AEC | ALGORITHM_STREAM_USE_AGC | ALGORITHM_STREAM_USE_NS), + // .algo_mask = (ALGORITHM_STREAM_USE_AGC | ALGORITHM_STREAM_USE_NS), + .algo_mask = (ALGORITHM_STREAM_USE_AEC | ALGORITHM_STREAM_USE_NS), + // .algo_mask = (ALGORITHM_STREAM_USE_NS), + .sample_rate = 16000, + .mic_ch = 1, + .agc_gain = 10, + .aec_low_cost = false, + }; + + // audio_element_handle_t algo_stream = algo_stream_init(&algo_cfg); + raw_stream_cfg_t raw_cfg = { .type = AUDIO_STREAM_READER, .out_rb_size = 8 * 1024, @@ -170,28 +169,33 @@ void ESPADFMicrophone::feed_task(void *params) { audio_pipeline_register(pipeline, i2s_stream_reader, "i2s"); audio_pipeline_register(pipeline, filter, "filter"); + // audio_pipeline_register(pipeline, algo_stream, "algo"); audio_pipeline_register(pipeline, raw_read, "raw"); - const char *link_tag[3] = {"i2s", "filter", "raw"}; + const char *link_tag[4] = { + "i2s", + "filter", + // "algo", + "raw", + }; audio_pipeline_link(pipeline, &link_tag[0], 3); audio_pipeline_run(pipeline); event.type = TaskEventType::STARTED; - xQueueSend(this_mic->feed_event_queue_, &event, portMAX_DELAY); + xQueueSend(this_mic->read_event_queue_, &event, portMAX_DELAY); CommandEvent command_event; - size_t fill_count = 0; while (true) { - if (xQueueReceive(this_mic->feed_command_queue_, &command_event, 0) == pdTRUE) { + if (xQueueReceive(this_mic->read_command_queue_, &command_event, 0) == pdTRUE) { if (command_event.stop) { // Stop signal from main thread break; } } - int bytes_read = raw_stream_read(raw_read, (char *) (afe_buffer + fill_count), buffer_size - fill_count); + int bytes_read = raw_stream_read(raw_read, (char *) buffer, BUFFER_SIZE); if (bytes_read == -2 || bytes_read == 0) { // No data in buffers to read. @@ -199,72 +203,43 @@ void ESPADFMicrophone::feed_task(void *params) { } else if (bytes_read < 0) { event.type = TaskEventType::WARNING; event.err = bytes_read; - xQueueSend(this_mic->feed_event_queue_, &event, 0); + xQueueSend(this_mic->read_event_queue_, &event, 0); continue; } event.type = TaskEventType::RUNNING; - event.err = ESP_OK; - xQueueSend(this_mic->feed_event_queue_, &event, 0); + event.err = bytes_read; + xQueueSend(this_mic->read_event_queue_, &event, 0); - fill_count += bytes_read; - - if (fill_count == buffer_size) { - this_mic->afe_handle_->feed(this_mic->afe_data_, afe_buffer); - fill_count -= buffer_size; + int available = rb_bytes_available(this_mic->ring_buffer_); + if (available < bytes_read) { + rb_read(this_mic->ring_buffer_, nullptr, bytes_read - available, 0); } + rb_write(this_mic->ring_buffer_, (char *) buffer, bytes_read, 0); } - allocator.deallocate(afe_buffer, this_mic->afe_chunk_size_); + allocator.deallocate(buffer, BUFFER_SIZE / sizeof(int16_t)); audio_pipeline_stop(pipeline); audio_pipeline_wait_for_stop(pipeline); audio_pipeline_terminate(pipeline); event.type = TaskEventType::STOPPING; - xQueueSend(this_mic->feed_event_queue_, &event, portMAX_DELAY); + xQueueSend(this_mic->read_event_queue_, &event, portMAX_DELAY); audio_pipeline_unregister(pipeline, i2s_stream_reader); audio_pipeline_unregister(pipeline, filter); + // audio_pipeline_unregister(pipeline, algo_stream); audio_pipeline_unregister(pipeline, raw_read); audio_pipeline_deinit(pipeline); audio_element_deinit(i2s_stream_reader); audio_element_deinit(filter); + // audio_element_deinit(algo_stream); audio_element_deinit(raw_read); event.type = TaskEventType::STOPPED; - xQueueSend(this_mic->feed_event_queue_, &event, portMAX_DELAY); - - while (true) { - delay(10); - } -} - -void ESPADFMicrophone::fetch_task(void *params) { - ESPADFMicrophone *this_mic = (ESPADFMicrophone *) params; - - CommandEvent event; - - while (true) { - if (xQueueReceive(this_mic->fetch_command_queue_, &event, 0) == pdTRUE) { - if (event.stop) { - // Stop signal from main thread - break; - } - } - afe_fetch_result_t *result = this_mic->afe_handle_->fetch(this_mic->afe_data_); - - if (result == nullptr) { - continue; - } - - int available = rb_bytes_available(this_mic->ring_buffer_); - if (available < result->data_size) { - rb_read(this_mic->ring_buffer_, nullptr, result->data_size - available, 0); - } - rb_write(this_mic->ring_buffer_, (char *) result->data, result->data_size, 0); - } + xQueueSend(this_mic->read_event_queue_, &event, portMAX_DELAY); while (true) { delay(10); @@ -272,31 +247,26 @@ void ESPADFMicrophone::fetch_task(void *params) { } void ESPADFMicrophone::stop() { - if (this->state_ == microphone::STATE_STOPPED || this->is_failed()) + if (this->state_ == microphone::STATE_STOPPED || this->state_ == microphone::STATE_STOPPING || this->is_failed()) return; - if (this->state_ == microphone::STATE_STARTING) { - this->state_ = microphone::STATE_STOPPED; - return; - } this->state_ = microphone::STATE_STOPPING; -} - -void ESPADFMicrophone::stop_() { - ESP_LOGD(TAG, "Stopping microphone"); CommandEvent command_event; command_event.stop = true; - xQueueSendToFront(this->feed_command_queue_, &command_event, portMAX_DELAY); - xQueueSendToFront(this->fetch_command_queue_, &command_event, portMAX_DELAY); + xQueueSendToFront(this->read_command_queue_, &command_event, portMAX_DELAY); + ESP_LOGD(TAG, "Stopping microphone"); } size_t ESPADFMicrophone::read(int16_t *buf, size_t len) { + if (rb_bytes_available(this->ring_buffer_) == 0) { + return 0; // No data + } int bytes_read = rb_read(this->ring_buffer_, (char *) buf, len, 0); - if (bytes_read == -2 || bytes_read == 0) { + if (bytes_read == -4 || bytes_read == -2 || bytes_read == 0) { // No data in buffers to read. return 0; } else if (bytes_read < 0) { - ESP_LOGW(TAG, "Error reading from I2S microphone"); + ESP_LOGW(TAG, "Error reading from I2S microphone %s (%d)", esp_err_to_name(bytes_read), bytes_read); this->status_set_warning(); return 0; } @@ -315,22 +285,24 @@ void ESPADFMicrophone::read_() { void ESPADFMicrophone::watch_() { TaskEvent event; - if (xQueueReceive(this->feed_event_queue_, &event, 0) == pdTRUE) { + if (xQueueReceive(this->read_event_queue_, &event, 0) == pdTRUE) { switch (event.type) { case TaskEventType::STARTING: - case TaskEventType::STARTED: case TaskEventType::STOPPING: break; + case TaskEventType::STARTED: + ESP_LOGD(TAG, "Microphone started"); + this->state_ = microphone::STATE_RUNNING; + break; case TaskEventType::RUNNING: this->status_clear_warning(); + // ESP_LOGD(TAG, "Putting %d bytes into ring buffer", event.err); break; case TaskEventType::STOPPED: this->parent_->unlock(); this->state_ = microphone::STATE_STOPPED; - vTaskDelete(this->feed_task_handle_); - vTaskDelete(this->fetch_task_handle_); - this->feed_task_handle_ = nullptr; - this->fetch_task_handle_ = nullptr; + vTaskDelete(this->read_task_handle_); + this->read_task_handle_ = nullptr; ESP_LOGD(TAG, "Microphone stopped"); break; case TaskEventType::WARNING: @@ -342,8 +314,10 @@ void ESPADFMicrophone::watch_() { } void ESPADFMicrophone::loop() { + this->watch_(); switch (this->state_) { case microphone::STATE_STOPPED: + case microphone::STATE_STOPPING: break; case microphone::STATE_STARTING: this->start_(); @@ -353,9 +327,6 @@ void ESPADFMicrophone::loop() { this->read_(); } break; - case microphone::STATE_STOPPING: - this->stop_(); - break; } } diff --git a/esphome/components/esp_adf/microphone/esp_adf_microphone.h b/esphome/components/esp_adf/microphone/esp_adf_microphone.h index b27b3baf08..7459242ff9 100644 --- a/esphome/components/esp_adf/microphone/esp_adf_microphone.h +++ b/esphome/components/esp_adf/microphone/esp_adf_microphone.h @@ -7,10 +7,6 @@ #include "esphome/components/microphone/microphone.h" #include "esphome/core/component.h" -#include -#include -#include -#include #include namespace esphome { @@ -28,25 +24,16 @@ class ESPADFMicrophone : public ESPADFPipeline, public microphone::Microphone, p protected: void start_(); - void stop_(); void read_(); void watch_(); - static void feed_task(void *params); - static void fetch_task(void *params); - - const esp_afe_sr_iface_t *afe_handle_{&ESP_AFE_SR_HANDLE}; - esp_afe_sr_data_t *afe_data_{nullptr}; - size_t afe_chunk_size_{0}; + static void read_task(void *params); ringbuf_handle_t ring_buffer_; - TaskHandle_t feed_task_handle_{nullptr}; - QueueHandle_t feed_event_queue_; - QueueHandle_t feed_command_queue_; - - TaskHandle_t fetch_task_handle_{nullptr}; - QueueHandle_t fetch_command_queue_; + TaskHandle_t read_task_handle_{nullptr}; + QueueHandle_t read_event_queue_; + QueueHandle_t read_command_queue_; }; } // namespace esp_adf diff --git a/esphome/components/esp_adf/speaker/esp_adf_speaker.cpp b/esphome/components/esp_adf/speaker/esp_adf_speaker.cpp index 8dadf20165..39bde1cdbf 100644 --- a/esphome/components/esp_adf/speaker/esp_adf_speaker.cpp +++ b/esphome/components/esp_adf/speaker/esp_adf_speaker.cpp @@ -32,7 +32,6 @@ void ESPADFSpeaker::start_() { if (!this->parent_->try_lock()) { return; // Waiting for another i2s component to return lock } - this->state_ = speaker::STATE_RUNNING; xTaskCreate(ESPADFSpeaker::player_task, "speaker_task", 8192, (void *) this, 0, &this->player_task_handle_); } @@ -211,9 +210,11 @@ void ESPADFSpeaker::watch_() { if (xQueueReceive(this->event_queue_, &event, 0) == pdTRUE) { switch (event.type) { case TaskEventType::STARTING: - case TaskEventType::STARTED: case TaskEventType::STOPPING: break; + case TaskEventType::STARTED: + this->state_ = speaker::STATE_RUNNING; + break; case TaskEventType::RUNNING: this->status_clear_warning(); break; @@ -232,13 +233,12 @@ void ESPADFSpeaker::watch_() { } void ESPADFSpeaker::loop() { + this->watch_(); switch (this->state_) { case speaker::STATE_STARTING: this->start_(); break; case speaker::STATE_RUNNING: - this->watch_(); - break; case speaker::STATE_STOPPING: case speaker::STATE_STOPPED: break;