sascha/esphome
1
0
Fork 0
mirror of https://github.com/esphome/esphome.git synced 2025-02-02 11:10:55 +00:00
Code Issues Releases Wiki Activity

Remove afe and try to use algorithm stream

Browse Source
This commit is contained in:
Jesse Hills 2023-08-31 17:38:49 +12:00
parent 0b60a1d9eb
commit bab9c7c70e
No known key found for this signature in database
GPG Key ID: BEAAE804EFD8E83A
3 changed files with 103 additions and 145 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

219
esphome/components/esp_adf/microphone/esp_adf_microphone.cpp
View File

@ -7,58 +7,32 @@
#include "esphome/core/hal.h" #include "esphome/core/hal.h"
#include "esphome/core/log.h" #include "esphome/core/log.h"
#include <algorithm_stream.h>
#include <audio_element.h>
#include <audio_hal.h> #include <audio_hal.h>
#include <audio_pipeline.h>
#include <filter_resample.h> #include <filter_resample.h>
#include <i2s_stream.h> #include <i2s_stream.h>
#include <raw_stream.h> #include <raw_stream.h>
#include <recorder_sr.h> #include <recorder_sr.h>
#include <board.h>
namespace esphome { namespace esphome {
namespace esp_adf { namespace esp_adf {
static const char *const TAG = "esp_adf.microphone"; static const char *const TAG = "esp_adf.microphone";
void ESPADFMicrophone::setup() { void ESPADFMicrophone::setup() {
this->ring_buffer_ = rb_create(BUFFER_SIZE, sizeof(int16_t)); this->ring_buffer_ = rb_create(8000, sizeof(int16_t));
if (this->ring_buffer_ == nullptr) { if (this->ring_buffer_ == nullptr) {
ESP_LOGW(TAG, "Could not allocate ring buffer."); ESP_LOGW(TAG, "Could not allocate ring buffer.");
this->mark_failed(); this->mark_failed();
return; return;
} }
this->feed_event_queue_ = xQueueCreate(20, sizeof(TaskEvent));
this->feed_command_queue_ = xQueueCreate(20, sizeof(CommandEvent));
this->fetch_command_queue_ = xQueueCreate(20, sizeof(CommandEvent));
afe_config_t cfg_afe = { this->read_event_queue_ = xQueueCreate(20, sizeof(TaskEvent));
.aec_init = false, this->read_command_queue_ = xQueueCreate(20, sizeof(CommandEvent));
.se_init = true,
.vad_init = true,
.wakenet_init = false,
.voice_communication_init = false,
.voice_communication_agc_init = false,
.voice_communication_agc_gain = 15,
.vad_mode = VAD_MODE_3,
.wakenet_model_name = nullptr,
.wakenet_mode = DET_MODE_2CH_90,
.afe_mode = SR_MODE_HIGH_PERF,
.afe_perferred_core = 1,
.afe_perferred_priority = 5,
.afe_ringbuf_size = 50,
.memory_alloc_mode = AFE_MEMORY_ALLOC_MORE_PSRAM,
.agc_mode = AFE_MN_PEAK_AGC_MODE_3,
.pcm_config =
{
.total_ch_num = 1,
.mic_num = 1,
.ref_num = 0,
.sample_rate = 16000,
},
.debug_init = false,
.debug_hook = {{AFE_DEBUG_HOOK_MASE_TASK_IN, NULL}, {AFE_DEBUG_HOOK_FETCH_TASK_IN, NULL}},
};
this->afe_data_ = this->afe_handle_->create_from_config(&cfg_afe);
this->afe_chunk_size_ = this->afe_handle_->get_feed_chunksize(this->afe_data_);
} }
void ESPADFMicrophone::start() { void ESPADFMicrophone::start() {
@ -74,42 +48,48 @@ void ESPADFMicrophone::start_() {
if (!this->parent_->try_lock()) { if (!this->parent_->try_lock()) {
return; return;
} }
this->state_ = microphone::STATE_RUNNING;
xTaskCreate(ESPADFMicrophone::feed_task, "feed_task", 8192, (void *) this, 0, &this->feed_task_handle_); xTaskCreate(ESPADFMicrophone::read_task, "read_task", 8192, (void *) this, 0, &this->read_task_handle_);
xTaskCreate(ESPADFMicrophone::fetch_task, "fetch_task", 8192, (void *) this, 0, &this->fetch_task_handle_);
} }
void ESPADFMicrophone::feed_task(void *params) { void ESPADFMicrophone::read_task(void *params) {
ESPADFMicrophone *this_mic = (ESPADFMicrophone *) params; ESPADFMicrophone *this_mic = (ESPADFMicrophone *) params;
TaskEvent event; TaskEvent event;
event.type = TaskEventType::STARTING;
xQueueSend(this_mic->feed_event_queue_, &event, portMAX_DELAY);
size_t buffer_size = this_mic->afe_chunk_size_ * sizeof(int16_t);
ExternalRAMAllocator<int16_t> allocator(ExternalRAMAllocator<int16_t>::ALLOW_FAILURE); ExternalRAMAllocator<int16_t> allocator(ExternalRAMAllocator<int16_t>::ALLOW_FAILURE);
int16_t *afe_buffer = allocator.allocate(this_mic->afe_chunk_size_); int16_t *buffer = allocator.allocate(BUFFER_SIZE / sizeof(int16_t));
if (afe_buffer == nullptr) { if (buffer == nullptr) {
event.type = TaskEventType::WARNING;
event.err = ESP_ERR_NO_MEM;
xQueueSend(this_mic->read_event_queue_, &event, portMAX_DELAY);
event.type = TaskEventType::STOPPED; event.type = TaskEventType::STOPPED;
xQueueSend(this_mic->feed_event_queue_, &event, portMAX_DELAY); event.err = ESP_OK;
xQueueSend(this_mic->read_event_queue_, &event, portMAX_DELAY);
while (true) { while (true) {
delay(10); delay(10);
}; }
return; return;
} }
event.type = TaskEventType::STARTING;
xQueueSend(this_mic->read_event_queue_, &event, portMAX_DELAY);
audio_pipeline_cfg_t pipeline_cfg = {
.rb_size = 8 * 1024,
};
audio_pipeline_handle_t pipeline = audio_pipeline_init(&pipeline_cfg);
i2s_driver_config_t i2s_config = { i2s_driver_config_t i2s_config = {
.mode = (i2s_mode_t) (I2S_MODE_MASTER | I2S_MODE_RX), .mode = (i2s_mode_t) (I2S_MODE_MASTER | I2S_MODE_RX),
.sample_rate = 44100, .sample_rate = 16000,
.bits_per_sample = I2S_BITS_PER_SAMPLE_16BIT, .bits_per_sample = I2S_BITS_PER_SAMPLE_16BIT,
.channel_format = I2S_CHANNEL_FMT_RIGHT_LEFT, .channel_format = I2S_CHANNEL_FMT_RIGHT_LEFT,
.communication_format = I2S_COMM_FORMAT_STAND_I2S, .communication_format = I2S_COMM_FORMAT_STAND_I2S,
.intr_alloc_flags = ESP_INTR_FLAG_LEVEL2 | ESP_INTR_FLAG_IRAM, .intr_alloc_flags = ESP_INTR_FLAG_LEVEL2 | ESP_INTR_FLAG_IRAM,
.dma_buf_count = 3, .dma_buf_count = 8,
.dma_buf_len = 300, .dma_buf_len = 128,
.use_apll = false, .use_apll = false,
.tx_desc_auto_clear = true, .tx_desc_auto_clear = true,
.fixed_mclk = 0, .fixed_mclk = 0,
@ -117,11 +97,6 @@ void ESPADFMicrophone::feed_task(void *params) {
.bits_per_chan = I2S_BITS_PER_CHAN_DEFAULT, .bits_per_chan = I2S_BITS_PER_CHAN_DEFAULT,
}; };
audio_pipeline_cfg_t pipeline_cfg = {
.rb_size = 8 * 1024,
};
audio_pipeline_handle_t pipeline = audio_pipeline_init(&pipeline_cfg);
i2s_stream_cfg_t i2s_cfg = { i2s_stream_cfg_t i2s_cfg = {
.type = AUDIO_STREAM_READER, .type = AUDIO_STREAM_READER,
.i2s_config = i2s_config, .i2s_config = i2s_config,
@ -141,7 +116,7 @@ void ESPADFMicrophone::feed_task(void *params) {
audio_element_handle_t i2s_stream_reader = i2s_stream_init(&i2s_cfg); audio_element_handle_t i2s_stream_reader = i2s_stream_init(&i2s_cfg);
rsp_filter_cfg_t rsp_cfg = { rsp_filter_cfg_t rsp_cfg = {
.src_rate = 44100, .src_rate = 16000,
.src_ch = 2, .src_ch = 2,
.dest_rate = 16000, .dest_rate = 16000,
.dest_bits = 16, .dest_bits = 16,
@ -162,6 +137,30 @@ void ESPADFMicrophone::feed_task(void *params) {
}; };
audio_element_handle_t filter = rsp_filter_init(&rsp_cfg); audio_element_handle_t filter = rsp_filter_init(&rsp_cfg);
algorithm_stream_cfg_t algo_cfg = {
.input_type = ALGORITHM_STREAM_INPUT_TYPE1,
.task_stack = 10 * 1024,
.task_prio = ALGORITHM_STREAM_TASK_PERIOD,
.task_core = ALGORITHM_STREAM_PINNED_TO_CORE,
.out_rb_size = ALGORITHM_STREAM_RINGBUFFER_SIZE,
.stack_in_ext = true,
.rec_linear_factor = 1,
.ref_linear_factor = 1,
.debug_input = false,
.swap_ch = false,
// .algo_mask = ALGORITHM_STREAM_USE_AGC,
// .algo_mask = (ALGORITHM_STREAM_USE_AEC | ALGORITHM_STREAM_USE_AGC | ALGORITHM_STREAM_USE_NS),
// .algo_mask = (ALGORITHM_STREAM_USE_AGC | ALGORITHM_STREAM_USE_NS),
.algo_mask = (ALGORITHM_STREAM_USE_AEC | ALGORITHM_STREAM_USE_NS),
// .algo_mask = (ALGORITHM_STREAM_USE_NS),
.sample_rate = 16000,
.mic_ch = 1,
.agc_gain = 10,
.aec_low_cost = false,
};
// audio_element_handle_t algo_stream = algo_stream_init(&algo_cfg);
raw_stream_cfg_t raw_cfg = { raw_stream_cfg_t raw_cfg = {
.type = AUDIO_STREAM_READER, .type = AUDIO_STREAM_READER,
.out_rb_size = 8 * 1024, .out_rb_size = 8 * 1024,
@ -170,28 +169,33 @@ void ESPADFMicrophone::feed_task(void *params) {
audio_pipeline_register(pipeline, i2s_stream_reader, "i2s"); audio_pipeline_register(pipeline, i2s_stream_reader, "i2s");
audio_pipeline_register(pipeline, filter, "filter"); audio_pipeline_register(pipeline, filter, "filter");
// audio_pipeline_register(pipeline, algo_stream, "algo");
audio_pipeline_register(pipeline, raw_read, "raw"); audio_pipeline_register(pipeline, raw_read, "raw");
const char *link_tag[3] = {"i2s", "filter", "raw"}; const char *link_tag[4] = {
"i2s",
"filter",
// "algo",
"raw",
};
audio_pipeline_link(pipeline, &link_tag[0], 3); audio_pipeline_link(pipeline, &link_tag[0], 3);
audio_pipeline_run(pipeline); audio_pipeline_run(pipeline);
event.type = TaskEventType::STARTED; event.type = TaskEventType::STARTED;
xQueueSend(this_mic->feed_event_queue_, &event, portMAX_DELAY); xQueueSend(this_mic->read_event_queue_, &event, portMAX_DELAY);
CommandEvent command_event; CommandEvent command_event;
size_t fill_count = 0;
while (true) { while (true) {
if (xQueueReceive(this_mic->feed_command_queue_, &command_event, 0) == pdTRUE) { if (xQueueReceive(this_mic->read_command_queue_, &command_event, 0) == pdTRUE) {
if (command_event.stop) { if (command_event.stop) {
// Stop signal from main thread // Stop signal from main thread
break; break;
} }
} }
int bytes_read = raw_stream_read(raw_read, (char *) (afe_buffer + fill_count), buffer_size - fill_count); int bytes_read = raw_stream_read(raw_read, (char *) buffer, BUFFER_SIZE);
if (bytes_read == -2 || bytes_read == 0) { if (bytes_read == -2 || bytes_read == 0) {
// No data in buffers to read. // No data in buffers to read.
@ -199,72 +203,43 @@ void ESPADFMicrophone::feed_task(void *params) {
} else if (bytes_read < 0) { } else if (bytes_read < 0) {
event.type = TaskEventType::WARNING; event.type = TaskEventType::WARNING;
event.err = bytes_read; event.err = bytes_read;
xQueueSend(this_mic->feed_event_queue_, &event, 0); xQueueSend(this_mic->read_event_queue_, &event, 0);
continue; continue;
} }
event.type = TaskEventType::RUNNING; event.type = TaskEventType::RUNNING;
event.err = ESP_OK; event.err = bytes_read;
xQueueSend(this_mic->feed_event_queue_, &event, 0); xQueueSend(this_mic->read_event_queue_, &event, 0);
fill_count += bytes_read; int available = rb_bytes_available(this_mic->ring_buffer_);
if (available < bytes_read) {
if (fill_count == buffer_size) { rb_read(this_mic->ring_buffer_, nullptr, bytes_read - available, 0);
this_mic->afe_handle_->feed(this_mic->afe_data_, afe_buffer);
fill_count -= buffer_size;
} }
rb_write(this_mic->ring_buffer_, (char *) buffer, bytes_read, 0);
} }
allocator.deallocate(afe_buffer, this_mic->afe_chunk_size_); allocator.deallocate(buffer, BUFFER_SIZE / sizeof(int16_t));
audio_pipeline_stop(pipeline); audio_pipeline_stop(pipeline);
audio_pipeline_wait_for_stop(pipeline); audio_pipeline_wait_for_stop(pipeline);
audio_pipeline_terminate(pipeline); audio_pipeline_terminate(pipeline);
event.type = TaskEventType::STOPPING; event.type = TaskEventType::STOPPING;
xQueueSend(this_mic->feed_event_queue_, &event, portMAX_DELAY); xQueueSend(this_mic->read_event_queue_, &event, portMAX_DELAY);
audio_pipeline_unregister(pipeline, i2s_stream_reader); audio_pipeline_unregister(pipeline, i2s_stream_reader);
audio_pipeline_unregister(pipeline, filter); audio_pipeline_unregister(pipeline, filter);
// audio_pipeline_unregister(pipeline, algo_stream);
audio_pipeline_unregister(pipeline, raw_read); audio_pipeline_unregister(pipeline, raw_read);
audio_pipeline_deinit(pipeline); audio_pipeline_deinit(pipeline);
audio_element_deinit(i2s_stream_reader); audio_element_deinit(i2s_stream_reader);
audio_element_deinit(filter); audio_element_deinit(filter);
// audio_element_deinit(algo_stream);
audio_element_deinit(raw_read); audio_element_deinit(raw_read);
event.type = TaskEventType::STOPPED; event.type = TaskEventType::STOPPED;
xQueueSend(this_mic->feed_event_queue_, &event, portMAX_DELAY); xQueueSend(this_mic->read_event_queue_, &event, portMAX_DELAY);
while (true) {
delay(10);
}
}
void ESPADFMicrophone::fetch_task(void *params) {
ESPADFMicrophone *this_mic = (ESPADFMicrophone *) params;
CommandEvent event;
while (true) {
if (xQueueReceive(this_mic->fetch_command_queue_, &event, 0) == pdTRUE) {
if (event.stop) {
// Stop signal from main thread
break;
}
}
afe_fetch_result_t *result = this_mic->afe_handle_->fetch(this_mic->afe_data_);
if (result == nullptr) {
continue;
}
int available = rb_bytes_available(this_mic->ring_buffer_);
if (available < result->data_size) {
rb_read(this_mic->ring_buffer_, nullptr, result->data_size - available, 0);
}
rb_write(this_mic->ring_buffer_, (char *) result->data, result->data_size, 0);
}
while (true) { while (true) {
delay(10); delay(10);
@ -272,31 +247,26 @@ void ESPADFMicrophone::fetch_task(void *params) {
} }
void ESPADFMicrophone::stop() { void ESPADFMicrophone::stop() {
if (this->state_ == microphone::STATE_STOPPED || this->is_failed()) if (this->state_ == microphone::STATE_STOPPED || this->state_ == microphone::STATE_STOPPING || this->is_failed())
return; return;
if (this->state_ == microphone::STATE_STARTING) {
this->state_ = microphone::STATE_STOPPED;
return;
}
this->state_ = microphone::STATE_STOPPING; this->state_ = microphone::STATE_STOPPING;
}
void ESPADFMicrophone::stop_() {
ESP_LOGD(TAG, "Stopping microphone");
CommandEvent command_event; CommandEvent command_event;
command_event.stop = true; command_event.stop = true;
xQueueSendToFront(this->feed_command_queue_, &command_event, portMAX_DELAY); xQueueSendToFront(this->read_command_queue_, &command_event, portMAX_DELAY);
xQueueSendToFront(this->fetch_command_queue_, &command_event, portMAX_DELAY); ESP_LOGD(TAG, "Stopping microphone");
} }
size_t ESPADFMicrophone::read(int16_t *buf, size_t len) { size_t ESPADFMicrophone::read(int16_t *buf, size_t len) {
if (rb_bytes_available(this->ring_buffer_) == 0) {
return 0; // No data
}
int bytes_read = rb_read(this->ring_buffer_, (char *) buf, len, 0); int bytes_read = rb_read(this->ring_buffer_, (char *) buf, len, 0);
if (bytes_read == -2 || bytes_read == 0) { if (bytes_read == -4 || bytes_read == -2 || bytes_read == 0) {
// No data in buffers to read. // No data in buffers to read.
return 0; return 0;
} else if (bytes_read < 0) { } else if (bytes_read < 0) {
ESP_LOGW(TAG, "Error reading from I2S microphone"); ESP_LOGW(TAG, "Error reading from I2S microphone %s (%d)", esp_err_to_name(bytes_read), bytes_read);
this->status_set_warning(); this->status_set_warning();
return 0; return 0;
} }
@ -315,22 +285,24 @@ void ESPADFMicrophone::read_() {
void ESPADFMicrophone::watch_() { void ESPADFMicrophone::watch_() {
TaskEvent event; TaskEvent event;
if (xQueueReceive(this->feed_event_queue_, &event, 0) == pdTRUE) { if (xQueueReceive(this->read_event_queue_, &event, 0) == pdTRUE) {
switch (event.type) { switch (event.type) {
case TaskEventType::STARTING: case TaskEventType::STARTING:
case TaskEventType::STARTED:
case TaskEventType::STOPPING: case TaskEventType::STOPPING:
break; break;
case TaskEventType::STARTED:
ESP_LOGD(TAG, "Microphone started");
this->state_ = microphone::STATE_RUNNING;
break;
case TaskEventType::RUNNING: case TaskEventType::RUNNING:
this->status_clear_warning(); this->status_clear_warning();
// ESP_LOGD(TAG, "Putting %d bytes into ring buffer", event.err);
break; break;
case TaskEventType::STOPPED: case TaskEventType::STOPPED:
this->parent_->unlock(); this->parent_->unlock();
this->state_ = microphone::STATE_STOPPED; this->state_ = microphone::STATE_STOPPED;
vTaskDelete(this->feed_task_handle_); vTaskDelete(this->read_task_handle_);
vTaskDelete(this->fetch_task_handle_); this->read_task_handle_ = nullptr;
this->feed_task_handle_ = nullptr;
this->fetch_task_handle_ = nullptr;
ESP_LOGD(TAG, "Microphone stopped"); ESP_LOGD(TAG, "Microphone stopped");
break; break;
case TaskEventType::WARNING: case TaskEventType::WARNING:
@ -342,8 +314,10 @@ void ESPADFMicrophone::watch_() {
} }
void ESPADFMicrophone::loop() { void ESPADFMicrophone::loop() {
this->watch_();
switch (this->state_) { switch (this->state_) {
case microphone::STATE_STOPPED: case microphone::STATE_STOPPED:
case microphone::STATE_STOPPING:
break; break;
case microphone::STATE_STARTING: case microphone::STATE_STARTING:
this->start_(); this->start_();
@ -353,9 +327,6 @@ void ESPADFMicrophone::loop() {
this->read_(); this->read_();
} }
break; break;
case microphone::STATE_STOPPING:
this->stop_();
break;
} }
} }

21
esphome/components/esp_adf/microphone/esp_adf_microphone.h
View File

@ -7,10 +7,6 @@
#include "esphome/components/microphone/microphone.h" #include "esphome/components/microphone/microphone.h"
#include "esphome/core/component.h" #include "esphome/core/component.h"
#include <audio_element.h>
#include <audio_pipeline.h>
#include <esp_afe_sr_iface.h>
#include <esp_afe_sr_models.h>
#include <ringbuf.h> #include <ringbuf.h>
namespace esphome { namespace esphome {
@ -28,25 +24,16 @@ class ESPADFMicrophone : public ESPADFPipeline, public microphone::Microphone, p
protected: protected:
void start_(); void start_();
void stop_();
void read_(); void read_();
void watch_(); void watch_();
static void feed_task(void *params); static void read_task(void *params);
static void fetch_task(void *params);
const esp_afe_sr_iface_t *afe_handle_{&ESP_AFE_SR_HANDLE};
esp_afe_sr_data_t *afe_data_{nullptr};
size_t afe_chunk_size_{0};
ringbuf_handle_t ring_buffer_; ringbuf_handle_t ring_buffer_;
TaskHandle_t feed_task_handle_{nullptr}; TaskHandle_t read_task_handle_{nullptr};
QueueHandle_t feed_event_queue_; QueueHandle_t read_event_queue_;
QueueHandle_t feed_command_queue_; QueueHandle_t read_command_queue_;
TaskHandle_t fetch_task_handle_{nullptr};
QueueHandle_t fetch_command_queue_;
}; };
} // namespace esp_adf } // namespace esp_adf

8
esphome/components/esp_adf/speaker/esp_adf_speaker.cpp
View File

@ -32,7 +32,6 @@ void ESPADFSpeaker::start_() {
if (!this->parent_->try_lock()) { if (!this->parent_->try_lock()) {
return; // Waiting for another i2s component to return lock return; // Waiting for another i2s component to return lock
} }
this->state_ = speaker::STATE_RUNNING;
xTaskCreate(ESPADFSpeaker::player_task, "speaker_task", 8192, (void *) this, 0, &this->player_task_handle_); xTaskCreate(ESPADFSpeaker::player_task, "speaker_task", 8192, (void *) this, 0, &this->player_task_handle_);
} }
@ -211,9 +210,11 @@ void ESPADFSpeaker::watch_() {
if (xQueueReceive(this->event_queue_, &event, 0) == pdTRUE) { if (xQueueReceive(this->event_queue_, &event, 0) == pdTRUE) {
switch (event.type) { switch (event.type) {
case TaskEventType::STARTING: case TaskEventType::STARTING:
case TaskEventType::STARTED:
case TaskEventType::STOPPING: case TaskEventType::STOPPING:
break; break;
case TaskEventType::STARTED:
this->state_ = speaker::STATE_RUNNING;
break;
case TaskEventType::RUNNING: case TaskEventType::RUNNING:
this->status_clear_warning(); this->status_clear_warning();
break; break;
@ -232,13 +233,12 @@ void ESPADFSpeaker::watch_() {
} }
void ESPADFSpeaker::loop() { void ESPADFSpeaker::loop() {
this->watch_();
switch (this->state_) { switch (this->state_) {
case speaker::STATE_STARTING: case speaker::STATE_STARTING:
this->start_(); this->start_();
break; break;
case speaker::STATE_RUNNING: case speaker::STATE_RUNNING:
this->watch_();
break;
case speaker::STATE_STOPPING: case speaker::STATE_STOPPING:
case speaker::STATE_STOPPED: case speaker::STATE_STOPPED:
break; break;