sascha/esphome
1
0
Fork 0
mirror of https://github.com/esphome/esphome.git synced 2025-11-04 09:01:49 +00:00
Code Issues Releases Wiki Activity

Remove afe and try to use algorithm stream

Browse Source
This commit is contained in:
Jesse Hills
2023-08-31 17:38:49 +12:00
parent 0b60a1d9eb
commit bab9c7c70e
3 changed files with 103 additions and 145 deletions
Download Patch File Download Diff File Expand all files Collapse all files

219
esphome/components/esp_adf/microphone/esp_adf_microphone.cpp
View File

@@ -7,58 +7,32 @@
#include "esphome/core/hal.h"
#include "esphome/core/log.h"
#include <algorithm_stream.h>
#include <audio_element.h>
#include <audio_hal.h>
#include <audio_pipeline.h>
#include <filter_resample.h>
#include <i2s_stream.h>
#include <raw_stream.h>
#include <recorder_sr.h>
#include <board.h>
namespace esphome {
namespace esp_adf {
static const char *const TAG = "esp_adf.microphone";
void ESPADFMicrophone::setup() {
this->ring_buffer_ = rb_create(BUFFER_SIZE, sizeof(int16_t));
this->ring_buffer_ = rb_create(8000, sizeof(int16_t));
if (this->ring_buffer_ == nullptr) {
ESP_LOGW(TAG, "Could not allocate ring buffer.");
this->mark_failed();
return;
}
this->feed_event_queue_ = xQueueCreate(20, sizeof(TaskEvent));
this->feed_command_queue_ = xQueueCreate(20, sizeof(CommandEvent));
this->fetch_command_queue_ = xQueueCreate(20, sizeof(CommandEvent));
afe_config_t cfg_afe = {
.aec_init = false,
.se_init = true,
.vad_init = true,
.wakenet_init = false,
.voice_communication_init = false,
.voice_communication_agc_init = false,
.voice_communication_agc_gain = 15,
.vad_mode = VAD_MODE_3,
.wakenet_model_name = nullptr,
.wakenet_mode = DET_MODE_2CH_90,
.afe_mode = SR_MODE_HIGH_PERF,
.afe_perferred_core = 1,
.afe_perferred_priority = 5,
.afe_ringbuf_size = 50,
.memory_alloc_mode = AFE_MEMORY_ALLOC_MORE_PSRAM,
.agc_mode = AFE_MN_PEAK_AGC_MODE_3,
.pcm_config =
{
.total_ch_num = 1,
.mic_num = 1,
.ref_num = 0,
.sample_rate = 16000,
},
.debug_init = false,
.debug_hook = {{AFE_DEBUG_HOOK_MASE_TASK_IN, NULL}, {AFE_DEBUG_HOOK_FETCH_TASK_IN, NULL}},
};
this->afe_data_ = this->afe_handle_->create_from_config(&cfg_afe);
this->afe_chunk_size_ = this->afe_handle_->get_feed_chunksize(this->afe_data_);
this->read_event_queue_ = xQueueCreate(20, sizeof(TaskEvent));
this->read_command_queue_ = xQueueCreate(20, sizeof(CommandEvent));
}
void ESPADFMicrophone::start() {
@@ -74,42 +48,48 @@ void ESPADFMicrophone::start_() {
if (!this->parent_->try_lock()) {
return;
}
this->state_ = microphone::STATE_RUNNING;
xTaskCreate(ESPADFMicrophone::feed_task, "feed_task", 8192, (void *) this, 0, &this->feed_task_handle_);
xTaskCreate(ESPADFMicrophone::fetch_task, "fetch_task", 8192, (void *) this, 0, &this->fetch_task_handle_);
xTaskCreate(ESPADFMicrophone::read_task, "read_task", 8192, (void *) this, 0, &this->read_task_handle_);
}
void ESPADFMicrophone::feed_task(void *params) {
void ESPADFMicrophone::read_task(void *params) {
ESPADFMicrophone *this_mic = (ESPADFMicrophone *) params;
TaskEvent event;
event.type = TaskEventType::STARTING;
xQueueSend(this_mic->feed_event_queue_, &event, portMAX_DELAY);
size_t buffer_size = this_mic->afe_chunk_size_ * sizeof(int16_t);
ExternalRAMAllocator<int16_t> allocator(ExternalRAMAllocator<int16_t>::ALLOW_FAILURE);
int16_t *afe_buffer = allocator.allocate(this_mic->afe_chunk_size_);
if (afe_buffer == nullptr) {
int16_t *buffer = allocator.allocate(BUFFER_SIZE / sizeof(int16_t));
if (buffer == nullptr) {
event.type = TaskEventType::WARNING;
event.err = ESP_ERR_NO_MEM;
xQueueSend(this_mic->read_event_queue_, &event, portMAX_DELAY);
event.type = TaskEventType::STOPPED;
xQueueSend(this_mic->feed_event_queue_, &event, portMAX_DELAY);
event.err = ESP_OK;
xQueueSend(this_mic->read_event_queue_, &event, portMAX_DELAY);
while (true) {
delay(10);
};
}
return;
}
event.type = TaskEventType::STARTING;
xQueueSend(this_mic->read_event_queue_, &event, portMAX_DELAY);
audio_pipeline_cfg_t pipeline_cfg = {
.rb_size = 8 * 1024,
};
audio_pipeline_handle_t pipeline = audio_pipeline_init(&pipeline_cfg);
i2s_driver_config_t i2s_config = {
.mode = (i2s_mode_t) (I2S_MODE_MASTER | I2S_MODE_RX),
.sample_rate = 44100,
.sample_rate = 16000,
.bits_per_sample = I2S_BITS_PER_SAMPLE_16BIT,
.channel_format = I2S_CHANNEL_FMT_RIGHT_LEFT,
.communication_format = I2S_COMM_FORMAT_STAND_I2S,
.intr_alloc_flags = ESP_INTR_FLAG_LEVEL2 | ESP_INTR_FLAG_IRAM,
.dma_buf_count = 3,
.dma_buf_len = 300,
.dma_buf_count = 8,
.dma_buf_len = 128,
.use_apll = false,
.tx_desc_auto_clear = true,
.fixed_mclk = 0,
@@ -117,11 +97,6 @@ void ESPADFMicrophone::feed_task(void *params) {
.bits_per_chan = I2S_BITS_PER_CHAN_DEFAULT,
};
audio_pipeline_cfg_t pipeline_cfg = {
.rb_size = 8 * 1024,
};
audio_pipeline_handle_t pipeline = audio_pipeline_init(&pipeline_cfg);
i2s_stream_cfg_t i2s_cfg = {
.type = AUDIO_STREAM_READER,
.i2s_config = i2s_config,
@@ -141,7 +116,7 @@ void ESPADFMicrophone::feed_task(void *params) {
audio_element_handle_t i2s_stream_reader = i2s_stream_init(&i2s_cfg);
rsp_filter_cfg_t rsp_cfg = {
.src_rate = 44100,
.src_rate = 16000,
.src_ch = 2,
.dest_rate = 16000,
.dest_bits = 16,
@@ -162,6 +137,30 @@ void ESPADFMicrophone::feed_task(void *params) {
};
audio_element_handle_t filter = rsp_filter_init(&rsp_cfg);
algorithm_stream_cfg_t algo_cfg = {
.input_type = ALGORITHM_STREAM_INPUT_TYPE1,
.task_stack = 10 * 1024,
.task_prio = ALGORITHM_STREAM_TASK_PERIOD,
.task_core = ALGORITHM_STREAM_PINNED_TO_CORE,
.out_rb_size = ALGORITHM_STREAM_RINGBUFFER_SIZE,
.stack_in_ext = true,
.rec_linear_factor = 1,
.ref_linear_factor = 1,
.debug_input = false,
.swap_ch = false,
// .algo_mask = ALGORITHM_STREAM_USE_AGC,
// .algo_mask = (ALGORITHM_STREAM_USE_AEC | ALGORITHM_STREAM_USE_AGC | ALGORITHM_STREAM_USE_NS),
// .algo_mask = (ALGORITHM_STREAM_USE_AGC | ALGORITHM_STREAM_USE_NS),
.algo_mask = (ALGORITHM_STREAM_USE_AEC | ALGORITHM_STREAM_USE_NS),
// .algo_mask = (ALGORITHM_STREAM_USE_NS),
.sample_rate = 16000,
.mic_ch = 1,
.agc_gain = 10,
.aec_low_cost = false,
};
// audio_element_handle_t algo_stream = algo_stream_init(&algo_cfg);
raw_stream_cfg_t raw_cfg = {
.type = AUDIO_STREAM_READER,
.out_rb_size = 8 * 1024,
@@ -170,28 +169,33 @@ void ESPADFMicrophone::feed_task(void *params) {
audio_pipeline_register(pipeline, i2s_stream_reader, "i2s");
audio_pipeline_register(pipeline, filter, "filter");
// audio_pipeline_register(pipeline, algo_stream, "algo");
audio_pipeline_register(pipeline, raw_read, "raw");
const char *link_tag[3] = {"i2s", "filter", "raw"};
const char *link_tag[4] = {
"i2s",
"filter",
// "algo",
"raw",
};
audio_pipeline_link(pipeline, &link_tag[0], 3);
audio_pipeline_run(pipeline);
event.type = TaskEventType::STARTED;
xQueueSend(this_mic->feed_event_queue_, &event, portMAX_DELAY);
xQueueSend(this_mic->read_event_queue_, &event, portMAX_DELAY);
CommandEvent command_event;
size_t fill_count = 0;
while (true) {
if (xQueueReceive(this_mic->feed_command_queue_, &command_event, 0) == pdTRUE) {
if (xQueueReceive(this_mic->read_command_queue_, &command_event, 0) == pdTRUE) {
if (command_event.stop) {
// Stop signal from main thread
break;
}
}
int bytes_read = raw_stream_read(raw_read, (char *) (afe_buffer + fill_count), buffer_size - fill_count);
int bytes_read = raw_stream_read(raw_read, (char *) buffer, BUFFER_SIZE);
if (bytes_read == -2 || bytes_read == 0) {
// No data in buffers to read.
@@ -199,72 +203,43 @@ void ESPADFMicrophone::feed_task(void *params) {
} else if (bytes_read < 0) {
event.type = TaskEventType::WARNING;
event.err = bytes_read;
xQueueSend(this_mic->feed_event_queue_, &event, 0);
xQueueSend(this_mic->read_event_queue_, &event, 0);
continue;
}
event.type = TaskEventType::RUNNING;
event.err = ESP_OK;
xQueueSend(this_mic->feed_event_queue_, &event, 0);
event.err = bytes_read;
xQueueSend(this_mic->read_event_queue_, &event, 0);
fill_count += bytes_read;
if (fill_count == buffer_size) {
this_mic->afe_handle_->feed(this_mic->afe_data_, afe_buffer);
fill_count -= buffer_size;
int available = rb_bytes_available(this_mic->ring_buffer_);
if (available < bytes_read) {
rb_read(this_mic->ring_buffer_, nullptr, bytes_read - available, 0);
}
rb_write(this_mic->ring_buffer_, (char *) buffer, bytes_read, 0);
}
allocator.deallocate(afe_buffer, this_mic->afe_chunk_size_);
allocator.deallocate(buffer, BUFFER_SIZE / sizeof(int16_t));
audio_pipeline_stop(pipeline);
audio_pipeline_wait_for_stop(pipeline);
audio_pipeline_terminate(pipeline);
event.type = TaskEventType::STOPPING;
xQueueSend(this_mic->feed_event_queue_, &event, portMAX_DELAY);
xQueueSend(this_mic->read_event_queue_, &event, portMAX_DELAY);
audio_pipeline_unregister(pipeline, i2s_stream_reader);
audio_pipeline_unregister(pipeline, filter);
// audio_pipeline_unregister(pipeline, algo_stream);
audio_pipeline_unregister(pipeline, raw_read);
audio_pipeline_deinit(pipeline);
audio_element_deinit(i2s_stream_reader);
audio_element_deinit(filter);
// audio_element_deinit(algo_stream);
audio_element_deinit(raw_read);
event.type = TaskEventType::STOPPED;
xQueueSend(this_mic->feed_event_queue_, &event, portMAX_DELAY);
while (true) {
delay(10);
}
}
void ESPADFMicrophone::fetch_task(void *params) {
ESPADFMicrophone *this_mic = (ESPADFMicrophone *) params;
CommandEvent event;
while (true) {
if (xQueueReceive(this_mic->fetch_command_queue_, &event, 0) == pdTRUE) {
if (event.stop) {
// Stop signal from main thread
break;
}
}
afe_fetch_result_t *result = this_mic->afe_handle_->fetch(this_mic->afe_data_);
if (result == nullptr) {
continue;
}
int available = rb_bytes_available(this_mic->ring_buffer_);
if (available < result->data_size) {
rb_read(this_mic->ring_buffer_, nullptr, result->data_size - available, 0);
}
rb_write(this_mic->ring_buffer_, (char *) result->data, result->data_size, 0);
}
xQueueSend(this_mic->read_event_queue_, &event, portMAX_DELAY);
while (true) {
delay(10);
@@ -272,31 +247,26 @@ void ESPADFMicrophone::fetch_task(void *params) {
}
void ESPADFMicrophone::stop() {
if (this->state_ == microphone::STATE_STOPPED || this->is_failed())
if (this->state_ == microphone::STATE_STOPPED || this->state_ == microphone::STATE_STOPPING || this->is_failed())
return;
if (this->state_ == microphone::STATE_STARTING) {
this->state_ = microphone::STATE_STOPPED;
return;
}
this->state_ = microphone::STATE_STOPPING;
}
void ESPADFMicrophone::stop_() {
ESP_LOGD(TAG, "Stopping microphone");
CommandEvent command_event;
command_event.stop = true;
xQueueSendToFront(this->feed_command_queue_, &command_event, portMAX_DELAY);
xQueueSendToFront(this->fetch_command_queue_, &command_event, portMAX_DELAY);
xQueueSendToFront(this->read_command_queue_, &command_event, portMAX_DELAY);
ESP_LOGD(TAG, "Stopping microphone");
}
size_t ESPADFMicrophone::read(int16_t *buf, size_t len) {
if (rb_bytes_available(this->ring_buffer_) == 0) {
return 0; // No data
}
int bytes_read = rb_read(this->ring_buffer_, (char *) buf, len, 0);
if (bytes_read == -2 || bytes_read == 0) {
if (bytes_read == -4 || bytes_read == -2 || bytes_read == 0) {
// No data in buffers to read.
return 0;
} else if (bytes_read < 0) {
ESP_LOGW(TAG, "Error reading from I2S microphone");
ESP_LOGW(TAG, "Error reading from I2S microphone %s (%d)", esp_err_to_name(bytes_read), bytes_read);
this->status_set_warning();
return 0;
}
@@ -315,22 +285,24 @@ void ESPADFMicrophone::read_() {
void ESPADFMicrophone::watch_() {
TaskEvent event;
if (xQueueReceive(this->feed_event_queue_, &event, 0) == pdTRUE) {
if (xQueueReceive(this->read_event_queue_, &event, 0) == pdTRUE) {
switch (event.type) {
case TaskEventType::STARTING:
case TaskEventType::STARTED:
case TaskEventType::STOPPING:
break;
case TaskEventType::STARTED:
ESP_LOGD(TAG, "Microphone started");
this->state_ = microphone::STATE_RUNNING;
break;
case TaskEventType::RUNNING:
this->status_clear_warning();
// ESP_LOGD(TAG, "Putting %d bytes into ring buffer", event.err);
break;
case TaskEventType::STOPPED:
this->parent_->unlock();
this->state_ = microphone::STATE_STOPPED;
vTaskDelete(this->feed_task_handle_);
vTaskDelete(this->fetch_task_handle_);
this->feed_task_handle_ = nullptr;
this->fetch_task_handle_ = nullptr;
vTaskDelete(this->read_task_handle_);
this->read_task_handle_ = nullptr;
ESP_LOGD(TAG, "Microphone stopped");
break;
case TaskEventType::WARNING:
@@ -342,8 +314,10 @@ void ESPADFMicrophone::watch_() {
}
void ESPADFMicrophone::loop() {
this->watch_();
switch (this->state_) {
case microphone::STATE_STOPPED:
case microphone::STATE_STOPPING:
break;
case microphone::STATE_STARTING:
this->start_();
@@ -353,9 +327,6 @@ void ESPADFMicrophone::loop() {
this->read_();
}
break;
case microphone::STATE_STOPPING:
this->stop_();
break;
}
}

21
esphome/components/esp_adf/microphone/esp_adf_microphone.h
View File

@@ -7,10 +7,6 @@
#include "esphome/components/microphone/microphone.h"
#include "esphome/core/component.h"
#include <audio_element.h>
#include <audio_pipeline.h>
#include <esp_afe_sr_iface.h>
#include <esp_afe_sr_models.h>
#include <ringbuf.h>
namespace esphome {
@@ -28,25 +24,16 @@ class ESPADFMicrophone : public ESPADFPipeline, public microphone::Microphone, p
protected:
void start_();
void stop_();
void read_();
void watch_();
static void feed_task(void *params);
static void fetch_task(void *params);
const esp_afe_sr_iface_t *afe_handle_{&ESP_AFE_SR_HANDLE};
esp_afe_sr_data_t *afe_data_{nullptr};
size_t afe_chunk_size_{0};
static void read_task(void *params);
ringbuf_handle_t ring_buffer_;
TaskHandle_t feed_task_handle_{nullptr};
QueueHandle_t feed_event_queue_;
QueueHandle_t feed_command_queue_;
TaskHandle_t fetch_task_handle_{nullptr};
QueueHandle_t fetch_command_queue_;
TaskHandle_t read_task_handle_{nullptr};
QueueHandle_t read_event_queue_;
QueueHandle_t read_command_queue_;
};
} // namespace esp_adf

8
esphome/components/esp_adf/speaker/esp_adf_speaker.cpp
View File

@@ -32,7 +32,6 @@ void ESPADFSpeaker::start_() {
if (!this->parent_->try_lock()) {
return; // Waiting for another i2s component to return lock
}
this->state_ = speaker::STATE_RUNNING;
xTaskCreate(ESPADFSpeaker::player_task, "speaker_task", 8192, (void *) this, 0, &this->player_task_handle_);
}
@@ -211,9 +210,11 @@ void ESPADFSpeaker::watch_() {
if (xQueueReceive(this->event_queue_, &event, 0) == pdTRUE) {
switch (event.type) {
case TaskEventType::STARTING:
case TaskEventType::STARTED:
case TaskEventType::STOPPING:
break;
case TaskEventType::STARTED:
this->state_ = speaker::STATE_RUNNING;
break;
case TaskEventType::RUNNING:
this->status_clear_warning();
break;
@@ -232,13 +233,12 @@ void ESPADFSpeaker::watch_() {
}
void ESPADFSpeaker::loop() {
this->watch_();
switch (this->state_) {
case speaker::STATE_STARTING:
this->start_();
break;
case speaker::STATE_RUNNING:
this->watch_();
break;
case speaker::STATE_STOPPING:
case speaker::STATE_STOPPED:
break;