sascha/esphome
1
0
Fork 0
mirror of https://github.com/esphome/esphome.git synced 2025-10-15 16:23:48 +01:00
Code Issues Releases Wiki Activity

[usb_host] Fix transfer slot exhaustion at high data rates and add configurable max_transfer_requests

Browse Source
This commit is contained in:
J. Nick Koston
2025-10-11 05:53:14 -10:00
parent b54beb357a
commit 460c41d9b8
4 changed files with 45 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
esphome/components/usb_host/__init__.py
View File

@@ -20,6 +20,7 @@ USBClient = usb_host_ns.class_("USBClient", Component)
CONF_VID = "vid" CONF_VID = "vid"
CONF_PID = "pid" CONF_PID = "pid"
CONF_ENABLE_HUBS = "enable_hubs" CONF_ENABLE_HUBS = "enable_hubs"
CONF_MAX_TRANSFER_REQUESTS = "max_transfer_requests"
def usb_device_schema(cls=USBClient, vid: int = None, pid: [int] = None) -> cv.Schema: def usb_device_schema(cls=USBClient, vid: int = None, pid: [int] = None) -> cv.Schema:
@@ -44,6 +45,9 @@ CONFIG_SCHEMA = cv.All(
{ {
cv.GenerateID(): cv.declare_id(USBHost), cv.GenerateID(): cv.declare_id(USBHost),
cv.Optional(CONF_ENABLE_HUBS, default=False): cv.boolean, cv.Optional(CONF_ENABLE_HUBS, default=False): cv.boolean,
cv.Optional(CONF_MAX_TRANSFER_REQUESTS, default=16): cv.int_range(
min=1, max=32
),
cv.Optional(CONF_DEVICES): cv.ensure_list(usb_device_schema()), cv.Optional(CONF_DEVICES): cv.ensure_list(usb_device_schema()),
} }
), ),
@@ -62,6 +66,10 @@ async def to_code(config):
add_idf_sdkconfig_option("CONFIG_USB_HOST_CONTROL_TRANSFER_MAX_SIZE", 1024) add_idf_sdkconfig_option("CONFIG_USB_HOST_CONTROL_TRANSFER_MAX_SIZE", 1024)
if config.get(CONF_ENABLE_HUBS): if config.get(CONF_ENABLE_HUBS):
add_idf_sdkconfig_option("CONFIG_USB_HOST_HUBS_SUPPORTED", True) add_idf_sdkconfig_option("CONFIG_USB_HOST_HUBS_SUPPORTED", True)
max_requests = config[CONF_MAX_TRANSFER_REQUESTS]
cg.add_define("USB_HOST_MAX_REQUESTS", max_requests)
var = cg.new_Pvariable(config[CONF_ID]) var = cg.new_Pvariable(config[CONF_ID])
await cg.register_component(var, config) await cg.register_component(var, config)
for device in config.get(CONF_DEVICES) or (): for device in config.get(CONF_DEVICES) or ():

37
esphome/components/usb_host/usb_host.h
View File

@@ -16,23 +16,25 @@ namespace usb_host {
// THREADING MODEL: // THREADING MODEL:
// This component uses a dedicated USB task for event processing to prevent data loss. // This component uses a dedicated USB task for event processing to prevent data loss.
// - USB Task (high priority): Handles USB events, executes transfer callbacks // - USB Task (high priority): Handles USB events, executes transfer callbacks, releases transfer slots
// - Main Loop Task: Initiates transfers, processes completion events // - Main Loop Task: Initiates transfers, processes device connect/disconnect events
// //
// Thread-safe communication: // Thread-safe communication:
// - Lock-free queues for USB task -> main loop events (SPSC pattern) // - Lock-free queues for USB task -> main loop events (SPSC pattern)
// - Lock-free TransferRequest pool using atomic bitmask (MCSP pattern) // - Lock-free TransferRequest pool using atomic bitmask (MCMP pattern - multi-consumer, multi-producer)
// //
// TransferRequest pool access pattern: // TransferRequest pool access pattern:
// - get_trq_() [allocate]: Called from BOTH USB task and main loop threads // - get_trq_() [allocate]: Called from BOTH USB task and main loop threads
// * USB task: via USB UART input callbacks that restart transfers immediately // * USB task: via USB UART input callbacks that restart transfers immediately
// * Main loop: for output transfers and flow-controlled input restarts // * Main loop: for output transfers and flow-controlled input restarts
// - release_trq() [deallocate]: Called from main loop thread only // - release_trq() [deallocate]: Called from BOTH USB task and main loop threads
// * USB task: immediately after transfer callback completes (critical for preventing slot exhaustion)
// * Main loop: when transfer submission fails
// //
// The multi-threaded allocation is intentional for performance: // The multi-threaded allocation/deallocation is intentional for performance:
// - USB task can immediately restart input transfers without context switching // - USB task can immediately restart input transfers and release slots without context switching
// - Main loop controls backpressure by deciding when to restart after consuming data // - Main loop controls backpressure by deciding when to restart after consuming data
// The atomic bitmask ensures thread-safe allocation without mutex blocking. // The atomic bitmask ensures thread-safe allocation/deallocation without mutex blocking.
static const char *const TAG = "usb_host"; static const char *const TAG = "usb_host";
@@ -52,8 +54,13 @@ static const uint8_t USB_DIR_IN = 1 << 7;
static const uint8_t USB_DIR_OUT = 0; static const uint8_t USB_DIR_OUT = 0;
static const size_t SETUP_PACKET_SIZE = 8; static const size_t SETUP_PACKET_SIZE = 8;
static const size_t MAX_REQUESTS = 16; // maximum number of outstanding requests possible. static const size_t MAX_REQUESTS = USB_HOST_MAX_REQUESTS; // maximum number of outstanding requests possible.
static_assert(MAX_REQUESTS <= 16, "MAX_REQUESTS must be <= 16 to fit in uint16_t bitmask"); static_assert(MAX_REQUESTS >= 1 && MAX_REQUESTS <= 32, "MAX_REQUESTS must be between 1 and 32");
// Select appropriate bitmask type based on MAX_REQUESTS
// uint16_t for <= 16 requests, uint32_t for 17-32 requests
using trq_bitmask_t = std::conditional<(MAX_REQUESTS <= 16), uint16_t, uint32_t>::type;
static constexpr size_t USB_EVENT_QUEUE_SIZE = 32; // Size of event queue between USB task and main loop static constexpr size_t USB_EVENT_QUEUE_SIZE = 32; // Size of event queue between USB task and main loop
static constexpr size_t USB_TASK_STACK_SIZE = 4096; // Stack size for USB task (same as ESP-IDF USB examples) static constexpr size_t USB_TASK_STACK_SIZE = 4096; // Stack size for USB task (same as ESP-IDF USB examples)
static constexpr UBaseType_t USB_TASK_PRIORITY = 5; // Higher priority than main loop (tskIDLE_PRIORITY + 5) static constexpr UBaseType_t USB_TASK_PRIORITY = 5; // Higher priority than main loop (tskIDLE_PRIORITY + 5)
@@ -83,8 +90,6 @@ struct TransferRequest {
enum EventType : uint8_t { enum EventType : uint8_t {
EVENT_DEVICE_NEW, EVENT_DEVICE_NEW,
EVENT_DEVICE_GONE, EVENT_DEVICE_GONE,
EVENT_TRANSFER_COMPLETE,
EVENT_CONTROL_COMPLETE,
}; };
struct UsbEvent { struct UsbEvent {
@@ -96,9 +101,6 @@ struct UsbEvent {
struct { struct {
usb_device_handle_t handle; usb_device_handle_t handle;
} device_gone; } device_gone;
struct {
TransferRequest *trq;
} transfer;
} data; } data;
// Required for EventPool - no cleanup needed for POD types // Required for EventPool - no cleanup needed for POD types
@@ -163,10 +165,9 @@ class USBClient : public Component {
uint16_t pid_{}; uint16_t pid_{};
// Lock-free pool management using atomic bitmask (no dynamic allocation) // Lock-free pool management using atomic bitmask (no dynamic allocation)
// Bit i = 1: requests_[i] is in use, Bit i = 0: requests_[i] is available // Bit i = 1: requests_[i] is in use, Bit i = 0: requests_[i] is available
// Supports multiple concurrent consumers (both threads can allocate) // Supports multiple concurrent consumers and producers (both threads can allocate/deallocate)
// Single producer for deallocation (main loop only) // Bitmask type automatically selected: uint16_t for <= 16 slots, uint32_t for 17-32 slots
// Limited to 16 slots by uint16_t size (enforced by static_assert) std::atomic<trq_bitmask_t> trq_in_use_;
std::atomic<uint16_t> trq_in_use_;
TransferRequest requests_[MAX_REQUESTS]{}; TransferRequest requests_[MAX_REQUESTS]{};
}; };
class USBHost : public Component { class USBHost : public Component {

56
esphome/components/usb_host/usb_host_client.cpp
View File

@@ -228,12 +228,6 @@ void USBClient::loop() {
case EVENT_DEVICE_GONE: case EVENT_DEVICE_GONE:
this->on_removed(event->data.device_gone.handle); this->on_removed(event->data.device_gone.handle);
break; break;
case EVENT_TRANSFER_COMPLETE:
case EVENT_CONTROL_COMPLETE: {
auto *trq = event->data.transfer.trq;
this->release_trq(trq);
break;
}
} }
// Return event to pool for reuse // Return event to pool for reuse
this->event_pool.release(event); this->event_pool.release(event);
@@ -313,25 +307,6 @@ void USBClient::on_removed(usb_device_handle_t handle) {
} }
} }
// Helper to queue transfer cleanup to main loop
static void queue_transfer_cleanup(TransferRequest *trq, EventType type) {
auto *client = trq->client;
// Allocate event from pool
UsbEvent *event = client->event_pool.allocate();
if (event == nullptr) {
// No events available - increment counter for periodic logging
client->event_queue.increment_dropped_count();
return;
}
event->type = type;
event->data.transfer.trq = trq;
// Push to lock-free queue (always succeeds since pool size == queue size)
client->event_queue.push(event);
}
// CALLBACK CONTEXT: USB task (called from usb_host_client_handle_events in USB task) // CALLBACK CONTEXT: USB task (called from usb_host_client_handle_events in USB task)
static void control_callback(const usb_transfer_t *xfer) { static void control_callback(const usb_transfer_t *xfer) {
auto *trq = static_cast<TransferRequest *>(xfer->context); auto *trq = static_cast<TransferRequest *>(xfer->context);
@@ -346,8 +321,9 @@ static void control_callback(const usb_transfer_t *xfer) {
trq->callback(trq->status); trq->callback(trq->status);
} }
// Queue cleanup to main loop // Release transfer slot immediately in USB task
queue_transfer_cleanup(trq, EVENT_CONTROL_COMPLETE); // The release_trq() uses thread-safe atomic operations
trq->client->release_trq(trq);
} }
// THREAD CONTEXT: Called from both USB task and main loop threads (multi-consumer) // THREAD CONTEXT: Called from both USB task and main loop threads (multi-consumer)
@@ -358,20 +334,20 @@ static void control_callback(const usb_transfer_t *xfer) {
// This multi-threaded access is intentional for performance - USB task can // This multi-threaded access is intentional for performance - USB task can
// immediately restart transfers without waiting for main loop scheduling. // immediately restart transfers without waiting for main loop scheduling.
TransferRequest *USBClient::get_trq_() { TransferRequest *USBClient::get_trq_() {
uint16_t mask = this->trq_in_use_.load(std::memory_order_relaxed); trq_bitmask_t mask = this->trq_in_use_.load(std::memory_order_relaxed);
// Find first available slot (bit = 0) and try to claim it atomically // Find first available slot (bit = 0) and try to claim it atomically
// We use a while loop to allow retrying the same slot after CAS failure // We use a while loop to allow retrying the same slot after CAS failure
size_t i = 0; size_t i = 0;
while (i != MAX_REQUESTS) { while (i != MAX_REQUESTS) {
if (mask & (1U << i)) { if (mask & (static_cast<trq_bitmask_t>(1) << i)) {
// Slot is in use, move to next slot // Slot is in use, move to next slot
i++; i++;
continue; continue;
} }
// Slot i appears available, try to claim it atomically // Slot i appears available, try to claim it atomically
uint16_t desired = mask | (1U << i); // Set bit i to mark as in-use trq_bitmask_t desired = mask | (static_cast<trq_bitmask_t>(1) << i); // Set bit i to mark as in-use
if (this->trq_in_use_.compare_exchange_weak(mask, desired, std::memory_order_acquire, std::memory_order_relaxed)) { if (this->trq_in_use_.compare_exchange_weak(mask, desired, std::memory_order_acquire, std::memory_order_relaxed)) {
// Successfully claimed slot i - prepare the TransferRequest // Successfully claimed slot i - prepare the TransferRequest
@@ -386,7 +362,7 @@ TransferRequest *USBClient::get_trq_() {
i = 0; i = 0;
} }
ESP_LOGE(TAG, "All %d transfer slots in use", MAX_REQUESTS); ESP_LOGE(TAG, "All %zu transfer slots in use", MAX_REQUESTS);
return nullptr; return nullptr;
} }
void USBClient::disconnect() { void USBClient::disconnect() {
@@ -452,8 +428,10 @@ static void transfer_callback(usb_transfer_t *xfer) {
trq->callback(trq->status); trq->callback(trq->status);
} }
// Queue cleanup to main loop // Release transfer slot immediately in USB task to prevent slot exhaustion
queue_transfer_cleanup(trq, EVENT_TRANSFER_COMPLETE); // This is critical for high-throughput transfers (e.g., USB UART at 115200 baud)
// The release_trq() uses thread-safe atomic operations
trq->client->release_trq(trq);
} }
/** /**
* Performs a transfer input operation. * Performs a transfer input operation.
@@ -521,12 +499,12 @@ void USBClient::dump_config() {
" Product id %04X", " Product id %04X",
this->vid_, this->pid_); this->vid_, this->pid_);
} }
// THREAD CONTEXT: Only called from main loop thread (single producer for deallocation) // THREAD CONTEXT: Called from both USB task and main loop threads
// - Via event processing when handling EVENT_TRANSFER_COMPLETE/EVENT_CONTROL_COMPLETE // - USB task: Immediately after transfer callback completes
// - Directly when transfer submission fails // - Main loop: When transfer submission fails
// //
// THREAD SAFETY: Lock-free using atomic AND to clear bit // THREAD SAFETY: Lock-free using atomic AND to clear bit
// Single-producer pattern makes this simpler than allocation // Thread-safe atomic operation allows multi-threaded deallocation
void USBClient::release_trq(TransferRequest *trq) { void USBClient::release_trq(TransferRequest *trq) {
if (trq == nullptr) if (trq == nullptr)
return; return;
@@ -540,8 +518,8 @@ void USBClient::release_trq(TransferRequest *trq) {
// Atomically clear bit i to mark slot as available // Atomically clear bit i to mark slot as available
// fetch_and with inverted bitmask clears the bit atomically // fetch_and with inverted bitmask clears the bit atomically
uint16_t bit = 1U << index; trq_bitmask_t bit = static_cast<trq_bitmask_t>(1) << index;
this->trq_in_use_.fetch_and(static_cast<uint16_t>(~bit), std::memory_order_release); this->trq_in_use_.fetch_and(static_cast<trq_bitmask_t>(~bit), std::memory_order_release);
} }
} // namespace usb_host } // namespace usb_host

1
esphome/core/defines.h
View File

@@ -191,6 +191,7 @@
#define USE_WEBSERVER_PORT 80 // NOLINT #define USE_WEBSERVER_PORT 80 // NOLINT
#define USE_WEBSERVER_SORTING #define USE_WEBSERVER_SORTING
#define USE_WIFI_11KV_SUPPORT #define USE_WIFI_11KV_SUPPORT
#define USB_HOST_MAX_REQUESTS 16
#ifdef USE_ARDUINO #ifdef USE_ARDUINO
#define USE_ARDUINO_VERSION_CODE VERSION_CODE(3, 2, 1) #define USE_ARDUINO_VERSION_CODE VERSION_CODE(3, 2, 1)