[usb_host] Fix transfer slot exhaustion at high data rates and add configurable max_transfer_requests (#11174)

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com>

esphome/components/usb_host/__init__.py

@@ -9,6 +9,7 @@ from esphome.components.esp32 import (
 import esphome.config_validation as cv
 from esphome.const import CONF_DEVICES, CONF_ID
 from esphome.cpp_types import Component
+from esphome.types import ConfigType
 
 AUTO_LOAD = ["bytebuffer"]
 CODEOWNERS = ["@clydebarrow"]
@@ -20,6 +21,7 @@ USBClient = usb_host_ns.class_("USBClient", Component)
 CONF_VID = "vid"
 CONF_PID = "pid"
 CONF_ENABLE_HUBS = "enable_hubs"
+CONF_MAX_TRANSFER_REQUESTS = "max_transfer_requests"
 
 
 def usb_device_schema(cls=USBClient, vid: int = None, pid: [int] = None) -> cv.Schema:
@@ -44,6 +46,9 @@ CONFIG_SCHEMA = cv.All(
         {
             cv.GenerateID(): cv.declare_id(USBHost),
             cv.Optional(CONF_ENABLE_HUBS, default=False): cv.boolean,
+            cv.Optional(CONF_MAX_TRANSFER_REQUESTS, default=16): cv.int_range(
+                min=1, max=32
+            ),
             cv.Optional(CONF_DEVICES): cv.ensure_list(usb_device_schema()),
         }
     ),
@@ -58,10 +63,14 @@ async def register_usb_client(config):
     return var
 
 
-async def to_code(config):
+async def to_code(config: ConfigType) -> None:
     add_idf_sdkconfig_option("CONFIG_USB_HOST_CONTROL_TRANSFER_MAX_SIZE", 1024)
     if config.get(CONF_ENABLE_HUBS):
         add_idf_sdkconfig_option("CONFIG_USB_HOST_HUBS_SUPPORTED", True)
+
+    max_requests = config[CONF_MAX_TRANSFER_REQUESTS]
+    cg.add_define("USB_HOST_MAX_REQUESTS", max_requests)
+
     var = cg.new_Pvariable(config[CONF_ID])
     await cg.register_component(var, config)
     for device in config.get(CONF_DEVICES) or ():

esphome/components/usb_host/usb_host.h

@@ -2,6 +2,7 @@
 
 // Should not be needed, but it's required to pass CI clang-tidy checks
 #if defined(USE_ESP32_VARIANT_ESP32S2) || defined(USE_ESP32_VARIANT_ESP32S3) || defined(USE_ESP32_VARIANT_ESP32P4)
+#include "esphome/core/defines.h"
 #include "esphome/core/component.h"
 #include <vector>
 #include "usb/usb_host.h"
@@ -16,23 +17,25 @@ namespace usb_host {
 
 // THREADING MODEL:
 // This component uses a dedicated USB task for event processing to prevent data loss.
-// - USB Task (high priority): Handles USB events, executes transfer callbacks
-// - Main Loop Task: Initiates transfers, processes completion events
+// - USB Task (high priority): Handles USB events, executes transfer callbacks, releases transfer slots
+// - Main Loop Task: Initiates transfers, processes device connect/disconnect events
 //
 // Thread-safe communication:
 // - Lock-free queues for USB task -> main loop events (SPSC pattern)
-// - Lock-free TransferRequest pool using atomic bitmask (MCSP pattern)
+// - Lock-free TransferRequest pool using atomic bitmask (MCMP pattern - multi-consumer, multi-producer)
 //
 // TransferRequest pool access pattern:
 // - get_trq_() [allocate]: Called from BOTH USB task and main loop threads
 //   * USB task: via USB UART input callbacks that restart transfers immediately
 //   * Main loop: for output transfers and flow-controlled input restarts
-// - release_trq() [deallocate]: Called from main loop thread only
+// - release_trq() [deallocate]: Called from BOTH USB task and main loop threads
+//   * USB task: immediately after transfer callback completes (critical for preventing slot exhaustion)
+//   * Main loop: when transfer submission fails
 //
-// The multi-threaded allocation is intentional for performance:
-// - USB task can immediately restart input transfers without context switching
+// The multi-threaded allocation/deallocation is intentional for performance:
+// - USB task can immediately restart input transfers and release slots without context switching
 // - Main loop controls backpressure by deciding when to restart after consuming data
-// The atomic bitmask ensures thread-safe allocation without mutex blocking.
+// The atomic bitmask ensures thread-safe allocation/deallocation without mutex blocking.
 
 static const char *const TAG = "usb_host";
 
@@ -52,8 +55,17 @@ static const uint8_t USB_DIR_IN = 1 << 7;
 static const uint8_t USB_DIR_OUT = 0;
 static const size_t SETUP_PACKET_SIZE = 8;
 
-static const size_t MAX_REQUESTS = 16;  // maximum number of outstanding requests possible.
-static_assert(MAX_REQUESTS <= 16, "MAX_REQUESTS must be <= 16 to fit in uint16_t bitmask");
+static const size_t MAX_REQUESTS = USB_HOST_MAX_REQUESTS;  // maximum number of outstanding requests possible.
+static_assert(MAX_REQUESTS >= 1 && MAX_REQUESTS <= 32, "MAX_REQUESTS must be between 1 and 32");
+
+// Select appropriate bitmask type for tracking allocation of TransferRequest slots.
+// The bitmask must have at least as many bits as MAX_REQUESTS, so:
+// - Use uint16_t for up to 16 requests (MAX_REQUESTS <= 16)
+// - Use uint32_t for 17-32 requests (MAX_REQUESTS > 16)
+// This is tied to the static_assert above, which enforces MAX_REQUESTS is between 1 and 32.
+// If MAX_REQUESTS is increased above 32, this logic and the static_assert must be updated.
+using trq_bitmask_t = std::conditional<(MAX_REQUESTS <= 16), uint16_t, uint32_t>::type;
+
 static constexpr size_t USB_EVENT_QUEUE_SIZE = 32;   // Size of event queue between USB task and main loop
 static constexpr size_t USB_TASK_STACK_SIZE = 4096;  // Stack size for USB task (same as ESP-IDF USB examples)
 static constexpr UBaseType_t USB_TASK_PRIORITY = 5;  // Higher priority than main loop (tskIDLE_PRIORITY + 5)
@@ -83,8 +95,6 @@ struct TransferRequest {
 enum EventType : uint8_t {
   EVENT_DEVICE_NEW,
   EVENT_DEVICE_GONE,
-  EVENT_TRANSFER_COMPLETE,
-  EVENT_CONTROL_COMPLETE,
 };
 
 struct UsbEvent {
@@ -96,9 +106,6 @@ struct UsbEvent {
     struct {
       usb_device_handle_t handle;
     } device_gone;
-    struct {
-      TransferRequest *trq;
-    } transfer;
   } data;
 
   // Required for EventPool - no cleanup needed for POD types
@@ -163,10 +170,9 @@ class USBClient : public Component {
   uint16_t pid_{};
   // Lock-free pool management using atomic bitmask (no dynamic allocation)
   // Bit i = 1: requests_[i] is in use, Bit i = 0: requests_[i] is available
-  // Supports multiple concurrent consumers (both threads can allocate)
-  // Single producer for deallocation (main loop only)
-  // Limited to 16 slots by uint16_t size (enforced by static_assert)
-  std::atomic<uint16_t> trq_in_use_;
+  // Supports multiple concurrent consumers and producers (both threads can allocate/deallocate)
+  // Bitmask type automatically selected: uint16_t for <= 16 slots, uint32_t for 17-32 slots
+  std::atomic<trq_bitmask_t> trq_in_use_;
   TransferRequest requests_[MAX_REQUESTS]{};
 };
 class USBHost : public Component {

esphome/components/usb_host/usb_host_client.cpp

@@ -228,12 +228,6 @@ void USBClient::loop() {
       case EVENT_DEVICE_GONE:
         this->on_removed(event->data.device_gone.handle);
         break;
-      case EVENT_TRANSFER_COMPLETE:
-      case EVENT_CONTROL_COMPLETE: {
-        auto *trq = event->data.transfer.trq;
-        this->release_trq(trq);
-        break;
-      }
     }
     // Return event to pool for reuse
     this->event_pool.release(event);
@@ -313,25 +307,6 @@ void USBClient::on_removed(usb_device_handle_t handle) {
   }
 }
 
-// Helper to queue transfer cleanup to main loop
-static void queue_transfer_cleanup(TransferRequest *trq, EventType type) {
-  auto *client = trq->client;
-
-  // Allocate event from pool
-  UsbEvent *event = client->event_pool.allocate();
-  if (event == nullptr) {
-    // No events available - increment counter for periodic logging
-    client->event_queue.increment_dropped_count();
-    return;
-  }
-
-  event->type = type;
-  event->data.transfer.trq = trq;
-
-  // Push to lock-free queue (always succeeds since pool size == queue size)
-  client->event_queue.push(event);
-}
-
 // CALLBACK CONTEXT: USB task (called from usb_host_client_handle_events in USB task)
 static void control_callback(const usb_transfer_t *xfer) {
   auto *trq = static_cast<TransferRequest *>(xfer->context);
@@ -346,8 +321,9 @@ static void control_callback(const usb_transfer_t *xfer) {
     trq->callback(trq->status);
   }
 
-  // Queue cleanup to main loop
-  queue_transfer_cleanup(trq, EVENT_CONTROL_COMPLETE);
+  // Release transfer slot immediately in USB task
+  // The release_trq() uses thread-safe atomic operations
+  trq->client->release_trq(trq);
 }
 
 // THREAD CONTEXT: Called from both USB task and main loop threads (multi-consumer)
@@ -358,20 +334,20 @@ static void control_callback(const usb_transfer_t *xfer) {
 // This multi-threaded access is intentional for performance - USB task can
 // immediately restart transfers without waiting for main loop scheduling.
 TransferRequest *USBClient::get_trq_() {
-  uint16_t mask = this->trq_in_use_.load(std::memory_order_relaxed);
+  trq_bitmask_t mask = this->trq_in_use_.load(std::memory_order_relaxed);
 
   // Find first available slot (bit = 0) and try to claim it atomically
   // We use a while loop to allow retrying the same slot after CAS failure
   size_t i = 0;
   while (i != MAX_REQUESTS) {
-    if (mask & (1U << i)) {
+    if (mask & (static_cast<trq_bitmask_t>(1) << i)) {
       // Slot is in use, move to next slot
       i++;
       continue;
     }
 
     // Slot i appears available, try to claim it atomically
-    uint16_t desired = mask | (1U << i);  // Set bit i to mark as in-use
+    trq_bitmask_t desired = mask | (static_cast<trq_bitmask_t>(1) << i);  // Set bit i to mark as in-use
 
     if (this->trq_in_use_.compare_exchange_weak(mask, desired, std::memory_order_acquire, std::memory_order_relaxed)) {
       // Successfully claimed slot i - prepare the TransferRequest
@@ -386,7 +362,7 @@ TransferRequest *USBClient::get_trq_() {
     i = 0;
   }
 
-  ESP_LOGE(TAG, "All %d transfer slots in use", MAX_REQUESTS);
+  ESP_LOGE(TAG, "All %zu transfer slots in use", MAX_REQUESTS);
   return nullptr;
 }
 void USBClient::disconnect() {
@@ -452,8 +428,11 @@ static void transfer_callback(usb_transfer_t *xfer) {
     trq->callback(trq->status);
   }
 
-  // Queue cleanup to main loop
-  queue_transfer_cleanup(trq, EVENT_TRANSFER_COMPLETE);
+  // Release transfer slot AFTER callback completes to prevent slot exhaustion
+  // This is critical for high-throughput transfers (e.g., USB UART at 115200 baud)
+  // The callback has finished accessing xfer->data_buffer, so it's safe to release
+  // The release_trq() uses thread-safe atomic operations
+  trq->client->release_trq(trq);
 }
 /**
  * Performs a transfer input operation.
@@ -521,12 +500,12 @@ void USBClient::dump_config() {
                 "  Product id %04X",
                 this->vid_, this->pid_);
 }
-// THREAD CONTEXT: Only called from main loop thread (single producer for deallocation)
-// - Via event processing when handling EVENT_TRANSFER_COMPLETE/EVENT_CONTROL_COMPLETE
-// - Directly when transfer submission fails
+// THREAD CONTEXT: Called from both USB task and main loop threads
+// - USB task: Immediately after transfer callback completes
+// - Main loop: When transfer submission fails
 //
 // THREAD SAFETY: Lock-free using atomic AND to clear bit
-// Single-producer pattern makes this simpler than allocation
+// Thread-safe atomic operation allows multi-threaded deallocation
 void USBClient::release_trq(TransferRequest *trq) {
   if (trq == nullptr)
     return;
@@ -540,8 +519,8 @@ void USBClient::release_trq(TransferRequest *trq) {
 
   // Atomically clear bit i to mark slot as available
   // fetch_and with inverted bitmask clears the bit atomically
-  uint16_t bit = 1U << index;
-  this->trq_in_use_.fetch_and(static_cast<uint16_t>(~bit), std::memory_order_release);
+  trq_bitmask_t bit = static_cast<trq_bitmask_t>(1) << index;
+  this->trq_in_use_.fetch_and(static_cast<trq_bitmask_t>(~bit), std::memory_order_release);
 }
 
 }  // namespace usb_host

esphome/core/defines.h

@@ -193,6 +193,7 @@
 #define USE_WEBSERVER_PORT 80  // NOLINT
 #define USE_WEBSERVER_SORTING
 #define USE_WIFI_11KV_SUPPORT
+#define USB_HOST_MAX_REQUESTS 16
 
 #ifdef USE_ARDUINO
 #define USE_ARDUINO_VERSION_CODE VERSION_CODE(3, 2, 1)

tests/components/usb_host/test.esp32-s3-idf.yaml

@@ -1,4 +1,5 @@
 usb_host:
+  max_transfer_requests: 32  # Test uint32_t bitmask path (17-32 requests)
   devices:
     - id: device_1
       vid: 0x1234