[esphome.ota] Fix ESP32-S3 OTA authentication with hardware SHA acceleration (#11011)

esphome/components/esphome/ota/ota_esphome.cpp

@@ -614,24 +614,67 @@ bool ESPHomeOTAComponent::handle_auth_send_() {
       return false;
     }
 
-    // Generate nonce with appropriate hasher
+    // Generate nonce - hasher must be created and used in same stack frame
-    bool success = false;
+    // CRITICAL ESP32-S3 HARDWARE SHA ACCELERATION REQUIREMENTS:
+    // 1. Hash objects must NEVER be passed to another function (different stack frame)
+    // 2. NO Variable Length Arrays (VLAs) - they corrupt the stack with hardware DMA
+    // 3. All hash operations (init/add/calculate) must happen in the SAME function where object is created
+    // Violating these causes truncated hash output (20 bytes instead of 32) or memory corruption.
+    //
+    // Buffer layout after AUTH_READ completes:
+    //   [0]: auth_type (1 byte)
+    //   [1...hex_size]: nonce (hex_size bytes) - our random nonce sent in AUTH_SEND
+    //   [1+hex_size...1+2*hex_size-1]: cnonce (hex_size bytes) - client's nonce
+    //   [1+2*hex_size...1+3*hex_size-1]: response (hex_size bytes) - client's hash
+
+    // Declare both hash objects in same stack frame, use pointer to select.
+    // NOTE: Both objects are declared here even though only one is used. This is REQUIRED for ESP32-S3
+    // hardware SHA acceleration - the object must exist in this stack frame for all operations.
+    // Do NOT try to "optimize" by creating the object inside the if block, as it would go out of scope.
+#ifdef USE_OTA_SHA256
+    sha256::SHA256 sha_hasher;
+#endif
+#ifdef USE_OTA_MD5
+    md5::MD5Digest md5_hasher;
+#endif
+    HashBase *hasher = nullptr;
+
 #ifdef USE_OTA_SHA256
     if (this->auth_type_ == ota::OTA_RESPONSE_REQUEST_SHA256_AUTH) {
-      sha256::SHA256 sha_hasher;
+      hasher = &sha_hasher;
-      success = this->prepare_auth_nonce_(&sha_hasher);
     }
 #endif
 #ifdef USE_OTA_MD5
     if (this->auth_type_ == ota::OTA_RESPONSE_REQUEST_AUTH) {
-      md5::MD5Digest md5_hasher;
+      hasher = &md5_hasher;
-      success = this->prepare_auth_nonce_(&md5_hasher);
     }
 #endif
 
-    if (!success) {
+    const size_t hex_size = hasher->get_size() * 2;
+    const size_t nonce_len = hasher->get_size() / 4;
+    const size_t auth_buf_size = 1 + 3 * hex_size;
+    this->auth_buf_ = std::make_unique<uint8_t[]>(auth_buf_size);
+    this->auth_buf_pos_ = 0;
+
+    char *buf = reinterpret_cast<char *>(this->auth_buf_.get() + 1);
+    if (!random_bytes(reinterpret_cast<uint8_t *>(buf), nonce_len)) {
+      this->log_auth_warning_(LOG_STR("Random failed"));
+      this->send_error_and_cleanup_(ota::OTA_RESPONSE_ERROR_UNKNOWN);
       return false;
     }
+
+    hasher->init();
+    hasher->add(buf, nonce_len);
+    hasher->calculate();
+    this->auth_buf_[0] = this->auth_type_;
+    hasher->get_hex(buf);
+
+#if ESPHOME_LOG_LEVEL >= ESPHOME_LOG_LEVEL_VERBOSE
+    char log_buf[65];  // Fixed size for SHA256 hex (64) + null, works for MD5 (32) too
+    memcpy(log_buf, buf, hex_size);
+    log_buf[hex_size] = '\0';
+    ESP_LOGV(TAG, "Auth: Nonce is %s", log_buf);
+#endif
   }
 
   // Try to write auth_type + nonce
@@ -678,89 +721,41 @@ bool ESPHomeOTAComponent::handle_auth_read_() {
   }
 
   // We have all the data, verify it
-  bool matches = false;
+  const char *nonce = reinterpret_cast<char *>(this->auth_buf_.get() + 1);
+  const char *cnonce = nonce + hex_size;
+  const char *response = cnonce + hex_size;
+
+  // CRITICAL ESP32-S3: Hash objects must stay in same stack frame (no passing to other functions).
+  // Declare both hash objects in same stack frame, use pointer to select.
+  // NOTE: Both objects are declared here even though only one is used. This is REQUIRED for ESP32-S3
+  // hardware SHA acceleration - the object must exist in this stack frame for all operations.
+  // Do NOT try to "optimize" by creating the object inside the if block, as it would go out of scope.
+#ifdef USE_OTA_SHA256
+  sha256::SHA256 sha_hasher;
+#endif
+#ifdef USE_OTA_MD5
+  md5::MD5Digest md5_hasher;
+#endif
+  HashBase *hasher = nullptr;
 
 #ifdef USE_OTA_SHA256
   if (this->auth_type_ == ota::OTA_RESPONSE_REQUEST_SHA256_AUTH) {
-    sha256::SHA256 sha_hasher;
+    hasher = &sha_hasher;
-    matches = this->verify_hash_auth_(&sha_hasher, hex_size);
   }
 #endif
 #ifdef USE_OTA_MD5
   if (this->auth_type_ == ota::OTA_RESPONSE_REQUEST_AUTH) {
-    md5::MD5Digest md5_hasher;
+    hasher = &md5_hasher;
-    matches = this->verify_hash_auth_(&md5_hasher, hex_size);
   }
 #endif
 
-  if (!matches) {
-    this->log_auth_warning_(LOG_STR("Password mismatch"));
-    this->send_error_and_cleanup_(ota::OTA_RESPONSE_ERROR_AUTH_INVALID);
-    return false;
-  }
-
-  // Authentication successful - clean up auth state
-  this->cleanup_auth_();
-
-  return true;
-}
-
-bool ESPHomeOTAComponent::prepare_auth_nonce_(HashBase *hasher) {
-  // Calculate required buffer size using the hasher
-  const size_t hex_size = hasher->get_size() * 2;
-  const size_t nonce_len = hasher->get_size() / 4;
-
-  // Buffer layout after AUTH_READ completes:
-  //   [0]: auth_type (1 byte)
-  //   [1...hex_size]: nonce (hex_size bytes) - our random nonce sent in AUTH_SEND
-  //   [1+hex_size...1+2*hex_size-1]: cnonce (hex_size bytes) - client's nonce
-  //   [1+2*hex_size...1+3*hex_size-1]: response (hex_size bytes) - client's hash
-  // Total: 1 + 3*hex_size
-  const size_t auth_buf_size = 1 + 3 * hex_size;
-  this->auth_buf_ = std::make_unique<uint8_t[]>(auth_buf_size);
-  this->auth_buf_pos_ = 0;
-
-  // Generate nonce
-  char *buf = reinterpret_cast<char *>(this->auth_buf_.get() + 1);
-  if (!random_bytes(reinterpret_cast<uint8_t *>(buf), nonce_len)) {
-    this->log_auth_warning_(LOG_STR("Random failed"));
-    this->send_error_and_cleanup_(ota::OTA_RESPONSE_ERROR_UNKNOWN);
-    return false;
-  }
-
-  hasher->init();
-  hasher->add(buf, nonce_len);
-  hasher->calculate();
-
-  // Prepare buffer: auth_type (1 byte) + nonce (hex_size bytes)
-  this->auth_buf_[0] = this->auth_type_;
-  hasher->get_hex(buf);
-
-#if ESPHOME_LOG_LEVEL >= ESPHOME_LOG_LEVEL_VERBOSE
-  char log_buf[hex_size + 1];
-  // Log nonce for debugging
-  memcpy(log_buf, buf, hex_size);
-  log_buf[hex_size] = '\0';
-  ESP_LOGV(TAG, "Auth: Nonce is %s", log_buf);
-#endif
-
-  return true;
-}
-
-bool ESPHomeOTAComponent::verify_hash_auth_(HashBase *hasher, size_t hex_size) {
-  // Get pointers to the data in the buffer (see prepare_auth_nonce_ for buffer layout)
-  const char *nonce = reinterpret_cast<char *>(this->auth_buf_.get() + 1);  // Skip auth_type byte
-  const char *cnonce = nonce + hex_size;                                    // CNonce immediately follows nonce
-  const char *response = cnonce + hex_size;                                 // Response immediately follows cnonce
-
-  // Calculate expected hash: password + nonce + cnonce
   hasher->init();
   hasher->add(this->password_.c_str(), this->password_.length());
   hasher->add(nonce, hex_size * 2);  // Add both nonce and cnonce (contiguous in buffer)
   hasher->calculate();
 
 #if ESPHOME_LOG_LEVEL >= ESPHOME_LOG_LEVEL_VERBOSE
-  char log_buf[hex_size + 1];
+  char log_buf[65];  // Fixed size for SHA256 hex (64) + null, works for MD5 (32) too
   // Log CNonce
   memcpy(log_buf, cnonce, hex_size);
   log_buf[hex_size] = '\0';
@@ -778,7 +773,18 @@ bool ESPHomeOTAComponent::verify_hash_auth_(HashBase *hasher, size_t hex_size) {
 #endif
 
   // Compare response
-  return hasher->equals_hex(response);
+  bool matches = hasher->equals_hex(response);
+
+  if (!matches) {
+    this->log_auth_warning_(LOG_STR("Password mismatch"));
+    this->send_error_and_cleanup_(ota::OTA_RESPONSE_ERROR_AUTH_INVALID);
+    return false;
+  }
+
+  // Authentication successful - clean up auth state
+  this->cleanup_auth_();
+
+  return true;
 }
 
 size_t ESPHomeOTAComponent::get_auth_hex_size_() const {

esphome/components/esphome/ota/ota_esphome.h

@@ -47,8 +47,6 @@ class ESPHomeOTAComponent : public ota::OTAComponent {
   bool handle_auth_send_();
   bool handle_auth_read_();
   bool select_auth_type_();
-  bool prepare_auth_nonce_(HashBase *hasher);
-  bool verify_hash_auth_(HashBase *hasher, size_t hex_size);
   size_t get_auth_hex_size_() const;
   void cleanup_auth_();
   void log_auth_warning_(const LogString *msg);

esphome/components/sha256/sha256.cpp

@@ -10,6 +10,39 @@ namespace esphome::sha256 {
 
 #if defined(USE_ESP32) || defined(USE_LIBRETINY)
 
+// CRITICAL ESP32-S3 HARDWARE SHA ACCELERATION REQUIREMENTS:
+//
+// The ESP32-S3 uses hardware DMA for SHA acceleration. The mbedtls_sha256_context structure contains
+// internal state that the DMA engine references. This imposes two critical constraints:
+//
+// 1. NO VARIABLE LENGTH ARRAYS (VLAs): VLAs corrupt the stack layout, causing the DMA engine to
+//    write to incorrect memory locations. This results in null pointer dereferences and crashes.
+//    ALWAYS use fixed-size arrays (e.g., char buf[65], not char buf[size+1]).
+//
+// 2. SAME STACK FRAME ONLY: The SHA256 object must be created and used entirely within the same
+//    function. NEVER pass the SHA256 object or HashBase pointer to another function. When the stack
+//    frame changes (function call/return), the DMA references become invalid and will produce
+//    truncated hash output (20 bytes instead of 32) or corrupt memory.
+//
+// CORRECT USAGE:
+//   void my_function() {
+//     sha256::SHA256 hasher;  // Created locally
+//     hasher.init();
+//     hasher.add(data, len);  // Any size, no chunking needed
+//     hasher.calculate();
+//     bool ok = hasher.equals_hex(expected);
+//     // hasher destroyed when function returns
+//   }
+//
+// INCORRECT USAGE (WILL FAIL ON ESP32-S3):
+//   void my_function() {
+//     sha256::SHA256 hasher;
+//     helper(&hasher);  // WRONG: Passed to different stack frame
+//   }
+//   void helper(HashBase *h) {
+//     h->init();  // WRONG: Will produce truncated/corrupted output
+//   }
+
 SHA256::~SHA256() { mbedtls_sha256_free(&this->ctx_); }
 
 void SHA256::init() {

esphome/components/sha256/sha256.h

@@ -39,6 +39,10 @@ class SHA256 : public esphome::HashBase {
 
  protected:
 #if defined(USE_ESP32) || defined(USE_LIBRETINY)
+  // CRITICAL: The mbedtls context MUST be stack-allocated (not a pointer) for ESP32-S3 hardware SHA acceleration.
+  // The ESP32-S3 DMA engine references this structure's memory addresses. If the context is passed to another
+  // function (crossing stack frames) or if VLAs are present, the DMA operations will corrupt memory and produce
+  // truncated/incorrect hash results.
   mbedtls_sha256_context ctx_{};
 #elif defined(USE_ESP8266) || defined(USE_RP2040)
   br_sha256_context ctx_{};

esphome/core/hash_base.h

@@ -39,7 +39,7 @@ class HashBase {
 
   /// Compare the hash against a provided hex-encoded hash
   bool equals_hex(const char *expected) {
-    uint8_t parsed[this->get_size()];
+    uint8_t parsed[32];  // Fixed size for max hash (SHA256 = 32 bytes)
     if (!parse_hex(expected, parsed, this->get_size())) {
       return false;
     }