working

esphome/components/ota/ota_backend_esp_idf.cpp

@@ -17,6 +17,10 @@ namespace ota {
 std::unique_ptr<ota::OTABackend> make_ota_backend() { return make_unique<ota::IDFOTABackend>(); }
 
 OTAResponseTypes IDFOTABackend::begin(size_t image_size) {
+  // Reset MD5 validation state
+  this->md5_set_ = false;
+  memset(this->expected_bin_md5_, 0, sizeof(this->expected_bin_md5_));
+
   this->partition_ = esp_ota_get_next_update_partition(nullptr);
   if (this->partition_ == nullptr) {
     return OTA_RESPONSE_ERROR_NO_UPDATE_PARTITION;
@@ -67,7 +71,10 @@ OTAResponseTypes IDFOTABackend::begin(size_t image_size) {
   return OTA_RESPONSE_OK;
 }
 
-void IDFOTABackend::set_update_md5(const char *expected_md5) { memcpy(this->expected_bin_md5_, expected_md5, 32); }
+void IDFOTABackend::set_update_md5(const char *expected_md5) {
+  memcpy(this->expected_bin_md5_, expected_md5, 32);
+  this->md5_set_ = true;
+}
 
 OTAResponseTypes IDFOTABackend::write(uint8_t *data, size_t len) {
   esp_err_t err = esp_ota_write(this->update_handle_, data, len);
@@ -85,10 +92,15 @@ OTAResponseTypes IDFOTABackend::write(uint8_t *data, size_t len) {
 
 OTAResponseTypes IDFOTABackend::end() {
   this->md5_.calculate();
-  if (!this->md5_.equals_hex(this->expected_bin_md5_)) {
-    this->abort();
-    return OTA_RESPONSE_ERROR_MD5_MISMATCH;
+
+  // Only validate MD5 if one was provided
+  if (this->md5_set_) {
+    if (!this->md5_.equals_hex(this->expected_bin_md5_)) {
+      this->abort();
+      return OTA_RESPONSE_ERROR_MD5_MISMATCH;
+    }
   }
+
   esp_err_t err = esp_ota_end(this->update_handle_);
   this->update_handle_ = 0;
   if (err == ESP_OK) {

esphome/components/ota/ota_backend_esp_idf.h

@@ -6,12 +6,14 @@
 #include "esphome/core/defines.h"
 
 #include <esp_ota_ops.h>
+#include <cstring>
 
 namespace esphome {
 namespace ota {
 
 class IDFOTABackend : public OTABackend {
  public:
+  IDFOTABackend() : md5_set_(false) { memset(expected_bin_md5_, 0, sizeof(expected_bin_md5_)); }
   OTAResponseTypes begin(size_t image_size) override;
   void set_update_md5(const char *md5) override;
   OTAResponseTypes write(uint8_t *data, size_t len) override;
@@ -24,6 +26,7 @@ class IDFOTABackend : public OTABackend {
   const esp_partition_t *partition_;
   md5::MD5Digest md5_{};
   char expected_bin_md5_[32];
+  bool md5_set_;
 };
 
 }  // namespace ota

esphome/components/web_server_base/web_server_base.cpp

@@ -4,6 +4,11 @@
 #include "esphome/core/helpers.h"
 #include "esphome/core/log.h"
 
+#ifdef USE_ESP_IDF
+#include "freertos/FreeRTOS.h"
+#include "freertos/task.h"
+#endif
+
 #ifdef USE_ARDUINO
 #include <StreamString.h>
 #if defined(USE_ESP32) || defined(USE_LIBRETINY)
@@ -117,6 +122,7 @@ void OTARequestHandler::handleUpload(AsyncWebServerRequest *request, const Strin
   if (index == 0) {
     this->ota_init_(filename.c_str());
     this->ota_started_ = false;
+    this->ota_success_ = false;
 
     // Create OTA backend
     auto backend = ota::make_ota_backend();
@@ -125,12 +131,14 @@ void OTARequestHandler::handleUpload(AsyncWebServerRequest *request, const Strin
     auto result = backend->begin(0);
     if (result != ota::OTA_RESPONSE_OK) {
       ESP_LOGE(TAG, "OTA begin failed: %d", result);
+      this->ota_success_ = false;
       return;
     }
 
     // Store the backend pointer
     this->ota_backend_ = backend.release();
     this->ota_started_ = true;
+    this->ota_success_ = false;  // Will be set to true only on successful completion
   } else if (!this->ota_started_ || !this->ota_backend_) {
     // Begin failed or was aborted
     return;
@@ -139,6 +147,29 @@ void OTARequestHandler::handleUpload(AsyncWebServerRequest *request, const Strin
   // Write data
   if (len > 0) {
     auto *backend = static_cast<ota::OTABackend *>(this->ota_backend_);
+
+    // Log first chunk of data received by OTA handler
+    if (this->ota_read_length_ == 0 && len >= 8) {
+      ESP_LOGD(TAG, "First data received by OTA handler: %02x %02x %02x %02x %02x %02x %02x %02x", data[0], data[1],
+               data[2], data[3], data[4], data[5], data[6], data[7]);
+      ESP_LOGD(TAG, "Data pointer in OTA handler: %p, len: %zu, index: %zu", data, len, index);
+    }
+
+    // Feed watchdog and yield periodically to prevent timeout during OTA
+    // Flash writes can be slow, especially for large chunks
+    static uint32_t last_ota_yield = 0;
+    static uint32_t ota_chunks_written = 0;
+    uint32_t now = millis();
+    ota_chunks_written++;
+
+    // Yield more frequently during OTA - every 25ms or every 2 chunks
+    if (now - last_ota_yield > 25 || ota_chunks_written >= 2) {
+      // Don't log during yield - logging itself can cause delays
+      vTaskDelay(2);  // Let other tasks run for 2 ticks
+      last_ota_yield = now;
+      ota_chunks_written = 0;
+    }
+
     auto result = backend->write(data, len);
     if (result != ota::OTA_RESPONSE_OK) {
       ESP_LOGE(TAG, "OTA write failed: %d", result);
@@ -146,6 +177,7 @@ void OTARequestHandler::handleUpload(AsyncWebServerRequest *request, const Strin
       delete backend;
       this->ota_backend_ = nullptr;
       this->ota_started_ = false;
+      this->ota_success_ = false;
       return;
     }
 
@@ -157,9 +189,11 @@ void OTARequestHandler::handleUpload(AsyncWebServerRequest *request, const Strin
     auto *backend = static_cast<ota::OTABackend *>(this->ota_backend_);
     auto result = backend->end();
     if (result == ota::OTA_RESPONSE_OK) {
+      this->ota_success_ = true;
       this->schedule_ota_reboot_();
     } else {
       ESP_LOGE(TAG, "OTA end failed: %d", result);
+      this->ota_success_ = false;
     }
     delete backend;
     this->ota_backend_ = nullptr;
@@ -170,6 +204,7 @@ void OTARequestHandler::handleUpload(AsyncWebServerRequest *request, const Strin
 }
 void OTARequestHandler::handleRequest(AsyncWebServerRequest *request) {
 #ifdef USE_WEBSERVER_OTA
+  ESP_LOGD(TAG, "OTA handleRequest called");
   AsyncWebServerResponse *response;
 #ifdef USE_ARDUINO
   if (!Update.hasError()) {
@@ -182,7 +217,12 @@ void OTARequestHandler::handleRequest(AsyncWebServerRequest *request) {
   }
 #endif  // USE_ARDUINO
 #ifdef USE_ESP_IDF
-  response = request->beginResponse(200, "text/plain", this->ota_started_ ? "Update Successful!" : "Update Failed!");
+  if (this->ota_success_) {
+    request->send(200, "text/plain", "Update Successful!");
+  } else {
+    request->send(200, "text/plain", "Update Failed!");
+  }
+  return;
 #endif  // USE_ESP_IDF
   response->addHeader("Connection", "close");
   request->send(response);

esphome/components/web_server_base/web_server_base.h

@@ -127,7 +127,13 @@ class WebServerBase : public Component {
 
 class OTARequestHandler : public AsyncWebHandler {
  public:
-  OTARequestHandler(WebServerBase *parent) : parent_(parent) {}
+  OTARequestHandler(WebServerBase *parent) : parent_(parent) {
+#if defined(USE_ESP_IDF) && defined(USE_WEBSERVER_OTA)
+    this->ota_backend_ = nullptr;
+    this->ota_started_ = false;
+    this->ota_success_ = false;
+#endif
+  }
   void handleRequest(AsyncWebServerRequest *request) override;
   void handleUpload(AsyncWebServerRequest *request, const String &filename, size_t index, uint8_t *data, size_t len,
                     bool final) override;
@@ -153,6 +159,7 @@ class OTARequestHandler : public AsyncWebHandler {
 #if defined(USE_ESP_IDF) && defined(USE_WEBSERVER_OTA)
   void *ota_backend_{nullptr};  // Actually ota::OTABackend*, stored as void* to avoid incomplete type issues
   bool ota_started_{false};
+  bool ota_success_{false};
 #endif
 };
 

esphome/components/web_server_idf/multipart_parser_utils.cpp

@@ -2,6 +2,7 @@
 #ifdef USE_ESP_IDF
 #ifdef USE_WEBSERVER_OTA
 #include "multipart_parser_utils.h"
+#include "esphome/core/log.h"
 
 namespace esphome {
 namespace web_server_idf {
@@ -181,6 +182,10 @@ bool parse_multipart_boundary(const char *content_type, const char **boundary_st
   }
 
   *boundary_start = start;
+
+  // Debug log the extracted boundary
+  ESP_LOGD("multipart_utils", "Extracted boundary: '%.*s' (len: %zu)", (int) *boundary_len, start, *boundary_len);
+
   return true;
 }
 

esphome/components/web_server_idf/multipart_reader.cpp

@@ -12,7 +12,7 @@ namespace web_server_idf {
 
 static const char *const TAG = "multipart_reader";
 
-MultipartReader::MultipartReader(const std::string &boundary) {
+MultipartReader::MultipartReader(const std::string &boundary) : first_data_logged_(false) {
   // Initialize settings with callbacks
   memset(&settings_, 0, sizeof(settings_));
   settings_.on_header_field = on_header_field;
@@ -22,10 +22,14 @@ MultipartReader::MultipartReader(const std::string &boundary) {
   settings_.on_part_data_end = on_part_data_end;
   settings_.on_headers_complete = on_headers_complete;
 
+  ESP_LOGV(TAG, "Initializing multipart parser with boundary: '%s' (len: %zu)", boundary.c_str(), boundary.length());
+
   // Create parser with boundary
   parser_ = multipart_parser_init(boundary.c_str(), &settings_);
   if (parser_) {
     multipart_parser_set_data(parser_, this);
+  } else {
+    ESP_LOGE(TAG, "Failed to initialize multipart parser");
   }
 }
 
@@ -37,9 +41,26 @@ MultipartReader::~MultipartReader() {
 
 size_t MultipartReader::parse(const char *data, size_t len) {
   if (!parser_) {
+    ESP_LOGE(TAG, "Parser not initialized");
     return 0;
   }
-  return multipart_parser_execute(parser_, data, len);
+
+  size_t parsed = multipart_parser_execute(parser_, data, len);
+
+  if (parsed != len) {
+    ESP_LOGD(TAG, "Parser consumed %zu of %zu bytes", parsed, len);
+    // Log the data around the error point
+    if (parsed < len && parsed < 32) {
+      ESP_LOGD(TAG, "Data at error point (offset %zu): %02x %02x %02x %02x", parsed,
+               parsed > 0 ? (uint8_t) data[parsed - 1] : 0, (uint8_t) data[parsed],
+               parsed + 1 < len ? (uint8_t) data[parsed + 1] : 0, parsed + 2 < len ? (uint8_t) data[parsed + 2] : 0);
+
+      // Log what we have vs what parser expects
+      ESP_LOGD(TAG, "Parser error at position %zu: got '%c' (0x%02x)", parsed, data[parsed], (uint8_t) data[parsed]);
+    }
+  }
+
+  return parsed;
 }
 
 void MultipartReader::process_header_() {
@@ -95,7 +116,7 @@ int MultipartReader::on_headers_complete(multipart_parser *parser) {
 
 int MultipartReader::on_part_data_begin(multipart_parser *parser) {
   MultipartReader *reader = static_cast<MultipartReader *>(multipart_parser_get_data(parser));
-  ESP_LOGD(TAG, "Part data begin");
+  ESP_LOGV(TAG, "Part data begin");
   return 0;
 }
 
@@ -104,6 +125,18 @@ int MultipartReader::on_part_data(multipart_parser *parser, const char *at, size
 
   // Only process file uploads
   if (reader->has_file() && reader->data_callback_) {
+    // IMPORTANT: The 'at' pointer points to data within the parser's input buffer.
+    // This data is only valid during this callback. The callback handler MUST
+    // process or copy the data immediately - it cannot store the pointer for
+    // later use as the buffer will be overwritten.
+    // Log first data bytes from multipart parser
+    if (!reader->first_data_logged_ && length >= 8) {
+      ESP_LOGD(TAG, "First part data from parser: %02x %02x %02x %02x %02x %02x %02x %02x", (uint8_t) at[0],
+               (uint8_t) at[1], (uint8_t) at[2], (uint8_t) at[3], (uint8_t) at[4], (uint8_t) at[5], (uint8_t) at[6],
+               (uint8_t) at[7]);
+      reader->first_data_logged_ = true;
+    }
+
     reader->data_callback_(reinterpret_cast<const uint8_t *>(at), length);
   }
 
@@ -113,7 +146,7 @@ int MultipartReader::on_part_data(multipart_parser *parser, const char *at, size
 int MultipartReader::on_part_data_end(multipart_parser *parser) {
   MultipartReader *reader = static_cast<MultipartReader *>(multipart_parser_get_data(parser));
 
-  ESP_LOGD(TAG, "Part data end");
+  ESP_LOGV(TAG, "Part data end");
 
   if (reader->part_complete_callback_) {
     reader->part_complete_callback_();
@@ -122,6 +155,9 @@ int MultipartReader::on_part_data_end(multipart_parser *parser) {
   // Clear part info for next part
   reader->current_part_ = Part{};
 
+  // Reset first_data flag for next upload
+  reader->first_data_logged_ = false;
+
   return 0;
 }
 

esphome/components/web_server_idf/multipart_reader.h

@@ -20,6 +20,11 @@ class MultipartReader {
     std::string content_type;
   };
 
+  // IMPORTANT: The data pointer in DataCallback is only valid during the callback!
+  // The multipart parser passes pointers to its internal buffer which will be
+  // overwritten after the callback returns. Callbacks MUST process or copy the
+  // data immediately - storing the pointer for deferred processing will result
+  // in use-after-free bugs.
   using DataCallback = std::function<void(const uint8_t *data, size_t len)>;
   using PartCompleteCallback = std::function<void()>;
 
@@ -58,6 +63,7 @@ class MultipartReader {
   PartCompleteCallback part_complete_callback_;
 
   bool in_headers_{false};
+  bool first_data_logged_{false};
 
   void process_header_();
 };

esphome/components/web_server_idf/web_server_idf.cpp

@@ -7,6 +7,8 @@
 #include "esphome/core/log.h"
 
 #include "esp_tls_crypto.h"
+#include "freertos/FreeRTOS.h"
+#include "freertos/task.h"
 
 #include "utils.h"
 #include "web_server_idf.h"
@@ -75,7 +77,7 @@ void AsyncWebServer::begin() {
 }
 
 esp_err_t AsyncWebServer::request_post_handler(httpd_req_t *r) {
-  ESP_LOGVV(TAG, "Enter AsyncWebServer::request_post_handler. uri=%s", r->uri);
+  ESP_LOGD(TAG, "Enter AsyncWebServer::request_post_handler. uri=%s", r->uri);
   auto content_type = request_get_header(r, "Content-Type");
 
 #ifdef USE_WEBSERVER_OTA
@@ -91,6 +93,7 @@ esp_err_t AsyncWebServer::request_post_handler(httpd_req_t *r) {
     if (parse_multipart_boundary(ct.c_str(), &boundary_start, &boundary_len)) {
       boundary.assign(boundary_start, boundary_len);
       is_multipart = true;
+      ESP_LOGD(TAG, "Multipart upload detected, boundary: '%s' (len: %zu)", boundary.c_str(), boundary_len);
     } else if (!is_form_urlencoded(ct.c_str())) {
       ESP_LOGW(TAG, "Unsupported content type for POST: %s", ct.c_str());
       // fallback to get handler to support backward compatibility
@@ -123,42 +126,93 @@ esp_err_t AsyncWebServer::request_post_handler(httpd_req_t *r) {
     for (auto *handler : server->handlers_) {
       if (handler->canHandle(&req)) {
         found_handler = handler;
+        ESP_LOGD(TAG, "Found handler for OTA request");
         break;
       }
     }
 
     if (!found_handler) {
+      ESP_LOGW(TAG, "No handler found for OTA request");
       httpd_resp_send_err(r, HTTPD_404_NOT_FOUND, nullptr);
       return ESP_OK;
     }
 
     // Handle multipart upload using the multipart-parser library
-    MultipartReader reader(boundary);
+    // The multipart data starts with "--" + boundary, so we need to prepend it
+    std::string full_boundary = "--" + boundary;
+    ESP_LOGV(TAG, "Initializing multipart reader with full boundary: '%s'", full_boundary.c_str());
+    MultipartReader reader(full_boundary);
     static constexpr size_t CHUNK_SIZE = 1024;
+    // IMPORTANT: chunk_buf is reused for each chunk read from the socket.
+    // The multipart parser will pass pointers into this buffer to callbacks.
+    // Those pointers are only valid during the callback execution!
     std::unique_ptr<char[]> chunk_buf(new char[CHUNK_SIZE]);
     size_t total_len = r->content_len;
     size_t remaining = total_len;
     std::string current_filename;
     bool upload_started = false;
 
+    // Track if we've started the upload
+    bool file_started = false;
+
     // Set up callbacks for the multipart reader
     reader.set_data_callback([&](const uint8_t *data, size_t len) {
-      if (!current_filename.empty()) {
-        found_handler->handleUpload(&req, current_filename, upload_started ? 1 : 0, const_cast<uint8_t *>(data), len,
-                                    false);
-        upload_started = true;
+      // CRITICAL: The data pointer is only valid during this callback!
+      // The multipart parser passes pointers into the chunk_buf buffer, which will be
+      // overwritten when we read the next chunk. We MUST process the data immediately
+      // within this callback - any deferred processing will result in use-after-free bugs
+      // where the data pointer points to corrupted/overwritten memory.
+
+      // By the time on_part_data is called, on_headers_complete has already been called
+      // so we can check for filename
+      if (reader.has_file()) {
+        if (current_filename.empty()) {
+          // First time we see data for this file
+          current_filename = reader.get_current_part().filename;
+          ESP_LOGD(TAG, "Processing file part: '%s'", current_filename.c_str());
+        }
+
+        // Log first few bytes of firmware data (only once)
+        static bool firmware_data_logged = false;
+        if (!firmware_data_logged && len >= 8) {
+          ESP_LOGD(TAG, "First firmware bytes from callback: %02x %02x %02x %02x %02x %02x %02x %02x", data[0], data[1],
+                   data[2], data[3], data[4], data[5], data[6], data[7]);
+          firmware_data_logged = true;
+        }
+
+        if (!file_started) {
+          // Initialize the upload with index=0
+          ESP_LOGD(TAG, "Starting upload for: '%s'", current_filename.c_str());
+          found_handler->handleUpload(&req, current_filename, 0, nullptr, 0, false);
+          file_started = true;
+          upload_started = true;
+        }
+
+        // Process the data chunk immediately - the pointer won't be valid after this callback returns!
+        // DO NOT store the data pointer for later use or pass it to any async/deferred operations.
+        if (len > 0) {
+          found_handler->handleUpload(&req, current_filename, 1, const_cast<uint8_t *>(data), len, false);
+        }
       }
     });
 
     reader.set_part_complete_callback([&]() {
       if (!current_filename.empty() && upload_started) {
-        // Signal end of this part
-        found_handler->handleUpload(&req, current_filename, 2, nullptr, 0, false);
+        ESP_LOGD(TAG, "Part complete callback called for: '%s'", current_filename.c_str());
+        // Signal end of this part - final=true signals completion
+        found_handler->handleUpload(&req, current_filename, 2, nullptr, 0, true);
         current_filename.clear();
         upload_started = false;
+        file_started = false;
       }
     });
 
+    // Track time to yield periodically
+    uint32_t last_yield = millis();
+    static constexpr uint32_t YIELD_INTERVAL_MS = 50;  // Yield every 50ms
+    uint32_t chunks_processed = 0;
+    static constexpr uint32_t CHUNKS_PER_YIELD = 5;  // Also yield every 5 chunks
+
     while (remaining > 0) {
       size_t to_read = std::min(remaining, CHUNK_SIZE);
       int recv_len = httpd_req_recv(r, chunk_buf.get(), to_read);
@@ -172,29 +226,69 @@ esp_err_t AsyncWebServer::request_post_handler(httpd_req_t *r) {
         return ESP_FAIL;
       }
 
-      // Parse multipart data
-      size_t parsed = reader.parse(chunk_buf.get(), recv_len);
-      if (parsed != recv_len) {
-        ESP_LOGW(TAG, "Multipart parser error at byte %zu", total_len - remaining + parsed);
-        httpd_resp_send_err(r, HTTPD_400_BAD_REQUEST, nullptr);
-        return ESP_FAIL;
+      // Yield periodically to prevent watchdog timeout
+      chunks_processed++;
+      uint32_t now = millis();
+      if (now - last_yield > YIELD_INTERVAL_MS || chunks_processed >= CHUNKS_PER_YIELD) {
+        // Don't log during yield - logging itself can cause delays
+        vTaskDelay(2);  // Yield for 2 ticks to give more time to other tasks
+        last_yield = now;
+        chunks_processed = 0;
       }
 
-      // Check if we found a new file part
-      if (reader.has_file() && current_filename.empty()) {
-        current_filename = reader.get_current_part().filename;
+      // Log received vs requested - only log every 100KB to reduce overhead
+      static size_t bytes_logged = 0;
+      bytes_logged += recv_len;
+      if (bytes_logged > 100000) {
+        ESP_LOGD(TAG, "OTA progress: %zu bytes remaining", remaining);
+        bytes_logged = 0;
+      }
+      // Log first few bytes for debugging
+      if (total_len == remaining) {
+        ESP_LOGD(TAG, "First chunk data (hex): %02x %02x %02x %02x %02x %02x %02x %02x", (uint8_t) chunk_buf[0],
+                 (uint8_t) chunk_buf[1], (uint8_t) chunk_buf[2], (uint8_t) chunk_buf[3], (uint8_t) chunk_buf[4],
+                 (uint8_t) chunk_buf[5], (uint8_t) chunk_buf[6], (uint8_t) chunk_buf[7]);
+        ESP_LOGD(TAG, "First chunk data (ascii): %.8s", chunk_buf.get());
+        ESP_LOGD(TAG, "Expected boundary start: %.8s", full_boundary.c_str());
+
+        // Log more of the first chunk to see the headers
+        ESP_LOGD(TAG, "First 256 bytes of upload:");
+        for (int i = 0; i < std::min(recv_len, 256); i += 16) {
+          char hex_buf[50];
+          char ascii_buf[17];
+          int n = std::min(16, recv_len - i);
+          for (int j = 0; j < n; j++) {
+            sprintf(hex_buf + j * 3, "%02x ", (uint8_t) chunk_buf[i + j]);
+            ascii_buf[j] = isprint(chunk_buf[i + j]) ? chunk_buf[i + j] : '.';
+          }
+          ascii_buf[n] = '\0';
+          ESP_LOGD(TAG, "%04x: %-48s %s", i, hex_buf, ascii_buf);
+        }
+      }
+
+      size_t parsed = reader.parse(chunk_buf.get(), recv_len);
+      if (parsed != recv_len) {
+        ESP_LOGW(TAG, "Multipart parser error at byte %zu (parsed %zu of %d bytes)", total_len - remaining + parsed,
+                 parsed, recv_len);
+        httpd_resp_send_err(r, HTTPD_400_BAD_REQUEST, nullptr);
+        return ESP_FAIL;
       }
 
       remaining -= recv_len;
     }
 
     // Final cleanup - send final signal if upload was in progress
+    // This should not be needed as part_complete_callback should handle it
     if (!current_filename.empty() && upload_started) {
+      ESP_LOGW(TAG, "Upload was not properly closed by part_complete_callback");
       found_handler->handleUpload(&req, current_filename, 2, nullptr, 0, true);
+      file_started = false;
     }
 
     // Let handler send response
+    ESP_LOGD(TAG, "Calling handleRequest for OTA response");
     found_handler->handleRequest(&req);
+    ESP_LOGD(TAG, "handleRequest completed");
     return ESP_OK;
   }
 #endif  // USE_WEBSERVER_OTA