Add JPEG encoder support via new camera_encoder component (#9459)

Co-authored-by: J. Nick Koston <nick@koston.org>
Co-authored-by: J. Nick Koston <nick@home-assistant.io>
Co-authored-by: J. Nick Koston <nick+github@koston.org>

CODEOWNERS

@@ -89,6 +89,7 @@ esphome/components/bp5758d/* @Cossid
 esphome/components/button/* @esphome/core
 esphome/components/bytebuffer/* @clydebarrow
 esphome/components/camera/* @DT-art1 @bdraco
+esphome/components/camera_encoder/* @DT-art1
 esphome/components/canbus/* @danielschramm @mvturnho
 esphome/components/cap1188/* @mreditor97
 esphome/components/captive_portal/* @esphome/core

esphome/components/camera/buffer.h

@@ -0,0 +1,18 @@
+#pragma once
+
+#include <cinttypes>
+#include <cstddef>
+
+namespace esphome::camera {
+
+/// Interface for a generic buffer that stores image data.
+class Buffer {
+ public:
+  /// Returns a pointer to the buffer's data.
+  virtual uint8_t *get_data_buffer() = 0;
+  /// Returns the length of the buffer in bytes.
+  virtual size_t get_data_length() = 0;
+  virtual ~Buffer() = default;
+};
+
+}  // namespace esphome::camera

esphome/components/camera/buffer_impl.cpp

@@ -0,0 +1,20 @@
+#include "buffer_impl.h"
+
+namespace esphome::camera {
+
+BufferImpl::BufferImpl(size_t size) {
+  this->data_ = this->allocator_.allocate(size);
+  this->size_ = size;
+}
+
+BufferImpl::BufferImpl(CameraImageSpec *spec) {
+  this->data_ = this->allocator_.allocate(spec->bytes_per_image());
+  this->size_ = spec->bytes_per_image();
+}
+
+BufferImpl::~BufferImpl() {
+  if (this->data_ != nullptr)
+    this->allocator_.deallocate(this->data_, this->size_);
+}
+
+}  // namespace esphome::camera

esphome/components/camera/buffer_impl.h

@@ -0,0 +1,26 @@
+#pragma once
+
+#include "buffer.h"
+#include "camera.h"
+
+namespace esphome::camera {
+
+/// Default implementation of Buffer Interface.
+/// Uses a RAMAllocator for memory reservation.
+class BufferImpl : public Buffer {
+ public:
+  explicit BufferImpl(size_t size);
+  explicit BufferImpl(CameraImageSpec *spec);
+  // -------- Buffer --------
+  uint8_t *get_data_buffer() override { return data_; }
+  size_t get_data_length() override { return size_; }
+  // ------------------------
+  ~BufferImpl() override;
+
+ protected:
+  RAMAllocator<uint8_t> allocator_;
+  size_t size_{};
+  uint8_t *data_{};
+};
+
+}  // namespace esphome::camera

esphome/components/camera/camera.h

@@ -15,6 +15,26 @@ namespace camera {
  */
 enum CameraRequester : uint8_t { IDLE, API_REQUESTER, WEB_REQUESTER };
 
+/// Enumeration of different pixel formats.
+enum PixelFormat : uint8_t {
+  PIXEL_FORMAT_GRAYSCALE = 0,  ///< 8-bit grayscale.
+  PIXEL_FORMAT_RGB565,         ///< 16-bit RGB (5-6-5).
+  PIXEL_FORMAT_BGR888,         ///< RGB pixel data in 8-bit format, stored as B, G, R (1 byte each).
+};
+
+/// Returns string name for a given PixelFormat.
+inline const char *to_string(PixelFormat format) {
+  switch (format) {
+    case PIXEL_FORMAT_GRAYSCALE:
+      return "PIXEL_FORMAT_GRAYSCALE";
+    case PIXEL_FORMAT_RGB565:
+      return "PIXEL_FORMAT_RGB565";
+    case PIXEL_FORMAT_BGR888:
+      return "PIXEL_FORMAT_BGR888";
+  }
+  return "PIXEL_FORMAT_UNKNOWN";
+}
+
 /** Abstract camera image base class.
  *  Encapsulates the JPEG encoded data and it is shared among
  *  all connected clients.
@@ -43,6 +63,29 @@ class CameraImageReader {
   virtual ~CameraImageReader() {}
 };
 
+/// Specification of a caputured camera image.
+/// This struct defines the format and size details for images captured
+/// or processed by a camera component.
+struct CameraImageSpec {
+  uint16_t width;
+  uint16_t height;
+  PixelFormat format;
+  size_t bytes_per_pixel() {
+    switch (format) {
+      case PIXEL_FORMAT_GRAYSCALE:
+        return 1;
+      case PIXEL_FORMAT_RGB565:
+        return 2;
+      case PIXEL_FORMAT_BGR888:
+        return 3;
+    }
+
+    return 1;
+  }
+  size_t bytes_per_row() { return bytes_per_pixel() * width; }
+  size_t bytes_per_image() { return bytes_per_pixel() * width * height; }
+};
+
 /** Abstract camera base class. Collaborates with API.
  *  1) API server starts and installs callback (add_image_callback)
  *     which is called by the camera when a new image is available.

esphome/components/camera/encoder.h

@@ -0,0 +1,69 @@
+#pragma once
+
+#include "buffer.h"
+#include "camera.h"
+
+namespace esphome::camera {
+
+/// Result codes from the encoder used to control camera pipeline flow.
+enum EncoderError : uint8_t {
+  ENCODER_ERROR_SUCCESS = 0,   ///< Encoding succeeded, continue pipeline normally.
+  ENCODER_ERROR_SKIP_FRAME,    ///< Skip current frame, try again on next frame.
+  ENCODER_ERROR_RETRY_FRAME,   ///< Retry current frame, after buffer growth or for incremental encoding.
+  ENCODER_ERROR_CONFIGURATION  ///< Fatal config error, shut down pipeline.
+};
+
+/// Converts EncoderError to string.
+inline const char *to_string(EncoderError error) {
+  switch (error) {
+    case ENCODER_ERROR_SUCCESS:
+      return "ENCODER_ERROR_SUCCESS";
+    case ENCODER_ERROR_SKIP_FRAME:
+      return "ENCODER_ERROR_SKIP_FRAME";
+    case ENCODER_ERROR_RETRY_FRAME:
+      return "ENCODER_ERROR_RETRY_FRAME";
+    case ENCODER_ERROR_CONFIGURATION:
+      return "ENCODER_ERROR_CONFIGURATION";
+  }
+  return "ENCODER_ERROR_INVALID";
+}
+
+/// Interface for an encoder buffer supporting resizing and variable-length data.
+class EncoderBuffer {
+ public:
+  ///  Sets logical buffer size, reallocates if needed.
+  ///  @param size Required size in bytes.
+  ///  @return true on success, false on allocation failure.
+  virtual bool set_buffer_size(size_t size) = 0;
+
+  /// Returns a pointer to the buffer data.
+  virtual uint8_t *get_data() const = 0;
+
+  /// Returns number of bytes currently used.
+  virtual size_t get_size() const = 0;
+
+  ///  Returns total allocated buffer size.
+  virtual size_t get_max_size() const = 0;
+
+  virtual ~EncoderBuffer() = default;
+};
+
+/// Interface for image encoders used in a camera pipeline.
+class Encoder {
+ public:
+  /// Encodes pixel data from a previous camera pipeline stage.
+  /// @param spec Specification of the input pixel data.
+  /// @param pixels Image pixels in RGB or grayscale format, as specified in @p spec.
+  /// @return EncoderError Indicating the result of the encoding operation.
+  virtual EncoderError encode_pixels(CameraImageSpec *spec, Buffer *pixels) = 0;
+
+  /// Returns the encoder's output buffer.
+  /// @return Pointer to an EncoderBuffer containing encoded data.
+  virtual EncoderBuffer *get_output_buffer() = 0;
+
+  ///  Prints the encoder's configuration to the log.
+  virtual void dump_config() = 0;
+  virtual ~Encoder() = default;
+};
+
+}  // namespace esphome::camera

esphome/components/camera_encoder/__init__.py

@@ -0,0 +1,62 @@
+import esphome.codegen as cg
+from esphome.components.esp32 import add_idf_component
+import esphome.config_validation as cv
+from esphome.const import CONF_BUFFER_SIZE, CONF_ID, CONF_TYPE
+from esphome.core import CORE
+from esphome.types import ConfigType
+
+CODEOWNERS = ["@DT-art1"]
+
+AUTO_LOAD = ["camera"]
+
+CONF_BUFFER_EXPAND_SIZE = "buffer_expand_size"
+CONF_ENCODER_BUFFER_ID = "encoder_buffer_id"
+CONF_QUALITY = "quality"
+
+ESP32_CAMERA_ENCODER = "esp32_camera"
+
+camera_ns = cg.esphome_ns.namespace("camera")
+camera_encoder_ns = cg.esphome_ns.namespace("camera_encoder")
+
+Encoder = camera_ns.class_("Encoder")
+EncoderBufferImpl = camera_encoder_ns.class_("EncoderBufferImpl")
+
+ESP32CameraJPEGEncoder = camera_encoder_ns.class_("ESP32CameraJPEGEncoder", Encoder)
+
+MAX_JPEG_BUFFER_SIZE_2MB = 2 * 1024 * 1024
+
+ESP32_CAMERA_ENCODER_SCHEMA = cv.Schema(
+    {
+        cv.GenerateID(): cv.declare_id(ESP32CameraJPEGEncoder),
+        cv.Optional(CONF_QUALITY, default=80): cv.int_range(1, 100),
+        cv.Optional(CONF_BUFFER_SIZE, default=4096): cv.int_range(
+            1024, MAX_JPEG_BUFFER_SIZE_2MB
+        ),
+        cv.Optional(CONF_BUFFER_EXPAND_SIZE, default=1024): cv.int_range(
+            0, MAX_JPEG_BUFFER_SIZE_2MB
+        ),
+        cv.GenerateID(CONF_ENCODER_BUFFER_ID): cv.declare_id(EncoderBufferImpl),
+    }
+)
+
+CONFIG_SCHEMA = cv.typed_schema(
+    {
+        ESP32_CAMERA_ENCODER: ESP32_CAMERA_ENCODER_SCHEMA,
+    },
+    default_type=ESP32_CAMERA_ENCODER,
+)
+
+
+async def to_code(config: ConfigType) -> None:
+    buffer = cg.new_Pvariable(config[CONF_ENCODER_BUFFER_ID])
+    cg.add(buffer.set_buffer_size(config[CONF_BUFFER_SIZE]))
+    if config[CONF_TYPE] == ESP32_CAMERA_ENCODER:
+        if CORE.using_esp_idf:
+            add_idf_component(name="espressif/esp32-camera", ref="2.1.0")
+        cg.add_build_flag("-DUSE_ESP32_CAMERA_JPEG_ENCODER")
+        var = cg.new_Pvariable(
+            config[CONF_ID],
+            config[CONF_QUALITY],
+            buffer,
+        )
+        cg.add(var.set_buffer_expand_size(config[CONF_BUFFER_EXPAND_SIZE]))

esphome/components/camera_encoder/encoder_buffer_impl.cpp

@@ -0,0 +1,23 @@
+#include "encoder_buffer_impl.h"
+
+namespace esphome::camera_encoder {
+
+bool EncoderBufferImpl::set_buffer_size(size_t size) {
+  if (size > this->capacity_) {
+    uint8_t *p = this->allocator_.reallocate(this->data_, size);
+    if (p == nullptr)
+      return false;
+
+    this->data_ = p;
+    this->capacity_ = size;
+  }
+  this->size_ = size;
+  return true;
+}
+
+EncoderBufferImpl::~EncoderBufferImpl() {
+  if (this->data_ != nullptr)
+    this->allocator_.deallocate(this->data_, this->capacity_);
+}
+
+}  // namespace esphome::camera_encoder

esphome/components/camera_encoder/encoder_buffer_impl.h

@@ -0,0 +1,25 @@
+#pragma once
+
+#include "esphome/components/camera/encoder.h"
+#include "esphome/core/helpers.h"
+
+namespace esphome::camera_encoder {
+
+class EncoderBufferImpl : public camera::EncoderBuffer {
+ public:
+  // --- EncoderBuffer  ---
+  bool set_buffer_size(size_t size) override;
+  uint8_t *get_data() const override { return this->data_; }
+  size_t get_size() const override { return this->size_; }
+  size_t get_max_size() const override { return this->capacity_; }
+  // ----------------------
+  ~EncoderBufferImpl() override;
+
+ protected:
+  RAMAllocator<uint8_t> allocator_;
+  size_t capacity_{};
+  size_t size_{};
+  uint8_t *data_{};
+};
+
+}  // namespace esphome::camera_encoder

esphome/components/camera_encoder/esp32_camera_jpeg_encoder.cpp

@@ -0,0 +1,82 @@
+#ifdef USE_ESP32_CAMERA_JPEG_ENCODER
+
+#include "esp32_camera_jpeg_encoder.h"
+
+namespace esphome::camera_encoder {
+
+static const char *const TAG = "camera_encoder";
+
+ESP32CameraJPEGEncoder::ESP32CameraJPEGEncoder(uint8_t quality, camera::EncoderBuffer *output) {
+  this->quality_ = quality;
+  this->output_ = output;
+}
+
+camera::EncoderError ESP32CameraJPEGEncoder::encode_pixels(camera::CameraImageSpec *spec, camera::Buffer *pixels) {
+  this->bytes_written_ = 0;
+  this->out_of_output_memory_ = false;
+  bool success = fmt2jpg_cb(pixels->get_data_buffer(), pixels->get_data_length(), spec->width, spec->height,
+                            to_internal_(spec->format), this->quality_, callback_, this);
+
+  if (!success)
+    return camera::ENCODER_ERROR_CONFIGURATION;
+
+  if (this->out_of_output_memory_) {
+    if (this->buffer_expand_size_ <= 0)
+      return camera::ENCODER_ERROR_SKIP_FRAME;
+
+    size_t current_size = this->output_->get_max_size();
+    size_t new_size = this->output_->get_max_size() + this->buffer_expand_size_;
+    if (!this->output_->set_buffer_size(new_size)) {
+      ESP_LOGE(TAG, "Failed to expand output buffer.");
+      this->buffer_expand_size_ = 0;
+      return camera::ENCODER_ERROR_SKIP_FRAME;
+    }
+
+    ESP_LOGD(TAG, "Output buffer expanded (%u -> %u).", current_size, this->output_->get_max_size());
+    return camera::ENCODER_ERROR_RETRY_FRAME;
+  }
+
+  this->output_->set_buffer_size(this->bytes_written_);
+  return camera::ENCODER_ERROR_SUCCESS;
+}
+
+void ESP32CameraJPEGEncoder::dump_config() {
+  ESP_LOGCONFIG(TAG,
+                "ESP32 Camera JPEG Encoder:\n"
+                "  Size: %zu\n"
+                "  Quality: %d\n"
+                "  Expand: %d\n",
+                this->output_->get_max_size(), this->quality_, this->buffer_expand_size_);
+}
+
+size_t ESP32CameraJPEGEncoder::callback_(void *arg, size_t index, const void *data, size_t len) {
+  ESP32CameraJPEGEncoder *that = reinterpret_cast<ESP32CameraJPEGEncoder *>(arg);
+  uint8_t *buffer = that->output_->get_data();
+  size_t buffer_length = that->output_->get_max_size();
+  if (index + len > buffer_length) {
+    that->out_of_output_memory_ = true;
+    return 0;
+  }
+
+  std::memcpy(&buffer[index], data, len);
+  that->bytes_written_ += len;
+  return len;
+}
+
+pixformat_t ESP32CameraJPEGEncoder::to_internal_(camera::PixelFormat format) {
+  switch (format) {
+    case camera::PIXEL_FORMAT_GRAYSCALE:
+      return PIXFORMAT_GRAYSCALE;
+    case camera::PIXEL_FORMAT_RGB565:
+      return PIXFORMAT_RGB565;
+    // Internal representation for RGB is in byte order: B, G, R
+    case camera::PIXEL_FORMAT_BGR888:
+      return PIXFORMAT_RGB888;
+  }
+
+  return PIXFORMAT_GRAYSCALE;
+}
+
+}  // namespace esphome::camera_encoder
+
+#endif

esphome/components/camera_encoder/esp32_camera_jpeg_encoder.h

@@ -0,0 +1,39 @@
+#pragma once
+
+#ifdef USE_ESP32_CAMERA_JPEG_ENCODER
+
+#include <esp_camera.h>
+
+#include "esphome/components/camera/encoder.h"
+
+namespace esphome::camera_encoder {
+
+/// Encoder that uses the software-based JPEG implementation from Espressif's esp32-camera component.
+class ESP32CameraJPEGEncoder : public camera::Encoder {
+ public:
+  /// Constructs a ESP32CameraJPEGEncoder instance.
+  /// @param quality Sets the quality of the encoded image (1-100).
+  /// @param output Pointer to preallocated output buffer.
+  ESP32CameraJPEGEncoder(uint8_t quality, camera::EncoderBuffer *output);
+  /// Sets the number of bytes to expand the output buffer on underflow during encoding.
+  /// @param buffer_expand_size Number of bytes to expand the buffer.
+  void set_buffer_expand_size(size_t buffer_expand_size) { this->buffer_expand_size_ = buffer_expand_size; }
+  // -------- Encoder --------
+  camera::EncoderError encode_pixels(camera::CameraImageSpec *spec, camera::Buffer *pixels) override;
+  camera::EncoderBuffer *get_output_buffer() override { return output_; }
+  void dump_config() override;
+  // -------------------------
+ protected:
+  static size_t callback_(void *arg, size_t index, const void *data, size_t len);
+  pixformat_t to_internal_(camera::PixelFormat format);
+
+  camera::EncoderBuffer *output_{};
+  size_t buffer_expand_size_{};
+  size_t bytes_written_{};
+  uint8_t quality_{};
+  bool out_of_output_memory_{};
+};
+
+}  // namespace esphome::camera_encoder
+
+#endif

tests/components/camera_encoder/common.yaml

@@ -0,0 +1,5 @@
+camera_encoder:
+  id: jpeg_encoder
+  quality: 80
+  buffer_size: 4096
+  buffer_expand_size: 1024

tests/components/camera_encoder/test.esp32-ard.yaml

@@ -0,0 +1 @@
+<<: !include common.yaml

tests/components/camera_encoder/test.esp32-idf.yaml

@@ -0,0 +1 @@
+<<: !include common.yaml