[online_image] Add JPEG support to online_image (#8127)

Co-authored-by: Jimmy Hedman <jimmy.hedman@gmail.com>
Co-authored-by: J. Nick Koston <nick@koston.org>
Co-authored-by: Rodrigo Martín <contact@rodrigomartin.dev>
Co-authored-by: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com>

esphome/components/online_image/__init__.py

@@ -60,6 +60,15 @@ class BMPFormat(Format):
         cg.add_define("USE_ONLINE_IMAGE_BMP_SUPPORT")
 
 
+class JPEGFormat(Format):
+    def __init__(self):
+        super().__init__("JPEG")
+
+    def actions(self):
+        cg.add_define("USE_ONLINE_IMAGE_JPEG_SUPPORT")
+        cg.add_library("JPEGDEC", "1.6.2", "https://github.com/bitbank2/JPEGDEC")
+
+
 class PNGFormat(Format):
     def __init__(self):
         super().__init__("PNG")
@@ -69,14 +78,15 @@ class PNGFormat(Format):
         cg.add_library("pngle", "1.0.2")
 
 
-# New formats can be added here.
 IMAGE_FORMATS = {
     x.image_type: x
     for x in (
         BMPFormat(),
+        JPEGFormat(),
         PNGFormat(),
     )
 }
+IMAGE_FORMATS.update({"JPG": IMAGE_FORMATS["JPEG"]})
 
 OnlineImage = online_image_ns.class_("OnlineImage", cg.PollingComponent, Image_)
 
@@ -116,7 +126,7 @@ ONLINE_IMAGE_SCHEMA = (
             cv.Required(CONF_URL): cv.url,
             cv.Required(CONF_FORMAT): cv.one_of(*IMAGE_FORMATS, upper=True),
             cv.Optional(CONF_PLACEHOLDER): cv.use_id(Image_),
-            cv.Optional(CONF_BUFFER_SIZE, default=2048): cv.int_range(256, 65536),
+            cv.Optional(CONF_BUFFER_SIZE, default=65536): cv.int_range(256, 65536),
             cv.Optional(CONF_ON_DOWNLOAD_FINISHED): automation.validate_automation(
                 {
                     cv.GenerateID(CONF_TRIGGER_ID): cv.declare_id(

esphome/components/online_image/image_decoder.cpp

@@ -41,5 +41,20 @@ size_t DownloadBuffer::read(size_t len) {
   return this->unread_;
 }
 
+size_t DownloadBuffer::resize(size_t size) {
+  if (this->size_ == size) {
+    return size;
+  }
+  this->allocator_.deallocate(this->buffer_, this->size_);
+  this->size_ = size;
+  this->buffer_ = this->allocator_.allocate(size);
+  this->reset();
+  if (this->buffer_) {
+    return size;
+  } else {
+    return 0;
+  }
+}
+
 }  // namespace online_image
 }  // namespace esphome

esphome/components/online_image/image_decoder.h

@@ -106,6 +106,8 @@ class DownloadBuffer {
 
   void reset() { this->unread_ = 0; }
 
+  size_t resize(size_t size);
+
  protected:
   RAMAllocator<uint8_t> allocator_{};
   uint8_t *buffer_;

esphome/components/online_image/jpeg_image.cpp

@@ -0,0 +1,89 @@
+#include "jpeg_image.h"
+#ifdef USE_ONLINE_IMAGE_JPEG_SUPPORT
+
+#include "esphome/components/display/display_buffer.h"
+#include "esphome/core/application.h"
+#include "esphome/core/helpers.h"
+#include "esphome/core/log.h"
+
+#include "online_image.h"
+static const char *const TAG = "online_image.jpeg";
+
+namespace esphome {
+namespace online_image {
+
+/**
+ * @brief Callback method that will be called by the JPEGDEC engine when a chunk
+ * of the image is decoded.
+ *
+ * @param jpeg  The JPEGDRAW object, including the context data.
+ */
+static int draw_callback(JPEGDRAW *jpeg) {
+  ImageDecoder *decoder = (ImageDecoder *) jpeg->pUser;
+
+  // Some very big images take too long to decode, so feed the watchdog on each callback
+  // to avoid crashing.
+  App.feed_wdt();
+  size_t position = 0;
+  for (size_t y = 0; y < jpeg->iHeight; y++) {
+    for (size_t x = 0; x < jpeg->iWidth; x++) {
+      auto rg = decode_value(jpeg->pPixels[position++]);
+      auto ba = decode_value(jpeg->pPixels[position++]);
+      Color color(rg[1], rg[0], ba[1], ba[0]);
+
+      if (!decoder) {
+        ESP_LOGE(TAG, "Decoder pointer is null!");
+        return 0;
+      }
+      decoder->draw(jpeg->x + x, jpeg->y + y, 1, 1, color);
+    }
+  }
+  return 1;
+}
+
+void JpegDecoder::prepare(size_t download_size) {
+  ImageDecoder::prepare(download_size);
+  auto size = this->image_->resize_download_buffer(download_size);
+  if (size < download_size) {
+    ESP_LOGE(TAG, "Resize failed!");
+    // TODO: return an error code;
+  }
+}
+
+int HOT JpegDecoder::decode(uint8_t *buffer, size_t size) {
+  if (size < this->download_size_) {
+    ESP_LOGV(TAG, "Download not complete. Size: %d/%d", size, this->download_size_);
+    return 0;
+  }
+
+  if (!this->jpeg_.openRAM(buffer, size, draw_callback)) {
+    ESP_LOGE(TAG, "Could not open image for decoding.");
+    return DECODE_ERROR_INVALID_TYPE;
+  }
+  auto jpeg_type = this->jpeg_.getJPEGType();
+  if (jpeg_type == JPEG_MODE_INVALID) {
+    ESP_LOGE(TAG, "Unsupported JPEG image");
+    return DECODE_ERROR_INVALID_TYPE;
+  } else if (jpeg_type == JPEG_MODE_PROGRESSIVE) {
+    ESP_LOGE(TAG, "Progressive JPEG images not supported");
+    return DECODE_ERROR_INVALID_TYPE;
+  }
+  ESP_LOGD(TAG, "Image size: %d x %d, bpp: %d", this->jpeg_.getWidth(), this->jpeg_.getHeight(), this->jpeg_.getBpp());
+
+  this->jpeg_.setUserPointer(this);
+  this->jpeg_.setPixelType(RGB8888);
+  this->set_size(this->jpeg_.getWidth(), this->jpeg_.getHeight());
+  if (!this->jpeg_.decode(0, 0, 0)) {
+    ESP_LOGE(TAG, "Error while decoding.");
+    this->jpeg_.close();
+    return DECODE_ERROR_UNSUPPORTED_FORMAT;
+  }
+  this->decoded_bytes_ = size;
+  this->jpeg_.close();
+  return size;
+}
+
+}  // namespace online_image
+}  // namespace esphome
+
+#endif  // USE_ONLINE_IMAGE_JPEG_SUPPORT

esphome/components/online_image/jpeg_image.h

@@ -0,0 +1,34 @@
+#pragma once
+
+#include "image_decoder.h"
+#include "esphome/core/defines.h"
+#ifdef USE_ONLINE_IMAGE_JPEG_SUPPORT
+#include <JPEGDEC.h>
+
+namespace esphome {
+namespace online_image {
+
+/**
+ * @brief Image decoder specialization for JPEG images.
+ */
+class JpegDecoder : public ImageDecoder {
+ public:
+  /**
+   * @brief Construct a new JPEG Decoder object.
+   *
+   * @param display The image to decode the stream into.
+   */
+  JpegDecoder(OnlineImage *image) : ImageDecoder(image) {}
+  ~JpegDecoder() override {}
+
+  void prepare(size_t download_size) override;
+  int HOT decode(uint8_t *buffer, size_t size) override;
+
+ protected:
+  JPEGDEC jpeg_{};
+};
+
+}  // namespace online_image
+}  // namespace esphome
+
+#endif  // USE_ONLINE_IMAGE_JPEG_SUPPORT

esphome/components/online_image/online_image.cpp

@@ -9,6 +9,9 @@ static const char *const TAG = "online_image";
 #ifdef USE_ONLINE_IMAGE_BMP_SUPPORT
 #include "bmp_image.h"
 #endif
+#ifdef USE_ONLINE_IMAGE_JPEG_SUPPORT
+#include "jpeg_image.h"
+#endif
 #ifdef USE_ONLINE_IMAGE_PNG_SUPPORT
 #include "png_image.h"
 #endif
@@ -32,6 +35,7 @@ OnlineImage::OnlineImage(const std::string &url, int width, int height, ImageFor
     : Image(nullptr, 0, 0, type, transparency),
       buffer_(nullptr),
       download_buffer_(download_buffer_size),
+      download_buffer_initial_size_(download_buffer_size),
       format_(format),
       fixed_width_(width),
       fixed_height_(height) {
@@ -123,23 +127,32 @@ void OnlineImage::update() {
 
 #ifdef USE_ONLINE_IMAGE_BMP_SUPPORT
   if (this->format_ == ImageFormat::BMP) {
+    ESP_LOGD(TAG, "Allocating BMP decoder");
     this->decoder_ = make_unique<BmpDecoder>(this);
   }
 #endif  // ONLINE_IMAGE_BMP_SUPPORT
+#ifdef USE_ONLINE_IMAGE_JPEG_SUPPORT
+  if (this->format_ == ImageFormat::JPEG) {
+    ESP_LOGD(TAG, "Allocating JPEG decoder");
+    this->decoder_ = esphome::make_unique<JpegDecoder>(this);
+  }
+#endif  // USE_ONLINE_IMAGE_JPEG_SUPPORT
 #ifdef USE_ONLINE_IMAGE_PNG_SUPPORT
   if (this->format_ == ImageFormat::PNG) {
+    ESP_LOGD(TAG, "Allocating PNG decoder");
     this->decoder_ = make_unique<PngDecoder>(this);
   }
 #endif  // ONLINE_IMAGE_PNG_SUPPORT
 
   if (!this->decoder_) {
-    ESP_LOGE(TAG, "Could not instantiate decoder. Image format unsupported.");
+    ESP_LOGE(TAG, "Could not instantiate decoder. Image format unsupported: %d", this->format_);
     this->end_connection_();
     this->download_error_callback_.call();
     return;
   }
   this->decoder_->prepare(total_size);
-  ESP_LOGI(TAG, "Downloading image");
+  ESP_LOGI(TAG, "Downloading image (Size: %d)", total_size);
+  this->start_time_ = ::time(nullptr);
 }
 
 void OnlineImage::loop() {
@@ -153,6 +166,7 @@ void OnlineImage::loop() {
     this->height_ = buffer_height_;
     ESP_LOGD(TAG, "Image fully downloaded, read %zu bytes, width/height = %d/%d", this->downloader_->get_bytes_read(),
              this->width_, this->height_);
+    ESP_LOGD(TAG, "Total time: %lds", ::time(nullptr) - this->start_time_);
     this->end_connection_();
     this->download_finished_callback_.call();
     return;
@@ -163,6 +177,10 @@ void OnlineImage::loop() {
   }
   size_t available = this->download_buffer_.free_capacity();
   if (available) {
+    // Some decoders need to fully download the image before downloading.
+    // In case of huge images, don't wait blocking until the whole image has been downloaded,
+    // use smaller chunks
+    available = std::min(available, this->download_buffer_initial_size_);
     auto len = this->downloader_->read(this->download_buffer_.append(), available);
     if (len > 0) {
       this->download_buffer_.write(len);

esphome/components/online_image/online_image.h

@@ -23,7 +23,7 @@ using t_http_codes = enum {
 enum ImageFormat {
   /** Automatically detect from MIME type. Not supported yet. */
   AUTO,
-  /** JPEG format. Not supported yet. */
+  /** JPEG format. */
   JPEG,
   /** PNG format. */
   PNG,
@@ -79,6 +79,13 @@ class OnlineImage : public PollingComponent,
    */
   void release();
 
+  /**
+   * Resize the download buffer
+   *
+   * @param size The new size for the download buffer.
+   */
+  size_t resize_download_buffer(size_t size) { return this->download_buffer_.resize(size); }
+
   void add_on_finished_callback(std::function<void()> &&callback);
   void add_on_error_callback(std::function<void()> &&callback);
 
@@ -119,6 +126,12 @@ class OnlineImage : public PollingComponent,
 
   uint8_t *buffer_;
   DownloadBuffer download_buffer_;
+  /**
+   * This is the *initial* size of the download buffer, not the current size.
+   * The download buffer can be resized at runtime; the download_buffer_initial_size_
+   * will *not* change even if the download buffer has been resized.
+   */
+  size_t download_buffer_initial_size_;
 
   const ImageFormat format_;
   image::Image *placeholder_{nullptr};
@@ -148,6 +161,8 @@ class OnlineImage : public PollingComponent,
    */
   int buffer_height_;
 
+  time_t start_time_;
+
   friend bool ImageDecoder::set_size(int width, int height);
   friend void ImageDecoder::draw(int x, int y, int w, int h, const Color &color);
 };

esphome/components/online_image/png_image.cpp

@@ -2,7 +2,6 @@
 #ifdef USE_ONLINE_IMAGE_PNG_SUPPORT
 
 #include "esphome/components/display/display_buffer.h"
-#include "esphome/core/application.h"
 #include "esphome/core/helpers.h"
 #include "esphome/core/log.h"
 

esphome/core/defines.h

@@ -62,6 +62,7 @@
 #define USE_NUMBER
 #define USE_ONLINE_IMAGE_BMP_SUPPORT
 #define USE_ONLINE_IMAGE_PNG_SUPPORT
+#define USE_ONLINE_IMAGE_JPEG_SUPPORT
 #define USE_OTA
 #define USE_OTA_PASSWORD
 #define USE_OTA_STATE_CALLBACK

platformio.ini

@@ -41,6 +41,8 @@ lib_deps =
     functionpointer/arduino-MLX90393@1.0.2 ; mlx90393
     pavlodn/HaierProtocol@0.9.31           ; haier
     kikuchan98/pngle@1.0.2                 ; online_image
+    ; Using the repository directly, otherwise ESP-IDF can't use the library
+    https://github.com/bitbank2/JPEGDEC.git#1.6.2            ; online_image
     ; This is using the repository until a new release is published to PlatformIO
     https://github.com/Sensirion/arduino-gas-index-algorithm.git#3.2.1 ; Sensirion Gas Index Algorithm Arduino Library
     lvgl/lvgl@8.4.0                                       ; lvgl

tests/components/online_image/common.yaml

@@ -30,6 +30,14 @@ online_image:
     url: https://samples-files.com/samples/images/bmp/480-360-sample.bmp
     format: BMP
     type: BINARY
+  - id: online_jpeg_image
+    url: http://www.faqs.org/images/library.jpg
+    format: JPEG
+    type: RGB
+  - id: online_jpg_image
+    url: http://www.faqs.org/images/library.jpg
+    format: JPG
+    type: RGB565
 
 # Check the set_url action
 esphome: