diff --git a/esphome/components/esp32_ble_server/ble_characteristic.cpp b/esphome/components/esp32_ble_server/ble_characteristic.cpp
index 373d57436e..4d0ada0ac2 100644
--- a/esphome/components/esp32_ble_server/ble_characteristic.cpp
+++ b/esphome/components/esp32_ble_server/ble_characteristic.cpp
@@ -51,11 +51,11 @@ void BLECharacteristic::notify() {
 
   for (auto &client : this->service_->get_server()->get_clients()) {
     size_t length = this->value_.size();
-    // If the client is not in the list of clients to notify, skip it
-    if (this->clients_to_notify_.count(client) == 0)
+    // Find the client in the list of clients to notify
+    auto *entry = this->find_client_in_notify_list_(client);
+    if (entry == nullptr)
       continue;
-    // If the client is in the list of clients to notify, check if it requires an ack (i.e. INDICATE)
-    bool require_ack = this->clients_to_notify_[client];
+    bool require_ack = entry->indicate;
     // TODO: Remove this block when INDICATE acknowledgment is supported
     if (require_ack) {
       ESP_LOGW(TAG, "INDICATE acknowledgment is not yet supported (i.e. it works as a NOTIFY)");
@@ -79,10 +79,11 @@ void BLECharacteristic::add_descriptor(BLEDescriptor *descriptor) {
       uint16_t cccd = encode_uint16(value[1], value[0]);
       bool notify = (cccd & 1) != 0;
       bool indicate = (cccd & 2) != 0;
+      // Remove existing entry if present
+      this->remove_client_from_notify_list_(conn_id);
+      // Add new entry if needed
       if (notify || indicate) {
-        this->clients_to_notify_[conn_id] = indicate;
-      } else {
-        this->clients_to_notify_.erase(conn_id);
+        this->clients_to_notify_.push_back({conn_id, indicate});
       }
     });
   }
@@ -307,6 +308,30 @@ void BLECharacteristic::gatts_event_handler(esp_gatts_cb_event_t event, esp_gatt
   }
 }
 
+void BLECharacteristic::remove_client_from_notify_list_(uint16_t conn_id) {
+  // Since we typically have very few clients (often just 1), we can optimize
+  // for the common case by swapping with the last element and popping
+  for (size_t i = 0; i < this->clients_to_notify_.size(); i++) {
+    if (this->clients_to_notify_[i].conn_id == conn_id) {
+      // Swap with last element and pop
+      if (i != this->clients_to_notify_.size() - 1) {
+        this->clients_to_notify_[i] = this->clients_to_notify_.back();
+      }
+      this->clients_to_notify_.pop_back();
+      return;
+    }
+  }
+}
+
+BLECharacteristic::ClientNotificationEntry *BLECharacteristic::find_client_in_notify_list_(uint16_t conn_id) {
+  for (auto &entry : this->clients_to_notify_) {
+    if (entry.conn_id == conn_id) {
+      return &entry;
+    }
+  }
+  return nullptr;
+}
+
 }  // namespace esp32_ble_server
 }  // namespace esphome
 
diff --git a/esphome/components/esp32_ble_server/ble_characteristic.h b/esphome/components/esp32_ble_server/ble_characteristic.h
index 3698b8c4aa..97b3af2a21 100644
--- a/esphome/components/esp32_ble_server/ble_characteristic.h
+++ b/esphome/components/esp32_ble_server/ble_characteristic.h
@@ -6,7 +6,6 @@
 #include "esphome/components/bytebuffer/bytebuffer.h"
 
 #include <vector>
-#include <unordered_map>
 
 #ifdef USE_ESP32
 
@@ -89,7 +88,15 @@ class BLECharacteristic : public EventEmitter<BLECharacteristicEvt::VectorEvt, s
   SemaphoreHandle_t set_value_lock_;
 
   std::vector<BLEDescriptor *> descriptors_;
-  std::unordered_map<uint16_t, bool> clients_to_notify_;
+
+  struct ClientNotificationEntry {
+    uint16_t conn_id;
+    bool indicate;  // true = indicate, false = notify
+  };
+  std::vector<ClientNotificationEntry> clients_to_notify_;
+
+  void remove_client_from_notify_list_(uint16_t conn_id);
+  ClientNotificationEntry *find_client_in_notify_list_(uint16_t conn_id);
 
   esp_gatt_perm_t permissions_ = ESP_GATT_PERM_READ | ESP_GATT_PERM_WRITE;
 
diff --git a/esphome/components/esp32_ble_server/ble_server_automations.cpp b/esphome/components/esp32_ble_server/ble_server_automations.cpp
index 41ef2b8bfe..ea6a074daa 100644
--- a/esphome/components/esp32_ble_server/ble_server_automations.cpp
+++ b/esphome/components/esp32_ble_server/ble_server_automations.cpp
@@ -45,17 +45,16 @@ Trigger<uint16_t> *BLETriggers::create_server_on_disconnect_trigger(BLEServer *s
 void BLECharacteristicSetValueActionManager::set_listener(BLECharacteristic *characteristic,
                                                           EventEmitterListenerID listener_id,
                                                           const std::function<void()> &pre_notify_listener) {
-  // Check if there is already a listener for this characteristic
-  if (this->listeners_.count(characteristic) > 0) {
-    // Unpack the pair listener_id, pre_notify_listener_id
-    auto listener_pairs = this->listeners_[characteristic];
-    EventEmitterListenerID old_listener_id = listener_pairs.first;
-    EventEmitterListenerID old_pre_notify_listener_id = listener_pairs.second;
+  // Find and remove existing listener for this characteristic
+  auto *existing = this->find_listener_(characteristic);
+  if (existing != nullptr) {
     // Remove the previous listener
     characteristic->EventEmitter<BLECharacteristicEvt::EmptyEvt, uint16_t>::off(BLECharacteristicEvt::EmptyEvt::ON_READ,
-                                                                                old_listener_id);
+                                                                                existing->listener_id);
     // Remove the pre-notify listener
-    this->off(BLECharacteristicSetValueActionEvt::PRE_NOTIFY, old_pre_notify_listener_id);
+    this->off(BLECharacteristicSetValueActionEvt::PRE_NOTIFY, existing->pre_notify_listener_id);
+    // Remove from vector
+    this->remove_listener_(characteristic);
   }
   // Create a new listener for the pre-notify event
   EventEmitterListenerID pre_notify_listener_id =
@@ -66,8 +65,32 @@ void BLECharacteristicSetValueActionManager::set_listener(BLECharacteristic *cha
                    pre_notify_listener();
                  }
                });
-  // Save the pair listener_id, pre_notify_listener_id to the map
-  this->listeners_[characteristic] = std::make_pair(listener_id, pre_notify_listener_id);
+  // Save the entry to the vector
+  this->listeners_.push_back({characteristic, listener_id, pre_notify_listener_id});
+}
+
+BLECharacteristicSetValueActionManager::ListenerEntry *BLECharacteristicSetValueActionManager::find_listener_(
+    BLECharacteristic *characteristic) {
+  for (auto &entry : this->listeners_) {
+    if (entry.characteristic == characteristic) {
+      return &entry;
+    }
+  }
+  return nullptr;
+}
+
+void BLECharacteristicSetValueActionManager::remove_listener_(BLECharacteristic *characteristic) {
+  // Since we typically have very few listeners, optimize by swapping with back and popping
+  for (size_t i = 0; i < this->listeners_.size(); i++) {
+    if (this->listeners_[i].characteristic == characteristic) {
+      // Swap with last element and pop
+      if (i != this->listeners_.size() - 1) {
+        this->listeners_[i] = this->listeners_.back();
+      }
+      this->listeners_.pop_back();
+      return;
+    }
+  }
 }
 
 }  // namespace esp32_ble_server_automations
diff --git a/esphome/components/esp32_ble_server/ble_server_automations.h b/esphome/components/esp32_ble_server/ble_server_automations.h
index eab6b05f05..54bc0f2632 100644
--- a/esphome/components/esp32_ble_server/ble_server_automations.h
+++ b/esphome/components/esp32_ble_server/ble_server_automations.h
@@ -8,7 +8,6 @@
 #include "esphome/core/automation.h"
 
 #include <vector>
-#include <unordered_map>
 #include <functional>
 
 #ifdef USE_ESP32
@@ -46,14 +45,27 @@ class BLECharacteristicSetValueActionManager
   void set_listener(BLECharacteristic *characteristic, EventEmitterListenerID listener_id,
                     const std::function<void()> &pre_notify_listener);
   EventEmitterListenerID get_listener(BLECharacteristic *characteristic) {
-    return this->listeners_[characteristic].first;
+    for (const auto &entry : this->listeners_) {
+      if (entry.characteristic == characteristic) {
+        return entry.listener_id;
+      }
+    }
+    return 0;
   }
   void emit_pre_notify(BLECharacteristic *characteristic) {
     this->emit_(BLECharacteristicSetValueActionEvt::PRE_NOTIFY, characteristic);
   }
 
  private:
-  std::unordered_map<BLECharacteristic *, std::pair<EventEmitterListenerID, EventEmitterListenerID>> listeners_;
+  struct ListenerEntry {
+    BLECharacteristic *characteristic;
+    EventEmitterListenerID listener_id;
+    EventEmitterListenerID pre_notify_listener_id;
+  };
+  std::vector<ListenerEntry> listeners_;
+
+  ListenerEntry *find_listener_(BLECharacteristic *characteristic);
+  void remove_listener_(BLECharacteristic *characteristic);
 };
 
 template<typename... Ts> class BLECharacteristicSetValueAction : public Action<Ts...> {