diff --git a/esphome/components/e131/e131.cpp b/esphome/components/e131/e131.cpp
index d18d945cec..c10c88faf2 100644
--- a/esphome/components/e131/e131.cpp
+++ b/esphome/components/e131/e131.cpp
@@ -3,6 +3,8 @@
 #include "e131_addressable_light_effect.h"
 #include "esphome/core/log.h"
 
+#include <algorithm>
+
 namespace esphome {
 namespace e131 {
 
@@ -76,14 +78,14 @@ void E131Component::loop() {
 }
 
 void E131Component::add_effect(E131AddressableLightEffect *light_effect) {
-  if (light_effects_.count(light_effect)) {
+  if (std::find(light_effects_.begin(), light_effects_.end(), light_effect) != light_effects_.end()) {
     return;
   }
 
   ESP_LOGD(TAG, "Registering '%s' for universes %d-%d.", light_effect->get_name(), light_effect->get_first_universe(),
            light_effect->get_last_universe());
 
-  light_effects_.insert(light_effect);
+  light_effects_.push_back(light_effect);
 
   for (auto universe = light_effect->get_first_universe(); universe <= light_effect->get_last_universe(); ++universe) {
     join_(universe);
@@ -91,14 +93,17 @@ void E131Component::add_effect(E131AddressableLightEffect *light_effect) {
 }
 
 void E131Component::remove_effect(E131AddressableLightEffect *light_effect) {
-  if (!light_effects_.count(light_effect)) {
+  auto it = std::find(light_effects_.begin(), light_effects_.end(), light_effect);
+  if (it == light_effects_.end()) {
     return;
   }
 
   ESP_LOGD(TAG, "Unregistering '%s' for universes %d-%d.", light_effect->get_name(), light_effect->get_first_universe(),
            light_effect->get_last_universe());
 
-  light_effects_.erase(light_effect);
+  // Swap with last element and pop for O(1) removal (order doesn't matter)
+  *it = light_effects_.back();
+  light_effects_.pop_back();
 
   for (auto universe = light_effect->get_first_universe(); universe <= light_effect->get_last_universe(); ++universe) {
     leave_(universe);
diff --git a/esphome/components/e131/e131.h b/esphome/components/e131/e131.h
index d0e38fa98c..831138a545 100644
--- a/esphome/components/e131/e131.h
+++ b/esphome/components/e131/e131.h
@@ -7,7 +7,6 @@
 #include <cinttypes>
 #include <map>
 #include <memory>
-#include <set>
 #include <vector>
 
 namespace esphome {
@@ -47,9 +46,8 @@ class E131Component : public esphome::Component {
 
   E131ListenMethod listen_method_{E131_MULTICAST};
   std::unique_ptr<socket::Socket> socket_;
-  std::set<E131AddressableLightEffect *> light_effects_;
+  std::vector<E131AddressableLightEffect *> light_effects_;
   std::map<int, int> universe_consumers_;
-  std::map<int, E131Packet> universe_packets_;
 };
 
 }  // namespace e131