diff --git a/esphome/components/esp32_ble/__init__.py b/esphome/components/esp32_ble/__init__.py
index dae9799028..0501d1c5ef 100644
--- a/esphome/components/esp32_ble/__init__.py
+++ b/esphome/components/esp32_ble/__init__.py
@@ -242,6 +242,19 @@ def final_validation(config):
                 f"Name '{name}' is too long, maximum length is {max_length} characters"
             )
 
+    # Set GATT Client/Server sdkconfig options based on which components are loaded
+    full_config = fv.full_config.get()
+
+    # Check if BLE Server is needed
+    has_ble_server = "esp32_ble_server" in full_config
+    add_idf_sdkconfig_option("CONFIG_BT_GATTS_ENABLE", has_ble_server)
+
+    # Check if BLE Client is needed (via esp32_ble_tracker or esp32_ble_client)
+    has_ble_client = (
+        "esp32_ble_tracker" in full_config or "esp32_ble_client" in full_config
+    )
+    add_idf_sdkconfig_option("CONFIG_BT_GATTC_ENABLE", has_ble_client)
+
     return config
 
 
diff --git a/esphome/components/esp32_ble/ble.cpp b/esphome/components/esp32_ble/ble.cpp
index e22d43c0cc..591ee0e42f 100644
--- a/esphome/components/esp32_ble/ble.cpp
+++ b/esphome/components/esp32_ble/ble.cpp
@@ -167,6 +167,7 @@ bool ESP32BLE::ble_setup_() {
     }
   }
 
+#ifdef USE_ESP32_BLE_SERVER
   if (!this->gatts_event_handlers_.empty()) {
     err = esp_ble_gatts_register_callback(ESP32BLE::gatts_event_handler);
     if (err != ESP_OK) {
@@ -174,7 +175,9 @@ bool ESP32BLE::ble_setup_() {
       return false;
     }
   }
+#endif
 
+#ifdef USE_ESP32_BLE_CLIENT
   if (!this->gattc_event_handlers_.empty()) {
     err = esp_ble_gattc_register_callback(ESP32BLE::gattc_event_handler);
     if (err != ESP_OK) {
@@ -182,6 +185,7 @@ bool ESP32BLE::ble_setup_() {
       return false;
     }
   }
+#endif
 
   std::string name;
   if (this->name_.has_value()) {
@@ -303,6 +307,7 @@ void ESP32BLE::loop() {
   BLEEvent *ble_event = this->ble_events_.pop();
   while (ble_event != nullptr) {
     switch (ble_event->type_) {
+#ifdef USE_ESP32_BLE_SERVER
       case BLEEvent::GATTS: {
         esp_gatts_cb_event_t event = ble_event->event_.gatts.gatts_event;
         esp_gatt_if_t gatts_if = ble_event->event_.gatts.gatts_if;
@@ -313,6 +318,7 @@ void ESP32BLE::loop() {
         }
         break;
       }
+#endif
       case BLEEvent::GATTC: {
         esp_gattc_cb_event_t event = ble_event->event_.gattc.gattc_event;
         esp_gatt_if_t gattc_if = ble_event->event_.gattc.gattc_if;
@@ -484,15 +490,19 @@ void ESP32BLE::gap_event_handler(esp_gap_ble_cb_event_t event, esp_ble_gap_cb_pa
   ESP_LOGW(TAG, "Ignoring unexpected GAP event type: %d", event);
 }
 
+#ifdef USE_ESP32_BLE_SERVER
 void ESP32BLE::gatts_event_handler(esp_gatts_cb_event_t event, esp_gatt_if_t gatts_if,
                                    esp_ble_gatts_cb_param_t *param) {
   enqueue_ble_event(event, gatts_if, param);
 }
+#endif
 
+#ifdef USE_ESP32_BLE_CLIENT
 void ESP32BLE::gattc_event_handler(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_if,
                                    esp_ble_gattc_cb_param_t *param) {
   enqueue_ble_event(event, gattc_if, param);
 }
+#endif
 
 float ESP32BLE::get_setup_priority() const { return setup_priority::BLUETOOTH; }
 
diff --git a/esphome/components/esp32_ble/ble.h b/esphome/components/esp32_ble/ble.h
index 712787fe53..2232c0b06c 100644
--- a/esphome/components/esp32_ble/ble.h
+++ b/esphome/components/esp32_ble/ble.h
@@ -131,8 +131,12 @@ class ESP32BLE : public Component {
   void set_enable_on_boot(bool enable_on_boot) { this->enable_on_boot_ = enable_on_boot; }
 
  protected:
+#ifdef USE_ESP32_BLE_SERVER
   static void gatts_event_handler(esp_gatts_cb_event_t event, esp_gatt_if_t gatts_if, esp_ble_gatts_cb_param_t *param);
+#endif
+#ifdef USE_ESP32_BLE_CLIENT
   static void gattc_event_handler(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_if, esp_ble_gattc_cb_param_t *param);
+#endif
   static void gap_event_handler(esp_gap_ble_cb_event_t event, esp_ble_gap_cb_param_t *param);
 
   bool ble_setup_();