diff --git a/app/Models/WebhookConfiguration.php b/app/Models/WebhookConfiguration.php
index d5a1b0ba0..60eade1ab 100644
--- a/app/Models/WebhookConfiguration.php
+++ b/app/Models/WebhookConfiguration.php
@@ -6,6 +6,7 @@ use Illuminate\Database\Eloquent\Factories\HasFactory;
 use Illuminate\Database\Eloquent\Model;
 use Illuminate\Database\Eloquent\Relations\HasMany;
 use Illuminate\Database\Eloquent\SoftDeletes;
+use Illuminate\Support\Collection;
 use Illuminate\Support\Facades\File;
 
 class WebhookConfiguration extends Model
@@ -40,12 +41,21 @@ class WebhookConfiguration extends Model
                 ...$webhookConfiguration->getOriginal('events', '[]'),
             ])->unique();
 
-            $changedEvents->each(function (string $event) {
-                cache()->forever("webhooks.$event", WebhookConfiguration::query()->whereJsonContains('events', $event)->get());
-            });
-
-            cache()->forever('watchedWebhooks', WebhookConfiguration::pluck('events')->flatten()->unique()->values()->all());
+            self::updateCache($changedEvents);
         });
+
+        self::deleted(static function (self $webhookConfiguration): void {
+            self::updateCache(collect((array) $webhookConfiguration->events));
+        });
+    }
+
+    private static function updateCache(Collection $eventList): void
+    {
+        $eventList->each(function (string $event) {
+            cache()->forever("webhooks.$event", WebhookConfiguration::query()->whereJsonContains('events', $event)->get());
+        });
+
+        cache()->forever('watchedWebhooks', WebhookConfiguration::pluck('events')->flatten()->unique()->values()->all());
     }
 
     public function webhooks(): HasMany
diff --git a/tests/Feature/DispatchWebhooksTest.php b/tests/Feature/Webhooks/DispatchWebhooksTest.php
similarity index 68%
rename from tests/Feature/DispatchWebhooksTest.php
rename to tests/Feature/Webhooks/DispatchWebhooksTest.php
index 5c032761c..7fcb1e326 100644
--- a/tests/Feature/DispatchWebhooksTest.php
+++ b/tests/Feature/Webhooks/DispatchWebhooksTest.php
@@ -1,6 +1,6 @@
 <?php
 
-namespace App\Tests\Feature;
+namespace App\Tests\Feature\Webhooks;
 
 use App\Jobs\ProcessWebhook;
 use App\Models\Server;
@@ -62,6 +62,32 @@ class DispatchWebhooksTest extends TestCase
         Queue::assertPushed(ProcessWebhook::class, 1);
     }
 
+    public function test_it_does_not_call_removed_events()
+    {
+        $webhookConfig = WebhookConfiguration::factory()->create([
+            'events' => ['eloquent.created: '.Server::class],
+        ]);
+
+        $webhookConfig->update(['events' => 'eloquent.deleted: '.Server::class]);
+
+        $this->createServer();
+
+        Queue::assertNothingPushed();
+    }
+
+    public function test_it_does_not_call_deleted_webhooks()
+    {
+        $webhookConfig = WebhookConfiguration::factory()->create([
+            'events' => ['eloquent.created: '.Server::class],
+        ]);
+
+        $webhookConfig->delete();
+
+        $this->createServer();
+
+        Queue::assertNothingPushed();
+    }
+
     public function createServer(): Server
     {
         return Server::factory()->withNode()->create();
diff --git a/tests/Feature/ProcessWebhooksTest.php b/tests/Feature/Webhooks/ProcessWebhooksTest.php
similarity index 99%
rename from tests/Feature/ProcessWebhooksTest.php
rename to tests/Feature/Webhooks/ProcessWebhooksTest.php
index 7eb7a8d9a..c15683d29 100644
--- a/tests/Feature/ProcessWebhooksTest.php
+++ b/tests/Feature/Webhooks/ProcessWebhooksTest.php
@@ -1,6 +1,6 @@
 <?php
 
-namespace App\Tests\Feature;
+namespace App\Tests\Feature\Webhooks;
 
 use App\Events\Server\Installed;
 use App\Jobs\ProcessWebhook;