feat: notification rate limiter

fix: limit server up / down notification limits

app/Console/Kernel.php

@@ -110,11 +110,11 @@ class Kernel extends ConsoleKernel
             $servers = $this->allServers->where('ip', '!=', '1.2.3.4');
         }
         foreach ($servers as $server) {
-            $last_sentinel_update = $server->sentinel_updated_at;
+            $lastSentinelUpdate = $server->sentinel_updated_at;
-            if (Carbon::parse($last_sentinel_update)->isBefore(now()->subSeconds($server->waitBeforeDoingSshCheck()))) {
+            $serverTimezone = $server->settings->server_timezone;
+            if (Carbon::parse($lastSentinelUpdate)->isBefore(now()->subSeconds($server->waitBeforeDoingSshCheck()))) {
                 $schedule->job(new ServerCheckJob($server))->everyMinute()->onOneServer();
             }
-            $serverTimezone = $server->settings->server_timezone;
             if ($server->settings->force_docker_cleanup) {
                 $schedule->job(new DockerCleanupJob($server))->cron($server->settings->docker_cleanup_frequency)->timezone($serverTimezone)->onOneServer();
             } else {

app/Jobs/PushServerUpdateJob.php

@@ -97,8 +97,6 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue
             }
             $data = collect($this->data);
 
-            $this->serverStatus();
-
             $this->server->sentinelHeartbeat();
 
             $this->containers = collect(data_get($data, 'containers'));
@@ -212,16 +210,6 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue
 
     }
 
-    private function serverStatus()
-    {
-        if ($this->server->isFunctional() === false) {
-            throw new \Exception('Server is not ready.');
-        }
-        if ($this->server->status() === false) {
-            throw new \Exception('Server is not reachable.');
-        }
-    }
-
     private function updateApplicationStatus(string $applicationId, string $containerStatus)
     {
         $application = $this->applications->where('id', $applicationId)->first();

app/Jobs/ServerCheckJob.php

@@ -43,22 +43,15 @@ class ServerCheckJob implements ShouldBeEncrypted, ShouldQueue
     public function handle()
     {
         try {
+            if ($this->server->serverStatus() === false) {
+                return 'Server is not reachable or not ready.';
+            }
+
             $this->applications = $this->server->applications();
             $this->databases = $this->server->databases();
             $this->services = $this->server->services()->get();
             $this->previews = $this->server->previews();
 
-            $up = $this->serverStatus();
-            if (! $up) {
-                ray('Server is not reachable.');
-
-                return 'Server is not reachable.';
-            }
-            if (! $this->server->isFunctional()) {
-                ray('Server is not ready.');
-
-                return 'Server is not ready.';
-            }
             if (! $this->server->isSwarmWorker() && ! $this->server->isBuildServer()) {
                 ['containers' => $this->containers, 'containerReplicates' => $containerReplicates] = $this->server->getContainers();
                 if (is_null($this->containers)) {
@@ -111,39 +104,6 @@ class ServerCheckJob implements ShouldBeEncrypted, ShouldQueue
 
     }
 
-    private function serverStatus()
-    {
-        ['uptime' => $uptime] = $this->server->validateConnection(false);
-        if ($uptime) {
-            if ($this->server->unreachable_notification_sent === true) {
-                $this->server->update(['unreachable_notification_sent' => false]);
-            }
-        } else {
-            // $this->server->team?->notify(new Unreachable($this->server));
-            foreach ($this->applications as $application) {
-                $application->update(['status' => 'exited']);
-            }
-            foreach ($this->databases as $database) {
-                $database->update(['status' => 'exited']);
-            }
-            foreach ($this->services as $service) {
-                $apps = $service->applications()->get();
-                $dbs = $service->databases()->get();
-                foreach ($apps as $app) {
-                    $app->update(['status' => 'exited']);
-                }
-                foreach ($dbs as $db) {
-                    $db->update(['status' => 'exited']);
-                }
-            }
-
-            return false;
-        }
-
-        return true;
-
-    }
-
     private function checkLogDrainContainer()
     {
         $foundLogDrainContainer = $this->containers->filter(function ($value, $key) {

app/Models/Server.php

@@ -6,6 +6,8 @@ use App\Actions\Server\InstallDocker;
 use App\Actions\Server\StartSentinel;
 use App\Enums\ProxyTypes;
 use App\Jobs\CheckAndStartSentinelJob;
+use App\Notifications\Server\Reachable;
+use App\Notifications\Server\Unreachable;
 use Illuminate\Database\Eloquent\Builder;
 use Illuminate\Database\Eloquent\Casts\Attribute;
 use Illuminate\Database\Eloquent\SoftDeletes;
@@ -61,6 +63,7 @@ class Server extends BaseModel
                 $payload['ip'] = str($server->ip)->trim();
             }
             $server->forceFill($payload);
+
         });
         static::created(function ($server) {
             ServerSetting::create([
@@ -107,12 +110,15 @@ class Server extends BaseModel
         });
     }
 
-    public $casts = [
+    protected $casts = [
         'proxy' => SchemalessAttributes::class,
         'logdrain_axiom_api_key' => 'encrypted',
         'logdrain_newrelic_license_key' => 'encrypted',
         'delete_unused_volumes' => 'boolean',
         'delete_unused_networks' => 'boolean',
+        'unreachable_notification_sent' => 'boolean',
+        'is_build_server' => 'boolean',
+        'force_disabled' => 'boolean',
     ];
 
     protected $schemalessAttributes = [
@@ -519,16 +525,14 @@ $schema://$host {
 
     public function forceEnableServer()
     {
-        $this->settings->update([
+        $this->settings->force_disabled = false;
-            'force_disabled' => false,
+        $this->settings->save();
-        ]);
     }
 
     public function forceDisableServer()
     {
-        $this->settings->update([
+        $this->settings->force_disabled = true;
-            'force_disabled' => true,
+        $this->settings->save();
-        ]);
         $sshKeyFileLocation = "id.root@{$this->uuid}";
         Storage::disk('ssh-keys')->delete($sshKeyFileLocation);
         Storage::disk('ssh-mux')->delete($this->muxFilename());
@@ -624,72 +628,6 @@ $schema://$host {
         }
     }
 
-    public function isServerReady(int $tries = 3)
-    {
-        if ($this->skipServer()) {
-            return false;
-        }
-        $serverUptimeCheckNumber = $this->unreachable_count;
-        if ($this->unreachable_count < $tries) {
-            $serverUptimeCheckNumber = $this->unreachable_count + 1;
-        }
-        if ($this->unreachable_count > $tries) {
-            $serverUptimeCheckNumber = $tries;
-        }
-
-        $serverUptimeCheckNumberMax = $tries;
-
-        // ray('server: ' . $this->name);
-        // ray('serverUptimeCheckNumber: ' . $serverUptimeCheckNumber);
-        // ray('serverUptimeCheckNumberMax: ' . $serverUptimeCheckNumberMax);
-
-        ['uptime' => $uptime] = $this->validateConnection();
-        if ($uptime) {
-            if ($this->unreachable_notification_sent === true) {
-                $this->update(['unreachable_notification_sent' => false]);
-            }
-
-            return true;
-        } else {
-            if ($serverUptimeCheckNumber >= $serverUptimeCheckNumberMax) {
-                // Reached max number of retries
-                if ($this->unreachable_notification_sent === false) {
-                    ray('Server unreachable, sending notification...');
-                    // $this->team?->notify(new Unreachable($this));
-                    $this->update(['unreachable_notification_sent' => true]);
-                }
-                if ($this->settings->is_reachable === true) {
-                    $this->settings()->update([
-                        'is_reachable' => false,
-                    ]);
-                }
-
-                foreach ($this->applications() as $application) {
-                    $application->update(['status' => 'exited']);
-                }
-                foreach ($this->databases() as $database) {
-                    $database->update(['status' => 'exited']);
-                }
-                foreach ($this->services()->get() as $service) {
-                    $apps = $service->applications()->get();
-                    $dbs = $service->databases()->get();
-                    foreach ($apps as $app) {
-                        $app->update(['status' => 'exited']);
-                    }
-                    foreach ($dbs as $db) {
-                        $db->update(['status' => 'exited']);
-                    }
-                }
-            } else {
-                $this->update([
-                    'unreachable_count' => $this->unreachable_count + 1,
-                ]);
-            }
-
-            return false;
-        }
-    }
-
     public function getDiskUsage(): ?string
     {
         return instant_remote_process(['df / --output=pcent | tr -cd 0-9'], $this, false);
@@ -1038,29 +976,43 @@ $schema://$host {
         return data_get($this, 'settings.is_swarm_worker');
     }
 
+    public function serverStatus(): bool
+    {
+        if ($this->status() === false) {
+            return false;
+        }
+        if ($this->isFunctional() === false) {
+            return false;
+        }
+
+        return true;
+    }
+
     public function status(): bool
     {
+        if ($this->skipServer()) {
+            return false;
+        }
         ['uptime' => $uptime] = $this->validateConnection(false);
-        if ($uptime) {
+        if ($uptime === false) {
-            if ($this->unreachable_notification_sent === true) {
+            foreach ($this->applications() as $application) {
-                $this->update(['unreachable_notification_sent' => false]);
+                $application->status = 'exited';
+                $application->save();
             }
-        } else {
+            foreach ($this->databases() as $database) {
-            // $this->server->team?->notify(new Unreachable($this->server));
+                $database->status = 'exited';
-            foreach ($this->applications as $application) {
+                $database->save();
-                $application->update(['status' => 'exited']);
             }
-            foreach ($this->databases as $database) {
+            foreach ($this->services() as $service) {
-                $database->update(['status' => 'exited']);
-            }
-            foreach ($this->services as $service) {
                 $apps = $service->applications()->get();
                 $dbs = $service->databases()->get();
                 foreach ($apps as $app) {
-                    $app->update(['status' => 'exited']);
+                    $app->status = 'exited';
+                    $app->save();
                 }
                 foreach ($dbs as $db) {
-                    $db->update(['status' => 'exited']);
+                    $db->status = 'exited';
+                    $db->save();
                 }
             }
 
@@ -1070,39 +1022,65 @@ $schema://$host {
         return true;
     }
 
+    public function isReachableChanged()
+    {
+        $this->refresh();
+        $unreachableNotificationSent = (bool) $this->unreachable_notification_sent;
+        $isReachable = (bool) $this->settings->is_reachable;
+        loggy('Server setting is_reachable changed to '.$isReachable.' for server '.$this->id.'. Unreachable notification sent: '.$unreachableNotificationSent);
+        // If the server is reachable, send the reachable notification if it was sent before
+        if ($isReachable === true) {
+            if ($unreachableNotificationSent === true) {
+                $this->sendReachableNotification();
+            }
+        } else {
+            // If the server is unreachable, send the unreachable notification if it was not sent before
+            if ($unreachableNotificationSent === false) {
+                $this->sendUnreachableNotification();
+            }
+        }
+    }
+
+    public function sendReachableNotification()
+    {
+        $this->unreachable_notification_sent = false;
+        $this->save();
+        $this->refresh();
+        $this->team->notify(new Reachable($this));
+    }
+
+    public function sendUnreachableNotification()
+    {
+        $this->unreachable_notification_sent = true;
+        $this->save();
+        $this->refresh();
+        $this->team->notify(new Unreachable($this));
+    }
+
     public function validateConnection($isManualCheck = true)
     {
         config()->set('constants.ssh.mux_enabled', ! $isManualCheck);
-        // ray('Manual Check: ' . ($isManualCheck ? 'true' : 'false'));
 
-        $server = Server::find($this->id);
+        if ($this->skipServer()) {
-        if (! $server) {
-            return ['uptime' => false, 'error' => 'Server not found.'];
-        }
-        if ($server->skipServer()) {
             return ['uptime' => false, 'error' => 'Server skipped.'];
         }
         try {
             // Make sure the private key is stored
-            if ($server->privateKey) {
+            if ($this->privateKey) {
-                $server->privateKey->storeInFileSystem();
+                $this->privateKey->storeInFileSystem();
             }
-            instant_remote_process(['ls /'], $server);
+            instant_remote_process(['ls /'], $this);
-            $server->settings()->update([
+            if ($this->settings->is_reachable === false) {
-                'is_reachable' => true,
+                $this->settings->is_reachable = true;
-            ]);
+                $this->settings->save();
-            $server->update([
-                'unreachable_count' => 0,
-            ]);
-            if (data_get($server, 'unreachable_notification_sent') === true) {
-                $server->update(['unreachable_notification_sent' => false]);
             }
 
             return ['uptime' => true, 'error' => null];
         } catch (\Throwable $e) {
-            $server->settings()->update([
+            if ($this->settings->is_reachable === true) {
-                'is_reachable' => false,
+                $this->settings->is_reachable = false;
-            ]);
+                $this->settings->save();
+            }
 
             return ['uptime' => false, 'error' => $e->getMessage()];
         }

app/Models/ServerSetting.php

@@ -54,6 +54,8 @@ class ServerSetting extends Model
         'force_docker_cleanup' => 'boolean',
         'docker_cleanup_threshold' => 'integer',
         'sentinel_token' => 'encrypted',
+        'is_reachable' => 'boolean',
+        'is_usable' => 'boolean',
     ];
 
     protected static function booted()
@@ -70,15 +72,18 @@ class ServerSetting extends Model
                 loggy('Error creating server setting: '.$e->getMessage());
             }
         });
-        static::updated(function ($setting) {
+        static::updated(function ($settings) {
             if (
-                $setting->isDirty('sentinel_token') ||
+                $settings->isDirty('sentinel_token') ||
-                $setting->isDirty('sentinel_custom_url') ||
+                $settings->isDirty('sentinel_custom_url') ||
-                $setting->isDirty('sentinel_metrics_refresh_rate_seconds') ||
+                $settings->isDirty('sentinel_metrics_refresh_rate_seconds') ||
-                $setting->isDirty('sentinel_metrics_history_days') ||
+                $settings->isDirty('sentinel_metrics_history_days') ||
-                $setting->isDirty('sentinel_push_interval_seconds')
+                $settings->isDirty('sentinel_push_interval_seconds')
             ) {
-                $setting->server->restartSentinel();
+                $settings->server->restartSentinel();
+            }
+            if ($settings->isDirty('is_reachable')) {
+                $settings->server->isReachableChanged();
             }
         });
     }

app/Notifications/Server/Reachable.php

@@ -2,8 +2,6 @@
 
 namespace App\Notifications\Server;
 
-use App\Actions\Docker\GetContainersStatus;
-use App\Jobs\ContainerStatusJob;
 use App\Models\Server;
 use App\Notifications\Channels\DiscordChannel;
 use App\Notifications\Channels\EmailChannel;
@@ -13,25 +11,28 @@ use Illuminate\Bus\Queueable;
 use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Notifications\Messages\MailMessage;
 use Illuminate\Notifications\Notification;
-use Illuminate\Support\Facades\RateLimiter;
 
-class Revived extends Notification implements ShouldQueue
+class Reachable extends Notification implements ShouldQueue
 {
     use Queueable;
 
     public $tries = 1;
 
+    protected bool $isRateLimited = false;
+
     public function __construct(public Server $server)
     {
-        if ($this->server->unreachable_notification_sent === false) {
+        $this->isRateLimited = isEmailRateLimited(
-            return;
+            limiterKey: 'server-reachable:'.$this->server->id,
-        }
+        );
-        GetContainersStatus::dispatch($server)->onQueue('high');
-        // dispatch(new ContainerStatusJob($server));
     }
 
     public function via(object $notifiable): array
     {
+        if ($this->isRateLimited) {
+            return [];
+        }
+
         $channels = [];
         $isEmailEnabled = isEmailEnabled($notifiable);
         $isDiscordEnabled = data_get($notifiable, 'discord_enabled');
@@ -46,20 +47,8 @@ class Revived extends Notification implements ShouldQueue
         if ($isTelegramEnabled) {
             $channels[] = TelegramChannel::class;
         }
-        $executed = RateLimiter::attempt(
-            'notification-server-revived-'.$this->server->uuid,
-            1,
-            function () use ($channels) {
-                return $channels;
-            },
-            7200,
-        );
 
-        if (! $executed) {
+        return $channels;
-            return [];
-        }
-
-        return $executed;
     }
 
     public function toMail(): MailMessage

app/Notifications/Server/Unreachable.php

@@ -11,7 +11,6 @@ use Illuminate\Bus\Queueable;
 use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Notifications\Messages\MailMessage;
 use Illuminate\Notifications\Notification;
-use Illuminate\Support\Facades\RateLimiter;
 
 class Unreachable extends Notification implements ShouldQueue
 {
@@ -19,10 +18,21 @@ class Unreachable extends Notification implements ShouldQueue
 
     public $tries = 1;
 
-    public function __construct(public Server $server) {}
+    protected bool $isRateLimited = false;
+
+    public function __construct(public Server $server)
+    {
+        $this->isRateLimited = isEmailRateLimited(
+            limiterKey: 'server-unreachable:'.$this->server->id,
+        );
+    }
 
     public function via(object $notifiable): array
     {
+        if ($this->isRateLimited) {
+            return [];
+        }
+
         $channels = [];
         $isEmailEnabled = isEmailEnabled($notifiable);
         $isDiscordEnabled = data_get($notifiable, 'discord_enabled');
@@ -37,23 +47,11 @@ class Unreachable extends Notification implements ShouldQueue
         if ($isTelegramEnabled) {
             $channels[] = TelegramChannel::class;
         }
-        $executed = RateLimiter::attempt(
-            'notification-server-unreachable-'.$this->server->uuid,
-            1,
-            function () use ($channels) {
-                return $channels;
-            },
-            7200,
-        );
 
-        if (! $executed) {
+        return $channels;
-            return [];
-        }
-
-        return $executed;
     }
 
-    public function toMail(): MailMessage
+    public function toMail(): ?MailMessage
     {
         $mail = new MailMessage;
         $mail->subject("Coolify: Your server ({$this->server->name}) is unreachable.");
@@ -64,7 +62,7 @@ class Unreachable extends Notification implements ShouldQueue
         return $mail;
     }
 
-    public function toDiscord(): DiscordMessage
+    public function toDiscord(): ?DiscordMessage
     {
         $message = new DiscordMessage(
             title: ':cross_mark: Server unreachable',
@@ -77,7 +75,7 @@ class Unreachable extends Notification implements ShouldQueue
         return $message;
     }
 
-    public function toTelegram(): array
+    public function toTelegram(): ?array
     {
         return [
             'message' => "Coolify: Your server '{$this->server->name}' is unreachable. All automations & integrations are turned off! Please check your server! IMPORTANT: We automatically try to revive your server and turn on all automations & integrations.",

bootstrap/helpers/shared.php

@@ -39,6 +39,7 @@ use Illuminate\Support\Facades\File;
 use Illuminate\Support\Facades\Http;
 use Illuminate\Support\Facades\Mail;
 use Illuminate\Support\Facades\Process;
+use Illuminate\Support\Facades\RateLimiter;
 use Illuminate\Support\Facades\Request;
 use Illuminate\Support\Facades\Route;
 use Illuminate\Support\Facades\Validator;
@@ -4038,3 +4039,30 @@ function sslipDomainWarning(string $domains)
 
     return $showSslipHttpsWarning;
 }
+
+function isEmailRateLimited(string $limiterKey, int $decaySeconds = 3600, ?callable $callbackOnSuccess = null): bool
+{
+    if (isDev()) {
+        $decaySeconds = 120;
+    }
+    $rateLimited = false;
+    $executed = RateLimiter::attempt(
+        $limiterKey,
+        $maxAttempts = 0,
+        function () use (&$rateLimited, &$limiterKey, $callbackOnSuccess) {
+            isDev() && loggy('Rate limit not reached for '.$limiterKey);
+            $rateLimited = false;
+
+            if ($callbackOnSuccess) {
+                $callbackOnSuccess();
+            }
+        },
+        $decaySeconds,
+    );
+    if (! $executed) {
+        isDev() && loggy('Rate limit reached for '.$limiterKey.'. Rate limiter will be disabled for '.$decaySeconds.' seconds.');
+        $rateLimited = true;
+    }
+
+    return $rateLimited;
+}

scripts/run

@@ -20,6 +20,9 @@ function help {
     compgen -A function | cat -n
 }
 
+function logs {
+    docker exec -t coolify tail -f storage/logs/laravel.log
+}
 function test {
     docker exec -t coolify php artisan test --testsuite=Feature
 }
@@ -35,11 +38,6 @@ function db:reset-prod {
     bash spin exec -u webuser coolify php artisan migrate:fresh --force --seed --seeder=ProductionSeeder ||
         php artisan migrate:fresh --force --seed --seeder=ProductionSeeder
 }
-
-function mfs {
-    db:reset
-}
-
 function coolify {
     bash spin exec -u webuser coolify bash
 }