Update server status check and notifications

app/Console/Kernel.php

@@ -9,6 +9,7 @@ use App\Jobs\DockerCleanupJob;
 use App\Jobs\InstanceAutoUpdateJob;
 use App\Jobs\ContainerStatusJob;
 use App\Jobs\PullHelperImageJob;
+use App\Jobs\ServerStatusJob;
 use App\Models\InstanceSettings;
 use App\Models\ScheduledDatabaseBackup;
 use App\Models\Server;
@@ -67,6 +68,7 @@ class Kernel extends ConsoleKernel
         }
         foreach ($servers as $server) {
             $schedule->job(new ContainerStatusJob($server))->everyMinute()->onOneServer();
+            $schedule->job(new ServerStatusJob($server))->everyFiveMinutes()->onOneServer();
         }
     }
     private function instance_auto_update($schedule)

app/Jobs/ContainerStatusJob.php

@@ -41,76 +41,7 @@ class ContainerStatusJob implements ShouldQueue, ShouldBeEncrypted
     {
         // ray("checking server status for {$this->server->id}");
         try {
-            // ray()->clearAll();
+            $this->server->checkServerRediness();
-            $serverUptimeCheckNumber = $this->server->unreachable_count;
-            $serverUptimeCheckNumberMax = 3;
-
-            // ray('checking # ' . $serverUptimeCheckNumber);
-            if ($serverUptimeCheckNumber >= $serverUptimeCheckNumberMax) {
-                if ($this->server->unreachable_email_sent === false) {
-                    ray('Server unreachable, sending notification...');
-                    $this->server->team->notify(new Unreachable($this->server));
-                    $this->server->update(['unreachable_email_sent' => true]);
-                }
-                $this->server->settings()->update([
-                    'is_reachable' => false,
-                ]);
-                $this->server->update([
-                    'unreachable_count' => 0,
-                ]);
-                // Update all applications, databases and services to exited
-                foreach ($this->server->applications() as $application) {
-                    $application->update(['status' => 'exited']);
-                }
-                foreach ($this->server->databases() as $database) {
-                    $database->update(['status' => 'exited']);
-                }
-                foreach ($this->server->services() as $service) {
-                    $apps = $service->applications()->get();
-                    $dbs = $service->databases()->get();
-                    foreach ($apps as $app) {
-                        $app->update(['status' => 'exited']);
-                    }
-                    foreach ($dbs as $db) {
-                        $db->update(['status' => 'exited']);
-                    }
-                }
-                return;
-            }
-            $result = $this->server->validateConnection();
-            if ($result) {
-                $this->server->settings()->update([
-                    'is_reachable' => true,
-                ]);
-                $this->server->update([
-                    'unreachable_count' => 0,
-                ]);
-            } else {
-                $serverUptimeCheckNumber++;
-                $this->server->settings()->update([
-                    'is_reachable' => false,
-                ]);
-                $this->server->update([
-                    'unreachable_count' => $serverUptimeCheckNumber,
-                ]);
-                return;
-            }
-
-            if (data_get($this->server, 'unreachable_email_sent') === true) {
-                ray('Server is reachable again, sending notification...');
-                $this->server->team->notify(new Revived($this->server));
-                $this->server->update(['unreachable_email_sent' => false]);
-            }
-            if (
-                data_get($this->server, 'settings.is_reachable') === false ||
-                data_get($this->server, 'settings.is_usable') === false
-            ) {
-                $this->server->settings()->update([
-                    'is_reachable' => true,
-                    'is_usable' => true
-                ]);
-            }
-            // $this->server->validateDockerEngine(true);
             $containers = instant_remote_process(["docker container ls -q"], $this->server);
             if (!$containers) {
                 return;

app/Jobs/DockerCleanupJob.php

@@ -3,6 +3,7 @@
 namespace App\Jobs;
 
 use App\Models\Server;
+use App\Notifications\Server\HighDiskUsage;
 use Exception;
 use Illuminate\Bus\Queueable;
 use Illuminate\Contracts\Queue\ShouldBeEncrypted;
@@ -18,7 +19,6 @@ class DockerCleanupJob implements ShouldQueue, ShouldBeEncrypted
     use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
 
     public $timeout = 300;
-    public ?string $dockerRootFilesystem = null;
     public ?int $usageBefore = null;
 
     public function __construct(public Server $server)
@@ -26,6 +26,7 @@ class DockerCleanupJob implements ShouldQueue, ShouldBeEncrypted
     }
     public function handle(): void
     {
+        try {
             $isInprogress = false;
             $this->server->applications()->each(function ($application) use (&$isInprogress) {
                 if ($application->isDeploymentInprogress()) {
@@ -36,18 +37,16 @@ class DockerCleanupJob implements ShouldQueue, ShouldBeEncrypted
             if ($isInprogress) {
                 throw new Exception('DockerCleanupJob: ApplicationDeploymentQueue is not empty, skipping...');
             }
-        try {
             if (!$this->server->isFunctional()) {
                 return;
             }
-            $this->dockerRootFilesystem = "/";
+
-            $this->usageBefore = $this->getFilesystemUsage();
             if ($this->usageBefore >= $this->server->settings->cleanup_after_percentage) {
                 ray('Cleaning up ' . $this->server->name);
                 instant_remote_process(['docker image prune -af'], $this->server);
                 instant_remote_process(['docker container prune -f --filter "label=coolify.managed=true"'], $this->server);
                 instant_remote_process(['docker builder prune -af'], $this->server);
-                $usageAfter = $this->getFilesystemUsage();
+                $usageAfter = $this->server->getDiskUsage();
                 if ($usageAfter <  $this->usageBefore) {
                     ray('Saved ' . ($this->usageBefore - $usageAfter) . '% disk space on ' . $this->server->name);
                     send_internal_notification('DockerCleanupJob done: Saved ' . ($this->usageBefore - $usageAfter) . '% disk space on ' . $this->server->name);
@@ -65,9 +64,4 @@ class DockerCleanupJob implements ShouldQueue, ShouldBeEncrypted
             throw $e;
         }
     }
-
-    private function getFilesystemUsage()
-    {
-        return instant_remote_process(["df '{$this->dockerRootFilesystem}'| tail -1 | awk '{ print $5}' | sed 's/%//g'"], $this->server, false);
-    }
 }

app/Jobs/ServerStatusJob.php

@@ -0,0 +1,54 @@
+<?php
+
+namespace App\Jobs;
+
+use App\Models\Server;
+use App\Notifications\Server\HighDiskUsage;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeEncrypted;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Illuminate\Queue\SerializesModels;
+
+class ServerStatusJob implements ShouldQueue, ShouldBeEncrypted
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    public function __construct(public Server $server)
+    {
+    }
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping($this->server->id))->dontRelease()];
+    }
+
+    public function uniqueId(): int
+    {
+        return $this->server->id;
+    }
+
+    public function handle(): void
+    {
+        ray("checking server status for {$this->server->id}");
+        try {
+
+            $this->server->checkServerRediness();
+            $disk_usage = $this->server->getDiskUsage();
+            ray($this->server->settings->cleanup_after_percentage);
+            if ($disk_usage >= $this->server->settings->cleanup_after_percentage) {
+                $this->server->high_disk_usage_notification_sent = true;
+                $this->server->save();
+                $this->server->team->notify(new HighDiskUsage($this->server, $disk_usage, $this->server->settings->cleanup_after_percentage));
+            } else {
+                $this->server->high_disk_usage_notification_sent = false;
+                $this->server->save();
+            }
+        } catch (\Throwable $e) {
+            send_internal_notification('ServerStatusJob failed with: ' . $e->getMessage());
+            ray($e->getMessage());
+            handleError($e);
+        }
+    }
+}

app/Models/Server.php

@@ -4,8 +4,11 @@ namespace App\Models;
 
 use App\Enums\ProxyStatus;
 use App\Enums\ProxyTypes;
+use App\Notifications\Server\Revived;
+use App\Notifications\Server\Unreachable;
 use Illuminate\Database\Eloquent\Builder;
 use Illuminate\Database\Eloquent\Casts\Attribute;
+use Illuminate\Support\Sleep;
 use Spatie\SchemalessAttributes\Casts\SchemalessAttributes;
 use Spatie\SchemalessAttributes\SchemalessAttributesTrait;
 use Illuminate\Support\Str;
@@ -109,6 +112,75 @@ class Server extends BaseModel
         return $this->proxy->modelScope();
     }
 
+    public function checkServerRediness()
+    {
+        $serverUptimeCheckNumber = $this->unreachable_count;
+        $serverUptimeCheckNumberMax = 5;
+        while (true) {
+            if ($serverUptimeCheckNumber >= $serverUptimeCheckNumberMax) {
+                if ($this->unreachable_notification_sent === false) {
+                    ray('Server unreachable, sending notification...');
+                    $this->team->notify(new Unreachable($this));
+                    $this->update(['unreachable_notification_sent' => true]);
+                }
+                $this->settings()->update([
+                    'is_reachable' => false,
+                ]);
+                $this->update([
+                    'unreachable_count' => 0,
+                ]);
+                foreach ($this->applications() as $application) {
+                    $application->update(['status' => 'exited']);
+                }
+                foreach ($this->databases() as $database) {
+                    $database->update(['status' => 'exited']);
+                }
+                foreach ($this->services() as $service) {
+                    $apps = $service->applications()->get();
+                    $dbs = $service->databases()->get();
+                    foreach ($apps as $app) {
+                        $app->update(['status' => 'exited']);
+                    }
+                    foreach ($dbs as $db) {
+                        $db->update(['status' => 'exited']);
+                    }
+                }
+                throw new \Exception('Server is not reachable.');
+            }
+            $result = $this->validateConnection();
+            ray('validateConnection: ' . $result);
+            if (!$result) {
+                $serverUptimeCheckNumber++;
+                $this->update([
+                    'unreachable_count' => $serverUptimeCheckNumber,
+                ]);
+                Sleep::for(5)->seconds();
+                return;
+            }
+            $this->update([
+                'unreachable_count' => 0,
+            ]);
+            if (data_get($this, 'unreachable_notification_sent') === true) {
+                ray('Server is reachable again, sending notification...');
+                $this->team->notify(new Revived($this));
+                $this->update(['unreachable_notification_sent' => false]);
+            }
+            if (
+                data_get($this, 'settings.is_reachable') === false ||
+                data_get($this, 'settings.is_usable') === false
+            ) {
+                $this->settings()->update([
+                    'is_reachable' => true,
+                    'is_usable' => true
+                ]);
+            }
+            break;
+        }
+    }
+    public function getDiskUsage()
+    {
+        return instant_remote_process(["df /| tail -1 | awk '{ print $5}' | sed 's/%//g'"], $this, false);
+    }
     public function hasDefinedResources()
     {
         $applications = $this->applications()->count() > 0;

app/Notifications/Server/HighDiskUsage.php

@@ -0,0 +1,68 @@
+<?php
+
+namespace App\Notifications\Server;
+
+use App\Models\Server;
+use Illuminate\Bus\Queueable;
+use App\Notifications\Channels\DiscordChannel;
+use App\Notifications\Channels\EmailChannel;
+use App\Notifications\Channels\TelegramChannel;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Notifications\Messages\MailMessage;
+use Illuminate\Notifications\Notification;
+
+class HighDiskUsage extends Notification implements ShouldQueue
+{
+    use Queueable;
+
+    public $tries = 1;
+    public function __construct(public Server $server, public int $disk_usage, public int $cleanup_after_percentage)
+    {
+        if ($this->server->high_disk_usage_notification_sent === false) {
+            return;
+        }
+    }
+
+    public function via(object $notifiable): array
+    {
+        $channels = [];
+        $isEmailEnabled = isEmailEnabled($notifiable);
+        $isDiscordEnabled = data_get($notifiable, 'discord_enabled');
+        $isTelegramEnabled = data_get($notifiable, 'telegram_enabled');
+
+        if ($isDiscordEnabled) {
+            $channels[] = DiscordChannel::class;
+        }
+        if ($isEmailEnabled) {
+            $channels[] = EmailChannel::class;
+        }
+        if ($isTelegramEnabled) {
+            $channels[] = TelegramChannel::class;
+        }
+        return $channels;
+    }
+
+    public function toMail(): MailMessage
+    {
+        $mail = new MailMessage();
+        $mail->subject("Coolify: Server ({$this->server->name}) high disk usage detected!");
+        $mail->view('emails.high-disk-usage', [
+            'name' => $this->server->name,
+            'disk_usage' => $this->disk_usage,
+            'threshold' => $this->cleanup_after_percentage,
+        ]);
+        return $mail;
+    }
+
+    public function toDiscord(): string
+    {
+        $message = "Coolify: Server '{$this->server->name}' high disk usage detected! \nDisk usage: {$this->disk_usage}";
+        return $message;
+    }
+    public function toTelegram(): array
+    {
+        return [
+            "message" => "Coolify: Server '{$this->server->name}' high disk usage detected! \n Disk usage: {$this->disk_usage}"
+        ];
+    }
+}

app/Notifications/Server/Revived.php

@@ -18,7 +18,7 @@ class Revived extends Notification implements ShouldQueue
     public $tries = 1;
     public function __construct(public Server $server)
     {
-        if ($this->server->unreachable_email_sent === false) {
+        if ($this->server->unreachable_notification_sent === false) {
             return;
         }
     }

bootstrap/helpers/remoteProcess.php

@@ -191,7 +191,7 @@ function refresh_server_connection(?PrivateKey $private_key = null)
 //         if (!$uptime) {
 //             $server->settings->is_reachable = false;
 //             $server->team->notify(new Unreachable($server));
-//             $server->unreachable_email_sent = true;
+//             $server->unreachable_notification_sent = true;
 //             $server->save();
 //             return [
 //                 "uptime" => null,
@@ -213,9 +213,9 @@ function refresh_server_connection(?PrivateKey $private_key = null)
 //             $server->settings->is_usable = false;
 //         } else {
 //             $server->settings->is_usable = true;
-//             if (data_get($server, 'unreachable_email_sent') === true) {
+//             if (data_get($server, 'unreachable_notification_sent') === true) {
 //                 $server->team->notify(new Revived($server));
-//                 $server->unreachable_email_sent = false;
+//                 $server->unreachable_notification_sent = false;
 //                 $server->save();
 //             }
 //         }

config/sentry.php

@@ -7,7 +7,7 @@ return [
 
     // The release version of your application
     // Example with dynamic git hash: trim(exec('git --git-dir ' . base_path('.git') . ' log --pretty="%h" -n1 HEAD'))
-    'release' => '4.0.0-beta.136',
+    'release' => '4.0.0-beta.137',
     // When left empty or `null` the Laravel environment will be used
     'environment' => config('app.env'),
 

config/version.php

@@ -1,3 +1,3 @@
 <?php
 
-return '4.0.0-beta.136';
+return '4.0.0-beta.137';

database/migrations/2023_11_16_101819_add_high_disk_usage_notification.php

@@ -0,0 +1,30 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+return new class extends Migration
+{
+    /**
+     * Run the migrations.
+     */
+    public function up(): void
+    {
+        Schema::table('servers', function (Blueprint $table) {
+            $table->boolean('high_disk_usage_notification_sent')->default(false);
+            $table->renameColumn('unreachable_email_sent', 'unreachable_notification_sent');
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     */
+    public function down(): void
+    {
+        Schema::table('servers', function (Blueprint $table) {
+            $table->dropColumn('high_disk_usage_notification_sent');
+            $table->renameColumn('unreachable_notification_sent', 'unreachable_email_sent');
+        });
+    }
+};

resources/views/emails/high-disk-usage.blade.php

@@ -0,0 +1,7 @@
+<x-emails.layout>
+
+Your server ({{ $name }}) has high disk usage ({{ $disk_usage }}%).
+
+Threshold is {{ $threshold }}% (you can change it in the Server Settings menu).
+
+</x-emails.layout>

versions.json

@@ -4,7 +4,7 @@
             "version": "3.12.36"
         },
         "v4": {
-            "version": "4.0.0-beta.136"
+            "version": "4.0.0-beta.137"
         }
     }
 }