feat: limit storage check emails

feat: sentinel should send storage usage

app/Actions/Docker/GetContainersStatus.php

@@ -3,8 +3,6 @@
 namespace App\Actions\Docker;
 
 use App\Actions\Database\StartDatabaseProxy;
-use App\Actions\Proxy\CheckProxy;
-use App\Actions\Proxy\StartProxy;
 use App\Actions\Shared\ComplexStatusCheck;
 use App\Models\ApplicationPreview;
 use App\Models\Server;
@@ -253,7 +251,7 @@ class GetContainersStatus
             $environmentName = data_get($service, 'environment.name');
 
             if ($projectUuid && $serviceUuid && $environmentName) {
-                $url = base_url() . '/project/' . $projectUuid . '/' . $environmentName . '/service/' . $serviceUuid;
+                $url = base_url().'/project/'.$projectUuid.'/'.$environmentName.'/service/'.$serviceUuid;
             } else {
                 $url = null;
             }
@@ -279,7 +277,7 @@ class GetContainersStatus
             $environment = data_get($application, 'environment.name');
 
             if ($projectUuid && $applicationUuid && $environment) {
-                $url = base_url() . '/project/' . $projectUuid . '/' . $environment . '/application/' . $applicationUuid;
+                $url = base_url().'/project/'.$projectUuid.'/'.$environment.'/application/'.$applicationUuid;
             } else {
                 $url = null;
             }
@@ -304,7 +302,7 @@ class GetContainersStatus
             $applicationUuid = data_get($preview, 'application.uuid');
 
             if ($projectUuid && $applicationUuid && $environmentName) {
-                $url = base_url() . '/project/' . $projectUuid . '/' . $environmentName . '/application/' . $applicationUuid;
+                $url = base_url().'/project/'.$projectUuid.'/'.$environmentName.'/application/'.$applicationUuid;
             } else {
                 $url = null;
             }
@@ -329,7 +327,7 @@ class GetContainersStatus
             $databaseUuid = data_get($database, 'uuid');
 
             if ($projectUuid && $databaseUuid && $environmentName) {
-                $url = base_url() . '/project/' . $projectUuid . '/' . $environmentName . '/database/' . $databaseUuid;
+                $url = base_url().'/project/'.$projectUuid.'/'.$environmentName.'/database/'.$databaseUuid;
             } else {
                 $url = null;
             }

app/Jobs/PushServerUpdateJob.php

@@ -101,6 +101,10 @@ class PushServerUpdateJob implements ShouldQueue
             $this->server->sentinelHeartbeat();
 
             $this->containers = collect(data_get($data, 'containers'));
+
+            $filesystemUsageRoot = data_get($data, 'filesystem_usage_root.used_percentage');
+            ServerStorageCheckJob::dispatch($this->server, $filesystemUsageRoot);
+
             if ($this->containers->isEmpty()) {
                 return;
             }

app/Jobs/ServerCheckJob.php

@@ -2,14 +2,11 @@
 
 namespace App\Jobs;
 
-use App\Actions\Database\StartDatabaseProxy;
 use App\Actions\Docker\GetContainersStatus;
 use App\Actions\Proxy\CheckProxy;
 use App\Actions\Proxy\StartProxy;
 use App\Actions\Server\InstallLogDrain;
-use App\Models\ApplicationPreview;
 use App\Models\Server;
-use App\Models\ServiceDatabase;
 use App\Notifications\Container\ContainerRestarted;
 use Illuminate\Bus\Queueable;
 use Illuminate\Contracts\Queue\ShouldBeEncrypted;
@@ -17,7 +14,6 @@ use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
-use Illuminate\Support\Arr;
 
 class ServerCheckJob implements ShouldBeEncrypted, ShouldQueue
 {
@@ -68,7 +64,9 @@ class ServerCheckJob implements ShouldBeEncrypted, ShouldQueue
                 if (is_null($this->containers)) {
                     return 'No containers found.';
                 }
+                ServerStorageCheckJob::dispatch($this->server);
                 GetContainersStatus::run($this->server, $this->containers, $containerReplicates);
+
                 if ($this->server->isLogDrainEnabled()) {
                     $this->checkLogDrainContainer();
                 }

app/Jobs/ServerStorageCheckJob.php

@@ -3,12 +3,14 @@
 namespace App\Jobs;
 
 use App\Models\Server;
+use App\Notifications\Server\HighDiskUsage;
 use Illuminate\Bus\Queueable;
 use Illuminate\Contracts\Queue\ShouldBeEncrypted;
 use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Facades\RateLimiter;
 
 class ServerStorageCheckJob implements ShouldBeEncrypted, ShouldQueue
 {
@@ -18,22 +20,12 @@ class ServerStorageCheckJob implements ShouldBeEncrypted, ShouldQueue
 
     public $timeout = 60;
 
-    public $containers;
-
-    public $applications;
-
-    public $databases;
-
-    public $services;
-
-    public $previews;
-
     public function backoff(): int
     {
         return isDev() ? 1 : 3;
     }
 
-    public function __construct(public Server $server) {}
+    public function __construct(public Server $server, public ?int $percentage = null) {}
 
     public function handle()
     {
@@ -43,15 +35,33 @@ class ServerStorageCheckJob implements ShouldBeEncrypted, ShouldQueue
 
                 return 'Server is not ready.';
             }
-            $team = $this->server->team;
+            $team = data_get($this->server, 'team');
-            $percentage = $this->server->storageCheck();
+            $serverDiskUsageNotificationThreshold = data_get($this->server, 'settings.server_disk_usage_notification_threshold');
-            if ($percentage > 1) {
+
-                ray('Server storage is at '.$percentage.'%');
+            if (is_null($this->percentage)) {
+                $this->percentage = $this->server->storageCheck();
+            }
+            if (! $this->percentage) {
+                throw new \Exception('No percentage could be retrieved.');
+            }
+            if ($this->percentage > $serverDiskUsageNotificationThreshold) {
+                $executed = RateLimiter::attempt(
+                    'high-disk-usage:'.$this->server->id,
+                    $maxAttempts = 0,
+                    function () use ($team, $serverDiskUsageNotificationThreshold) {
+                        $team->notify(new HighDiskUsage($this->server, $this->percentage, $serverDiskUsageNotificationThreshold));
+                    },
+                    $decaySeconds = 3600,
+                );
+
+                if (! $executed) {
+                    throw new \Exception('Too many messages sent!');
+                }
+            } else {
+                RateLimiter::hit('high-disk-usage:'.$this->server->id, 600);
             }
 
         } catch (\Throwable $e) {
-            ray($e->getMessage());
-
             return handleError($e);
         }
 

app/Livewire/Notifications/Email.php

@@ -4,6 +4,7 @@ namespace App\Livewire\Notifications;
 
 use App\Models\Team;
 use App\Notifications\Test;
+use Illuminate\Support\Facades\RateLimiter;
 use Livewire\Component;
 
 class Email extends Component
@@ -74,8 +75,23 @@ class Email extends Component
 
     public function sendTestNotification()
     {
+        try {
+            $executed = RateLimiter::attempt(
+                'test-email:'.$this->team->id,
+                $perMinute = 0,
+                function () {
                     $this->team?->notify(new Test($this->emails));
                     $this->dispatch('success', 'Test Email sent.');
+                },
+                $decaySeconds = 10,
+            );
+
+            if (! $executed) {
+                throw new \Exception('Too many messages sent!');
+            }
+        } catch (\Throwable $e) {
+            return handleError($e, $this);
+        }
     }
 
     public function instantSaveInstance()

app/Livewire/Server/Advanced.php

@@ -16,6 +16,7 @@ class Advanced extends Component
         'server.settings.force_docker_cleanup' => 'required|boolean',
         'server.settings.docker_cleanup_frequency' => 'required_if:server.settings.force_docker_cleanup,true|string',
         'server.settings.docker_cleanup_threshold' => 'required_if:server.settings.force_docker_cleanup,false|integer|min:1|max:100',
+        'server.settings.server_disk_usage_notification_threshold' => 'required|integer|min:50|max:100',
         'server.settings.delete_unused_volumes' => 'boolean',
         'server.settings.delete_unused_networks' => 'boolean',
     ];
@@ -27,6 +28,7 @@ class Advanced extends Component
         'server.settings.force_docker_cleanup' => 'Force Docker Cleanup',
         'server.settings.docker_cleanup_frequency' => 'Docker Cleanup Frequency',
         'server.settings.docker_cleanup_threshold' => 'Docker Cleanup Threshold',
+        'server.settings.server_disk_usage_notification_threshold' => 'Server Disk Usage Notification Threshold',
         'server.settings.delete_unused_volumes' => 'Delete Unused Volumes',
         'server.settings.delete_unused_networks' => 'Delete Unused Networks',
     ];

app/Livewire/Server/Form.php

@@ -246,11 +246,6 @@ class Form extends Component
             }
             refresh_server_connection($this->server->privateKey);
             $this->server->settings->wildcard_domain = $this->wildcard_domain;
-            // if ($this->server->settings->force_docker_cleanup) {
-            //     $this->server->settings->docker_cleanup_frequency = $this->server->settings->docker_cleanup_frequency;
-            // } else {
-            //     $this->server->settings->docker_cleanup_threshold = $this->server->settings->docker_cleanup_threshold;
-            // }
             $currentTimezone = $this->server->settings->getOriginal('server_timezone');
             $newTimezone = $this->server->settings->server_timezone;
             if ($currentTimezone !== $newTimezone || $currentTimezone === '') {

app/Livewire/SettingsEmail.php

@@ -3,7 +3,6 @@
 namespace App\Livewire;
 
 use App\Models\InstanceSettings;
-use App\Notifications\TransactionalEmails\Test;
 use Livewire\Component;
 
 class SettingsEmail extends Component
@@ -124,10 +123,4 @@ class SettingsEmail extends Component
             return handleError($e, $this);
         }
     }
-
-    public function sendTestNotification()
-    {
-        $this->settings?->notify(new Test($this->emails));
-        $this->dispatch('success', 'Test email sent.');
-    }
 }

app/Models/Server.php

@@ -700,7 +700,8 @@ $schema://$host {
 
     public function getDiskUsage(): ?string
     {
-        return instant_remote_process(["df /| tail -1 | awk '{ print $5}' | sed 's/%//g'"], $this, false);
+        return instant_remote_process(['df / --output=pcent | tr -cd 0-9'], $this, false);
+        // return instant_remote_process(["df /| tail -1 | awk '{ print $5}' | sed 's/%//g'"], $this, false);
     }
 
     public function definedResources()

app/Notifications/Server/HighDiskUsage.php

@@ -18,7 +18,7 @@ class HighDiskUsage extends Notification implements ShouldQueue
 
     public $tries = 1;
 
-    public function __construct(public Server $server, public int $disk_usage, public int $docker_cleanup_threshold) {}
+    public function __construct(public Server $server, public int $disk_usage, public int $server_disk_usage_notification_threshold) {}
 
     public function via(object $notifiable): array
     {
@@ -47,7 +47,7 @@ class HighDiskUsage extends Notification implements ShouldQueue
         $mail->view('emails.high-disk-usage', [
             'name' => $this->server->name,
             'disk_usage' => $this->disk_usage,
-            'threshold' => $this->docker_cleanup_threshold,
+            'threshold' => $this->server_disk_usage_notification_threshold,
         ]);
 
         return $mail;
@@ -62,7 +62,7 @@ class HighDiskUsage extends Notification implements ShouldQueue
         );
 
         $message->addField('Disk usage', "{$this->disk_usage}%");
-        $message->addField('Threshold', "{$this->docker_cleanup_threshold}%");
+        $message->addField('Threshold', "{$this->server_disk_usage_notification_threshold}%");
         $message->addField('Tips', '[Link](https://coolify.io/docs/knowledge-base/server/automated-cleanup)');
 
         return $message;
@@ -71,7 +71,7 @@ class HighDiskUsage extends Notification implements ShouldQueue
     public function toTelegram(): array
     {
         return [
-            'message' => "Coolify: Server '{$this->server->name}' high disk usage detected!\nDisk usage: {$this->disk_usage}%. Threshold: {$this->docker_cleanup_threshold}%.\nPlease cleanup your disk to prevent data-loss.\nHere are some tips: https://coolify.io/docs/knowledge-base/server/automated-cleanup.",
+            'message' => "Coolify: Server '{$this->server->name}' high disk usage detected!\nDisk usage: {$this->disk_usage}%. Threshold: {$this->server_disk_usage_notification_threshold}%.\nPlease cleanup your disk to prevent data-loss.\nHere are some tips: https://coolify.io/docs/knowledge-base/server/automated-cleanup.",
         ];
     }
 }

app/Notifications/Test.php

@@ -7,6 +7,7 @@ use Illuminate\Bus\Queueable;
 use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Notifications\Messages\MailMessage;
 use Illuminate\Notifications\Notification;
+use Illuminate\Queue\Middleware\RateLimited;
 
 class Test extends Notification implements ShouldQueue
 {
@@ -21,6 +22,14 @@ class Test extends Notification implements ShouldQueue
         return setNotificationChannels($notifiable, 'test');
     }
 
+    public function middleware(object $notifiable, string $channel)
+    {
+        return match ($channel) {
+            'App\Notifications\Channels\EmailChannel' => [new RateLimited('email')],
+            default => [],
+        };
+    }
+
     public function toMail(): MailMessage
     {
         $mail = new MailMessage;

database/migrations/2024_10_22_105745_add_server_disk_usage_threshold.php

@@ -0,0 +1,28 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+return new class extends Migration
+{
+    /**
+     * Run the migrations.
+     */
+    public function up(): void
+    {
+        Schema::table('server_settings', function (Blueprint $table) {
+            $table->integer('server_disk_usage_notification_threshold')->default(80)->after('docker_cleanup_threshold');
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     */
+    public function down(): void
+    {
+        Schema::table('server_settings', function (Blueprint $table) {
+            $table->dropColumn('server_disk_usage_notification_threshold');
+        });
+    }
+};

docker/prod/Dockerfile

@@ -1,4 +1,4 @@
-FROM serversideup/php:8.2-fpm-nginx-v2.2.1 as base
+FROM serversideup/php:8.2-fpm-nginx-v2.2.1 AS base
 WORKDIR /var/www/html
 
 COPY composer.json composer.lock ./

resources/views/livewire/server/advanced.blade.php

@@ -16,11 +16,18 @@
         </div>
         <div>Advanced configuration for your server.</div>
     </div>
-    <div class="flex flex-col gap-4 pt-4">
+
+    <div class="flex flex-col gap-4">
+        <div class="flex flex-col">
+            <div class="flex flex-wrap gap-2 sm:flex-nowrap pt-4">
+                <x-forms.input id="server.settings.server_disk_usage_notification_threshold"
+                    label="Server disk usage notification threshold (%)" required
+                    helper="If the server disk usage exceeds this threshold, Coolify will send a notification to the team members." />
+            </div>
+        </div>
         <div class="flex flex-col gap-2">
             <div class="flex items-center gap-2">
                 <h3>Docker Cleanup</h3>
-
             </div>
             <div class="flex flex-wrap items-center gap-4">
                 @if ($server->settings->force_docker_cleanup)
@@ -70,6 +77,7 @@
             </ul>" />
             </div>
         </div>
+
         <div class="flex flex-col">
             <h3>Builds</h3>
             <div>Customize the build process.</div>