From ac768e53130cf2833bd1b506ecd02238d44c2b8c Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Tue, 22 Oct 2024 14:01:36 +0200 Subject: [PATCH] feat: limit storage check emails feat: sentinel should send storage usage --- app/Actions/Docker/GetContainersStatus.php | 10 ++--- app/Jobs/PushServerUpdateJob.php | 4 ++ app/Jobs/ServerCheckJob.php | 6 +-- app/Jobs/ServerStorageCheckJob.php | 44 ++++++++++++------- app/Livewire/Notifications/Email.php | 20 ++++++++- app/Livewire/Server/Advanced.php | 2 + app/Livewire/Server/Form.php | 5 --- app/Livewire/SettingsEmail.php | 7 --- app/Models/Server.php | 3 +- app/Notifications/Server/HighDiskUsage.php | 8 ++-- app/Notifications/Test.php | 9 ++++ ...105745_add_server_disk_usage_threshold.php | 28 ++++++++++++ docker/prod/Dockerfile | 2 +- .../views/livewire/server/advanced.blade.php | 12 ++++- 14 files changed, 111 insertions(+), 49 deletions(-) create mode 100644 database/migrations/2024_10_22_105745_add_server_disk_usage_threshold.php diff --git a/app/Actions/Docker/GetContainersStatus.php b/app/Actions/Docker/GetContainersStatus.php index ed72701be..1cabc6418 100644 --- a/app/Actions/Docker/GetContainersStatus.php +++ b/app/Actions/Docker/GetContainersStatus.php @@ -3,8 +3,6 @@ namespace App\Actions\Docker; use App\Actions\Database\StartDatabaseProxy; -use App\Actions\Proxy\CheckProxy; -use App\Actions\Proxy\StartProxy; use App\Actions\Shared\ComplexStatusCheck; use App\Models\ApplicationPreview; use App\Models\Server; @@ -253,7 +251,7 @@ class GetContainersStatus $environmentName = data_get($service, 'environment.name'); if ($projectUuid && $serviceUuid && $environmentName) { - $url = base_url() . '/project/' . $projectUuid . '/' . $environmentName . '/service/' . $serviceUuid; + $url = base_url().'/project/'.$projectUuid.'/'.$environmentName.'/service/'.$serviceUuid; } else { $url = null; } @@ -279,7 +277,7 @@ class GetContainersStatus $environment = data_get($application, 'environment.name'); if ($projectUuid && $applicationUuid && $environment) { - $url = base_url() . '/project/' . $projectUuid . '/' . $environment . '/application/' . $applicationUuid; + $url = base_url().'/project/'.$projectUuid.'/'.$environment.'/application/'.$applicationUuid; } else { $url = null; } @@ -304,7 +302,7 @@ class GetContainersStatus $applicationUuid = data_get($preview, 'application.uuid'); if ($projectUuid && $applicationUuid && $environmentName) { - $url = base_url() . '/project/' . $projectUuid . '/' . $environmentName . '/application/' . $applicationUuid; + $url = base_url().'/project/'.$projectUuid.'/'.$environmentName.'/application/'.$applicationUuid; } else { $url = null; } @@ -329,7 +327,7 @@ class GetContainersStatus $databaseUuid = data_get($database, 'uuid'); if ($projectUuid && $databaseUuid && $environmentName) { - $url = base_url() . '/project/' . $projectUuid . '/' . $environmentName . '/database/' . $databaseUuid; + $url = base_url().'/project/'.$projectUuid.'/'.$environmentName.'/database/'.$databaseUuid; } else { $url = null; } diff --git a/app/Jobs/PushServerUpdateJob.php b/app/Jobs/PushServerUpdateJob.php index 9a3905928..549f1b09f 100644 --- a/app/Jobs/PushServerUpdateJob.php +++ b/app/Jobs/PushServerUpdateJob.php @@ -101,6 +101,10 @@ class PushServerUpdateJob implements ShouldQueue $this->server->sentinelHeartbeat(); $this->containers = collect(data_get($data, 'containers')); + + $filesystemUsageRoot = data_get($data, 'filesystem_usage_root.used_percentage'); + ServerStorageCheckJob::dispatch($this->server, $filesystemUsageRoot); + if ($this->containers->isEmpty()) { return; } diff --git a/app/Jobs/ServerCheckJob.php b/app/Jobs/ServerCheckJob.php index 596847958..0013ab784 100644 --- a/app/Jobs/ServerCheckJob.php +++ b/app/Jobs/ServerCheckJob.php @@ -2,14 +2,11 @@ namespace App\Jobs; -use App\Actions\Database\StartDatabaseProxy; use App\Actions\Docker\GetContainersStatus; use App\Actions\Proxy\CheckProxy; use App\Actions\Proxy\StartProxy; use App\Actions\Server\InstallLogDrain; -use App\Models\ApplicationPreview; use App\Models\Server; -use App\Models\ServiceDatabase; use App\Notifications\Container\ContainerRestarted; use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldBeEncrypted; @@ -17,7 +14,6 @@ use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; -use Illuminate\Support\Arr; class ServerCheckJob implements ShouldBeEncrypted, ShouldQueue { @@ -68,7 +64,9 @@ class ServerCheckJob implements ShouldBeEncrypted, ShouldQueue if (is_null($this->containers)) { return 'No containers found.'; } + ServerStorageCheckJob::dispatch($this->server); GetContainersStatus::run($this->server, $this->containers, $containerReplicates); + if ($this->server->isLogDrainEnabled()) { $this->checkLogDrainContainer(); } diff --git a/app/Jobs/ServerStorageCheckJob.php b/app/Jobs/ServerStorageCheckJob.php index 376cb8532..1330b7cde 100644 --- a/app/Jobs/ServerStorageCheckJob.php +++ b/app/Jobs/ServerStorageCheckJob.php @@ -3,12 +3,14 @@ namespace App\Jobs; use App\Models\Server; +use App\Notifications\Server\HighDiskUsage; use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldBeEncrypted; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; +use Illuminate\Support\Facades\RateLimiter; class ServerStorageCheckJob implements ShouldBeEncrypted, ShouldQueue { @@ -18,22 +20,12 @@ class ServerStorageCheckJob implements ShouldBeEncrypted, ShouldQueue public $timeout = 60; - public $containers; - - public $applications; - - public $databases; - - public $services; - - public $previews; - public function backoff(): int { return isDev() ? 1 : 3; } - public function __construct(public Server $server) {} + public function __construct(public Server $server, public ?int $percentage = null) {} public function handle() { @@ -43,15 +35,33 @@ class ServerStorageCheckJob implements ShouldBeEncrypted, ShouldQueue return 'Server is not ready.'; } - $team = $this->server->team; - $percentage = $this->server->storageCheck(); - if ($percentage > 1) { - ray('Server storage is at '.$percentage.'%'); + $team = data_get($this->server, 'team'); + $serverDiskUsageNotificationThreshold = data_get($this->server, 'settings.server_disk_usage_notification_threshold'); + + if (is_null($this->percentage)) { + $this->percentage = $this->server->storageCheck(); + } + if (! $this->percentage) { + throw new \Exception('No percentage could be retrieved.'); + } + if ($this->percentage > $serverDiskUsageNotificationThreshold) { + $executed = RateLimiter::attempt( + 'high-disk-usage:'.$this->server->id, + $maxAttempts = 0, + function () use ($team, $serverDiskUsageNotificationThreshold) { + $team->notify(new HighDiskUsage($this->server, $this->percentage, $serverDiskUsageNotificationThreshold)); + }, + $decaySeconds = 3600, + ); + + if (! $executed) { + throw new \Exception('Too many messages sent!'); + } + } else { + RateLimiter::hit('high-disk-usage:'.$this->server->id, 600); } } catch (\Throwable $e) { - ray($e->getMessage()); - return handleError($e); } diff --git a/app/Livewire/Notifications/Email.php b/app/Livewire/Notifications/Email.php index 53673292e..acb1d69a2 100644 --- a/app/Livewire/Notifications/Email.php +++ b/app/Livewire/Notifications/Email.php @@ -4,6 +4,7 @@ namespace App\Livewire\Notifications; use App\Models\Team; use App\Notifications\Test; +use Illuminate\Support\Facades\RateLimiter; use Livewire\Component; class Email extends Component @@ -74,8 +75,23 @@ class Email extends Component public function sendTestNotification() { - $this->team?->notify(new Test($this->emails)); - $this->dispatch('success', 'Test Email sent.'); + try { + $executed = RateLimiter::attempt( + 'test-email:'.$this->team->id, + $perMinute = 0, + function () { + $this->team?->notify(new Test($this->emails)); + $this->dispatch('success', 'Test Email sent.'); + }, + $decaySeconds = 10, + ); + + if (! $executed) { + throw new \Exception('Too many messages sent!'); + } + } catch (\Throwable $e) { + return handleError($e, $this); + } } public function instantSaveInstance() diff --git a/app/Livewire/Server/Advanced.php b/app/Livewire/Server/Advanced.php index b8003803a..4d2d777d4 100644 --- a/app/Livewire/Server/Advanced.php +++ b/app/Livewire/Server/Advanced.php @@ -16,6 +16,7 @@ class Advanced extends Component 'server.settings.force_docker_cleanup' => 'required|boolean', 'server.settings.docker_cleanup_frequency' => 'required_if:server.settings.force_docker_cleanup,true|string', 'server.settings.docker_cleanup_threshold' => 'required_if:server.settings.force_docker_cleanup,false|integer|min:1|max:100', + 'server.settings.server_disk_usage_notification_threshold' => 'required|integer|min:50|max:100', 'server.settings.delete_unused_volumes' => 'boolean', 'server.settings.delete_unused_networks' => 'boolean', ]; @@ -27,6 +28,7 @@ class Advanced extends Component 'server.settings.force_docker_cleanup' => 'Force Docker Cleanup', 'server.settings.docker_cleanup_frequency' => 'Docker Cleanup Frequency', 'server.settings.docker_cleanup_threshold' => 'Docker Cleanup Threshold', + 'server.settings.server_disk_usage_notification_threshold' => 'Server Disk Usage Notification Threshold', 'server.settings.delete_unused_volumes' => 'Delete Unused Volumes', 'server.settings.delete_unused_networks' => 'Delete Unused Networks', ]; diff --git a/app/Livewire/Server/Form.php b/app/Livewire/Server/Form.php index 5687bbf39..289531fcb 100644 --- a/app/Livewire/Server/Form.php +++ b/app/Livewire/Server/Form.php @@ -246,11 +246,6 @@ class Form extends Component } refresh_server_connection($this->server->privateKey); $this->server->settings->wildcard_domain = $this->wildcard_domain; - // if ($this->server->settings->force_docker_cleanup) { - // $this->server->settings->docker_cleanup_frequency = $this->server->settings->docker_cleanup_frequency; - // } else { - // $this->server->settings->docker_cleanup_threshold = $this->server->settings->docker_cleanup_threshold; - // } $currentTimezone = $this->server->settings->getOriginal('server_timezone'); $newTimezone = $this->server->settings->server_timezone; if ($currentTimezone !== $newTimezone || $currentTimezone === '') { diff --git a/app/Livewire/SettingsEmail.php b/app/Livewire/SettingsEmail.php index 4515df9a7..2b515bf68 100644 --- a/app/Livewire/SettingsEmail.php +++ b/app/Livewire/SettingsEmail.php @@ -3,7 +3,6 @@ namespace App\Livewire; use App\Models\InstanceSettings; -use App\Notifications\TransactionalEmails\Test; use Livewire\Component; class SettingsEmail extends Component @@ -124,10 +123,4 @@ class SettingsEmail extends Component return handleError($e, $this); } } - - public function sendTestNotification() - { - $this->settings?->notify(new Test($this->emails)); - $this->dispatch('success', 'Test email sent.'); - } } diff --git a/app/Models/Server.php b/app/Models/Server.php index c3f8347ba..bc93676b6 100644 --- a/app/Models/Server.php +++ b/app/Models/Server.php @@ -700,7 +700,8 @@ $schema://$host { public function getDiskUsage(): ?string { - return instant_remote_process(["df /| tail -1 | awk '{ print $5}' | sed 's/%//g'"], $this, false); + return instant_remote_process(['df / --output=pcent | tr -cd 0-9'], $this, false); + // return instant_remote_process(["df /| tail -1 | awk '{ print $5}' | sed 's/%//g'"], $this, false); } public function definedResources() diff --git a/app/Notifications/Server/HighDiskUsage.php b/app/Notifications/Server/HighDiskUsage.php index 54fd7b160..108eead9c 100644 --- a/app/Notifications/Server/HighDiskUsage.php +++ b/app/Notifications/Server/HighDiskUsage.php @@ -18,7 +18,7 @@ class HighDiskUsage extends Notification implements ShouldQueue public $tries = 1; - public function __construct(public Server $server, public int $disk_usage, public int $docker_cleanup_threshold) {} + public function __construct(public Server $server, public int $disk_usage, public int $server_disk_usage_notification_threshold) {} public function via(object $notifiable): array { @@ -47,7 +47,7 @@ class HighDiskUsage extends Notification implements ShouldQueue $mail->view('emails.high-disk-usage', [ 'name' => $this->server->name, 'disk_usage' => $this->disk_usage, - 'threshold' => $this->docker_cleanup_threshold, + 'threshold' => $this->server_disk_usage_notification_threshold, ]); return $mail; @@ -62,7 +62,7 @@ class HighDiskUsage extends Notification implements ShouldQueue ); $message->addField('Disk usage', "{$this->disk_usage}%"); - $message->addField('Threshold', "{$this->docker_cleanup_threshold}%"); + $message->addField('Threshold', "{$this->server_disk_usage_notification_threshold}%"); $message->addField('Tips', '[Link](https://coolify.io/docs/knowledge-base/server/automated-cleanup)'); return $message; @@ -71,7 +71,7 @@ class HighDiskUsage extends Notification implements ShouldQueue public function toTelegram(): array { return [ - 'message' => "Coolify: Server '{$this->server->name}' high disk usage detected!\nDisk usage: {$this->disk_usage}%. Threshold: {$this->docker_cleanup_threshold}%.\nPlease cleanup your disk to prevent data-loss.\nHere are some tips: https://coolify.io/docs/knowledge-base/server/automated-cleanup.", + 'message' => "Coolify: Server '{$this->server->name}' high disk usage detected!\nDisk usage: {$this->disk_usage}%. Threshold: {$this->server_disk_usage_notification_threshold}%.\nPlease cleanup your disk to prevent data-loss.\nHere are some tips: https://coolify.io/docs/knowledge-base/server/automated-cleanup.", ]; } } diff --git a/app/Notifications/Test.php b/app/Notifications/Test.php index 0446f8e52..4c069ffd6 100644 --- a/app/Notifications/Test.php +++ b/app/Notifications/Test.php @@ -7,6 +7,7 @@ use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Notifications\Messages\MailMessage; use Illuminate\Notifications\Notification; +use Illuminate\Queue\Middleware\RateLimited; class Test extends Notification implements ShouldQueue { @@ -21,6 +22,14 @@ class Test extends Notification implements ShouldQueue return setNotificationChannels($notifiable, 'test'); } + public function middleware(object $notifiable, string $channel) + { + return match ($channel) { + 'App\Notifications\Channels\EmailChannel' => [new RateLimited('email')], + default => [], + }; + } + public function toMail(): MailMessage { $mail = new MailMessage; diff --git a/database/migrations/2024_10_22_105745_add_server_disk_usage_threshold.php b/database/migrations/2024_10_22_105745_add_server_disk_usage_threshold.php new file mode 100644 index 000000000..76ccd1352 --- /dev/null +++ b/database/migrations/2024_10_22_105745_add_server_disk_usage_threshold.php @@ -0,0 +1,28 @@ +integer('server_disk_usage_notification_threshold')->default(80)->after('docker_cleanup_threshold'); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('server_settings', function (Blueprint $table) { + $table->dropColumn('server_disk_usage_notification_threshold'); + }); + } +}; diff --git a/docker/prod/Dockerfile b/docker/prod/Dockerfile index d0cebcbca..22a5b143a 100644 --- a/docker/prod/Dockerfile +++ b/docker/prod/Dockerfile @@ -1,4 +1,4 @@ -FROM serversideup/php:8.2-fpm-nginx-v2.2.1 as base +FROM serversideup/php:8.2-fpm-nginx-v2.2.1 AS base WORKDIR /var/www/html COPY composer.json composer.lock ./ diff --git a/resources/views/livewire/server/advanced.blade.php b/resources/views/livewire/server/advanced.blade.php index b1d21402a..29b6cd34c 100644 --- a/resources/views/livewire/server/advanced.blade.php +++ b/resources/views/livewire/server/advanced.blade.php @@ -16,11 +16,18 @@
Advanced configuration for your server.
-
+ +
+
+
+ +
+

Docker Cleanup

-
@if ($server->settings->force_docker_cleanup) @@ -70,6 +77,7 @@ " />
+

Builds

Customize the build process.