dave/coolify
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

feat: limit storage check emails

Browse Source 
feat: sentinel should send storage usage
This commit is contained in:
Andras Bacsai
2024-10-22 14:01:36 +02:00
parent 0a26598093
commit ac768e5313
14 changed files with 111 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
app/Actions/Docker/GetContainersStatus.php
View File

@@ -3,8 +3,6 @@
namespace App\Actions\Docker;
use App\Actions\Database\StartDatabaseProxy;
use App\Actions\Proxy\CheckProxy;
use App\Actions\Proxy\StartProxy;
use App\Actions\Shared\ComplexStatusCheck;
use App\Models\ApplicationPreview;
use App\Models\Server;

4
app/Jobs/PushServerUpdateJob.php
View File

@@ -101,6 +101,10 @@ class PushServerUpdateJob implements ShouldQueue
$this->server->sentinelHeartbeat();
$this->containers = collect(data_get($data, 'containers'));
$filesystemUsageRoot = data_get($data, 'filesystem_usage_root.used_percentage');
ServerStorageCheckJob::dispatch($this->server, $filesystemUsageRoot);
if ($this->containers->isEmpty()) {
return;
}

6
app/Jobs/ServerCheckJob.php
View File

@@ -2,14 +2,11 @@
namespace App\Jobs;
use App\Actions\Database\StartDatabaseProxy;
use App\Actions\Docker\GetContainersStatus;
use App\Actions\Proxy\CheckProxy;
use App\Actions\Proxy\StartProxy;
use App\Actions\Server\InstallLogDrain;
use App\Models\ApplicationPreview;
use App\Models\Server;
use App\Models\ServiceDatabase;
use App\Notifications\Container\ContainerRestarted;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
@@ -17,7 +14,6 @@ use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Arr;
class ServerCheckJob implements ShouldBeEncrypted, ShouldQueue
{
@@ -68,7 +64,9 @@ class ServerCheckJob implements ShouldBeEncrypted, ShouldQueue
if (is_null($this->containers)) {
return 'No containers found.';
}
ServerStorageCheckJob::dispatch($this->server);
GetContainersStatus::run($this->server, $this->containers, $containerReplicates);
if ($this->server->isLogDrainEnabled()) {
$this->checkLogDrainContainer();
}

44
app/Jobs/ServerStorageCheckJob.php
View File

@@ -3,12 +3,14 @@
namespace App\Jobs;
use App\Models\Server;
use App\Notifications\Server\HighDiskUsage;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\RateLimiter;
class ServerStorageCheckJob implements ShouldBeEncrypted, ShouldQueue
{
@@ -18,22 +20,12 @@ class ServerStorageCheckJob implements ShouldBeEncrypted, ShouldQueue
public $timeout = 60;
public $containers;
public $applications;
public $databases;
public $services;
public $previews;
public function backoff(): int
{
return isDev() ? 1 : 3;
}
public function __construct(public Server $server) {}
public function __construct(public Server $server, public ?int $percentage = null) {}
public function handle()
{
@@ -43,15 +35,33 @@ class ServerStorageCheckJob implements ShouldBeEncrypted, ShouldQueue
return 'Server is not ready.';
}
$team = $this->server->team;
$percentage = $this->server->storageCheck();
if ($percentage > 1) {
ray('Server storage is at '.$percentage.'%');
$team = data_get($this->server, 'team');
$serverDiskUsageNotificationThreshold = data_get($this->server, 'settings.server_disk_usage_notification_threshold');
if (is_null($this->percentage)) {
$this->percentage = $this->server->storageCheck();
}
if (! $this->percentage) {
throw new \Exception('No percentage could be retrieved.');
}
if ($this->percentage > $serverDiskUsageNotificationThreshold) {
$executed = RateLimiter::attempt(
'high-disk-usage:'.$this->server->id,
$maxAttempts = 0,
function () use ($team, $serverDiskUsageNotificationThreshold) {
$team->notify(new HighDiskUsage($this->server, $this->percentage, $serverDiskUsageNotificationThreshold));
},
$decaySeconds = 3600,
);
if (! $executed) {
throw new \Exception('Too many messages sent!');
}
} else {
RateLimiter::hit('high-disk-usage:'.$this->server->id, 600);
}
} catch (\Throwable $e) {
ray($e->getMessage());
return handleError($e);
}

16
app/Livewire/Notifications/Email.php
View File

@@ -4,6 +4,7 @@ namespace App\Livewire\Notifications;
use App\Models\Team;
use App\Notifications\Test;
use Illuminate\Support\Facades\RateLimiter;
use Livewire\Component;
class Email extends Component
@@ -74,8 +75,23 @@ class Email extends Component
public function sendTestNotification()
{
try {
$executed = RateLimiter::attempt(
'test-email:'.$this->team->id,
$perMinute = 0,
function () {
$this->team?->notify(new Test($this->emails));
$this->dispatch('success', 'Test Email sent.');
},
$decaySeconds = 10,
);
if (! $executed) {
throw new \Exception('Too many messages sent!');
}
} catch (\Throwable $e) {
return handleError($e, $this);
}
}
public function instantSaveInstance()

2
app/Livewire/Server/Advanced.php
View File

@@ -16,6 +16,7 @@ class Advanced extends Component
'server.settings.force_docker_cleanup' => 'required|boolean',
'server.settings.docker_cleanup_frequency' => 'required_if:server.settings.force_docker_cleanup,true|string',
'server.settings.docker_cleanup_threshold' => 'required_if:server.settings.force_docker_cleanup,false|integer|min:1|max:100',
'server.settings.server_disk_usage_notification_threshold' => 'required|integer|min:50|max:100',
'server.settings.delete_unused_volumes' => 'boolean',
'server.settings.delete_unused_networks' => 'boolean',
];
@@ -27,6 +28,7 @@ class Advanced extends Component
'server.settings.force_docker_cleanup' => 'Force Docker Cleanup',
'server.settings.docker_cleanup_frequency' => 'Docker Cleanup Frequency',
'server.settings.docker_cleanup_threshold' => 'Docker Cleanup Threshold',
'server.settings.server_disk_usage_notification_threshold' => 'Server Disk Usage Notification Threshold',
'server.settings.delete_unused_volumes' => 'Delete Unused Volumes',
'server.settings.delete_unused_networks' => 'Delete Unused Networks',
];

5
app/Livewire/Server/Form.php
View File

@@ -246,11 +246,6 @@ class Form extends Component
}
refresh_server_connection($this->server->privateKey);
$this->server->settings->wildcard_domain = $this->wildcard_domain;
// if ($this->server->settings->force_docker_cleanup) {
// $this->server->settings->docker_cleanup_frequency = $this->server->settings->docker_cleanup_frequency;
// } else {
// $this->server->settings->docker_cleanup_threshold = $this->server->settings->docker_cleanup_threshold;
// }
$currentTimezone = $this->server->settings->getOriginal('server_timezone');
$newTimezone = $this->server->settings->server_timezone;
if ($currentTimezone !== $newTimezone || $currentTimezone === '') {

7
app/Livewire/SettingsEmail.php
View File

@@ -3,7 +3,6 @@
namespace App\Livewire;
use App\Models\InstanceSettings;
use App\Notifications\TransactionalEmails\Test;
use Livewire\Component;
class SettingsEmail extends Component
@@ -124,10 +123,4 @@ class SettingsEmail extends Component
return handleError($e, $this);
}
}
public function sendTestNotification()
{
$this->settings?->notify(new Test($this->emails));
$this->dispatch('success', 'Test email sent.');
}
}

3
app/Models/Server.php
View File

@@ -700,7 +700,8 @@ $schema://$host {
public function getDiskUsage(): ?string
{
return instant_remote_process(["df /| tail -1 | awk '{ print $5}' | sed 's/%//g'"], $this, false);
return instant_remote_process(['df / --output=pcent | tr -cd 0-9'], $this, false);
// return instant_remote_process(["df /| tail -1 | awk '{ print $5}' | sed 's/%//g'"], $this, false);
}
public function definedResources()

8
app/Notifications/Server/HighDiskUsage.php
View File

@@ -18,7 +18,7 @@ class HighDiskUsage extends Notification implements ShouldQueue
public $tries = 1;
public function __construct(public Server $server, public int $disk_usage, public int $docker_cleanup_threshold) {}
public function __construct(public Server $server, public int $disk_usage, public int $server_disk_usage_notification_threshold) {}
public function via(object $notifiable): array
{
@@ -47,7 +47,7 @@ class HighDiskUsage extends Notification implements ShouldQueue
$mail->view('emails.high-disk-usage', [
'name' => $this->server->name,
'disk_usage' => $this->disk_usage,
'threshold' => $this->docker_cleanup_threshold,
'threshold' => $this->server_disk_usage_notification_threshold,
]);
return $mail;
@@ -62,7 +62,7 @@ class HighDiskUsage extends Notification implements ShouldQueue
);
$message->addField('Disk usage', "{$this->disk_usage}%");
$message->addField('Threshold', "{$this->docker_cleanup_threshold}%");
$message->addField('Threshold', "{$this->server_disk_usage_notification_threshold}%");
$message->addField('Tips', '[Link](https://coolify.io/docs/knowledge-base/server/automated-cleanup)');
return $message;
@@ -71,7 +71,7 @@ class HighDiskUsage extends Notification implements ShouldQueue
public function toTelegram(): array
{
return [
'message' => "Coolify: Server '{$this->server->name}' high disk usage detected!\nDisk usage: {$this->disk_usage}%. Threshold: {$this->docker_cleanup_threshold}%.\nPlease cleanup your disk to prevent data-loss.\nHere are some tips: https://coolify.io/docs/knowledge-base/server/automated-cleanup.",
'message' => "Coolify: Server '{$this->server->name}' high disk usage detected!\nDisk usage: {$this->disk_usage}%. Threshold: {$this->server_disk_usage_notification_threshold}%.\nPlease cleanup your disk to prevent data-loss.\nHere are some tips: https://coolify.io/docs/knowledge-base/server/automated-cleanup.",
];
}
}

9
app/Notifications/Test.php
View File

@@ -7,6 +7,7 @@ use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Notifications\Messages\MailMessage;
use Illuminate\Notifications\Notification;
use Illuminate\Queue\Middleware\RateLimited;
class Test extends Notification implements ShouldQueue
{
@@ -21,6 +22,14 @@ class Test extends Notification implements ShouldQueue
return setNotificationChannels($notifiable, 'test');
}
public function middleware(object $notifiable, string $channel)
{
return match ($channel) {
'App\Notifications\Channels\EmailChannel' => [new RateLimited('email')],
default => [],
};
}
public function toMail(): MailMessage
{
$mail = new MailMessage;

28
database/migrations/2024_10_22_105745_add_server_disk_usage_threshold.php Normal file
View File

@@ -0,0 +1,28 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
/**
* Run the migrations.
*/
public function up(): void
{
Schema::table('server_settings', function (Blueprint $table) {
$table->integer('server_disk_usage_notification_threshold')->default(80)->after('docker_cleanup_threshold');
});
}
/**
* Reverse the migrations.
*/
public function down(): void
{
Schema::table('server_settings', function (Blueprint $table) {
$table->dropColumn('server_disk_usage_notification_threshold');
});
}
};

2
docker/prod/Dockerfile
View File

@@ -1,4 +1,4 @@
FROM serversideup/php:8.2-fpm-nginx-v2.2.1 as base
FROM serversideup/php:8.2-fpm-nginx-v2.2.1 AS base
WORKDIR /var/www/html
COPY composer.json composer.lock ./

12
resources/views/livewire/server/advanced.blade.php
View File

@@ -16,11 +16,18 @@
</div>
<div>Advanced configuration for your server.</div>
</div>
<div class="flex flex-col gap-4 pt-4">
<div class="flex flex-col gap-4">
<div class="flex flex-col">
<div class="flex flex-wrap gap-2 sm:flex-nowrap pt-4">
<x-forms.input id="server.settings.server_disk_usage_notification_threshold"
label="Server disk usage notification threshold (%)" required
helper="If the server disk usage exceeds this threshold, Coolify will send a notification to the team members." />
</div>
</div>
<div class="flex flex-col gap-2">
<div class="flex items-center gap-2">
<h3>Docker Cleanup</h3>
</div>
<div class="flex flex-wrap items-center gap-4">
@if ($server->settings->force_docker_cleanup)
@@ -70,6 +77,7 @@
</ul>" />
</div>
</div>
<div class="flex flex-col">
<h3>Builds</h3>
<div>Customize the build process.</div>