dave/coolify
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

feat: limit storage check emails

Browse Source 
feat: sentinel should send storage usage
This commit is contained in:
Andras Bacsai
2024-10-22 14:01:36 +02:00
parent 0a26598093
commit ac768e5313
14 changed files with 111 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
app/Jobs/PushServerUpdateJob.php
View File

@@ -101,6 +101,10 @@ class PushServerUpdateJob implements ShouldQueue
$this->server->sentinelHeartbeat();
$this->containers = collect(data_get($data, 'containers'));
$filesystemUsageRoot = data_get($data, 'filesystem_usage_root.used_percentage');
ServerStorageCheckJob::dispatch($this->server, $filesystemUsageRoot);
if ($this->containers->isEmpty()) {
return;
}

6
app/Jobs/ServerCheckJob.php
View File

@@ -2,14 +2,11 @@
namespace App\Jobs;
use App\Actions\Database\StartDatabaseProxy;
use App\Actions\Docker\GetContainersStatus;
use App\Actions\Proxy\CheckProxy;
use App\Actions\Proxy\StartProxy;
use App\Actions\Server\InstallLogDrain;
use App\Models\ApplicationPreview;
use App\Models\Server;
use App\Models\ServiceDatabase;
use App\Notifications\Container\ContainerRestarted;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
@@ -17,7 +14,6 @@ use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Arr;
class ServerCheckJob implements ShouldBeEncrypted, ShouldQueue
{
@@ -68,7 +64,9 @@ class ServerCheckJob implements ShouldBeEncrypted, ShouldQueue
if (is_null($this->containers)) {
return 'No containers found.';
}
ServerStorageCheckJob::dispatch($this->server);
GetContainersStatus::run($this->server, $this->containers, $containerReplicates);
if ($this->server->isLogDrainEnabled()) {
$this->checkLogDrainContainer();
}

44
app/Jobs/ServerStorageCheckJob.php
View File

@@ -3,12 +3,14 @@
namespace App\Jobs;
use App\Models\Server;
use App\Notifications\Server\HighDiskUsage;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\RateLimiter;
class ServerStorageCheckJob implements ShouldBeEncrypted, ShouldQueue
{
@@ -18,22 +20,12 @@ class ServerStorageCheckJob implements ShouldBeEncrypted, ShouldQueue
public $timeout = 60;
public $containers;
public $applications;
public $databases;
public $services;
public $previews;
public function backoff(): int
{
return isDev() ? 1 : 3;
}
public function __construct(public Server $server) {}
public function __construct(public Server $server, public ?int $percentage = null) {}
public function handle()
{
@@ -43,15 +35,33 @@ class ServerStorageCheckJob implements ShouldBeEncrypted, ShouldQueue
return 'Server is not ready.';
}
$team = $this->server->team;
$percentage = $this->server->storageCheck();
if ($percentage > 1) {
ray('Server storage is at '.$percentage.'%');
$team = data_get($this->server, 'team');
$serverDiskUsageNotificationThreshold = data_get($this->server, 'settings.server_disk_usage_notification_threshold');
if (is_null($this->percentage)) {
$this->percentage = $this->server->storageCheck();
}
if (! $this->percentage) {
throw new \Exception('No percentage could be retrieved.');
}
if ($this->percentage > $serverDiskUsageNotificationThreshold) {
$executed = RateLimiter::attempt(
'high-disk-usage:'.$this->server->id,
$maxAttempts = 0,
function () use ($team, $serverDiskUsageNotificationThreshold) {
$team->notify(new HighDiskUsage($this->server, $this->percentage, $serverDiskUsageNotificationThreshold));
},
$decaySeconds = 3600,
);
if (! $executed) {
throw new \Exception('Too many messages sent!');
}
} else {
RateLimiter::hit('high-disk-usage:'.$this->server->id, 600);
}
} catch (\Throwable $e) {
ray($e->getMessage());
return handleError($e);
}