dave/coolify
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

feat: notification rate limiter

Browse Source 
fix: limit server up / down notification limits
This commit is contained in:
Andras Bacsai
2024-10-25 15:13:23 +02:00
parent fb75741aa8
commit 8c96ab52d7
9 changed files with 161 additions and 217 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
app/Console/Kernel.php
View File

@@ -110,11 +110,11 @@ class Kernel extends ConsoleKernel
$servers = $this->allServers->where('ip', '!=', '1.2.3.4');
}
foreach ($servers as $server) {
$last_sentinel_update = $server->sentinel_updated_at;
if (Carbon::parse($last_sentinel_update)->isBefore(now()->subSeconds($server->waitBeforeDoingSshCheck()))) {
$lastSentinelUpdate = $server->sentinel_updated_at;
$serverTimezone = $server->settings->server_timezone;
if (Carbon::parse($lastSentinelUpdate)->isBefore(now()->subSeconds($server->waitBeforeDoingSshCheck()))) {
$schedule->job(new ServerCheckJob($server))->everyMinute()->onOneServer();
}
$serverTimezone = $server->settings->server_timezone;
if ($server->settings->force_docker_cleanup) {
$schedule->job(new DockerCleanupJob($server))->cron($server->settings->docker_cleanup_frequency)->timezone($serverTimezone)->onOneServer();
} else {

12
app/Jobs/PushServerUpdateJob.php
View File

@@ -97,8 +97,6 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue
}
$data = collect($this->data);
$this->serverStatus();
$this->server->sentinelHeartbeat();
$this->containers = collect(data_get($data, 'containers'));
@@ -212,16 +210,6 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue
}
private function serverStatus()
{
if ($this->server->isFunctional() === false) {
throw new \Exception('Server is not ready.');
}
if ($this->server->status() === false) {
throw new \Exception('Server is not reachable.');
}
}
private function updateApplicationStatus(string $applicationId, string $containerStatus)
{
$application = $this->applications->where('id', $applicationId)->first();

48
app/Jobs/ServerCheckJob.php
View File

@@ -43,22 +43,15 @@ class ServerCheckJob implements ShouldBeEncrypted, ShouldQueue
public function handle()
{
try {
if ($this->server->serverStatus() === false) {
return 'Server is not reachable or not ready.';
}
$this->applications = $this->server->applications();
$this->databases = $this->server->databases();
$this->services = $this->server->services()->get();
$this->previews = $this->server->previews();
$up = $this->serverStatus();
if (! $up) {
ray('Server is not reachable.');
return 'Server is not reachable.';
}
if (! $this->server->isFunctional()) {
ray('Server is not ready.');
return 'Server is not ready.';
}
if (! $this->server->isSwarmWorker() && ! $this->server->isBuildServer()) {
['containers' => $this->containers, 'containerReplicates' => $containerReplicates] = $this->server->getContainers();
if (is_null($this->containers)) {
@@ -111,39 +104,6 @@ class ServerCheckJob implements ShouldBeEncrypted, ShouldQueue
}
private function serverStatus()
{
['uptime' => $uptime] = $this->server->validateConnection(false);
if ($uptime) {
if ($this->server->unreachable_notification_sent === true) {
$this->server->update(['unreachable_notification_sent' => false]);
}
} else {
// $this->server->team?->notify(new Unreachable($this->server));
foreach ($this->applications as $application) {
$application->update(['status' => 'exited']);
}
foreach ($this->databases as $database) {
$database->update(['status' => 'exited']);
}
foreach ($this->services as $service) {
$apps = $service->applications()->get();
$dbs = $service->databases()->get();
foreach ($apps as $app) {
$app->update(['status' => 'exited']);
}
foreach ($dbs as $db) {
$db->update(['status' => 'exited']);
}
}
return false;
}
return true;
}
private function checkLogDrainContainer()
{
$foundLogDrainContainer = $this->containers->filter(function ($value, $key) {

190
app/Models/Server.php
View File

@@ -6,6 +6,8 @@ use App\Actions\Server\InstallDocker;
use App\Actions\Server\StartSentinel;
use App\Enums\ProxyTypes;
use App\Jobs\CheckAndStartSentinelJob;
use App\Notifications\Server\Reachable;
use App\Notifications\Server\Unreachable;
use Illuminate\Database\Eloquent\Builder;
use Illuminate\Database\Eloquent\Casts\Attribute;
use Illuminate\Database\Eloquent\SoftDeletes;
@@ -61,6 +63,7 @@ class Server extends BaseModel
$payload['ip'] = str($server->ip)->trim();
}
$server->forceFill($payload);
});
static::created(function ($server) {
ServerSetting::create([
@@ -107,12 +110,15 @@ class Server extends BaseModel
});
}
public $casts = [
protected $casts = [
'proxy' => SchemalessAttributes::class,
'logdrain_axiom_api_key' => 'encrypted',
'logdrain_newrelic_license_key' => 'encrypted',
'delete_unused_volumes' => 'boolean',
'delete_unused_networks' => 'boolean',
'unreachable_notification_sent' => 'boolean',
'is_build_server' => 'boolean',
'force_disabled' => 'boolean',
];
protected $schemalessAttributes = [
@@ -519,16 +525,14 @@ $schema://$host {
public function forceEnableServer()
{
$this->settings->update([
'force_disabled' => false,
]);
$this->settings->force_disabled = false;
$this->settings->save();
}
public function forceDisableServer()
{
$this->settings->update([
'force_disabled' => true,
]);
$this->settings->force_disabled = true;
$this->settings->save();
$sshKeyFileLocation = "id.root@{$this->uuid}";
Storage::disk('ssh-keys')->delete($sshKeyFileLocation);
Storage::disk('ssh-mux')->delete($this->muxFilename());
@@ -624,72 +628,6 @@ $schema://$host {
}
}
public function isServerReady(int $tries = 3)
{
if ($this->skipServer()) {
return false;
}
$serverUptimeCheckNumber = $this->unreachable_count;
if ($this->unreachable_count < $tries) {
$serverUptimeCheckNumber = $this->unreachable_count + 1;
}
if ($this->unreachable_count > $tries) {
$serverUptimeCheckNumber = $tries;
}
$serverUptimeCheckNumberMax = $tries;
// ray('server: ' . $this->name);
// ray('serverUptimeCheckNumber: ' . $serverUptimeCheckNumber);
// ray('serverUptimeCheckNumberMax: ' . $serverUptimeCheckNumberMax);
['uptime' => $uptime] = $this->validateConnection();
if ($uptime) {
if ($this->unreachable_notification_sent === true) {
$this->update(['unreachable_notification_sent' => false]);
}
return true;
} else {
if ($serverUptimeCheckNumber >= $serverUptimeCheckNumberMax) {
// Reached max number of retries
if ($this->unreachable_notification_sent === false) {
ray('Server unreachable, sending notification...');
// $this->team?->notify(new Unreachable($this));
$this->update(['unreachable_notification_sent' => true]);
}
if ($this->settings->is_reachable === true) {
$this->settings()->update([
'is_reachable' => false,
]);
}
foreach ($this->applications() as $application) {
$application->update(['status' => 'exited']);
}
foreach ($this->databases() as $database) {
$database->update(['status' => 'exited']);
}
foreach ($this->services()->get() as $service) {
$apps = $service->applications()->get();
$dbs = $service->databases()->get();
foreach ($apps as $app) {
$app->update(['status' => 'exited']);
}
foreach ($dbs as $db) {
$db->update(['status' => 'exited']);
}
}
} else {
$this->update([
'unreachable_count' => $this->unreachable_count + 1,
]);
}
return false;
}
}
public function getDiskUsage(): ?string
{
return instant_remote_process(['df / --output=pcent | tr -cd 0-9'], $this, false);
@@ -1038,29 +976,43 @@ $schema://$host {
return data_get($this, 'settings.is_swarm_worker');
}
public function serverStatus(): bool
{
if ($this->status() === false) {
return false;
}
if ($this->isFunctional() === false) {
return false;
}
return true;
}
public function status(): bool
{
if ($this->skipServer()) {
return false;
}
['uptime' => $uptime] = $this->validateConnection(false);
if ($uptime) {
if ($this->unreachable_notification_sent === true) {
$this->update(['unreachable_notification_sent' => false]);
if ($uptime === false) {
foreach ($this->applications() as $application) {
$application->status = 'exited';
$application->save();
}
} else {
// $this->server->team?->notify(new Unreachable($this->server));
foreach ($this->applications as $application) {
$application->update(['status' => 'exited']);
foreach ($this->databases() as $database) {
$database->status = 'exited';
$database->save();
}
foreach ($this->databases as $database) {
$database->update(['status' => 'exited']);
}
foreach ($this->services as $service) {
foreach ($this->services() as $service) {
$apps = $service->applications()->get();
$dbs = $service->databases()->get();
foreach ($apps as $app) {
$app->update(['status' => 'exited']);
$app->status = 'exited';
$app->save();
}
foreach ($dbs as $db) {
$db->update(['status' => 'exited']);
$db->status = 'exited';
$db->save();
}
}
@@ -1070,39 +1022,65 @@ $schema://$host {
return true;
}
public function isReachableChanged()
{
$this->refresh();
$unreachableNotificationSent = (bool) $this->unreachable_notification_sent;
$isReachable = (bool) $this->settings->is_reachable;
loggy('Server setting is_reachable changed to '.$isReachable.' for server '.$this->id.'. Unreachable notification sent: '.$unreachableNotificationSent);
// If the server is reachable, send the reachable notification if it was sent before
if ($isReachable === true) {
if ($unreachableNotificationSent === true) {
$this->sendReachableNotification();
}
} else {
// If the server is unreachable, send the unreachable notification if it was not sent before
if ($unreachableNotificationSent === false) {
$this->sendUnreachableNotification();
}
}
}
public function sendReachableNotification()
{
$this->unreachable_notification_sent = false;
$this->save();
$this->refresh();
$this->team->notify(new Reachable($this));
}
public function sendUnreachableNotification()
{
$this->unreachable_notification_sent = true;
$this->save();
$this->refresh();
$this->team->notify(new Unreachable($this));
}
public function validateConnection($isManualCheck = true)
{
config()->set('constants.ssh.mux_enabled', ! $isManualCheck);
// ray('Manual Check: ' . ($isManualCheck ? 'true' : 'false'));
$server = Server::find($this->id);
if (! $server) {
return ['uptime' => false, 'error' => 'Server not found.'];
}
if ($server->skipServer()) {
if ($this->skipServer()) {
return ['uptime' => false, 'error' => 'Server skipped.'];
}
try {
// Make sure the private key is stored
if ($server->privateKey) {
$server->privateKey->storeInFileSystem();
if ($this->privateKey) {
$this->privateKey->storeInFileSystem();
}
instant_remote_process(['ls /'], $server);
$server->settings()->update([
'is_reachable' => true,
]);
$server->update([
'unreachable_count' => 0,
]);
if (data_get($server, 'unreachable_notification_sent') === true) {
$server->update(['unreachable_notification_sent' => false]);
instant_remote_process(['ls /'], $this);
if ($this->settings->is_reachable === false) {
$this->settings->is_reachable = true;
$this->settings->save();
}
return ['uptime' => true, 'error' => null];
} catch (\Throwable $e) {
$server->settings()->update([
'is_reachable' => false,
]);
if ($this->settings->is_reachable === true) {
$this->settings->is_reachable = false;
$this->settings->save();
}
return ['uptime' => false, 'error' => $e->getMessage()];
}

19
app/Models/ServerSetting.php
View File

@@ -54,6 +54,8 @@ class ServerSetting extends Model
'force_docker_cleanup' => 'boolean',
'docker_cleanup_threshold' => 'integer',
'sentinel_token' => 'encrypted',
'is_reachable' => 'boolean',
'is_usable' => 'boolean',
];
protected static function booted()
@@ -70,15 +72,18 @@ class ServerSetting extends Model
loggy('Error creating server setting: '.$e->getMessage());
}
});
static::updated(function ($setting) {
static::updated(function ($settings) {
if (
$setting->isDirty('sentinel_token') ||
$setting->isDirty('sentinel_custom_url') ||
$setting->isDirty('sentinel_metrics_refresh_rate_seconds') ||
$setting->isDirty('sentinel_metrics_history_days') ||
$setting->isDirty('sentinel_push_interval_seconds')
$settings->isDirty('sentinel_token') ||
$settings->isDirty('sentinel_custom_url') ||
$settings->isDirty('sentinel_metrics_refresh_rate_seconds') ||
$settings->isDirty('sentinel_metrics_history_days') ||
$settings->isDirty('sentinel_push_interval_seconds')
) {
$setting->server->restartSentinel();
$settings->server->restartSentinel();
}
if ($settings->isDirty('is_reachable')) {
$settings->server->isReachableChanged();
}
});
}

33
app/Notifications/Server/Revived.php → app/Notifications/Server/Reachable.php
View File

@@ -2,8 +2,6 @@
namespace App\Notifications\Server;
use App\Actions\Docker\GetContainersStatus;
use App\Jobs\ContainerStatusJob;
use App\Models\Server;
use App\Notifications\Channels\DiscordChannel;
use App\Notifications\Channels\EmailChannel;
@@ -13,25 +11,28 @@ use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Notifications\Messages\MailMessage;
use Illuminate\Notifications\Notification;
use Illuminate\Support\Facades\RateLimiter;
class Revived extends Notification implements ShouldQueue
class Reachable extends Notification implements ShouldQueue
{
use Queueable;
public $tries = 1;
protected bool $isRateLimited = false;
public function __construct(public Server $server)
{
if ($this->server->unreachable_notification_sent === false) {
return;
}
GetContainersStatus::dispatch($server)->onQueue('high');
// dispatch(new ContainerStatusJob($server));
$this->isRateLimited = isEmailRateLimited(
limiterKey: 'server-reachable:'.$this->server->id,
);
}
public function via(object $notifiable): array
{
if ($this->isRateLimited) {
return [];
}
$channels = [];
$isEmailEnabled = isEmailEnabled($notifiable);
$isDiscordEnabled = data_get($notifiable, 'discord_enabled');
@@ -46,20 +47,8 @@ class Revived extends Notification implements ShouldQueue
if ($isTelegramEnabled) {
$channels[] = TelegramChannel::class;
}
$executed = RateLimiter::attempt(
'notification-server-revived-'.$this->server->uuid,
1,
function () use ($channels) {
return $channels;
},
7200,
);
if (! $executed) {
return [];
}
return $executed;
return $channels;
}
public function toMail(): MailMessage

34
app/Notifications/Server/Unreachable.php
View File

@@ -11,7 +11,6 @@ use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Notifications\Messages\MailMessage;
use Illuminate\Notifications\Notification;
use Illuminate\Support\Facades\RateLimiter;
class Unreachable extends Notification implements ShouldQueue
{
@@ -19,10 +18,21 @@ class Unreachable extends Notification implements ShouldQueue
public $tries = 1;
public function __construct(public Server $server) {}
protected bool $isRateLimited = false;
public function __construct(public Server $server)
{
$this->isRateLimited = isEmailRateLimited(
limiterKey: 'server-unreachable:'.$this->server->id,
);
}
public function via(object $notifiable): array
{
if ($this->isRateLimited) {
return [];
}
$channels = [];
$isEmailEnabled = isEmailEnabled($notifiable);
$isDiscordEnabled = data_get($notifiable, 'discord_enabled');
@@ -37,23 +47,11 @@ class Unreachable extends Notification implements ShouldQueue
if ($isTelegramEnabled) {
$channels[] = TelegramChannel::class;
}
$executed = RateLimiter::attempt(
'notification-server-unreachable-'.$this->server->uuid,
1,
function () use ($channels) {
return $channels;
},
7200,
);
if (! $executed) {
return [];
}
return $executed;
return $channels;
}
public function toMail(): MailMessage
public function toMail(): ?MailMessage
{
$mail = new MailMessage;
$mail->subject("Coolify: Your server ({$this->server->name}) is unreachable.");
@@ -64,7 +62,7 @@ class Unreachable extends Notification implements ShouldQueue
return $mail;
}
public function toDiscord(): DiscordMessage
public function toDiscord(): ?DiscordMessage
{
$message = new DiscordMessage(
title: ':cross_mark: Server unreachable',
@@ -77,7 +75,7 @@ class Unreachable extends Notification implements ShouldQueue
return $message;
}
public function toTelegram(): array
public function toTelegram(): ?array
{
return [
'message' => "Coolify: Your server '{$this->server->name}' is unreachable. All automations & integrations are turned off! Please check your server! IMPORTANT: We automatically try to revive your server and turn on all automations & integrations.",

28
bootstrap/helpers/shared.php
View File

@@ -39,6 +39,7 @@ use Illuminate\Support\Facades\File;
use Illuminate\Support\Facades\Http;
use Illuminate\Support\Facades\Mail;
use Illuminate\Support\Facades\Process;
use Illuminate\Support\Facades\RateLimiter;
use Illuminate\Support\Facades\Request;
use Illuminate\Support\Facades\Route;
use Illuminate\Support\Facades\Validator;
@@ -4038,3 +4039,30 @@ function sslipDomainWarning(string $domains)
return $showSslipHttpsWarning;
}
function isEmailRateLimited(string $limiterKey, int $decaySeconds = 3600, ?callable $callbackOnSuccess = null): bool
{
if (isDev()) {
$decaySeconds = 120;
}
$rateLimited = false;
$executed = RateLimiter::attempt(
$limiterKey,
$maxAttempts = 0,
function () use (&$rateLimited, &$limiterKey, $callbackOnSuccess) {
isDev() && loggy('Rate limit not reached for '.$limiterKey);
$rateLimited = false;
if ($callbackOnSuccess) {
$callbackOnSuccess();
}
},
$decaySeconds,
);
if (! $executed) {
isDev() && loggy('Rate limit reached for '.$limiterKey.'. Rate limiter will be disabled for '.$decaySeconds.' seconds.');
$rateLimited = true;
}
return $rateLimited;
}

8
scripts/run
View File

@@ -20,6 +20,9 @@ function help {
compgen -A function | cat -n
}
function logs {
docker exec -t coolify tail -f storage/logs/laravel.log
}
function test {
docker exec -t coolify php artisan test --testsuite=Feature
}
@@ -35,11 +38,6 @@ function db:reset-prod {
bash spin exec -u webuser coolify php artisan migrate:fresh --force --seed --seeder=ProductionSeeder ||
php artisan migrate:fresh --force --seed --seeder=ProductionSeeder
}
function mfs {
db:reset
}
function coolify {
bash spin exec -u webuser coolify bash
}