diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index 3ad8af98c..1e55aa57f 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -110,11 +110,11 @@ class Kernel extends ConsoleKernel $servers = $this->allServers->where('ip', '!=', '1.2.3.4'); } foreach ($servers as $server) { - $last_sentinel_update = $server->sentinel_updated_at; - if (Carbon::parse($last_sentinel_update)->isBefore(now()->subSeconds($server->waitBeforeDoingSshCheck()))) { + $lastSentinelUpdate = $server->sentinel_updated_at; + $serverTimezone = $server->settings->server_timezone; + if (Carbon::parse($lastSentinelUpdate)->isBefore(now()->subSeconds($server->waitBeforeDoingSshCheck()))) { $schedule->job(new ServerCheckJob($server))->everyMinute()->onOneServer(); } - $serverTimezone = $server->settings->server_timezone; if ($server->settings->force_docker_cleanup) { $schedule->job(new DockerCleanupJob($server))->cron($server->settings->docker_cleanup_frequency)->timezone($serverTimezone)->onOneServer(); } else { diff --git a/app/Jobs/PushServerUpdateJob.php b/app/Jobs/PushServerUpdateJob.php index 384deb80e..62f059129 100644 --- a/app/Jobs/PushServerUpdateJob.php +++ b/app/Jobs/PushServerUpdateJob.php @@ -97,8 +97,6 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue } $data = collect($this->data); - $this->serverStatus(); - $this->server->sentinelHeartbeat(); $this->containers = collect(data_get($data, 'containers')); @@ -212,16 +210,6 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue } - private function serverStatus() - { - if ($this->server->isFunctional() === false) { - throw new \Exception('Server is not ready.'); - } - if ($this->server->status() === false) { - throw new \Exception('Server is not reachable.'); - } - } - private function updateApplicationStatus(string $applicationId, string $containerStatus) { $application = $this->applications->where('id', $applicationId)->first(); diff --git a/app/Jobs/ServerCheckJob.php b/app/Jobs/ServerCheckJob.php index f4720d035..5ac98e954 100644 --- a/app/Jobs/ServerCheckJob.php +++ b/app/Jobs/ServerCheckJob.php @@ -43,22 +43,15 @@ class ServerCheckJob implements ShouldBeEncrypted, ShouldQueue public function handle() { try { + if ($this->server->serverStatus() === false) { + return 'Server is not reachable or not ready.'; + } + $this->applications = $this->server->applications(); $this->databases = $this->server->databases(); $this->services = $this->server->services()->get(); $this->previews = $this->server->previews(); - $up = $this->serverStatus(); - if (! $up) { - ray('Server is not reachable.'); - - return 'Server is not reachable.'; - } - if (! $this->server->isFunctional()) { - ray('Server is not ready.'); - - return 'Server is not ready.'; - } if (! $this->server->isSwarmWorker() && ! $this->server->isBuildServer()) { ['containers' => $this->containers, 'containerReplicates' => $containerReplicates] = $this->server->getContainers(); if (is_null($this->containers)) { @@ -111,39 +104,6 @@ class ServerCheckJob implements ShouldBeEncrypted, ShouldQueue } - private function serverStatus() - { - ['uptime' => $uptime] = $this->server->validateConnection(false); - if ($uptime) { - if ($this->server->unreachable_notification_sent === true) { - $this->server->update(['unreachable_notification_sent' => false]); - } - } else { - // $this->server->team?->notify(new Unreachable($this->server)); - foreach ($this->applications as $application) { - $application->update(['status' => 'exited']); - } - foreach ($this->databases as $database) { - $database->update(['status' => 'exited']); - } - foreach ($this->services as $service) { - $apps = $service->applications()->get(); - $dbs = $service->databases()->get(); - foreach ($apps as $app) { - $app->update(['status' => 'exited']); - } - foreach ($dbs as $db) { - $db->update(['status' => 'exited']); - } - } - - return false; - } - - return true; - - } - private function checkLogDrainContainer() { $foundLogDrainContainer = $this->containers->filter(function ($value, $key) { diff --git a/app/Models/Server.php b/app/Models/Server.php index bd8177dd0..2f023a248 100644 --- a/app/Models/Server.php +++ b/app/Models/Server.php @@ -6,6 +6,8 @@ use App\Actions\Server\InstallDocker; use App\Actions\Server\StartSentinel; use App\Enums\ProxyTypes; use App\Jobs\CheckAndStartSentinelJob; +use App\Notifications\Server\Reachable; +use App\Notifications\Server\Unreachable; use Illuminate\Database\Eloquent\Builder; use Illuminate\Database\Eloquent\Casts\Attribute; use Illuminate\Database\Eloquent\SoftDeletes; @@ -61,6 +63,7 @@ class Server extends BaseModel $payload['ip'] = str($server->ip)->trim(); } $server->forceFill($payload); + }); static::created(function ($server) { ServerSetting::create([ @@ -107,12 +110,15 @@ class Server extends BaseModel }); } - public $casts = [ + protected $casts = [ 'proxy' => SchemalessAttributes::class, 'logdrain_axiom_api_key' => 'encrypted', 'logdrain_newrelic_license_key' => 'encrypted', 'delete_unused_volumes' => 'boolean', 'delete_unused_networks' => 'boolean', + 'unreachable_notification_sent' => 'boolean', + 'is_build_server' => 'boolean', + 'force_disabled' => 'boolean', ]; protected $schemalessAttributes = [ @@ -519,16 +525,14 @@ $schema://$host { public function forceEnableServer() { - $this->settings->update([ - 'force_disabled' => false, - ]); + $this->settings->force_disabled = false; + $this->settings->save(); } public function forceDisableServer() { - $this->settings->update([ - 'force_disabled' => true, - ]); + $this->settings->force_disabled = true; + $this->settings->save(); $sshKeyFileLocation = "id.root@{$this->uuid}"; Storage::disk('ssh-keys')->delete($sshKeyFileLocation); Storage::disk('ssh-mux')->delete($this->muxFilename()); @@ -624,72 +628,6 @@ $schema://$host { } } - public function isServerReady(int $tries = 3) - { - if ($this->skipServer()) { - return false; - } - $serverUptimeCheckNumber = $this->unreachable_count; - if ($this->unreachable_count < $tries) { - $serverUptimeCheckNumber = $this->unreachable_count + 1; - } - if ($this->unreachable_count > $tries) { - $serverUptimeCheckNumber = $tries; - } - - $serverUptimeCheckNumberMax = $tries; - - // ray('server: ' . $this->name); - // ray('serverUptimeCheckNumber: ' . $serverUptimeCheckNumber); - // ray('serverUptimeCheckNumberMax: ' . $serverUptimeCheckNumberMax); - - ['uptime' => $uptime] = $this->validateConnection(); - if ($uptime) { - if ($this->unreachable_notification_sent === true) { - $this->update(['unreachable_notification_sent' => false]); - } - - return true; - } else { - if ($serverUptimeCheckNumber >= $serverUptimeCheckNumberMax) { - // Reached max number of retries - if ($this->unreachable_notification_sent === false) { - ray('Server unreachable, sending notification...'); - // $this->team?->notify(new Unreachable($this)); - $this->update(['unreachable_notification_sent' => true]); - } - if ($this->settings->is_reachable === true) { - $this->settings()->update([ - 'is_reachable' => false, - ]); - } - - foreach ($this->applications() as $application) { - $application->update(['status' => 'exited']); - } - foreach ($this->databases() as $database) { - $database->update(['status' => 'exited']); - } - foreach ($this->services()->get() as $service) { - $apps = $service->applications()->get(); - $dbs = $service->databases()->get(); - foreach ($apps as $app) { - $app->update(['status' => 'exited']); - } - foreach ($dbs as $db) { - $db->update(['status' => 'exited']); - } - } - } else { - $this->update([ - 'unreachable_count' => $this->unreachable_count + 1, - ]); - } - - return false; - } - } - public function getDiskUsage(): ?string { return instant_remote_process(['df / --output=pcent | tr -cd 0-9'], $this, false); @@ -1038,29 +976,43 @@ $schema://$host { return data_get($this, 'settings.is_swarm_worker'); } + public function serverStatus(): bool + { + if ($this->status() === false) { + return false; + } + if ($this->isFunctional() === false) { + return false; + } + + return true; + } + public function status(): bool { + if ($this->skipServer()) { + return false; + } ['uptime' => $uptime] = $this->validateConnection(false); - if ($uptime) { - if ($this->unreachable_notification_sent === true) { - $this->update(['unreachable_notification_sent' => false]); + if ($uptime === false) { + foreach ($this->applications() as $application) { + $application->status = 'exited'; + $application->save(); } - } else { - // $this->server->team?->notify(new Unreachable($this->server)); - foreach ($this->applications as $application) { - $application->update(['status' => 'exited']); + foreach ($this->databases() as $database) { + $database->status = 'exited'; + $database->save(); } - foreach ($this->databases as $database) { - $database->update(['status' => 'exited']); - } - foreach ($this->services as $service) { + foreach ($this->services() as $service) { $apps = $service->applications()->get(); $dbs = $service->databases()->get(); foreach ($apps as $app) { - $app->update(['status' => 'exited']); + $app->status = 'exited'; + $app->save(); } foreach ($dbs as $db) { - $db->update(['status' => 'exited']); + $db->status = 'exited'; + $db->save(); } } @@ -1070,39 +1022,65 @@ $schema://$host { return true; } + public function isReachableChanged() + { + $this->refresh(); + $unreachableNotificationSent = (bool) $this->unreachable_notification_sent; + $isReachable = (bool) $this->settings->is_reachable; + loggy('Server setting is_reachable changed to '.$isReachable.' for server '.$this->id.'. Unreachable notification sent: '.$unreachableNotificationSent); + // If the server is reachable, send the reachable notification if it was sent before + if ($isReachable === true) { + if ($unreachableNotificationSent === true) { + $this->sendReachableNotification(); + } + } else { + // If the server is unreachable, send the unreachable notification if it was not sent before + if ($unreachableNotificationSent === false) { + $this->sendUnreachableNotification(); + } + } + } + + public function sendReachableNotification() + { + $this->unreachable_notification_sent = false; + $this->save(); + $this->refresh(); + $this->team->notify(new Reachable($this)); + } + + public function sendUnreachableNotification() + { + $this->unreachable_notification_sent = true; + $this->save(); + $this->refresh(); + $this->team->notify(new Unreachable($this)); + } + public function validateConnection($isManualCheck = true) { config()->set('constants.ssh.mux_enabled', ! $isManualCheck); - // ray('Manual Check: ' . ($isManualCheck ? 'true' : 'false')); - $server = Server::find($this->id); - if (! $server) { - return ['uptime' => false, 'error' => 'Server not found.']; - } - if ($server->skipServer()) { + if ($this->skipServer()) { return ['uptime' => false, 'error' => 'Server skipped.']; } try { // Make sure the private key is stored - if ($server->privateKey) { - $server->privateKey->storeInFileSystem(); + if ($this->privateKey) { + $this->privateKey->storeInFileSystem(); } - instant_remote_process(['ls /'], $server); - $server->settings()->update([ - 'is_reachable' => true, - ]); - $server->update([ - 'unreachable_count' => 0, - ]); - if (data_get($server, 'unreachable_notification_sent') === true) { - $server->update(['unreachable_notification_sent' => false]); + instant_remote_process(['ls /'], $this); + if ($this->settings->is_reachable === false) { + $this->settings->is_reachable = true; + $this->settings->save(); } return ['uptime' => true, 'error' => null]; } catch (\Throwable $e) { - $server->settings()->update([ - 'is_reachable' => false, - ]); + if ($this->settings->is_reachable === true) { + $this->settings->is_reachable = false; + $this->settings->save(); + } return ['uptime' => false, 'error' => $e->getMessage()]; } diff --git a/app/Models/ServerSetting.php b/app/Models/ServerSetting.php index b72fa3acf..7a8e7b8ed 100644 --- a/app/Models/ServerSetting.php +++ b/app/Models/ServerSetting.php @@ -54,6 +54,8 @@ class ServerSetting extends Model 'force_docker_cleanup' => 'boolean', 'docker_cleanup_threshold' => 'integer', 'sentinel_token' => 'encrypted', + 'is_reachable' => 'boolean', + 'is_usable' => 'boolean', ]; protected static function booted() @@ -70,15 +72,18 @@ class ServerSetting extends Model loggy('Error creating server setting: '.$e->getMessage()); } }); - static::updated(function ($setting) { + static::updated(function ($settings) { if ( - $setting->isDirty('sentinel_token') || - $setting->isDirty('sentinel_custom_url') || - $setting->isDirty('sentinel_metrics_refresh_rate_seconds') || - $setting->isDirty('sentinel_metrics_history_days') || - $setting->isDirty('sentinel_push_interval_seconds') + $settings->isDirty('sentinel_token') || + $settings->isDirty('sentinel_custom_url') || + $settings->isDirty('sentinel_metrics_refresh_rate_seconds') || + $settings->isDirty('sentinel_metrics_history_days') || + $settings->isDirty('sentinel_push_interval_seconds') ) { - $setting->server->restartSentinel(); + $settings->server->restartSentinel(); + } + if ($settings->isDirty('is_reachable')) { + $settings->server->isReachableChanged(); } }); } diff --git a/app/Notifications/Server/Revived.php b/app/Notifications/Server/Reachable.php similarity index 74% rename from app/Notifications/Server/Revived.php rename to app/Notifications/Server/Reachable.php index 8d1ceeaef..9b54501d9 100644 --- a/app/Notifications/Server/Revived.php +++ b/app/Notifications/Server/Reachable.php @@ -2,8 +2,6 @@ namespace App\Notifications\Server; -use App\Actions\Docker\GetContainersStatus; -use App\Jobs\ContainerStatusJob; use App\Models\Server; use App\Notifications\Channels\DiscordChannel; use App\Notifications\Channels\EmailChannel; @@ -13,25 +11,28 @@ use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Notifications\Messages\MailMessage; use Illuminate\Notifications\Notification; -use Illuminate\Support\Facades\RateLimiter; -class Revived extends Notification implements ShouldQueue +class Reachable extends Notification implements ShouldQueue { use Queueable; public $tries = 1; + protected bool $isRateLimited = false; + public function __construct(public Server $server) { - if ($this->server->unreachable_notification_sent === false) { - return; - } - GetContainersStatus::dispatch($server)->onQueue('high'); - // dispatch(new ContainerStatusJob($server)); + $this->isRateLimited = isEmailRateLimited( + limiterKey: 'server-reachable:'.$this->server->id, + ); } public function via(object $notifiable): array { + if ($this->isRateLimited) { + return []; + } + $channels = []; $isEmailEnabled = isEmailEnabled($notifiable); $isDiscordEnabled = data_get($notifiable, 'discord_enabled'); @@ -46,20 +47,8 @@ class Revived extends Notification implements ShouldQueue if ($isTelegramEnabled) { $channels[] = TelegramChannel::class; } - $executed = RateLimiter::attempt( - 'notification-server-revived-'.$this->server->uuid, - 1, - function () use ($channels) { - return $channels; - }, - 7200, - ); - if (! $executed) { - return []; - } - - return $executed; + return $channels; } public function toMail(): MailMessage diff --git a/app/Notifications/Server/Unreachable.php b/app/Notifications/Server/Unreachable.php index 65ea6a2ff..5bc568e82 100644 --- a/app/Notifications/Server/Unreachable.php +++ b/app/Notifications/Server/Unreachable.php @@ -11,7 +11,6 @@ use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Notifications\Messages\MailMessage; use Illuminate\Notifications\Notification; -use Illuminate\Support\Facades\RateLimiter; class Unreachable extends Notification implements ShouldQueue { @@ -19,10 +18,21 @@ class Unreachable extends Notification implements ShouldQueue public $tries = 1; - public function __construct(public Server $server) {} + protected bool $isRateLimited = false; + + public function __construct(public Server $server) + { + $this->isRateLimited = isEmailRateLimited( + limiterKey: 'server-unreachable:'.$this->server->id, + ); + } public function via(object $notifiable): array { + if ($this->isRateLimited) { + return []; + } + $channels = []; $isEmailEnabled = isEmailEnabled($notifiable); $isDiscordEnabled = data_get($notifiable, 'discord_enabled'); @@ -37,23 +47,11 @@ class Unreachable extends Notification implements ShouldQueue if ($isTelegramEnabled) { $channels[] = TelegramChannel::class; } - $executed = RateLimiter::attempt( - 'notification-server-unreachable-'.$this->server->uuid, - 1, - function () use ($channels) { - return $channels; - }, - 7200, - ); - if (! $executed) { - return []; - } - - return $executed; + return $channels; } - public function toMail(): MailMessage + public function toMail(): ?MailMessage { $mail = new MailMessage; $mail->subject("Coolify: Your server ({$this->server->name}) is unreachable."); @@ -64,7 +62,7 @@ class Unreachable extends Notification implements ShouldQueue return $mail; } - public function toDiscord(): DiscordMessage + public function toDiscord(): ?DiscordMessage { $message = new DiscordMessage( title: ':cross_mark: Server unreachable', @@ -77,7 +75,7 @@ class Unreachable extends Notification implements ShouldQueue return $message; } - public function toTelegram(): array + public function toTelegram(): ?array { return [ 'message' => "Coolify: Your server '{$this->server->name}' is unreachable. All automations & integrations are turned off! Please check your server! IMPORTANT: We automatically try to revive your server and turn on all automations & integrations.", diff --git a/bootstrap/helpers/shared.php b/bootstrap/helpers/shared.php index 7ae9df749..dbab6861d 100644 --- a/bootstrap/helpers/shared.php +++ b/bootstrap/helpers/shared.php @@ -39,6 +39,7 @@ use Illuminate\Support\Facades\File; use Illuminate\Support\Facades\Http; use Illuminate\Support\Facades\Mail; use Illuminate\Support\Facades\Process; +use Illuminate\Support\Facades\RateLimiter; use Illuminate\Support\Facades\Request; use Illuminate\Support\Facades\Route; use Illuminate\Support\Facades\Validator; @@ -4038,3 +4039,30 @@ function sslipDomainWarning(string $domains) return $showSslipHttpsWarning; } + +function isEmailRateLimited(string $limiterKey, int $decaySeconds = 3600, ?callable $callbackOnSuccess = null): bool +{ + if (isDev()) { + $decaySeconds = 120; + } + $rateLimited = false; + $executed = RateLimiter::attempt( + $limiterKey, + $maxAttempts = 0, + function () use (&$rateLimited, &$limiterKey, $callbackOnSuccess) { + isDev() && loggy('Rate limit not reached for '.$limiterKey); + $rateLimited = false; + + if ($callbackOnSuccess) { + $callbackOnSuccess(); + } + }, + $decaySeconds, + ); + if (! $executed) { + isDev() && loggy('Rate limit reached for '.$limiterKey.'. Rate limiter will be disabled for '.$decaySeconds.' seconds.'); + $rateLimited = true; + } + + return $rateLimited; +} diff --git a/scripts/run b/scripts/run index f8ac0f97d..f7e7b5264 100755 --- a/scripts/run +++ b/scripts/run @@ -20,6 +20,9 @@ function help { compgen -A function | cat -n } +function logs { + docker exec -t coolify tail -f storage/logs/laravel.log +} function test { docker exec -t coolify php artisan test --testsuite=Feature } @@ -35,11 +38,6 @@ function db:reset-prod { bash spin exec -u webuser coolify php artisan migrate:fresh --force --seed --seeder=ProductionSeeder || php artisan migrate:fresh --force --seed --seeder=ProductionSeeder } - -function mfs { - db:reset -} - function coolify { bash spin exec -u webuser coolify bash }