From 55ff00e02839f4049cc7af1b3d7867abe680e324 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Wed, 15 Nov 2023 15:19:31 +0100 Subject: [PATCH 01/12] Add logging configuration to compose file --- app/Jobs/ApplicationDeploymentJob.php | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/app/Jobs/ApplicationDeploymentJob.php b/app/Jobs/ApplicationDeploymentJob.php index a846543c8..9c0475610 100644 --- a/app/Jobs/ApplicationDeploymentJob.php +++ b/app/Jobs/ApplicationDeploymentJob.php @@ -827,6 +827,13 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted 'networks' => [ $this->destination->network, ], + 'logging' => [ + 'driver' => 'fluentd', + 'options' => [ + 'fluentd-async' => 'true', + 'tag' => $this->application->name . '-' . $this->application->uuid + ] + ], 'healthcheck' => [ 'test' => [ 'CMD-SHELL', From ecbfc4d790f1814a8dbfd2fa74443c60717832be Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Wed, 15 Nov 2023 15:45:37 +0100 Subject: [PATCH 02/12] Add Fluent Bit and New Relic configurations --- examples/fluent-bit/fluent-bit.conf | 16 ++++++++++++++++ examples/fluent-bit/fluent-bit.yaml | 9 +++++++++ examples/newrelic.yaml | 21 +++++++++++++++++++++ 3 files changed, 46 insertions(+) create mode 100644 examples/fluent-bit/fluent-bit.conf create mode 100644 examples/fluent-bit/fluent-bit.yaml create mode 100644 examples/newrelic.yaml diff --git a/examples/fluent-bit/fluent-bit.conf b/examples/fluent-bit/fluent-bit.conf new file mode 100644 index 000000000..5d10f559a --- /dev/null +++ b/examples/fluent-bit/fluent-bit.conf @@ -0,0 +1,16 @@ +[SERVICE] + Flush 1 + Daemon off +[INPUT] + Name forward + Buffer_Chunk_Size 1M + Buffer_Max_Size 6M +# [OUTPUT] +# Name nrlogs +# Match * +# license_key ${LICENSE_KEY} +# base_uri https://log-api.eu.newrelic.com/log/v1 + +[OUTPUT] + Name stdout + Match * diff --git a/examples/fluent-bit/fluent-bit.yaml b/examples/fluent-bit/fluent-bit.yaml new file mode 100644 index 000000000..ca573635e --- /dev/null +++ b/examples/fluent-bit/fluent-bit.yaml @@ -0,0 +1,9 @@ +version: '3' +services: + coolify-fluent-bit: + image: cr.fluentbit.io/fluent/fluent-bit:2.0 + command: -c /fluent-bit.conf + volumes: + - ./fluent-bit.conf:/fluent-bit.conf + ports: + - 24224:24224 diff --git a/examples/newrelic.yaml b/examples/newrelic.yaml new file mode 100644 index 000000000..40bd5b0f2 --- /dev/null +++ b/examples/newrelic.yaml @@ -0,0 +1,21 @@ +version: '3' +services: + newrelic-infra: + container_name: newrelic-infra + image: newrelic/infrastructure:latest + networks: + - coolify + cap_add: + - SYS_PTRACE + privileged: true + pid: host + volumes: + - "/:/host:ro" + - "/var/run/docker.sock:/var/run/docker.sock" + - "newrelic-infra:/etc/newrelic-infra" + environment: + - NRIA_LICENSE_KEY=${NRIA_LICENSE_KEY} + - NRIA_DISPLAY_NAME=${HOSTNAME} + +networks: + coolify: From f5de21a3435f8fb6854d1a1e0adb722ba0be46a2 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Thu, 16 Nov 2023 11:16:41 +0100 Subject: [PATCH 03/12] Add OTLP exporter and host metrics receiver configuration to config.yaml. --- examples/otl/config.yaml | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 examples/otl/config.yaml diff --git a/examples/otl/config.yaml b/examples/otl/config.yaml new file mode 100644 index 000000000..a1b8b7ec4 --- /dev/null +++ b/examples/otl/config.yaml @@ -0,0 +1,34 @@ +receivers: + hostmetrics: + collection_interval: 5s + scrapers: + cpu: + metrics: + system.cpu.utilization: + enabled: true +processors: + resourcedetection: + detectors: [env, system] + system: + hostname_sources: ["os"] + resource_attributes: + host.id: + enabled: true + batch: + memory_limiter: + check_interval: 1s + limit_mib: 1000 + spike_limit_mib: 200 +exporters: + debug: + verbosity: detailed + otlp: + endpoint: ${OTLP_ENDPOINT} + headers: + api-key: ${OTLP_API_KEY} +service: + pipelines: + metrics: + receivers: [hostmetrics] + processors: [memory_limiter, resourcedetection, batch] + exporters: [debug, otlp] From 7cec6330cf2bb3f812e6974788305aa668923be4 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Thu, 16 Nov 2023 11:53:37 +0100 Subject: [PATCH 04/12] Update server status check and notifications --- app/Console/Kernel.php | 2 + app/Jobs/ContainerStatusJob.php | 71 +----------------- app/Jobs/DockerCleanupJob.php | 32 ++++----- app/Jobs/ServerStatusJob.php | 54 ++++++++++++++ app/Models/Server.php | 72 +++++++++++++++++++ app/Notifications/Server/HighDiskUsage.php | 68 ++++++++++++++++++ app/Notifications/Server/Revived.php | 2 +- bootstrap/helpers/remoteProcess.php | 6 +- config/sentry.php | 2 +- config/version.php | 2 +- ...01819_add_high_disk_usage_notification.php | 30 ++++++++ .../views/emails/high-disk-usage.blade.php | 7 ++ versions.json | 2 +- 13 files changed, 254 insertions(+), 96 deletions(-) create mode 100644 app/Jobs/ServerStatusJob.php create mode 100644 app/Notifications/Server/HighDiskUsage.php create mode 100644 database/migrations/2023_11_16_101819_add_high_disk_usage_notification.php create mode 100644 resources/views/emails/high-disk-usage.blade.php diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index c39cb626a..3439630d5 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -9,6 +9,7 @@ use App\Jobs\DockerCleanupJob; use App\Jobs\InstanceAutoUpdateJob; use App\Jobs\ContainerStatusJob; use App\Jobs\PullHelperImageJob; +use App\Jobs\ServerStatusJob; use App\Models\InstanceSettings; use App\Models\ScheduledDatabaseBackup; use App\Models\Server; @@ -67,6 +68,7 @@ class Kernel extends ConsoleKernel } foreach ($servers as $server) { $schedule->job(new ContainerStatusJob($server))->everyMinute()->onOneServer(); + $schedule->job(new ServerStatusJob($server))->everyFiveMinutes()->onOneServer(); } } private function instance_auto_update($schedule) diff --git a/app/Jobs/ContainerStatusJob.php b/app/Jobs/ContainerStatusJob.php index c4a6c02b6..25c1e40c9 100644 --- a/app/Jobs/ContainerStatusJob.php +++ b/app/Jobs/ContainerStatusJob.php @@ -41,76 +41,7 @@ class ContainerStatusJob implements ShouldQueue, ShouldBeEncrypted { // ray("checking server status for {$this->server->id}"); try { - // ray()->clearAll(); - $serverUptimeCheckNumber = $this->server->unreachable_count; - $serverUptimeCheckNumberMax = 3; - - // ray('checking # ' . $serverUptimeCheckNumber); - if ($serverUptimeCheckNumber >= $serverUptimeCheckNumberMax) { - if ($this->server->unreachable_email_sent === false) { - ray('Server unreachable, sending notification...'); - $this->server->team->notify(new Unreachable($this->server)); - $this->server->update(['unreachable_email_sent' => true]); - } - $this->server->settings()->update([ - 'is_reachable' => false, - ]); - $this->server->update([ - 'unreachable_count' => 0, - ]); - // Update all applications, databases and services to exited - foreach ($this->server->applications() as $application) { - $application->update(['status' => 'exited']); - } - foreach ($this->server->databases() as $database) { - $database->update(['status' => 'exited']); - } - foreach ($this->server->services() as $service) { - $apps = $service->applications()->get(); - $dbs = $service->databases()->get(); - foreach ($apps as $app) { - $app->update(['status' => 'exited']); - } - foreach ($dbs as $db) { - $db->update(['status' => 'exited']); - } - } - return; - } - $result = $this->server->validateConnection(); - if ($result) { - $this->server->settings()->update([ - 'is_reachable' => true, - ]); - $this->server->update([ - 'unreachable_count' => 0, - ]); - } else { - $serverUptimeCheckNumber++; - $this->server->settings()->update([ - 'is_reachable' => false, - ]); - $this->server->update([ - 'unreachable_count' => $serverUptimeCheckNumber, - ]); - return; - } - - if (data_get($this->server, 'unreachable_email_sent') === true) { - ray('Server is reachable again, sending notification...'); - $this->server->team->notify(new Revived($this->server)); - $this->server->update(['unreachable_email_sent' => false]); - } - if ( - data_get($this->server, 'settings.is_reachable') === false || - data_get($this->server, 'settings.is_usable') === false - ) { - $this->server->settings()->update([ - 'is_reachable' => true, - 'is_usable' => true - ]); - } - // $this->server->validateDockerEngine(true); + $this->server->checkServerRediness(); $containers = instant_remote_process(["docker container ls -q"], $this->server); if (!$containers) { return; diff --git a/app/Jobs/DockerCleanupJob.php b/app/Jobs/DockerCleanupJob.php index cbdbab095..081c1d863 100644 --- a/app/Jobs/DockerCleanupJob.php +++ b/app/Jobs/DockerCleanupJob.php @@ -3,6 +3,7 @@ namespace App\Jobs; use App\Models\Server; +use App\Notifications\Server\HighDiskUsage; use Exception; use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldBeEncrypted; @@ -18,7 +19,6 @@ class DockerCleanupJob implements ShouldQueue, ShouldBeEncrypted use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; public $timeout = 300; - public ?string $dockerRootFilesystem = null; public ?int $usageBefore = null; public function __construct(public Server $server) @@ -26,28 +26,27 @@ class DockerCleanupJob implements ShouldQueue, ShouldBeEncrypted } public function handle(): void { - $isInprogress = false; - $this->server->applications()->each(function ($application) use (&$isInprogress) { - if ($application->isDeploymentInprogress()) { - $isInprogress = true; - return; - } - }); - if ($isInprogress) { - throw new Exception('DockerCleanupJob: ApplicationDeploymentQueue is not empty, skipping...'); - } try { + $isInprogress = false; + $this->server->applications()->each(function ($application) use (&$isInprogress) { + if ($application->isDeploymentInprogress()) { + $isInprogress = true; + return; + } + }); + if ($isInprogress) { + throw new Exception('DockerCleanupJob: ApplicationDeploymentQueue is not empty, skipping...'); + } if (!$this->server->isFunctional()) { return; } - $this->dockerRootFilesystem = "/"; - $this->usageBefore = $this->getFilesystemUsage(); + if ($this->usageBefore >= $this->server->settings->cleanup_after_percentage) { ray('Cleaning up ' . $this->server->name); instant_remote_process(['docker image prune -af'], $this->server); instant_remote_process(['docker container prune -f --filter "label=coolify.managed=true"'], $this->server); instant_remote_process(['docker builder prune -af'], $this->server); - $usageAfter = $this->getFilesystemUsage(); + $usageAfter = $this->server->getDiskUsage(); if ($usageAfter < $this->usageBefore) { ray('Saved ' . ($this->usageBefore - $usageAfter) . '% disk space on ' . $this->server->name); send_internal_notification('DockerCleanupJob done: Saved ' . ($this->usageBefore - $usageAfter) . '% disk space on ' . $this->server->name); @@ -65,9 +64,4 @@ class DockerCleanupJob implements ShouldQueue, ShouldBeEncrypted throw $e; } } - - private function getFilesystemUsage() - { - return instant_remote_process(["df '{$this->dockerRootFilesystem}'| tail -1 | awk '{ print $5}' | sed 's/%//g'"], $this->server, false); - } } diff --git a/app/Jobs/ServerStatusJob.php b/app/Jobs/ServerStatusJob.php new file mode 100644 index 000000000..1d9c4a682 --- /dev/null +++ b/app/Jobs/ServerStatusJob.php @@ -0,0 +1,54 @@ +server->id))->dontRelease()]; + } + + public function uniqueId(): int + { + return $this->server->id; + } + + public function handle(): void + { + ray("checking server status for {$this->server->id}"); + try { + + $this->server->checkServerRediness(); + $disk_usage = $this->server->getDiskUsage(); + ray($this->server->settings->cleanup_after_percentage); + if ($disk_usage >= $this->server->settings->cleanup_after_percentage) { + $this->server->high_disk_usage_notification_sent = true; + $this->server->save(); + $this->server->team->notify(new HighDiskUsage($this->server, $disk_usage, $this->server->settings->cleanup_after_percentage)); + } else { + $this->server->high_disk_usage_notification_sent = false; + $this->server->save(); + } + } catch (\Throwable $e) { + send_internal_notification('ServerStatusJob failed with: ' . $e->getMessage()); + ray($e->getMessage()); + handleError($e); + } + } +} diff --git a/app/Models/Server.php b/app/Models/Server.php index 6890c0fe7..b3544989d 100644 --- a/app/Models/Server.php +++ b/app/Models/Server.php @@ -4,8 +4,11 @@ namespace App\Models; use App\Enums\ProxyStatus; use App\Enums\ProxyTypes; +use App\Notifications\Server\Revived; +use App\Notifications\Server\Unreachable; use Illuminate\Database\Eloquent\Builder; use Illuminate\Database\Eloquent\Casts\Attribute; +use Illuminate\Support\Sleep; use Spatie\SchemalessAttributes\Casts\SchemalessAttributes; use Spatie\SchemalessAttributes\SchemalessAttributesTrait; use Illuminate\Support\Str; @@ -109,6 +112,75 @@ class Server extends BaseModel return $this->proxy->modelScope(); } + public function checkServerRediness() + { + $serverUptimeCheckNumber = $this->unreachable_count; + $serverUptimeCheckNumberMax = 5; + while (true) { + if ($serverUptimeCheckNumber >= $serverUptimeCheckNumberMax) { + if ($this->unreachable_notification_sent === false) { + ray('Server unreachable, sending notification...'); + $this->team->notify(new Unreachable($this)); + $this->update(['unreachable_notification_sent' => true]); + } + $this->settings()->update([ + 'is_reachable' => false, + ]); + $this->update([ + 'unreachable_count' => 0, + ]); + foreach ($this->applications() as $application) { + $application->update(['status' => 'exited']); + } + foreach ($this->databases() as $database) { + $database->update(['status' => 'exited']); + } + foreach ($this->services() as $service) { + $apps = $service->applications()->get(); + $dbs = $service->databases()->get(); + foreach ($apps as $app) { + $app->update(['status' => 'exited']); + } + foreach ($dbs as $db) { + $db->update(['status' => 'exited']); + } + } + throw new \Exception('Server is not reachable.'); + } + $result = $this->validateConnection(); + ray('validateConnection: ' . $result); + if (!$result) { + $serverUptimeCheckNumber++; + $this->update([ + 'unreachable_count' => $serverUptimeCheckNumber, + ]); + Sleep::for(5)->seconds(); + return; + } + $this->update([ + 'unreachable_count' => 0, + ]); + if (data_get($this, 'unreachable_notification_sent') === true) { + ray('Server is reachable again, sending notification...'); + $this->team->notify(new Revived($this)); + $this->update(['unreachable_notification_sent' => false]); + } + if ( + data_get($this, 'settings.is_reachable') === false || + data_get($this, 'settings.is_usable') === false + ) { + $this->settings()->update([ + 'is_reachable' => true, + 'is_usable' => true + ]); + } + break; + } + } + public function getDiskUsage() + { + return instant_remote_process(["df /| tail -1 | awk '{ print $5}' | sed 's/%//g'"], $this, false); + } public function hasDefinedResources() { $applications = $this->applications()->count() > 0; diff --git a/app/Notifications/Server/HighDiskUsage.php b/app/Notifications/Server/HighDiskUsage.php new file mode 100644 index 000000000..e638bc6c9 --- /dev/null +++ b/app/Notifications/Server/HighDiskUsage.php @@ -0,0 +1,68 @@ +server->high_disk_usage_notification_sent === false) { + return; + } + } + + public function via(object $notifiable): array + { + $channels = []; + $isEmailEnabled = isEmailEnabled($notifiable); + $isDiscordEnabled = data_get($notifiable, 'discord_enabled'); + $isTelegramEnabled = data_get($notifiable, 'telegram_enabled'); + + if ($isDiscordEnabled) { + $channels[] = DiscordChannel::class; + } + if ($isEmailEnabled) { + $channels[] = EmailChannel::class; + } + if ($isTelegramEnabled) { + $channels[] = TelegramChannel::class; + } + return $channels; + } + + public function toMail(): MailMessage + { + $mail = new MailMessage(); + $mail->subject("Coolify: Server ({$this->server->name}) high disk usage detected!"); + $mail->view('emails.high-disk-usage', [ + 'name' => $this->server->name, + 'disk_usage' => $this->disk_usage, + 'threshold' => $this->cleanup_after_percentage, + ]); + return $mail; + } + + public function toDiscord(): string + { + $message = "Coolify: Server '{$this->server->name}' high disk usage detected! \nDisk usage: {$this->disk_usage}"; + return $message; + } + public function toTelegram(): array + { + return [ + "message" => "Coolify: Server '{$this->server->name}' high disk usage detected! \n Disk usage: {$this->disk_usage}" + ]; + } +} diff --git a/app/Notifications/Server/Revived.php b/app/Notifications/Server/Revived.php index 21fe6d40d..400ef8377 100644 --- a/app/Notifications/Server/Revived.php +++ b/app/Notifications/Server/Revived.php @@ -18,7 +18,7 @@ class Revived extends Notification implements ShouldQueue public $tries = 1; public function __construct(public Server $server) { - if ($this->server->unreachable_email_sent === false) { + if ($this->server->unreachable_notification_sent === false) { return; } } diff --git a/bootstrap/helpers/remoteProcess.php b/bootstrap/helpers/remoteProcess.php index e3d263a11..c1ed577b5 100644 --- a/bootstrap/helpers/remoteProcess.php +++ b/bootstrap/helpers/remoteProcess.php @@ -191,7 +191,7 @@ function refresh_server_connection(?PrivateKey $private_key = null) // if (!$uptime) { // $server->settings->is_reachable = false; // $server->team->notify(new Unreachable($server)); -// $server->unreachable_email_sent = true; +// $server->unreachable_notification_sent = true; // $server->save(); // return [ // "uptime" => null, @@ -213,9 +213,9 @@ function refresh_server_connection(?PrivateKey $private_key = null) // $server->settings->is_usable = false; // } else { // $server->settings->is_usable = true; -// if (data_get($server, 'unreachable_email_sent') === true) { +// if (data_get($server, 'unreachable_notification_sent') === true) { // $server->team->notify(new Revived($server)); -// $server->unreachable_email_sent = false; +// $server->unreachable_notification_sent = false; // $server->save(); // } // } diff --git a/config/sentry.php b/config/sentry.php index 6e7ff7cbc..1afa9d1ea 100644 --- a/config/sentry.php +++ b/config/sentry.php @@ -7,7 +7,7 @@ return [ // The release version of your application // Example with dynamic git hash: trim(exec('git --git-dir ' . base_path('.git') . ' log --pretty="%h" -n1 HEAD')) - 'release' => '4.0.0-beta.136', + 'release' => '4.0.0-beta.137', // When left empty or `null` the Laravel environment will be used 'environment' => config('app.env'), diff --git a/config/version.php b/config/version.php index 781d975f1..20308c282 100644 --- a/config/version.php +++ b/config/version.php @@ -1,3 +1,3 @@ boolean('high_disk_usage_notification_sent')->default(false); + $table->renameColumn('unreachable_email_sent', 'unreachable_notification_sent'); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('servers', function (Blueprint $table) { + $table->dropColumn('high_disk_usage_notification_sent'); + $table->renameColumn('unreachable_notification_sent', 'unreachable_email_sent'); + }); + } +}; diff --git a/resources/views/emails/high-disk-usage.blade.php b/resources/views/emails/high-disk-usage.blade.php new file mode 100644 index 000000000..cff1590db --- /dev/null +++ b/resources/views/emails/high-disk-usage.blade.php @@ -0,0 +1,7 @@ + + +Your server ({{ $name }}) has high disk usage ({{ $disk_usage }}%). + +Threshold is {{ $threshold }}% (you can change it in the Server Settings menu). + + diff --git a/versions.json b/versions.json index 3b07acf3f..826bf73cc 100644 --- a/versions.json +++ b/versions.json @@ -4,7 +4,7 @@ "version": "3.12.36" }, "v4": { - "version": "4.0.0-beta.136" + "version": "4.0.0-beta.137" } } } From c866213f34513f4caa912a41f228efc6f2bb680a Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Thu, 16 Nov 2023 13:22:12 +0100 Subject: [PATCH 05/12] fix: when to pull image --- app/Jobs/ApplicationDeploymentJob.php | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/app/Jobs/ApplicationDeploymentJob.php b/app/Jobs/ApplicationDeploymentJob.php index 9c0475610..7851d3fe5 100644 --- a/app/Jobs/ApplicationDeploymentJob.php +++ b/app/Jobs/ApplicationDeploymentJob.php @@ -1096,21 +1096,19 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf"); private function start_by_compose_file() { - if ( - !$this->application->dockerfile && - ( - $this->application->build_pack === 'dockerimage' || - $this->application->build_pack === 'dockerfile') - ) { + if ($this->application->dockerfile || $this->application->build_pack === 'dockerfile') { + $this->execute_remote_command( + ["echo -n 'Starting application (could take a while).'"], + [executeInDocker($this->deployment_uuid, "docker compose --project-directory {$this->workdir} up --build -d"), "hidden" => true], + ); + } else if ($this->application->build_pack === 'dockerimage') { $this->execute_remote_command( ["echo -n 'Pulling latest images from the registry.'"], - [executeInDocker($this->deployment_uuid, "docker compose --project-directory {$this->workdir}"), "hidden" => true], + [executeInDocker($this->deployment_uuid, "docker compose --project-directory {$this->workdir} pull"), "hidden" => true], + ["echo -n 'Starting application (could take a while).'"], + [executeInDocker($this->deployment_uuid, "docker compose --project-directory {$this->workdir} up --build -d"), "hidden" => true], ); } - $this->execute_remote_command( - ["echo -n 'Starting application (could take a while).'"], - [executeInDocker($this->deployment_uuid, "docker compose --project-directory {$this->workdir} up --build -d"), "hidden" => true], - ); } private function generate_build_env_variables() From 4f0b2140424e7ebc51002043d68cbcc0b7fe7e9f Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Thu, 16 Nov 2023 13:27:51 +0100 Subject: [PATCH 06/12] Add timeout to ApplicationDeploymentJob --- app/Jobs/ApplicationDeploymentJob.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/Jobs/ApplicationDeploymentJob.php b/app/Jobs/ApplicationDeploymentJob.php index 7851d3fe5..0d22bf354 100644 --- a/app/Jobs/ApplicationDeploymentJob.php +++ b/app/Jobs/ApplicationDeploymentJob.php @@ -33,6 +33,8 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted { use Dispatchable, InteractsWithQueue, Queueable, SerializesModels, ExecuteRemoteCommand; + public $timeout = 3600; + public static int $batch_counter = 0; private int $application_deployment_queue_id; From 2fe429fe92c5ed01e21c07a4f6832763bcd8ed54 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Thu, 16 Nov 2023 13:32:07 +0100 Subject: [PATCH 07/12] Comment out logging configuration in ApplicationDeploymentJob.php --- app/Jobs/ApplicationDeploymentJob.php | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/app/Jobs/ApplicationDeploymentJob.php b/app/Jobs/ApplicationDeploymentJob.php index 0d22bf354..2f175553e 100644 --- a/app/Jobs/ApplicationDeploymentJob.php +++ b/app/Jobs/ApplicationDeploymentJob.php @@ -829,13 +829,13 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted 'networks' => [ $this->destination->network, ], - 'logging' => [ - 'driver' => 'fluentd', - 'options' => [ - 'fluentd-async' => 'true', - 'tag' => $this->application->name . '-' . $this->application->uuid - ] - ], + // 'logging' => [ + // 'driver' => 'fluentd', + // 'options' => [ + // 'fluentd-async' => 'true', + // 'tag' => $this->application->name . '-' . $this->application->uuid + // ] + // ], 'healthcheck' => [ 'test' => [ 'CMD-SHELL', From 81437e682232fb5b9d2be5f4ba4a4b7e79d72a35 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Thu, 16 Nov 2023 13:49:08 +0100 Subject: [PATCH 08/12] Fix high disk usage notification bug in ServerStatusJob.php and HighDiskUsage.php --- app/Jobs/ServerStatusJob.php | 6 ++++-- app/Notifications/Server/HighDiskUsage.php | 7 ++----- resources/views/emails/high-disk-usage.blade.php | 6 ++++-- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/app/Jobs/ServerStatusJob.php b/app/Jobs/ServerStatusJob.php index 1d9c4a682..d3599049d 100644 --- a/app/Jobs/ServerStatusJob.php +++ b/app/Jobs/ServerStatusJob.php @@ -33,11 +33,13 @@ class ServerStatusJob implements ShouldQueue, ShouldBeEncrypted { ray("checking server status for {$this->server->id}"); try { - $this->server->checkServerRediness(); $disk_usage = $this->server->getDiskUsage(); - ray($this->server->settings->cleanup_after_percentage); if ($disk_usage >= $this->server->settings->cleanup_after_percentage) { + if ($this->server->high_disk_usage_notification_sent) { + ray('high disk usage notification already sent'); + return; + } $this->server->high_disk_usage_notification_sent = true; $this->server->save(); $this->server->team->notify(new HighDiskUsage($this->server, $disk_usage, $this->server->settings->cleanup_after_percentage)); diff --git a/app/Notifications/Server/HighDiskUsage.php b/app/Notifications/Server/HighDiskUsage.php index e638bc6c9..d23a2a0a4 100644 --- a/app/Notifications/Server/HighDiskUsage.php +++ b/app/Notifications/Server/HighDiskUsage.php @@ -18,9 +18,6 @@ class HighDiskUsage extends Notification implements ShouldQueue public $tries = 1; public function __construct(public Server $server, public int $disk_usage, public int $cleanup_after_percentage) { - if ($this->server->high_disk_usage_notification_sent === false) { - return; - } } public function via(object $notifiable): array @@ -56,13 +53,13 @@ class HighDiskUsage extends Notification implements ShouldQueue public function toDiscord(): string { - $message = "Coolify: Server '{$this->server->name}' high disk usage detected! \nDisk usage: {$this->disk_usage}"; + $message = "Coolify: Server '{$this->server->name}' high disk usage detected! \nDisk usage: {$this->disk_usage}. Please cleanup your disk to prevent data-loss. Here are some tips: https://coolify.io/docs/automated-cleanup."; return $message; } public function toTelegram(): array { return [ - "message" => "Coolify: Server '{$this->server->name}' high disk usage detected! \n Disk usage: {$this->disk_usage}" + "message" => "Coolify: Server '{$this->server->name}' high disk usage detected! \n Disk usage: {$this->disk_usage}. Please cleanup your disk to prevent data-loss. Here are some tips: https://coolify.io/docs/automated-cleanup." ]; } } diff --git a/resources/views/emails/high-disk-usage.blade.php b/resources/views/emails/high-disk-usage.blade.php index cff1590db..0a4baa300 100644 --- a/resources/views/emails/high-disk-usage.blade.php +++ b/resources/views/emails/high-disk-usage.blade.php @@ -1,7 +1,9 @@ -Your server ({{ $name }}) has high disk usage ({{ $disk_usage }}%). +Your server ({{ $name }}) has high disk usage ({{ $disk_usage }}% used). Threshold is {{ $threshold }}%. -Threshold is {{ $threshold }}% (you can change it in the Server Settings menu). +Please cleanup your disk to prevent data-loss. Here are some [tips](https://coolify.io/docs/automated-cleanup). + +(You can change the threshold in the Server Settings menu.) From fb42c439535633ababaa0d8f7050ad81584bd308 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Thu, 16 Nov 2023 14:28:26 +0100 Subject: [PATCH 09/12] Add isLocalhost method to Server model and conditionally show Cloudflare Tunnel checkbox in server form view --- app/Models/Server.php | 6 ++++++ resources/views/livewire/server/form.blade.php | 12 +++++++----- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/app/Models/Server.php b/app/Models/Server.php index b3544989d..3ba40ecc2 100644 --- a/app/Models/Server.php +++ b/app/Models/Server.php @@ -112,6 +112,12 @@ class Server extends BaseModel return $this->proxy->modelScope(); } + public function isLocalhost() { + if (isDev()) { + return $this->ip === 'coolify-testing-host'; + } + return $this->ip === 'host.docker.internal'; + } public function checkServerRediness() { $serverUptimeCheckNumber = $this->unreachable_count; diff --git a/resources/views/livewire/server/form.blade.php b/resources/views/livewire/server/form.blade.php index c9ed332de..c36d34cbc 100644 --- a/resources/views/livewire/server/form.blade.php +++ b/resources/views/livewire/server/form.blade.php @@ -49,11 +49,13 @@ -
- -
+ @if (!$server->isLocalhost()) +
+ +
+ @endif @if ($server->isFunctional()) From 2b666ff121ed6bc28f597dc10e6aebb483084acd Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Thu, 16 Nov 2023 14:29:01 +0100 Subject: [PATCH 10/12] Refactor server and docker cleanup jobs --- app/Console/Kernel.php | 13 +------- app/Jobs/DockerCleanupJob.php | 9 +++--- app/Jobs/ServerStatusJob.php | 37 ++++++++++++++-------- app/Models/Server.php | 16 ++++++---- app/Models/ServiceDatabase.php | 2 +- app/Notifications/Server/HighDiskUsage.php | 4 +-- 6 files changed, 42 insertions(+), 39 deletions(-) diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index 3439630d5..e0d9f2752 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -2,10 +2,8 @@ namespace App\Console; -use App\Jobs\CheckResaleLicenseJob; use App\Jobs\CleanupInstanceStuffsJob; use App\Jobs\DatabaseBackupJob; -use App\Jobs\DockerCleanupJob; use App\Jobs\InstanceAutoUpdateJob; use App\Jobs\ContainerStatusJob; use App\Jobs\PullHelperImageJob; @@ -28,7 +26,6 @@ class Kernel extends ConsoleKernel // Server Jobs $this->check_scheduled_backups($schedule); $this->check_resources($schedule); - $this->cleanup_servers($schedule); $this->check_scheduled_backups($schedule); $this->pull_helper_image($schedule); } else { @@ -41,7 +38,6 @@ class Kernel extends ConsoleKernel $this->instance_auto_update($schedule); $this->check_scheduled_backups($schedule); $this->check_resources($schedule); - $this->cleanup_servers($schedule); $this->pull_helper_image($schedule); } } @@ -52,13 +48,6 @@ class Kernel extends ConsoleKernel $schedule->job(new PullHelperImageJob($server))->everyTenMinutes()->onOneServer(); } } - private function cleanup_servers($schedule) - { - $servers = Server::all()->where('settings.is_usable', true)->where('settings.is_reachable', true); - foreach ($servers as $server) { - $schedule->job(new DockerCleanupJob($server))->everyTenMinutes()->onOneServer(); - } - } private function check_resources($schedule) { if (isCloud()) { @@ -67,8 +56,8 @@ class Kernel extends ConsoleKernel $servers = Server::all(); } foreach ($servers as $server) { + $schedule->job(new ServerStatusJob($server))->everyTenMinutes()->onOneServer(); $schedule->job(new ContainerStatusJob($server))->everyMinute()->onOneServer(); - $schedule->job(new ServerStatusJob($server))->everyFiveMinutes()->onOneServer(); } } private function instance_auto_update($schedule) diff --git a/app/Jobs/DockerCleanupJob.php b/app/Jobs/DockerCleanupJob.php index 081c1d863..14ca11b22 100644 --- a/app/Jobs/DockerCleanupJob.php +++ b/app/Jobs/DockerCleanupJob.php @@ -40,12 +40,13 @@ class DockerCleanupJob implements ShouldQueue, ShouldBeEncrypted if (!$this->server->isFunctional()) { return; } - + $this->usageBefore = $this->server->getDiskUsage(); + ray('Usage before: ' . $this->usageBefore); if ($this->usageBefore >= $this->server->settings->cleanup_after_percentage) { ray('Cleaning up ' . $this->server->name); - instant_remote_process(['docker image prune -af'], $this->server); - instant_remote_process(['docker container prune -f --filter "label=coolify.managed=true"'], $this->server); - instant_remote_process(['docker builder prune -af'], $this->server); + instant_remote_process(['docker image prune -af'], $this->server, false); + instant_remote_process(['docker container prune -f --filter "label=coolify.managed=true"'], $this->server, false); + instant_remote_process(['docker builder prune -af'], $this->server, false); $usageAfter = $this->server->getDiskUsage(); if ($usageAfter < $this->usageBefore) { ray('Saved ' . ($this->usageBefore - $usageAfter) . '% disk space on ' . $this->server->name); diff --git a/app/Jobs/ServerStatusJob.php b/app/Jobs/ServerStatusJob.php index d3599049d..7d591bc83 100644 --- a/app/Jobs/ServerStatusJob.php +++ b/app/Jobs/ServerStatusJob.php @@ -16,6 +16,7 @@ class ServerStatusJob implements ShouldQueue, ShouldBeEncrypted { use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; + public ?int $disk_usage = null; public function __construct(public Server $server) { } @@ -34,23 +35,33 @@ class ServerStatusJob implements ShouldQueue, ShouldBeEncrypted ray("checking server status for {$this->server->id}"); try { $this->server->checkServerRediness(); - $disk_usage = $this->server->getDiskUsage(); - if ($disk_usage >= $this->server->settings->cleanup_after_percentage) { - if ($this->server->high_disk_usage_notification_sent) { - ray('high disk usage notification already sent'); - return; - } - $this->server->high_disk_usage_notification_sent = true; - $this->server->save(); - $this->server->team->notify(new HighDiskUsage($this->server, $disk_usage, $this->server->settings->cleanup_after_percentage)); - } else { - $this->server->high_disk_usage_notification_sent = false; - $this->server->save(); - } + $this->cleanup(notify: false); } catch (\Throwable $e) { send_internal_notification('ServerStatusJob failed with: ' . $e->getMessage()); ray($e->getMessage()); handleError($e); } } + public function cleanup(bool $notify = false): void + { + $this->disk_usage = $this->server->getDiskUsage(); + if ($this->disk_usage >= $this->server->settings->cleanup_after_percentage) { + if ($notify) { + if ($this->server->high_disk_usage_notification_sent) { + ray('high disk usage notification already sent'); + return; + } else { + $this->server->high_disk_usage_notification_sent = true; + $this->server->save(); + $this->server->team->notify(new HighDiskUsage($this->server, $this->disk_usage, $this->server->settings->cleanup_after_percentage)); + } + } else { + DockerCleanupJob::dispatchSync($this->server); + $this->cleanup(notify: true); + } + } else { + $this->server->high_disk_usage_notification_sent = false; + $this->server->save(); + } + } } diff --git a/app/Models/Server.php b/app/Models/Server.php index 3ba40ecc2..ff6af7c46 100644 --- a/app/Models/Server.php +++ b/app/Models/Server.php @@ -112,17 +112,19 @@ class Server extends BaseModel return $this->proxy->modelScope(); } - public function isLocalhost() { - if (isDev()) { - return $this->ip === 'coolify-testing-host'; - } - return $this->ip === 'host.docker.internal'; + public function isLocalhost() + { + return $this->ip === 'host.docker.internal' || $this->id === 0; } public function checkServerRediness() { $serverUptimeCheckNumber = $this->unreachable_count; $serverUptimeCheckNumberMax = 5; - while (true) { + + $currentTime = now()->timestamp; + $runtime5Minutes = 5 * 60; + // Run for 5 minutes max and check every 5 seconds + while ($currentTime + $runtime5Minutes > now()->timestamp) { if ($serverUptimeCheckNumber >= $serverUptimeCheckNumberMax) { if ($this->unreachable_notification_sent === false) { ray('Server unreachable, sending notification...'); @@ -226,7 +228,7 @@ class Server extends BaseModel if (isDev()) { return '127.0.0.1'; } - if ($this->ip === 'host.docker.internal') { + if ($this->isLocalhost()) { return base_ip(); } return $this->ip; diff --git a/app/Models/ServiceDatabase.php b/app/Models/ServiceDatabase.php index c86334e39..2feaeb947 100644 --- a/app/Models/ServiceDatabase.php +++ b/app/Models/ServiceDatabase.php @@ -36,7 +36,7 @@ class ServiceDatabase extends BaseModel { $port = $this->public_port; $realIp = $this->service->server->ip; - if ($realIp === 'host.docker.internal' || isDev()) { + if ($this->service->server->isLocalhost() || isDev()) { $realIp = base_ip(); } $url = "{$realIp}:{$port}"; diff --git a/app/Notifications/Server/HighDiskUsage.php b/app/Notifications/Server/HighDiskUsage.php index d23a2a0a4..d8794600d 100644 --- a/app/Notifications/Server/HighDiskUsage.php +++ b/app/Notifications/Server/HighDiskUsage.php @@ -53,13 +53,13 @@ class HighDiskUsage extends Notification implements ShouldQueue public function toDiscord(): string { - $message = "Coolify: Server '{$this->server->name}' high disk usage detected! \nDisk usage: {$this->disk_usage}. Please cleanup your disk to prevent data-loss. Here are some tips: https://coolify.io/docs/automated-cleanup."; + $message = "Coolify: Server '{$this->server->name}' high disk usage detected!\nDisk usage: {$this->disk_usage}%. Threshold: {$this->cleanup_after_percentage}%.\nPlease cleanup your disk to prevent data-loss.\nHere are some tips: https://coolify.io/docs/automated-cleanup."; return $message; } public function toTelegram(): array { return [ - "message" => "Coolify: Server '{$this->server->name}' high disk usage detected! \n Disk usage: {$this->disk_usage}. Please cleanup your disk to prevent data-loss. Here are some tips: https://coolify.io/docs/automated-cleanup." + "message" => "Coolify: Server '{$this->server->name}' high disk usage detected!\nDisk usage: {$this->disk_usage}%. Threshold: {$this->cleanup_after_percentage}%.\nPlease cleanup your disk to prevent data-loss.\nHere are some tips: https://coolify.io/docs/automated-cleanup." ]; } } From d56c28c8d905ec7a151dfb17d88e0e240cea297e Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Thu, 16 Nov 2023 14:29:23 +0100 Subject: [PATCH 11/12] Remove unused notifications from ContainerStatusJob --- app/Jobs/ContainerStatusJob.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/Jobs/ContainerStatusJob.php b/app/Jobs/ContainerStatusJob.php index 25c1e40c9..74d300c38 100644 --- a/app/Jobs/ContainerStatusJob.php +++ b/app/Jobs/ContainerStatusJob.php @@ -8,8 +8,6 @@ use App\Models\ApplicationPreview; use App\Models\Server; use App\Notifications\Container\ContainerRestarted; use App\Notifications\Container\ContainerStopped; -use App\Notifications\Server\Revived; -use App\Notifications\Server\Unreachable; use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldBeEncrypted; use Illuminate\Contracts\Queue\ShouldQueue; From 5c0f239f62ff54e714d8dd59e5ef5761ee5cdb65 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Thu, 16 Nov 2023 14:36:43 +0100 Subject: [PATCH 12/12] Update server readiness check runtime to 1 minute --- app/Models/Server.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/Models/Server.php b/app/Models/Server.php index ff6af7c46..d1b11a080 100644 --- a/app/Models/Server.php +++ b/app/Models/Server.php @@ -122,8 +122,8 @@ class Server extends BaseModel $serverUptimeCheckNumberMax = 5; $currentTime = now()->timestamp; - $runtime5Minutes = 5 * 60; - // Run for 5 minutes max and check every 5 seconds + $runtime5Minutes = 1 * 60; + // Run for 1 minutes max and check every 5 seconds while ($currentTime + $runtime5Minutes > now()->timestamp) { if ($serverUptimeCheckNumber >= $serverUptimeCheckNumberMax) { if ($this->unreachable_notification_sent === false) {