diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index c39cb626a..e0d9f2752 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -2,13 +2,12 @@ namespace App\Console; -use App\Jobs\CheckResaleLicenseJob; use App\Jobs\CleanupInstanceStuffsJob; use App\Jobs\DatabaseBackupJob; -use App\Jobs\DockerCleanupJob; use App\Jobs\InstanceAutoUpdateJob; use App\Jobs\ContainerStatusJob; use App\Jobs\PullHelperImageJob; +use App\Jobs\ServerStatusJob; use App\Models\InstanceSettings; use App\Models\ScheduledDatabaseBackup; use App\Models\Server; @@ -27,7 +26,6 @@ class Kernel extends ConsoleKernel // Server Jobs $this->check_scheduled_backups($schedule); $this->check_resources($schedule); - $this->cleanup_servers($schedule); $this->check_scheduled_backups($schedule); $this->pull_helper_image($schedule); } else { @@ -40,7 +38,6 @@ class Kernel extends ConsoleKernel $this->instance_auto_update($schedule); $this->check_scheduled_backups($schedule); $this->check_resources($schedule); - $this->cleanup_servers($schedule); $this->pull_helper_image($schedule); } } @@ -51,13 +48,6 @@ class Kernel extends ConsoleKernel $schedule->job(new PullHelperImageJob($server))->everyTenMinutes()->onOneServer(); } } - private function cleanup_servers($schedule) - { - $servers = Server::all()->where('settings.is_usable', true)->where('settings.is_reachable', true); - foreach ($servers as $server) { - $schedule->job(new DockerCleanupJob($server))->everyTenMinutes()->onOneServer(); - } - } private function check_resources($schedule) { if (isCloud()) { @@ -66,6 +56,7 @@ class Kernel extends ConsoleKernel $servers = Server::all(); } foreach ($servers as $server) { + $schedule->job(new ServerStatusJob($server))->everyTenMinutes()->onOneServer(); $schedule->job(new ContainerStatusJob($server))->everyMinute()->onOneServer(); } } diff --git a/app/Jobs/ApplicationDeploymentJob.php b/app/Jobs/ApplicationDeploymentJob.php index a846543c8..2f175553e 100644 --- a/app/Jobs/ApplicationDeploymentJob.php +++ b/app/Jobs/ApplicationDeploymentJob.php @@ -33,6 +33,8 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted { use Dispatchable, InteractsWithQueue, Queueable, SerializesModels, ExecuteRemoteCommand; + public $timeout = 3600; + public static int $batch_counter = 0; private int $application_deployment_queue_id; @@ -827,6 +829,13 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted 'networks' => [ $this->destination->network, ], + // 'logging' => [ + // 'driver' => 'fluentd', + // 'options' => [ + // 'fluentd-async' => 'true', + // 'tag' => $this->application->name . '-' . $this->application->uuid + // ] + // ], 'healthcheck' => [ 'test' => [ 'CMD-SHELL', @@ -1089,21 +1098,19 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf"); private function start_by_compose_file() { - if ( - !$this->application->dockerfile && - ( - $this->application->build_pack === 'dockerimage' || - $this->application->build_pack === 'dockerfile') - ) { + if ($this->application->dockerfile || $this->application->build_pack === 'dockerfile') { + $this->execute_remote_command( + ["echo -n 'Starting application (could take a while).'"], + [executeInDocker($this->deployment_uuid, "docker compose --project-directory {$this->workdir} up --build -d"), "hidden" => true], + ); + } else if ($this->application->build_pack === 'dockerimage') { $this->execute_remote_command( ["echo -n 'Pulling latest images from the registry.'"], - [executeInDocker($this->deployment_uuid, "docker compose --project-directory {$this->workdir}"), "hidden" => true], + [executeInDocker($this->deployment_uuid, "docker compose --project-directory {$this->workdir} pull"), "hidden" => true], + ["echo -n 'Starting application (could take a while).'"], + [executeInDocker($this->deployment_uuid, "docker compose --project-directory {$this->workdir} up --build -d"), "hidden" => true], ); } - $this->execute_remote_command( - ["echo -n 'Starting application (could take a while).'"], - [executeInDocker($this->deployment_uuid, "docker compose --project-directory {$this->workdir} up --build -d"), "hidden" => true], - ); } private function generate_build_env_variables() diff --git a/app/Jobs/ContainerStatusJob.php b/app/Jobs/ContainerStatusJob.php index c4a6c02b6..74d300c38 100644 --- a/app/Jobs/ContainerStatusJob.php +++ b/app/Jobs/ContainerStatusJob.php @@ -8,8 +8,6 @@ use App\Models\ApplicationPreview; use App\Models\Server; use App\Notifications\Container\ContainerRestarted; use App\Notifications\Container\ContainerStopped; -use App\Notifications\Server\Revived; -use App\Notifications\Server\Unreachable; use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldBeEncrypted; use Illuminate\Contracts\Queue\ShouldQueue; @@ -41,76 +39,7 @@ class ContainerStatusJob implements ShouldQueue, ShouldBeEncrypted { // ray("checking server status for {$this->server->id}"); try { - // ray()->clearAll(); - $serverUptimeCheckNumber = $this->server->unreachable_count; - $serverUptimeCheckNumberMax = 3; - - // ray('checking # ' . $serverUptimeCheckNumber); - if ($serverUptimeCheckNumber >= $serverUptimeCheckNumberMax) { - if ($this->server->unreachable_email_sent === false) { - ray('Server unreachable, sending notification...'); - $this->server->team->notify(new Unreachable($this->server)); - $this->server->update(['unreachable_email_sent' => true]); - } - $this->server->settings()->update([ - 'is_reachable' => false, - ]); - $this->server->update([ - 'unreachable_count' => 0, - ]); - // Update all applications, databases and services to exited - foreach ($this->server->applications() as $application) { - $application->update(['status' => 'exited']); - } - foreach ($this->server->databases() as $database) { - $database->update(['status' => 'exited']); - } - foreach ($this->server->services() as $service) { - $apps = $service->applications()->get(); - $dbs = $service->databases()->get(); - foreach ($apps as $app) { - $app->update(['status' => 'exited']); - } - foreach ($dbs as $db) { - $db->update(['status' => 'exited']); - } - } - return; - } - $result = $this->server->validateConnection(); - if ($result) { - $this->server->settings()->update([ - 'is_reachable' => true, - ]); - $this->server->update([ - 'unreachable_count' => 0, - ]); - } else { - $serverUptimeCheckNumber++; - $this->server->settings()->update([ - 'is_reachable' => false, - ]); - $this->server->update([ - 'unreachable_count' => $serverUptimeCheckNumber, - ]); - return; - } - - if (data_get($this->server, 'unreachable_email_sent') === true) { - ray('Server is reachable again, sending notification...'); - $this->server->team->notify(new Revived($this->server)); - $this->server->update(['unreachable_email_sent' => false]); - } - if ( - data_get($this->server, 'settings.is_reachable') === false || - data_get($this->server, 'settings.is_usable') === false - ) { - $this->server->settings()->update([ - 'is_reachable' => true, - 'is_usable' => true - ]); - } - // $this->server->validateDockerEngine(true); + $this->server->checkServerRediness(); $containers = instant_remote_process(["docker container ls -q"], $this->server); if (!$containers) { return; diff --git a/app/Jobs/DockerCleanupJob.php b/app/Jobs/DockerCleanupJob.php index cbdbab095..14ca11b22 100644 --- a/app/Jobs/DockerCleanupJob.php +++ b/app/Jobs/DockerCleanupJob.php @@ -3,6 +3,7 @@ namespace App\Jobs; use App\Models\Server; +use App\Notifications\Server\HighDiskUsage; use Exception; use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldBeEncrypted; @@ -18,7 +19,6 @@ class DockerCleanupJob implements ShouldQueue, ShouldBeEncrypted use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; public $timeout = 300; - public ?string $dockerRootFilesystem = null; public ?int $usageBefore = null; public function __construct(public Server $server) @@ -26,28 +26,28 @@ class DockerCleanupJob implements ShouldQueue, ShouldBeEncrypted } public function handle(): void { - $isInprogress = false; - $this->server->applications()->each(function ($application) use (&$isInprogress) { - if ($application->isDeploymentInprogress()) { - $isInprogress = true; - return; - } - }); - if ($isInprogress) { - throw new Exception('DockerCleanupJob: ApplicationDeploymentQueue is not empty, skipping...'); - } try { + $isInprogress = false; + $this->server->applications()->each(function ($application) use (&$isInprogress) { + if ($application->isDeploymentInprogress()) { + $isInprogress = true; + return; + } + }); + if ($isInprogress) { + throw new Exception('DockerCleanupJob: ApplicationDeploymentQueue is not empty, skipping...'); + } if (!$this->server->isFunctional()) { return; } - $this->dockerRootFilesystem = "/"; - $this->usageBefore = $this->getFilesystemUsage(); + $this->usageBefore = $this->server->getDiskUsage(); + ray('Usage before: ' . $this->usageBefore); if ($this->usageBefore >= $this->server->settings->cleanup_after_percentage) { ray('Cleaning up ' . $this->server->name); - instant_remote_process(['docker image prune -af'], $this->server); - instant_remote_process(['docker container prune -f --filter "label=coolify.managed=true"'], $this->server); - instant_remote_process(['docker builder prune -af'], $this->server); - $usageAfter = $this->getFilesystemUsage(); + instant_remote_process(['docker image prune -af'], $this->server, false); + instant_remote_process(['docker container prune -f --filter "label=coolify.managed=true"'], $this->server, false); + instant_remote_process(['docker builder prune -af'], $this->server, false); + $usageAfter = $this->server->getDiskUsage(); if ($usageAfter < $this->usageBefore) { ray('Saved ' . ($this->usageBefore - $usageAfter) . '% disk space on ' . $this->server->name); send_internal_notification('DockerCleanupJob done: Saved ' . ($this->usageBefore - $usageAfter) . '% disk space on ' . $this->server->name); @@ -65,9 +65,4 @@ class DockerCleanupJob implements ShouldQueue, ShouldBeEncrypted throw $e; } } - - private function getFilesystemUsage() - { - return instant_remote_process(["df '{$this->dockerRootFilesystem}'| tail -1 | awk '{ print $5}' | sed 's/%//g'"], $this->server, false); - } } diff --git a/app/Jobs/ServerStatusJob.php b/app/Jobs/ServerStatusJob.php new file mode 100644 index 000000000..7d591bc83 --- /dev/null +++ b/app/Jobs/ServerStatusJob.php @@ -0,0 +1,67 @@ +server->id))->dontRelease()]; + } + + public function uniqueId(): int + { + return $this->server->id; + } + + public function handle(): void + { + ray("checking server status for {$this->server->id}"); + try { + $this->server->checkServerRediness(); + $this->cleanup(notify: false); + } catch (\Throwable $e) { + send_internal_notification('ServerStatusJob failed with: ' . $e->getMessage()); + ray($e->getMessage()); + handleError($e); + } + } + public function cleanup(bool $notify = false): void + { + $this->disk_usage = $this->server->getDiskUsage(); + if ($this->disk_usage >= $this->server->settings->cleanup_after_percentage) { + if ($notify) { + if ($this->server->high_disk_usage_notification_sent) { + ray('high disk usage notification already sent'); + return; + } else { + $this->server->high_disk_usage_notification_sent = true; + $this->server->save(); + $this->server->team->notify(new HighDiskUsage($this->server, $this->disk_usage, $this->server->settings->cleanup_after_percentage)); + } + } else { + DockerCleanupJob::dispatchSync($this->server); + $this->cleanup(notify: true); + } + } else { + $this->server->high_disk_usage_notification_sent = false; + $this->server->save(); + } + } +} diff --git a/app/Models/Server.php b/app/Models/Server.php index 6890c0fe7..d1b11a080 100644 --- a/app/Models/Server.php +++ b/app/Models/Server.php @@ -4,8 +4,11 @@ namespace App\Models; use App\Enums\ProxyStatus; use App\Enums\ProxyTypes; +use App\Notifications\Server\Revived; +use App\Notifications\Server\Unreachable; use Illuminate\Database\Eloquent\Builder; use Illuminate\Database\Eloquent\Casts\Attribute; +use Illuminate\Support\Sleep; use Spatie\SchemalessAttributes\Casts\SchemalessAttributes; use Spatie\SchemalessAttributes\SchemalessAttributesTrait; use Illuminate\Support\Str; @@ -109,6 +112,83 @@ class Server extends BaseModel return $this->proxy->modelScope(); } + public function isLocalhost() + { + return $this->ip === 'host.docker.internal' || $this->id === 0; + } + public function checkServerRediness() + { + $serverUptimeCheckNumber = $this->unreachable_count; + $serverUptimeCheckNumberMax = 5; + + $currentTime = now()->timestamp; + $runtime5Minutes = 1 * 60; + // Run for 1 minutes max and check every 5 seconds + while ($currentTime + $runtime5Minutes > now()->timestamp) { + if ($serverUptimeCheckNumber >= $serverUptimeCheckNumberMax) { + if ($this->unreachable_notification_sent === false) { + ray('Server unreachable, sending notification...'); + $this->team->notify(new Unreachable($this)); + $this->update(['unreachable_notification_sent' => true]); + } + $this->settings()->update([ + 'is_reachable' => false, + ]); + $this->update([ + 'unreachable_count' => 0, + ]); + foreach ($this->applications() as $application) { + $application->update(['status' => 'exited']); + } + foreach ($this->databases() as $database) { + $database->update(['status' => 'exited']); + } + foreach ($this->services() as $service) { + $apps = $service->applications()->get(); + $dbs = $service->databases()->get(); + foreach ($apps as $app) { + $app->update(['status' => 'exited']); + } + foreach ($dbs as $db) { + $db->update(['status' => 'exited']); + } + } + throw new \Exception('Server is not reachable.'); + } + $result = $this->validateConnection(); + ray('validateConnection: ' . $result); + if (!$result) { + $serverUptimeCheckNumber++; + $this->update([ + 'unreachable_count' => $serverUptimeCheckNumber, + ]); + Sleep::for(5)->seconds(); + return; + } + $this->update([ + 'unreachable_count' => 0, + ]); + if (data_get($this, 'unreachable_notification_sent') === true) { + ray('Server is reachable again, sending notification...'); + $this->team->notify(new Revived($this)); + $this->update(['unreachable_notification_sent' => false]); + } + if ( + data_get($this, 'settings.is_reachable') === false || + data_get($this, 'settings.is_usable') === false + ) { + $this->settings()->update([ + 'is_reachable' => true, + 'is_usable' => true + ]); + } + break; + } + } + public function getDiskUsage() + { + return instant_remote_process(["df /| tail -1 | awk '{ print $5}' | sed 's/%//g'"], $this, false); + } public function hasDefinedResources() { $applications = $this->applications()->count() > 0; @@ -148,7 +228,7 @@ class Server extends BaseModel if (isDev()) { return '127.0.0.1'; } - if ($this->ip === 'host.docker.internal') { + if ($this->isLocalhost()) { return base_ip(); } return $this->ip; diff --git a/app/Models/ServiceDatabase.php b/app/Models/ServiceDatabase.php index c86334e39..2feaeb947 100644 --- a/app/Models/ServiceDatabase.php +++ b/app/Models/ServiceDatabase.php @@ -36,7 +36,7 @@ class ServiceDatabase extends BaseModel { $port = $this->public_port; $realIp = $this->service->server->ip; - if ($realIp === 'host.docker.internal' || isDev()) { + if ($this->service->server->isLocalhost() || isDev()) { $realIp = base_ip(); } $url = "{$realIp}:{$port}"; diff --git a/app/Notifications/Server/HighDiskUsage.php b/app/Notifications/Server/HighDiskUsage.php new file mode 100644 index 000000000..d8794600d --- /dev/null +++ b/app/Notifications/Server/HighDiskUsage.php @@ -0,0 +1,65 @@ +subject("Coolify: Server ({$this->server->name}) high disk usage detected!"); + $mail->view('emails.high-disk-usage', [ + 'name' => $this->server->name, + 'disk_usage' => $this->disk_usage, + 'threshold' => $this->cleanup_after_percentage, + ]); + return $mail; + } + + public function toDiscord(): string + { + $message = "Coolify: Server '{$this->server->name}' high disk usage detected!\nDisk usage: {$this->disk_usage}%. Threshold: {$this->cleanup_after_percentage}%.\nPlease cleanup your disk to prevent data-loss.\nHere are some tips: https://coolify.io/docs/automated-cleanup."; + return $message; + } + public function toTelegram(): array + { + return [ + "message" => "Coolify: Server '{$this->server->name}' high disk usage detected!\nDisk usage: {$this->disk_usage}%. Threshold: {$this->cleanup_after_percentage}%.\nPlease cleanup your disk to prevent data-loss.\nHere are some tips: https://coolify.io/docs/automated-cleanup." + ]; + } +} diff --git a/app/Notifications/Server/Revived.php b/app/Notifications/Server/Revived.php index 21fe6d40d..400ef8377 100644 --- a/app/Notifications/Server/Revived.php +++ b/app/Notifications/Server/Revived.php @@ -18,7 +18,7 @@ class Revived extends Notification implements ShouldQueue public $tries = 1; public function __construct(public Server $server) { - if ($this->server->unreachable_email_sent === false) { + if ($this->server->unreachable_notification_sent === false) { return; } } diff --git a/bootstrap/helpers/remoteProcess.php b/bootstrap/helpers/remoteProcess.php index e3d263a11..c1ed577b5 100644 --- a/bootstrap/helpers/remoteProcess.php +++ b/bootstrap/helpers/remoteProcess.php @@ -191,7 +191,7 @@ function refresh_server_connection(?PrivateKey $private_key = null) // if (!$uptime) { // $server->settings->is_reachable = false; // $server->team->notify(new Unreachable($server)); -// $server->unreachable_email_sent = true; +// $server->unreachable_notification_sent = true; // $server->save(); // return [ // "uptime" => null, @@ -213,9 +213,9 @@ function refresh_server_connection(?PrivateKey $private_key = null) // $server->settings->is_usable = false; // } else { // $server->settings->is_usable = true; -// if (data_get($server, 'unreachable_email_sent') === true) { +// if (data_get($server, 'unreachable_notification_sent') === true) { // $server->team->notify(new Revived($server)); -// $server->unreachable_email_sent = false; +// $server->unreachable_notification_sent = false; // $server->save(); // } // } diff --git a/config/sentry.php b/config/sentry.php index 6e7ff7cbc..1afa9d1ea 100644 --- a/config/sentry.php +++ b/config/sentry.php @@ -7,7 +7,7 @@ return [ // The release version of your application // Example with dynamic git hash: trim(exec('git --git-dir ' . base_path('.git') . ' log --pretty="%h" -n1 HEAD')) - 'release' => '4.0.0-beta.136', + 'release' => '4.0.0-beta.137', // When left empty or `null` the Laravel environment will be used 'environment' => config('app.env'), diff --git a/config/version.php b/config/version.php index 781d975f1..20308c282 100644 --- a/config/version.php +++ b/config/version.php @@ -1,3 +1,3 @@ boolean('high_disk_usage_notification_sent')->default(false); + $table->renameColumn('unreachable_email_sent', 'unreachable_notification_sent'); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('servers', function (Blueprint $table) { + $table->dropColumn('high_disk_usage_notification_sent'); + $table->renameColumn('unreachable_notification_sent', 'unreachable_email_sent'); + }); + } +}; diff --git a/examples/fluent-bit/fluent-bit.conf b/examples/fluent-bit/fluent-bit.conf new file mode 100644 index 000000000..5d10f559a --- /dev/null +++ b/examples/fluent-bit/fluent-bit.conf @@ -0,0 +1,16 @@ +[SERVICE] + Flush 1 + Daemon off +[INPUT] + Name forward + Buffer_Chunk_Size 1M + Buffer_Max_Size 6M +# [OUTPUT] +# Name nrlogs +# Match * +# license_key ${LICENSE_KEY} +# base_uri https://log-api.eu.newrelic.com/log/v1 + +[OUTPUT] + Name stdout + Match * diff --git a/examples/fluent-bit/fluent-bit.yaml b/examples/fluent-bit/fluent-bit.yaml new file mode 100644 index 000000000..ca573635e --- /dev/null +++ b/examples/fluent-bit/fluent-bit.yaml @@ -0,0 +1,9 @@ +version: '3' +services: + coolify-fluent-bit: + image: cr.fluentbit.io/fluent/fluent-bit:2.0 + command: -c /fluent-bit.conf + volumes: + - ./fluent-bit.conf:/fluent-bit.conf + ports: + - 24224:24224 diff --git a/examples/newrelic.yaml b/examples/newrelic.yaml new file mode 100644 index 000000000..40bd5b0f2 --- /dev/null +++ b/examples/newrelic.yaml @@ -0,0 +1,21 @@ +version: '3' +services: + newrelic-infra: + container_name: newrelic-infra + image: newrelic/infrastructure:latest + networks: + - coolify + cap_add: + - SYS_PTRACE + privileged: true + pid: host + volumes: + - "/:/host:ro" + - "/var/run/docker.sock:/var/run/docker.sock" + - "newrelic-infra:/etc/newrelic-infra" + environment: + - NRIA_LICENSE_KEY=${NRIA_LICENSE_KEY} + - NRIA_DISPLAY_NAME=${HOSTNAME} + +networks: + coolify: diff --git a/examples/otl/config.yaml b/examples/otl/config.yaml new file mode 100644 index 000000000..a1b8b7ec4 --- /dev/null +++ b/examples/otl/config.yaml @@ -0,0 +1,34 @@ +receivers: + hostmetrics: + collection_interval: 5s + scrapers: + cpu: + metrics: + system.cpu.utilization: + enabled: true +processors: + resourcedetection: + detectors: [env, system] + system: + hostname_sources: ["os"] + resource_attributes: + host.id: + enabled: true + batch: + memory_limiter: + check_interval: 1s + limit_mib: 1000 + spike_limit_mib: 200 +exporters: + debug: + verbosity: detailed + otlp: + endpoint: ${OTLP_ENDPOINT} + headers: + api-key: ${OTLP_API_KEY} +service: + pipelines: + metrics: + receivers: [hostmetrics] + processors: [memory_limiter, resourcedetection, batch] + exporters: [debug, otlp] diff --git a/resources/views/emails/high-disk-usage.blade.php b/resources/views/emails/high-disk-usage.blade.php new file mode 100644 index 000000000..0a4baa300 --- /dev/null +++ b/resources/views/emails/high-disk-usage.blade.php @@ -0,0 +1,9 @@ + + +Your server ({{ $name }}) has high disk usage ({{ $disk_usage }}% used). Threshold is {{ $threshold }}%. + +Please cleanup your disk to prevent data-loss. Here are some [tips](https://coolify.io/docs/automated-cleanup). + +(You can change the threshold in the Server Settings menu.) + + diff --git a/resources/views/livewire/server/form.blade.php b/resources/views/livewire/server/form.blade.php index c9ed332de..c36d34cbc 100644 --- a/resources/views/livewire/server/form.blade.php +++ b/resources/views/livewire/server/form.blade.php @@ -49,11 +49,13 @@ -
- -
+ @if (!$server->isLocalhost()) +
+ +
+ @endif @if ($server->isFunctional()) diff --git a/versions.json b/versions.json index 3b07acf3f..826bf73cc 100644 --- a/versions.json +++ b/versions.json @@ -4,7 +4,7 @@ "version": "3.12.36" }, "v4": { - "version": "4.0.0-beta.136" + "version": "4.0.0-beta.137" } } }