dave/coolify
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

fix: docker cleanup job

Browse Source
This commit is contained in:
Andras Bacsai
2024-08-26 12:23:03 +02:00
parent 4d7877ce1d
commit 3d73c98779
10 changed files with 231 additions and 179 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
app/Actions/Server/CleanupDocker.php
View File

@@ -12,28 +12,20 @@ class CleanupDocker
public function handle(Server $server)
{
$commands = $this->getCommands($force);
$commands = $this->getCommands();
foreach ($commands as $command) {
instant_remote_process([$command], $server, false);
}
}
private function getCommands(bool $force): array
private function getCommands(): array
{
$commonCommands = [
'docker container prune -f --filter "label=coolify.managed=true"',
'docker image prune -f',
'docker builder prune -f',
];
if ($force) {
return array_merge([
'docker container prune -f --filter "label=coolify.managed=true"',
'docker image prune -af',
'docker builder prune -af',
], $commonCommands);
}
];
return $commonCommands;
}

10
app/Console/Kernel.php
View File

@@ -106,7 +106,11 @@ class Kernel extends ConsoleKernel
//The lines below need to be added as soon as timzone is merged!!
//$serverTimezone = $server->settings->server_timezone;
//$schedule->job(new DockerCleanupJob($server))->cron($server->settings->docker_cleanup_frequency)->timezone($serverTimezone)->onOneServer();
if ($server->settings->force_docker_cleanup) {
$schedule->job(new DockerCleanupJob($server))->cron($server->settings->docker_cleanup_frequency)->onOneServer();
} else {
$schedule->job(new DockerCleanupJob($server))->everyTenMinutes()->onOneServer();
}
}
}
@@ -118,7 +122,7 @@ class Kernel extends ConsoleKernel
return;
}
foreach ($scheduled_backups as $scheduled_backup) {
if (!$scheduled_backup->enabled) {
if (! $scheduled_backup->enabled) {
continue;
}
if (is_null(data_get($scheduled_backup, 'database'))) {
@@ -150,7 +154,7 @@ class Kernel extends ConsoleKernel
$service = $scheduled_task->service;
$application = $scheduled_task->application;
if (!$application && !$service) {
if (! $application && ! $service) {
ray('application/service attached to scheduled task does not exist');
$scheduled_task->delete();
@@ -177,7 +181,7 @@ class Kernel extends ConsoleKernel
protected function commands(): void
{
$this->load(__DIR__ . '/Commands');
$this->load(__DIR__.'/Commands');
require base_path('routes/console.php');
}

37
app/Jobs/DockerCleanupJob.php
View File

@@ -10,6 +10,7 @@ use Illuminate\Contracts\Queue\ShouldBeEncrypted;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Log;
@@ -17,14 +18,24 @@ class DockerCleanupJob implements ShouldBeEncrypted, ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public $timeout = 300;
public $timeout = 600;
public $tries = 2;
public $tries = 1;
public ?string $usageBefore = null;
public function __construct(public Server $server) {}
public function middleware(): array
{
return [new WithoutOverlapping($this->server->id)];
}
public function uniqueId(): int
{
return $this->server->id;
}
public function handle(): void
{
try {
@@ -32,34 +43,30 @@ class DockerCleanupJob implements ShouldBeEncrypted, ShouldQueue
return;
}
if ($this->server->settings->force_docker_cleanup) {
Log::info('DockerCleanupJob force cleanup on ' . $this->server->name);
CleanupDocker::run(server: $this->server, force: true);
Log::info('DockerCleanupJob force cleanup on '.$this->server->name);
CleanupDocker::run(server: $this->server);
return;
}
return;
}
try {
$this->usageBefore = $this->server->getDiskUsage();
if (str($this->usageBefore)->isEmpty() || $this->usageBefore === null || $this->usageBefore === 0) {
Log::info('DockerCleanupJob force cleanup on ' . $this->server->name);
CleanupDocker::run(server: $this->server, force: true);
Log::info('DockerCleanupJob force cleanup on '.$this->server->name);
CleanupDocker::run(server: $this->server);
return;
}
if ($this->usageBefore >= $this->server->settings->cleanup_after_percentage) {
if ($this->usageBefore >= $this->server->settings->docker_cleanup_threshold) {
CleanupDocker::run(server: $this->server);
$usageAfter = $this->server->getDiskUsage();
if ($usageAfter < $this->usageBefore) {
$this->server->team?->notify(new DockerCleanup($this->server, 'Saved ' . ($this->usageBefore - $usageAfter) . '% disk space.'));
Log::info('DockerCleanupJob done: Saved ' . ($this->usageBefore - $usageAfter) . '% disk space on ' . $this->server->name);
$this->server->team?->notify(new DockerCleanup($this->server, 'Saved '.($this->usageBefore - $usageAfter).'% disk space.'));
Log::info('DockerCleanupJob done: Saved '.($this->usageBefore - $usageAfter).'% disk space on '.$this->server->name);
} else {
Log::info('DockerCleanupJob failed to save disk space on ' . $this->server->name);
Log::info('DockerCleanupJob failed to save disk space on '.$this->server->name);
}
} else {
Log::info('No need to clean up ' . $this->server->name);
Log::info('No need to clean up '.$this->server->name);
}
} catch (\Throwable $e) {
CleanupDocker::run(server: $this->server);

12
app/Livewire/Server/Form.php
View File

@@ -18,8 +18,6 @@ class Form extends Component
public ?string $wildcard_domain = null;
public int $cleanup_after_percentage;
public bool $dockerInstallationStarted = false;
public bool $revalidate = false;
@@ -81,7 +79,7 @@ class Form extends Component
{
if ($field === 'server.settings.docker_cleanup_frequency') {
$frequency = $this->server->settings->docker_cleanup_frequency;
if (empty($frequency) || !validate_cron_expression($frequency)) {
if (empty($frequency) || ! validate_cron_expression($frequency)) {
$this->dispatch('error', 'Invalid Cron / Human expression for Docker Cleanup Frequency. Resetting to default 10 minutes.');
$this->server->settings->docker_cleanup_frequency = '*/10 * * * *';
}
@@ -169,7 +167,7 @@ class Form extends Component
$this->server->settings->save();
$this->dispatch('proxyStatusUpdated');
} else {
$this->dispatch('error', 'Server is not reachable.', 'Please validate your configuration and connection.<br><br>Check this <a target="_blank" class="underline" href="https://coolify.io/docs/knowledge-base/server/openssh">documentation</a> for further help. <br><br>Error: ' . $error);
$this->dispatch('error', 'Server is not reachable.', 'Please validate your configuration and connection.<br><br>Check this <a target="_blank" class="underline" href="https://coolify.io/docs/knowledge-base/server/openssh">documentation</a> for further help. <br><br>Error: '.$error);
return;
}
@@ -185,7 +183,8 @@ class Form extends Component
public function submit()
{
if (isCloud() && !isDev()) {
try {
if (isCloud() && ! isDev()) {
$this->validate();
$this->validate([
'server.ip' => 'required',
@@ -211,5 +210,8 @@ class Form extends Component
$this->server->settings->save();
$this->server->save();
$this->dispatch('success', 'Server updated.');
} catch (\Throwable $e) {
return handleError($e, $this);
}
}
}

4
app/Models/Service.php
View File

@@ -2,13 +2,11 @@
namespace App\Models;
use App\Enums\ProxyTypes;
use Illuminate\Database\Eloquent\Casts\Attribute;
use Illuminate\Database\Eloquent\Factories\HasFactory;
use Illuminate\Database\Eloquent\Relations\HasMany;
use Illuminate\Database\Eloquent\SoftDeletes;
use Illuminate\Support\Collection;
use Illuminate\Support\Str;
use OpenApi\Attributes as OA;
use Spatie\Url\Url;
use Symfony\Component\Yaml\Yaml;
@@ -25,7 +23,7 @@ use Symfony\Component\Yaml\Yaml;
'description' => ['type' => 'string', 'description' => 'The description of the service.'],
'docker_compose_raw' => ['type' => 'string', 'description' => 'The raw docker-compose.yml file of the service.'],
'docker_compose' => ['type' => 'string', 'description' => 'The docker-compose.yml file that is parsed and modified by Coolify.'],
'destination_type' => ['type' => 'integer', 'description' => 'The unique identifier of the destination where the service is running.'],
'destination_type' => ['type' => 'string', 'description' => 'Destination type.'],
'destination_id' => ['type' => 'integer', 'description' => 'The unique identifier of the destination where the service is running.'],
'connect_to_docker_network' => ['type' => 'boolean', 'description' => 'The flag to connect the service to the predefined Docker network.'],
'is_container_label_escape_enabled' => ['type' => 'boolean', 'description' => 'The flag to enable the container label escape.'],

2
app/Notifications/Server/DockerCleanup.php
View File

@@ -44,7 +44,7 @@ class DockerCleanup extends Notification implements ShouldQueue
// $mail->view('emails.high-disk-usage', [
// 'name' => $this->server->name,
// 'disk_usage' => $this->disk_usage,
// 'threshold' => $this->cleanup_after_percentage,
// 'threshold' => $this->docker_cleanup_threshold,
// ]);
// return $mail;
// }

8
app/Notifications/Server/HighDiskUsage.php
View File

@@ -17,7 +17,7 @@ class HighDiskUsage extends Notification implements ShouldQueue
public $tries = 1;
public function __construct(public Server $server, public int $disk_usage, public int $cleanup_after_percentage) {}
public function __construct(public Server $server, public int $disk_usage, public int $docker_cleanup_threshold) {}
public function via(object $notifiable): array
{
@@ -46,7 +46,7 @@ class HighDiskUsage extends Notification implements ShouldQueue
$mail->view('emails.high-disk-usage', [
'name' => $this->server->name,
'disk_usage' => $this->disk_usage,
'threshold' => $this->cleanup_after_percentage,
'threshold' => $this->docker_cleanup_threshold,
]);
return $mail;
@@ -54,7 +54,7 @@ class HighDiskUsage extends Notification implements ShouldQueue
public function toDiscord(): string
{
$message = "Coolify: Server '{$this->server->name}' high disk usage detected!\nDisk usage: {$this->disk_usage}%. Threshold: {$this->cleanup_after_percentage}%.\nPlease cleanup your disk to prevent data-loss.\nHere are some tips: https://coolify.io/docs/knowledge-base/server/automated-cleanup.";
$message = "Coolify: Server '{$this->server->name}' high disk usage detected!\nDisk usage: {$this->disk_usage}%. Threshold: {$this->docker_cleanup_threshold}%.\nPlease cleanup your disk to prevent data-loss.\nHere are some tips: https://coolify.io/docs/knowledge-base/server/automated-cleanup.";
return $message;
}
@@ -62,7 +62,7 @@ class HighDiskUsage extends Notification implements ShouldQueue
public function toTelegram(): array
{
return [
'message' => "Coolify: Server '{$this->server->name}' high disk usage detected!\nDisk usage: {$this->disk_usage}%. Threshold: {$this->cleanup_after_percentage}%.\nPlease cleanup your disk to prevent data-loss.\nHere are some tips: https://coolify.io/docs/knowledge-base/server/automated-cleanup.",
'message' => "Coolify: Server '{$this->server->name}' high disk usage detected!\nDisk usage: {$this->disk_usage}%. Threshold: {$this->docker_cleanup_threshold}%.\nPlease cleanup your disk to prevent data-loss.\nHere are some tips: https://coolify.io/docs/knowledge-base/server/automated-cleanup.",
];
}
}

18
database/migrations/2024_08_09_215659_add_server_cleanup_fields_to_server_settings_table.php
View File

@@ -1,5 +1,6 @@
<?php
use App\Models\ServerSetting;
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
@@ -13,16 +14,23 @@ class AddServerCleanupFieldsToServerSettingsTable extends Migration
*/
public function up()
{
$serverSettings = ServerSetting::all();
Schema::table('server_settings', function (Blueprint $table) {
$table->boolean('force_docker_cleanup')->default(false);
$table->string('docker_cleanup_frequency')->default('*/10 * * * *');
$table->integer('docker_cleanup_threshold')->default(80);
// Remove old columns
$table->dropColumn('cleanup_after_percentage');
$table->dropColumn('is_force_cleanup_enabled');
});
foreach ($serverSettings as $serverSetting) {
$serverSetting->force_docker_cleanup = $serverSetting->is_force_cleanup_enabled;
$serverSetting->docker_cleanup_threshold = $serverSetting->cleanup_after_percentage;
$serverSetting->save();
}
}
/**
@@ -32,15 +40,21 @@ class AddServerCleanupFieldsToServerSettingsTable extends Migration
*/
public function down()
{
$serverSettings = ServerSetting::all();
Schema::table('server_settings', function (Blueprint $table) {
$table->dropColumn('force_docker_cleanup');
$table->dropColumn('docker_cleanup_frequency');
$table->dropColumn('docker_cleanup_threshold');
// Add back old columns
$table->integer('cleanup_after_percentage')->default(80);
$table->boolean('force_server_cleanup')->default(false);
$table->boolean('is_force_cleanup_enabled')->default(false);
});
foreach ($serverSettings as $serverSetting) {
$serverSetting->is_force_cleanup_enabled = $serverSetting->force_docker_cleanup;
$serverSetting->cleanup_after_percentage = $serverSetting->docker_cleanup_threshold;
$serverSetting->save();
}
}
}

11
openapi.yaml
View File

@@ -4523,14 +4523,14 @@ components:
properties:
id:
type: integer
cleanup_after_percentage:
type: integer
concurrent_builds:
type: integer
dynamic_timeout:
type: integer
force_disabled:
type: boolean
force_server_cleanup:
type: boolean
is_build_server:
type: boolean
is_cloudflare_tunnel:
@@ -4577,6 +4577,10 @@ components:
type: integer
metrics_token:
type: string
docker_cleanup_frequency:
type: string
docker_cleanup_threshold:
type: integer
server_id:
type: integer
wildcard_domain:
@@ -4613,6 +4617,9 @@ components:
docker_compose:
type: string
description: 'The docker-compose.yml file that is parsed and modified by Coolify.'
destination_type:
type: string
description: 'Destination type.'
destination_id:
type: integer
description: 'The unique identifier of the destination where the service is running.'

62
resources/views/livewire/server/form.blade.php
View File

@@ -34,7 +34,9 @@
<x-slot:content>
<livewire:server.validate-and-install :server="$server" />
</x-slot:content>
<x-forms.button @click="slideOverOpen=true" class="w-full mt-8 mb-4 font-bold box-without-bg bg-coollabs hover:bg-coollabs-100" wire:click.prevent='validateServer' isHighlighted>
<x-forms.button @click="slideOverOpen=true"
class="w-full mt-8 mb-4 font-bold box-without-bg bg-coollabs hover:bg-coollabs-100"
wire:click.prevent='validateServer' isHighlighted>
Validate Server & Install Docker Engine
</x-forms.button>
</x-slide-over>
@@ -46,7 +48,8 @@
@endif
@endif
@if ((!$server->settings->is_reachable || !$server->settings->is_usable) && $server->id === 0)
<x-forms.button class="mt-8 mb-4 font-bold box-without-bg bg-coollabs hover:bg-coollabs-100" wire:click.prevent='checkLocalhostConnection' isHighlighted>
<x-forms.button class="mt-8 mb-4 font-bold box-without-bg bg-coollabs hover:bg-coollabs-100"
wire:click.prevent='checkLocalhostConnection' isHighlighted>
Validate Server
</x-forms.button>
@endif
@@ -59,12 +62,14 @@
<x-forms.input id="server.name" label="Name" required />
<x-forms.input id="server.description" label="Description" />
@if (!$server->settings->is_swarm_worker && !$server->settings->is_build_server)
<x-forms.input placeholder="https://example.com" id="wildcard_domain" label="Wildcard Domain" helper='A wildcard domain allows you to receive a randomly generated domain for your new applications. <br><br>For instance, if you set "https://example.com" as your wildcard domain, your applications will receive domains like "https://randomId.example.com".' />
<x-forms.input placeholder="https://example.com" id="wildcard_domain" label="Wildcard Domain"
helper='A wildcard domain allows you to receive a randomly generated domain for your new applications. <br><br>For instance, if you set "https://example.com" as your wildcard domain, your applications will receive domains like "https://randomId.example.com".' />
@endif
</div>
<div class="flex flex-col w-full gap-2 lg:flex-row">
<x-forms.input type="password" id="server.ip" label="IP Address/Domain" helper="An IP Address (127.0.0.1) or domain (example.com)." required />
<x-forms.input type="password" id="server.ip" label="IP Address/Domain"
helper="An IP Address (127.0.0.1) or domain (example.com)." required />
<div class="flex gap-2">
<x-forms.input id="server.user" label="User" required />
<x-forms.input type="number" id="server.port" label="Port" required />
@@ -73,11 +78,13 @@
<div class="w-64">
@if ($server->isFunctional())
@if (!$server->isLocalhost())
<x-forms.checkbox instantSave id="server.settings.is_build_server" label="Use it as a build server?" />
<x-forms.checkbox instantSave id="server.settings.is_build_server"
label="Use it as a build server?" />
<div class="flex items-center gap-1 pt-6">
<h3 class="">Cloudflare Tunnels
</h3>
<x-helper class="inline-flex" helper="If you are using Cloudflare Tunnels, enable this. It will proxy all SSH requests to your server through Cloudflare.<br><span class='dark:text-warning'>Coolify does not install or set up Cloudflare (cloudflared) on your server.</span>" />
<x-helper class="inline-flex"
helper="If you are using Cloudflare Tunnels, enable this. It will proxy all SSH requests to your server through Cloudflare.<br><span class='dark:text-warning'>Coolify does not install or set up Cloudflare (cloudflared) on your server.</span>" />
</div>
@if ($server->settings->is_cloudflare_tunnel)
<x-forms.checkbox instantSave id="server.settings.is_cloudflare_tunnel" label="Enabled" />
@@ -88,18 +95,29 @@
@endif
@if (!$server->isBuildServer())
<h3 class="pt-6">Swarm <span class="text-xs text-neutral-500">(experimental)</span></h3>
<div class="pb-4">Read the docs <a class='underline dark:text-white' href='https://coolify.io/docs/knowledge-base/docker/swarm' target='_blank'>here</a>.
<div class="pb-4">Read the docs <a class='underline dark:text-white'
href='https://coolify.io/docs/knowledge-base/docker/swarm' target='_blank'>here</a>.
</div>
@if ($server->settings->is_swarm_worker)
<x-forms.checkbox disabled instantSave type="checkbox" id="server.settings.is_swarm_manager" helper="For more information, please read the documentation <a class='dark:text-white' href='https://coolify.io/docs/knowledge-base/docker/swarm' target='_blank'>here</a>." label="Is it a Swarm Manager?" />
<x-forms.checkbox disabled instantSave type="checkbox"
id="server.settings.is_swarm_manager"
helper="For more information, please read the documentation <a class='dark:text-white' href='https://coolify.io/docs/knowledge-base/docker/swarm' target='_blank'>here</a>."
label="Is it a Swarm Manager?" />
@else
<x-forms.checkbox instantSave type="checkbox" id="server.settings.is_swarm_manager" helper="For more information, please read the documentation <a class='dark:text-white' href='https://coolify.io/docs/knowledge-base/docker/swarm' target='_blank'>here</a>." label="Is it a Swarm Manager?" />
<x-forms.checkbox instantSave type="checkbox" id="server.settings.is_swarm_manager"
helper="For more information, please read the documentation <a class='dark:text-white' href='https://coolify.io/docs/knowledge-base/docker/swarm' target='_blank'>here</a>."
label="Is it a Swarm Manager?" />
@endif
@if ($server->settings->is_swarm_manager)
<x-forms.checkbox disabled instantSave type="checkbox" id="server.settings.is_swarm_worker" helper="For more information, please read the documentation <a class='dark:text-white' href='https://coolify.io/docs/knowledge-base/docker/swarm' target='_blank'>here</a>." label="Is it a Swarm Worker?" />
<x-forms.checkbox disabled instantSave type="checkbox"
id="server.settings.is_swarm_worker"
helper="For more information, please read the documentation <a class='dark:text-white' href='https://coolify.io/docs/knowledge-base/docker/swarm' target='_blank'>here</a>."
label="Is it a Swarm Worker?" />
@else
<x-forms.checkbox instantSave type="checkbox" id="server.settings.is_swarm_worker" helper="For more information, please read the documentation <a class='dark:text-white' href='https://coolify.io/docs/knowledge-base/docker/swarm' target='_blank'>here</a>." label="Is it a Swarm Worker?" />
<x-forms.checkbox instantSave type="checkbox" id="server.settings.is_swarm_worker"
helper="For more information, please read the documentation <a class='dark:text-white' href='https://coolify.io/docs/knowledge-base/docker/swarm' target='_blank'>here</a>."
label="Is it a Swarm Worker?" />
@endif
@endif
@endif
@@ -107,7 +125,8 @@
<div class="flex items-center gap-1 pt-6">
<h3 class="">Cloudflare Tunnels
</h3>
<x-helper class="inline-flex" helper="If you are using Cloudflare Tunnels, enable this. It will proxy all SSH requests to your server through Cloudflare.<br><span class='dark:text-warning'>Coolify does not install or set up Cloudflare (cloudflared) on your server.</span>" />
<x-helper class="inline-flex"
helper="If you are using Cloudflare Tunnels, enable this. It will proxy all SSH requests to your server through Cloudflare.<br><span class='dark:text-warning'>Coolify does not install or set up Cloudflare (cloudflared) on your server.</span>" />
</div>
@if ($server->settings->is_cloudflare_tunnel)
<x-forms.checkbox instantSave id="server.settings.is_cloudflare_tunnel" label="Enabled" />
@@ -126,16 +145,25 @@
<div class="flex flex-col gap-2">
<div class="flex flex-col flex-wrap gap-2 sm:flex-nowrap">
<div class="w-64">
<x-forms.checkbox helper="Enable force Docker Cleanup. This will cleanup build caches / unused images / etc." instantSave id="server.settings.force_docker_cleanup" label="Force Docker Cleanup" /> </div>
<x-forms.checkbox
helper="Enable force Docker Cleanup. This will cleanup build caches / unused images / etc."
instantSave id="server.settings.force_docker_cleanup" label="Force Docker Cleanup" />
</div>
@if ($server->settings->force_docker_cleanup)
<x-forms.input placeholder="*/10 * * * *" id="server.settings.docker_cleanup_frequency" label="Docker Cleanup Frequency" required helper="Cron expression for Docker Cleanup.<br>You can use every_minute, hourly, daily, weekly, monthly, yearly.<br><br>Default is every 10 minutes." />
<x-forms.input placeholder="*/10 * * * *" id="server.settings.docker_cleanup_frequency"
label="Docker cleanup frequency" required
helper="Cron expression for Docker Cleanup.<br>You can use every_minute, hourly, daily, weekly, monthly, yearly.<br><br>Default is every 10 minutes." />
@else
<x-forms.input id="server.settings.docker_cleanup_threshold" label="Docker cleanup threshold (%)" required helper="The Docker cleanup tasks will run when the disk usage exceeds this threshold." />
<x-forms.input id="server.settings.docker_cleanup_threshold"
label="Docker cleanup threshold (%)" required
helper="The Docker cleanup tasks will run when the disk usage exceeds this threshold." />
@endif
</div>
<div class="flex flex-wrap gap-2 sm:flex-nowrap">
<x-forms.input id="server.settings.concurrent_builds" label="Number of concurrent builds" required helper="You can specify the number of simultaneous build processes/deployments that should run concurrently." />
<x-forms.input id="server.settings.dynamic_timeout" label="Deployment timeout (seconds)" required helper="You can define the maximum duration for a deployment to run before timing it out." />
<x-forms.input id="server.settings.concurrent_builds" label="Number of concurrent builds" required
helper="You can specify the number of simultaneous build processes/deployments that should run concurrently." />
<x-forms.input id="server.settings.dynamic_timeout" label="Deployment timeout (seconds)" required
helper="You can define the maximum duration for a deployment to run before timing it out." />
</div>
</div>
<div class="flex items-center gap-2 pt-4 pb-2">