dave/coolify
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

feat(core): add error logging and cron parsing to docker/server schedules

Browse Source
This commit is contained in:
Andras Bacsai
2025-02-15 13:21:23 +01:00
parent 570b46d50e
commit 694b0ac249
1 changed files with 79 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

134
app/Console/Kernel.php
View File

@@ -6,13 +6,11 @@ use App\Jobs\CheckAndStartSentinelJob;
use App\Jobs\CheckForUpdatesJob; use App\Jobs\CheckForUpdatesJob;
use App\Jobs\CheckHelperImageJob; use App\Jobs\CheckHelperImageJob;
use App\Jobs\CleanupInstanceStuffsJob; use App\Jobs\CleanupInstanceStuffsJob;
use App\Jobs\CleanupStaleMultiplexedConnections;
use App\Jobs\DatabaseBackupJob; use App\Jobs\DatabaseBackupJob;
use App\Jobs\DockerCleanupJob; use App\Jobs\DockerCleanupJob;
use App\Jobs\PullTemplatesFromCDN; use App\Jobs\PullTemplatesFromCDN;
use App\Jobs\ScheduledTaskJob; use App\Jobs\ScheduledTaskJob;
use App\Jobs\ServerCheckJob; use App\Jobs\ServerCheckJob;
use App\Jobs\ServerCleanupMux;
use App\Jobs\ServerStorageCheckJob; use App\Jobs\ServerStorageCheckJob;
use App\Jobs\UpdateCoolifyJob; use App\Jobs\UpdateCoolifyJob;
use App\Models\InstanceSettings; use App\Models\InstanceSettings;
@@ -23,6 +21,7 @@ use App\Models\Team;
use Illuminate\Console\Scheduling\Schedule; use Illuminate\Console\Scheduling\Schedule;
use Illuminate\Foundation\Console\Kernel as ConsoleKernel; use Illuminate\Foundation\Console\Kernel as ConsoleKernel;
use Illuminate\Support\Carbon; use Illuminate\Support\Carbon;
use Illuminate\Support\Facades\Log;
class Kernel extends ConsoleKernel class Kernel extends ConsoleKernel
{ {
@@ -99,10 +98,14 @@ class Kernel extends ConsoleKernel
$servers = $this->allServers->whereRelation('settings', 'is_usable', true)->whereRelation('settings', 'is_reachable', true)->get(); $servers = $this->allServers->whereRelation('settings', 'is_usable', true)->whereRelation('settings', 'is_reachable', true)->get();
} }
foreach ($servers as $server) { foreach ($servers as $server) {
if ($server->isSentinelEnabled()) { try {
$this->scheduleInstance->job(function () use ($server) { if ($server->isSentinelEnabled()) {
CheckAndStartSentinelJob::dispatch($server); $this->scheduleInstance->job(function () use ($server) {
})->cron($this->updateCheckFrequency)->timezone($this->instanceTimezone)->onOneServer(); CheckAndStartSentinelJob::dispatch($server);
})->cron($this->updateCheckFrequency)->timezone($this->instanceTimezone)->onOneServer();
}
} catch (\Exception $e) {
Log::error('Error pulling images: '.$e->getMessage());
} }
} }
$this->scheduleInstance->job(new CheckHelperImageJob) $this->scheduleInstance->job(new CheckHelperImageJob)
@@ -138,35 +141,47 @@ class Kernel extends ConsoleKernel
} }
foreach ($servers as $server) { foreach ($servers as $server) {
$serverTimezone = data_get($server->settings, 'server_timezone', $this->instanceTimezone); try {
if (validate_timezone($serverTimezone) === false) { $serverTimezone = data_get($server->settings, 'server_timezone', $this->instanceTimezone);
$serverTimezone = config('app.timezone'); if (validate_timezone($serverTimezone) === false) {
} $serverTimezone = config('app.timezone');
// Sentinel check
$lastSentinelUpdate = $server->sentinel_updated_at;
if (Carbon::parse($lastSentinelUpdate)->isBefore(now()->subSeconds($server->waitBeforeDoingSshCheck()))) {
// Check container status every minute if Sentinel does not activated
if (isCloud()) {
$this->scheduleInstance->job(new ServerCheckJob($server))->timezone($serverTimezone)->everyFiveMinutes()->onOneServer();
} else {
$this->scheduleInstance->job(new ServerCheckJob($server))->timezone($serverTimezone)->everyMinute()->onOneServer();
} }
// $this->scheduleInstance->job(new \App\Jobs\ServerCheckNewJob($server))->everyFiveMinutes()->onOneServer();
$this->scheduleInstance->job(new ServerStorageCheckJob($server))->cron($server->settings->server_disk_usage_check_frequency)->timezone($serverTimezone)->onOneServer(); // Sentinel check
} $lastSentinelUpdate = $server->sentinel_updated_at;
if (Carbon::parse($lastSentinelUpdate)->isBefore(now()->subSeconds($server->waitBeforeDoingSshCheck()))) {
// Check container status every minute if Sentinel does not activated
if (isCloud()) {
$this->scheduleInstance->job(new ServerCheckJob($server))->timezone($serverTimezone)->everyFiveMinutes()->onOneServer();
} else {
$this->scheduleInstance->job(new ServerCheckJob($server))->timezone($serverTimezone)->everyMinute()->onOneServer();
}
// $this->scheduleInstance->job(new \App\Jobs\ServerCheckNewJob($server))->everyFiveMinutes()->onOneServer();
$this->scheduleInstance->job(new DockerCleanupJob($server))->cron($server->settings->docker_cleanup_frequency)->timezone($serverTimezone)->onOneServer(); $serverDiskUsageCheckFrequency = data_get($server->settings, 'server_disk_usage_check_frequency', '0 * * * *');
if (isset(VALID_CRON_STRINGS[$serverDiskUsageCheckFrequency])) {
$serverDiskUsageCheckFrequency = VALID_CRON_STRINGS[$serverDiskUsageCheckFrequency];
}
$this->scheduleInstance->job(new ServerStorageCheckJob($server))->cron($serverDiskUsageCheckFrequency)->timezone($serverTimezone)->onOneServer();
}
// Cleanup multiplexed connections every hour $dockerCleanupFrequency = data_get($server->settings, 'docker_cleanup_frequency', '0 * * * *');
// $this->scheduleInstance->job(new ServerCleanupMux($server))->hourly()->onOneServer(); if (isset(VALID_CRON_STRINGS[$dockerCleanupFrequency])) {
$dockerCleanupFrequency = VALID_CRON_STRINGS[$dockerCleanupFrequency];
}
$this->scheduleInstance->job(new DockerCleanupJob($server))->cron($dockerCleanupFrequency)->timezone($serverTimezone)->onOneServer();
// Temporary solution until we have better memory management for Sentinel // Cleanup multiplexed connections every hour
if ($server->isSentinelEnabled()) { // $this->scheduleInstance->job(new ServerCleanupMux($server))->hourly()->onOneServer();
$this->scheduleInstance->job(function () use ($server) {
$server->restartContainer('coolify-sentinel'); // Temporary solution until we have better memory management for Sentinel
})->daily()->onOneServer(); if ($server->isSentinelEnabled()) {
$this->scheduleInstance->job(function () use ($server) {
$server->restartContainer('coolify-sentinel');
})->daily()->onOneServer();
}
} catch (\Exception $e) {
Log::error('Error checking resources: '.$e->getMessage());
} }
} }
} }
@@ -200,24 +215,28 @@ class Kernel extends ConsoleKernel
} }
foreach ($finalScheduledBackups as $scheduled_backup) { foreach ($finalScheduledBackups as $scheduled_backup) {
if (isset(VALID_CRON_STRINGS[$scheduled_backup->frequency])) { try {
$scheduled_backup->frequency = VALID_CRON_STRINGS[$scheduled_backup->frequency]; if (isset(VALID_CRON_STRINGS[$scheduled_backup->frequency])) {
} $scheduled_backup->frequency = VALID_CRON_STRINGS[$scheduled_backup->frequency];
}
$server = $scheduled_backup->server();
$serverTimezone = data_get($server->settings, 'server_timezone', $this->instanceTimezone);
$server = $scheduled_backup->server(); if (validate_timezone($serverTimezone) === false) {
$serverTimezone = data_get($server->settings, 'server_timezone', $this->instanceTimezone); $serverTimezone = config('app.timezone');
}
if (validate_timezone($serverTimezone) === false) { if (isset(VALID_CRON_STRINGS[$scheduled_backup->frequency])) {
$serverTimezone = config('app.timezone'); $scheduled_backup->frequency = VALID_CRON_STRINGS[$scheduled_backup->frequency];
}
$serverTimezone = data_get($server->settings, 'server_timezone', $this->instanceTimezone);
$this->scheduleInstance->job(new DatabaseBackupJob(
backup: $scheduled_backup
))->cron($scheduled_backup->frequency)->timezone($serverTimezone)->onOneServer();
} catch (\Exception $e) {
Log::error('Error scheduling backup: '.$e->getMessage());
Log::error($e->getTraceAsString());
} }
if (isset(VALID_CRON_STRINGS[$scheduled_backup->frequency])) {
$scheduled_backup->frequency = VALID_CRON_STRINGS[$scheduled_backup->frequency];
}
$serverTimezone = data_get($server->settings, 'server_timezone', $this->instanceTimezone);
$this->scheduleInstance->job(new DatabaseBackupJob(
backup: $scheduled_backup
))->cron($scheduled_backup->frequency)->timezone($serverTimezone)->onOneServer();
} }
} }
@@ -264,18 +283,23 @@ class Kernel extends ConsoleKernel
} }
foreach ($finalScheduledTasks as $scheduled_task) { foreach ($finalScheduledTasks as $scheduled_task) {
$server = $scheduled_task->server(); try {
if (isset(VALID_CRON_STRINGS[$scheduled_task->frequency])) { $server = $scheduled_task->server();
$scheduled_task->frequency = VALID_CRON_STRINGS[$scheduled_task->frequency]; if (isset(VALID_CRON_STRINGS[$scheduled_task->frequency])) {
} $scheduled_task->frequency = VALID_CRON_STRINGS[$scheduled_task->frequency];
$serverTimezone = data_get($server->settings, 'server_timezone', $this->instanceTimezone); }
$serverTimezone = data_get($server->settings, 'server_timezone', $this->instanceTimezone);
if (validate_timezone($serverTimezone) === false) { if (validate_timezone($serverTimezone) === false) {
$serverTimezone = config('app.timezone'); $serverTimezone = config('app.timezone');
}
$this->scheduleInstance->job(new ScheduledTaskJob(
task: $scheduled_task
))->cron($scheduled_task->frequency)->timezone($serverTimezone)->onOneServer();
} catch (\Exception $e) {
Log::error('Error scheduling task: '.$e->getMessage());
Log::error($e->getTraceAsString());
} }
$this->scheduleInstance->job(new ScheduledTaskJob(
task: $scheduled_task
))->cron($scheduled_task->frequency)->timezone($serverTimezone)->onOneServer();
} }
} }