refactor(scheduling): replace deprecated job checks with ScheduledJobManager and ServerResourceManager for improved scheduling efficiency
This commit is contained in:
Andras Bacsai
@@ -6,24 +6,16 @@ use App\Jobs\CheckAndStartSentinelJob;
|
|||||||
use App\Jobs\CheckForUpdatesJob;
|
use App\Jobs\CheckForUpdatesJob;
|
||||||
use App\Jobs\CheckHelperImageJob;
|
use App\Jobs\CheckHelperImageJob;
|
||||||
use App\Jobs\CleanupInstanceStuffsJob;
|
use App\Jobs\CleanupInstanceStuffsJob;
|
||||||
use App\Jobs\DatabaseBackupJob;
|
|
||||||
use App\Jobs\DockerCleanupJob;
|
|
||||||
use App\Jobs\PullTemplatesFromCDN;
|
use App\Jobs\PullTemplatesFromCDN;
|
||||||
use App\Jobs\RegenerateSslCertJob;
|
use App\Jobs\RegenerateSslCertJob;
|
||||||
use App\Jobs\ScheduledTaskJob;
|
use App\Jobs\ScheduledJobManager;
|
||||||
use App\Jobs\ServerCheckJob;
|
use App\Jobs\ServerResourceManager;
|
||||||
use App\Jobs\ServerPatchCheckJob;
|
|
||||||
use App\Jobs\ServerStorageCheckJob;
|
|
||||||
use App\Jobs\UpdateCoolifyJob;
|
use App\Jobs\UpdateCoolifyJob;
|
||||||
use App\Models\InstanceSettings;
|
use App\Models\InstanceSettings;
|
||||||
use App\Models\ScheduledDatabaseBackup;
|
|
||||||
use App\Models\ScheduledTask;
|
|
||||||
use App\Models\Server;
|
use App\Models\Server;
|
||||||
use App\Models\Team;
|
use App\Models\Team;
|
||||||
use Cron\CronExpression;
|
|
||||||
use Illuminate\Console\Scheduling\Schedule;
|
use Illuminate\Console\Scheduling\Schedule;
|
||||||
use Illuminate\Foundation\Console\Kernel as ConsoleKernel;
|
use Illuminate\Foundation\Console\Kernel as ConsoleKernel;
|
||||||
use Illuminate\Support\Carbon;
|
|
||||||
use Illuminate\Support\Facades\Log;
|
use Illuminate\Support\Facades\Log;
|
||||||
|
|
||||||
class Kernel extends ConsoleKernel
|
class Kernel extends ConsoleKernel
|
||||||
@@ -62,10 +54,10 @@ class Kernel extends ConsoleKernel
|
|||||||
$this->scheduleInstance->job(new CheckHelperImageJob)->everyTenMinutes()->onOneServer();
|
$this->scheduleInstance->job(new CheckHelperImageJob)->everyTenMinutes()->onOneServer();
|
||||||
|
|
||||||
// Server Jobs
|
// Server Jobs
|
||||||
$this->checkResources();
|
$this->scheduleInstance->job(new ServerResourceManager)->everyMinute()->onOneServer();
|
||||||
|
|
||||||
$this->checkScheduledBackups();
|
// Scheduled Jobs (Backups & Tasks)
|
||||||
$this->checkScheduledTasks();
|
$this->scheduleInstance->job(new ScheduledJobManager)->everyMinute()->onOneServer();
|
||||||
|
|
||||||
$this->scheduleInstance->command('uploads:clear')->everyTwoMinutes();
|
$this->scheduleInstance->command('uploads:clear')->everyTwoMinutes();
|
||||||
|
|
||||||
@@ -80,12 +72,12 @@ class Kernel extends ConsoleKernel
|
|||||||
$this->scheduleUpdates();
|
$this->scheduleUpdates();
|
||||||
|
|
||||||
// Server Jobs
|
// Server Jobs
|
||||||
$this->checkResources();
|
$this->scheduleInstance->job(new ServerResourceManager)->everyMinute()->onOneServer();
|
||||||
|
|
||||||
$this->pullImages();
|
$this->pullImages();
|
||||||
|
|
||||||
$this->checkScheduledBackups();
|
// Scheduled Jobs (Backups & Tasks)
|
||||||
$this->checkScheduledTasks();
|
$this->scheduleInstance->job(new ScheduledJobManager)->everyMinute()->onOneServer();
|
||||||
|
|
||||||
$this->scheduleInstance->job(new RegenerateSslCertJob)->twiceDaily();
|
$this->scheduleInstance->job(new RegenerateSslCertJob)->twiceDaily();
|
||||||
|
|
||||||
@@ -136,231 +128,6 @@ class Kernel extends ConsoleKernel
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private function checkResources(): void
|
|
||||||
{
|
|
||||||
if (isCloud()) {
|
|
||||||
$servers = $this->allServers->whereRelation('team.subscription', 'stripe_invoice_paid', true)->get();
|
|
||||||
$own = Team::find(0)->servers;
|
|
||||||
$servers = $servers->merge($own);
|
|
||||||
} else {
|
|
||||||
$servers = $this->allServers->get();
|
|
||||||
}
|
|
||||||
|
|
||||||
foreach ($servers as $server) {
|
|
||||||
try {
|
|
||||||
$serverTimezone = data_get($server->settings, 'server_timezone', $this->instanceTimezone);
|
|
||||||
if (validate_timezone($serverTimezone) === false) {
|
|
||||||
$serverTimezone = config('app.timezone');
|
|
||||||
}
|
|
||||||
|
|
||||||
// Sentinel check
|
|
||||||
$lastSentinelUpdate = $server->sentinel_updated_at;
|
|
||||||
if (Carbon::parse($lastSentinelUpdate)->isBefore(now()->subSeconds($server->waitBeforeDoingSshCheck()))) {
|
|
||||||
// Check container status every minute if Sentinel does not activated
|
|
||||||
if (isCloud()) {
|
|
||||||
$this->scheduleInstance->job(new ServerCheckJob($server))->timezone($serverTimezone)->everyFiveMinutes()->onOneServer();
|
|
||||||
} else {
|
|
||||||
$this->scheduleInstance->job(new ServerCheckJob($server))->timezone($serverTimezone)->everyMinute()->onOneServer();
|
|
||||||
}
|
|
||||||
// $this->scheduleInstance->job(new \App\Jobs\ServerCheckNewJob($server))->everyFiveMinutes()->onOneServer();
|
|
||||||
|
|
||||||
$serverDiskUsageCheckFrequency = data_get($server->settings, 'server_disk_usage_check_frequency', '0 * * * *');
|
|
||||||
if (isset(VALID_CRON_STRINGS[$serverDiskUsageCheckFrequency])) {
|
|
||||||
$serverDiskUsageCheckFrequency = VALID_CRON_STRINGS[$serverDiskUsageCheckFrequency];
|
|
||||||
}
|
|
||||||
$this->scheduleInstance->job(new ServerStorageCheckJob($server))->cron($serverDiskUsageCheckFrequency)->timezone($serverTimezone)->onOneServer();
|
|
||||||
}
|
|
||||||
|
|
||||||
$dockerCleanupFrequency = data_get($server->settings, 'docker_cleanup_frequency', '0 * * * *');
|
|
||||||
if (isset(VALID_CRON_STRINGS[$dockerCleanupFrequency])) {
|
|
||||||
$dockerCleanupFrequency = VALID_CRON_STRINGS[$dockerCleanupFrequency];
|
|
||||||
}
|
|
||||||
$this->scheduleInstance->job(new DockerCleanupJob($server))->cron($dockerCleanupFrequency)->timezone($serverTimezone)->onOneServer();
|
|
||||||
|
|
||||||
// Server patch check - weekly
|
|
||||||
$this->scheduleInstance->job(new ServerPatchCheckJob($server))->weekly()->timezone($serverTimezone)->onOneServer();
|
|
||||||
|
|
||||||
// Cleanup multiplexed connections every hour
|
|
||||||
// $this->scheduleInstance->job(new ServerCleanupMux($server))->hourly()->onOneServer();
|
|
||||||
|
|
||||||
// Temporary solution until we have better memory management for Sentinel
|
|
||||||
if ($server->isSentinelEnabled()) {
|
|
||||||
$this->scheduleInstance->job(function () use ($server) {
|
|
||||||
$server->restartContainer('coolify-sentinel');
|
|
||||||
})->daily()->onOneServer();
|
|
||||||
}
|
|
||||||
} catch (\Exception $e) {
|
|
||||||
Log::error('Error checking resources: '.$e->getMessage());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private function checkScheduledBackups(): void
|
|
||||||
{
|
|
||||||
$scheduled_backups = ScheduledDatabaseBackup::where('enabled', true)->get();
|
|
||||||
|
|
||||||
if ($scheduled_backups->isEmpty()) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
$finalScheduledBackups = collect();
|
|
||||||
foreach ($scheduled_backups as $scheduled_backup) {
|
|
||||||
if (blank(data_get($scheduled_backup, 'database'))) {
|
|
||||||
$scheduled_backup->delete();
|
|
||||||
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
$server = $scheduled_backup->server();
|
|
||||||
if (blank($server)) {
|
|
||||||
$scheduled_backup->delete();
|
|
||||||
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if ($server->isFunctional() === false) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if (isCloud() && data_get($server->team->subscription, 'stripe_invoice_paid', false) === false && $server->team->id !== 0) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
$finalScheduledBackups->push($scheduled_backup);
|
|
||||||
}
|
|
||||||
|
|
||||||
foreach ($finalScheduledBackups as $scheduled_backup) {
|
|
||||||
try {
|
|
||||||
if (isset(VALID_CRON_STRINGS[$scheduled_backup->frequency])) {
|
|
||||||
$scheduled_backup->frequency = VALID_CRON_STRINGS[$scheduled_backup->frequency];
|
|
||||||
}
|
|
||||||
$server = $scheduled_backup->server();
|
|
||||||
$serverTimezone = data_get($server->settings, 'server_timezone', $this->instanceTimezone);
|
|
||||||
|
|
||||||
if (validate_timezone($serverTimezone) === false) {
|
|
||||||
$serverTimezone = config('app.timezone');
|
|
||||||
}
|
|
||||||
|
|
||||||
if (isset(VALID_CRON_STRINGS[$scheduled_backup->frequency])) {
|
|
||||||
$scheduled_backup->frequency = VALID_CRON_STRINGS[$scheduled_backup->frequency];
|
|
||||||
}
|
|
||||||
$serverTimezone = data_get($server->settings, 'server_timezone', $this->instanceTimezone);
|
|
||||||
|
|
||||||
// Check if the backup should run now
|
|
||||||
$cron = new CronExpression($scheduled_backup->frequency);
|
|
||||||
$now = Carbon::now($serverTimezone);
|
|
||||||
|
|
||||||
if ($cron->isDue($now)) {
|
|
||||||
Log::channel('scheduled')->info('Backup job running now', [
|
|
||||||
'backup_id' => $scheduled_backup->id,
|
|
||||||
'backup_name' => $scheduled_backup->name ?? 'unnamed',
|
|
||||||
'server_name' => $server->name,
|
|
||||||
'frequency' => $scheduled_backup->frequency,
|
|
||||||
'timezone' => $serverTimezone,
|
|
||||||
'current_time' => $now->format('Y-m-d H:i:s T'),
|
|
||||||
]);
|
|
||||||
}
|
|
||||||
|
|
||||||
$this->scheduleInstance->job(new DatabaseBackupJob(
|
|
||||||
backup: $scheduled_backup
|
|
||||||
))->cron($scheduled_backup->frequency)->timezone($serverTimezone)->onOneServer();
|
|
||||||
|
|
||||||
} catch (\Exception $e) {
|
|
||||||
Log::channel('scheduled-errors')->error('Error scheduling backup', [
|
|
||||||
'backup_id' => $scheduled_backup->id,
|
|
||||||
'backup_name' => $scheduled_backup->name ?? 'unnamed',
|
|
||||||
'error' => $e->getMessage(),
|
|
||||||
'trace' => $e->getTraceAsString(),
|
|
||||||
]);
|
|
||||||
Log::error('Error scheduling backup: '.$e->getMessage());
|
|
||||||
Log::error($e->getTraceAsString());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private function checkScheduledTasks(): void
|
|
||||||
{
|
|
||||||
$scheduled_tasks = ScheduledTask::where('enabled', true)->get();
|
|
||||||
|
|
||||||
if ($scheduled_tasks->isEmpty()) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
$finalScheduledTasks = collect();
|
|
||||||
foreach ($scheduled_tasks as $scheduled_task) {
|
|
||||||
$service = $scheduled_task->service;
|
|
||||||
$application = $scheduled_task->application;
|
|
||||||
|
|
||||||
$server = $scheduled_task->server();
|
|
||||||
if (blank($server)) {
|
|
||||||
$scheduled_task->delete();
|
|
||||||
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
if ($server->isFunctional() === false) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (isCloud() && data_get($server->team->subscription, 'stripe_invoice_paid', false) === false && $server->team->id !== 0) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (! $service && ! $application) {
|
|
||||||
$scheduled_task->delete();
|
|
||||||
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
if ($application && str($application->status)->contains('running') === false) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if ($service && str($service->status)->contains('running') === false) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
$finalScheduledTasks->push($scheduled_task);
|
|
||||||
}
|
|
||||||
|
|
||||||
foreach ($finalScheduledTasks as $scheduled_task) {
|
|
||||||
try {
|
|
||||||
$server = $scheduled_task->server();
|
|
||||||
if (isset(VALID_CRON_STRINGS[$scheduled_task->frequency])) {
|
|
||||||
$scheduled_task->frequency = VALID_CRON_STRINGS[$scheduled_task->frequency];
|
|
||||||
}
|
|
||||||
$serverTimezone = data_get($server->settings, 'server_timezone', $this->instanceTimezone);
|
|
||||||
|
|
||||||
if (validate_timezone($serverTimezone) === false) {
|
|
||||||
$serverTimezone = config('app.timezone');
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check if the task should run now
|
|
||||||
$cron = new CronExpression($scheduled_task->frequency);
|
|
||||||
$now = Carbon::now($serverTimezone);
|
|
||||||
|
|
||||||
if ($cron->isDue($now)) {
|
|
||||||
Log::channel('scheduled')->info('Task job running now', [
|
|
||||||
'task_id' => $scheduled_task->id,
|
|
||||||
'task_name' => $scheduled_task->name ?? 'unnamed',
|
|
||||||
'server_name' => $server->name,
|
|
||||||
'frequency' => $scheduled_task->frequency,
|
|
||||||
'timezone' => $serverTimezone,
|
|
||||||
'type' => $scheduled_task->service ? 'service' : 'application',
|
|
||||||
'current_time' => $now->format('Y-m-d H:i:s T'),
|
|
||||||
]);
|
|
||||||
}
|
|
||||||
|
|
||||||
$this->scheduleInstance->job(new ScheduledTaskJob(
|
|
||||||
task: $scheduled_task
|
|
||||||
))->cron($scheduled_task->frequency)->timezone($serverTimezone)->onOneServer();
|
|
||||||
|
|
||||||
} catch (\Exception $e) {
|
|
||||||
Log::channel('scheduled-errors')->error('Error scheduling task', [
|
|
||||||
'task_id' => $scheduled_task->id,
|
|
||||||
'task_name' => $scheduled_task->name ?? 'unnamed',
|
|
||||||
'error' => $e->getMessage(),
|
|
||||||
'trace' => $e->getTraceAsString(),
|
|
||||||
]);
|
|
||||||
Log::error('Error scheduling task: '.$e->getMessage());
|
|
||||||
Log::error($e->getTraceAsString());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
protected function commands(): void
|
protected function commands(): void
|
||||||
{
|
{
|
||||||
$this->load(__DIR__.'/Commands');
|
$this->load(__DIR__.'/Commands');
|
||||||
|
|||||||
Reference in New Issue
Block a user