onQueue($this->determineQueue()); } private function determineQueue(): string { $preferredQueue = 'crons'; $fallbackQueue = 'high'; $configuredQueues = explode(',', env('HORIZON_QUEUES', 'high,default')); return in_array($preferredQueue, $configuredQueues) ? $preferredQueue : $fallbackQueue; } /** * Get the middleware the job should pass through. */ public function middleware(): array { return [ (new WithoutOverlapping('scheduled-job-manager')) ->releaseAfter(60), // Release the lock after 60 seconds if job fails ]; } public function handle(): void { // Freeze the execution time at the start of the job $this->executionTime = Carbon::now(); // Process backups - don't let failures stop task processing try { $this->processScheduledBackups(); } catch (\Exception $e) { Log::channel('scheduled-errors')->error('Failed to process scheduled backups', [ 'error' => $e->getMessage(), 'trace' => $e->getTraceAsString(), ]); } // Process tasks - don't let failures stop the job manager try { $this->processScheduledTasks(); } catch (\Exception $e) { Log::channel('scheduled-errors')->error('Failed to process scheduled tasks', [ 'error' => $e->getMessage(), 'trace' => $e->getTraceAsString(), ]); } // Process Docker cleanups - don't let failures stop the job manager try { $this->processDockerCleanups(); } catch (\Exception $e) { Log::channel('scheduled-errors')->error('Failed to process docker cleanups', [ 'error' => $e->getMessage(), 'trace' => $e->getTraceAsString(), ]); } } private function processScheduledBackups(): void { $backups = ScheduledDatabaseBackup::with(['database']) ->where('enabled', true) ->get(); foreach ($backups as $backup) { try { // Apply the same filtering logic as the original if (! $this->shouldProcessBackup($backup)) { continue; } $server = $backup->server(); $serverTimezone = data_get($server->settings, 'server_timezone', config('app.timezone')); if (validate_timezone($serverTimezone) === false) { $serverTimezone = config('app.timezone'); } $frequency = $backup->frequency; if (isset(VALID_CRON_STRINGS[$frequency])) { $frequency = VALID_CRON_STRINGS[$frequency]; } if ($this->shouldRunNow($frequency, $serverTimezone)) { DatabaseBackupJob::dispatch($backup); } } catch (\Exception $e) { Log::channel('scheduled-errors')->error('Error processing backup', [ 'backup_id' => $backup->id, 'error' => $e->getMessage(), ]); } } } private function processScheduledTasks(): void { $tasks = ScheduledTask::with(['service', 'application']) ->where('enabled', true) ->get(); foreach ($tasks as $task) { try { if (! $this->shouldProcessTask($task)) { continue; } $server = $task->server(); $serverTimezone = data_get($server->settings, 'server_timezone', config('app.timezone')); if (validate_timezone($serverTimezone) === false) { $serverTimezone = config('app.timezone'); } $frequency = $task->frequency; if (isset(VALID_CRON_STRINGS[$frequency])) { $frequency = VALID_CRON_STRINGS[$frequency]; } if ($this->shouldRunNow($frequency, $serverTimezone)) { ScheduledTaskJob::dispatch($task); } } catch (\Exception $e) { Log::channel('scheduled-errors')->error('Error processing task', [ 'task_id' => $task->id, 'error' => $e->getMessage(), ]); } } } private function shouldProcessBackup(ScheduledDatabaseBackup $backup): bool { if (blank(data_get($backup, 'database'))) { $backup->delete(); return false; } $server = $backup->server(); if (blank($server)) { $backup->delete(); return false; } if ($server->isFunctional() === false) { return false; } if (isCloud() && data_get($server->team->subscription, 'stripe_invoice_paid', false) === false && $server->team->id !== 0) { return false; } return true; } private function shouldProcessTask(ScheduledTask $task): bool { $service = $task->service; $application = $task->application; $server = $task->server(); if (blank($server)) { $task->delete(); return false; } if ($server->isFunctional() === false) { return false; } if (isCloud() && data_get($server->team->subscription, 'stripe_invoice_paid', false) === false && $server->team->id !== 0) { return false; } if (! $service && ! $application) { $task->delete(); return false; } if ($application && str($application->status)->contains('running') === false) { return false; } if ($service && str($service->status)->contains('running') === false) { return false; } return true; } private function shouldRunNow(string $frequency, string $timezone): bool { $cron = new CronExpression($frequency); // Use the frozen execution time, not the current time // Fallback to current time if execution time is not set (shouldn't happen) $baseTime = $this->executionTime ?? Carbon::now(); $executionTime = $baseTime->copy()->setTimezone($timezone); return $cron->isDue($executionTime); } private function processDockerCleanups(): void { // Get all servers that need cleanup checks $servers = $this->getServersForCleanup(); foreach ($servers as $server) { try { if (! $this->shouldProcessDockerCleanup($server)) { continue; } $serverTimezone = data_get($server->settings, 'server_timezone', config('app.timezone')); if (validate_timezone($serverTimezone) === false) { $serverTimezone = config('app.timezone'); } $frequency = data_get($server->settings, 'docker_cleanup_frequency', '0 * * * *'); if (isset(VALID_CRON_STRINGS[$frequency])) { $frequency = VALID_CRON_STRINGS[$frequency]; } // Use the frozen execution time for consistent evaluation if ($this->shouldRunNow($frequency, $serverTimezone)) { DockerCleanupJob::dispatch( $server, false, $server->settings->delete_unused_volumes, $server->settings->delete_unused_networks ); } } catch (\Exception $e) { Log::channel('scheduled-errors')->error('Error processing docker cleanup', [ 'server_id' => $server->id, 'server_name' => $server->name, 'error' => $e->getMessage(), ]); } } } private function getServersForCleanup(): Collection { $query = Server::with('settings') ->where('ip', '!=', '1.2.3.4'); if (isCloud()) { $servers = $query->whereRelation('team.subscription', 'stripe_invoice_paid', true)->get(); $own = Team::find(0)->servers()->with('settings')->get(); return $servers->merge($own); } return $query->get(); } private function shouldProcessDockerCleanup(Server $server): bool { if (! $server->isFunctional()) { return false; } // In cloud, check subscription status (except team 0) if (isCloud() && $server->team_id !== 0) { if (data_get($server->team->subscription, 'stripe_invoice_paid', false) === false) { return false; } } return true; } }