diff --git a/app/Console/Commands/CleanupStuckedResources.php b/app/Console/Commands/CleanupStuckedResources.php index def3d5a2c..0b5eef84d 100644 --- a/app/Console/Commands/CleanupStuckedResources.php +++ b/app/Console/Commands/CleanupStuckedResources.php @@ -39,6 +39,11 @@ class CleanupStuckedResources extends Command $servers = Server::all()->filter(function ($server) { return $server->isFunctional(); }); + if (isCloud()) { + $servers = $servers->filter(function ($server) { + return data_get($server->team->subscription, 'stripe_invoice_paid', false) === true; + }); + } foreach ($servers as $server) { CleanupHelperContainersJob::dispatch($server); } diff --git a/app/Console/Commands/HorizonManage.php b/app/Console/Commands/HorizonManage.php new file mode 100644 index 000000000..ca2da147c --- /dev/null +++ b/app/Console/Commands/HorizonManage.php @@ -0,0 +1,178 @@ +option('can-i-restart-this-worker')) { + return $this->isThereAJobInProgress(); + } + + if ($this->option('job-status')) { + return $this->getJobStatus($this->option('job-status')); + } + + $action = select( + label: 'What to do?', + options: [ + 'pending' => 'Pending Jobs', + 'running' => 'Running Jobs', + 'can-i-restart-this-worker' => 'Can I restart this worker?', + 'job-status' => 'Job Status', + 'workers' => 'Workers', + 'failed' => 'Failed Jobs', + 'failed-delete' => 'Failed Jobs - Delete', + 'purge-queues' => 'Purge Queues', + ] + ); + + if ($action === 'can-i-restart-this-worker') { + $this->isThereAJobInProgress(); + } + + if ($action === 'job-status') { + $jobId = text('Which job to check?'); + $jobStatus = $this->getJobStatus($jobId); + $this->info('Job Status: '.$jobStatus); + } + + if ($action === 'pending') { + $pendingJobs = app(JobRepository::class)->getPending(); + $pendingJobsTable = []; + if (count($pendingJobs) === 0) { + $this->info('No pending jobs found.'); + + return; + } + foreach ($pendingJobs as $pendingJob) { + $pendingJobsTable[] = [ + 'id' => $pendingJob->id, + 'name' => $pendingJob->name, + 'status' => $pendingJob->status, + 'reserved_at' => $pendingJob->reserved_at ? now()->parse($pendingJob->reserved_at)->format('Y-m-d H:i:s') : null, + ]; + } + table($pendingJobsTable); + } + + if ($action === 'failed') { + $failedJobs = app(JobRepository::class)->getFailed(); + $failedJobsTable = []; + if (count($failedJobs) === 0) { + $this->info('No failed jobs found.'); + + return; + } + foreach ($failedJobs as $failedJob) { + $failedJobsTable[] = [ + 'id' => $failedJob->id, + 'name' => $failedJob->name, + 'failed_at' => $failedJob->failed_at ? now()->parse($failedJob->failed_at)->format('Y-m-d H:i:s') : null, + ]; + } + table($failedJobsTable); + } + + if ($action === 'failed-delete') { + $failedJobs = app(JobRepository::class)->getFailed(); + $failedJobsTable = []; + foreach ($failedJobs as $failedJob) { + $failedJobsTable[] = [ + 'id' => $failedJob->id, + 'name' => $failedJob->name, + 'failed_at' => $failedJob->failed_at ? now()->parse($failedJob->failed_at)->format('Y-m-d H:i:s') : null, + ]; + } + app(MetricsRepository::class)->clear(); + if (count($failedJobsTable) === 0) { + $this->info('No failed jobs found.'); + + return; + } + $jobIds = multiselect( + label: 'Which job to delete?', + options: collect($failedJobsTable)->mapWithKeys(fn ($job) => [$job['id'] => $job['id'].' - '.$job['name']])->toArray(), + ); + foreach ($jobIds as $jobId) { + Artisan::queue('horizon:forget', ['id' => $jobId]); + } + } + + if ($action === 'running') { + $redisJobRepository = app(CustomJobRepository::class); + $runningJobs = $redisJobRepository->getReservedJobs(); + $runningJobsTable = []; + if (count($runningJobs) === 0) { + $this->info('No running jobs found.'); + + return; + } + foreach ($runningJobs as $runningJob) { + $runningJobsTable[] = [ + 'id' => $runningJob->id, + 'name' => $runningJob->name, + 'reserved_at' => $runningJob->reserved_at ? now()->parse($runningJob->reserved_at)->format('Y-m-d H:i:s') : null, + ]; + } + table($runningJobsTable); + } + + if ($action === 'workers') { + $redisJobRepository = app(CustomJobRepository::class); + $workers = $redisJobRepository->getHorizonWorkers(); + $workersTable = []; + foreach ($workers as $worker) { + $workersTable[] = [ + 'name' => $worker->name, + ]; + } + table($workersTable); + } + + if ($action === 'purge-queues') { + $getQueues = app(CustomJobRepository::class)->getQueues(); + $queueName = select( + label: 'Which queue to purge?', + options: $getQueues, + ); + $redisJobRepository = app(RedisJobRepository::class); + $redisJobRepository->purge($queueName); + } + } + + public function isThereAJobInProgress() + { + $runningJobs = ApplicationDeploymentQueue::where('horizon_job_worker', gethostname())->where('status', ApplicationDeploymentStatus::IN_PROGRESS->value)->get(); + $count = $runningJobs->count(); + if ($count === 0) { + return false; + } + + return true; + } + + public function getJobStatus(string $jobId) + { + return getJobStatus($jobId); + } +} diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index e8781b01e..8b4240412 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -91,7 +91,13 @@ class Kernel extends ConsoleKernel private function pullImages(): void { - $servers = $this->allServers->whereRelation('settings', 'is_usable', true)->whereRelation('settings', 'is_reachable', true)->get(); + if (isCloud()) { + $servers = $this->allServers->whereRelation('team.subscription', 'stripe_invoice_paid', true)->whereRelation('settings', 'is_usable', true)->whereRelation('settings', 'is_reachable', true)->get(); + $own = Team::find(0)->servers; + $servers = $servers->merge($own); + } else { + $servers = $this->allServers->whereRelation('settings', 'is_usable', true)->whereRelation('settings', 'is_reachable', true)->get(); + } foreach ($servers as $server) { if ($server->isSentinelEnabled()) { $this->scheduleInstance->job(function () use ($server) { @@ -124,7 +130,7 @@ class Kernel extends ConsoleKernel private function checkResources(): void { if (isCloud()) { - $servers = $this->allServers->whereHas('team.subscription')->get(); + $servers = $this->allServers->whereRelation('team.subscription', 'stripe_invoice_paid', true)->get(); $own = Team::find(0)->servers; $servers = $servers->merge($own); } else { @@ -171,18 +177,40 @@ class Kernel extends ConsoleKernel if ($scheduled_backups->isEmpty()) { return; } + $finalScheduledBackups = collect(); foreach ($scheduled_backups as $scheduled_backup) { - if (is_null(data_get($scheduled_backup, 'database'))) { + if (blank(data_get($scheduled_backup, 'database'))) { $scheduled_backup->delete(); continue; } - $server = $scheduled_backup->server(); + if (blank($server)) { + $scheduled_backup->delete(); - if (is_null($server)) { continue; } + if ($server->isFunctional() === false) { + continue; + } + if (isCloud() && data_get($server->team->subscription, 'stripe_invoice_paid', false) === false && $server->team->id !== 0) { + continue; + } + $finalScheduledBackups->push($scheduled_backup); + } + + foreach ($finalScheduledBackups as $scheduled_backup) { + if (isset(VALID_CRON_STRINGS[$scheduled_backup->frequency])) { + $scheduled_backup->frequency = VALID_CRON_STRINGS[$scheduled_backup->frequency]; + } + + $server = $scheduled_backup->server(); + $serverTimezone = data_get($server->settings, 'server_timezone', $this->instanceTimezone); + + if (validate_timezone($serverTimezone) === false) { + $serverTimezone = config('app.timezone'); + } + if (isset(VALID_CRON_STRINGS[$scheduled_backup->frequency])) { $scheduled_backup->frequency = VALID_CRON_STRINGS[$scheduled_backup->frequency]; } @@ -199,35 +227,52 @@ class Kernel extends ConsoleKernel if ($scheduled_tasks->isEmpty()) { return; } + $finalScheduledTasks = collect(); foreach ($scheduled_tasks as $scheduled_task) { $service = $scheduled_task->service; $application = $scheduled_task->application; - if (! $application && ! $service) { + $server = $scheduled_task->server(); + if (blank($server)) { $scheduled_task->delete(); continue; } - if ($application) { - if (str($application->status)->contains('running') === false) { - continue; - } - } - if ($service) { - if (str($service->status)->contains('running') === false) { - continue; - } - } - $server = $scheduled_task->server(); - if (! $server) { + if ($server->isFunctional() === false) { continue; } + if (isCloud() && data_get($server->team->subscription, 'stripe_invoice_paid', false) === false && $server->team->id !== 0) { + continue; + } + + if (! $service && ! $application) { + $scheduled_task->delete(); + + continue; + } + + if ($application && str($application->status)->contains('running') === false) { + continue; + } + if ($service && str($service->status)->contains('running') === false) { + continue; + } + + $finalScheduledTasks->push($scheduled_task); + } + + foreach ($finalScheduledTasks as $scheduled_task) { + $server = $scheduled_task->server(); if (isset(VALID_CRON_STRINGS[$scheduled_task->frequency])) { $scheduled_task->frequency = VALID_CRON_STRINGS[$scheduled_task->frequency]; } $serverTimezone = data_get($server->settings, 'server_timezone', $this->instanceTimezone); + + if (validate_timezone($serverTimezone) === false) { + $serverTimezone = config('app.timezone'); + } $this->scheduleInstance->job(new ScheduledTaskJob( task: $scheduled_task ))->cron($scheduled_task->frequency)->timezone($serverTimezone)->onOneServer(); diff --git a/app/Contracts/CustomJobRepositoryInterface.php b/app/Contracts/CustomJobRepositoryInterface.php new file mode 100644 index 000000000..1fbd71f46 --- /dev/null +++ b/app/Contracts/CustomJobRepositoryInterface.php @@ -0,0 +1,19 @@ +application_deployment_queue_id]; + } + + public function __construct(public int $application_deployment_queue_id) { $this->onQueue('high'); + $this->application_deployment_queue = ApplicationDeploymentQueue::find($this->application_deployment_queue_id); $this->nixpacks_plan_json = collect([]); - $this->application_deployment_queue = ApplicationDeploymentQueue::find($application_deployment_queue_id); $this->application = Application::find($this->application_deployment_queue->application_id); $this->build_pack = data_get($this->application, 'build_pack'); $this->build_args = collect([]); - $this->application_deployment_queue_id = $application_deployment_queue_id; $this->deployment_uuid = $this->application_deployment_queue->deployment_uuid; $this->pull_request_id = $this->application_deployment_queue->pull_request_id; $this->commit = $this->application_deployment_queue->commit; @@ -237,15 +242,11 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue } } - public function tags(): array - { - return ['server:'.gethostname()]; - } - public function handle(): void { $this->application_deployment_queue->update([ 'status' => ApplicationDeploymentStatus::IN_PROGRESS->value, + 'horizon_job_worker' => gethostname(), ]); if ($this->server->isFunctional() === false) { $this->application_deployment_queue->addLogEntry('Server is not functional.'); @@ -2391,7 +2392,8 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf"); queue_next_deployment($this->application); // If the deployment is cancelled by the user, don't update the status if ( - $this->application_deployment_queue->status !== ApplicationDeploymentStatus::CANCELLED_BY_USER->value && $this->application_deployment_queue->status !== ApplicationDeploymentStatus::FAILED->value + $this->application_deployment_queue->status !== ApplicationDeploymentStatus::CANCELLED_BY_USER->value && + $this->application_deployment_queue->status !== ApplicationDeploymentStatus::FAILED->value ) { $this->application_deployment_queue->update([ 'status' => $status, diff --git a/app/Jobs/CheckAndStartSentinelJob.php b/app/Jobs/CheckAndStartSentinelJob.php index 788db89ea..304b2a15c 100644 --- a/app/Jobs/CheckAndStartSentinelJob.php +++ b/app/Jobs/CheckAndStartSentinelJob.php @@ -24,7 +24,7 @@ class CheckAndStartSentinelJob implements ShouldBeEncrypted, ShouldQueue $latestVersion = get_latest_sentinel_version(); // Check if sentinel is running - $sentinelFound = instant_remote_process(['docker inspect coolify-sentinel'], $this->server, false); + $sentinelFound = instant_remote_process_with_timeout(['docker inspect coolify-sentinel'], $this->server, false, 10); $sentinelFoundJson = json_decode($sentinelFound, true); $sentinelStatus = data_get($sentinelFoundJson, '0.State.Status', 'exited'); if ($sentinelStatus !== 'running') { @@ -33,7 +33,7 @@ class CheckAndStartSentinelJob implements ShouldBeEncrypted, ShouldQueue return; } // If sentinel is running, check if it needs an update - $runningVersion = instant_remote_process(['docker exec coolify-sentinel sh -c "curl http://127.0.0.1:8888/api/version"'], $this->server, false); + $runningVersion = instant_remote_process_with_timeout(['docker exec coolify-sentinel sh -c "curl http://127.0.0.1:8888/api/version"'], $this->server, false); if (empty($runningVersion)) { $runningVersion = '0.0.0'; } diff --git a/app/Jobs/CleanupHelperContainersJob.php b/app/Jobs/CleanupHelperContainersJob.php index f185ab781..0e1fcb4d7 100644 --- a/app/Jobs/CleanupHelperContainersJob.php +++ b/app/Jobs/CleanupHelperContainersJob.php @@ -20,11 +20,11 @@ class CleanupHelperContainersJob implements ShouldBeEncrypted, ShouldBeUnique, S public function handle(): void { try { - $containers = instant_remote_process(['docker container ps --format \'{{json .}}\' | jq -s \'map(select(.Image | contains("ghcr.io/coollabsio/coolify-helper")))\''], $this->server, false); + $containers = instant_remote_process_with_timeout(['docker container ps --format \'{{json .}}\' | jq -s \'map(select(.Image | contains("ghcr.io/coollabsio/coolify-helper")))\''], $this->server, false); $containerIds = collect(json_decode($containers))->pluck('ID'); if ($containerIds->count() > 0) { foreach ($containerIds as $containerId) { - instant_remote_process(['docker container rm -f '.$containerId], $this->server, false); + instant_remote_process_with_timeout(['docker container rm -f '.$containerId], $this->server, false); } } } catch (\Throwable $e) { diff --git a/app/Jobs/PushServerUpdateJob.php b/app/Jobs/PushServerUpdateJob.php index 24f8d1e6b..93b203fcb 100644 --- a/app/Jobs/PushServerUpdateJob.php +++ b/app/Jobs/PushServerUpdateJob.php @@ -19,6 +19,7 @@ use Illuminate\Contracts\Queue\ShouldBeEncrypted; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; +use Illuminate\Queue\Middleware\WithoutOverlapping; use Illuminate\Queue\SerializesModels; use Illuminate\Support\Collection; @@ -68,6 +69,11 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue public bool $foundLogDrainContainer = false; + public function middleware(): array + { + return [(new WithoutOverlapping($this->server->uuid))->dontRelease()]; + } + public function backoff(): int { return isDev() ? 1 : 3; diff --git a/app/Livewire/Project/Application/Deployment/Show.php b/app/Livewire/Project/Application/Deployment/Show.php index 4e86b5c7e..7b2ac09d3 100644 --- a/app/Livewire/Project/Application/Deployment/Show.php +++ b/app/Livewire/Project/Application/Deployment/Show.php @@ -14,6 +14,8 @@ class Show extends Component public string $deployment_uuid; + public string $horizon_job_status; + public $isKeepAliveOn = true; protected $listeners = ['refreshQueue']; @@ -44,7 +46,9 @@ class Show extends Component } $this->application = $application; $this->application_deployment_queue = $application_deployment_queue; + $this->horizon_job_status = $this->application_deployment_queue->getHorizonJobStatus(); $this->deployment_uuid = $deploymentUuid; + $this->isKeepAliveOn(); } public function refreshQueue() @@ -52,13 +56,21 @@ class Show extends Component $this->application_deployment_queue->refresh(); } + private function isKeepAliveOn() + { + if (data_get($this->application_deployment_queue, 'status') === 'finished' || data_get($this->application_deployment_queue, 'status') === 'failed') { + $this->isKeepAliveOn = false; + } else { + $this->isKeepAliveOn = true; + } + } + public function polling() { $this->dispatch('deploymentFinished'); $this->application_deployment_queue->refresh(); - if (data_get($this->application_deployment_queue, 'status') === 'finished' || data_get($this->application_deployment_queue, 'status') === 'failed') { - $this->isKeepAliveOn = false; - } + $this->horizon_job_status = $this->application_deployment_queue->getHorizonJobStatus(); + $this->isKeepAliveOn(); } public function getLogLinesProperty() diff --git a/app/Models/ApplicationDeploymentQueue.php b/app/Models/ApplicationDeploymentQueue.php index c261c30c6..95c78d725 100644 --- a/app/Models/ApplicationDeploymentQueue.php +++ b/app/Models/ApplicationDeploymentQueue.php @@ -70,6 +70,11 @@ class ApplicationDeploymentQueue extends Model return collect(json_decode($this->logs))->where('name', $name)->first()?->output ?? null; } + public function getHorizonJobStatus() + { + return getJobStatus($this->horizon_job_id); + } + public function commitMessage() { if (empty($this->commit_message) || is_null($this->commit_message)) { diff --git a/app/Models/Server.php b/app/Models/Server.php index 8d11e23a9..cc6bff2cf 100644 --- a/app/Models/Server.php +++ b/app/Models/Server.php @@ -656,9 +656,9 @@ $schema://$host { $containers = collect([]); $containerReplicates = collect([]); if ($this->isSwarm()) { - $containers = instant_remote_process(["docker service inspect $(docker service ls -q) --format '{{json .}}'"], $this, false); + $containers = instant_remote_process_with_timeout(["docker service inspect $(docker service ls -q) --format '{{json .}}'"], $this, false); $containers = format_docker_command_output_to_json($containers); - $containerReplicates = instant_remote_process(["docker service ls --format '{{json .}}'"], $this, false); + $containerReplicates = instant_remote_process_with_timeout(["docker service ls --format '{{json .}}'"], $this, false); if ($containerReplicates) { $containerReplicates = format_docker_command_output_to_json($containerReplicates); foreach ($containerReplicates as $containerReplica) { @@ -682,7 +682,7 @@ $schema://$host { } } } else { - $containers = instant_remote_process(["docker container inspect $(docker container ls -aq) --format '{{json .}}'"], $this, false); + $containers = instant_remote_process_with_timeout(["docker container inspect $(docker container ls -aq) --format '{{json .}}'"], $this, false); $containers = format_docker_command_output_to_json($containers); $containerReplicates = collect([]); } diff --git a/app/Providers/HorizonServiceProvider.php b/app/Providers/HorizonServiceProvider.php index 2e2b79a59..0caa3a3a9 100644 --- a/app/Providers/HorizonServiceProvider.php +++ b/app/Providers/HorizonServiceProvider.php @@ -2,32 +2,54 @@ namespace App\Providers; +use App\Contracts\CustomJobRepositoryInterface; +use App\Models\ApplicationDeploymentQueue; use App\Models\User; +use App\Repositories\CustomJobRepository; +use Illuminate\Support\Facades\Event; use Illuminate\Support\Facades\Gate; -use Laravel\Horizon\Horizon; +use Laravel\Horizon\Contracts\JobRepository; +use Laravel\Horizon\Events\JobReserved; use Laravel\Horizon\HorizonApplicationServiceProvider; class HorizonServiceProvider extends HorizonApplicationServiceProvider { /** - * Bootstrap any application services. + * Register services. + */ + public function register(): void + { + $this->app->singleton(JobRepository::class, CustomJobRepository::class); + $this->app->singleton(CustomJobRepositoryInterface::class, CustomJobRepository::class); + } + + /** + * Bootstrap services. */ public function boot(): void { parent::boot(); - - // Horizon::routeSmsNotificationsTo('15556667777'); - // Horizon::routeMailNotificationsTo('example@example.com'); - // Horizon::routeSlackNotificationsTo('slack-webhook-url', '#channel'); - - Horizon::night(); + Event::listen(function (JobReserved $event) { + $payload = $event->payload->decoded; + $jobName = $payload['displayName']; + if ($jobName === 'App\Jobs\ApplicationDeploymentJob') { + $tags = $payload['tags']; + $id = $payload['id']; + $deploymentQueueId = collect($tags)->first(function ($tag) { + return str_contains($tag, 'App\Models\ApplicationDeploymentQueue'); + }); + if (blank($deploymentQueueId)) { + return; + } + $deploymentQueueId = explode(':', $deploymentQueueId)[1]; + $deploymentQueue = ApplicationDeploymentQueue::find($deploymentQueueId); + $deploymentQueue->update([ + 'horizon_job_id' => $id, + ]); + } + }); } - /** - * Register the Horizon gate. - * - * This gate determines who can access Horizon in non-local environments. - */ protected function gate(): void { Gate::define('viewHorizon', function ($user) { diff --git a/app/Repositories/CustomJobRepository.php b/app/Repositories/CustomJobRepository.php new file mode 100644 index 000000000..502dd252b --- /dev/null +++ b/app/Repositories/CustomJobRepository.php @@ -0,0 +1,51 @@ +all(); + } + + public function getReservedJobs(): Collection + { + return $this->getJobsByStatus('reserved'); + } + + public function getJobsByStatus(string $status): Collection + { + $jobs = new Collection; + + $this->getRecent()->each(function ($job) use ($jobs, $status) { + if ($job->status === $status) { + $jobs->push($job); + } + }); + + return $jobs; + } + + public function countJobsByStatus(string $status): int + { + return $this->getJobsByStatus($status)->count(); + } + + public function getQueues(): array + { + $queues = $this->connection()->keys('queue:*'); + $queues = array_map(function ($queue) { + return explode(':', $queue)[2]; + }, $queues); + + return $queues; + } +} diff --git a/bootstrap/helpers/remoteProcess.php b/bootstrap/helpers/remoteProcess.php index c7dd2cb83..d1cb93d9a 100644 --- a/bootstrap/helpers/remoteProcess.php +++ b/bootstrap/helpers/remoteProcess.php @@ -71,6 +71,31 @@ function instant_scp(string $source, string $dest, Server $server, $throwError = return $output === 'null' ? null : $output; } +function instant_remote_process_with_timeout(Collection|array $command, Server $server, bool $throwError = true, bool $no_sudo = false): ?string +{ + $command = $command instanceof Collection ? $command->toArray() : $command; + if ($server->isNonRoot() && ! $no_sudo) { + $command = parseCommandsByLineForSudo(collect($command), $server); + } + $command_string = implode("\n", $command); + + // $start_time = microtime(true); + $sshCommand = SshMultiplexingHelper::generateSshCommand($server, $command_string); + $process = Process::timeout(30)->run($sshCommand); + // $end_time = microtime(true); + + // $execution_time = ($end_time - $start_time) * 1000; // Convert to milliseconds + // ray('SSH command execution time:', $execution_time.' ms')->orange(); + + $output = trim($process->output()); + $exitCode = $process->exitCode(); + + if ($exitCode !== 0) { + return $throwError ? excludeCertainErrors($process->errorOutput(), $exitCode) : null; + } + + return $output === 'null' ? null : $output; +} function instant_remote_process(Collection|array $command, Server $server, bool $throwError = true, bool $no_sudo = false): ?string { $command = $command instanceof Collection ? $command->toArray() : $command; diff --git a/bootstrap/helpers/shared.php b/bootstrap/helpers/shared.php index 4cd40f203..5d34c63b6 100644 --- a/bootstrap/helpers/shared.php +++ b/bootstrap/helpers/shared.php @@ -41,6 +41,7 @@ use Illuminate\Support\Facades\Route; use Illuminate\Support\Facades\Validator; use Illuminate\Support\Str; use Illuminate\Support\Stringable; +use Laravel\Horizon\Contracts\JobRepository; use Lcobucci\JWT\Encoding\ChainedFormatter; use Lcobucci\JWT\Encoding\JoseEncoder; use Lcobucci\JWT\Signer\Hmac\Sha256; @@ -1257,14 +1258,22 @@ function get_public_ips() function isAnyDeploymentInprogress() { - // Only use it in the deployment script - $count = ApplicationDeploymentQueue::whereIn('status', [ApplicationDeploymentStatus::IN_PROGRESS, ApplicationDeploymentStatus::QUEUED])->count(); - if ($count > 0) { - echo "There are $count deployments in progress. Exiting...\n"; - exit(1); + $runningJobs = ApplicationDeploymentQueue::where('horizon_job_worker', gethostname())->where('status', ApplicationDeploymentStatus::IN_PROGRESS->value)->get(); + $horizonJobIds = []; + foreach ($runningJobs as $runningJob) { + $horizonJobStatus = getJobStatus($runningJob->horizon_job_id); + if ($horizonJobStatus === 'unknown') { + return true; + } + $horizonJobIds[] = $runningJob->horizon_job_id; } - echo "No deployments in progress.\n"; - exit(0); + if (count($horizonJobIds) === 0) { + echo "No deployments in progress.\n"; + exit(0); + } + $horizonJobIds = collect($horizonJobIds)->unique()->toArray(); + echo 'There are '.count($horizonJobIds)." deployments in progress.\n"; + exit(1); } function isBase64Encoded($strValue) @@ -4124,3 +4133,16 @@ function convertGitUrl(string $gitRepository, string $deploymentType, ?GithubApp 'port' => $providerInfo['port'], ]; } + +function getJobStatus(?string $jobId = null) +{ + if (blank($jobId)) { + return 'unknown'; + } + $jobFound = app(JobRepository::class)->getJobs([$jobId]); + if ($jobFound->isEmpty()) { + return 'unknown'; + } + + return $jobFound->first()->status; +} diff --git a/composer.json b/composer.json index 5a51db531..055059a56 100644 --- a/composer.json +++ b/composer.json @@ -30,7 +30,7 @@ "league/flysystem-sftp-v3": "^3.0", "livewire/livewire": "^3.5", "log1x/laravel-webfonts": "^1.0", - "lorisleiva/laravel-actions": "^2.7", + "lorisleiva/laravel-actions": "^2.8", "nubs/random-name-generator": "^2.2", "phpseclib/phpseclib": "^3.0", "pion/laravel-chunk-upload": "^1.5", @@ -124,4 +124,4 @@ "@php artisan key:generate --ansi" ] } -} +} \ No newline at end of file diff --git a/composer.lock b/composer.lock index 6406b0837..7ca8219c7 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "ccced2490c39e4f6f1bf9b036bfa3ef0", + "content-hash": "85a775fb1a4b9ea329d8d893f43621c2", "packages": [ { "name": "3sidedcube/laravel-redoc", @@ -928,16 +928,16 @@ }, { "name": "aws/aws-sdk-php", - "version": "3.336.11", + "version": "3.336.12", "source": { "type": "git", "url": "https://github.com/aws/aws-sdk-php.git", - "reference": "442039c766a82f06ecfecb0ac2c610d6aaba228d" + "reference": "a173ab3af8d9186d266e4937d8254597f36a9e15" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/aws/aws-sdk-php/zipball/442039c766a82f06ecfecb0ac2c610d6aaba228d", - "reference": "442039c766a82f06ecfecb0ac2c610d6aaba228d", + "url": "https://api.github.com/repos/aws/aws-sdk-php/zipball/a173ab3af8d9186d266e4937d8254597f36a9e15", + "reference": "a173ab3af8d9186d266e4937d8254597f36a9e15", "shasum": "" }, "require": { @@ -1020,9 +1020,9 @@ "support": { "forum": "https://forums.aws.amazon.com/forum.jspa?forumID=80", "issues": "https://github.com/aws/aws-sdk-php/issues", - "source": "https://github.com/aws/aws-sdk-php/tree/3.336.11" + "source": "https://github.com/aws/aws-sdk-php/tree/3.336.12" }, - "time": "2025-01-08T19:06:59+00:00" + "time": "2025-01-09T19:04:34+00:00" }, { "name": "bacon/bacon-qr-code", @@ -13198,16 +13198,16 @@ }, { "name": "phpstan/phpstan", - "version": "2.1.1", + "version": "1.12.15", "source": { "type": "git", "url": "https://github.com/phpstan/phpstan.git", - "reference": "cd6e973e04b4c2b94c86e8612b5a65f0da0e08e7" + "reference": "c91d4e8bc056f46cf653656e6f71004b254574d1" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpstan/phpstan/zipball/cd6e973e04b4c2b94c86e8612b5a65f0da0e08e7", - "reference": "cd6e973e04b4c2b94c86e8612b5a65f0da0e08e7", + "url": "https://api.github.com/repos/phpstan/phpstan/zipball/c91d4e8bc056f46cf653656e6f71004b254574d1", + "reference": "c91d4e8bc056f46cf653656e6f71004b254574d1", "shasum": "" }, "require": { @@ -13252,7 +13252,7 @@ "type": "github" } ], - "time": "2025-01-05T16:43:48+00:00" + "time": "2025-01-05T16:40:22+00:00" }, { "name": "phpunit/php-code-coverage", diff --git a/database/migrations/2025_01_10_135244_add_horizon_job_details_to_queue.php b/database/migrations/2025_01_10_135244_add_horizon_job_details_to_queue.php new file mode 100644 index 000000000..8ce5821ef --- /dev/null +++ b/database/migrations/2025_01_10_135244_add_horizon_job_details_to_queue.php @@ -0,0 +1,30 @@ +string('horizon_job_id')->nullable(); + $table->string('horizon_job_worker')->nullable(); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('application_deployment_queues', function (Blueprint $table) { + $table->dropColumn('horizon_job_id'); + $table->dropColumn('horizon_job_worker'); + }); + } +}; diff --git a/routes/api.php b/routes/api.php index c3f7a8f26..b884f4007 100644 --- a/routes/api.php +++ b/routes/api.php @@ -138,13 +138,29 @@ Route::group([ return response()->json(['message' => 'Unauthorized'], 401); } $naked_token = str_replace('Bearer ', '', $token); - $decrypted = decrypt($naked_token); - $decrypted_token = json_decode($decrypted, true); + try { + $decrypted = decrypt($naked_token); + $decrypted_token = json_decode($decrypted, true); + } catch (\Exception $e) { + return response()->json(['message' => 'Invalid token'], 401); + } $server_uuid = data_get($decrypted_token, 'server_uuid'); + if (! $server_uuid) { + return response()->json(['message' => 'Invalid token'], 401); + } $server = Server::where('uuid', $server_uuid)->first(); if (! $server) { return response()->json(['message' => 'Server not found'], 404); } + + if (isCloud() && data_get($server->team->subscription, 'stripe_invoice_paid', false) === false && $server->team->id !== 0) { + return response()->json(['message' => 'Unauthorized'], 401); + } + + if ($server->isFunctional() === false) { + return response()->json(['message' => 'Server is not functional'], 401); + } + if ($server->settings->sentinel_token !== $naked_token) { return response()->json(['message' => 'Unauthorized'], 401); } @@ -160,22 +176,3 @@ Route::group([ Route::any('/{any}', function () { return response()->json(['message' => 'Not found.', 'docs' => 'https://coolify.io/docs'], 404); })->where('any', '.*'); - -// Route::middleware(['throttle:5'])->group(function () { -// Route::get('/unsubscribe/{token}', function () { -// try { -// $token = request()->token; -// $email = decrypt($token); -// if (!User::whereEmail($email)->exists()) { -// return redirect(RouteServiceProvider::HOME); -// } -// if (User::whereEmail($email)->first()->marketing_emails === false) { -// return 'You have already unsubscribed from marketing emails.'; -// } -// User::whereEmail($email)->update(['marketing_emails' => false]); -// return 'You have been unsubscribed from marketing emails.'; -// } catch (\Throwable $e) { -// return 'Something went wrong. Please try again or contact support.'; -// } -// })->name('unsubscribe.marketing.emails'); -// }); diff --git a/scripts/cloud_upgrade.sh b/scripts/cloud_upgrade.sh index 8bab73b98..4cb326cbb 100644 --- a/scripts/cloud_upgrade.sh +++ b/scripts/cloud_upgrade.sh @@ -3,7 +3,4 @@ export IMAGE=$1 docker system prune -af docker compose pull read -p "Press Enter to update Coolify to $IMAGE..." last_version -docker compose logs -f +while ! (docker exec coolify sh -c "php artisan tinker --execute='isAnyDeploymentInprogress()'" && docker compose up --remove-orphans --force-recreate -d --wait && echo $IMAGE > last_version); do sleep 1; done \ No newline at end of file