+
diff --git a/app/Actions/CoolifyTask/RunRemoteProcess.php b/app/Actions/CoolifyTask/RunRemoteProcess.php
index 981b81378..926d30fe6 100644
--- a/app/Actions/CoolifyTask/RunRemoteProcess.php
+++ b/app/Actions/CoolifyTask/RunRemoteProcess.php
@@ -91,16 +91,9 @@ class RunRemoteProcess
} else {
if ($processResult->exitCode() == 0) {
$status = ProcessStatus::FINISHED;
- }
- if ($processResult->exitCode() != 0 && ! $this->ignore_errors) {
+ } else {
$status = ProcessStatus::ERROR;
}
- // if (($processResult->exitCode() == 0 && $this->is_finished) || $this->activity->properties->get('status') === ProcessStatus::FINISHED->value) {
- // $status = ProcessStatus::FINISHED;
- // }
- // if ($processResult->exitCode() != 0 && !$this->ignore_errors) {
- // $status = ProcessStatus::ERROR;
- // }
}
$this->activity->properties = $this->activity->properties->merge([
@@ -110,9 +103,6 @@ class RunRemoteProcess
'status' => $status->value,
]);
$this->activity->save();
- if ($processResult->exitCode() != 0 && ! $this->ignore_errors) {
- throw new \RuntimeException($processResult->errorOutput(), $processResult->exitCode());
- }
if ($this->call_event_on_finish) {
try {
if ($this->call_event_data) {
@@ -128,6 +118,9 @@ class RunRemoteProcess
Log::error('Error calling event: '.$e->getMessage());
}
}
+ if ($processResult->exitCode() != 0 && ! $this->ignore_errors) {
+ throw new \RuntimeException($processResult->errorOutput(), $processResult->exitCode());
+ }
return $processResult;
}
diff --git a/app/Console/Commands/CleanupStuckedResources.php b/app/Console/Commands/CleanupStuckedResources.php
index def3d5a2c..0b5eef84d 100644
--- a/app/Console/Commands/CleanupStuckedResources.php
+++ b/app/Console/Commands/CleanupStuckedResources.php
@@ -39,6 +39,11 @@ class CleanupStuckedResources extends Command
$servers = Server::all()->filter(function ($server) {
return $server->isFunctional();
});
+ if (isCloud()) {
+ $servers = $servers->filter(function ($server) {
+ return data_get($server->team->subscription, 'stripe_invoice_paid', false) === true;
+ });
+ }
foreach ($servers as $server) {
CleanupHelperContainersJob::dispatch($server);
}
diff --git a/app/Console/Commands/HorizonManage.php b/app/Console/Commands/HorizonManage.php
new file mode 100644
index 000000000..ca2da147c
--- /dev/null
+++ b/app/Console/Commands/HorizonManage.php
@@ -0,0 +1,178 @@
+option('can-i-restart-this-worker')) {
+ return $this->isThereAJobInProgress();
+ }
+
+ if ($this->option('job-status')) {
+ return $this->getJobStatus($this->option('job-status'));
+ }
+
+ $action = select(
+ label: 'What to do?',
+ options: [
+ 'pending' => 'Pending Jobs',
+ 'running' => 'Running Jobs',
+ 'can-i-restart-this-worker' => 'Can I restart this worker?',
+ 'job-status' => 'Job Status',
+ 'workers' => 'Workers',
+ 'failed' => 'Failed Jobs',
+ 'failed-delete' => 'Failed Jobs - Delete',
+ 'purge-queues' => 'Purge Queues',
+ ]
+ );
+
+ if ($action === 'can-i-restart-this-worker') {
+ $this->isThereAJobInProgress();
+ }
+
+ if ($action === 'job-status') {
+ $jobId = text('Which job to check?');
+ $jobStatus = $this->getJobStatus($jobId);
+ $this->info('Job Status: '.$jobStatus);
+ }
+
+ if ($action === 'pending') {
+ $pendingJobs = app(JobRepository::class)->getPending();
+ $pendingJobsTable = [];
+ if (count($pendingJobs) === 0) {
+ $this->info('No pending jobs found.');
+
+ return;
+ }
+ foreach ($pendingJobs as $pendingJob) {
+ $pendingJobsTable[] = [
+ 'id' => $pendingJob->id,
+ 'name' => $pendingJob->name,
+ 'status' => $pendingJob->status,
+ 'reserved_at' => $pendingJob->reserved_at ? now()->parse($pendingJob->reserved_at)->format('Y-m-d H:i:s') : null,
+ ];
+ }
+ table($pendingJobsTable);
+ }
+
+ if ($action === 'failed') {
+ $failedJobs = app(JobRepository::class)->getFailed();
+ $failedJobsTable = [];
+ if (count($failedJobs) === 0) {
+ $this->info('No failed jobs found.');
+
+ return;
+ }
+ foreach ($failedJobs as $failedJob) {
+ $failedJobsTable[] = [
+ 'id' => $failedJob->id,
+ 'name' => $failedJob->name,
+ 'failed_at' => $failedJob->failed_at ? now()->parse($failedJob->failed_at)->format('Y-m-d H:i:s') : null,
+ ];
+ }
+ table($failedJobsTable);
+ }
+
+ if ($action === 'failed-delete') {
+ $failedJobs = app(JobRepository::class)->getFailed();
+ $failedJobsTable = [];
+ foreach ($failedJobs as $failedJob) {
+ $failedJobsTable[] = [
+ 'id' => $failedJob->id,
+ 'name' => $failedJob->name,
+ 'failed_at' => $failedJob->failed_at ? now()->parse($failedJob->failed_at)->format('Y-m-d H:i:s') : null,
+ ];
+ }
+ app(MetricsRepository::class)->clear();
+ if (count($failedJobsTable) === 0) {
+ $this->info('No failed jobs found.');
+
+ return;
+ }
+ $jobIds = multiselect(
+ label: 'Which job to delete?',
+ options: collect($failedJobsTable)->mapWithKeys(fn ($job) => [$job['id'] => $job['id'].' - '.$job['name']])->toArray(),
+ );
+ foreach ($jobIds as $jobId) {
+ Artisan::queue('horizon:forget', ['id' => $jobId]);
+ }
+ }
+
+ if ($action === 'running') {
+ $redisJobRepository = app(CustomJobRepository::class);
+ $runningJobs = $redisJobRepository->getReservedJobs();
+ $runningJobsTable = [];
+ if (count($runningJobs) === 0) {
+ $this->info('No running jobs found.');
+
+ return;
+ }
+ foreach ($runningJobs as $runningJob) {
+ $runningJobsTable[] = [
+ 'id' => $runningJob->id,
+ 'name' => $runningJob->name,
+ 'reserved_at' => $runningJob->reserved_at ? now()->parse($runningJob->reserved_at)->format('Y-m-d H:i:s') : null,
+ ];
+ }
+ table($runningJobsTable);
+ }
+
+ if ($action === 'workers') {
+ $redisJobRepository = app(CustomJobRepository::class);
+ $workers = $redisJobRepository->getHorizonWorkers();
+ $workersTable = [];
+ foreach ($workers as $worker) {
+ $workersTable[] = [
+ 'name' => $worker->name,
+ ];
+ }
+ table($workersTable);
+ }
+
+ if ($action === 'purge-queues') {
+ $getQueues = app(CustomJobRepository::class)->getQueues();
+ $queueName = select(
+ label: 'Which queue to purge?',
+ options: $getQueues,
+ );
+ $redisJobRepository = app(RedisJobRepository::class);
+ $redisJobRepository->purge($queueName);
+ }
+ }
+
+ public function isThereAJobInProgress()
+ {
+ $runningJobs = ApplicationDeploymentQueue::where('horizon_job_worker', gethostname())->where('status', ApplicationDeploymentStatus::IN_PROGRESS->value)->get();
+ $count = $runningJobs->count();
+ if ($count === 0) {
+ return false;
+ }
+
+ return true;
+ }
+
+ public function getJobStatus(string $jobId)
+ {
+ return getJobStatus($jobId);
+ }
+}
diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php
index e8781b01e..8b4240412 100644
--- a/app/Console/Kernel.php
+++ b/app/Console/Kernel.php
@@ -91,7 +91,13 @@ class Kernel extends ConsoleKernel
private function pullImages(): void
{
- $servers = $this->allServers->whereRelation('settings', 'is_usable', true)->whereRelation('settings', 'is_reachable', true)->get();
+ if (isCloud()) {
+ $servers = $this->allServers->whereRelation('team.subscription', 'stripe_invoice_paid', true)->whereRelation('settings', 'is_usable', true)->whereRelation('settings', 'is_reachable', true)->get();
+ $own = Team::find(0)->servers;
+ $servers = $servers->merge($own);
+ } else {
+ $servers = $this->allServers->whereRelation('settings', 'is_usable', true)->whereRelation('settings', 'is_reachable', true)->get();
+ }
foreach ($servers as $server) {
if ($server->isSentinelEnabled()) {
$this->scheduleInstance->job(function () use ($server) {
@@ -124,7 +130,7 @@ class Kernel extends ConsoleKernel
private function checkResources(): void
{
if (isCloud()) {
- $servers = $this->allServers->whereHas('team.subscription')->get();
+ $servers = $this->allServers->whereRelation('team.subscription', 'stripe_invoice_paid', true)->get();
$own = Team::find(0)->servers;
$servers = $servers->merge($own);
} else {
@@ -171,18 +177,40 @@ class Kernel extends ConsoleKernel
if ($scheduled_backups->isEmpty()) {
return;
}
+ $finalScheduledBackups = collect();
foreach ($scheduled_backups as $scheduled_backup) {
- if (is_null(data_get($scheduled_backup, 'database'))) {
+ if (blank(data_get($scheduled_backup, 'database'))) {
$scheduled_backup->delete();
continue;
}
-
$server = $scheduled_backup->server();
+ if (blank($server)) {
+ $scheduled_backup->delete();
- if (is_null($server)) {
continue;
}
+ if ($server->isFunctional() === false) {
+ continue;
+ }
+ if (isCloud() && data_get($server->team->subscription, 'stripe_invoice_paid', false) === false && $server->team->id !== 0) {
+ continue;
+ }
+ $finalScheduledBackups->push($scheduled_backup);
+ }
+
+ foreach ($finalScheduledBackups as $scheduled_backup) {
+ if (isset(VALID_CRON_STRINGS[$scheduled_backup->frequency])) {
+ $scheduled_backup->frequency = VALID_CRON_STRINGS[$scheduled_backup->frequency];
+ }
+
+ $server = $scheduled_backup->server();
+ $serverTimezone = data_get($server->settings, 'server_timezone', $this->instanceTimezone);
+
+ if (validate_timezone($serverTimezone) === false) {
+ $serverTimezone = config('app.timezone');
+ }
+
if (isset(VALID_CRON_STRINGS[$scheduled_backup->frequency])) {
$scheduled_backup->frequency = VALID_CRON_STRINGS[$scheduled_backup->frequency];
}
@@ -199,35 +227,52 @@ class Kernel extends ConsoleKernel
if ($scheduled_tasks->isEmpty()) {
return;
}
+ $finalScheduledTasks = collect();
foreach ($scheduled_tasks as $scheduled_task) {
$service = $scheduled_task->service;
$application = $scheduled_task->application;
- if (! $application && ! $service) {
+ $server = $scheduled_task->server();
+ if (blank($server)) {
$scheduled_task->delete();
continue;
}
- if ($application) {
- if (str($application->status)->contains('running') === false) {
- continue;
- }
- }
- if ($service) {
- if (str($service->status)->contains('running') === false) {
- continue;
- }
- }
- $server = $scheduled_task->server();
- if (! $server) {
+ if ($server->isFunctional() === false) {
continue;
}
+ if (isCloud() && data_get($server->team->subscription, 'stripe_invoice_paid', false) === false && $server->team->id !== 0) {
+ continue;
+ }
+
+ if (! $service && ! $application) {
+ $scheduled_task->delete();
+
+ continue;
+ }
+
+ if ($application && str($application->status)->contains('running') === false) {
+ continue;
+ }
+ if ($service && str($service->status)->contains('running') === false) {
+ continue;
+ }
+
+ $finalScheduledTasks->push($scheduled_task);
+ }
+
+ foreach ($finalScheduledTasks as $scheduled_task) {
+ $server = $scheduled_task->server();
if (isset(VALID_CRON_STRINGS[$scheduled_task->frequency])) {
$scheduled_task->frequency = VALID_CRON_STRINGS[$scheduled_task->frequency];
}
$serverTimezone = data_get($server->settings, 'server_timezone', $this->instanceTimezone);
+
+ if (validate_timezone($serverTimezone) === false) {
+ $serverTimezone = config('app.timezone');
+ }
$this->scheduleInstance->job(new ScheduledTaskJob(
task: $scheduled_task
))->cron($scheduled_task->frequency)->timezone($serverTimezone)->onOneServer();
diff --git a/app/Contracts/CustomJobRepositoryInterface.php b/app/Contracts/CustomJobRepositoryInterface.php
new file mode 100644
index 000000000..1fbd71f46
--- /dev/null
+++ b/app/Contracts/CustomJobRepositoryInterface.php
@@ -0,0 +1,19 @@
+application_deployment_queue_id];
+ }
+
+ public function __construct(public int $application_deployment_queue_id)
{
$this->onQueue('high');
+ $this->application_deployment_queue = ApplicationDeploymentQueue::find($this->application_deployment_queue_id);
$this->nixpacks_plan_json = collect([]);
- $this->application_deployment_queue = ApplicationDeploymentQueue::find($application_deployment_queue_id);
$this->application = Application::find($this->application_deployment_queue->application_id);
$this->build_pack = data_get($this->application, 'build_pack');
$this->build_args = collect([]);
- $this->application_deployment_queue_id = $application_deployment_queue_id;
$this->deployment_uuid = $this->application_deployment_queue->deployment_uuid;
$this->pull_request_id = $this->application_deployment_queue->pull_request_id;
$this->commit = $this->application_deployment_queue->commit;
@@ -237,15 +240,11 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
}
}
- public function tags(): array
- {
- return ['server:'.gethostname()];
- }
-
public function handle(): void
{
$this->application_deployment_queue->update([
'status' => ApplicationDeploymentStatus::IN_PROGRESS->value,
+ 'horizon_job_worker' => gethostname(),
]);
if ($this->server->isFunctional() === false) {
$this->application_deployment_queue->addLogEntry('Server is not functional.');
@@ -2391,7 +2390,8 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf");
queue_next_deployment($this->application);
// If the deployment is cancelled by the user, don't update the status
if (
- $this->application_deployment_queue->status !== ApplicationDeploymentStatus::CANCELLED_BY_USER->value && $this->application_deployment_queue->status !== ApplicationDeploymentStatus::FAILED->value
+ $this->application_deployment_queue->status !== ApplicationDeploymentStatus::CANCELLED_BY_USER->value &&
+ $this->application_deployment_queue->status !== ApplicationDeploymentStatus::FAILED->value
) {
$this->application_deployment_queue->update([
'status' => $status,
diff --git a/app/Jobs/CheckAndStartSentinelJob.php b/app/Jobs/CheckAndStartSentinelJob.php
index 788db89ea..304b2a15c 100644
--- a/app/Jobs/CheckAndStartSentinelJob.php
+++ b/app/Jobs/CheckAndStartSentinelJob.php
@@ -24,7 +24,7 @@ class CheckAndStartSentinelJob implements ShouldBeEncrypted, ShouldQueue
$latestVersion = get_latest_sentinel_version();
// Check if sentinel is running
- $sentinelFound = instant_remote_process(['docker inspect coolify-sentinel'], $this->server, false);
+ $sentinelFound = instant_remote_process_with_timeout(['docker inspect coolify-sentinel'], $this->server, false, 10);
$sentinelFoundJson = json_decode($sentinelFound, true);
$sentinelStatus = data_get($sentinelFoundJson, '0.State.Status', 'exited');
if ($sentinelStatus !== 'running') {
@@ -33,7 +33,7 @@ class CheckAndStartSentinelJob implements ShouldBeEncrypted, ShouldQueue
return;
}
// If sentinel is running, check if it needs an update
- $runningVersion = instant_remote_process(['docker exec coolify-sentinel sh -c "curl http://127.0.0.1:8888/api/version"'], $this->server, false);
+ $runningVersion = instant_remote_process_with_timeout(['docker exec coolify-sentinel sh -c "curl http://127.0.0.1:8888/api/version"'], $this->server, false);
if (empty($runningVersion)) {
$runningVersion = '0.0.0';
}
diff --git a/app/Jobs/CleanupHelperContainersJob.php b/app/Jobs/CleanupHelperContainersJob.php
index f185ab781..0e1fcb4d7 100644
--- a/app/Jobs/CleanupHelperContainersJob.php
+++ b/app/Jobs/CleanupHelperContainersJob.php
@@ -20,11 +20,11 @@ class CleanupHelperContainersJob implements ShouldBeEncrypted, ShouldBeUnique, S
public function handle(): void
{
try {
- $containers = instant_remote_process(['docker container ps --format \'{{json .}}\' | jq -s \'map(select(.Image | contains("ghcr.io/coollabsio/coolify-helper")))\''], $this->server, false);
+ $containers = instant_remote_process_with_timeout(['docker container ps --format \'{{json .}}\' | jq -s \'map(select(.Image | contains("ghcr.io/coollabsio/coolify-helper")))\''], $this->server, false);
$containerIds = collect(json_decode($containers))->pluck('ID');
if ($containerIds->count() > 0) {
foreach ($containerIds as $containerId) {
- instant_remote_process(['docker container rm -f '.$containerId], $this->server, false);
+ instant_remote_process_with_timeout(['docker container rm -f '.$containerId], $this->server, false);
}
}
} catch (\Throwable $e) {
diff --git a/app/Jobs/PushServerUpdateJob.php b/app/Jobs/PushServerUpdateJob.php
index 24f8d1e6b..93b203fcb 100644
--- a/app/Jobs/PushServerUpdateJob.php
+++ b/app/Jobs/PushServerUpdateJob.php
@@ -19,6 +19,7 @@ use Illuminate\Contracts\Queue\ShouldBeEncrypted;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Collection;
@@ -68,6 +69,11 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue
public bool $foundLogDrainContainer = false;
+ public function middleware(): array
+ {
+ return [(new WithoutOverlapping($this->server->uuid))->dontRelease()];
+ }
+
public function backoff(): int
{
return isDev() ? 1 : 3;
diff --git a/app/Livewire/ActivityMonitor.php b/app/Livewire/ActivityMonitor.php
index 2e36f34ee..024f53c3d 100644
--- a/app/Livewire/ActivityMonitor.php
+++ b/app/Livewire/ActivityMonitor.php
@@ -42,14 +42,8 @@ class ActivityMonitor extends Component
public function polling()
{
$this->hydrateActivity();
- // $this->setStatus(ProcessStatus::IN_PROGRESS);
$exit_code = data_get($this->activity, 'properties.exitCode');
if ($exit_code !== null) {
- // if ($exit_code === 0) {
- // // $this->setStatus(ProcessStatus::FINISHED);
- // } else {
- // // $this->setStatus(ProcessStatus::ERROR);
- // }
$this->isPollingActive = false;
if ($exit_code === 0) {
if ($this->eventToDispatch !== null) {
@@ -70,12 +64,4 @@ class ActivityMonitor extends Component
}
}
}
-
- // protected function setStatus($status)
- // {
- // $this->activity->properties = $this->activity->properties->merge([
- // 'status' => $status,
- // ]);
- // $this->activity->save();
- // }
}
diff --git a/app/Livewire/Destination/New/Docker.php b/app/Livewire/Destination/New/Docker.php
index 337f1d067..0e60025e5 100644
--- a/app/Livewire/Destination/New/Docker.php
+++ b/app/Livewire/Destination/New/Docker.php
@@ -83,9 +83,7 @@ class Docker extends Component
]);
}
}
- $connectProxyToDockerNetworks = connectProxyToNetworks($this->selectedServer);
- instant_remote_process($connectProxyToDockerNetworks, $this->selectedServer, false);
- $this->dispatch('reloadWindow');
+ $this->redirect(route('destination.show', $docker->uuid));
} catch (\Throwable $e) {
return handleError($e, $this);
}
diff --git a/app/Livewire/Project/Application/Deployment/Show.php b/app/Livewire/Project/Application/Deployment/Show.php
index 4e86b5c7e..7b2ac09d3 100644
--- a/app/Livewire/Project/Application/Deployment/Show.php
+++ b/app/Livewire/Project/Application/Deployment/Show.php
@@ -14,6 +14,8 @@ class Show extends Component
public string $deployment_uuid;
+ public string $horizon_job_status;
+
public $isKeepAliveOn = true;
protected $listeners = ['refreshQueue'];
@@ -44,7 +46,9 @@ class Show extends Component
}
$this->application = $application;
$this->application_deployment_queue = $application_deployment_queue;
+ $this->horizon_job_status = $this->application_deployment_queue->getHorizonJobStatus();
$this->deployment_uuid = $deploymentUuid;
+ $this->isKeepAliveOn();
}
public function refreshQueue()
@@ -52,13 +56,21 @@ class Show extends Component
$this->application_deployment_queue->refresh();
}
+ private function isKeepAliveOn()
+ {
+ if (data_get($this->application_deployment_queue, 'status') === 'finished' || data_get($this->application_deployment_queue, 'status') === 'failed') {
+ $this->isKeepAliveOn = false;
+ } else {
+ $this->isKeepAliveOn = true;
+ }
+ }
+
public function polling()
{
$this->dispatch('deploymentFinished');
$this->application_deployment_queue->refresh();
- if (data_get($this->application_deployment_queue, 'status') === 'finished' || data_get($this->application_deployment_queue, 'status') === 'failed') {
- $this->isKeepAliveOn = false;
- }
+ $this->horizon_job_status = $this->application_deployment_queue->getHorizonJobStatus();
+ $this->isKeepAliveOn();
}
public function getLogLinesProperty()
diff --git a/app/Livewire/Project/Database/BackupExecutions.php b/app/Livewire/Project/Database/BackupExecutions.php
index f91b8bfaf..ce168a352 100644
--- a/app/Livewire/Project/Database/BackupExecutions.php
+++ b/app/Livewire/Project/Database/BackupExecutions.php
@@ -83,8 +83,10 @@ class BackupExecutions extends Component
public function refreshBackupExecutions(): void
{
- if ($this->backup) {
- $this->executions = $this->backup->executions()->get();
+ if ($this->backup && $this->backup->exists) {
+ $this->executions = $this->backup->executions()->get()->toArray();
+ } else {
+ $this->executions = [];
}
}
diff --git a/app/Livewire/Project/Service/Configuration.php b/app/Livewire/Project/Service/Configuration.php
index 4a5e8627f..d1744b178 100644
--- a/app/Livewire/Project/Service/Configuration.php
+++ b/app/Livewire/Project/Service/Configuration.php
@@ -99,7 +99,7 @@ class Configuration extends Component
$this->service->databases->each(function ($database) {
$database->refresh();
});
- $this->dispatch('$refresh');
+ $this->dispatch('refresh');
} catch (\Exception $e) {
return handleError($e, $this);
}
diff --git a/app/Livewire/Project/Service/Navbar.php b/app/Livewire/Project/Service/Navbar.php
index 22fc1c0d6..915fb54c6 100644
--- a/app/Livewire/Project/Service/Navbar.php
+++ b/app/Livewire/Project/Service/Navbar.php
@@ -5,6 +5,7 @@ namespace App\Livewire\Project\Service;
use App\Actions\Service\StartService;
use App\Actions\Service\StopService;
use App\Actions\Shared\PullImage;
+use App\Enums\ProcessStatus;
use App\Events\ServiceStatusChanged;
use App\Models\Service;
use Illuminate\Support\Facades\Auth;
@@ -68,11 +69,9 @@ class Navbar extends Component
public function checkDeployments()
{
try {
- // TODO: This is a temporary solution. We need to refactor this.
- // We need to delete null bytes somehow.
$activity = Activity::where('properties->type_uuid', $this->service->uuid)->latest()->first();
$status = data_get($activity, 'properties.status');
- if ($status === 'queued' || $status === 'in_progress') {
+ if ($status === ProcessStatus::QUEUED->value || $status === ProcessStatus::IN_PROGRESS->value) {
$this->isDeploymentProgress = true;
} else {
$this->isDeploymentProgress = false;
@@ -80,25 +79,46 @@ class Navbar extends Component
} catch (\Throwable) {
$this->isDeploymentProgress = false;
}
+
+ return $this->isDeploymentProgress;
}
public function start()
{
- $this->checkDeployments();
- if ($this->isDeploymentProgress) {
- $this->dispatch('error', 'There is a deployment in progress.');
-
- return;
- }
$this->service->parse();
$activity = StartService::run($this->service);
$this->dispatch('activityMonitor', $activity->id);
}
- public function stop()
+ public function forceDeploy()
{
- StopService::run($this->service, false, $this->docker_cleanup);
- ServiceStatusChanged::dispatch();
+ try {
+ $activities = Activity::where('properties->type_uuid', $this->service->uuid)->where('properties->status', ProcessStatus::IN_PROGRESS->value)->orWhere('properties->status', ProcessStatus::QUEUED->value)->get();
+ foreach ($activities as $activity) {
+ $activity->properties->status = ProcessStatus::ERROR->value;
+ $activity->save();
+ }
+ $this->service->parse();
+ $activity = StartService::run($this->service);
+ $this->dispatch('activityMonitor', $activity->id);
+ } catch (\Exception $e) {
+ $this->dispatch('error', $e->getMessage());
+ }
+ }
+
+ public function stop($cleanupContainers = false)
+ {
+ try {
+ StopService::run($this->service, false, $this->docker_cleanup);
+ ServiceStatusChanged::dispatch();
+ if ($cleanupContainers) {
+ $this->dispatch('success', 'Containers cleaned up.');
+ } else {
+ $this->dispatch('success', 'Service stopped.');
+ }
+ } catch (\Exception $e) {
+ $this->dispatch('error', $e->getMessage());
+ }
}
public function restart()
diff --git a/app/Models/ApplicationDeploymentQueue.php b/app/Models/ApplicationDeploymentQueue.php
index c261c30c6..95c78d725 100644
--- a/app/Models/ApplicationDeploymentQueue.php
+++ b/app/Models/ApplicationDeploymentQueue.php
@@ -70,6 +70,11 @@ class ApplicationDeploymentQueue extends Model
return collect(json_decode($this->logs))->where('name', $name)->first()?->output ?? null;
}
+ public function getHorizonJobStatus()
+ {
+ return getJobStatus($this->horizon_job_id);
+ }
+
public function commitMessage()
{
if (empty($this->commit_message) || is_null($this->commit_message)) {
diff --git a/app/Models/Server.php b/app/Models/Server.php
index 8d11e23a9..2867f95cb 100644
--- a/app/Models/Server.php
+++ b/app/Models/Server.php
@@ -54,6 +54,8 @@ class Server extends BaseModel
public static $batch_counter = 0;
+ protected $appends = ['is_coolify_host'];
+
protected static function booted()
{
static::saving(function ($server) {
@@ -156,6 +158,15 @@ class Server extends BaseModel
return 'server';
}
+ protected function isCoolifyHost(): Attribute
+ {
+ return Attribute::make(
+ get: function () {
+ return $this->id === 0;
+ }
+ );
+ }
+
public static function isReachable()
{
return Server::ownedByCurrentTeam()->whereRelation('settings', 'is_reachable', true);
@@ -656,9 +667,9 @@ $schema://$host {
$containers = collect([]);
$containerReplicates = collect([]);
if ($this->isSwarm()) {
- $containers = instant_remote_process(["docker service inspect $(docker service ls -q) --format '{{json .}}'"], $this, false);
+ $containers = instant_remote_process_with_timeout(["docker service inspect $(docker service ls -q) --format '{{json .}}'"], $this, false);
$containers = format_docker_command_output_to_json($containers);
- $containerReplicates = instant_remote_process(["docker service ls --format '{{json .}}'"], $this, false);
+ $containerReplicates = instant_remote_process_with_timeout(["docker service ls --format '{{json .}}'"], $this, false);
if ($containerReplicates) {
$containerReplicates = format_docker_command_output_to_json($containerReplicates);
foreach ($containerReplicates as $containerReplica) {
@@ -682,7 +693,7 @@ $schema://$host {
}
}
} else {
- $containers = instant_remote_process(["docker container inspect $(docker container ls -aq) --format '{{json .}}'"], $this, false);
+ $containers = instant_remote_process_with_timeout(["docker container inspect $(docker container ls -aq) --format '{{json .}}'"], $this, false);
$containers = format_docker_command_output_to_json($containers);
$containerReplicates = collect([]);
}
diff --git a/app/Models/Service.php b/app/Models/Service.php
index 1df579802..25e6b92ea 100644
--- a/app/Models/Service.php
+++ b/app/Models/Service.php
@@ -1050,10 +1050,11 @@ class Service extends BaseModel
$fields->put('MySQL', $data->toArray());
break;
case $image->contains('mariadb'):
- $userVariables = ['SERVICE_USER_MARIADB', 'SERVICE_USER_WORDPRESS', '_APP_DB_USER', 'SERVICE_USER_MYSQL', 'MYSQL_USER'];
+ $userVariables = ['SERVICE_USER_MARIADB', 'SERVICE_USER_WORDPRESS', 'SERVICE_USER_MYSQL', 'MYSQL_USER'];
$passwordVariables = ['SERVICE_PASSWORD_MARIADB', 'SERVICE_PASSWORD_WORDPRESS', '_APP_DB_PASS', 'MYSQL_PASSWORD'];
$rootPasswordVariables = ['SERVICE_PASSWORD_MARIADBROOT', 'SERVICE_PASSWORD_ROOT', '_APP_DB_ROOT_PASS', 'MYSQL_ROOT_PASSWORD'];
$dbNameVariables = ['SERVICE_DATABASE_MARIADB', 'SERVICE_DATABASE_WORDPRESS', '_APP_DB_SCHEMA', 'MYSQL_DATABASE'];
+
$mariadb_user = $this->environment_variables()->whereIn('key', $userVariables)->first();
$mariadb_password = $this->environment_variables()->whereIn('key', $passwordVariables)->first();
$mariadb_root_password = $this->environment_variables()->whereIn('key', $rootPasswordVariables)->first();
@@ -1102,6 +1103,23 @@ class Service extends BaseModel
break;
}
}
+ $fields = collect($fields)->map(function ($extraFields) {
+ if (is_array($extraFields)) {
+ $extraFields = collect($extraFields)->map(function ($field) {
+ if (filled($field['value']) && str($field['value'])->startsWith('$SERVICE_')) {
+ $searchValue = str($field['value'])->after('$')->value;
+ $newValue = $this->environment_variables()->where('key', $searchValue)->first();
+ if ($newValue) {
+ $field['value'] = $newValue->value;
+ }
+ }
+
+ return $field;
+ });
+ }
+
+ return $extraFields;
+ });
return $fields;
}
diff --git a/app/Models/StandaloneDocker.php b/app/Models/StandaloneDocker.php
index 1ef6ff587..9db6a2d29 100644
--- a/app/Models/StandaloneDocker.php
+++ b/app/Models/StandaloneDocker.php
@@ -6,6 +6,19 @@ class StandaloneDocker extends BaseModel
{
protected $guarded = [];
+ protected static function boot()
+ {
+ parent::boot();
+ static::created(function ($newStandaloneDocker) {
+ $server = $newStandaloneDocker->server;
+ instant_remote_process([
+ "docker network inspect $newStandaloneDocker->network >/dev/null 2>&1 || docker network create --driver overlay --attachable $newStandaloneDocker->network >/dev/null",
+ ], $server, false);
+ $connectProxyToDockerNetworks = connectProxyToNetworks($server);
+ instant_remote_process($connectProxyToDockerNetworks, $server, false);
+ });
+ }
+
public function applications()
{
return $this->morphMany(Application::class, 'destination');
diff --git a/app/Models/Team.php b/app/Models/Team.php
index 8651df3c8..07959dd16 100644
--- a/app/Models/Team.php
+++ b/app/Models/Team.php
@@ -247,8 +247,17 @@ class Team extends Model implements SendsDiscord, SendsEmail, SendsPushover, Sen
public function sources()
{
$sources = collect([]);
- $github_apps = $this->hasMany(GithubApp::class)->whereisPublic(false)->get();
- $gitlab_apps = $this->hasMany(GitlabApp::class)->whereisPublic(false)->get();
+ $github_apps = GithubApp::where(function ($query) {
+ $query->where('team_id', $this->id)
+ ->Where('is_public', false)
+ ->orWhere('is_system_wide', true);
+ })->get();
+
+ $gitlab_apps = GitlabApp::where(function ($query) {
+ $query->where('team_id', $this->id)
+ ->Where('is_public', false)
+ ->orWhere('is_system_wide', true);
+ })->get();
return $sources->merge($github_apps)->merge($gitlab_apps);
}
diff --git a/app/Providers/HorizonServiceProvider.php b/app/Providers/HorizonServiceProvider.php
index f851f1f3b..0caa3a3a9 100644
--- a/app/Providers/HorizonServiceProvider.php
+++ b/app/Providers/HorizonServiceProvider.php
@@ -2,15 +2,52 @@
namespace App\Providers;
+use App\Contracts\CustomJobRepositoryInterface;
+use App\Models\ApplicationDeploymentQueue;
use App\Models\User;
+use App\Repositories\CustomJobRepository;
+use Illuminate\Support\Facades\Event;
use Illuminate\Support\Facades\Gate;
+use Laravel\Horizon\Contracts\JobRepository;
+use Laravel\Horizon\Events\JobReserved;
use Laravel\Horizon\HorizonApplicationServiceProvider;
class HorizonServiceProvider extends HorizonApplicationServiceProvider
{
+ /**
+ * Register services.
+ */
+ public function register(): void
+ {
+ $this->app->singleton(JobRepository::class, CustomJobRepository::class);
+ $this->app->singleton(CustomJobRepositoryInterface::class, CustomJobRepository::class);
+ }
+
+ /**
+ * Bootstrap services.
+ */
public function boot(): void
{
parent::boot();
+ Event::listen(function (JobReserved $event) {
+ $payload = $event->payload->decoded;
+ $jobName = $payload['displayName'];
+ if ($jobName === 'App\Jobs\ApplicationDeploymentJob') {
+ $tags = $payload['tags'];
+ $id = $payload['id'];
+ $deploymentQueueId = collect($tags)->first(function ($tag) {
+ return str_contains($tag, 'App\Models\ApplicationDeploymentQueue');
+ });
+ if (blank($deploymentQueueId)) {
+ return;
+ }
+ $deploymentQueueId = explode(':', $deploymentQueueId)[1];
+ $deploymentQueue = ApplicationDeploymentQueue::find($deploymentQueueId);
+ $deploymentQueue->update([
+ 'horizon_job_id' => $id,
+ ]);
+ }
+ });
}
protected function gate(): void
diff --git a/app/Repositories/CustomJobRepository.php b/app/Repositories/CustomJobRepository.php
new file mode 100644
index 000000000..502dd252b
--- /dev/null
+++ b/app/Repositories/CustomJobRepository.php
@@ -0,0 +1,51 @@
+all();
+ }
+
+ public function getReservedJobs(): Collection
+ {
+ return $this->getJobsByStatus('reserved');
+ }
+
+ public function getJobsByStatus(string $status): Collection
+ {
+ $jobs = new Collection;
+
+ $this->getRecent()->each(function ($job) use ($jobs, $status) {
+ if ($job->status === $status) {
+ $jobs->push($job);
+ }
+ });
+
+ return $jobs;
+ }
+
+ public function countJobsByStatus(string $status): int
+ {
+ return $this->getJobsByStatus($status)->count();
+ }
+
+ public function getQueues(): array
+ {
+ $queues = $this->connection()->keys('queue:*');
+ $queues = array_map(function ($queue) {
+ return explode(':', $queue)[2];
+ }, $queues);
+
+ return $queues;
+ }
+}
diff --git a/app/View/Components/services/advanced.php b/app/View/Components/services/advanced.php
new file mode 100644
index 000000000..7e68c6dda
--- /dev/null
+++ b/app/View/Components/services/advanced.php
@@ -0,0 +1,26 @@
+toArray() : $command;
+ if ($server->isNonRoot() && ! $no_sudo) {
+ $command = parseCommandsByLineForSudo(collect($command), $server);
+ }
+ $command_string = implode("\n", $command);
+
+ // $start_time = microtime(true);
+ $sshCommand = SshMultiplexingHelper::generateSshCommand($server, $command_string);
+ $process = Process::timeout(30)->run($sshCommand);
+ // $end_time = microtime(true);
+
+ // $execution_time = ($end_time - $start_time) * 1000; // Convert to milliseconds
+ // ray('SSH command execution time:', $execution_time.' ms')->orange();
+
+ $output = trim($process->output());
+ $exitCode = $process->exitCode();
+
+ if ($exitCode !== 0) {
+ return $throwError ? excludeCertainErrors($process->errorOutput(), $exitCode) : null;
+ }
+
+ return $output === 'null' ? null : $output;
+}
function instant_remote_process(Collection|array $command, Server $server, bool $throwError = true, bool $no_sudo = false): ?string
{
$command = $command instanceof Collection ? $command->toArray() : $command;
diff --git a/bootstrap/helpers/shared.php b/bootstrap/helpers/shared.php
index 4cd40f203..5d34c63b6 100644
--- a/bootstrap/helpers/shared.php
+++ b/bootstrap/helpers/shared.php
@@ -41,6 +41,7 @@ use Illuminate\Support\Facades\Route;
use Illuminate\Support\Facades\Validator;
use Illuminate\Support\Str;
use Illuminate\Support\Stringable;
+use Laravel\Horizon\Contracts\JobRepository;
use Lcobucci\JWT\Encoding\ChainedFormatter;
use Lcobucci\JWT\Encoding\JoseEncoder;
use Lcobucci\JWT\Signer\Hmac\Sha256;
@@ -1257,14 +1258,22 @@ function get_public_ips()
function isAnyDeploymentInprogress()
{
- // Only use it in the deployment script
- $count = ApplicationDeploymentQueue::whereIn('status', [ApplicationDeploymentStatus::IN_PROGRESS, ApplicationDeploymentStatus::QUEUED])->count();
- if ($count > 0) {
- echo "There are $count deployments in progress. Exiting...\n";
- exit(1);
+ $runningJobs = ApplicationDeploymentQueue::where('horizon_job_worker', gethostname())->where('status', ApplicationDeploymentStatus::IN_PROGRESS->value)->get();
+ $horizonJobIds = [];
+ foreach ($runningJobs as $runningJob) {
+ $horizonJobStatus = getJobStatus($runningJob->horizon_job_id);
+ if ($horizonJobStatus === 'unknown') {
+ return true;
+ }
+ $horizonJobIds[] = $runningJob->horizon_job_id;
}
- echo "No deployments in progress.\n";
- exit(0);
+ if (count($horizonJobIds) === 0) {
+ echo "No deployments in progress.\n";
+ exit(0);
+ }
+ $horizonJobIds = collect($horizonJobIds)->unique()->toArray();
+ echo 'There are '.count($horizonJobIds)." deployments in progress.\n";
+ exit(1);
}
function isBase64Encoded($strValue)
@@ -4124,3 +4133,16 @@ function convertGitUrl(string $gitRepository, string $deploymentType, ?GithubApp
'port' => $providerInfo['port'],
];
}
+
+function getJobStatus(?string $jobId = null)
+{
+ if (blank($jobId)) {
+ return 'unknown';
+ }
+ $jobFound = app(JobRepository::class)->getJobs([$jobId]);
+ if ($jobFound->isEmpty()) {
+ return 'unknown';
+ }
+
+ return $jobFound->first()->status;
+}
diff --git a/composer.json b/composer.json
index 5a51db531..055059a56 100644
--- a/composer.json
+++ b/composer.json
@@ -30,7 +30,7 @@
"league/flysystem-sftp-v3": "^3.0",
"livewire/livewire": "^3.5",
"log1x/laravel-webfonts": "^1.0",
- "lorisleiva/laravel-actions": "^2.7",
+ "lorisleiva/laravel-actions": "^2.8",
"nubs/random-name-generator": "^2.2",
"phpseclib/phpseclib": "^3.0",
"pion/laravel-chunk-upload": "^1.5",
@@ -124,4 +124,4 @@
"@php artisan key:generate --ansi"
]
}
-}
+}
\ No newline at end of file
diff --git a/composer.lock b/composer.lock
index 8c05ebb36..7ca8219c7 100644
--- a/composer.lock
+++ b/composer.lock
@@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
- "content-hash": "ccced2490c39e4f6f1bf9b036bfa3ef0",
+ "content-hash": "85a775fb1a4b9ea329d8d893f43621c2",
"packages": [
{
"name": "3sidedcube/laravel-redoc",
@@ -928,16 +928,16 @@
},
{
"name": "aws/aws-sdk-php",
- "version": "3.336.11",
+ "version": "3.336.12",
"source": {
"type": "git",
"url": "https://github.com/aws/aws-sdk-php.git",
- "reference": "442039c766a82f06ecfecb0ac2c610d6aaba228d"
+ "reference": "a173ab3af8d9186d266e4937d8254597f36a9e15"
},
"dist": {
"type": "zip",
- "url": "https://api.github.com/repos/aws/aws-sdk-php/zipball/442039c766a82f06ecfecb0ac2c610d6aaba228d",
- "reference": "442039c766a82f06ecfecb0ac2c610d6aaba228d",
+ "url": "https://api.github.com/repos/aws/aws-sdk-php/zipball/a173ab3af8d9186d266e4937d8254597f36a9e15",
+ "reference": "a173ab3af8d9186d266e4937d8254597f36a9e15",
"shasum": ""
},
"require": {
@@ -1020,9 +1020,9 @@
"support": {
"forum": "https://forums.aws.amazon.com/forum.jspa?forumID=80",
"issues": "https://github.com/aws/aws-sdk-php/issues",
- "source": "https://github.com/aws/aws-sdk-php/tree/3.336.11"
+ "source": "https://github.com/aws/aws-sdk-php/tree/3.336.12"
},
- "time": "2025-01-08T19:06:59+00:00"
+ "time": "2025-01-09T19:04:34+00:00"
},
{
"name": "bacon/bacon-qr-code",
@@ -13198,16 +13198,16 @@
},
{
"name": "phpstan/phpstan",
- "version": "2.1.1",
+ "version": "1.12.15",
"source": {
"type": "git",
"url": "https://github.com/phpstan/phpstan.git",
- "reference": "cd6e973e04b4c2b94c86e8612b5a65f0da0e08e7"
+ "reference": "c91d4e8bc056f46cf653656e6f71004b254574d1"
},
"dist": {
"type": "zip",
- "url": "https://api.github.com/repos/phpstan/phpstan/zipball/cd6e973e04b4c2b94c86e8612b5a65f0da0e08e7",
- "reference": "cd6e973e04b4c2b94c86e8612b5a65f0da0e08e7",
+ "url": "https://api.github.com/repos/phpstan/phpstan/zipball/c91d4e8bc056f46cf653656e6f71004b254574d1",
+ "reference": "c91d4e8bc056f46cf653656e6f71004b254574d1",
"shasum": ""
},
"require": {
@@ -13252,7 +13252,7 @@
"type": "github"
}
],
- "time": "2025-01-05T16:43:48+00:00"
+ "time": "2025-01-05T16:40:22+00:00"
},
{
"name": "phpunit/php-code-coverage",
@@ -15362,12 +15362,12 @@
],
"aliases": [],
"minimum-stability": "stable",
- "stability-flags": {},
+ "stability-flags": [],
"prefer-stable": true,
"prefer-lowest": false,
"platform": {
"php": "^8.4"
},
- "platform-dev": {},
+ "platform-dev": [],
"plugin-api-version": "2.6.0"
}
diff --git a/database/migrations/2025_01_10_135244_add_horizon_job_details_to_queue.php b/database/migrations/2025_01_10_135244_add_horizon_job_details_to_queue.php
new file mode 100644
index 000000000..8ce5821ef
--- /dev/null
+++ b/database/migrations/2025_01_10_135244_add_horizon_job_details_to_queue.php
@@ -0,0 +1,30 @@
+string('horizon_job_id')->nullable();
+ $table->string('horizon_job_worker')->nullable();
+ });
+ }
+
+ /**
+ * Reverse the migrations.
+ */
+ public function down(): void
+ {
+ Schema::table('application_deployment_queues', function (Blueprint $table) {
+ $table->dropColumn('horizon_job_id');
+ $table->dropColumn('horizon_job_worker');
+ });
+ }
+};
diff --git a/resources/views/components/forms/checkbox.blade.php b/resources/views/components/forms/checkbox.blade.php
index 39704a122..29c97735f 100644
--- a/resources/views/components/forms/checkbox.blade.php
+++ b/resources/views/components/forms/checkbox.blade.php
@@ -11,20 +11,21 @@
])