diff --git a/app/Actions/Application/StopApplication.php b/app/Actions/Application/StopApplication.php index a98384f45..1d09f0daf 100644 --- a/app/Actions/Application/StopApplication.php +++ b/app/Actions/Application/StopApplication.php @@ -25,5 +25,15 @@ class StopApplication // TODO: make notification for application // $application->environment->project->team->notify(new StatusChanged($application)); } + // Delete Preview Deployments + $previewDeployments = $application->previews; + foreach ($previewDeployments as $previewDeployment) { + $containers = getCurrentApplicationContainerStatus($server, $application->id, $previewDeployment->pull_request_id); + foreach ($containers as $container) { + $name = str_replace('/', '', $container['Names']); + instant_remote_process(["docker rm -f $name"], $application->destination->server, throwError: false); + } + } + } } diff --git a/app/Actions/Server/InstallDocker.php b/app/Actions/Server/InstallDocker.php index 0713ed086..a580c3473 100644 --- a/app/Actions/Server/InstallDocker.php +++ b/app/Actions/Server/InstallDocker.php @@ -15,7 +15,7 @@ class InstallDocker if (!$supported_os_type) { throw new \Exception('Server OS type is not supported for automated installation. Please install Docker manually before continuing: documentation.'); } - ray('Installing Docker on server: ' . $server->name . ' (' . $server->ip . ')' . ' with OS: ' . $supported_os_type); + ray('Installing Docker on server: ' . $server->name . ' (' . $server->ip . ')' . ' with OS type: ' . $supported_os_type); $dockerVersion = '24.0'; $config = base64_encode('{ "log-driver": "json-file", @@ -44,17 +44,23 @@ class InstallDocker "ls -l /tmp" ]); } else { - if ($supported_os_type === 'debian') { + if ($supported_os_type->contains('debian')) { $command = $command->merge([ "echo 'Installing Prerequisites...'", - "command -v jq >/dev/null || apt-get update", - "command -v jq >/dev/null || apt install -y jq", + "command -v jq >/dev/null || apt-get update -y", + "command -v jq >/dev/null || apt install -y curl wget git jq", ]); - } else if ($supported_os_type === 'rhel') { + } else if ($supported_os_type->contains('rhel')) { $command = $command->merge([ "echo 'Installing Prerequisites...'", - "command -v jq >/dev/null || dnf install -y jq", + "command -v jq >/dev/null || dnf install -y curl wget git jq", + ]); + } else if ($supported_os_type->contains('sles')) { + $command = $command->merge([ + "echo 'Installing Prerequisites...'", + "command -v jq >/dev/null || zypper update -y", + "command -v jq >/dev/null || zypper install -y curl wget git jq", ]); } else { throw new \Exception('Unsupported OS'); diff --git a/app/Actions/Service/StartService.php b/app/Actions/Service/StartService.php index ef473e578..50f7fba82 100644 --- a/app/Actions/Service/StartService.php +++ b/app/Actions/Service/StartService.php @@ -16,13 +16,13 @@ class StartService $commands[] = "cd " . $service->workdir(); $commands[] = "echo 'Saved configuration files to {$service->workdir()}.'"; $commands[] = "echo 'Creating Docker network.'"; - $commands[] = "docker network create --attachable '{$service->uuid}' >/dev/null || true"; + $commands[] = "docker network create --attachable '{$service->uuid}' >/dev/null 2>&1 || true"; $commands[] = "echo 'Starting service {$service->name} on {$service->server->name}.'"; $commands[] = "echo 'Pulling images.'"; $commands[] = "docker compose pull"; $commands[] = "echo 'Starting containers.'"; $commands[] = "docker compose up -d --remove-orphans --force-recreate"; - $commands[] = "docker network connect $service->uuid coolify-proxy || true"; + $commands[] = "docker network connect $service->uuid coolify-proxy >/dev/null 2>&1 || true"; $compose = data_get($service,'docker_compose',[]); $serviceNames = data_get(Yaml::parse($compose),'services',[]); foreach($serviceNames as $serviceName => $serviceConfig){ diff --git a/app/Console/Commands/ServicesGenerate.php b/app/Console/Commands/ServicesGenerate.php index 530dd259a..802b3180f 100644 --- a/app/Console/Commands/ServicesGenerate.php +++ b/app/Console/Commands/ServicesGenerate.php @@ -26,7 +26,7 @@ class ServicesGenerate extends Command */ public function handle() { - ray()->clearAll(); + // ray()->clearAll(); $files = array_diff(scandir(base_path('templates/compose')), ['.', '..']); $files = array_filter($files, function ($file) { return strpos($file, '.yaml') !== false; diff --git a/app/Console/Commands/SyncBunny.php b/app/Console/Commands/SyncBunny.php index f3ddaaffe..a59d13ab8 100644 --- a/app/Console/Commands/SyncBunny.php +++ b/app/Console/Commands/SyncBunny.php @@ -71,6 +71,15 @@ class SyncBunny extends Command ]); }); try { + if (!$only_template && !$only_version) { + $this->info('About to sync files (docker-compose.prod.yaml, upgrade.sh, install.sh, etc) to BunnyCDN.'); + } + if ($only_template) { + $this->info('About to sync service-templates.json to BunnyCDN.'); + } + if ($only_version) { + $this->info('About to sync versions.json to BunnyCDN.'); + } $confirmed = confirm('Are you sure you want to sync?'); if (!$confirmed) { return; diff --git a/app/Http/Controllers/ProjectController.php b/app/Http/Controllers/ProjectController.php index 73cf55fcd..4e2fc5878 100644 --- a/app/Http/Controllers/ProjectController.php +++ b/app/Http/Controllers/ProjectController.php @@ -67,7 +67,7 @@ class ProjectController extends Controller $database = create_standalone_mongodb($environment->id, $destination_uuid); } else if ($type->value() === 'mysql') { $database = create_standalone_mysql($environment->id, $destination_uuid); - }else if ($type->value() === 'mariadb') { + } else if ($type->value() === 'mariadb') { $database = create_standalone_mariadb($environment->id, $destination_uuid); } return redirect()->route('project.database.configuration', [ @@ -104,27 +104,7 @@ class ProjectController extends Controller $generatedValue = $value; if ($value->contains('SERVICE_')) { $command = $value->after('SERVICE_')->beforeLast('_'); - // TODO: make it shared with Service.php - switch ($command->value()) { - case 'PASSWORD': - $generatedValue = Str::password(symbols: false); - break; - case 'PASSWORD_64': - $generatedValue = Str::password(length: 64, symbols: false); - break; - case 'BASE64_64': - $generatedValue = Str::random(64); - break; - case 'BASE64_128': - $generatedValue = Str::random(128); - break; - case 'BASE64': - $generatedValue = Str::random(32); - break; - case 'USER': - $generatedValue = Str::random(16); - break; - } + $generatedValue = generateEnvValue($command->value()); } EnvironmentVariable::create([ 'key' => $key, @@ -137,7 +117,7 @@ class ProjectController extends Controller } $service->parse(isNew: true); - return redirect()->route('project.service', [ + return redirect()->route('project.service.configuration', [ 'service_uuid' => $service->uuid, 'environment_name' => $environment->name, 'project_uuid' => $project->uuid, diff --git a/app/Http/Livewire/Project/Application/General.php b/app/Http/Livewire/Project/Application/General.php index 27c5023c3..87b894bf4 100644 --- a/app/Http/Livewire/Project/Application/General.php +++ b/app/Http/Livewire/Project/Application/General.php @@ -6,6 +6,7 @@ use App\Models\Application; use Illuminate\Support\Collection; use Illuminate\Support\Str; use Livewire\Component; +use Visus\Cuid2\Cuid2; class General extends Component { @@ -25,8 +26,14 @@ class General extends Component public bool $labelsChanged = false; public bool $isConfigurationChanged = false; + public ?string $initialDockerComposeLocation = null; + public ?string $initialDockerComposePrLocation = null; + public bool $is_static; + public $parsedServices = []; + public $parsedServiceDomains = []; + protected $listeners = [ 'resetDefaultLabels' ]; @@ -50,6 +57,12 @@ class General extends Component 'application.docker_registry_image_name' => 'nullable', 'application.docker_registry_image_tag' => 'nullable', 'application.dockerfile_location' => 'nullable', + 'application.docker_compose_location' => 'nullable', + 'application.docker_compose_pr_location' => 'nullable', + 'application.docker_compose' => 'nullable', + 'application.docker_compose_pr' => 'nullable', + 'application.docker_compose_raw' => 'nullable', + 'application.docker_compose_pr_raw' => 'nullable', 'application.custom_labels' => 'nullable', 'application.dockerfile_target_build' => 'nullable', 'application.settings.is_static' => 'boolean|required', @@ -74,6 +87,12 @@ class General extends Component 'application.docker_registry_image_name' => 'Docker registry image name', 'application.docker_registry_image_tag' => 'Docker registry image tag', 'application.dockerfile_location' => 'Dockerfile location', + 'application.docker_compose_location' => 'Docker compose location', + 'application.docker_compose_pr_location' => 'Docker compose location', + 'application.docker_compose' => 'Docker compose', + 'application.docker_compose_pr' => 'Docker compose', + 'application.docker_compose_raw' => 'Docker compose raw', + 'application.docker_compose_pr_raw' => 'Docker compose raw', 'application.custom_labels' => 'Custom labels', 'application.dockerfile_target_build' => 'Dockerfile target build', 'application.settings.is_static' => 'Is static', @@ -81,6 +100,13 @@ class General extends Component public function mount() { + try { + $this->parsedServices = $this->application->parseCompose(); + } catch (\Throwable $e) { + $this->emit('error', $e->getMessage()); + } + $this->parsedServiceDomains = $this->application->docker_compose_domains ? json_decode($this->application->docker_compose_domains, true) : []; + $this->ports_exposes = $this->application->ports_exposes; if (str($this->application->status)->startsWith('running') && is_null($this->application->config_hash)) { $this->application->isConfigurationChanged(true); @@ -91,6 +117,7 @@ class General extends Component } else { $this->customLabels = str($this->application->custom_labels)->replace(',', "\n"); } + $this->initialDockerComposeLocation = $this->application->docker_compose_location; $this->checkLabelUpdates(); } public function instantSave() @@ -98,12 +125,44 @@ class General extends Component $this->application->settings->save(); $this->emit('success', 'Settings saved.'); } + public function loadComposeFile($isInit = false) + { + try { + if ($isInit && $this->application->docker_compose_raw) { + return; + } + ['parsedServices' => $this->parsedServices, 'initialDockerComposeLocation' => $this->initialDockerComposeLocation, 'initialDockerComposePrLocation' => $this->initialDockerComposePrLocation] = $this->application->loadComposeFile($isInit); + $this->emit('success', 'Docker compose file loaded.'); + } catch (\Throwable $e) { + $this->application->docker_compose_location = $this->initialDockerComposeLocation; + $this->application->docker_compose_pr_location = $this->initialDockerComposePrLocation; + $this->application->save(); + return handleError($e, $this); + } + } + public function generateDomain(string $serviceName) + { + $domain = $this->parsedServiceDomains[$serviceName]['domain'] ?? null; + if (!$domain) { + $uuid = new Cuid2(7); + $domain = generateFqdn($this->application->destination->server, $uuid); + $this->parsedServiceDomains[$serviceName]['domain'] = $domain; + $this->application->docker_compose_domains = json_encode($this->parsedServiceDomains); + $this->application->save(); + $this->emit('success', 'Domain generated.'); + } + return $domain; + } public function updatedApplicationBuildPack() { if ($this->application->build_pack !== 'nixpacks') { $this->application->settings->is_static = $this->is_static = false; $this->application->settings->save(); } + if ($this->application->build_pack === 'dockercompose') { + $this->application->fqdn = null; + $this->application->settings->save(); + } $this->submit(); } public function checkLabelUpdates() @@ -140,6 +199,9 @@ class General extends Component public function submit($showToaster = true) { try { + if ($this->application->build_pack === 'dockercompose' && ($this->initialDockerComposeLocation !== $this->application->docker_compose_location || $this->initialDockerComposePrLocation !== $this->application->docker_compose_pr_location)) { + $this->loadComposeFile(); + } $this->validate(); if ($this->ports_exposes !== $this->application->ports_exposes) { $this->resetDefaultLabels(false); @@ -172,6 +234,10 @@ class General extends Component $this->customLabels = str($this->customLabels)->replace(',', "\n"); } $this->application->custom_labels = $this->customLabels->explode("\n")->implode(','); + if ($this->application->build_pack === 'dockercompose') { + $this->application->docker_compose_domains = json_encode($this->parsedServiceDomains); + $this->parsedServices = $this->application->parseCompose(); + } $this->application->save(); $showToaster && $this->emit('success', 'Application settings updated!'); } catch (\Throwable $e) { diff --git a/app/Http/Livewire/Project/Application/Heading.php b/app/Http/Livewire/Project/Application/Heading.php index e09e469a9..14bdf1db0 100644 --- a/app/Http/Livewire/Project/Application/Heading.php +++ b/app/Http/Livewire/Project/Application/Heading.php @@ -41,6 +41,10 @@ class Heading extends Component public function deploy(bool $force_rebuild = false) { + if ($this->application->build_pack === 'dockercompose' && is_null($this->application->docker_compose_raw)) { + $this->emit('error', 'Please load a Compose file first.'); + return; + } $this->setDeploymentUuid(); queue_application_deployment( application_id: $this->application->id, @@ -68,7 +72,8 @@ class Heading extends Component $this->application->save(); $this->application->refresh(); } - public function restart() { + public function restart() + { $this->setDeploymentUuid(); queue_application_deployment( application_id: $this->application->id, diff --git a/app/Http/Livewire/Project/Application/Previews.php b/app/Http/Livewire/Project/Application/Previews.php index 1c9674011..effdd7f7c 100644 --- a/app/Http/Livewire/Project/Application/Previews.php +++ b/app/Http/Livewire/Project/Application/Previews.php @@ -72,10 +72,12 @@ class Previews extends Component public function stop(int $pull_request_id) { try { - $container_name = generateApplicationContainerName($this->application, $pull_request_id); - - instant_remote_process(["docker rm -f $container_name"], $this->application->destination->server, throwError: false); - ApplicationPreview::where('application_id', $this->application->id)->where('pull_request_id', $pull_request_id)->delete(); + $containers = getCurrentApplicationContainerStatus($this->application->destination->server, $this->application->id, $pull_request_id); + foreach ($containers as $container) { + $name = str_replace('/', '', $container['Names']); + instant_remote_process(["docker rm -f $name"], $this->application->destination->server, throwError: false); + } + ApplicationPreview::where('application_id', $this->application->id)->where('pull_request_id', $pull_request_id)->first()->delete(); $this->application->refresh(); } catch (\Throwable $e) { return handleError($e, $this); diff --git a/app/Http/Livewire/Project/New/DockerCompose.php b/app/Http/Livewire/Project/New/DockerCompose.php index 81c487ccf..cd0d86c08 100644 --- a/app/Http/Livewire/Project/New/DockerCompose.php +++ b/app/Http/Livewire/Project/New/DockerCompose.php @@ -129,7 +129,7 @@ class DockerCompose extends Component $service->parse(isNew: true); - return redirect()->route('project.service', [ + return redirect()->route('project.service.configuration', [ 'service_uuid' => $service->uuid, 'environment_name' => $environment->name, 'project_uuid' => $project->uuid, diff --git a/app/Http/Livewire/Project/New/Select.php b/app/Http/Livewire/Project/New/Select.php index a4b80ce69..13fbb8886 100644 --- a/app/Http/Livewire/Project/New/Select.php +++ b/app/Http/Livewire/Project/New/Select.php @@ -47,7 +47,7 @@ class Select extends Component } public function render() { - $this->loadServices(); + if ($this->search) $this->loadServices(); return view('livewire.project.new.select'); } @@ -69,10 +69,10 @@ class Select extends Component // } // } - public function loadServices(bool $force = false) + public function loadServices() { try { - if (count($this->allServices) > 0 && !$force) { + if (count($this->allServices) > 0) { if (!$this->search) { $this->services = $this->allServices; return; diff --git a/app/Http/Livewire/Project/Service/Application.php b/app/Http/Livewire/Project/Service/Application.php index 14fd3d5ab..622b1e0e2 100644 --- a/app/Http/Livewire/Project/Service/Application.php +++ b/app/Http/Livewire/Project/Service/Application.php @@ -41,7 +41,7 @@ class Application extends Component try { $this->application->delete(); $this->emit('success', 'Application deleted successfully.'); - return redirect()->route('project.service', $this->parameters); + return redirect()->route('project.service.configuration', $this->parameters); } catch (\Throwable $e) { return handleError($e, $this); } diff --git a/app/Http/Livewire/Project/Service/StackForm.php b/app/Http/Livewire/Project/Service/StackForm.php index ebdb2d481..1014756ce 100644 --- a/app/Http/Livewire/Project/Service/StackForm.php +++ b/app/Http/Livewire/Project/Service/StackForm.php @@ -23,7 +23,7 @@ class StackForm extends Component foreach ($fields as $fieldKey => $field) { $key = data_get($field, 'key'); $value = data_get($field, 'value'); - $rules = data_get($field, 'rules'); + $rules = data_get($field, 'rules', 'nullable'); $isPassword = data_get($field, 'isPassword'); $this->fields[$key] = [ "serviceName" => $serviceName, @@ -31,6 +31,7 @@ class StackForm extends Component "name" => $fieldKey, "value" => $value, "isPassword" => $isPassword, + "rules" => $rules ]; $this->rules["fields.$key.value"] = $rules; $this->validationAttributes["fields.$key.value"] = $fieldKey; diff --git a/app/Http/Livewire/Project/Shared/EnvironmentVariable/Show.php b/app/Http/Livewire/Project/Shared/EnvironmentVariable/Show.php index eed0f7052..dc2808d08 100644 --- a/app/Http/Livewire/Project/Shared/EnvironmentVariable/Show.php +++ b/app/Http/Livewire/Project/Shared/EnvironmentVariable/Show.php @@ -59,6 +59,7 @@ class Show extends Component { $this->validate(); $this->env->save(); + ray($this->env); $this->emit('success', 'Environment variable updated successfully.'); $this->emit('refreshEnvs'); } diff --git a/app/Http/Livewire/Project/Shared/Logs.php b/app/Http/Livewire/Project/Shared/Logs.php index f58ec672a..982f729b4 100644 --- a/app/Http/Livewire/Project/Shared/Logs.php +++ b/app/Http/Livewire/Project/Shared/Logs.php @@ -17,13 +17,15 @@ class Logs extends Component public ?string $type = null; public Application|Service|StandalonePostgresql|StandaloneRedis|StandaloneMongodb|StandaloneMysql|StandaloneMariadb $resource; public Server $server; - public ?string $container = null; + public $container = []; + public $containers; public $parameters; public $query; public $status; public function mount() { + $this->containers = collect(); $this->parameters = get_route_parameters(); $this->query = request()->query(); if (data_get($this->parameters, 'application_uuid')) { @@ -33,7 +35,9 @@ class Logs extends Component $this->server = $this->resource->destination->server; $containers = getCurrentApplicationContainerStatus($this->server, $this->resource->id, 0); if ($containers->count() > 0) { - $this->container = data_get($containers[0], 'Names'); + $containers->each(function ($container) { + $this->containers->push(str_replace('/', '', $container['Names'])); + }); } } else if (data_get($this->parameters, 'database_uuid')) { $this->type = 'database'; diff --git a/app/Http/Livewire/Server/Show.php b/app/Http/Livewire/Server/Show.php index 3863381b2..a72abdf94 100644 --- a/app/Http/Livewire/Server/Show.php +++ b/app/Http/Livewire/Server/Show.php @@ -19,6 +19,7 @@ class Show extends Component if (is_null($this->server)) { return redirect()->route('server.all'); } + } catch (\Throwable $e) { return handleError($e, $this); } diff --git a/app/Jobs/ApplicationDeployDockerImageJob.php b/app/Jobs/ApplicationDeployDockerImageJob.php deleted file mode 100644 index dda5629c8..000000000 --- a/app/Jobs/ApplicationDeployDockerImageJob.php +++ /dev/null @@ -1,111 +0,0 @@ -applicationDeploymentQueueId = $applicationDeploymentQueueId; - } - public function handle() - { - ray()->clearAll(); - ray('Deploying Docker Image'); - try { - $applicationDeploymentQueue = ApplicationDeploymentQueue::find($this->applicationDeploymentQueueId); - $application = Application::find($applicationDeploymentQueue->application_id); - - $deploymentUuid = data_get($applicationDeploymentQueue, 'deployment_uuid'); - $dockerImage = data_get($application, 'docker_registry_image_name'); - $dockerImageTag = data_get($application, 'docker_registry_image_tag'); - $productionImageName = str("{$dockerImage}:{$dockerImageTag}"); - $destination = $application->destination->getMorphClass()::where('id', $application->destination->id)->first(); - $pullRequestId = data_get($applicationDeploymentQueue, 'pull_request_id'); - - $server = data_get($destination, 'server'); - $network = data_get($destination, 'network'); - - $containerName = generateApplicationContainerName($application, $pullRequestId); - savePrivateKeyToFs($server); - - ray("echo 'Starting deployment of {$productionImageName}.'"); - - $applicationDeploymentQueue->update([ - 'status' => ApplicationDeploymentStatus::IN_PROGRESS->value, - ]); - $this->executeRemoteCommand( - server: $server, - logModel: $applicationDeploymentQueue, - commands: prepareHelperContainer($server, $network, $deploymentUuid) - ); - - $this->executeRemoteCommand( - server: $server, - logModel: $applicationDeploymentQueue, - commands: generateComposeFile( - deploymentUuid: $deploymentUuid, - server: $server, - network: $network, - application: $application, - containerName: $containerName, - imageName: $productionImageName, - pullRequestId: $pullRequestId - ) - ); - $this->executeRemoteCommand( - server: $server, - logModel: $applicationDeploymentQueue, - commands: rollingUpdate(application: $application, deploymentUuid: $deploymentUuid) - ); - } catch (Throwable $e) { - $this->executeRemoteCommand( - server: $server, - logModel: $applicationDeploymentQueue, - commands: [ - "echo 'Oops something is not okay, are you okay? 😢'", - "echo '{$e->getMessage()}'", - "echo -n 'Deployment failed. Removing the new version of your application.'", - executeInDocker($deploymentUuid, "docker rm -f $containerName >/dev/null 2>&1"), - ] - ); - // $this->next(ApplicationDeploymentStatus::FAILED->value); - throw $e; - } - } - // private function next(string $status) - // { - // // If the deployment is cancelled by the user, don't update the status - // if ($this->application_deployment_queue->status !== ApplicationDeploymentStatus::CANCELLED_BY_USER->value) { - // $this->application_deployment_queue->update([ - // 'status' => $status, - // ]); - // } - // queue_next_deployment($this->application); - // if ($status === ApplicationDeploymentStatus::FINISHED->value) { - // $this->application->environment->project->team->notify(new DeploymentSuccess($this->application, $this->deployment_uuid, $this->preview)); - // } - // if ($status === ApplicationDeploymentStatus::FAILED->value) { - // $this->application->environment->project->team->notify(new DeploymentFailed($this->application, $this->deployment_uuid, $this->preview)); - // } - // } -} diff --git a/app/Jobs/ApplicationDeploySimpleDockerfileJob.php b/app/Jobs/ApplicationDeploySimpleDockerfileJob.php deleted file mode 100644 index a9e17bc80..000000000 --- a/app/Jobs/ApplicationDeploySimpleDockerfileJob.php +++ /dev/null @@ -1,29 +0,0 @@ -applicationDeploymentQueueId = $applicationDeploymentQueueId; - } - public function handle() { - ray('Deploying Simple Dockerfile'); - } -} diff --git a/app/Jobs/ApplicationDeploymentJob.php b/app/Jobs/ApplicationDeploymentJob.php index 1dcd51b22..7d229e879 100644 --- a/app/Jobs/ApplicationDeploymentJob.php +++ b/app/Jobs/ApplicationDeploymentJob.php @@ -73,9 +73,9 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted private $docker_compose; private $docker_compose_base64; private string $dockerfile_location = '/Dockerfile'; + private string $docker_compose_location = '/docker-compose.yml'; private ?string $addHosts = null; private ?string $buildTarget = null; - private $log_model; private Collection $saved_outputs; private ?string $full_healthcheck_url = null; @@ -92,9 +92,7 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted public $tries = 1; public function __construct(int $application_deployment_queue_id) { - // ray()->clearScreen(); $this->application_deployment_queue = ApplicationDeploymentQueue::find($application_deployment_queue_id); - $this->log_model = $this->application_deployment_queue; $this->application = Application::find($this->application_deployment_queue->application_id); $this->build_pack = data_get($this->application, 'build_pack'); @@ -114,7 +112,7 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted $this->destination = $this->application->destination->getMorphClass()::where('id', $this->application->destination->id)->first(); $this->server = $this->mainServer = $this->destination->server; $this->serverUser = $this->server->user; - $this->basedir = "/artifacts/{$this->deployment_uuid}"; + $this->basedir = $this->application->generateBaseDir($this->deployment_uuid); $this->workdir = "{$this->basedir}" . rtrim($this->application->base_directory, '/'); $this->configuration_dir = application_configuration_dir() . "/{$this->application->uuid}"; $this->is_debug_enabled = $this->application->settings->is_debug_enabled; @@ -183,15 +181,8 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted } // Check custom port - preg_match('/(?<=:)\d+(?=\/)/', $this->application->git_repository, $matches); - if (count($matches) === 1) { - $this->customPort = $matches[0]; - $gitHost = str($this->application->git_repository)->before(':'); - $gitRepo = str($this->application->git_repository)->after('/'); - $this->customRepository = "$gitHost:$gitRepo"; - } else { - $this->customRepository = $this->application->git_repository; - } + ['repository' => $this->customRepository, 'port' => $this->customPort] = $this->application->customRepository(); + try { if ($this->restart_only && $this->application->build_pack !== 'dockerimage') { $this->just_restart(); @@ -203,6 +194,8 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted return; } else if ($this->application->dockerfile) { $this->deploy_simple_dockerfile(); + } else if ($this->application->build_pack === 'dockercompose') { + $this->deploy_docker_compose_buildpack(); } else if ($this->application->build_pack === 'dockerimage') { $this->deploy_dockerimage_buildpack(); } else if ($this->application->build_pack === 'dockerfile') { @@ -397,19 +390,27 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted ]); } } - // private function save_environment_variables() - // { - // $envs = collect([]); - // foreach ($this->application->environment_variables as $env) { - // $envs->push($env->key . '=' . $env->value); - // } - // $envs_base64 = base64_encode($envs->implode("\n")); - // $this->execute_remote_command( - // [ - // executeInDocker($this->deployment_uuid, "echo '$envs_base64' | base64 -d > $this->workdir/.env") - // ], - // ); - // } + private function save_environment_variables() + { + $envs = collect([]); + if ($this->pull_request_id !== 0) { + foreach ($this->application->environment_variables_preview as $env) { + $envs->push($env->key . '=' . $env->value); + } + } else { + foreach ($this->application->environment_variables as $env) { + $envs->push($env->key . '=' . $env->value); + } + } + ray($envs); + $envs_base64 = base64_encode($envs->implode("\n")); + $this->execute_remote_command( + [ + executeInDocker($this->deployment_uuid, "echo '$envs_base64' | base64 -d > $this->workdir/.env") + ], + ); + } + private function deploy_simple_dockerfile() { $dockerfile_base64 = base64_encode($this->application->dockerfile); @@ -447,7 +448,45 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted $this->generate_compose_file(); $this->rolling_update(); } + private function deploy_docker_compose_buildpack() + { + if (data_get($this->application, 'docker_compose_location')) { + $this->docker_compose_location = $this->application->docker_compose_location; + } + if ($this->pull_request_id === 0) { + $this->application_deployment_queue->addLogEntry("Starting deployment of {$this->application->name}."); + } else { + $this->application_deployment_queue->addLogEntry("Starting pull request (#{$this->pull_request_id}) deployment of {$this->customRepository}:{$this->application->git_branch}."); + } + $this->server->executeRemoteCommand( + commands: $this->application->prepareHelperImage($this->deployment_uuid), + loggingModel: $this->application_deployment_queue + ); + $this->check_git_if_build_needed(); + $this->clone_repository(); + $this->generate_image_names(); + $this->cleanup_git(); + $composeFile = $this->application->parseCompose(pull_request_id: $this->pull_request_id); + $yaml = Yaml::dump($composeFile->toArray(), 10); + $this->docker_compose_base64 = base64_encode($yaml); + $this->execute_remote_command([ + executeInDocker($this->deployment_uuid, "echo '{$this->docker_compose_base64}' | base64 -d > {$this->workdir}/docker-compose.yaml"), "hidden" => true + ]); + $this->save_environment_variables(); + $this->stop_running_container(force: true); + $networkId = $this->application->uuid; + if ($this->pull_request_id !== 0) { + $networkId = "{$this->application->uuid}-{$this->pull_request_id}"; + } + $this->execute_remote_command([ + "docker network create --attachable '{$networkId}' >/dev/null || true", "hidden" => true, "ignore_errors" => true + ], [ + "docker network connect {$networkId} coolify-proxy || true", "hidden" => true, "ignore_errors" => true + ]); + $this->start_by_compose_file(); + $this->application->loadComposeFile(isInit: false); + } private function deploy_dockerfile_buildpack() { if (data_get($this->application, 'dockerfile_location')) { @@ -472,7 +511,7 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted // $this->push_to_docker_registry(); // $this->deploy_to_additional_destinations(); // } else { - $this->rolling_update(); + $this->rolling_update(); // } } private function deploy_nixpacks_buildpack() @@ -618,8 +657,8 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted } $this->generate_compose_file(); // Needs separate preview variables - // $this->generate_build_env_variables(); - // $this->add_build_env_variables_to_dockerfile(); + $this->generate_build_env_variables(); + $this->add_build_env_variables_to_dockerfile(); $this->build_image(); $this->stop_running_container(); $this->execute_remote_command( @@ -725,13 +764,12 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted private function clone_repository() { $importCommands = $this->generate_git_import_commands(); + $this->application_deployment_queue->addLogEntry("\n----------------------------------------"); + $this->application_deployment_queue->addLogEntry("Importing {$this->customRepository}:{$this->application->git_branch} (commit sha {$this->application->git_commit_sha}) to {$this->basedir}."); + if ($this->pull_request_id !== 0) { + $this->application_deployment_queue->addLogEntry("Checking out tag pull/{$this->pull_request_id}/head."); + } $this->execute_remote_command( - [ - "echo '\n----------------------------------------'", - ], - [ - "echo -n 'Importing {$this->customRepository}:{$this->application->git_branch} (commit sha {$this->application->git_commit_sha}) to {$this->basedir}. '" - ], [ $importCommands, "hidden" => true ] @@ -740,90 +778,13 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted private function generate_git_import_commands() { - $this->branch = $this->application->git_branch; - $commands = collect([]); - $git_clone_command = "git clone -q -b {$this->application->git_branch}"; - if ($this->pull_request_id !== 0) { - $pr_branch_name = "pr-{$this->pull_request_id}-coolify"; - } - - if ($this->application->deploymentType() === 'source') { - $source_html_url = data_get($this->application, 'source.html_url'); - $url = parse_url(filter_var($source_html_url, FILTER_SANITIZE_URL)); - $source_html_url_host = $url['host']; - $source_html_url_scheme = $url['scheme']; - - if ($this->source->getMorphClass() == 'App\Models\GithubApp') { - if ($this->source->is_public) { - $this->fullRepoUrl = "{$this->source->html_url}/{$this->customRepository}"; - $git_clone_command = "{$git_clone_command} {$this->source->html_url}/{$this->customRepository} {$this->basedir}"; - $git_clone_command = $this->set_git_import_settings($git_clone_command); - - $commands->push(executeInDocker($this->deployment_uuid, $git_clone_command)); - } else { - $github_access_token = generate_github_installation_token($this->source); - $commands->push(executeInDocker($this->deployment_uuid, "git clone -q -b {$this->application->git_branch} $source_html_url_scheme://x-access-token:$github_access_token@$source_html_url_host/{$this->customRepository}.git {$this->basedir}")); - $this->fullRepoUrl = "$source_html_url_scheme://x-access-token:$github_access_token@$source_html_url_host/{$this->customRepository}.git"; - } - if ($this->pull_request_id !== 0) { - $this->branch = "pull/{$this->pull_request_id}/head:$pr_branch_name"; - $commands->push(executeInDocker($this->deployment_uuid, "cd {$this->basedir} && git fetch origin $this->branch && git checkout $pr_branch_name")); - } - return $commands->implode(' && '); - } - } - if ($this->application->deploymentType() === 'deploy_key') { - $this->fullRepoUrl = $this->customRepository; - $private_key = data_get($this->application, 'private_key.private_key'); - if (is_null($private_key)) { - throw new RuntimeException('Private key not found. Please add a private key to the application and try again.'); - } - $private_key = base64_encode($private_key); - $git_clone_command_base = "GIT_SSH_COMMAND=\"ssh -o ConnectTimeout=30 -p {$this->customPort} -o Port={$this->customPort} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /root/.ssh/id_rsa\" {$git_clone_command} {$this->customRepository} {$this->basedir}"; - $git_clone_command = $this->set_git_import_settings($git_clone_command_base); - $commands = collect([ - executeInDocker($this->deployment_uuid, "mkdir -p /root/.ssh"), - executeInDocker($this->deployment_uuid, "echo '{$private_key}' | base64 -d > /root/.ssh/id_rsa"), - executeInDocker($this->deployment_uuid, "chmod 600 /root/.ssh/id_rsa"), - ]); - if ($this->pull_request_id !== 0) { - ray($this->git_type); - if ($this->git_type === 'gitlab') { - $this->branch = "merge-requests/{$this->pull_request_id}/head:$pr_branch_name"; - $commands->push(executeInDocker($this->deployment_uuid, "echo 'Checking out $this->branch'")); - $git_clone_command = "{$git_clone_command} && cd {$this->basedir} && GIT_SSH_COMMAND=\"ssh -o ConnectTimeout=30 -p {$this->customPort} -o Port={$this->customPort} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /root/.ssh/id_rsa\" git fetch origin $this->branch && git checkout $pr_branch_name"; - } - if ($this->git_type === 'github') { - $this->branch = "pull/{$this->pull_request_id}/head:$pr_branch_name"; - $commands->push(executeInDocker($this->deployment_uuid, "echo 'Checking out $this->branch'")); - $git_clone_command = "{$git_clone_command} && cd {$this->basedir} && GIT_SSH_COMMAND=\"ssh -o ConnectTimeout=30 -p {$this->customPort} -o Port={$this->customPort} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /root/.ssh/id_rsa\" git fetch origin $this->branch && git checkout $pr_branch_name"; - } - } - - $commands->push(executeInDocker($this->deployment_uuid, $git_clone_command)); - return $commands->implode(' && '); - } - if ($this->application->deploymentType() === 'other') { - $this->fullRepoUrl = $this->customRepository; - $git_clone_command = "{$git_clone_command} {$this->customRepository} {$this->basedir}"; - $git_clone_command = $this->set_git_import_settings($git_clone_command); - $commands->push(executeInDocker($this->deployment_uuid, $git_clone_command)); - return $commands->implode(' && '); - } + ['commands' => $commands, 'branch' => $this->branch, 'fullRepoUrl' => $this->fullRepoUrl] = $this->application->generateGitImportCommands($this->deployment_uuid, $this->pull_request_id, $this->git_type); + return $commands; } private function set_git_import_settings($git_clone_command) { - if ($this->application->git_commit_sha !== 'HEAD') { - $git_clone_command = "{$git_clone_command} && cd {$this->basedir} && git -c advice.detachedHead=false checkout {$this->application->git_commit_sha} >/dev/null 2>&1"; - } - if ($this->application->settings->is_git_submodules_enabled) { - $git_clone_command = "{$git_clone_command} && cd {$this->basedir} && git submodule update --init --recursive"; - } - if ($this->application->settings->is_git_lfs_enabled) { - $git_clone_command = "{$git_clone_command} && cd {$this->basedir} && git lfs pull"; - } - return $git_clone_command; + return $this->application->setGitImportSettings($this->deployment_uuid, $git_clone_command); } private function cleanup_git() @@ -849,7 +810,11 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted private function nixpacks_build_cmd() { $this->generate_env_variables(); - $nixpacks_command = "nixpacks build --cache-key '{$this->application->uuid}' -o {$this->workdir} {$this->env_args} --no-error-without-start"; + $cacheKey = $this->application->uuid; + if ($this->pull_request_id !== 0) { + $cacheKey = "{$this->application->uuid}-pr-{$this->pull_request_id}"; + } + $nixpacks_command = "nixpacks build --cache-key '{$cacheKey}' -o {$this->workdir} {$this->env_args} --no-error-without-start"; if ($this->application->build_command) { $nixpacks_command .= " --build-cmd \"{$this->application->build_command}\""; } @@ -879,6 +844,26 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted $this->env_args = $this->env_args->implode(' '); } + private function modify_compose_file() + { + // ray("{$this->workdir}{$this->docker_compose_location}"); + $this->execute_remote_command([executeInDocker($this->deployment_uuid, "cat {$this->workdir}{$this->docker_compose_location}"), "hidden" => true, "save" => 'compose_file']); + if ($this->saved_outputs->get('compose_file')) { + $compose = $this->saved_outputs->get('compose_file'); + } + try { + $yaml = Yaml::parse($compose); + } catch (\Exception $e) { + throw new \Exception($e->getMessage()); + } + $services = data_get($yaml, 'services'); + $topLevelNetworks = collect(data_get($yaml, 'networks', [])); + $definedNetwork = collect([$this->application->uuid]); + + $services = collect($services)->map(function ($service, $serviceName) use ($topLevelNetworks, $definedNetwork) { + $serviceNetworks = collect(data_get($service, 'networks', [])); + }); + } private function generate_compose_file() { $ports = $this->application->settings->is_static ? [80] : $this->application->ports_exposes_array; @@ -967,7 +952,6 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted ]; } if ($this->application->settings->is_gpu_enabled) { - ray('asd'); $docker_compose['services'][$this->container_name]['deploy']['resources']['reservations']['devices'] = [ [ 'driver' => data_get($this->application, 'settings.gpu_driver', 'nvidia'), @@ -1206,14 +1190,10 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf"); private function stop_running_container(bool $force = false) { - $this->execute_remote_command(["echo -n 'Removing old container.'"]); + $this->application_deployment_queue->addLogEntry("Removing old containers."); if ($this->newVersionIsHealthy || $force) { $containers = getCurrentApplicationContainerStatus($this->server, $this->application->id, $this->pull_request_id); - if ($this->pull_request_id !== 0) { - $containers = $containers->filter(function ($container) { - return data_get($container, 'Names') === $this->container_name; - }); - } else { + if ($this->pull_request_id === 0) { $containers = $containers->filter(function ($container) { return data_get($container, 'Names') !== $this->container_name; }); @@ -1224,14 +1204,10 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf"); [executeInDocker($this->deployment_uuid, "docker rm -f $containerName >/dev/null 2>&1"), "hidden" => true, "ignore_errors" => true], ); }); - $this->execute_remote_command( - [ - "echo 'Rolling update completed.'" - ], - ); + $this->application_deployment_queue->addLogEntry("Rolling update completed."); } else { + $this->application_deployment_queue->addLogEntry("New container is not healthy, rolling back to the old container."); $this->execute_remote_command( - ["echo -n 'New container is not healthy, rolling back to the old container.'"], [executeInDocker($this->deployment_uuid, "docker rm -f $this->container_name >/dev/null 2>&1"), "hidden" => true, "ignore_errors" => true], ); } @@ -1240,8 +1216,8 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf"); private function start_by_compose_file() { if ($this->application->build_pack === 'dockerimage') { + $this->application_deployment_queue->addLogEntry("Pulling latest images from the registry."); $this->execute_remote_command( - ["echo -n 'Pulling latest images from the registry.'"], [executeInDocker($this->deployment_uuid, "docker compose --project-directory {$this->workdir} pull"), "hidden" => true], [executeInDocker($this->deployment_uuid, "docker compose --project-directory {$this->workdir} up --build -d"), "hidden" => true], ); @@ -1274,10 +1250,16 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf"); executeInDocker($this->deployment_uuid, "cat {$this->workdir}{$this->dockerfile_location}"), "hidden" => true, "save" => 'dockerfile' ]); $dockerfile = collect(Str::of($this->saved_outputs->get('dockerfile'))->trim()->explode("\n")); - - foreach ($this->application->build_environment_variables as $env) { - $dockerfile->splice(1, 0, "ARG {$env->key}={$env->value}"); + if ($this->pull_request_id === 0) { + foreach ($this->application->build_environment_variables as $env) { + $dockerfile->splice(1, 0, "ARG {$env->key}={$env->value}"); + } + } else { + foreach ($this->application->build_environment_variables_preview as $env) { + $dockerfile->splice(1, 0, "ARG {$env->key}={$env->value}"); + } } + ray($dockerfile->implode("\n")); $dockerfile_base64 = base64_encode($dockerfile->implode("\n")); $this->execute_remote_command([ executeInDocker($this->deployment_uuid, "echo '{$dockerfile_base64}' | base64 -d > {$this->workdir}{$this->dockerfile_location}"), diff --git a/app/Jobs/ContainerStatusJob.php b/app/Jobs/ContainerStatusJob.php index a45bebf8e..79265f7a1 100644 --- a/app/Jobs/ContainerStatusJob.php +++ b/app/Jobs/ContainerStatusJob.php @@ -35,17 +35,13 @@ class ContainerStatusJob implements ShouldQueue, ShouldBeEncrypted return $this->server->id; } - public function handle(): void + public function handle() { // ray("checking container statuses for {$this->server->id}"); try { if (!$this->server->isServerReady()) { return; }; - $containers = instant_remote_process(["docker container ls -q"], $this->server); - if (!$containers) { - return; - } $containers = instant_remote_process(["docker container inspect $(docker container ls -q) --format '{{json .}}'"], $this->server); $containers = format_docker_command_output_to_json($containers); $applications = $this->server->applications(); @@ -167,7 +163,7 @@ class ContainerStatusJob implements ShouldQueue, ShouldBeEncrypted } else { $url = null; } - $this->server->team->notify(new ContainerStopped($containerName, $this->server, $url)); + $this->server->team?->notify(new ContainerStopped($containerName, $this->server, $url)); $exitedService->update(['status' => 'exited']); } @@ -194,7 +190,7 @@ class ContainerStatusJob implements ShouldQueue, ShouldBeEncrypted $url = null; } - $this->server->team->notify(new ContainerStopped($containerName, $this->server, $url)); + $this->server->team?->notify(new ContainerStopped($containerName, $this->server, $url)); } $notRunningApplicationPreviews = $previews->pluck('id')->diff($foundApplicationPreviews); foreach ($notRunningApplicationPreviews as $previewId) { @@ -219,7 +215,7 @@ class ContainerStatusJob implements ShouldQueue, ShouldBeEncrypted $url = null; } - $this->server->team->notify(new ContainerStopped($containerName, $this->server, $url)); + $this->server->team?->notify(new ContainerStopped($containerName, $this->server, $url)); } $notRunningDatabases = $databases->pluck('id')->diff($foundDatabases); foreach ($notRunningDatabases as $database) { @@ -243,7 +239,7 @@ class ContainerStatusJob implements ShouldQueue, ShouldBeEncrypted } else { $url = null; } - $this->server->team->notify(new ContainerStopped($containerName, $this->server, $url)); + $this->server->team?->notify(new ContainerStopped($containerName, $this->server, $url)); } // Check if proxy is running @@ -256,7 +252,7 @@ class ContainerStatusJob implements ShouldQueue, ShouldBeEncrypted $shouldStart = CheckProxy::run($this->server); if ($shouldStart) { StartProxy::run($this->server, false); - $this->server->team->notify(new ContainerRestarted('coolify-proxy', $this->server)); + $this->server->team?->notify(new ContainerRestarted('coolify-proxy', $this->server)); } else { ray('Proxy could not be started.'); } @@ -272,7 +268,7 @@ class ContainerStatusJob implements ShouldQueue, ShouldBeEncrypted } catch (\Throwable $e) { send_internal_notification("ContainerStatusJob failed on ({$this->server->id}) with: " . $e->getMessage()); ray($e->getMessage()); - handleError($e); + return handleError($e); } } } diff --git a/app/Jobs/DockerCleanupJob.php b/app/Jobs/DockerCleanupJob.php index 14ca11b22..a48ddd248 100644 --- a/app/Jobs/DockerCleanupJob.php +++ b/app/Jobs/DockerCleanupJob.php @@ -3,8 +3,6 @@ namespace App\Jobs; use App\Models\Server; -use App\Notifications\Server\HighDiskUsage; -use Exception; use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldBeEncrypted; use Illuminate\Contracts\Queue\ShouldQueue; @@ -13,6 +11,7 @@ use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\Middleware\WithoutOverlapping; use Illuminate\Queue\SerializesModels; use Illuminate\Support\Facades\Log; +use RuntimeException; class DockerCleanupJob implements ShouldQueue, ShouldBeEncrypted { @@ -35,7 +34,7 @@ class DockerCleanupJob implements ShouldQueue, ShouldBeEncrypted } }); if ($isInprogress) { - throw new Exception('DockerCleanupJob: ApplicationDeploymentQueue is not empty, skipping...'); + throw new RuntimeException('DockerCleanupJob: ApplicationDeploymentQueue is not empty, skipping...'); } if (!$this->server->isFunctional()) { return; diff --git a/app/Jobs/MultipleApplicationDeploymentJob.php b/app/Jobs/MultipleApplicationDeploymentJob.php deleted file mode 100644 index 32c98d3b0..000000000 --- a/app/Jobs/MultipleApplicationDeploymentJob.php +++ /dev/null @@ -1,1165 +0,0 @@ -clearScreen(); - $this->application_deployment_queue = ApplicationDeploymentQueue::find($application_deployment_queue_id); - $this->log_model = $this->application_deployment_queue; - $this->application = Application::find($this->application_deployment_queue->application_id); - $this->build_pack = data_get($this->application, 'build_pack'); - - $this->application_deployment_queue_id = $application_deployment_queue_id; - $this->deployment_uuid = $this->application_deployment_queue->deployment_uuid; - $this->commit = $this->application_deployment_queue->commit; - $this->force_rebuild = $this->application_deployment_queue->force_rebuild; - $this->restart_only = $this->application_deployment_queue->restart_only; - - $this->git_type = data_get($this->application_deployment_queue, 'git_type'); - - $source = data_get($this->application, 'source'); - if ($source) { - $this->source = $source->getMorphClass()::where('id', $this->application->source->id)->first(); - } - $this->destination = $this->application->destination->getMorphClass()::where('id', $this->application->destination->id)->first(); - $this->server = $this->mainServer = $this->destination->server; - $this->serverUser = $this->server->user; - $this->basedir = generateBaseDir($this->deployment_uuid); - $this->workdir = "{$this->basedir}" . rtrim($this->application->base_directory, '/'); - $this->configuration_dir = application_configuration_dir() . "/{$this->application->uuid}"; - $this->is_debug_enabled = $this->application->settings->is_debug_enabled; - $this->saved_outputs = collect(); - $this->container_name = generateApplicationContainerName($this->application, 0); - } - - public function handle(): void - { - savePrivateKeyToFs($this->server); - $this->application_deployment_queue->update([ - 'status' => ApplicationDeploymentStatus::IN_PROGRESS->value, - ]); - - $this->addHosts = generateHostIpMapping($this->server, $this->destination->network); - - if ($this->application->dockerfile_target_build) { - $this->buildTarget = " --target {$this->application->dockerfile_target_build} "; - } - - // Check custom port - preg_match('/(?<=:)\d+(?=\/)/', $this->application->git_repository, $matches); - if (count($matches) === 1) { - $this->customPort = $matches[0]; - $gitHost = str($this->application->git_repository)->before(':'); - $gitRepo = str($this->application->git_repository)->after('/'); - $this->customRepository = "$gitHost:$gitRepo"; - } else { - $this->customRepository = $this->application->git_repository; - } - try { - if ($this->application->isMultipleServerDeployment()) { - if ($this->application->build_pack === 'dockerimage') { - $this->dockerImage = $this->application->docker_registry_image_name; - $this->dockerImageTag = $this->application->docker_registry_image_tag; - $this->execute_remote_command( - [ - "echo 'Starting deployment of {$this->dockerImage}:{$this->dockerImageTag}.'" - ], - ); - $this->production_image_name = Str::lower("{$this->dockerImage}:{$this->dockerImageTag}"); - ray(prepareHelperContainer($this->server, $this->deployment_uuid)); - $this->execute_remote_command( - [prepareHelperContainer($this->server, $this->deployment_uuid)] - ); - } - } else { - throw new RuntimeException('Missing configuration for multiple server deployment.'); - } - // if ($this->restart_only && $this->application->build_pack !== 'dockerimage') { - // $this->just_restart(); - // if ($this->server->isProxyShouldRun()) { - // dispatch(new ContainerStatusJob($this->server)); - // } - // $this->next(ApplicationDeploymentStatus::FINISHED->value); - // $this->application->isConfigurationChanged(true); - // return; - // } else if ($this->application->dockerfile) { - // $this->deploy_simple_dockerfile(); - // } else if ($this->application->build_pack === 'dockerimage') { - // $this->deploy_dockerimage_buildpack(); - // } else if ($this->application->build_pack === 'dockerfile') { - // $this->deploy_dockerfile_buildpack(); - // } else if ($this->application->build_pack === 'static') { - // $this->deploy_static_buildpack(); - // } else { - // $this->deploy_nixpacks_buildpack(); - // } - // if ($this->server->isProxyShouldRun()) { - // dispatch(new ContainerStatusJob($this->server)); - // } - // if ($this->application->docker_registry_image_name) { - // $this->push_to_docker_registry(); - // } - // $this->next(ApplicationDeploymentStatus::FINISHED->value); - // $this->application->isConfigurationChanged(true); - } catch (Exception $e) { - $this->fail($e); - throw $e; - } finally { - // if (isset($this->docker_compose_base64)) { - // $readme = generate_readme_file($this->application->name, $this->application_deployment_queue->updated_at); - // $composeFileName = "$this->configuration_dir/docker-compose.yml"; - // $this->execute_remote_command( - // [ - // "mkdir -p $this->configuration_dir" - // ], - // [ - // "echo '{$this->docker_compose_base64}' | base64 -d > $composeFileName", - // ], - // [ - // "echo '{$readme}' > $this->configuration_dir/README.md", - // ] - // ); - // } - // $this->execute_remote_command( - // [ - // "docker rm -f {$this->deployment_uuid} >/dev/null 2>&1", - // "hidden" => true, - // "ignore_errors" => true, - // ] - // ); - // $this->execute_remote_command( - // [ - // "docker image prune -f >/dev/null 2>&1", - // "hidden" => true, - // "ignore_errors" => true, - // ] - // ); - } - } - private function push_to_docker_registry() - { - try { - instant_remote_process(["docker images --format '{{json .}}' {$this->production_image_name}"], $this->server); - $this->execute_remote_command( - [ - "echo '\n----------------------------------------'", - ], - ["echo -n 'Pushing image to docker registry ({$this->production_image_name}).'"], - [ - executeInDocker($this->deployment_uuid, "docker push {$this->production_image_name}"), 'hidden' => true - ], - ); - if ($this->application->docker_registry_image_tag) { - // Tag image with latest - $this->execute_remote_command( - ['echo -n "Tagging and pushing image with latest tag."'], - [ - executeInDocker($this->deployment_uuid, "docker tag {$this->production_image_name} {$this->application->docker_registry_image_name}:{$this->application->docker_registry_image_tag}"), 'ignore_errors' => true, 'hidden' => true - ], - [ - executeInDocker($this->deployment_uuid, "docker push {$this->application->docker_registry_image_name}:{$this->application->docker_registry_image_tag}"), 'ignore_errors' => true, 'hidden' => true - ], - ); - } - $this->execute_remote_command([ - "echo -n 'Image pushed to docker registry.'" - ]); - } catch (Exception $e) { - $this->execute_remote_command( - ["echo -n 'Failed to push image to docker registry. Please check debug logs for more information.'"], - ); - ray($e); - } - } - // private function deploy_docker_compose() - // { - // $dockercompose_base64 = base64_encode($this->application->dockercompose); - // $this->execute_remote_command( - // [ - // "echo 'Starting deployment of {$this->application->name}.'" - // ], - // ); - // $this->prepare_builder_image(); - // $this->execute_remote_command( - // [ - // executeInDocker($this->deployment_uuid, "echo '$dockercompose_base64' | base64 -d > $this->workdir/docker-compose.yaml") - // ], - // ); - // $this->build_image_name = Str::lower("{$this->customRepository}:build"); - // $this->production_image_name = Str::lower("{$this->application->uuid}:latest"); - // $this->save_environment_variables(); - // $containers = getCurrentApplicationContainerStatus($this->application->destination->server, $this->application->id); - // ray($containers); - // if ($containers->count() > 0) { - // foreach ($containers as $container) { - // $containerName = data_get($container, 'Names'); - // if ($containerName) { - // instant_remote_process( - // ["docker rm -f {$containerName}"], - // $this->application->destination->server - // ); - // } - // } - // } - - // $this->execute_remote_command( - // ["echo -n 'Starting services (could take a while)...'"], - // [executeInDocker($this->deployment_uuid, "docker compose --project-directory {$this->workdir} up -d"), "hidden" => true], - // ); - // } - private function generate_image_names() - { - if ($this->application->dockerfile) { - if ($this->application->docker_registry_image_name) { - $this->build_image_name = Str::lower("{$this->application->docker_registry_image_name}:build"); - $this->production_image_name = Str::lower("{$this->application->docker_registry_image_name}:latest"); - } else { - $this->build_image_name = Str::lower("{$this->application->uuid}:build"); - $this->production_image_name = Str::lower("{$this->application->uuid}:latest"); - } - } else if ($this->application->build_pack === 'dockerimage') { - $this->production_image_name = Str::lower("{$this->dockerImage}:{$this->dockerImageTag}"); - } else { - $this->dockerImageTag = str($this->commit)->substr(0, 128); - if ($this->application->docker_registry_image_name) { - $this->build_image_name = Str::lower("{$this->application->docker_registry_image_name}:{$this->dockerImageTag}-build"); - $this->production_image_name = Str::lower("{$this->application->docker_registry_image_name}:{$this->dockerImageTag}"); - } else { - $this->build_image_name = Str::lower("{$this->application->uuid}:{$this->dockerImageTag}-build"); - $this->production_image_name = Str::lower("{$this->application->uuid}:{$this->dockerImageTag}"); - } - } - } - private function just_restart() - { - $this->execute_remote_command( - [ - "echo 'Starting deployment of {$this->customRepository}:{$this->application->git_branch}.'" - ], - ); - $this->prepare_builder_image(); - $this->check_git_if_build_needed(); - $this->set_base_dir(); - $this->generate_image_names(); - $this->check_image_locally_or_remotely(); - if (str($this->saved_outputs->get('local_image_found'))->isNotEmpty()) { - $this->generate_compose_file(); - $this->rolling_update(); - return; - } - throw new RuntimeException('Cannot find image anywhere. Please redeploy the application.'); - } - private function check_image_locally_or_remotely() - { - $this->execute_remote_command([ - "docker images -q {$this->production_image_name} 2>/dev/null", "hidden" => true, "save" => "local_image_found" - ]); - if (str($this->saved_outputs->get('local_image_found'))->isEmpty() && $this->application->docker_registry_image_name) { - $this->execute_remote_command([ - "docker pull {$this->production_image_name} 2>/dev/null", "ignore_errors" => true, "hidden" => true - ]); - $this->execute_remote_command([ - "docker images -q {$this->production_image_name} 2>/dev/null", "hidden" => true, "save" => "local_image_found" - ]); - } - } - // private function save_environment_variables() - // { - // $envs = collect([]); - // foreach ($this->application->environment_variables as $env) { - // $envs->push($env->key . '=' . $env->value); - // } - // $envs_base64 = base64_encode($envs->implode("\n")); - // $this->execute_remote_command( - // [ - // executeInDocker($this->deployment_uuid, "echo '$envs_base64' | base64 -d > $this->workdir/.env") - // ], - // ); - // } - private function deploy_simple_dockerfile() - { - $dockerfile_base64 = base64_encode($this->application->dockerfile); - $this->execute_remote_command( - [ - "echo 'Starting deployment of {$this->application->name}.'" - ], - ); - $this->prepare_builder_image(); - $this->execute_remote_command( - [ - executeInDocker($this->deployment_uuid, "echo '$dockerfile_base64' | base64 -d > $this->workdir$this->dockerfile_location") - ], - ); - $this->generate_image_names(); - $this->generate_compose_file(); - $this->generate_build_env_variables(); - $this->add_build_env_variables_to_dockerfile(); - $this->build_image(); - $this->rolling_update(); - } - - private function deploy_dockerimage_buildpack() - { - // $this->dockerImage = $this->application->docker_registry_image_name; - // $this->dockerImageTag = $this->application->docker_registry_image_tag; - // ray("echo 'Starting deployment of {$this->dockerImage}:{$this->dockerImageTag}.'"); - // $this->execute_remote_command( - // [ - // "echo 'Starting deployment of {$this->dockerImage}:{$this->dockerImageTag}.'" - // ], - // ); - // $this->generate_image_names(); - // $this->prepare_builder_image(); - $this->generate_compose_file(); - $this->rolling_update(); - } - - private function deploy_dockerfile_buildpack() - { - if (data_get($this->application, 'dockerfile_location')) { - $this->dockerfile_location = $this->application->dockerfile_location; - } - $this->execute_remote_command( - [ - "echo 'Starting deployment of {$this->customRepository}:{$this->application->git_branch}.'" - ], - ); - $this->prepare_builder_image(); - $this->check_git_if_build_needed(); - $this->clone_repository(); - $this->set_base_dir(); - $this->generate_image_names(); - $this->cleanup_git(); - $this->generate_compose_file(); - $this->generate_build_env_variables(); - $this->add_build_env_variables_to_dockerfile(); - $this->build_image(); - // if ($this->application->additional_destinations) { - // $this->push_to_docker_registry(); - // $this->deploy_to_additional_destinations(); - // } else { - $this->rolling_update(); - // } - } - private function deploy_nixpacks_buildpack() - { - $this->execute_remote_command( - [ - "echo 'Starting deployment of {$this->customRepository}:{$this->application->git_branch}.'" - ], - ); - $this->prepare_builder_image(); - $this->check_git_if_build_needed(); - $this->set_base_dir(); - $this->generate_image_names(); - if (!$this->force_rebuild) { - $this->check_image_locally_or_remotely(); - if (str($this->saved_outputs->get('local_image_found'))->isNotEmpty() && !$this->application->isConfigurationChanged()) { - $this->execute_remote_command([ - "echo 'No configuration changed & image found ({$this->production_image_name}) with the same Git Commit SHA. Build step skipped.'", - ]); - $this->generate_compose_file(); - $this->rolling_update(); - return; - } - if ($this->application->isConfigurationChanged()) { - $this->execute_remote_command([ - "echo 'Configuration changed. Rebuilding image.'", - ]); - } - } - $this->clone_repository(); - $this->cleanup_git(); - $this->generate_nixpacks_confs(); - $this->generate_compose_file(); - $this->generate_build_env_variables(); - $this->add_build_env_variables_to_dockerfile(); - $this->build_image(); - $this->rolling_update(); - } - private function deploy_static_buildpack() - { - $this->execute_remote_command( - [ - "echo 'Starting deployment of {$this->customRepository}:{$this->application->git_branch}.'" - ], - ); - $this->prepare_builder_image(); - $this->check_git_if_build_needed(); - $this->set_base_dir(); - $this->generate_image_names(); - $this->clone_repository(); - $this->cleanup_git(); - $this->build_image(); - $this->generate_compose_file(); - $this->rolling_update(); - } - - private function rolling_update() - { - if (count($this->application->ports_mappings_array) > 0) { - $this->execute_remote_command( - [ - "echo '\n----------------------------------------'", - ], - ["echo -n 'Application has ports mapped to the host system, rolling update is not supported.'"], - ); - $this->stop_running_container(force: true); - $this->start_by_compose_file(); - } else { - $this->execute_remote_command( - [ - "echo '\n----------------------------------------'", - ], - ["echo -n 'Rolling update started.'"], - ); - $this->start_by_compose_file(); - $this->health_check(); - $this->stop_running_container(); - } - } - private function health_check() - { - if ($this->application->isHealthcheckDisabled()) { - $this->newVersionIsHealthy = true; - return; - } - // ray('New container name: ', $this->container_name); - if ($this->container_name) { - $counter = 1; - $this->execute_remote_command( - [ - "echo 'Waiting for healthcheck to pass on the new container.'" - ] - ); - if ($this->full_healthcheck_url) { - $this->execute_remote_command( - [ - "echo 'Healthcheck URL (inside the container): {$this->full_healthcheck_url}'" - ] - ); - } - while ($counter < $this->application->health_check_retries) { - $this->execute_remote_command( - [ - "docker inspect --format='{{json .State.Health.Status}}' {$this->container_name}", - "hidden" => true, - "save" => "health_check" - ], - - ); - $this->execute_remote_command( - [ - "echo 'Attempt {$counter} of {$this->application->health_check_retries} | Healthcheck status: {$this->saved_outputs->get('health_check')}'" - ], - ); - if (Str::of($this->saved_outputs->get('health_check'))->contains('healthy')) { - $this->newVersionIsHealthy = true; - $this->application->update(['status' => 'running']); - $this->execute_remote_command( - [ - "echo 'New container is healthy.'" - ], - ); - break; - } - $counter++; - sleep($this->application->health_check_interval); - } - } - } - - private function prepare_builder_image() - { - $helperImage = config('coolify.helper_image'); - // Get user home directory - $this->serverUserHomeDir = instant_remote_process(["echo \$HOME"], $this->server); - $this->dockerConfigFileExists = instant_remote_process(["test -f {$this->serverUserHomeDir}/.docker/config.json && echo 'OK' || echo 'NOK'"], $this->server); - - if ($this->dockerConfigFileExists === 'OK') { - $runCommand = "docker run -d --network {$this->destination->network} -v /:/host --name {$this->deployment_uuid} --rm -v {$this->serverUserHomeDir}/.docker/config.json:/root/.docker/config.json:ro -v /var/run/docker.sock:/var/run/docker.sock {$helperImage}"; - } else { - $runCommand = "docker run -d --network {$this->destination->network} -v /:/host --name {$this->deployment_uuid} --rm -v /var/run/docker.sock:/var/run/docker.sock {$helperImage}"; - } - $this->execute_remote_command( - [ - "echo -n 'Preparing container with helper image: $helperImage.'", - ], - [ - $runCommand, - "hidden" => true, - ], - [ - "command" => executeInDocker($this->deployment_uuid, "mkdir -p {$this->basedir}") - ], - ); - } - private function deploy_to_additional_destinations() - { - $destination_ids = collect(str($this->application->additional_destinations)->explode(',')); - foreach ($destination_ids as $destination_id) { - $destination = StandaloneDocker::find($destination_id); - $server = $destination->server; - if ($server->team_id !== $this->mainServer->team_id) { - $this->execute_remote_command( - [ - "echo -n 'Skipping deployment to {$server->name}. Not in the same team?!'", - ], - ); - continue; - } - $this->server = $server; - $this->execute_remote_command( - [ - "echo -n 'Deploying to {$this->server->name}.'", - ], - ); - $this->prepare_builder_image(); - $this->generate_image_names(); - $this->rolling_update(); - } - } - private function set_base_dir() - { - $this->execute_remote_command( - [ - "echo -n 'Setting base directory to {$this->workdir}.'" - ], - ); - } - private function check_git_if_build_needed() - { - $this->generate_git_import_commands(); - $private_key = data_get($this->application, 'private_key.private_key'); - if ($private_key) { - $private_key = base64_encode($private_key); - $this->execute_remote_command( - [ - executeInDocker($this->deployment_uuid, "mkdir -p /root/.ssh") - ], - [ - executeInDocker($this->deployment_uuid, "echo '{$private_key}' | base64 -d > /root/.ssh/id_rsa") - ], - [ - executeInDocker($this->deployment_uuid, "chmod 600 /root/.ssh/id_rsa") - ], - [ - executeInDocker($this->deployment_uuid, "GIT_SSH_COMMAND=\"ssh -o ConnectTimeout=30 -p {$this->customPort} -o Port={$this->customPort} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /root/.ssh/id_rsa\" git ls-remote {$this->fullRepoUrl} {$this->branch}"), - "hidden" => true, - "save" => "git_commit_sha" - ], - ); - } else { - $this->execute_remote_command( - [ - executeInDocker($this->deployment_uuid, "GIT_SSH_COMMAND=\"ssh -o ConnectTimeout=30 -p {$this->customPort} -o Port={$this->customPort} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null\" git ls-remote {$this->fullRepoUrl} {$this->branch}"), - "hidden" => true, - "save" => "git_commit_sha" - ], - ); - } - - if ($this->saved_outputs->get('git_commit_sha')) { - $this->commit = $this->saved_outputs->get('git_commit_sha')->before("\t"); - } - } - private function clone_repository() - { - $importCommands = $this->generate_git_import_commands(); - $this->execute_remote_command( - [ - "echo '\n----------------------------------------'", - ], - [ - "echo -n 'Importing {$this->customRepository}:{$this->application->git_branch} (commit sha {$this->application->git_commit_sha}) to {$this->basedir}. '" - ], - [ - $importCommands, "hidden" => true - ] - ); - } - - private function generate_git_import_commands() - { - $this->branch = $this->application->git_branch; - $commands = collect([]); - $git_clone_command = "git clone -q -b {$this->application->git_branch}"; - - if ($this->application->deploymentType() === 'source') { - $source_html_url = data_get($this->application, 'source.html_url'); - $url = parse_url(filter_var($source_html_url, FILTER_SANITIZE_URL)); - $source_html_url_host = $url['host']; - $source_html_url_scheme = $url['scheme']; - - if ($this->source->getMorphClass() == 'App\Models\GithubApp') { - if ($this->source->is_public) { - $this->fullRepoUrl = "{$this->source->html_url}/{$this->customRepository}"; - $git_clone_command = "{$git_clone_command} {$this->source->html_url}/{$this->customRepository} {$this->basedir}"; - $git_clone_command = $this->set_git_import_settings($git_clone_command); - - $commands->push(executeInDocker($this->deployment_uuid, $git_clone_command)); - } else { - $github_access_token = generate_github_installation_token($this->source); - $commands->push(executeInDocker($this->deployment_uuid, "git clone -q -b {$this->application->git_branch} $source_html_url_scheme://x-access-token:$github_access_token@$source_html_url_host/{$this->customRepository}.git {$this->basedir}")); - $this->fullRepoUrl = "$source_html_url_scheme://x-access-token:$github_access_token@$source_html_url_host/{$this->customRepository}.git"; - } - return $commands->implode(' && '); - } - } - if ($this->application->deploymentType() === 'deploy_key') { - $this->fullRepoUrl = $this->customRepository; - $private_key = data_get($this->application, 'private_key.private_key'); - if (is_null($private_key)) { - throw new RuntimeException('Private key not found. Please add a private key to the application and try again.'); - } - $private_key = base64_encode($private_key); - $git_clone_command_base = "GIT_SSH_COMMAND=\"ssh -o ConnectTimeout=30 -p {$this->customPort} -o Port={$this->customPort} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /root/.ssh/id_rsa\" {$git_clone_command} {$this->customRepository} {$this->basedir}"; - $git_clone_command = $this->set_git_import_settings($git_clone_command_base); - $commands = collect([ - executeInDocker($this->deployment_uuid, "mkdir -p /root/.ssh"), - executeInDocker($this->deployment_uuid, "echo '{$private_key}' | base64 -d > /root/.ssh/id_rsa"), - executeInDocker($this->deployment_uuid, "chmod 600 /root/.ssh/id_rsa"), - ]); - - $commands->push(executeInDocker($this->deployment_uuid, $git_clone_command)); - return $commands->implode(' && '); - } - if ($this->application->deploymentType() === 'other') { - $this->fullRepoUrl = $this->customRepository; - $git_clone_command = "{$git_clone_command} {$this->customRepository} {$this->basedir}"; - $git_clone_command = $this->set_git_import_settings($git_clone_command); - $commands->push(executeInDocker($this->deployment_uuid, $git_clone_command)); - return $commands->implode(' && '); - } - } - - private function set_git_import_settings($git_clone_command) - { - if ($this->application->git_commit_sha !== 'HEAD') { - $git_clone_command = "{$git_clone_command} && cd {$this->basedir} && git -c advice.detachedHead=false checkout {$this->application->git_commit_sha} >/dev/null 2>&1"; - } - if ($this->application->settings->is_git_submodules_enabled) { - $git_clone_command = "{$git_clone_command} && cd {$this->basedir} && git submodule update --init --recursive"; - } - if ($this->application->settings->is_git_lfs_enabled) { - $git_clone_command = "{$git_clone_command} && cd {$this->basedir} && git lfs pull"; - } - return $git_clone_command; - } - - private function cleanup_git() - { - $this->execute_remote_command( - [executeInDocker($this->deployment_uuid, "rm -fr {$this->basedir}/.git")], - ); - } - - private function generate_nixpacks_confs() - { - $nixpacks_command = $this->nixpacks_build_cmd(); - $this->execute_remote_command( - [ - "echo -n 'Generating nixpacks configuration with: $nixpacks_command'", - ], - [executeInDocker($this->deployment_uuid, $nixpacks_command)], - [executeInDocker($this->deployment_uuid, "cp {$this->workdir}/.nixpacks/Dockerfile {$this->workdir}/Dockerfile")], - [executeInDocker($this->deployment_uuid, "rm -f {$this->workdir}/.nixpacks/Dockerfile")] - ); - } - - private function nixpacks_build_cmd() - { - $this->generate_env_variables(); - $nixpacks_command = "nixpacks build --cache-key '{$this->application->uuid}' -o {$this->workdir} {$this->env_args} --no-error-without-start"; - if ($this->application->build_command) { - $nixpacks_command .= " --build-cmd \"{$this->application->build_command}\""; - } - if ($this->application->start_command) { - $nixpacks_command .= " --start-cmd \"{$this->application->start_command}\""; - } - if ($this->application->install_command) { - $nixpacks_command .= " --install-cmd \"{$this->application->install_command}\""; - } - $nixpacks_command .= " {$this->workdir}"; - return $nixpacks_command; - } - - private function generate_env_variables() - { - $this->env_args = collect([]); - foreach ($this->application->nixpacks_environment_variables_preview as $env) { - $this->env_args->push("--env {$env->key}={$env->value}"); - } - $this->env_args = $this->env_args->implode(' '); - } - - private function generate_compose_file() - { - $ports = $this->application->settings->is_static ? [80] : $this->application->ports_exposes_array; - - $persistent_storages = $this->generate_local_persistent_volumes(); - $volume_names = $this->generate_local_persistent_volumes_only_volume_names(); - $environment_variables = $this->generate_environment_variables($ports); - - if (data_get($this->application, 'custom_labels')) { - $labels = collect(str($this->application->custom_labels)->explode(',')); - $labels = $labels->filter(function ($value, $key) { - return !Str::startsWith($value, 'coolify.'); - }); - $this->application->custom_labels = $labels->implode(','); - $this->application->save(); - } else { - $labels = collect(generateLabelsApplication($this->application, $this->preview)); - } - - $labels = $labels->merge(defaultLabels($this->application->id, $this->application->uuid, 0))->toArray(); - $docker_compose = [ - 'version' => '3.8', - 'services' => [ - $this->container_name => [ - 'image' => $this->production_image_name, - 'container_name' => $this->container_name, - 'restart' => RESTART_MODE, - 'environment' => $environment_variables, - 'labels' => $labels, - 'expose' => $ports, - 'networks' => [ - $this->destination->network, - ], - 'healthcheck' => [ - 'test' => [ - 'CMD-SHELL', - $this->generate_healthcheck_commands() - ], - 'interval' => $this->application->health_check_interval . 's', - 'timeout' => $this->application->health_check_timeout . 's', - 'retries' => $this->application->health_check_retries, - 'start_period' => $this->application->health_check_start_period . 's' - ], - 'mem_limit' => $this->application->limits_memory, - 'memswap_limit' => $this->application->limits_memory_swap, - 'mem_swappiness' => $this->application->limits_memory_swappiness, - 'mem_reservation' => $this->application->limits_memory_reservation, - 'cpus' => (int) $this->application->limits_cpus, - 'cpuset' => $this->application->limits_cpuset, - 'cpu_shares' => $this->application->limits_cpu_shares, - ] - ], - 'networks' => [ - $this->destination->network => [ - 'external' => true, - 'name' => $this->destination->network, - 'attachable' => true - ] - ] - ]; - if ($this->server->isLogDrainEnabled() && $this->application->isLogDrainEnabled()) { - $docker_compose['services'][$this->container_name]['logging'] = [ - 'driver' => 'fluentd', - 'options' => [ - 'fluentd-address' => "tcp://127.0.0.1:24224", - 'fluentd-async' => "true", - 'fluentd-sub-second-precision' => "true", - ] - ]; - } - if ($this->application->settings->is_gpu_enabled) { - ray('asd'); - $docker_compose['services'][$this->container_name]['deploy']['resources']['reservations']['devices'] = [ - [ - 'driver' => data_get($this->application, 'settings.gpu_driver', 'nvidia'), - 'capabilities' => ['gpu'], - 'options' => data_get($this->application, 'settings.gpu_options', []) - ] - ]; - if (data_get($this->application, 'settings.gpu_count')) { - $count = data_get($this->application, 'settings.gpu_count'); - if ($count === 'all') { - $docker_compose['services'][$this->container_name]['deploy']['resources']['reservations']['devices'][0]['count'] = $count; - } else { - $docker_compose['services'][$this->container_name]['deploy']['resources']['reservations']['devices'][0]['count'] = (int) $count; - } - } else if (data_get($this->application, 'settings.gpu_device_ids')) { - $docker_compose['services'][$this->container_name]['deploy']['resources']['reservations']['devices'][0]['ids'] = data_get($this->application, 'settings.gpu_device_ids'); - } - } - if ($this->application->isHealthcheckDisabled()) { - data_forget($docker_compose, 'services.' . $this->container_name . '.healthcheck'); - } - if (count($persistent_storages) > 0) { - $docker_compose['services'][$this->container_name]['volumes'] = $persistent_storages; - } - if (count($volume_names) > 0) { - $docker_compose['volumes'] = $volume_names; - } - // if ($this->build_pack === 'dockerfile') { - // $docker_compose['services'][$this->container_name]['build'] = [ - // 'context' => $this->workdir, - // 'dockerfile' => $this->workdir . $this->dockerfile_location, - // ]; - // } - $this->docker_compose = Yaml::dump($docker_compose, 10); - $this->docker_compose_base64 = base64_encode($this->docker_compose); - $this->execute_remote_command([executeInDocker($this->deployment_uuid, "echo '{$this->docker_compose_base64}' | base64 -d > {$this->workdir}/docker-compose.yml"), "hidden" => true]); - } - - private function generate_local_persistent_volumes() - { - $local_persistent_volumes = []; - foreach ($this->application->persistentStorages as $persistentStorage) { - $volume_name = $persistentStorage->host_path ?? $persistentStorage->name; - $local_persistent_volumes[] = $volume_name . ':' . $persistentStorage->mount_path; - } - return $local_persistent_volumes; - } - - private function generate_local_persistent_volumes_only_volume_names() - { - $local_persistent_volumes_names = []; - foreach ($this->application->persistentStorages as $persistentStorage) { - if ($persistentStorage->host_path) { - continue; - } - $name = $persistentStorage->name; - $local_persistent_volumes_names[$name] = [ - 'name' => $name, - 'external' => false, - ]; - } - return $local_persistent_volumes_names; - } - - private function generate_environment_variables($ports) - { - $environment_variables = collect(); - foreach ($this->application->runtime_environment_variables_preview as $env) { - $environment_variables->push("$env->key=$env->value"); - } - foreach ($this->application->nixpacks_environment_variables_preview as $env) { - $environment_variables->push("$env->key=$env->value"); - } - // Add PORT if not exists, use the first port as default - if ($environment_variables->filter(fn ($env) => Str::of($env)->contains('PORT'))->isEmpty()) { - $environment_variables->push("PORT={$ports[0]}"); - } - return $environment_variables->all(); - } - - private function generate_healthcheck_commands() - { - if ($this->application->dockerfile || $this->application->build_pack === 'dockerfile' || $this->application->build_pack === 'dockerimage') { - // TODO: disabled HC because there are several ways to hc a simple docker image, hard to figure out a good way. Like some docker images (pocketbase) does not have curl. - return 'exit 0'; - } - if (!$this->application->health_check_port) { - $health_check_port = $this->application->ports_exposes_array[0]; - } else { - $health_check_port = $this->application->health_check_port; - } - if ($this->application->health_check_path) { - $this->full_healthcheck_url = "{$this->application->health_check_method}: {$this->application->health_check_scheme}://{$this->application->health_check_host}:{$health_check_port}{$this->application->health_check_path}"; - $generated_healthchecks_commands = [ - "curl -s -X {$this->application->health_check_method} -f {$this->application->health_check_scheme}://{$this->application->health_check_host}:{$health_check_port}{$this->application->health_check_path} > /dev/null" - ]; - } else { - $this->full_healthcheck_url = "{$this->application->health_check_method}: {$this->application->health_check_scheme}://{$this->application->health_check_host}:{$health_check_port}/"; - $generated_healthchecks_commands = [ - "curl -s -X {$this->application->health_check_method} -f {$this->application->health_check_scheme}://{$this->application->health_check_host}:{$health_check_port}/" - ]; - } - return implode(' ', $generated_healthchecks_commands); - } - private function pull_latest_image($image) - { - $this->execute_remote_command( - ["echo -n 'Pulling latest image ($image) from the registry.'"], - - [ - executeInDocker($this->deployment_uuid, "docker pull {$image}"), "hidden" => true - ] - ); - } - private function build_image() - { - if ($this->application->build_pack === 'static') { - $this->execute_remote_command([ - "echo -n 'Static deployment. Copying static assets to the image.'", - ]); - } else { - $this->execute_remote_command( - [ - "echo -n 'Building docker image started.'", - ], - ["echo -n 'To check the current progress, click on Show Debug Logs.'"] - ); - } - - if ($this->application->settings->is_static || $this->application->build_pack === 'static') { - if ($this->application->static_image) { - $this->pull_latest_image($this->application->static_image); - } - if ($this->application->build_pack === 'static') { - $dockerfile = base64_encode("FROM {$this->application->static_image} -WORKDIR /usr/share/nginx/html/ -LABEL coolify.deploymentId={$this->deployment_uuid} -COPY . . -RUN rm -f /usr/share/nginx/html/nginx.conf -RUN rm -f /usr/share/nginx/html/Dockerfile -COPY ./nginx.conf /etc/nginx/conf.d/default.conf"); - $nginx_config = base64_encode("server { - listen 80; - listen [::]:80; - server_name localhost; - - location / { - root /usr/share/nginx/html; - index index.html; - try_files \$uri \$uri.html \$uri/index.html \$uri/ /index.html =404; - } - - error_page 500 502 503 504 /50x.html; - location = /50x.html { - root /usr/share/nginx/html; - } - }"); - } else { - $this->execute_remote_command([ - executeInDocker($this->deployment_uuid, "docker build $this->buildTarget $this->addHosts --network host -f {$this->workdir}{$this->dockerfile_location} {$this->build_args} --progress plain -t $this->build_image_name {$this->workdir}"), "hidden" => true - ]); - - $dockerfile = base64_encode("FROM {$this->application->static_image} -WORKDIR /usr/share/nginx/html/ -LABEL coolify.deploymentId={$this->deployment_uuid} -COPY --from=$this->build_image_name /app/{$this->application->publish_directory} . -COPY ./nginx.conf /etc/nginx/conf.d/default.conf"); - - $nginx_config = base64_encode("server { - listen 80; - listen [::]:80; - server_name localhost; - - location / { - root /usr/share/nginx/html; - index index.html; - try_files \$uri \$uri.html \$uri/index.html \$uri/ /index.html =404; - } - - error_page 500 502 503 504 /50x.html; - location = /50x.html { - root /usr/share/nginx/html; - } - }"); - } - $this->execute_remote_command( - [ - executeInDocker($this->deployment_uuid, "echo '{$dockerfile}' | base64 -d > {$this->workdir}/Dockerfile") - ], - [ - executeInDocker($this->deployment_uuid, "echo '{$nginx_config}' | base64 -d > {$this->workdir}/nginx.conf") - ], - [ - executeInDocker($this->deployment_uuid, "docker build $this->addHosts --network host -f {$this->workdir}/Dockerfile {$this->build_args} --progress plain -t $this->production_image_name {$this->workdir}"), "hidden" => true - ] - ); - } else { - // Pure Dockerfile based deployment - if ($this->application->dockerfile) { - $this->execute_remote_command([ - executeInDocker($this->deployment_uuid, "docker build --pull $this->buildTarget $this->addHosts --network host -f {$this->workdir}{$this->dockerfile_location} {$this->build_args} --progress plain -t $this->production_image_name {$this->workdir}"), "hidden" => true - ]); - } else { - $this->execute_remote_command([ - executeInDocker($this->deployment_uuid, "docker build $this->buildTarget $this->addHosts --network host -f {$this->workdir}{$this->dockerfile_location} {$this->build_args} --progress plain -t $this->production_image_name {$this->workdir}"), "hidden" => true - ]); - } - } - $this->execute_remote_command([ - "echo -n 'Building docker image completed.'", - ]); - } - - private function stop_running_container(bool $force = false) - { - $this->execute_remote_command(["echo -n 'Removing old container.'"]); - if ($this->newVersionIsHealthy || $force) { - $containers = getCurrentApplicationContainerStatus($this->server, $this->application->id, 0); - $containers = $containers->filter(function ($container) { - return data_get($container, 'Names') !== $this->container_name; - }); - $containers->each(function ($container) { - $containerName = data_get($container, 'Names'); - $this->execute_remote_command( - [executeInDocker($this->deployment_uuid, "docker rm -f $containerName >/dev/null 2>&1"), "hidden" => true, "ignore_errors" => true], - ); - }); - $this->execute_remote_command( - [ - "echo 'Rolling update completed.'" - ], - ); - } else { - $this->execute_remote_command( - ["echo -n 'New container is not healthy, rolling back to the old container.'"], - [executeInDocker($this->deployment_uuid, "docker rm -f $this->container_name >/dev/null 2>&1"), "hidden" => true, "ignore_errors" => true], - ); - } - } - - private function start_by_compose_file() - { - if ($this->application->build_pack === 'dockerimage') { - $this->execute_remote_command( - ["echo -n 'Pulling latest images from the registry.'"], - [executeInDocker($this->deployment_uuid, "docker compose --project-directory {$this->workdir} pull"), "hidden" => true], - [executeInDocker($this->deployment_uuid, "docker compose --project-directory {$this->workdir} up --build -d"), "hidden" => true], - ); - } else { - $this->execute_remote_command( - [executeInDocker($this->deployment_uuid, "docker compose --project-directory {$this->workdir} up --build -d"), "hidden" => true], - ); - } - } - - private function generate_build_env_variables() - { - $this->build_args = collect(["--build-arg SOURCE_COMMIT=\"{$this->commit}\""]); - foreach ($this->application->build_environment_variables_preview as $env) { - $this->build_args->push("--build-arg {$env->key}=\"{$env->value}\""); - } - $this->build_args = $this->build_args->implode(' '); - } - - private function add_build_env_variables_to_dockerfile() - { - $this->execute_remote_command([ - executeInDocker($this->deployment_uuid, "cat {$this->workdir}{$this->dockerfile_location}"), "hidden" => true, "save" => 'dockerfile' - ]); - $dockerfile = collect(Str::of($this->saved_outputs->get('dockerfile'))->trim()->explode("\n")); - - foreach ($this->application->build_environment_variables as $env) { - $dockerfile->splice(1, 0, "ARG {$env->key}={$env->value}"); - } - $dockerfile_base64 = base64_encode($dockerfile->implode("\n")); - $this->execute_remote_command([ - executeInDocker($this->deployment_uuid, "echo '{$dockerfile_base64}' | base64 -d > {$this->workdir}{$this->dockerfile_location}"), - "hidden" => true - ]); - } - - private function next(string $status) - { - // If the deployment is cancelled by the user, don't update the status - if ($this->application_deployment_queue->status !== ApplicationDeploymentStatus::CANCELLED_BY_USER->value) { - $this->application_deployment_queue->update([ - 'status' => $status, - ]); - } - queue_next_deployment($this->application); - if ($status === ApplicationDeploymentStatus::FINISHED->value) { - $this->application->environment->project->team->notify(new DeploymentSuccess($this->application, $this->deployment_uuid, $this->preview)); - } - if ($status === ApplicationDeploymentStatus::FAILED->value) { - $this->application->environment->project->team->notify(new DeploymentFailed($this->application, $this->deployment_uuid, $this->preview)); - } - } - - public function failed(Throwable $exception): void - { - $this->execute_remote_command( - ["echo 'Oops something is not okay, are you okay? 😢'", 'type' => 'err'], - ["echo '{$exception->getMessage()}'", 'type' => 'err'], - ["echo -n 'Deployment failed. Removing the new version of your application.'", 'type' => 'err'], - [executeInDocker($this->deployment_uuid, "docker rm -f $this->container_name >/dev/null 2>&1"), "hidden" => true, "ignore_errors" => true] - ); - - $this->next(ApplicationDeploymentStatus::FAILED->value); - } -} diff --git a/app/Models/Application.php b/app/Models/Application.php index 785ef3040..ea7e0a930 100644 --- a/app/Models/Application.php +++ b/app/Models/Application.php @@ -6,6 +6,9 @@ use Illuminate\Database\Eloquent\Casts\Attribute; use Illuminate\Database\Eloquent\Relations\HasMany; use Spatie\Activitylog\Models\Activity; use Illuminate\Support\Str; +use RuntimeException; +use Symfony\Component\Yaml\Yaml; +use Visus\Cuid2\Cuid2; class Application extends BaseModel { @@ -45,7 +48,14 @@ class Application extends BaseModel $application->environment_variables_preview()->delete(); }); } - + public function link() + { + return route('project.application.configuration', [ + 'project_uuid' => $this->environment->project->uuid, + 'environment_name' => $this->environment->name, + 'application_uuid' => $this->uuid + ]); + } public function settings() { return $this->hasOne(ApplicationSetting::class); @@ -123,6 +133,36 @@ class Application extends BaseModel } ); } + public function dockerComposeLocation(): Attribute + { + return Attribute::make( + set: function ($value) { + if (is_null($value) || $value === '') { + return '/docker-compose.yaml'; + } else { + if ($value !== '/') { + return Str::start(Str::replaceEnd('/', '', $value), '/'); + } + return Str::start($value, '/'); + } + } + ); + } + public function dockerComposePrLocation(): Attribute + { + return Attribute::make( + set: function ($value) { + if (is_null($value) || $value === '') { + return '/docker-compose.yaml'; + } else { + if ($value !== '/') { + return Str::start(Str::replaceEnd('/', '', $value), '/'); + } + return Str::start($value, '/'); + } + } + ); + } public function baseDirectory(): Attribute { return Attribute::make( @@ -157,7 +197,16 @@ class Application extends BaseModel : explode(',', $this->ports_exposes) ); } - + public function serviceType() + { + $found = str(collect(SPECIFIC_SERVICES)->filter(function ($service) { + return str($this->image)->before(':')->value() === $service; + })->first()); + if ($found->isNotEmpty()) { + return $found; + } + return null; + } public function environment_variables(): HasMany { return $this->hasMany(EnvironmentVariable::class)->where('is_preview', false)->orderBy('key', 'asc'); @@ -224,7 +273,6 @@ class Application extends BaseModel { return $this->morphTo(); } - public function isDeploymentInprogress() { $deployments = ApplicationDeploymentQueue::where('application_id', $this->id)->where('status', 'in_progress')->count(); @@ -342,4 +390,289 @@ class Application extends BaseModel } return false; } + public function healthCheckUrl() + { + if ($this->dockerfile || $this->build_pack === 'dockerfile' || $this->build_pack === 'dockerimage') { + return null; + } + if (!$this->health_check_port) { + $health_check_port = $this->ports_exposes_array[0]; + } else { + $health_check_port = $this->health_check_port; + } + if ($this->health_check_path) { + $full_healthcheck_url = "{$this->health_check_scheme}://{$this->health_check_host}:{$health_check_port}{$this->health_check_path}"; + } else { + $full_healthcheck_url = "{$this->health_check_scheme}://{$this->health_check_host}:{$health_check_port}/"; + } + return $full_healthcheck_url; + } + function customRepository() + { + preg_match('/(?<=:)\d+(?=\/)/', $this->git_repository, $matches); + $port = 22; + if (count($matches) === 1) { + $port = $matches[0]; + $gitHost = str($this->git_repository)->before(':'); + $gitRepo = str($this->git_repository)->after('/'); + $repository = "$gitHost:$gitRepo"; + } else { + $repository = $this->git_repository; + } + return [ + 'repository' => $repository, + 'port' => $port + ]; + } + function generateBaseDir(string $uuid) + { + return "/artifacts/{$uuid}"; + } + function setGitImportSettings(string $deployment_uuid, string $git_clone_command) + { + $baseDir = $this->generateBaseDir($deployment_uuid); + if ($this->git_commit_sha !== 'HEAD') { + $git_clone_command = "{$git_clone_command} && cd {$baseDir} && git -c advice.detachedHead=false checkout {$this->git_commit_sha} >/dev/null 2>&1"; + } + if ($this->settings->is_git_submodules_enabled) { + $git_clone_command = "{$git_clone_command} && cd {$baseDir} && git submodule update --init --recursive"; + } + if ($this->settings->is_git_lfs_enabled) { + $git_clone_command = "{$git_clone_command} && cd {$baseDir} && git lfs pull"; + } + return $git_clone_command; + } + function generateGitImportCommands(string $deployment_uuid, int $pull_request_id = 0, ?string $git_type = null, bool $exec_in_docker = true, bool $only_checkout = false, ?string $custom_base_dir = null) + { + $branch = $this->git_branch; + ['repository' => $customRepository, 'port' => $customPort] = $this->customRepository(); + $baseDir = $custom_base_dir ?? $this->generateBaseDir($deployment_uuid); + $commands = collect([]); + $git_clone_command = "git clone -b {$this->git_branch}"; + if ($only_checkout) { + $git_clone_command = "git clone --no-checkout -b {$this->git_branch}"; + } + if ($pull_request_id !== 0) { + $pr_branch_name = "pr-{$pull_request_id}-coolify"; + } + + if ($this->deploymentType() === 'source') { + $source_html_url = data_get($this, 'source.html_url'); + $url = parse_url(filter_var($source_html_url, FILTER_SANITIZE_URL)); + $source_html_url_host = $url['host']; + $source_html_url_scheme = $url['scheme']; + + if ($this->source->getMorphClass() == 'App\Models\GithubApp') { + if ($this->source->is_public) { + $fullRepoUrl = "{$this->source->html_url}/{$customRepository}"; + $git_clone_command = "{$git_clone_command} {$this->source->html_url}/{$customRepository} {$baseDir}"; + if (!$only_checkout) { + $git_clone_command = $this->setGitImportSettings($deployment_uuid, $git_clone_command); + } + if ($exec_in_docker) { + $commands->push(executeInDocker($deployment_uuid, $git_clone_command)); + } else { + $commands->push($git_clone_command); + } + } else { + $github_access_token = generate_github_installation_token($this->source); + if ($exec_in_docker) { + $commands->push(executeInDocker($deployment_uuid, "{$git_clone_command} $source_html_url_scheme://x-access-token:$github_access_token@$source_html_url_host/{$customRepository}.git {$baseDir}")); + $fullRepoUrl = "$source_html_url_scheme://x-access-token:$github_access_token@$source_html_url_host/{$customRepository}.git"; + } else { + $commands->push("{$git_clone_command} $source_html_url_scheme://x-access-token:$github_access_token@$source_html_url_host/{$customRepository} {$baseDir}"); + $fullRepoUrl = "$source_html_url_scheme://x-access-token:$github_access_token@$source_html_url_host/{$customRepository}"; + } + } + if ($pull_request_id !== 0) { + $branch = "pull/{$pull_request_id}/head:$pr_branch_name"; + if ($exec_in_docker) { + $commands->push(executeInDocker($deployment_uuid, "cd {$baseDir} && git fetch origin {$branch} && git checkout $pr_branch_name")); + } else { + $commands->push("cd {$baseDir} && git fetch origin {$branch} && git checkout $pr_branch_name"); + } + } + return [ + 'commands' => $commands->implode(' && '), + 'branch' => $branch, + 'fullRepoUrl' => $fullRepoUrl + ]; + } + } + if ($this->deploymentType() === 'deploy_key') { + $fullRepoUrl = $customRepository; + $private_key = data_get($this, 'private_key.private_key'); + if (is_null($private_key)) { + throw new RuntimeException('Private key not found. Please add a private key to the application and try again.'); + } + $private_key = base64_encode($private_key); + $git_clone_command_base = "GIT_SSH_COMMAND=\"ssh -o ConnectTimeout=30 -p {$customPort} -o Port={$customPort} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /root/.ssh/id_rsa\" {$git_clone_command} {$customRepository} {$baseDir}"; + if (!$only_checkout) { + $git_clone_command = $this->setGitImportSettings($deployment_uuid, $git_clone_command_base); + } + if ($exec_in_docker) { + $commands = collect([ + executeInDocker($deployment_uuid, "mkdir -p /root/.ssh"), + executeInDocker($deployment_uuid, "echo '{$private_key}' | base64 -d > /root/.ssh/id_rsa"), + executeInDocker($deployment_uuid, "chmod 600 /root/.ssh/id_rsa"), + ]); + } else { + $commands = collect([ + "mkdir -p /root/.ssh", + "echo '{$private_key}' | base64 -d > /root/.ssh/id_rsa", + "chmod 600 /root/.ssh/id_rsa", + ]); + } + if ($pull_request_id !== 0) { + if ($git_type === 'gitlab') { + $branch = "merge-requests/{$pull_request_id}/head:$pr_branch_name"; + if ($exec_in_docker) { + $commands->push(executeInDocker($deployment_uuid, "echo 'Checking out $branch'")); + } else { + $commands->push("echo 'Checking out $branch'"); + } + $git_clone_command = "{$git_clone_command} && cd {$baseDir} && GIT_SSH_COMMAND=\"ssh -o ConnectTimeout=30 -p {$customPort} -o Port={$customPort} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /root/.ssh/id_rsa\" git fetch origin $branch && git checkout $pr_branch_name"; + } + if ($git_type === 'github') { + $branch = "pull/{$pull_request_id}/head:$pr_branch_name"; + if ($exec_in_docker) { + $commands->push(executeInDocker($deployment_uuid, "echo 'Checking out $branch'")); + } else { + $commands->push("echo 'Checking out $branch'"); + } + $git_clone_command = "{$git_clone_command} && cd {$baseDir} && GIT_SSH_COMMAND=\"ssh -o ConnectTimeout=30 -p {$customPort} -o Port={$customPort} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /root/.ssh/id_rsa\" git fetch origin $branch && git checkout $pr_branch_name"; + } + } + + if ($exec_in_docker) { + $commands->push(executeInDocker($deployment_uuid, $git_clone_command)); + } else { + $commands->push($git_clone_command); + } + return [ + 'commands' => $commands->implode(' && '), + 'branch' => $branch, + 'fullRepoUrl' => $fullRepoUrl + ]; + } + if ($this->deploymentType() === 'other') { + $fullRepoUrl = $customRepository; + $git_clone_command = "{$git_clone_command} {$customRepository} {$baseDir}"; + $git_clone_command = $this->setGitImportSettings($deployment_uuid, $git_clone_command); + if ($exec_in_docker) { + $commands->push(executeInDocker($deployment_uuid, $git_clone_command)); + } else { + $commands->push($git_clone_command); + } + return [ + 'commands' => $commands->implode(' && '), + 'branch' => $branch, + 'fullRepoUrl' => $fullRepoUrl + ]; + } + } + public function prepareHelperImage(string $deploymentUuid) + { + $basedir = $this->generateBaseDir($deploymentUuid); + $helperImage = config('coolify.helper_image'); + $server = data_get($this, 'destination.server'); + $network = data_get($this, 'destination.network'); + + $serverUserHomeDir = instant_remote_process(["echo \$HOME"], $server); + $dockerConfigFileExists = instant_remote_process(["test -f {$serverUserHomeDir}/.docker/config.json && echo 'OK' || echo 'NOK'"], $server); + + $commands = collect([]); + if ($dockerConfigFileExists === 'OK') { + $commands->push([ + "command" => "docker run -d --network $network -v /:/host --name $deploymentUuid --rm -v {$serverUserHomeDir}/.docker/config.json:/root/.docker/config.json:ro -v /var/run/docker.sock:/var/run/docker.sock $helperImage", + "hidden" => true, + ]); + } else { + $commands->push([ + "command" => "docker run -d --network {$network} -v /:/host --name {$deploymentUuid} --rm -v /var/run/docker.sock:/var/run/docker.sock {$helperImage}", + "hidden" => true, + ]); + } + $commands->push([ + "command" => executeInDocker($deploymentUuid, "mkdir -p {$basedir}"), + "hidden" => true, + ]); + return $commands; + } + function parseCompose(int $pull_request_id = 0) + { + if ($this->docker_compose_raw) { + $mainCompose = parseDockerComposeFile(resource: $this, isNew: false, pull_request_id: $pull_request_id); + if ($this->getMorphClass() === 'App\Models\Application' && $this->docker_compose_pr_raw) { + parseDockerComposeFile(resource: $this, isNew: false, pull_request_id: $pull_request_id, is_pr: true); + } + return $mainCompose; + } else { + return collect([]); + } + } + function loadComposeFile($isInit = false) + { + $initialDockerComposeLocation = $this->docker_compose_location; + $initialDockerComposePrLocation = $this->docker_compose_pr_location; + if ($this->build_pack === 'dockercompose') { + if ($isInit && $this->docker_compose_raw) { + return; + } + $uuid = new Cuid2(); + ['commands' => $cloneCommand] = $this->generateGitImportCommands(deployment_uuid: $uuid, only_checkout: true, exec_in_docker: false, custom_base_dir: '.'); + $workdir = rtrim($this->base_directory, '/'); + $composeFile = $this->docker_compose_location; + $prComposeFile = $this->docker_compose_pr_location; + $fileList = collect([".$composeFile"]); + if ($composeFile !== $prComposeFile) { + $fileList->push(".$prComposeFile"); + } + $commands = collect([ + "mkdir -p /tmp/{$uuid} && cd /tmp/{$uuid}", + $cloneCommand, + "git sparse-checkout init --cone", + "git sparse-checkout set {$fileList->implode(' ')}", + "git read-tree -mu HEAD", + "cat .$workdir$composeFile", + ]); + $composeFileContent = instant_remote_process($commands, $this->destination->server, false); + if (!$composeFileContent) { + $this->docker_compose_location = $initialDockerComposeLocation; + $this->save(); + throw new \Exception("Could not load base compose file from $workdir$composeFile"); + } else { + $this->docker_compose_raw = $composeFileContent; + $this->save(); + } + if ($composeFile === $prComposeFile) { + $this->docker_compose_pr_raw = $composeFileContent; + $this->save(); + } else { + $commands = collect([ + "cd /tmp/{$uuid}", + "cat .$workdir$prComposeFile", + ]); + $composePrFileContent = instant_remote_process($commands, $this->destination->server, false); + if (!$composePrFileContent) { + $this->docker_compose_pr_location = $initialDockerComposePrLocation; + $this->save(); + throw new \Exception("Could not load compose file from $workdir$prComposeFile"); + } else { + $this->docker_compose_pr_raw = $composePrFileContent; + $this->save(); + } + } + + $commands = collect([ + "rm -rf /tmp/{$uuid}", + ]); + instant_remote_process($commands, $this->destination->server, false); + return [ + 'parsedServices' => $this->parseCompose(), + 'initialDockerComposeLocation' => $this->docker_compose_location, + 'initialDockerComposePrLocation' => $this->docker_compose_pr_location, + ]; + } + } } diff --git a/app/Models/ApplicationDeploymentQueue.php b/app/Models/ApplicationDeploymentQueue.php index 1b7ae4781..99aed3750 100644 --- a/app/Models/ApplicationDeploymentQueue.php +++ b/app/Models/ApplicationDeploymentQueue.php @@ -3,8 +3,48 @@ namespace App\Models; use Illuminate\Database\Eloquent\Model; +use Illuminate\Support\Carbon; class ApplicationDeploymentQueue extends Model { protected $guarded = []; + + public function getOutput($name) + { + if (!$this->logs) { + return null; + } + return collect(json_decode($this->logs))->where('name', $name)->first()?->output ?? null; + } + + public function addLogEntry(string $message, string $type = 'stdout', bool $hidden = false) + { + if ($type === 'error') { + $type = 'stderr'; + } + $message = str($message)->trim(); + if ($message->startsWith('╔')) { + $message = "\n" . $message; + } + $newLogEntry = [ + 'command' => null, + 'output' => remove_iip($message), + 'type' => $type, + 'timestamp' => Carbon::now('UTC'), + 'hidden' => $hidden, + 'batch' => 1, + ]; + if ($this->logs) { + $previousLogs = json_decode($this->logs, associative: true, flags: JSON_THROW_ON_ERROR); + $newLogEntry['order'] = count($previousLogs) + 1; + $previousLogs[] = $newLogEntry; + $this->update([ + 'logs' => json_encode($previousLogs, flags: JSON_THROW_ON_ERROR), + ]); + } else { + $this->update([ + 'logs' => json_encode([$newLogEntry], flags: JSON_THROW_ON_ERROR), + ]); + } + } } diff --git a/app/Models/ApplicationPreview.php b/app/Models/ApplicationPreview.php index 13775abae..87dce056e 100644 --- a/app/Models/ApplicationPreview.php +++ b/app/Models/ApplicationPreview.php @@ -5,7 +5,26 @@ namespace App\Models; class ApplicationPreview extends BaseModel { protected $guarded = []; - + protected static function booted() + { + static::deleting(function ($preview) { + if ($preview->application->build_pack === 'dockercompose') { + $server = $preview->application->destination->server; + $composeFile = $preview->application->parseCompose(pull_request_id: $preview->pull_request_id); + $volumes = data_get($composeFile, 'volumes'); + $networks = data_get($composeFile, 'networks'); + $networkKeys = collect($networks)->keys(); + $volumeKeys = collect($volumes)->keys(); + $volumeKeys->each(function ($key) use ($server) { + instant_remote_process(["docker volume rm -f $key"], $server, false); + }); + $networkKeys->each(function ($key) use ($server) { + instant_remote_process(["docker network disconnect $key coolify-proxy"], $server, false); + instant_remote_process(["docker network rm $key"], $server, false); + }); + } + }); + } static function findPreviewByApplicationAndPullId(int $application_id, int $pull_request_id) { return self::where('application_id', $application_id)->where('pull_request_id', $pull_request_id)->firstOrFail(); diff --git a/app/Models/InstanceSettings.php b/app/Models/InstanceSettings.php index 057595351..846148159 100644 --- a/app/Models/InstanceSettings.php +++ b/app/Models/InstanceSettings.php @@ -3,8 +3,10 @@ namespace App\Models; use App\Notifications\Channels\SendsEmail; +use Illuminate\Database\Eloquent\Casts\Attribute; use Illuminate\Database\Eloquent\Model; use Illuminate\Notifications\Notifiable; +use Spatie\Url\Url; class InstanceSettings extends Model implements SendsEmail { @@ -16,6 +18,18 @@ class InstanceSettings extends Model implements SendsEmail 'smtp_password' => 'encrypted', ]; + public function fqdn(): Attribute + { + return Attribute::make( + set: function ($value) { + if ($value) { + $url = Url::fromString($value); + $host = $url->getHost(); + return $url->getScheme() . '://' . $host; + } + } + ); + } public static function get() { return InstanceSettings::findOrFail(0); diff --git a/app/Models/Server.php b/app/Models/Server.php index 30e07c1d8..07714caa7 100644 --- a/app/Models/Server.php +++ b/app/Models/Server.php @@ -4,20 +4,26 @@ namespace App\Models; use App\Actions\Server\InstallLogDrain; use App\Actions\Server\InstallNewRelic; +use App\Enums\ApplicationDeploymentStatus; use App\Enums\ProxyStatus; use App\Enums\ProxyTypes; use App\Notifications\Server\Revived; use App\Notifications\Server\Unreachable; use Illuminate\Database\Eloquent\Builder; use Illuminate\Database\Eloquent\Casts\Attribute; +use Illuminate\Support\Carbon; +use Illuminate\Support\Collection; +use Illuminate\Support\Facades\Process; use Illuminate\Support\Sleep; use Spatie\SchemalessAttributes\Casts\SchemalessAttributes; use Spatie\SchemalessAttributes\SchemalessAttributesTrait; use Illuminate\Support\Str; +use Stringable; class Server extends BaseModel { use SchemalessAttributesTrait; + public static $batch_counter = 0; protected static function booted() { @@ -189,6 +195,13 @@ class Server extends BaseModel { return instant_remote_process(["df /| tail -1 | awk '{ print $5}' | sed 's/%//g'"], $this, false); } + public function definedResources() + { + $applications = $this->applications(); + $databases = $this->databases(); + $services = $this->services(); + return $applications->concat($databases)->concat($services->get()); + } public function hasDefinedResources() { $applications = $this->applications()->count() > 0; @@ -217,6 +230,23 @@ class Server extends BaseModel return $standaloneDocker->applications; })->flatten(); } + public function dockerComposeBasedApplications() + { + return $this->applications()->filter(function ($application) { + return data_get($application, 'build_pack') === 'dockercompose'; + }); + } + public function dockerComposeBasedPreviewDeployments() + { + return $this->previews()->filter(function ($preview) { + $applicationId = data_get($preview, 'application_id'); + $application = Application::find($applicationId); + if (!$application) { + return false; + } + return data_get($application, 'build_pack') === 'dockercompose'; + }); + } public function services() { return $this->hasMany(Service::class); @@ -304,7 +334,7 @@ class Server extends BaseModel { return $this->settings->is_logdrain_newrelic_enabled || $this->settings->is_logdrain_highlight_enabled || $this->settings->is_logdrain_axiom_enabled; } - public function validateOS() + public function validateOS(): bool | Stringable { $os_release = instant_remote_process(['cat /etc/os-release'], $this); $datas = collect(explode("\n", $os_release)); @@ -314,12 +344,16 @@ class Server extends BaseModel $collectedData->put($item->before('=')->value(), $item->after('=')->lower()->replace('"', '')->value()); } $ID = data_get($collectedData, 'ID'); - $ID_LIKE = data_get($collectedData, 'ID_LIKE'); - $VERSION_ID = data_get($collectedData, 'VERSION_ID'); - // ray($ID, $ID_LIKE, $VERSION_ID); - if (collect(SUPPORTED_OS)->contains($ID_LIKE)) { + // $ID_LIKE = data_get($collectedData, 'ID_LIKE'); + // $VERSION_ID = data_get($collectedData, 'VERSION_ID'); + $supported = collect(SUPPORTED_OS)->filter(function ($supportedOs) use ($ID) { + if (str($supportedOs)->contains($ID)) { + return str($ID); + } + }); + if ($supported->count() === 1) { ray('supported'); - return str($ID_LIKE)->explode(' ')->first(); + return str($supported->first()); } else { ray('not supported'); return false; @@ -387,4 +421,83 @@ class Server extends BaseModel { return instant_remote_process(["docker network create coolify --attachable >/dev/null 2>&1 || true"], $this, false); } + public function executeRemoteCommand(Collection $commands, ?ApplicationDeploymentQueue $loggingModel = null) + { + static::$batch_counter++; + foreach ($commands as $command) { + $realCommand = data_get($command, 'command'); + if (is_null($realCommand)) { + throw new \RuntimeException('Command is not set'); + } + $hidden = data_get($command, 'hidden', false); + $ignoreErrors = data_get($command, 'ignoreErrors', false); + $customOutputType = data_get($command, 'customOutputType'); + $name = data_get($command, 'name'); + $remoteCommand = generateSshCommand($this, $realCommand); + + $process = Process::timeout(3600)->idleTimeout(3600)->start($remoteCommand, function (string $type, string $output) use ($realCommand, $hidden, $customOutputType, $loggingModel, $name) { + $output = str($output)->trim(); + if ($output->startsWith('╔')) { + $output = "\n" . $output; + } + $newLogEntry = [ + 'command' => remove_iip($realCommand), + 'output' => remove_iip($output), + 'type' => $customOutputType ?? $type === 'err' ? 'stderr' : 'stdout', + 'timestamp' => Carbon::now('UTC'), + 'hidden' => $hidden, + 'batch' => static::$batch_counter, + ]; + if ($loggingModel) { + if (!$loggingModel->logs) { + $newLogEntry['order'] = 1; + } else { + $previousLogs = json_decode($loggingModel->logs, associative: true, flags: JSON_THROW_ON_ERROR); + $newLogEntry['order'] = count($previousLogs) + 1; + } + if ($name) { + $newLogEntry['name'] = $name; + } + + $previousLogs[] = $newLogEntry; + $loggingModel->logs = json_encode($previousLogs, flags: JSON_THROW_ON_ERROR); + $loggingModel->save(); + } + }); + if ($loggingModel) { + $loggingModel->update([ + 'current_process_id' => $process->id(), + ]); + } + $processResult = $process->wait(); + if ($processResult->exitCode() !== 0) { + if (!$ignoreErrors) { + if ($loggingModel) { + $status = ApplicationDeploymentStatus::FAILED->value; + $loggingModel->status = $status; + $loggingModel->save(); + } + throw new \RuntimeException($processResult->errorOutput()); + } + } + } + } + public function stopApplicationRelatedRunningContainers(string $applicationId, string $containerName) + { + $containers = getCurrentApplicationContainerStatus($this, $applicationId, 0); + $containers = $containers->filter(function ($container) use ($containerName) { + return data_get($container, 'Names') !== $containerName; + }); + $containers->each(function ($container) { + $removableContainer = data_get($container, 'Names'); + $this->server->executeRemoteCommand( + commands: collect([ + 'command' => "docker rm -f $removableContainer >/dev/null 2>&1", + 'hidden' => true, + 'ignoreErrors' => true + ]), + loggingModel: $this->deploymentQueueEntry + ); + }); + } } diff --git a/app/Models/Service.php b/app/Models/Service.php index 8cd195bce..3a02ed560 100644 --- a/app/Models/Service.php +++ b/app/Models/Service.php @@ -52,7 +52,7 @@ class Service extends BaseModel foreach ($applications as $application) { $image = str($application->image)->before(':')->value(); switch ($image) { - case str($image)->contains('minio'): + case str($image)?->contains('minio'): $data = collect([]); $console_url = $this->environment_variables()->where('key', 'MINIO_BROWSER_REDIRECT_URL')->first(); $s3_api_url = $this->environment_variables()->where('key', 'MINIO_SERVER_URL')->first(); @@ -105,7 +105,7 @@ class Service extends BaseModel $fields->put('MinIO', $data->toArray()); break; - case str($image)->contains('weblate'): + case str($image)?->contains('weblate'): $data = collect([]); $admin_email = $this->environment_variables()->where('key', 'WEBLATE_ADMIN_EMAIL')->first(); $admin_password = $this->environment_variables()->where('key', 'SERVICE_PASSWORD_WEBLATE')->first(); @@ -130,6 +130,67 @@ class Service extends BaseModel ]); } $fields->put('Weblate', $data); + break; + case str($image)?->contains('ghost'): + $data = collect([]); + $MAIL_OPTIONS_AUTH_PASS = $this->environment_variables()->where('key', 'MAIL_OPTIONS_AUTH_PASS')->first(); + $MAIL_OPTIONS_AUTH_USER = $this->environment_variables()->where('key', 'MAIL_OPTIONS_AUTH_USER')->first(); + $MAIL_OPTIONS_SECURE = $this->environment_variables()->where('key', 'MAIL_OPTIONS_SECURE')->first(); + $MAIL_OPTIONS_PORT = $this->environment_variables()->where('key', 'MAIL_OPTIONS_PORT')->first(); + $MAIL_OPTIONS_SERVICE = $this->environment_variables()->where('key', 'MAIL_OPTIONS_SERVICE')->first(); + $MAIL_OPTIONS_HOST = $this->environment_variables()->where('key', 'MAIL_OPTIONS_HOST')->first(); + if ($MAIL_OPTIONS_AUTH_PASS) { + $data = $data->merge([ + 'Mail Password' => [ + 'key' => data_get($MAIL_OPTIONS_AUTH_PASS, 'key'), + 'value' => data_get($MAIL_OPTIONS_AUTH_PASS, 'value'), + 'isPassword' => true, + ], + ]); + } + if ($MAIL_OPTIONS_AUTH_USER) { + $data = $data->merge([ + 'Mail User' => [ + 'key' => data_get($MAIL_OPTIONS_AUTH_USER, 'key'), + 'value' => data_get($MAIL_OPTIONS_AUTH_USER, 'value'), + ], + ]); + } + if ($MAIL_OPTIONS_SECURE) { + $data = $data->merge([ + 'Mail Secure' => [ + 'key' => data_get($MAIL_OPTIONS_SECURE, 'key'), + 'value' => data_get($MAIL_OPTIONS_SECURE, 'value'), + ], + ]); + } + if ($MAIL_OPTIONS_PORT) { + $data = $data->merge([ + 'Mail Port' => [ + 'key' => data_get($MAIL_OPTIONS_PORT, 'key'), + 'value' => data_get($MAIL_OPTIONS_PORT, 'value'), + ], + ]); + } + if ($MAIL_OPTIONS_SERVICE) { + $data = $data->merge([ + 'Mail Service' => [ + 'key' => data_get($MAIL_OPTIONS_SERVICE, 'key'), + 'value' => data_get($MAIL_OPTIONS_SERVICE, 'value'), + ], + ]); + } + if ($MAIL_OPTIONS_HOST) { + $data = $data->merge([ + 'Mail Host' => [ + 'key' => data_get($MAIL_OPTIONS_HOST, 'key'), + 'value' => data_get($MAIL_OPTIONS_HOST, 'value'), + ], + ]); + } + + $fields->put('Ghost', $data); + break; } } $databases = $this->databases()->get(); @@ -300,6 +361,14 @@ class Service extends BaseModel } } } + public function link() + { + return route('project.service.configuration', [ + 'project_uuid' => $this->environment->project->uuid, + 'environment_name' => $this->environment->name, + 'service_uuid' => $this->uuid + ]); + } public function documentation() { $services = getServiceTemplates(); @@ -371,521 +440,12 @@ class Service extends BaseModel public function parse(bool $isNew = false): Collection { - // ray()->clearAll(); - if ($this->docker_compose_raw) { - try { - $yaml = Yaml::parse($this->docker_compose_raw); - } catch (\Exception $e) { - throw new \Exception($e->getMessage()); - } - - $topLevelVolumes = collect(data_get($yaml, 'volumes', [])); - $topLevelNetworks = collect(data_get($yaml, 'networks', [])); - $dockerComposeVersion = data_get($yaml, 'version') ?? '3.8'; - $services = data_get($yaml, 'services'); - - $generatedServiceFQDNS = collect([]); - if (is_null($this->destination)) { - $destination = $this->server->destinations()->first(); - if ($destination) { - $this->destination()->associate($destination); - $this->save(); - } - } - $definedNetwork = collect([$this->uuid]); - - $services = collect($services)->map(function ($service, $serviceName) use ($topLevelVolumes, $topLevelNetworks, $definedNetwork, $isNew, $generatedServiceFQDNS) { - $serviceVolumes = collect(data_get($service, 'volumes', [])); - $servicePorts = collect(data_get($service, 'ports', [])); - $serviceNetworks = collect(data_get($service, 'networks', [])); - $serviceVariables = collect(data_get($service, 'environment', [])); - $serviceLabels = collect(data_get($service, 'labels', [])); - if ($serviceLabels->count() > 0) { - $removedLabels = collect([]); - $serviceLabels = $serviceLabels->filter(function ($serviceLabel, $serviceLabelName) use ($removedLabels) { - if (!str($serviceLabel)->contains('=')) { - $removedLabels->put($serviceLabelName, $serviceLabel); - return false; - } - return $serviceLabel; - }); - foreach($removedLabels as $removedLabelName =>$removedLabel) { - $serviceLabels->push("$removedLabelName=$removedLabel"); - } - } - - $containerName = "$serviceName-{$this->uuid}"; - - // Decide if the service is a database - $isDatabase = false; - $image = data_get_str($service, 'image'); - if ($image->contains(':')) { - $image = Str::of($image); - } else { - $image = Str::of($image)->append(':latest'); - } - $imageName = $image->before(':'); - - if (collect(DATABASE_DOCKER_IMAGES)->contains($imageName)) { - $isDatabase = true; - } - data_set($service, 'is_database', $isDatabase); - - // Create new serviceApplication or serviceDatabase - if ($isDatabase) { - if ($isNew) { - $savedService = ServiceDatabase::create([ - 'name' => $serviceName, - 'image' => $image, - 'service_id' => $this->id - ]); - } else { - $savedService = ServiceDatabase::where([ - 'name' => $serviceName, - 'service_id' => $this->id - ])->first(); - } - } else { - if ($isNew) { - $savedService = ServiceApplication::create([ - 'name' => $serviceName, - 'image' => $image, - 'service_id' => $this->id - ]); - } else { - $savedService = ServiceApplication::where([ - 'name' => $serviceName, - 'service_id' => $this->id - ])->first(); - } - } - if (is_null($savedService)) { - if ($isDatabase) { - $savedService = ServiceDatabase::create([ - 'name' => $serviceName, - 'image' => $image, - 'service_id' => $this->id - ]); - } else { - $savedService = ServiceApplication::create([ - 'name' => $serviceName, - 'image' => $image, - 'service_id' => $this->id - ]); - } - } - - // Check if image changed - if ($savedService->image !== $image) { - $savedService->image = $image; - $savedService->save(); - } - - // Collect/create/update networks - if ($serviceNetworks->count() > 0) { - foreach ($serviceNetworks as $networkName => $networkDetails) { - $networkExists = $topLevelNetworks->contains(function ($value, $key) use ($networkName) { - return $value == $networkName || $key == $networkName; - }); - if (!$networkExists) { - $topLevelNetworks->put($networkDetails, null); - } - } - } - - // Collect/create/update ports - $collectedPorts = collect([]); - if ($servicePorts->count() > 0) { - foreach ($servicePorts as $sport) { - if (is_string($sport) || is_numeric($sport)) { - $collectedPorts->push($sport); - } - if (is_array($sport)) { - $target = data_get($sport, 'target'); - $published = data_get($sport, 'published'); - $protocol = data_get($sport, 'protocol'); - $collectedPorts->push("$target:$published/$protocol"); - } - } - } - $savedService->ports = $collectedPorts->implode(','); - $savedService->save(); - - // Add Coolify specific networks - $definedNetworkExists = $topLevelNetworks->contains(function ($value, $_) use ($definedNetwork) { - return $value == $definedNetwork; - }); - if (!$definedNetworkExists) { - foreach ($definedNetwork as $network) { - $topLevelNetworks->put($network, [ - 'name' => $network, - 'external' => true - ]); - } - } - $networks = collect(); - foreach ($serviceNetworks as $key => $serviceNetwork) { - if (gettype($serviceNetwork) === 'string') { - // networks: - // - appwrite - $networks->put($serviceNetwork, null); - } else if (gettype($serviceNetwork) === 'array') { - // networks: - // default: - // ipv4_address: 192.168.203.254 - // $networks->put($serviceNetwork, null); - ray($key); - $networks->put($key, $serviceNetwork); - } - } - foreach ($definedNetwork as $key => $network) { - $networks->put($network, null); - } - data_set($service, 'networks', $networks->toArray()); - - // Collect/create/update volumes - if ($serviceVolumes->count() > 0) { - $serviceVolumes = $serviceVolumes->map(function ($volume) use ($savedService, $topLevelVolumes) { - $type = null; - $source = null; - $target = null; - $content = null; - $isDirectory = false; - if (is_string($volume)) { - $source = Str::of($volume)->before(':'); - $target = Str::of($volume)->after(':')->beforeLast(':'); - if ($source->startsWith('./') || $source->startsWith('/') || $source->startsWith('~')) { - $type = Str::of('bind'); - } else { - $type = Str::of('volume'); - } - } else if (is_array($volume)) { - $type = data_get_str($volume, 'type'); - $source = data_get_str($volume, 'source'); - $target = data_get_str($volume, 'target'); - $content = data_get($volume, 'content'); - $isDirectory = (bool) data_get($volume, 'isDirectory', false); - $foundConfig = $savedService->fileStorages()->whereMountPath($target)->first(); - if ($foundConfig) { - $contentNotNull = data_get($foundConfig, 'content'); - if ($contentNotNull) { - $content = $contentNotNull; - } - $isDirectory = (bool) data_get($foundConfig, 'is_directory'); - } - } - if ($type->value() === 'bind') { - if ($source->value() === "/var/run/docker.sock") { - return $volume; - } - if ($source->value() === '/tmp' || $source->value() === '/tmp/') { - return $volume; - } - LocalFileVolume::updateOrCreate( - [ - 'mount_path' => $target, - 'resource_id' => $savedService->id, - 'resource_type' => get_class($savedService) - ], - [ - 'fs_path' => $source, - 'mount_path' => $target, - 'content' => $content, - 'is_directory' => $isDirectory, - 'resource_id' => $savedService->id, - 'resource_type' => get_class($savedService) - ] - ); - } else if ($type->value() === 'volume') { - $slugWithoutUuid = Str::slug($source, '-'); - $name = "{$savedService->service->uuid}_{$slugWithoutUuid}"; - if (is_string($volume)) { - $source = Str::of($volume)->before(':'); - $target = Str::of($volume)->after(':')->beforeLast(':'); - $source = $name; - $volume = "$source:$target"; - } else if (is_array($volume)) { - data_set($volume, 'source', $name); - } - $topLevelVolumes->put($name, [ - 'name' => $name, - ]); - LocalPersistentVolume::updateOrCreate( - [ - 'mount_path' => $target, - 'resource_id' => $savedService->id, - 'resource_type' => get_class($savedService) - ], - [ - 'name' => $name, - 'mount_path' => $target, - 'resource_id' => $savedService->id, - 'resource_type' => get_class($savedService) - ] - ); - } - $savedService->getFilesFromServer(isInit: true); - return $volume; - }); - data_set($service, 'volumes', $serviceVolumes->toArray()); - } - - // Add env_file with at least .env to the service - // $envFile = collect(data_get($service, 'env_file', [])); - // if ($envFile->count() > 0) { - // if (!$envFile->contains('.env')) { - // $envFile->push('.env'); - // } - // } else { - // $envFile = collect(['.env']); - // } - // data_set($service, 'env_file', $envFile->toArray()); - - - // Get variables from the service - foreach ($serviceVariables as $variableName => $variable) { - if (is_numeric($variableName)) { - $variable = Str::of($variable); - if ($variable->contains('=')) { - // - SESSION_SECRET=123 - // - SESSION_SECRET= - $key = $variable->before('='); - $value = $variable->after('='); - } else { - // - SESSION_SECRET - $key = $variable; - $value = null; - } - } else { - // SESSION_SECRET: 123 - // SESSION_SECRET: - $key = Str::of($variableName); - $value = Str::of($variable); - } - // TODO: here is the problem - if ($key->startsWith('SERVICE_FQDN')) { - if ($isNew || $savedService->fqdn === null) { - $name = $key->after('SERVICE_FQDN_')->beforeLast('_')->lower(); - $fqdn = generateFqdn($this->server, "{$name->value()}-{$this->uuid}"); - if (substr_count($key->value(), '_') === 3) { - // SERVICE_FQDN_UMAMI_1000 - $port = $key->afterLast('_'); - } else { - // SERVICE_FQDN_UMAMI - $port = null; - } - if ($port) { - $fqdn = "$fqdn:$port"; - } - if (substr_count($key->value(), '_') >= 2) { - if (is_null($value)) { - $value = Str::of('/'); - } - $path = $value->value(); - if ($generatedServiceFQDNS->count() > 0) { - $alreadyGenerated = $generatedServiceFQDNS->has($key->value()); - if ($alreadyGenerated) { - $fqdn = $generatedServiceFQDNS->get($key->value()); - } else { - $generatedServiceFQDNS->put($key->value(), $fqdn); - } - } else { - $generatedServiceFQDNS->put($key->value(), $fqdn); - } - $fqdn = "$fqdn$path"; - } - - if (!$isDatabase) { - if ($savedService->fqdn) { - $fqdn = $savedService->fqdn . ',' . $fqdn; - } else { - $fqdn = $fqdn; - } - $savedService->fqdn = $fqdn; - $savedService->save(); - } - } - // data_forget($service, "environment.$variableName"); - // $yaml = data_forget($yaml, "services.$serviceName.environment.$variableName"); - // if (count(data_get($yaml, 'services.' . $serviceName . '.environment')) === 0) { - // $yaml = data_forget($yaml, "services.$serviceName.environment"); - // } - continue; - } - if ($value?->startsWith('$')) { - $value = Str::of(replaceVariables($value)); - $key = $value; - $foundEnv = EnvironmentVariable::where([ - 'key' => $key, - 'service_id' => $this->id, - ])->first(); - if ($value->startsWith('SERVICE_')) { - // Count _ in $value - $count = substr_count($value->value(), '_'); - if ($count === 2) { - // SERVICE_FQDN_UMAMI - $command = $value->after('SERVICE_')->beforeLast('_'); - $forService = $value->afterLast('_'); - $generatedValue = null; - $port = null; - } - if ($count === 3) { - // SERVICE_FQDN_UMAMI_1000 - $command = $value->after('SERVICE_')->before('_'); - $forService = $value->after('SERVICE_')->after('_')->before('_'); - $generatedValue = null; - $port = $value->afterLast('_'); - } - if ($command->value() === 'FQDN' || $command->value() === 'URL') { - if (Str::lower($forService) === $serviceName) { - $fqdn = generateFqdn($this->server, $containerName); - } else { - $fqdn = generateFqdn($this->server, Str::lower($forService) . '-' . $this->uuid); - } - if ($port) { - $fqdn = "$fqdn:$port"; - } - if ($foundEnv) { - $fqdn = data_get($foundEnv, 'value'); - } else { - if ($command->value() === 'URL') { - $fqdn = Str::of($fqdn)->after('://')->value(); - } - EnvironmentVariable::create([ - 'key' => $key, - 'value' => $fqdn, - 'is_build_time' => false, - 'service_id' => $this->id, - 'is_preview' => false, - ]); - } - if (!$isDatabase) { - if ($command->value() === 'FQDN' && is_null($savedService->fqdn)) { - $savedService->fqdn = $fqdn; - $savedService->save(); - } - } - } else { - switch ($command) { - case 'PASSWORD': - $generatedValue = Str::password(symbols: false); - break; - case 'PASSWORD_64': - $generatedValue = Str::password(length: 64, symbols: false); - break; - case 'BASE64_64': - $generatedValue = Str::random(64); - break; - case 'BASE64_128': - $generatedValue = Str::random(128); - break; - case 'BASE64': - $generatedValue = Str::random(32); - break; - case 'USER': - $generatedValue = Str::random(16); - break; - } - - if (!$foundEnv) { - EnvironmentVariable::create([ - 'key' => $key, - 'value' => $generatedValue, - 'is_build_time' => false, - 'service_id' => $this->id, - 'is_preview' => false, - ]); - } - } - } else { - if ($value->contains(':-')) { - $key = $value->before(':'); - $defaultValue = $value->after(':-'); - } else if ($value->contains('-')) { - $key = $value->before('-'); - $defaultValue = $value->after('-'); - } else if ($value->contains(':?')) { - $key = $value->before(':'); - $defaultValue = $value->after(':?'); - } else if ($value->contains('?')) { - $key = $value->before('?'); - $defaultValue = $value->after('?'); - } else { - $key = $value; - $defaultValue = null; - } - if ($foundEnv) { - $defaultValue = data_get($foundEnv, 'value'); - } - EnvironmentVariable::updateOrCreate([ - 'key' => $key, - 'service_id' => $this->id, - ], [ - 'value' => $defaultValue, - 'is_build_time' => false, - 'service_id' => $this->id, - 'is_preview' => false, - ]); - } - } - } - - // Add labels to the service - if ($savedService->serviceType()) { - $fqdns = generateServiceSpecificFqdns($savedService, forTraefik: true); - } else { - $fqdns = collect(data_get($savedService, 'fqdns')); - } - $defaultLabels = defaultLabels($this->id, $containerName, type: 'service', subType: $isDatabase ? 'database' : 'application', subId: $savedService->id); - $serviceLabels = $serviceLabels->merge($defaultLabels); - if (!$isDatabase && $fqdns->count() > 0) { - if ($fqdns) { - $serviceLabels = $serviceLabels->merge(fqdnLabelsForTraefik($this->uuid, $fqdns, true)); - } - } - if ($this->server->isLogDrainEnabled() && $savedService->isLogDrainEnabled()) { - data_set($service, 'logging', [ - 'driver' => 'fluentd', - 'options' => [ - 'fluentd-address' => "tcp://127.0.0.1:24224", - 'fluentd-async' => "true", - 'fluentd-sub-second-precision' => "true", - ] - ]); - } - data_set($service, 'labels', $serviceLabels->toArray()); - data_forget($service, 'is_database'); - data_set($service, 'restart', RESTART_MODE); - data_set($service, 'container_name', $containerName); - data_forget($service, 'volumes.*.content'); - data_forget($service, 'volumes.*.isDirectory'); - // Remove unnecessary variables from service.environment - // $withoutServiceEnvs = collect([]); - // collect(data_get($service, 'environment'))->each(function ($value, $key) use ($withoutServiceEnvs) { - // ray($key, $value); - // if (!Str::of($key)->startsWith('$SERVICE_') && !Str::of($value)->startsWith('SERVICE_')) { - // $k = Str::of($value)->before("="); - // $v = Str::of($value)->after("="); - // $withoutServiceEnvs->put($k->value(), $v->value()); - // } - // }); - // ray($withoutServiceEnvs); - // data_set($service, 'environment', $withoutServiceEnvs->toArray()); - return $service; - }); - $finalServices = [ - 'version' => $dockerComposeVersion, - 'services' => $services->toArray(), - 'volumes' => $topLevelVolumes->toArray(), - 'networks' => $topLevelNetworks->toArray(), - ]; - $this->docker_compose_raw = Yaml::dump($yaml, 10, 2); - $this->docker_compose = Yaml::dump($finalServices, 10, 2); - $this->save(); - $this->saveComposeConfigs(); - return collect([]); - } else { - return collect([]); - } + return parseDockerComposeFile($this, $isNew); + } + public function networks() + { + $networks = getTopLevelNetworks($this); + // ray($networks); + return $networks; } } diff --git a/app/Models/StandaloneMariadb.php b/app/Models/StandaloneMariadb.php index cb9fa9aa2..ca7221d7c 100644 --- a/app/Models/StandaloneMariadb.php +++ b/app/Models/StandaloneMariadb.php @@ -41,6 +41,14 @@ class StandaloneMariadb extends BaseModel $database->environment_variables()->delete(); }); } + public function link() + { + return route('project.database.configuration', [ + 'project_uuid' => $this->environment->project->uuid, + 'environment_name' => $this->environment->name, + 'database_uuid' => $this->uuid + ]); + } public function isLogDrainEnabled() { return data_get($this, 'is_log_drain_enabled', false); diff --git a/app/Models/StandaloneMongodb.php b/app/Models/StandaloneMongodb.php index 3a5b7a52a..598719acf 100644 --- a/app/Models/StandaloneMongodb.php +++ b/app/Models/StandaloneMongodb.php @@ -48,6 +48,14 @@ class StandaloneMongodb extends BaseModel { return data_get($this, 'is_log_drain_enabled', false); } + public function link() + { + return route('project.database.configuration', [ + 'project_uuid' => $this->environment->project->uuid, + 'environment_name' => $this->environment->name, + 'database_uuid' => $this->uuid + ]); + } public function mongoInitdbRootPassword(): Attribute { return Attribute::make( diff --git a/app/Models/StandaloneMysql.php b/app/Models/StandaloneMysql.php index 6de57cb73..4cbee006e 100644 --- a/app/Models/StandaloneMysql.php +++ b/app/Models/StandaloneMysql.php @@ -41,6 +41,14 @@ class StandaloneMysql extends BaseModel $database->environment_variables()->delete(); }); } + public function link() + { + return route('project.database.configuration', [ + 'project_uuid' => $this->environment->project->uuid, + 'environment_name' => $this->environment->name, + 'database_uuid' => $this->uuid + ]); + } public function type(): string { return 'standalone-mysql'; diff --git a/app/Models/StandalonePostgresql.php b/app/Models/StandalonePostgresql.php index 1bef3b6c9..0056f9b51 100644 --- a/app/Models/StandalonePostgresql.php +++ b/app/Models/StandalonePostgresql.php @@ -41,7 +41,14 @@ class StandalonePostgresql extends BaseModel $database->environment_variables()->delete(); }); } - + public function link() + { + return route('project.database.configuration', [ + 'project_uuid' => $this->environment->project->uuid, + 'environment_name' => $this->environment->name, + 'database_uuid' => $this->uuid + ]); + } public function isLogDrainEnabled() { return data_get($this, 'is_log_drain_enabled', false); diff --git a/app/Models/StandaloneRedis.php b/app/Models/StandaloneRedis.php index fe1281c22..b2038d8af 100644 --- a/app/Models/StandaloneRedis.php +++ b/app/Models/StandaloneRedis.php @@ -36,7 +36,14 @@ class StandaloneRedis extends BaseModel $database->environment_variables()->delete(); }); } - + public function link() + { + return route('project.database.configuration', [ + 'project_uuid' => $this->environment->project->uuid, + 'environment_name' => $this->environment->name, + 'database_uuid' => $this->uuid + ]); + } public function isLogDrainEnabled() { return data_get($this, 'is_log_drain_enabled', false); diff --git a/app/Traits/ExecuteRemoteCommand.php b/app/Traits/ExecuteRemoteCommand.php index 1306f645c..6c0dc5d03 100644 --- a/app/Traits/ExecuteRemoteCommand.php +++ b/app/Traits/ExecuteRemoteCommand.php @@ -12,6 +12,7 @@ use Illuminate\Support\Str; trait ExecuteRemoteCommand { public ?string $save = null; + public static int $batch_counter = 0; public function execute_remote_command(...$commands) { static::$batch_counter++; @@ -23,8 +24,6 @@ trait ExecuteRemoteCommand if ($this->server instanceof Server === false) { throw new \RuntimeException('Server is not set or is not an instance of Server model'); } - - $commandsText->each(function ($single_command) { $command = data_get($single_command, 'command') ?? $single_command[0] ?? null; if ($command === null) { @@ -49,32 +48,29 @@ trait ExecuteRemoteCommand 'hidden' => $hidden, 'batch' => static::$batch_counter, ]; - - if (!$this->log_model->logs) { + if (!$this->application_deployment_queue->logs) { $new_log_entry['order'] = 1; } else { - $previous_logs = json_decode($this->log_model->logs, associative: true, flags: JSON_THROW_ON_ERROR); + $previous_logs = json_decode($this->application_deployment_queue->logs, associative: true, flags: JSON_THROW_ON_ERROR); $new_log_entry['order'] = count($previous_logs) + 1; } - $previous_logs[] = $new_log_entry; - $this->log_model->logs = json_encode($previous_logs, flags: JSON_THROW_ON_ERROR); - $this->log_model->save(); + $this->application_deployment_queue->logs = json_encode($previous_logs, flags: JSON_THROW_ON_ERROR); + $this->application_deployment_queue->save(); if ($this->save) { $this->saved_outputs[$this->save] = Str::of($output)->trim(); } }); - $this->log_model->update([ + $this->application_deployment_queue->update([ 'current_process_id' => $process->id(), ]); $process_result = $process->wait(); if ($process_result->exitCode() !== 0) { if (!$ignore_errors) { - $status = ApplicationDeploymentStatus::FAILED->value; - $this->log_model->status = $status; - $this->log_model->save(); + $this->application_deployment_queue->status = ApplicationDeploymentStatus::FAILED->value; + $this->application_deployment_queue->save(); throw new \RuntimeException($process_result->errorOutput()); } } diff --git a/bootstrap/helpers/applications.php b/bootstrap/helpers/applications.php index 1cead8a54..bc0de5c47 100644 --- a/bootstrap/helpers/applications.php +++ b/bootstrap/helpers/applications.php @@ -36,8 +36,6 @@ function queue_application_deployment(int $application_id, string $deployment_uu if ($running_deployments->count() > 0) { return; } - // New deployment - // dispatchDeploymentJob($deployment->id); dispatch(new ApplicationDeploymentJob( application_deployment_queue_id: $deployment->id, ))->onConnection('long-running')->onQueue('long-running'); @@ -48,34 +46,11 @@ function queue_next_deployment(Application $application) { $next_found = ApplicationDeploymentQueue::where('application_id', $application->id)->where('status', 'queued')->first(); if ($next_found) { - // New deployment - // dispatchDeploymentJob($next_found->id); dispatch(new ApplicationDeploymentJob( application_deployment_queue_id: $next_found->id, ))->onConnection('long-running')->onQueue('long-running'); - } } -function dispatchDeploymentJob($id) -{ - $applicationQueue = ApplicationDeploymentQueue::find($id); - $application = Application::find($applicationQueue->application_id); - - $isRestartOnly = data_get($applicationQueue, 'restart_only'); - $isSimpleDockerFile = data_get($application, 'dockerfile'); - $isDockerImage = data_get($application, 'build_pack') === 'dockerimage'; - - if ($isRestartOnly) { - ApplicationRestartJob::dispatch(applicationDeploymentQueueId: $id)->onConnection('long-running')->onQueue('long-running'); - } else if ($isSimpleDockerFile) { - ApplicationDeploySimpleDockerfileJob::dispatch(applicationDeploymentQueueId: $id)->onConnection('long-running')->onQueue('long-running'); - } else if ($isDockerImage) { - ApplicationDeployDockerImageJob::dispatch(applicationDeploymentQueueId: $id)->onConnection('long-running')->onQueue('long-running'); - } else { - throw new Exception('Unknown build pack'); - } -} - // Deployment things function generateHostIpMapping(Server $server, string $network) { @@ -201,7 +176,6 @@ function generateComposeFile(string $deploymentUuid, Server $server, string $net ]; } if ($application->settings->is_gpu_enabled) { - ray('asd'); $docker_compose['services'][$containerName]['deploy']['resources']['reservations']['devices'] = [ [ 'driver' => data_get($application, 'settings.gpu_driver', 'nvidia'), @@ -302,39 +276,37 @@ function generateEnvironmentVariables(Application $application, $ports, int $pul return $environment_variables->all(); } -function rollingUpdate(Application $application, string $deploymentUuid) +function startNewApplication(Application $application, string $deploymentUuid, ApplicationDeploymentQueue $loggingModel) { $commands = collect([]); $workDir = generateWorkdir($deploymentUuid, $application); - if (count($application->ports_mappings_array) > 0) { - // $this->execute_remote_command( - // [ - // "echo '\n----------------------------------------'", - // ], - // ["echo -n 'Application has ports mapped to the host system, rolling update is not supported.'"], - // ); - // $this->stop_running_container(force: true); - // $this->start_by_compose_file(); + if ($application->build_pack === 'dockerimage') { + $loggingModel->addLogEntry('Pulling latest images from the registry.'); + $commands->push( + [ + "command" => executeInDocker($deploymentUuid, "docker compose --project-directory {$workDir} pull"), + "hidden" => true + ], + [ + "command" => executeInDocker($deploymentUuid, "docker compose --project-directory {$workDir} up --build -d"), + "hidden" => true + ], + ); } else { $commands->push( [ - "command" => "echo '\n----------------------------------------'" + "command" => executeInDocker($deploymentUuid, "docker compose --project-directory {$workDir} up --build -d"), + "hidden" => true ], - [ - "command" => "echo -n 'Rolling update started.'" - ] ); - if ($application->build_pack === 'dockerimage') { - $commands->push( - ["echo -n 'Pulling latest images from the registry.'"], - [executeInDocker($deploymentUuid, "docker compose --project-directory {$workDir} pull"), "hidden" => true], - [executeInDocker($deploymentUuid, "docker compose --project-directory {$workDir} up --build -d"), "hidden" => true], - ); - } else { - $commands->push( - [executeInDocker($deploymentUuid, "docker compose --project-directory {$workDir} up --build -d"), "hidden" => true], - ); - } - return $commands; } + return $commands; +} +function removeOldDeployment(string $containerName) +{ + $commands = collect([]); + $commands->push( + ["docker rm -f $containerName >/dev/null 2>&1"], + ); + return $commands; } diff --git a/bootstrap/helpers/constants.php b/bootstrap/helpers/constants.php index 299d3acb9..380168005 100644 --- a/bootstrap/helpers/constants.php +++ b/bootstrap/helpers/constants.php @@ -28,7 +28,9 @@ const SPECIFIC_SERVICES = [ 'quay.io/minio/minio', ]; +// Based on /etc/os-release const SUPPORTED_OS = [ - 'debian', - 'rhel centos fedora' + 'ubuntu debian raspbian', + 'centos fedora rhel ol rocky', + 'sles opensuse-leap opensuse-tumbleweed' ]; diff --git a/bootstrap/helpers/databases.php b/bootstrap/helpers/databases.php index 007c414bd..cef5ac7fd 100644 --- a/bootstrap/helpers/databases.php +++ b/bootstrap/helpers/databases.php @@ -24,7 +24,7 @@ function create_standalone_postgresql($environment_id, $destination_uuid): Stand } return StandalonePostgresql::create([ 'name' => generate_database_name('postgresql'), - 'postgres_password' => \Illuminate\Support\Str::password(symbols: false), + 'postgres_password' => \Illuminate\Support\Str::password(length: 64, symbols: false), 'environment_id' => $environment_id, 'destination_id' => $destination->id, 'destination_type' => $destination->getMorphClass(), @@ -39,7 +39,7 @@ function create_standalone_redis($environment_id, $destination_uuid): Standalone } return StandaloneRedis::create([ 'name' => generate_database_name('redis'), - 'redis_password' => \Illuminate\Support\Str::password(symbols: false), + 'redis_password' => \Illuminate\Support\Str::password(length: 64, symbols: false), 'environment_id' => $environment_id, 'destination_id' => $destination->id, 'destination_type' => $destination->getMorphClass(), @@ -54,7 +54,7 @@ function create_standalone_mongodb($environment_id, $destination_uuid): Standalo } return StandaloneMongodb::create([ 'name' => generate_database_name('mongodb'), - 'mongo_initdb_root_password' => \Illuminate\Support\Str::password(symbols: false), + 'mongo_initdb_root_password' => \Illuminate\Support\Str::password(length: 64, symbols: false), 'environment_id' => $environment_id, 'destination_id' => $destination->id, 'destination_type' => $destination->getMorphClass(), @@ -68,8 +68,8 @@ function create_standalone_mysql($environment_id, $destination_uuid): Standalone } return StandaloneMysql::create([ 'name' => generate_database_name('mysql'), - 'mysql_root_password' => \Illuminate\Support\Str::password(symbols: false), - 'mysql_password' => \Illuminate\Support\Str::password(symbols: false), + 'mysql_root_password' => \Illuminate\Support\Str::password(length: 64, symbols: false), + 'mysql_password' => \Illuminate\Support\Str::password(length: 64, symbols: false), 'environment_id' => $environment_id, 'destination_id' => $destination->id, 'destination_type' => $destination->getMorphClass(), @@ -83,8 +83,8 @@ function create_standalone_mariadb($environment_id, $destination_uuid): Standalo } return StandaloneMariadb::create([ 'name' => generate_database_name('mariadb'), - 'mariadb_root_password' => \Illuminate\Support\Str::password(symbols: false), - 'mariadb_password' => \Illuminate\Support\Str::password(symbols: false), + 'mariadb_root_password' => \Illuminate\Support\Str::password(length: 64, symbols: false), + 'mariadb_password' => \Illuminate\Support\Str::password(length: 64, symbols: false), 'environment_id' => $environment_id, 'destination_id' => $destination->id, 'destination_type' => $destination->getMorphClass(), diff --git a/bootstrap/helpers/docker.php b/bootstrap/helpers/docker.php index 36f8733b3..030aa6aa7 100644 --- a/bootstrap/helpers/docker.php +++ b/bootstrap/helpers/docker.php @@ -3,6 +3,9 @@ use App\Models\Application; use App\Models\ApplicationPreview; use App\Models\Server; +use App\Models\Service; +use App\Models\ServiceApplication; +use App\Models\ServiceDatabase; use Illuminate\Support\Collection; use Illuminate\Support\Str; use Spatie\Url\Url; @@ -137,18 +140,28 @@ function defaultLabels($id, $name, $pull_request_id = 0, string $type = 'applica $labels->push('coolify.name=' . $name); $labels->push('coolify.pullRequestId=' . $pull_request_id); if ($type === 'service') { - $labels->push('coolify.service.subId=' . $subId); - $labels->push('coolify.service.subType=' . $subType); + $subId && $labels->push('coolify.service.subId=' . $subId); + $subType && $labels->push('coolify.service.subType=' . $subType); } return $labels; } -function generateServiceSpecificFqdns($service, $forTraefik = false) +function generateServiceSpecificFqdns(ServiceApplication|Application $resource, $forTraefik = false) { - $variables = collect($service->service->environment_variables); - $type = $service->serviceType(); + if ($resource->getMorphClass() === 'App\Models\ServiceApplication') { + $uuid = $resource->uuid; + $server = $resource->service->server; + $environment_variables = $resource->service->environment_variables; + $type = $resource->serviceType(); + } else if ($resource->getMorphClass() === 'App\Models\Application') { + $uuid = $resource->uuid; + $server = $resource->destination->server; + $environment_variables = $resource->environment_variables; + $type = $resource->serviceType(); + } + $variables = collect($environment_variables); $payload = collect([]); switch ($type) { - case $type->contains('minio'): + case $type?->contains('minio'): $MINIO_BROWSER_REDIRECT_URL = $variables->where('key', 'MINIO_BROWSER_REDIRECT_URL')->first(); $MINIO_SERVER_URL = $variables->where('key', 'MINIO_SERVER_URL')->first(); if (is_null($MINIO_BROWSER_REDIRECT_URL) || is_null($MINIO_SERVER_URL)) { @@ -156,12 +169,12 @@ function generateServiceSpecificFqdns($service, $forTraefik = false) } if (is_null($MINIO_BROWSER_REDIRECT_URL?->value)) { $MINIO_BROWSER_REDIRECT_URL?->update([ - "value" => generateFqdn($service->service->server, 'console-' . $service->uuid) + "value" => generateFqdn($server, 'console-' . $uuid) ]); } if (is_null($MINIO_SERVER_URL?->value)) { $MINIO_SERVER_URL?->update([ - "value" => generateFqdn($service->service->server, 'minio-' . $service->uuid) + "value" => generateFqdn($server, 'minio-' . $uuid) ]); } if ($forTraefik) { @@ -175,10 +188,11 @@ function generateServiceSpecificFqdns($service, $forTraefik = false) $MINIO_SERVER_URL->value, ]); } + break; } return $payload; } -function fqdnLabelsForTraefik(string $uuid, Collection $domains, bool $is_force_https_enabled, $onlyPort = null) +function fqdnLabelsForTraefik(string $uuid, Collection $domains, bool $is_force_https_enabled = false, $onlyPort = null) { $labels = collect([]); $labels->push('traefik.enable=true'); @@ -260,3 +274,18 @@ function generateLabelsApplication(Application $application, ?ApplicationPreview } return $labels->all(); } + +function isDatabaseImage(string $image) +{ + $image = str($image); + if ($image->contains(':')) { + $image = str($image); + } else { + $image = str($image)->append(':latest'); + } + $imageName = $image->before(':'); + if (collect(DATABASE_DOCKER_IMAGES)->contains($imageName)) { + return true; + } + return false; +} diff --git a/bootstrap/helpers/proxy.php b/bootstrap/helpers/proxy.php index ee9c624f5..2f595d1c6 100644 --- a/bootstrap/helpers/proxy.php +++ b/bootstrap/helpers/proxy.php @@ -1,6 +1,7 @@ standaloneDockers)->map(function ($docker) { return $docker['network']; - })->unique(); + }); + // Service networks + foreach ($server->services()->get() as $service) { + $networks->push($service->networks()); + } + // Docker compose based apps + $docker_compose_apps = $server->dockerComposeBasedApplications(); + foreach ($docker_compose_apps as $app) { + $networks->push($app->uuid); + } + // Docker compose based preview deployments + $docker_compose_previews = $server->dockerComposeBasedPreviewDeployments(); + foreach ($docker_compose_previews as $preview) { + $pullRequestId = $preview->pull_request_id; + $applicationId = $preview->application_id; + $application = Application::find($applicationId); + if (!$application) { + continue; + } + $network = "{$application->uuid}-{$pullRequestId}"; + $networks->push($network); + } + $networks = collect($networks)->flatten()->unique(); if ($networks->count() === 0) { $networks = collect(['coolify']); } diff --git a/bootstrap/helpers/remoteProcess.php b/bootstrap/helpers/remoteProcess.php index 948e47329..45c8ae0e5 100644 --- a/bootstrap/helpers/remoteProcess.php +++ b/bootstrap/helpers/remoteProcess.php @@ -151,6 +151,7 @@ function decode_remote_command_output(?ApplicationDeploymentQueue $application_d if (is_null($application_deployment_queue)) { return collect([]); } + // ray(data_get($application_deployment_queue, 'logs')); try { $decoded = json_decode( data_get($application_deployment_queue, 'logs'), @@ -160,14 +161,15 @@ function decode_remote_command_output(?ApplicationDeploymentQueue $application_d } catch (\JsonException $exception) { return collect([]); } + // ray($decoded ); $formatted = collect($decoded); if (!$is_debug_enabled) { $formatted = $formatted->filter(fn ($i) => $i['hidden'] === false ?? false); } $formatted = $formatted - ->sortBy(fn ($i) => $i['order']) + ->sortBy(fn ($i) => data_get($i, 'order')) ->map(function ($i) { - $i['timestamp'] = Carbon::parse($i['timestamp'])->format('Y-M-d H:i:s.u'); + data_set($i, 'timestamp', Carbon::parse(data_get($i, 'timestamp'))->format('Y-M-d H:i:s.u')); return $i; }); return $formatted; diff --git a/bootstrap/helpers/shared.php b/bootstrap/helpers/shared.php index 071c252ff..8eb1fd443 100644 --- a/bootstrap/helpers/shared.php +++ b/bootstrap/helpers/shared.php @@ -1,9 +1,15 @@ notify(new GeneralNotification("👀 {$baseUrl}: " . $message)); + $team?->notify(new GeneralNotification("👀 {$baseUrl}: " . $message)); ray("👀 {$baseUrl}: " . $message); } catch (\Throwable $e) { ray($e->getMessage()); @@ -472,7 +479,8 @@ function generateDeployWebhook($resource) $url = $api . $endpoint . "?uuid=$uuid&force=false"; return $url; } -function generateGitManualWebhook($resource, $type) { +function generateGitManualWebhook($resource, $type) +{ if ($resource->source_id !== 0 && !is_null($resource->source_id)) { return null; } @@ -487,3 +495,993 @@ function removeAnsiColors($text) { return preg_replace('/\e[[][A-Za-z0-9];?[0-9]*m?/', '', $text); } + +function getTopLevelNetworks(Service|Application $resource) +{ + if ($resource->getMorphClass() === 'App\Models\Service') { + if ($resource->docker_compose_raw) { + try { + $yaml = Yaml::parse($resource->docker_compose_raw); + } catch (\Exception $e) { + throw new \Exception($e->getMessage()); + } + $services = data_get($yaml, 'services'); + $topLevelNetworks = collect(data_get($yaml, 'networks', [])); + $definedNetwork = collect([$resource->uuid]); + $services = collect($services)->map(function ($service, $_) use ($topLevelNetworks, $definedNetwork) { + $serviceNetworks = collect(data_get($service, 'networks', [])); + + // Collect/create/update networks + if ($serviceNetworks->count() > 0) { + foreach ($serviceNetworks as $networkName => $networkDetails) { + $networkExists = $topLevelNetworks->contains(function ($value, $key) use ($networkName) { + return $value == $networkName || $key == $networkName; + }); + if (!$networkExists) { + $topLevelNetworks->put($networkDetails, null); + } + } + } + + $definedNetworkExists = $topLevelNetworks->contains(function ($value, $_) use ($definedNetwork) { + return $value == $definedNetwork; + }); + if (!$definedNetworkExists) { + foreach ($definedNetwork as $network) { + $topLevelNetworks->put($network, [ + 'name' => $network, + 'external' => true + ]); + } + } + + return $service; + }); + return $topLevelNetworks->keys(); + } + } else if ($resource->getMorphClass() === 'App\Models\Application') { + try { + $yaml = Yaml::parse($resource->docker_compose_raw); + } catch (\Exception $e) { + throw new \Exception($e->getMessage()); + } + $server = $resource->destination->server; + $topLevelNetworks = collect(data_get($yaml, 'networks', [])); + $services = data_get($yaml, 'services'); + $definedNetwork = collect([$resource->uuid]); + $services = collect($services)->map(function ($service, $_) use ($topLevelNetworks, $definedNetwork) { + $serviceNetworks = collect(data_get($service, 'networks', [])); + + // Collect/create/update networks + if ($serviceNetworks->count() > 0) { + foreach ($serviceNetworks as $networkName => $networkDetails) { + $networkExists = $topLevelNetworks->contains(function ($value, $key) use ($networkName) { + return $value == $networkName || $key == $networkName; + }); + if (!$networkExists) { + $topLevelNetworks->put($networkDetails, null); + } + } + } + $definedNetworkExists = $topLevelNetworks->contains(function ($value, $_) use ($definedNetwork) { + return $value == $definedNetwork; + }); + if (!$definedNetworkExists) { + foreach ($definedNetwork as $network) { + $topLevelNetworks->put($network, [ + 'name' => $network, + 'external' => true + ]); + } + } + return $service; + }); + return $topLevelNetworks->keys(); + } +} +function parseDockerComposeFile(Service|Application $resource, bool $isNew = false, int $pull_request_id = 0, bool $is_pr = false) +{ + // ray()->clearAll(); + if ($resource->getMorphClass() === 'App\Models\Service') { + if ($resource->docker_compose_raw) { + try { + $yaml = Yaml::parse($resource->docker_compose_raw); + } catch (\Exception $e) { + throw new \Exception($e->getMessage()); + } + + $topLevelVolumes = collect(data_get($yaml, 'volumes', [])); + $topLevelNetworks = collect(data_get($yaml, 'networks', [])); + $dockerComposeVersion = data_get($yaml, 'version') ?? '3.8'; + $services = data_get($yaml, 'services'); + + $generatedServiceFQDNS = collect([]); + if (is_null($resource->destination)) { + $destination = $resource->server->destinations()->first(); + if ($destination) { + $resource->destination()->associate($destination); + $resource->save(); + } + } + $definedNetwork = collect([$resource->uuid]); + + $services = collect($services)->map(function ($service, $serviceName) use ($topLevelVolumes, $topLevelNetworks, $definedNetwork, $isNew, $generatedServiceFQDNS, $resource) { + $serviceVolumes = collect(data_get($service, 'volumes', [])); + $servicePorts = collect(data_get($service, 'ports', [])); + $serviceNetworks = collect(data_get($service, 'networks', [])); + $serviceVariables = collect(data_get($service, 'environment', [])); + $serviceLabels = collect(data_get($service, 'labels', [])); + if ($serviceLabels->count() > 0) { + $removedLabels = collect([]); + $serviceLabels = $serviceLabels->filter(function ($serviceLabel, $serviceLabelName) use ($removedLabels) { + if (!str($serviceLabel)->contains('=')) { + $removedLabels->put($serviceLabelName, $serviceLabel); + return false; + } + return $serviceLabel; + }); + foreach ($removedLabels as $removedLabelName => $removedLabel) { + $serviceLabels->push("$removedLabelName=$removedLabel"); + } + } + + $containerName = "$serviceName-{$resource->uuid}"; + + // Decide if the service is a database + $isDatabase = isDatabaseImage(data_get_str($service, 'image')); + $image = data_get_str($service, 'image'); + data_set($service, 'is_database', $isDatabase); + + // Create new serviceApplication or serviceDatabase + if ($isDatabase) { + if ($isNew) { + $savedService = ServiceDatabase::create([ + 'name' => $serviceName, + 'image' => $image, + 'service_id' => $resource->id + ]); + } else { + $savedService = ServiceDatabase::where([ + 'name' => $serviceName, + 'service_id' => $resource->id + ])->first(); + } + } else { + if ($isNew) { + $savedService = ServiceApplication::create([ + 'name' => $serviceName, + 'image' => $image, + 'service_id' => $resource->id + ]); + } else { + $savedService = ServiceApplication::where([ + 'name' => $serviceName, + 'service_id' => $resource->id + ])->first(); + } + } + if (is_null($savedService)) { + if ($isDatabase) { + $savedService = ServiceDatabase::create([ + 'name' => $serviceName, + 'image' => $image, + 'service_id' => $resource->id + ]); + } else { + $savedService = ServiceApplication::create([ + 'name' => $serviceName, + 'image' => $image, + 'service_id' => $resource->id + ]); + } + } + + // Check if image changed + if ($savedService->image !== $image) { + $savedService->image = $image; + $savedService->save(); + } + + // Collect/create/update networks + if ($serviceNetworks->count() > 0) { + foreach ($serviceNetworks as $networkName => $networkDetails) { + $networkExists = $topLevelNetworks->contains(function ($value, $key) use ($networkName) { + return $value == $networkName || $key == $networkName; + }); + if (!$networkExists) { + $topLevelNetworks->put($networkDetails, null); + } + } + } + + // Collect/create/update ports + $collectedPorts = collect([]); + if ($servicePorts->count() > 0) { + foreach ($servicePorts as $sport) { + if (is_string($sport) || is_numeric($sport)) { + $collectedPorts->push($sport); + } + if (is_array($sport)) { + $target = data_get($sport, 'target'); + $published = data_get($sport, 'published'); + $protocol = data_get($sport, 'protocol'); + $collectedPorts->push("$target:$published/$protocol"); + } + } + } + $savedService->ports = $collectedPorts->implode(','); + $savedService->save(); + + // Add Coolify specific networks + $definedNetworkExists = $topLevelNetworks->contains(function ($value, $_) use ($definedNetwork) { + return $value == $definedNetwork; + }); + if (!$definedNetworkExists) { + foreach ($definedNetwork as $network) { + $topLevelNetworks->put($network, [ + 'name' => $network, + 'external' => true + ]); + } + } + $networks = collect(); + foreach ($serviceNetworks as $key => $serviceNetwork) { + if (gettype($serviceNetwork) === 'string') { + // networks: + // - appwrite + $networks->put($serviceNetwork, null); + } else if (gettype($serviceNetwork) === 'array') { + // networks: + // default: + // ipv4_address: 192.168.203.254 + // $networks->put($serviceNetwork, null); + ray($key); + $networks->put($key, $serviceNetwork); + } + } + foreach ($definedNetwork as $key => $network) { + $networks->put($network, null); + } + data_set($service, 'networks', $networks->toArray()); + + // Collect/create/update volumes + if ($serviceVolumes->count() > 0) { + $serviceVolumes = $serviceVolumes->map(function ($volume) use ($savedService, $topLevelVolumes) { + $type = null; + $source = null; + $target = null; + $content = null; + $isDirectory = false; + if (is_string($volume)) { + $source = Str::of($volume)->before(':'); + $target = Str::of($volume)->after(':')->beforeLast(':'); + if ($source->startsWith('./') || $source->startsWith('/') || $source->startsWith('~')) { + $type = Str::of('bind'); + } else { + $type = Str::of('volume'); + } + } else if (is_array($volume)) { + $type = data_get_str($volume, 'type'); + $source = data_get_str($volume, 'source'); + $target = data_get_str($volume, 'target'); + $content = data_get($volume, 'content'); + $isDirectory = (bool) data_get($volume, 'isDirectory', false); + $foundConfig = $savedService->fileStorages()->whereMountPath($target)->first(); + if ($foundConfig) { + $contentNotNull = data_get($foundConfig, 'content'); + if ($contentNotNull) { + $content = $contentNotNull; + } + $isDirectory = (bool) data_get($foundConfig, 'is_directory'); + } + } + if ($type->value() === 'bind') { + if ($source->value() === "/var/run/docker.sock") { + return $volume; + } + if ($source->value() === '/tmp' || $source->value() === '/tmp/') { + return $volume; + } + LocalFileVolume::updateOrCreate( + [ + 'mount_path' => $target, + 'resource_id' => $savedService->id, + 'resource_type' => get_class($savedService) + ], + [ + 'fs_path' => $source, + 'mount_path' => $target, + 'content' => $content, + 'is_directory' => $isDirectory, + 'resource_id' => $savedService->id, + 'resource_type' => get_class($savedService) + ] + ); + } else if ($type->value() === 'volume') { + $slugWithoutUuid = Str::slug($source, '-'); + $name = "{$savedService->service->uuid}_{$slugWithoutUuid}"; + if (is_string($volume)) { + $source = Str::of($volume)->before(':'); + $target = Str::of($volume)->after(':')->beforeLast(':'); + $source = $name; + $volume = "$source:$target"; + } else if (is_array($volume)) { + data_set($volume, 'source', $name); + } + $topLevelVolumes->put($name, [ + 'name' => $name, + ]); + LocalPersistentVolume::updateOrCreate( + [ + 'mount_path' => $target, + 'resource_id' => $savedService->id, + 'resource_type' => get_class($savedService) + ], + [ + 'name' => $name, + 'mount_path' => $target, + 'resource_id' => $savedService->id, + 'resource_type' => get_class($savedService) + ] + ); + } + $savedService->getFilesFromServer(isInit: true); + return $volume; + }); + data_set($service, 'volumes', $serviceVolumes->toArray()); + } + + // Add env_file with at least .env to the service + // $envFile = collect(data_get($service, 'env_file', [])); + // if ($envFile->count() > 0) { + // if (!$envFile->contains('.env')) { + // $envFile->push('.env'); + // } + // } else { + // $envFile = collect(['.env']); + // } + // data_set($service, 'env_file', $envFile->toArray()); + + + // Get variables from the service + foreach ($serviceVariables as $variableName => $variable) { + if (is_numeric($variableName)) { + $variable = Str::of($variable); + if ($variable->contains('=')) { + // - SESSION_SECRET=123 + // - SESSION_SECRET= + $key = $variable->before('='); + $value = $variable->after('='); + } else { + // - SESSION_SECRET + $key = $variable; + $value = null; + } + } else { + // SESSION_SECRET: 123 + // SESSION_SECRET: + $key = Str::of($variableName); + $value = Str::of($variable); + } + // TODO: here is the problem + if ($key->startsWith('SERVICE_FQDN')) { + if ($isNew || $savedService->fqdn === null) { + $name = $key->after('SERVICE_FQDN_')->beforeLast('_')->lower(); + $fqdn = generateFqdn($resource->server, "{$name->value()}-{$resource->uuid}"); + if (substr_count($key->value(), '_') === 3) { + // SERVICE_FQDN_UMAMI_1000 + $port = $key->afterLast('_'); + } else { + // SERVICE_FQDN_UMAMI + $port = null; + } + if ($port) { + $fqdn = "$fqdn:$port"; + } + if (substr_count($key->value(), '_') >= 2) { + if (is_null($value)) { + $value = Str::of('/'); + } + $path = $value->value(); + if ($generatedServiceFQDNS->count() > 0) { + $alreadyGenerated = $generatedServiceFQDNS->has($key->value()); + if ($alreadyGenerated) { + $fqdn = $generatedServiceFQDNS->get($key->value()); + } else { + $generatedServiceFQDNS->put($key->value(), $fqdn); + } + } else { + $generatedServiceFQDNS->put($key->value(), $fqdn); + } + $fqdn = "$fqdn$path"; + } + + if (!$isDatabase) { + if ($savedService->fqdn) { + $fqdn = $savedService->fqdn . ',' . $fqdn; + } else { + $fqdn = $fqdn; + } + $savedService->fqdn = $fqdn; + $savedService->save(); + } + } + // data_forget($service, "environment.$variableName"); + // $yaml = data_forget($yaml, "services.$serviceName.environment.$variableName"); + // if (count(data_get($yaml, 'services.' . $serviceName . '.environment')) === 0) { + // $yaml = data_forget($yaml, "services.$serviceName.environment"); + // } + continue; + } + if ($value?->startsWith('$')) { + $value = Str::of(replaceVariables($value)); + $key = $value; + $foundEnv = EnvironmentVariable::where([ + 'key' => $key, + 'service_id' => $resource->id, + ])->first(); + if ($value->startsWith('SERVICE_')) { + ['command' => $command, 'forService' => $forService, 'generatedValue' => $generatedValue, 'port' => $port] = parseEnvVariable($value); + if ($command->value() === 'FQDN' || $command->value() === 'URL') { + if (Str::lower($forService) === $serviceName) { + $fqdn = generateFqdn($resource->server, $containerName); + } else { + $fqdn = generateFqdn($resource->server, Str::lower($forService) . '-' . $resource->uuid); + } + if ($port) { + $fqdn = "$fqdn:$port"; + } + if ($foundEnv) { + $fqdn = data_get($foundEnv, 'value'); + } else { + if ($command->value() === 'URL') { + $fqdn = Str::of($fqdn)->after('://')->value(); + } + EnvironmentVariable::create([ + 'key' => $key, + 'value' => $fqdn, + 'is_build_time' => false, + 'service_id' => $resource->id, + 'is_preview' => false, + ]); + } + if (!$isDatabase) { + if ($command->value() === 'FQDN' && is_null($savedService->fqdn)) { + $savedService->fqdn = $fqdn; + $savedService->save(); + } + } + } else { + $generatedValue = generateEnvValue($command); + if (!$foundEnv) { + EnvironmentVariable::create([ + 'key' => $key, + 'value' => $generatedValue, + 'is_build_time' => false, + 'service_id' => $resource->id, + 'is_preview' => false, + ]); + } + } + } else { + if ($value->contains(':-')) { + $key = $value->before(':'); + $defaultValue = $value->after(':-'); + } else if ($value->contains('-')) { + $key = $value->before('-'); + $defaultValue = $value->after('-'); + } else if ($value->contains(':?')) { + $key = $value->before(':'); + $defaultValue = $value->after(':?'); + } else if ($value->contains('?')) { + $key = $value->before('?'); + $defaultValue = $value->after('?'); + } else { + $key = $value; + $defaultValue = null; + } + if ($foundEnv) { + $defaultValue = data_get($foundEnv, 'value'); + } + EnvironmentVariable::updateOrCreate([ + 'key' => $key, + 'service_id' => $resource->id, + ], [ + 'value' => $defaultValue, + 'is_build_time' => false, + 'service_id' => $resource->id, + 'is_preview' => false, + ]); + } + } + } + + // Add labels to the service + if ($savedService->serviceType()) { + $fqdns = generateServiceSpecificFqdns($savedService, forTraefik: true); + } else { + $fqdns = collect(data_get($savedService, 'fqdns')); + } + $defaultLabels = defaultLabels($resource->id, $containerName, type: 'service', subType: $isDatabase ? 'database' : 'application', subId: $savedService->id); + $serviceLabels = $serviceLabels->merge($defaultLabels); + if (!$isDatabase && $fqdns->count() > 0) { + if ($fqdns) { + $serviceLabels = $serviceLabels->merge(fqdnLabelsForTraefik($resource->uuid, $fqdns, true)); + } + } + if ($resource->server->isLogDrainEnabled() && $savedService->isLogDrainEnabled()) { + data_set($service, 'logging', [ + 'driver' => 'fluentd', + 'options' => [ + 'fluentd-address' => "tcp://127.0.0.1:24224", + 'fluentd-async' => "true", + 'fluentd-sub-second-precision' => "true", + ] + ]); + } + data_set($service, 'labels', $serviceLabels->toArray()); + data_forget($service, 'is_database'); + data_set($service, 'restart', RESTART_MODE); + data_set($service, 'container_name', $containerName); + data_forget($service, 'volumes.*.content'); + data_forget($service, 'volumes.*.isDirectory'); + // Remove unnecessary variables from service.environment + // $withoutServiceEnvs = collect([]); + // collect(data_get($service, 'environment'))->each(function ($value, $key) use ($withoutServiceEnvs) { + // ray($key, $value); + // if (!Str::of($key)->startsWith('$SERVICE_') && !Str::of($value)->startsWith('SERVICE_')) { + // $k = Str::of($value)->before("="); + // $v = Str::of($value)->after("="); + // $withoutServiceEnvs->put($k->value(), $v->value()); + // } + // }); + // ray($withoutServiceEnvs); + // data_set($service, 'environment', $withoutServiceEnvs->toArray()); + return $service; + }); + $finalServices = [ + 'version' => $dockerComposeVersion, + 'services' => $services->toArray(), + 'volumes' => $topLevelVolumes->toArray(), + 'networks' => $topLevelNetworks->toArray(), + ]; + $resource->docker_compose_raw = Yaml::dump($yaml, 10, 2); + $resource->docker_compose = Yaml::dump($finalServices, 10, 2); + $resource->save(); + $resource->saveComposeConfigs(); + return collect($finalServices); + } else { + return collect([]); + } + } else if ($resource->getMorphClass() === 'App\Models\Application') { + $isSameDockerComposeFile = false; + if ($resource->dockerComposePrLocation() === $resource->dockerComposeLocation()) { + $isSameDockerComposeFile = true; + $is_pr = false; + } + if ($is_pr) { + try { + $yaml = Yaml::parse($resource->docker_compose_pr_raw); + } catch (\Exception $e) { + throw new \Exception($e->getMessage()); + } + } else { + try { + $yaml = Yaml::parse($resource->docker_compose_raw); + } catch (\Exception $e) { + throw new \Exception($e->getMessage()); + } + } + $server = $resource->destination->server; + $topLevelVolumes = collect(data_get($yaml, 'volumes', [])); + if ($pull_request_id !== 0) { + $topLevelVolumes = collect([]); + } + $topLevelNetworks = collect(data_get($yaml, 'networks', [])); + $dockerComposeVersion = data_get($yaml, 'version') ?? '3.8'; + $services = data_get($yaml, 'services'); + + $generatedServiceFQDNS = collect([]); + if (is_null($resource->destination)) { + $destination = $server->destinations()->first(); + if ($destination) { + $resource->destination()->associate($destination); + $resource->save(); + } + } + $definedNetwork = collect([$resource->uuid]); + if ($pull_request_id !== 0) { + $definedNetwork = collect(["{$resource->uuid}-$pull_request_id"]); + } + $services = collect($services)->map(function ($service, $serviceName) use ($topLevelVolumes, $topLevelNetworks, $definedNetwork, $isNew, $generatedServiceFQDNS, $resource, $server, $pull_request_id) { + $serviceVolumes = collect(data_get($service, 'volumes', [])); + $servicePorts = collect(data_get($service, 'ports', [])); + $serviceNetworks = collect(data_get($service, 'networks', [])); + $serviceVariables = collect(data_get($service, 'environment', [])); + $serviceLabels = collect(data_get($service, 'labels', [])); + if ($serviceLabels->count() > 0) { + $removedLabels = collect([]); + $serviceLabels = $serviceLabels->filter(function ($serviceLabel, $serviceLabelName) use ($removedLabels) { + if (!str($serviceLabel)->contains('=')) { + $removedLabels->put($serviceLabelName, $serviceLabel); + return false; + } + return $serviceLabel; + }); + foreach ($removedLabels as $removedLabelName => $removedLabel) { + $serviceLabels->push("$removedLabelName=$removedLabel"); + } + } + $baseName = generateApplicationContainerName($resource, $pull_request_id); + $containerName = "$serviceName-$baseName"; + if ($pull_request_id !== 0) { + if (count($serviceVolumes) > 0) { + $serviceVolumes = $serviceVolumes->map(function ($volume) use ($resource, $pull_request_id, $topLevelVolumes) { + if (is_string($volume)) { + $volume = str($volume); + if ($volume->contains(':')) { + $name = $volume->before(':'); + $mount = $volume->after(':'); + $newName = $resource->uuid . "-{$name}-pr-$pull_request_id"; + $volume = str("$newName:$mount"); + $topLevelVolumes->put($newName, [ + 'name' => $newName, + ]); + } + } else if (is_array($volume)) { + $source = data_get($volume, 'source'); + if ($source) { + $newSource = $resource->uuid . "-{$source}-pr-$pull_request_id"; + data_set($volume, 'source', $newSource); + $topLevelVolumes->put($newSource, [ + 'name' => $newSource, + ]); + } + } + + + return $volume->value(); + }); + data_set($service, 'volumes', $serviceVolumes->toArray()); + } + } else { + } + // Decide if the service is a database + $isDatabase = isDatabaseImage(data_get_str($service, 'image')); + data_set($service, 'is_database', $isDatabase); + + // Collect/create/update networks + if ($serviceNetworks->count() > 0) { + foreach ($serviceNetworks as $networkName => $networkDetails) { + ray($networkDetails); + $networkExists = $topLevelNetworks->contains(function ($value, $key) use ($networkName) { + return $value == $networkName || $key == $networkName; + }); + if (!$networkExists) { + $topLevelNetworks->put($networkDetails, null); + } + } + } + // Collect/create/update ports + $collectedPorts = collect([]); + if ($servicePorts->count() > 0) { + foreach ($servicePorts as $sport) { + if (is_string($sport) || is_numeric($sport)) { + $collectedPorts->push($sport); + } + if (is_array($sport)) { + $target = data_get($sport, 'target'); + $published = data_get($sport, 'published'); + $protocol = data_get($sport, 'protocol'); + $collectedPorts->push("$target:$published/$protocol"); + } + } + } + if ($collectedPorts->count() > 0) { + // ray($collectedPorts->implode(',')); + } + $definedNetworkExists = $topLevelNetworks->contains(function ($value, $_) use ($definedNetwork) { + return $value == $definedNetwork; + }); + if (!$definedNetworkExists) { + foreach ($definedNetwork as $network) { + if ($pull_request_id !== 0) { + $topLevelNetworks->put($network, [ + 'name' => $network, + 'external' => true + ]); + } else { + $topLevelNetworks->put($network, [ + 'name' => $network, + 'external' => true + ]); + } + } + } + $networks = collect(); + foreach ($serviceNetworks as $key => $serviceNetwork) { + if (gettype($serviceNetwork) === 'string') { + // networks: + // - appwrite + $networks->put($serviceNetwork, null); + } else if (gettype($serviceNetwork) === 'array') { + // networks: + // default: + // ipv4_address: 192.168.203.254 + // $networks->put($serviceNetwork, null); + $networks->put($key, $serviceNetwork); + } + } + foreach ($definedNetwork as $key => $network) { + $networks->put($network, null); + } + data_set($service, 'networks', $networks->toArray()); + // Get variables from the service + foreach ($serviceVariables as $variableName => $variable) { + if (is_numeric($variableName)) { + $variable = Str::of($variable); + if ($variable->contains('=')) { + // - SESSION_SECRET=123 + // - SESSION_SECRET= + $key = $variable->before('='); + $value = $variable->after('='); + } else { + // - SESSION_SECRET + $key = $variable; + $value = null; + } + } else { + // SESSION_SECRET: 123 + // SESSION_SECRET: + $key = Str::of($variableName); + $value = Str::of($variable); + } + if ($key->startsWith('SERVICE_FQDN')) { + if ($isNew) { + $name = $key->after('SERVICE_FQDN_')->beforeLast('_')->lower(); + $fqdn = generateFqdn($server, "{$name->value()}-{$resource->uuid}"); + if (substr_count($key->value(), '_') === 3) { + // SERVICE_FQDN_UMAMI_1000 + $port = $key->afterLast('_'); + } else { + // SERVICE_FQDN_UMAMI + $port = null; + } + if ($port) { + $fqdn = "$fqdn:$port"; + } + if (substr_count($key->value(), '_') >= 2) { + if (is_null($value)) { + $value = Str::of('/'); + } + $path = $value->value(); + if ($generatedServiceFQDNS->count() > 0) { + $alreadyGenerated = $generatedServiceFQDNS->has($key->value()); + if ($alreadyGenerated) { + $fqdn = $generatedServiceFQDNS->get($key->value()); + } else { + $generatedServiceFQDNS->put($key->value(), $fqdn); + } + } else { + $generatedServiceFQDNS->put($key->value(), $fqdn); + } + $fqdn = "$fqdn$path"; + } + } + continue; + } + if ($value?->startsWith('$')) { + $value = Str::of(replaceVariables($value)); + $key = $value; + $foundEnv = EnvironmentVariable::where([ + 'key' => $key, + 'application_id' => $resource->id, + ])->first(); + if ($value->startsWith('SERVICE_')) { + ['command' => $command, 'forService' => $forService, 'generatedValue' => $generatedValue, 'port' => $port] = parseEnvVariable($value); + if ($command->value() === 'FQDN' || $command->value() === 'URL') { + if (Str::lower($forService) === $serviceName) { + $fqdn = generateFqdn($server, $containerName); + } else { + $fqdn = generateFqdn($server, Str::lower($forService) . '-' . $resource->uuid); + } + if ($port) { + $fqdn = "$fqdn:$port"; + } + if ($foundEnv) { + $fqdn = data_get($foundEnv, 'value'); + } else { + if ($command->value() === 'URL') { + $fqdn = Str::of($fqdn)->after('://')->value(); + } + EnvironmentVariable::create([ + 'key' => $key, + 'value' => $fqdn, + 'is_build_time' => false, + 'application_id' => $resource->id, + 'is_preview' => false, + ]); + } + } else { + $generatedValue = generateEnvValue($command); + if (!$foundEnv) { + EnvironmentVariable::create([ + 'key' => $key, + 'value' => $generatedValue, + 'is_build_time' => false, + 'application_id' => $resource->id, + 'is_preview' => false, + ]); + } + } + } else { + if ($value->contains(':-')) { + $key = $value->before(':'); + $defaultValue = $value->after(':-'); + } else if ($value->contains('-')) { + $key = $value->before('-'); + $defaultValue = $value->after('-'); + } else if ($value->contains(':?')) { + $key = $value->before(':'); + $defaultValue = $value->after(':?'); + } else if ($value->contains('?')) { + $key = $value->before('?'); + $defaultValue = $value->after('?'); + } else { + $key = $value; + $defaultValue = null; + } + if ($foundEnv) { + $defaultValue = data_get($foundEnv, 'value'); + } + $isBuildTime = data_get($foundEnv, 'is_build_time', false); + EnvironmentVariable::updateOrCreate([ + 'key' => $key->value(), + 'application_id' => $resource->id, + ], [ + 'value' => $defaultValue, + 'is_build_time' => $isBuildTime, + 'application_id' => $resource->id, + ]); + } + } + } + // Add labels to the service + if ($resource->serviceType()) { + $fqdns = generateServiceSpecificFqdns($resource, forTraefik: true); + } else { + $domains = collect(json_decode($resource->docker_compose_domains)) ?? []; + if ($domains) { + $fqdns = data_get($domains, "$serviceName.domain"); + if ($fqdns) { + $fqdns = str($fqdns)->explode(','); + $uuid = new Cuid2(7); + if ($pull_request_id !== 0) { + $fqdns = $fqdns->map(function ($fqdn) use ($pull_request_id, $resource) { + $preview = ApplicationPreview::findPreviewByApplicationAndPullId($resource->id, $pull_request_id); + $url = Url::fromString($fqdn); + $template = $resource->preview_url_template; + $host = $url->getHost(); + $schema = $url->getScheme(); + $random = new Cuid2(7); + $preview_fqdn = str_replace('{{random}}', $random, $template); + $preview_fqdn = str_replace('{{domain}}', $host, $preview_fqdn); + $preview_fqdn = str_replace('{{pr_id}}', $pull_request_id, $preview_fqdn); + $preview_fqdn = "$schema://$preview_fqdn"; + $preview->fqdn = $preview_fqdn; + $preview->save(); + return $preview_fqdn; + }); + } + $serviceLabels = $serviceLabels->merge(fqdnLabelsForTraefik($uuid, $fqdns)); + } + } + } + $defaultLabels = defaultLabels($resource->id, $containerName, $pull_request_id, type: 'application'); + $serviceLabels = $serviceLabels->merge($defaultLabels); + + if ($server->isLogDrainEnabled() && $resource->isLogDrainEnabled()) { + data_set($service, 'logging', [ + 'driver' => 'fluentd', + 'options' => [ + 'fluentd-address' => "tcp://127.0.0.1:24224", + 'fluentd-async' => "true", + 'fluentd-sub-second-precision' => "true", + ] + ]); + } + data_set($service, 'labels', $serviceLabels->toArray()); + data_forget($service, 'is_database'); + data_set($service, 'restart', RESTART_MODE); + data_set($service, 'container_name', $containerName); + data_forget($service, 'volumes.*.content'); + data_forget($service, 'volumes.*.isDirectory'); + + return $service; + }); + $finalServices = [ + 'version' => $dockerComposeVersion, + 'services' => $services->toArray(), + 'volumes' => $topLevelVolumes->toArray(), + 'networks' => $topLevelNetworks->toArray(), + ]; + if ($isSameDockerComposeFile) { + $resource->docker_compose_pr_raw = Yaml::dump($yaml, 10, 2); + $resource->docker_compose_pr = Yaml::dump($finalServices, 10, 2); + $resource->docker_compose_raw = Yaml::dump($yaml, 10, 2); + $resource->docker_compose = Yaml::dump($finalServices, 10, 2); + } else { + if ($is_pr) { + $resource->docker_compose_pr_raw = Yaml::dump($yaml, 10, 2); + $resource->docker_compose_pr = Yaml::dump($finalServices, 10, 2); + } else { + $resource->docker_compose_raw = Yaml::dump($yaml, 10, 2); + $resource->docker_compose = Yaml::dump($finalServices, 10, 2); + } + } + $resource->save(); + return collect($finalServices); + } +} + +function parseEnvVariable(Str|string $value) +{ + $value = str($value); + $count = substr_count($value->value(), '_'); + $command = null; + $forService = null; + $generatedValue = null; + $port = null; + + if ($count === 2) { + if ($value->startsWith('SERVICE_FQDN') || $value->startsWith('SERVICE_URL')) { + // SERVICE_FQDN_UMAMI + $command = $value->after('SERVICE_')->beforeLast('_'); + $forService = $value->afterLast('_'); + } else { + // SERVICE_BASE64_UMAMI + $command = $value->after('SERVICE_')->beforeLast('_'); + } + } + if ($count === 3) { + if ($value->startsWith('SERVICE_FQDN') || $value->startsWith('SERVICE_URL')) { + // SERVICE_FQDN_UMAMI_1000 + $command = $value->after('SERVICE_')->before('_'); + $forService = $value->after('SERVICE_')->after('_')->before('_'); + $port = $value->afterLast('_'); + } else { + // SERVICE_BASE64_64_UMAMI + $command = $value->after('SERVICE_')->beforeLast('_'); + } + } + return [ + 'command' => $command, + 'forService' => $forService, + 'generatedValue' => $generatedValue, + 'port' => $port, + ]; +} +function generateEnvValue(string $command) +{ + switch ($command) { + case 'PASSWORD': + $generatedValue = Str::password(symbols: false); + break; + case 'PASSWORD_64': + $generatedValue = Str::password(length: 64, symbols: false); + break; + case 'BASE64_64': + $generatedValue = Str::random(64); + break; + case 'BASE64_128': + $generatedValue = Str::random(128); + break; + case 'BASE64': + $generatedValue = Str::random(32); + break; + case 'USER': + $generatedValue = Str::random(16); + break; + } + return $generatedValue; +} diff --git a/config/constants.php b/config/constants.php index d125ce1cc..4febbf043 100644 --- a/config/constants.php +++ b/config/constants.php @@ -26,9 +26,9 @@ return [ 'server' => [ 'zero' => 0, 'self-hosted' => 999999999999, - 'basic' => 1, - 'pro' => 10, - 'ultimate' => 25, + 'basic' => env('LIMIT_SERVER_BASIC', 2), + 'pro' => env('LIMIT_SERVER_PRO', 10), + 'ultimate' => env('LIMIT_SERVER_ULTIMATE', 25), ], 'email' => [ 'zero' => true, diff --git a/config/sentry.php b/config/sentry.php index efd043adf..c07fda49b 100644 --- a/config/sentry.php +++ b/config/sentry.php @@ -7,7 +7,7 @@ return [ // The release version of your application // Example with dynamic git hash: trim(exec('git --git-dir ' . base_path('.git') . ' log --pretty="%h" -n1 HEAD')) - 'release' => '4.0.0-beta.146', + 'release' => '4.0.0-beta.147', // When left empty or `null` the Laravel environment will be used 'environment' => config('app.env'), diff --git a/config/version.php b/config/version.php index 8c257643a..cf8ab7438 100644 --- a/config/version.php +++ b/config/version.php @@ -1,3 +1,3 @@ string('docker_compose_location')->nullable()->default('/docker-compose.yaml')->after('dockerfile_location'); + $table->string('docker_compose_pr_location')->nullable()->default('/docker-compose.yaml')->after('docker_compose_location'); + + $table->longText('docker_compose')->nullable()->after('docker_compose_location'); + $table->longText('docker_compose_pr')->nullable()->after('docker_compose_location'); + $table->longText('docker_compose_raw')->nullable()->after('docker_compose'); + $table->longText('docker_compose_pr_raw')->nullable()->after('docker_compose'); + + $table->text('docker_compose_domains')->nullable()->after('docker_compose_raw'); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('applications', function (Blueprint $table) { + $table->dropColumn('docker_compose_location'); + $table->dropColumn('docker_compose_pr_location'); + $table->dropColumn('docker_compose'); + $table->dropColumn('docker_compose_pr'); + $table->dropColumn('docker_compose_raw'); + $table->dropColumn('docker_compose_pr_raw'); + $table->dropColumn('docker_compose_domains'); + }); + } +}; diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index be972721f..ca6b42348 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -28,6 +28,8 @@ services: - REDIS_HOST - REDIS_PASSWORD - HORIZON_MAX_PROCESSES + - HORIZON_BALANCE_MAX_SHIFT + - HORIZON_BALANCE_COOLDOWN - SSL_MODE=off - PHP_PM_CONTROL=dynamic - PHP_PM_START_SERVERS=1 diff --git a/resources/views/components/applications/links.blade.php b/resources/views/components/applications/links.blade.php index 1c716a3af..c5bdc4a48 100644 --- a/resources/views/components/applications/links.blade.php +++ b/resources/views/components/applications/links.blade.php @@ -1,44 +1,50 @@