dave/coolify
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

wip: compose based apps

Browse Source
This commit is contained in:
Andras Bacsai
2023-11-24 15:48:23 +01:00
parent 65a1961722
commit f96a91eb31
14 changed files with 1389 additions and 705 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
app/Console/Commands/SyncBunny.php
View File

@@ -71,6 +71,15 @@ class SyncBunny extends Command
]);
});
try {
if (!$only_template && !$only_version) {
$this->info('About to sync files (docker-compose.prod.yaml, upgrade.sh, install.sh, etc) to BunnyCDN.');
}
if ($only_template) {
$this->info('About to sync service-templates.json to BunnyCDN.');
}
if ($only_version) {
$this->info('About to sync versions.json to BunnyCDN.');
}
$confirmed = confirm('Are you sure you want to sync?');
if (!$confirmed) {
return;

52
app/Http/Livewire/Project/Application/General.php
View File

@@ -6,6 +6,7 @@ use App\Models\Application;
use Illuminate\Support\Collection;
use Illuminate\Support\Str;
use Livewire\Component;
use Visus\Cuid2\Cuid2;
class General extends Component
{
@@ -27,6 +28,9 @@ class General extends Component
public bool $is_static;
public $parsedServices = [];
public $parsedServiceDomains = [];
protected $listeners = [
'resetDefaultLabels'
];
@@ -50,6 +54,9 @@ class General extends Component
'application.docker_registry_image_name' => 'nullable',
'application.docker_registry_image_tag' => 'nullable',
'application.dockerfile_location' => 'nullable',
'application.docker_compose_location' => 'nullable',
'application.docker_compose' => 'nullable',
'application.docker_compose_raw' => 'nullable',
'application.custom_labels' => 'nullable',
'application.dockerfile_target_build' => 'nullable',
'application.settings.is_static' => 'boolean|required',
@@ -74,6 +81,9 @@ class General extends Component
'application.docker_registry_image_name' => 'Docker registry image name',
'application.docker_registry_image_tag' => 'Docker registry image tag',
'application.dockerfile_location' => 'Dockerfile location',
'application.docker_compose_location' => 'Docker compose location',
'application.docker_compose' => 'Docker compose',
'application.docker_compose_raw' => 'Docker compose raw',
'application.custom_labels' => 'Custom labels',
'application.dockerfile_target_build' => 'Dockerfile target build',
'application.settings.is_static' => 'Is static',
@@ -81,6 +91,14 @@ class General extends Component
public function mount()
{
try {
$this->parsedServices = $this->application->parseCompose();
ray($this->parsedServices);
} catch (\Throwable $e) {
$this->emit('error', $e->getMessage());
}
$this->parsedServiceDomains = $this->application->docker_compose_domains ? json_decode($this->application->docker_compose_domains, true) : [];
$this->ports_exposes = $this->application->ports_exposes;
if (str($this->application->status)->startsWith('running') && is_null($this->application->config_hash)) {
$this->application->isConfigurationChanged(true);
@@ -98,6 +116,38 @@ class General extends Component
$this->application->settings->save();
$this->emit('success', 'Settings saved.');
}
public function loadComposeFile($isInit = false)
{
if ($isInit && $this->application->docker_compose_raw) {
return;
}
$uuid = new Cuid2();
['commands' => $cloneCommand] = $this->application->generateGitImportCommands(deployment_uuid: $uuid, only_checkout: true, exec_in_docker: false, custom_base_dir: '.');
$workdir = rtrim($this->application->base_directory, '/');
$composeFile = $this->application->docker_compose_location;
$commands = collect([
"mkdir -p /tmp/{$uuid} && cd /tmp/{$uuid}",
$cloneCommand,
"git sparse-checkout init --cone",
"git sparse-checkout set .$workdir$composeFile",
"git read-tree -mu HEAD",
"cat .$workdir$composeFile",
]);
$composeFileContent = instant_remote_process($commands, $this->application->destination->server, false);
if (!$composeFileContent) {
$this->emit('error', "Could not load compose file from $workdir$composeFile");
return;
} else {
$this->application->docker_compose_raw = $composeFileContent;
$this->application->save();
}
$commands = collect([
"rm -rf /tmp/{$uuid}",
]);
instant_remote_process($commands, $this->application->destination->server, false);
$this->parsedServices = $this->application->parseCompose();
$this->emit('success', 'Compose file loaded.');
}
public function updatedApplicationBuildPack()
{
if ($this->application->build_pack !== 'nixpacks') {
@@ -172,8 +222,10 @@ class General extends Component
$this->customLabels = str($this->customLabels)->replace(',', "\n");
}
$this->application->custom_labels = $this->customLabels->explode("\n")->implode(',');
$this->application->docker_compose_domains = json_encode($this->parsedServiceDomains);
$this->application->save();
$showToaster && $this->emit('success', 'Application settings updated!');
$this->parsedServices = $this->application->parseCompose();
} catch (\Throwable $e) {
return handleError($e, $this);
} finally {

179
app/Jobs/ApplicationDeploymentJob.php
View File

@@ -73,6 +73,7 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted
private $docker_compose;
private $docker_compose_base64;
private string $dockerfile_location = '/Dockerfile';
private string $docker_compose_location = '/docker-compose.yml';
private ?string $addHosts = null;
private ?string $buildTarget = null;
private $log_model;
@@ -114,7 +115,7 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted
$this->destination = $this->application->destination->getMorphClass()::where('id', $this->application->destination->id)->first();
$this->server = $this->mainServer = $this->destination->server;
$this->serverUser = $this->server->user;
$this->basedir = "/artifacts/{$this->deployment_uuid}";
$this->basedir = $this->application->generateBaseDir($this->deployment_uuid);
$this->workdir = "{$this->basedir}" . rtrim($this->application->base_directory, '/');
$this->configuration_dir = application_configuration_dir() . "/{$this->application->uuid}";
$this->is_debug_enabled = $this->application->settings->is_debug_enabled;
@@ -183,16 +184,10 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted
}
// Check custom port
preg_match('/(?<=:)\d+(?=\/)/', $this->application->git_repository, $matches);
if (count($matches) === 1) {
$this->customPort = $matches[0];
$gitHost = str($this->application->git_repository)->before(':');
$gitRepo = str($this->application->git_repository)->after('/');
$this->customRepository = "$gitHost:$gitRepo";
} else {
$this->customRepository = $this->application->git_repository;
}
['repository' => $this->customRepository, 'port' => $this->customPort] = $this->application->customRepository();
try {
ray($this->application->build_pack);
if ($this->restart_only && $this->application->build_pack !== 'dockerimage') {
$this->just_restart();
if ($this->server->isProxyShouldRun()) {
@@ -203,6 +198,8 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted
return;
} else if ($this->application->dockerfile) {
$this->deploy_simple_dockerfile();
} else if ($this->application->build_pack === 'dockercompose') {
$this->deploy_docker_compose_buildpack();
} else if ($this->application->build_pack === 'dockerimage') {
$this->deploy_dockerimage_buildpack();
} else if ($this->application->build_pack === 'dockerfile') {
@@ -397,19 +394,20 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted
]);
}
}
// private function save_environment_variables()
// {
// $envs = collect([]);
// foreach ($this->application->environment_variables as $env) {
// $envs->push($env->key . '=' . $env->value);
// }
// $envs_base64 = base64_encode($envs->implode("\n"));
// $this->execute_remote_command(
// [
// executeInDocker($this->deployment_uuid, "echo '$envs_base64' | base64 -d > $this->workdir/.env")
// ],
// );
// }
private function save_environment_variables()
{
$envs = collect([]);
foreach ($this->application->environment_variables as $env) {
$envs->push($env->key . '=' . $env->value);
}
$envs_base64 = base64_encode($envs->implode("\n"));
$this->execute_remote_command(
[
executeInDocker($this->deployment_uuid, "echo '$envs_base64' | base64 -d > $this->workdir/.env")
],
);
}
private function deploy_simple_dockerfile()
{
$dockerfile_base64 = base64_encode($this->application->dockerfile);
@@ -447,7 +445,36 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted
$this->generate_compose_file();
$this->rolling_update();
}
private function deploy_docker_compose_buildpack()
{
if (data_get($this->application, 'docker_compose_location')) {
$this->docker_compose_location = $this->application->docker_compose_location;
}
$this->application_deployment_queue->addLogEntry("Starting deployment of {$this->application->name}.");
$this->server->executeRemoteCommand(
commands: $this->application->prepareHelperImage($this->deployment_uuid),
loggingModel: $this->application_deployment_queue
);
$this->check_git_if_build_needed();
$this->clone_repository();
$this->generate_image_names();
$this->cleanup_git();
$composeFile = $this->application->parseCompose();
$yaml = Yaml::dump($composeFile->toArray(), 10);
$this->docker_compose_base64 = base64_encode($yaml);
$this->execute_remote_command([
executeInDocker($this->deployment_uuid, "echo '{$this->docker_compose_base64}' | base64 -d > {$this->workdir}/docker-compose.yaml"), "hidden" => true
]);
$this->execute_remote_command([
"docker network create --attachable '{$this->application->uuid}' >/dev/null || true", "hidden" => true, "ignore_errors" => true
], [
"docker network connect {$this->application->uuid} coolify-proxy || true", "hidden" => true, "ignore_errors" => true
]);
$this->save_environment_variables();
$this->stop_running_container(force: true);
$this->start_by_compose_file();
}
private function deploy_dockerfile_buildpack()
{
if (data_get($this->application, 'dockerfile_location')) {
@@ -472,7 +499,7 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted
// $this->push_to_docker_registry();
// $this->deploy_to_additional_destinations();
// } else {
$this->rolling_update();
$this->rolling_update();
// }
}
private function deploy_nixpacks_buildpack()
@@ -725,6 +752,7 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted
private function clone_repository()
{
$importCommands = $this->generate_git_import_commands();
ray($importCommands);
$this->execute_remote_command(
[
"echo '\n----------------------------------------'",
@@ -740,90 +768,14 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted
private function generate_git_import_commands()
{
$this->branch = $this->application->git_branch;
$commands = collect([]);
$git_clone_command = "git clone -q -b {$this->application->git_branch}";
if ($this->pull_request_id !== 0) {
$pr_branch_name = "pr-{$this->pull_request_id}-coolify";
}
if ($this->application->deploymentType() === 'source') {
$source_html_url = data_get($this->application, 'source.html_url');
$url = parse_url(filter_var($source_html_url, FILTER_SANITIZE_URL));
$source_html_url_host = $url['host'];
$source_html_url_scheme = $url['scheme'];
if ($this->source->getMorphClass() == 'App\Models\GithubApp') {
if ($this->source->is_public) {
$this->fullRepoUrl = "{$this->source->html_url}/{$this->customRepository}";
$git_clone_command = "{$git_clone_command} {$this->source->html_url}/{$this->customRepository} {$this->basedir}";
$git_clone_command = $this->set_git_import_settings($git_clone_command);
$commands->push(executeInDocker($this->deployment_uuid, $git_clone_command));
} else {
$github_access_token = generate_github_installation_token($this->source);
$commands->push(executeInDocker($this->deployment_uuid, "git clone -q -b {$this->application->git_branch} $source_html_url_scheme://x-access-token:$github_access_token@$source_html_url_host/{$this->customRepository}.git {$this->basedir}"));
$this->fullRepoUrl = "$source_html_url_scheme://x-access-token:$github_access_token@$source_html_url_host/{$this->customRepository}.git";
}
if ($this->pull_request_id !== 0) {
$this->branch = "pull/{$this->pull_request_id}/head:$pr_branch_name";
$commands->push(executeInDocker($this->deployment_uuid, "cd {$this->basedir} && git fetch origin $this->branch && git checkout $pr_branch_name"));
}
return $commands->implode(' && ');
}
}
if ($this->application->deploymentType() === 'deploy_key') {
$this->fullRepoUrl = $this->customRepository;
$private_key = data_get($this->application, 'private_key.private_key');
if (is_null($private_key)) {
throw new RuntimeException('Private key not found. Please add a private key to the application and try again.');
}
$private_key = base64_encode($private_key);
$git_clone_command_base = "GIT_SSH_COMMAND=\"ssh -o ConnectTimeout=30 -p {$this->customPort} -o Port={$this->customPort} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /root/.ssh/id_rsa\" {$git_clone_command} {$this->customRepository} {$this->basedir}";
$git_clone_command = $this->set_git_import_settings($git_clone_command_base);
$commands = collect([
executeInDocker($this->deployment_uuid, "mkdir -p /root/.ssh"),
executeInDocker($this->deployment_uuid, "echo '{$private_key}' | base64 -d > /root/.ssh/id_rsa"),
executeInDocker($this->deployment_uuid, "chmod 600 /root/.ssh/id_rsa"),
]);
if ($this->pull_request_id !== 0) {
ray($this->git_type);
if ($this->git_type === 'gitlab') {
$this->branch = "merge-requests/{$this->pull_request_id}/head:$pr_branch_name";
$commands->push(executeInDocker($this->deployment_uuid, "echo 'Checking out $this->branch'"));
$git_clone_command = "{$git_clone_command} && cd {$this->basedir} && GIT_SSH_COMMAND=\"ssh -o ConnectTimeout=30 -p {$this->customPort} -o Port={$this->customPort} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /root/.ssh/id_rsa\" git fetch origin $this->branch && git checkout $pr_branch_name";
}
if ($this->git_type === 'github') {
$this->branch = "pull/{$this->pull_request_id}/head:$pr_branch_name";
$commands->push(executeInDocker($this->deployment_uuid, "echo 'Checking out $this->branch'"));
$git_clone_command = "{$git_clone_command} && cd {$this->basedir} && GIT_SSH_COMMAND=\"ssh -o ConnectTimeout=30 -p {$this->customPort} -o Port={$this->customPort} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /root/.ssh/id_rsa\" git fetch origin $this->branch && git checkout $pr_branch_name";
}
}
$commands->push(executeInDocker($this->deployment_uuid, $git_clone_command));
return $commands->implode(' && ');
}
if ($this->application->deploymentType() === 'other') {
$this->fullRepoUrl = $this->customRepository;
$git_clone_command = "{$git_clone_command} {$this->customRepository} {$this->basedir}";
$git_clone_command = $this->set_git_import_settings($git_clone_command);
$commands->push(executeInDocker($this->deployment_uuid, $git_clone_command));
return $commands->implode(' && ');
}
['commands' => $commands, 'branch' => $this->branch, 'fullRepoUrl' => $this->fullRepoUrl] = $this->application->generateGitImportCommands($this->deployment_uuid, $this->pull_request_id, $this->git_type);
return $commands;
}
private function set_git_import_settings($git_clone_command)
{
if ($this->application->git_commit_sha !== 'HEAD') {
$git_clone_command = "{$git_clone_command} && cd {$this->basedir} && git -c advice.detachedHead=false checkout {$this->application->git_commit_sha} >/dev/null 2>&1";
}
if ($this->application->settings->is_git_submodules_enabled) {
$git_clone_command = "{$git_clone_command} && cd {$this->basedir} && git submodule update --init --recursive";
}
if ($this->application->settings->is_git_lfs_enabled) {
$git_clone_command = "{$git_clone_command} && cd {$this->basedir} && git lfs pull";
}
return $git_clone_command;
return $this->application->setGitImportSettings($this->deployment_uuid, $git_clone_command);
}
private function cleanup_git()
@@ -879,6 +831,26 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted
$this->env_args = $this->env_args->implode(' ');
}
private function modify_compose_file()
{
// ray("{$this->workdir}{$this->docker_compose_location}");
$this->execute_remote_command([executeInDocker($this->deployment_uuid, "cat {$this->workdir}{$this->docker_compose_location}"), "hidden" => true, "save" => 'compose_file']);
if ($this->saved_outputs->get('compose_file')) {
$compose = $this->saved_outputs->get('compose_file');
}
try {
$yaml = Yaml::parse($compose);
} catch (\Exception $e) {
throw new \Exception($e->getMessage());
}
$services = data_get($yaml, 'services');
$topLevelNetworks = collect(data_get($yaml, 'networks', []));
$definedNetwork = collect([$this->application->uuid]);
$services = collect($services)->map(function ($service, $serviceName) use ($topLevelNetworks, $definedNetwork) {
$serviceNetworks = collect(data_get($service, 'networks', []));
});
}
private function generate_compose_file()
{
$ports = $this->application->settings->is_static ? [80] : $this->application->ports_exposes_array;
@@ -1209,6 +1181,7 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf");
$this->execute_remote_command(["echo -n 'Removing old container.'"]);
if ($this->newVersionIsHealthy || $force) {
$containers = getCurrentApplicationContainerStatus($this->server, $this->application->id, $this->pull_request_id);
ray($containers);
if ($this->pull_request_id !== 0) {
$containers = $containers->filter(function ($container) {
return data_get($container, 'Names') === $this->container_name;

231
app/Models/Application.php
View File

@@ -6,6 +6,8 @@ use Illuminate\Database\Eloquent\Casts\Attribute;
use Illuminate\Database\Eloquent\Relations\HasMany;
use Spatie\Activitylog\Models\Activity;
use Illuminate\Support\Str;
use RuntimeException;
use Symfony\Component\Yaml\Yaml;
class Application extends BaseModel
{
@@ -123,6 +125,21 @@ class Application extends BaseModel
}
);
}
public function dockerComposeLocation(): Attribute
{
return Attribute::make(
set: function ($value) {
if (is_null($value) || $value === '') {
return '/docker-compose.yml';
} else {
if ($value !== '/') {
return Str::start(Str::replaceEnd('/', '', $value), '/');
}
return Str::start($value, '/');
}
}
);
}
public function baseDirectory(): Attribute
{
return Attribute::make(
@@ -157,7 +174,16 @@ class Application extends BaseModel
: explode(',', $this->ports_exposes)
);
}
public function serviceType()
{
$found = str(collect(SPECIFIC_SERVICES)->filter(function ($service) {
return str($this->image)->before(':')->value() === $service;
})->first());
if ($found->isNotEmpty()) {
return $found;
}
return null;
}
public function environment_variables(): HasMany
{
return $this->hasMany(EnvironmentVariable::class)->where('is_preview', false)->orderBy('key', 'asc');
@@ -342,7 +368,8 @@ class Application extends BaseModel
}
return false;
}
public function healthCheckUrl() {
public function healthCheckUrl()
{
if ($this->dockerfile || $this->build_pack === 'dockerfile' || $this->build_pack === 'dockerimage') {
return null;
}
@@ -358,4 +385,204 @@ class Application extends BaseModel
}
return $full_healthcheck_url;
}
function customRepository()
{
preg_match('/(?<=:)\d+(?=\/)/', $this->git_repository, $matches);
$port = 22;
if (count($matches) === 1) {
$port = $matches[0];
$gitHost = str($this->git_repository)->before(':');
$gitRepo = str($this->git_repository)->after('/');
$repository = "$gitHost:$gitRepo";
} else {
$repository = $this->git_repository;
}
return [
'repository' => $repository,
'port' => $port
];
}
function generateBaseDir(string $uuid)
{
return "/artifacts/{$uuid}";
}
function setGitImportSettings(string $deployment_uuid, string $git_clone_command)
{
$baseDir = $this->generateBaseDir($deployment_uuid);
if ($this->git_commit_sha !== 'HEAD') {
$git_clone_command = "{$git_clone_command} && cd {$baseDir} && git -c advice.detachedHead=false checkout {$this->git_commit_sha} >/dev/null 2>&1";
}
if ($this->settings->is_git_submodules_enabled) {
$git_clone_command = "{$git_clone_command} && cd {$baseDir} && git submodule update --init --recursive";
}
if ($this->settings->is_git_lfs_enabled) {
$git_clone_command = "{$git_clone_command} && cd {$baseDir} && git lfs pull";
}
return $git_clone_command;
}
function generateGitImportCommands(string $deployment_uuid, int $pull_request_id = 0, ?string $git_type = null, bool $exec_in_docker = true, bool $only_checkout = false, ?string $custom_base_dir = null)
{
$branch = $this->git_branch;
['repository' => $customRepository, 'port' => $customPort] = $this->customRepository();
$baseDir = $custom_base_dir ?? $this->generateBaseDir($deployment_uuid);
$commands = collect([]);
$git_clone_command = "git clone -b {$this->git_branch}";
if ($only_checkout) {
$git_clone_command = "git clone --no-checkout -b {$this->git_branch}";
}
if ($pull_request_id !== 0) {
$pr_branch_name = "pr-{$pull_request_id}-coolify";
}
if ($this->deploymentType() === 'source') {
$source_html_url = data_get($this, 'source.html_url');
$url = parse_url(filter_var($source_html_url, FILTER_SANITIZE_URL));
$source_html_url_host = $url['host'];
$source_html_url_scheme = $url['scheme'];
if ($this->source->getMorphClass() == 'App\Models\GithubApp') {
if ($this->source->is_public) {
$fullRepoUrl = "{$this->source->html_url}/{$customRepository}";
$git_clone_command = "{$git_clone_command} {$this->source->html_url}/{$customRepository} {$baseDir}";
if (!$only_checkout) {
$git_clone_command = $this->setGitImportSettings($deployment_uuid, $git_clone_command);
}
if ($exec_in_docker) {
$commands->push(executeInDocker($deployment_uuid, $git_clone_command));
} else {
$commands->push($git_clone_command);
}
} else {
$github_access_token = generate_github_installation_token($this->source);
if ($exec_in_docker) {
$commands->push(executeInDocker($deployment_uuid, "{$git_clone_command} $source_html_url_scheme://x-access-token:$github_access_token@$source_html_url_host/{$customRepository}.git {$baseDir}"));
$fullRepoUrl = "$source_html_url_scheme://x-access-token:$github_access_token@$source_html_url_host/{$customRepository}.git";
} else {
$commands->push("{$git_clone_command} $source_html_url_scheme://x-access-token:$github_access_token@$source_html_url_host/{$customRepository} {$baseDir}");
$fullRepoUrl = "$source_html_url_scheme://x-access-token:$github_access_token@$source_html_url_host/{$customRepository}";
}
}
if ($pull_request_id !== 0) {
$branch = "pull/{$pull_request_id}/head:$pr_branch_name";
if ($exec_in_docker) {
$commands->push(executeInDocker($deployment_uuid, "cd {$baseDir} && git fetch origin {$branch} && git checkout $pr_branch_name"));
} else {
$commands->push("cd {$baseDir} && git fetch origin {$branch} && git checkout $pr_branch_name");
}
}
return [
'commands' => $commands->implode(' && '),
'branch' => $branch,
'fullRepoUrl' => $fullRepoUrl
];
}
}
if ($this->deploymentType() === 'deploy_key') {
$fullRepoUrl = $customRepository;
$private_key = data_get($this, 'private_key.private_key');
if (is_null($private_key)) {
throw new RuntimeException('Private key not found. Please add a private key to the application and try again.');
}
$private_key = base64_encode($private_key);
$git_clone_command_base = "GIT_SSH_COMMAND=\"ssh -o ConnectTimeout=30 -p {$customPort} -o Port={$customPort} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /root/.ssh/id_rsa\" {$git_clone_command} {$customRepository} {$baseDir}";
if (!$only_checkout) {
$git_clone_command = $this->setGitImportSettings($deployment_uuid, $git_clone_command_base);
}
if ($exec_in_docker) {
$commands = collect([
executeInDocker($deployment_uuid, "mkdir -p /root/.ssh"),
executeInDocker($deployment_uuid, "echo '{$private_key}' | base64 -d > /root/.ssh/id_rsa"),
executeInDocker($deployment_uuid, "chmod 600 /root/.ssh/id_rsa"),
]);
} else {
$commands = collect([
"mkdir -p /root/.ssh",
"echo '{$private_key}' | base64 -d > /root/.ssh/id_rsa",
"chmod 600 /root/.ssh/id_rsa",
]);
}
if ($pull_request_id !== 0) {
if ($git_type === 'gitlab') {
$branch = "merge-requests/{$pull_request_id}/head:$pr_branch_name";
if ($exec_in_docker) {
$commands->push(executeInDocker($deployment_uuid, "echo 'Checking out $branch'"));
} else {
$commands->push("echo 'Checking out $branch'");
}
$git_clone_command = "{$git_clone_command} && cd {$baseDir} && GIT_SSH_COMMAND=\"ssh -o ConnectTimeout=30 -p {$customPort} -o Port={$customPort} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /root/.ssh/id_rsa\" git fetch origin $branch && git checkout $pr_branch_name";
}
if ($git_type === 'github') {
$branch = "pull/{$pull_request_id}/head:$pr_branch_name";
if ($exec_in_docker) {
$commands->push(executeInDocker($deployment_uuid, "echo 'Checking out $branch'"));
} else {
$commands->push("echo 'Checking out $branch'");
}
$git_clone_command = "{$git_clone_command} && cd {$baseDir} && GIT_SSH_COMMAND=\"ssh -o ConnectTimeout=30 -p {$customPort} -o Port={$customPort} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /root/.ssh/id_rsa\" git fetch origin $branch && git checkout $pr_branch_name";
}
}
if ($exec_in_docker) {
$commands->push(executeInDocker($deployment_uuid, $git_clone_command));
} else {
$commands->push($git_clone_command);
}
return [
'commands' => $commands->implode(' && '),
'branch' => $branch,
'fullRepoUrl' => $fullRepoUrl
];
}
if ($this->deploymentType() === 'other') {
$fullRepoUrl = $customRepository;
$git_clone_command = "{$git_clone_command} {$customRepository} {$baseDir}";
$git_clone_command = $this->setGitImportSettings($deployment_uuid, $git_clone_command);
if ($exec_in_docker) {
$commands->push(executeInDocker($deployment_uuid, $git_clone_command));
} else {
$commands->push($git_clone_command);
}
return [
'commands' => $commands->implode(' && '),
'branch' => $branch,
'fullRepoUrl' => $fullRepoUrl
];
}
}
public function prepareHelperImage(string $deploymentUuid)
{
$basedir = $this->generateBaseDir($deploymentUuid);
$helperImage = config('coolify.helper_image');
$server = data_get($this, 'destination.server');
$network = data_get($this, 'destination.network');
$serverUserHomeDir = instant_remote_process(["echo \$HOME"], $server);
$dockerConfigFileExists = instant_remote_process(["test -f {$serverUserHomeDir}/.docker/config.json && echo 'OK' || echo 'NOK'"], $server);
$commands = collect([]);
if ($dockerConfigFileExists === 'OK') {
$commands->push([
"command" => "docker run -d --network $network -v /:/host --name $deploymentUuid --rm -v {$serverUserHomeDir}/.docker/config.json:/root/.docker/config.json:ro -v /var/run/docker.sock:/var/run/docker.sock $helperImage",
"hidden" => true,
]);
} else {
$commands->push([
"command" => "docker run -d --network {$network} -v /:/host --name {$deploymentUuid} --rm -v /var/run/docker.sock:/var/run/docker.sock {$helperImage}",
"hidden" => true,
]);
}
$commands->push([
"command" => executeInDocker($deploymentUuid, "mkdir -p {$basedir}"),
"hidden" => true,
]);
return $commands;
}
function parseCompose()
{
if ($this->docker_compose_raw) {
return parseDockerComposeFile($this);
} else {
return collect([]);
}
}
}

48
app/Models/Server.php
View File

@@ -392,7 +392,7 @@ class Server extends BaseModel
{
return instant_remote_process(["docker network create coolify --attachable >/dev/null 2>&1 || true"], $this, false);
}
public function executeRemoteCommand(Collection $commands, ApplicationDeploymentQueue $loggingModel)
public function executeRemoteCommand(Collection $commands, ?ApplicationDeploymentQueue $loggingModel = null)
{
static::$batch_counter++;
foreach ($commands as $command) {
@@ -419,33 +419,35 @@ class Server extends BaseModel
'hidden' => $hidden,
'batch' => static::$batch_counter,
];
if (!$loggingModel->logs) {
$newLogEntry['order'] = 1;
} else {
$previousLogs = json_decode($loggingModel->logs, associative: true, flags: JSON_THROW_ON_ERROR);
$newLogEntry['order'] = count($previousLogs) + 1;
}
if ($name) {
$newLogEntry['name'] = $name;
}
if ($loggingModel) {
if (!$loggingModel->logs) {
$newLogEntry['order'] = 1;
} else {
$previousLogs = json_decode($loggingModel->logs, associative: true, flags: JSON_THROW_ON_ERROR);
$newLogEntry['order'] = count($previousLogs) + 1;
}
if ($name) {
$newLogEntry['name'] = $name;
}
$previousLogs[] = $newLogEntry;
$loggingModel->logs = json_encode($previousLogs, flags: JSON_THROW_ON_ERROR);
$loggingModel->save();
// if ($name) {
// $loggingModel['savedOutputs'][$name] = str($output)->trim();
// }
$previousLogs[] = $newLogEntry;
$loggingModel->logs = json_encode($previousLogs, flags: JSON_THROW_ON_ERROR);
$loggingModel->save();
}
});
$loggingModel->update([
'current_process_id' => $process->id(),
]);
if ($loggingModel) {
$loggingModel->update([
'current_process_id' => $process->id(),
]);
}
$processResult = $process->wait();
if ($processResult->exitCode() !== 0) {
if (!$ignoreErrors) {
$status = ApplicationDeploymentStatus::FAILED->value;
$loggingModel->status = $status;
$loggingModel->save();
if ($loggingModel) {
$status = ApplicationDeploymentStatus::FAILED->value;
$loggingModel->status = $status;
$loggingModel->save();
}
throw new \RuntimeException($processResult->errorOutput());
}
}

517
app/Models/Service.php
View File

@@ -371,521 +371,6 @@ class Service extends BaseModel
public function parse(bool $isNew = false): Collection
{
// ray()->clearAll();
if ($this->docker_compose_raw) {
try {
$yaml = Yaml::parse($this->docker_compose_raw);
} catch (\Exception $e) {
throw new \Exception($e->getMessage());
}
$topLevelVolumes = collect(data_get($yaml, 'volumes', []));
$topLevelNetworks = collect(data_get($yaml, 'networks', []));
$dockerComposeVersion = data_get($yaml, 'version') ?? '3.8';
$services = data_get($yaml, 'services');
$generatedServiceFQDNS = collect([]);
if (is_null($this->destination)) {
$destination = $this->server->destinations()->first();
if ($destination) {
$this->destination()->associate($destination);
$this->save();
}
}
$definedNetwork = collect([$this->uuid]);
$services = collect($services)->map(function ($service, $serviceName) use ($topLevelVolumes, $topLevelNetworks, $definedNetwork, $isNew, $generatedServiceFQDNS) {
$serviceVolumes = collect(data_get($service, 'volumes', []));
$servicePorts = collect(data_get($service, 'ports', []));
$serviceNetworks = collect(data_get($service, 'networks', []));
$serviceVariables = collect(data_get($service, 'environment', []));
$serviceLabels = collect(data_get($service, 'labels', []));
if ($serviceLabels->count() > 0) {
$removedLabels = collect([]);
$serviceLabels = $serviceLabels->filter(function ($serviceLabel, $serviceLabelName) use ($removedLabels) {
if (!str($serviceLabel)->contains('=')) {
$removedLabels->put($serviceLabelName, $serviceLabel);
return false;
}
return $serviceLabel;
});
foreach($removedLabels as $removedLabelName =>$removedLabel) {
$serviceLabels->push("$removedLabelName=$removedLabel");
}
}
$containerName = "$serviceName-{$this->uuid}";
// Decide if the service is a database
$isDatabase = false;
$image = data_get_str($service, 'image');
if ($image->contains(':')) {
$image = Str::of($image);
} else {
$image = Str::of($image)->append(':latest');
}
$imageName = $image->before(':');
if (collect(DATABASE_DOCKER_IMAGES)->contains($imageName)) {
$isDatabase = true;
}
data_set($service, 'is_database', $isDatabase);
// Create new serviceApplication or serviceDatabase
if ($isDatabase) {
if ($isNew) {
$savedService = ServiceDatabase::create([
'name' => $serviceName,
'image' => $image,
'service_id' => $this->id
]);
} else {
$savedService = ServiceDatabase::where([
'name' => $serviceName,
'service_id' => $this->id
])->first();
}
} else {
if ($isNew) {
$savedService = ServiceApplication::create([
'name' => $serviceName,
'image' => $image,
'service_id' => $this->id
]);
} else {
$savedService = ServiceApplication::where([
'name' => $serviceName,
'service_id' => $this->id
])->first();
}
}
if (is_null($savedService)) {
if ($isDatabase) {
$savedService = ServiceDatabase::create([
'name' => $serviceName,
'image' => $image,
'service_id' => $this->id
]);
} else {
$savedService = ServiceApplication::create([
'name' => $serviceName,
'image' => $image,
'service_id' => $this->id
]);
}
}
// Check if image changed
if ($savedService->image !== $image) {
$savedService->image = $image;
$savedService->save();
}
// Collect/create/update networks
if ($serviceNetworks->count() > 0) {
foreach ($serviceNetworks as $networkName => $networkDetails) {
$networkExists = $topLevelNetworks->contains(function ($value, $key) use ($networkName) {
return $value == $networkName || $key == $networkName;
});
if (!$networkExists) {
$topLevelNetworks->put($networkDetails, null);
}
}
}
// Collect/create/update ports
$collectedPorts = collect([]);
if ($servicePorts->count() > 0) {
foreach ($servicePorts as $sport) {
if (is_string($sport) || is_numeric($sport)) {
$collectedPorts->push($sport);
}
if (is_array($sport)) {
$target = data_get($sport, 'target');
$published = data_get($sport, 'published');
$protocol = data_get($sport, 'protocol');
$collectedPorts->push("$target:$published/$protocol");
}
}
}
$savedService->ports = $collectedPorts->implode(',');
$savedService->save();
// Add Coolify specific networks
$definedNetworkExists = $topLevelNetworks->contains(function ($value, $_) use ($definedNetwork) {
return $value == $definedNetwork;
});
if (!$definedNetworkExists) {
foreach ($definedNetwork as $network) {
$topLevelNetworks->put($network, [
'name' => $network,
'external' => true
]);
}
}
$networks = collect();
foreach ($serviceNetworks as $key => $serviceNetwork) {
if (gettype($serviceNetwork) === 'string') {
// networks:
// - appwrite
$networks->put($serviceNetwork, null);
} else if (gettype($serviceNetwork) === 'array') {
// networks:
// default:
// ipv4_address: 192.168.203.254
// $networks->put($serviceNetwork, null);
ray($key);
$networks->put($key, $serviceNetwork);
}
}
foreach ($definedNetwork as $key => $network) {
$networks->put($network, null);
}
data_set($service, 'networks', $networks->toArray());
// Collect/create/update volumes
if ($serviceVolumes->count() > 0) {
$serviceVolumes = $serviceVolumes->map(function ($volume) use ($savedService, $topLevelVolumes) {
$type = null;
$source = null;
$target = null;
$content = null;
$isDirectory = false;
if (is_string($volume)) {
$source = Str::of($volume)->before(':');
$target = Str::of($volume)->after(':')->beforeLast(':');
if ($source->startsWith('./') || $source->startsWith('/') || $source->startsWith('~')) {
$type = Str::of('bind');
} else {
$type = Str::of('volume');
}
} else if (is_array($volume)) {
$type = data_get_str($volume, 'type');
$source = data_get_str($volume, 'source');
$target = data_get_str($volume, 'target');
$content = data_get($volume, 'content');
$isDirectory = (bool) data_get($volume, 'isDirectory', false);
$foundConfig = $savedService->fileStorages()->whereMountPath($target)->first();
if ($foundConfig) {
$contentNotNull = data_get($foundConfig, 'content');
if ($contentNotNull) {
$content = $contentNotNull;
}
$isDirectory = (bool) data_get($foundConfig, 'is_directory');
}
}
if ($type->value() === 'bind') {
if ($source->value() === "/var/run/docker.sock") {
return $volume;
}
if ($source->value() === '/tmp' || $source->value() === '/tmp/') {
return $volume;
}
LocalFileVolume::updateOrCreate(
[
'mount_path' => $target,
'resource_id' => $savedService->id,
'resource_type' => get_class($savedService)
],
[
'fs_path' => $source,
'mount_path' => $target,
'content' => $content,
'is_directory' => $isDirectory,
'resource_id' => $savedService->id,
'resource_type' => get_class($savedService)
]
);
} else if ($type->value() === 'volume') {
$slugWithoutUuid = Str::slug($source, '-');
$name = "{$savedService->service->uuid}_{$slugWithoutUuid}";
if (is_string($volume)) {
$source = Str::of($volume)->before(':');
$target = Str::of($volume)->after(':')->beforeLast(':');
$source = $name;
$volume = "$source:$target";
} else if (is_array($volume)) {
data_set($volume, 'source', $name);
}
$topLevelVolumes->put($name, [
'name' => $name,
]);
LocalPersistentVolume::updateOrCreate(
[
'mount_path' => $target,
'resource_id' => $savedService->id,
'resource_type' => get_class($savedService)
],
[
'name' => $name,
'mount_path' => $target,
'resource_id' => $savedService->id,
'resource_type' => get_class($savedService)
]
);
}
$savedService->getFilesFromServer(isInit: true);
return $volume;
});
data_set($service, 'volumes', $serviceVolumes->toArray());
}
// Add env_file with at least .env to the service
// $envFile = collect(data_get($service, 'env_file', []));
// if ($envFile->count() > 0) {
// if (!$envFile->contains('.env')) {
// $envFile->push('.env');
// }
// } else {
// $envFile = collect(['.env']);
// }
// data_set($service, 'env_file', $envFile->toArray());
// Get variables from the service
foreach ($serviceVariables as $variableName => $variable) {
if (is_numeric($variableName)) {
$variable = Str::of($variable);
if ($variable->contains('=')) {
// - SESSION_SECRET=123
// - SESSION_SECRET=
$key = $variable->before('=');
$value = $variable->after('=');
} else {
// - SESSION_SECRET
$key = $variable;
$value = null;
}
} else {
// SESSION_SECRET: 123
// SESSION_SECRET:
$key = Str::of($variableName);
$value = Str::of($variable);
}
// TODO: here is the problem
if ($key->startsWith('SERVICE_FQDN')) {
if ($isNew || $savedService->fqdn === null) {
$name = $key->after('SERVICE_FQDN_')->beforeLast('_')->lower();
$fqdn = generateFqdn($this->server, "{$name->value()}-{$this->uuid}");
if (substr_count($key->value(), '_') === 3) {
// SERVICE_FQDN_UMAMI_1000
$port = $key->afterLast('_');
} else {
// SERVICE_FQDN_UMAMI
$port = null;
}
if ($port) {
$fqdn = "$fqdn:$port";
}
if (substr_count($key->value(), '_') >= 2) {
if (is_null($value)) {
$value = Str::of('/');
}
$path = $value->value();
if ($generatedServiceFQDNS->count() > 0) {
$alreadyGenerated = $generatedServiceFQDNS->has($key->value());
if ($alreadyGenerated) {
$fqdn = $generatedServiceFQDNS->get($key->value());
} else {
$generatedServiceFQDNS->put($key->value(), $fqdn);
}
} else {
$generatedServiceFQDNS->put($key->value(), $fqdn);
}
$fqdn = "$fqdn$path";
}
if (!$isDatabase) {
if ($savedService->fqdn) {
$fqdn = $savedService->fqdn . ',' . $fqdn;
} else {
$fqdn = $fqdn;
}
$savedService->fqdn = $fqdn;
$savedService->save();
}
}
// data_forget($service, "environment.$variableName");
// $yaml = data_forget($yaml, "services.$serviceName.environment.$variableName");
// if (count(data_get($yaml, 'services.' . $serviceName . '.environment')) === 0) {
// $yaml = data_forget($yaml, "services.$serviceName.environment");
// }
continue;
}
if ($value?->startsWith('$')) {
$value = Str::of(replaceVariables($value));
$key = $value;
$foundEnv = EnvironmentVariable::where([
'key' => $key,
'service_id' => $this->id,
])->first();
if ($value->startsWith('SERVICE_')) {
// Count _ in $value
$count = substr_count($value->value(), '_');
if ($count === 2) {
// SERVICE_FQDN_UMAMI
$command = $value->after('SERVICE_')->beforeLast('_');
$forService = $value->afterLast('_');
$generatedValue = null;
$port = null;
}
if ($count === 3) {
// SERVICE_FQDN_UMAMI_1000
$command = $value->after('SERVICE_')->before('_');
$forService = $value->after('SERVICE_')->after('_')->before('_');
$generatedValue = null;
$port = $value->afterLast('_');
}
if ($command->value() === 'FQDN' || $command->value() === 'URL') {
if (Str::lower($forService) === $serviceName) {
$fqdn = generateFqdn($this->server, $containerName);
} else {
$fqdn = generateFqdn($this->server, Str::lower($forService) . '-' . $this->uuid);
}
if ($port) {
$fqdn = "$fqdn:$port";
}
if ($foundEnv) {
$fqdn = data_get($foundEnv, 'value');
} else {
if ($command->value() === 'URL') {
$fqdn = Str::of($fqdn)->after('://')->value();
}
EnvironmentVariable::create([
'key' => $key,
'value' => $fqdn,
'is_build_time' => false,
'service_id' => $this->id,
'is_preview' => false,
]);
}
if (!$isDatabase) {
if ($command->value() === 'FQDN' && is_null($savedService->fqdn)) {
$savedService->fqdn = $fqdn;
$savedService->save();
}
}
} else {
switch ($command) {
case 'PASSWORD':
$generatedValue = Str::password(symbols: false);
break;
case 'PASSWORD_64':
$generatedValue = Str::password(length: 64, symbols: false);
break;
case 'BASE64_64':
$generatedValue = Str::random(64);
break;
case 'BASE64_128':
$generatedValue = Str::random(128);
break;
case 'BASE64':
$generatedValue = Str::random(32);
break;
case 'USER':
$generatedValue = Str::random(16);
break;
}
if (!$foundEnv) {
EnvironmentVariable::create([
'key' => $key,
'value' => $generatedValue,
'is_build_time' => false,
'service_id' => $this->id,
'is_preview' => false,
]);
}
}
} else {
if ($value->contains(':-')) {
$key = $value->before(':');
$defaultValue = $value->after(':-');
} else if ($value->contains('-')) {
$key = $value->before('-');
$defaultValue = $value->after('-');
} else if ($value->contains(':?')) {
$key = $value->before(':');
$defaultValue = $value->after(':?');
} else if ($value->contains('?')) {
$key = $value->before('?');
$defaultValue = $value->after('?');
} else {
$key = $value;
$defaultValue = null;
}
if ($foundEnv) {
$defaultValue = data_get($foundEnv, 'value');
}
EnvironmentVariable::updateOrCreate([
'key' => $key,
'service_id' => $this->id,
], [
'value' => $defaultValue,
'is_build_time' => false,
'service_id' => $this->id,
'is_preview' => false,
]);
}
}
}
// Add labels to the service
if ($savedService->serviceType()) {
$fqdns = generateServiceSpecificFqdns($savedService, forTraefik: true);
} else {
$fqdns = collect(data_get($savedService, 'fqdns'));
}
$defaultLabels = defaultLabels($this->id, $containerName, type: 'service', subType: $isDatabase ? 'database' : 'application', subId: $savedService->id);
$serviceLabels = $serviceLabels->merge($defaultLabels);
if (!$isDatabase && $fqdns->count() > 0) {
if ($fqdns) {
$serviceLabels = $serviceLabels->merge(fqdnLabelsForTraefik($this->uuid, $fqdns, true));
}
}
if ($this->server->isLogDrainEnabled() && $savedService->isLogDrainEnabled()) {
data_set($service, 'logging', [
'driver' => 'fluentd',
'options' => [
'fluentd-address' => "tcp://127.0.0.1:24224",
'fluentd-async' => "true",
'fluentd-sub-second-precision' => "true",
]
]);
}
data_set($service, 'labels', $serviceLabels->toArray());
data_forget($service, 'is_database');
data_set($service, 'restart', RESTART_MODE);
data_set($service, 'container_name', $containerName);
data_forget($service, 'volumes.*.content');
data_forget($service, 'volumes.*.isDirectory');
// Remove unnecessary variables from service.environment
// $withoutServiceEnvs = collect([]);
// collect(data_get($service, 'environment'))->each(function ($value, $key) use ($withoutServiceEnvs) {
// ray($key, $value);
// if (!Str::of($key)->startsWith('$SERVICE_') && !Str::of($value)->startsWith('SERVICE_')) {
// $k = Str::of($value)->before("=");
// $v = Str::of($value)->after("=");
// $withoutServiceEnvs->put($k->value(), $v->value());
// }
// });
// ray($withoutServiceEnvs);
// data_set($service, 'environment', $withoutServiceEnvs->toArray());
return $service;
});
$finalServices = [
'version' => $dockerComposeVersion,
'services' => $services->toArray(),
'volumes' => $topLevelVolumes->toArray(),
'networks' => $topLevelNetworks->toArray(),
];
$this->docker_compose_raw = Yaml::dump($yaml, 10, 2);
$this->docker_compose = Yaml::dump($finalServices, 10, 2);
$this->save();
$this->saveComposeConfigs();
return collect([]);
} else {
return collect([]);
}
return parseDockerComposeFile($this, $isNew);
}
}

2
app/Traits/ExecuteRemoteCommand.php
View File

@@ -12,6 +12,7 @@ use Illuminate\Support\Str;
trait ExecuteRemoteCommand
{
public ?string $save = null;
public static int $batch_counter = 0;
public function execute_remote_command(...$commands)
{
static::$batch_counter++;
@@ -24,7 +25,6 @@ trait ExecuteRemoteCommand
throw new \RuntimeException('Server is not set or is not an instance of Server model');
}
$commandsText->each(function ($single_command) {
$command = data_get($single_command, 'command') ?? $single_command[0] ?? null;
if ($command === null) {