dave/coolify
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Merge branch 'next' into auto-cleanup-improvements

Browse Source
This commit is contained in:
Andras Bacsai
2024-08-26 11:13:40 +02:00
committed by GitHub
parent 0337c4e79d 1a9e1bcc88
commit d748d8b7c6
116 changed files with 3666 additions and 1157 deletions
Download Patch File Download Diff File Expand all files Collapse all files

122
app/Jobs/ApplicationDeploymentJob.php
View File

@@ -157,7 +157,7 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
private ?string $coolify_variables = null;
private bool $preserveRepository = true;
private bool $preserveRepository = false;
public $tries = 1;
@@ -198,9 +198,13 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
$this->container_name = generateApplicationContainerName($this->application, $this->pull_request_id);
if ($this->application->settings->custom_internal_name && ! $this->application->settings->is_consistent_container_name_enabled) {
$this->container_name = $this->application->settings->custom_internal_name;
if ($this->pull_request_id === 0) {
$this->container_name = $this->application->settings->custom_internal_name;
} else {
$this->container_name = "{$this->application->settings->custom_internal_name}-pr-{$this->pull_request_id}";
}
}
ray('New container name: ', $this->container_name);
ray('New container name: ', $this->container_name)->green();
savePrivateKeyToFs($this->server);
$this->saved_outputs = collect();
@@ -276,6 +280,7 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
$this->original_server = $this->server;
} else {
$this->build_server = $buildServers->random();
$this->application_deployment_queue->build_server_id = $this->build_server->id;
$this->application_deployment_queue->addLogEntry("Found a suitable build server ({$this->build_server->name}).");
$this->original_server = $this->server;
$this->use_build_server = true;
@@ -414,15 +419,43 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
$this->prepare_builder_image();
$this->check_git_if_build_needed();
$this->clone_repository();
if ($this->preserveRepository) {
foreach ($this->application->fileStorages as $fileStorage) {
$path = $fileStorage->fs_path;
$saveName = 'file_stat_'.$fileStorage->id;
$realPathInGit = str($path)->replace($this->application->workdir(), $this->workdir)->value();
// check if the file is a directory or a file inside the repository
$this->execute_remote_command(
[executeInDocker($this->deployment_uuid, "stat -c '%F' {$realPathInGit}"), 'hidden' => true, 'ignore_errors' => true, 'save' => $saveName]
);
if ($this->saved_outputs->has($saveName)) {
$fileStat = $this->saved_outputs->get($saveName);
if ($fileStat->value() === 'directory' && ! $fileStorage->is_directory) {
$fileStorage->is_directory = true;
$fileStorage->content = null;
$fileStorage->save();
$fileStorage->deleteStorageOnServer();
$fileStorage->saveStorageOnServer();
} elseif ($fileStat->value() === 'regular file' && $fileStorage->is_directory) {
$fileStorage->is_directory = false;
$fileStorage->is_based_on_git = true;
$fileStorage->save();
$fileStorage->deleteStorageOnServer();
$fileStorage->saveStorageOnServer();
}
}
}
}
$this->generate_image_names();
$this->cleanup_git();
$this->application->loadComposeFile(isInit: false);
if ($this->application->settings->is_raw_compose_deployment_enabled) {
$this->application->parseRawCompose();
$this->application->oldRawParser();
$yaml = $composeFile = $this->application->docker_compose_raw;
$this->save_environment_variables();
} else {
$composeFile = $this->application->parseCompose(pull_request_id: $this->pull_request_id, preview_id: data_get($this, 'preview.id'));
$composeFile = $this->application->oldParser(pull_request_id: $this->pull_request_id, preview_id: data_get($this->preview, 'id'));
$this->save_environment_variables();
if (! is_null($this->env_filename)) {
$services = collect($composeFile['services']);
@@ -439,7 +472,7 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
return;
}
$yaml = Yaml::dump($composeFile->toArray(), 10);
$yaml = Yaml::dump(convertToArray($composeFile), 10);
}
$this->docker_compose_base64 = base64_encode($yaml);
$this->execute_remote_command([
@@ -480,6 +513,7 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
}
// Start compose file
$server_workdir = $this->application->workdir();
if ($this->application->settings->is_raw_compose_deployment_enabled) {
if ($this->docker_compose_custom_start_command) {
$this->write_deployment_configurations();
@@ -488,7 +522,6 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
);
} else {
$this->write_deployment_configurations();
$server_workdir = $this->application->workdir();
$this->docker_compose_location = '/docker-compose.yaml';
$command = "{$this->coolify_variables} docker compose";
@@ -508,15 +541,27 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
);
} else {
$command = "{$this->coolify_variables} docker compose";
if ($this->env_filename) {
$command .= " --env-file {$this->workdir}/{$this->env_filename}";
}
$command .= " --project-name {$this->application->uuid} --project-directory {$this->workdir} -f {$this->workdir}{$this->docker_compose_location} up -d";
if ($this->preserveRepository) {
if ($this->env_filename) {
$command .= " --env-file {$server_workdir}/{$this->env_filename}";
}
$command .= " --project-name {$this->application->uuid} --project-directory {$server_workdir} -f {$server_workdir}{$this->docker_compose_location} up -d";
$this->write_deployment_configurations();
$this->execute_remote_command(
['command' => $command, 'hidden' => true],
);
} else {
if ($this->env_filename) {
$command .= " --env-file {$this->workdir}/{$this->env_filename}";
}
$command .= " --project-name {$this->application->uuid} --project-directory {$this->workdir} -f {$this->workdir}{$this->docker_compose_location} up -d";
$this->execute_remote_command(
[executeInDocker($this->deployment_uuid, $command), 'hidden' => true],
);
$this->write_deployment_configurations();
}
$this->write_deployment_configurations();
$this->execute_remote_command(
[executeInDocker($this->deployment_uuid, $command), 'hidden' => true],
);
}
}
@@ -610,15 +655,16 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
[
"mkdir -p $this->configuration_dir",
],
// removing this now as we are using docker cp
// [
// "rm -rf $this->configuration_dir/{*,.*}",
// ],
[
"docker cp {$this->deployment_uuid}:{$this->workdir}/. {$this->configuration_dir}",
],
);
}
foreach ($this->application->fileStorages as $fileStorage) {
if (! $fileStorage->is_based_on_git && ! $fileStorage->is_directory) {
$fileStorage->saveStorageOnServer();
}
}
if ($this->use_build_server) {
$this->server = $this->build_server;
}
@@ -851,11 +897,13 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
$url = str($this->preview->fqdn)->replace('http://', '')->replace('https://', '');
$envs->push("COOLIFY_URL={$url}");
}
if ($this->application->environment_variables_preview->where('key', 'COOLIFY_BRANCH')->isEmpty()) {
$envs->push("COOLIFY_BRANCH={$local_branch}");
}
if ($this->application->environment_variables_preview->where('key', 'COOLIFY_CONTAINER_NAME')->isEmpty()) {
$envs->push("COOLIFY_CONTAINER_NAME={$this->container_name}");
if ($this->application->compose_parsing_version === '1' || $this->application->compose_parsing_version === '2') {
if ($this->application->environment_variables_preview->where('key', 'COOLIFY_BRANCH')->isEmpty()) {
$envs->push("COOLIFY_BRANCH={$local_branch}");
}
if ($this->application->environment_variables_preview->where('key', 'COOLIFY_CONTAINER_NAME')->isEmpty()) {
$envs->push("COOLIFY_CONTAINER_NAME={$this->container_name}");
}
}
foreach ($sorted_environment_variables_preview as $env) {
$real_value = $env->real_value;
@@ -897,11 +945,13 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
$url = str($this->application->fqdn)->replace('http://', '')->replace('https://', '');
$envs->push("COOLIFY_URL={$url}");
}
if ($this->application->environment_variables->where('key', 'COOLIFY_BRANCH')->isEmpty()) {
$envs->push("COOLIFY_BRANCH={$local_branch}");
}
if ($this->application->environment_variables->where('key', 'COOLIFY_CONTAINER_NAME')->isEmpty()) {
$envs->push("COOLIFY_CONTAINER_NAME={$this->container_name}");
if ($this->application->compose_parsing_version === '1' || $this->application->compose_parsing_version === '2') {
if ($this->application->environment_variables->where('key', 'COOLIFY_BRANCH')->isEmpty()) {
$envs->push("COOLIFY_BRANCH={$local_branch}");
}
if ($this->application->environment_variables->where('key', 'COOLIFY_CONTAINER_NAME')->isEmpty()) {
$envs->push("COOLIFY_CONTAINER_NAME={$this->container_name}");
}
}
foreach ($sorted_environment_variables as $env) {
$real_value = $env->real_value;
@@ -1708,13 +1758,20 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
if (count($this->application->ports_mappings_array) > 0 && $this->pull_request_id === 0) {
$docker_compose['services'][$this->container_name]['ports'] = $this->application->ports_mappings_array;
}
if (count($persistent_storages) > 0) {
$docker_compose['services'][$this->container_name]['volumes'] = $persistent_storages;
if (! data_get($docker_compose, 'services.'.$this->container_name.'.volumes')) {
$docker_compose['services'][$this->container_name]['volumes'] = [];
}
$docker_compose['services'][$this->container_name]['volumes'] = array_merge($docker_compose['services'][$this->container_name]['volumes'], $persistent_storages);
}
if (count($persistent_file_volumes) > 0) {
$docker_compose['services'][$this->container_name]['volumes'] = $persistent_file_volumes->map(function ($item) {
if (! data_get($docker_compose, 'services.'.$this->container_name.'.volumes')) {
$docker_compose['services'][$this->container_name]['volumes'] = [];
}
$docker_compose['services'][$this->container_name]['volumes'] = array_merge($docker_compose['services'][$this->container_name]['volumes'], $persistent_file_volumes->map(function ($item) {
return "$item->fs_path:$item->mount_path";
})->toArray();
})->toArray());
}
if (count($volume_names) > 0) {
$docker_compose['volumes'] = $volume_names;
@@ -2140,7 +2197,6 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf");
} else {
$dockerfile->splice(1, 0, "ARG {$env->key}={$env->real_value}");
}
$dockerfile->splice(1, 0, "ARG {$env->key}={$env->real_value}");
}
}
$dockerfile_base64 = base64_encode($dockerfile->implode("\n"));

30
app/Jobs/DatabaseBackupJob.php
View File

@@ -56,6 +56,8 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
public ?string $backup_output = null;
public ?string $postgres_password = null;
public ?S3Storage $s3 = null;
public function __construct($backup)
@@ -89,8 +91,6 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
public function handle(): void
{
try {
BackupCreated::dispatch($this->team->id);
// Check if team is exists
if (is_null($this->team)) {
$this->backup->update(['status' => 'failed']);
@@ -99,6 +99,9 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
return;
}
BackupCreated::dispatch($this->team->id);
$status = str(data_get($this->database, 'status'));
if (! $status->startsWith('running') && $this->database->id !== 0) {
ray('database not running');
@@ -134,6 +137,13 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
} else {
$databasesToBackup = $this->database->postgres_user;
}
$this->postgres_password = $envs->filter(function ($env) {
return str($env)->startsWith('POSTGRES_PASSWORD=');
})->first();
if ($this->postgres_password) {
$this->postgres_password = str($this->postgres_password)->after('POSTGRES_PASSWORD=')->value();
}
} elseif (str($databaseType)->contains('mysql')) {
$this->container_name = "{$this->database->name}-$serviceUuid";
$this->directory_name = $serviceName.'-'.$this->container_name;
@@ -336,7 +346,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
$url = $this->database->internal_db_url;
if ($databaseWithCollections === 'all') {
$commands[] = 'mkdir -p '.$this->backup_dir;
if (str($this->database->image)->startsWith('mongo:4.0')) {
if (str($this->database->image)->startsWith('mongo:4')) {
$commands[] = "docker exec $this->container_name mongodump --uri=$url --gzip --archive > $this->backup_location";
} else {
$commands[] = "docker exec $this->container_name mongodump --authenticationDatabase=admin --uri=$url --gzip --archive > $this->backup_location";
@@ -351,13 +361,13 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
}
$commands[] = 'mkdir -p '.$this->backup_dir;
if ($collectionsToExclude->count() === 0) {
if (str($this->database->image)->startsWith('mongo:4.0')) {
if (str($this->database->image)->startsWith('mongo:4')) {
$commands[] = "docker exec $this->container_name mongodump --uri=$url --gzip --archive > $this->backup_location";
} else {
$commands[] = "docker exec $this->container_name mongodump --authenticationDatabase=admin --uri=$url --db $databaseName --gzip --archive > $this->backup_location";
}
} else {
if (str($this->database->image)->startsWith('mongo:4.0')) {
if (str($this->database->image)->startsWith('mongo:4')) {
$commands[] = "docker exec $this->container_name mongodump --uri=$url --gzip --excludeCollection ".$collectionsToExclude->implode(' --excludeCollection ')." --archive > $this->backup_location";
} else {
$commands[] = "docker exec $this->container_name mongodump --authenticationDatabase=admin --uri=$url --db $databaseName --gzip --excludeCollection ".$collectionsToExclude->implode(' --excludeCollection ')." --archive > $this->backup_location";
@@ -381,7 +391,13 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
{
try {
$commands[] = 'mkdir -p '.$this->backup_dir;
$commands[] = "docker exec $this->container_name pg_dump --format=custom --no-acl --no-owner --username {$this->database->postgres_user} $database > $this->backup_location";
$backupCommand = 'docker exec';
if ($this->postgres_password) {
$backupCommand .= " -e PGPASSWORD=$this->postgres_password";
}
$backupCommand .= " $this->container_name pg_dump --format=custom --no-acl --no-owner --username {$this->database->postgres_user} $database > $this->backup_location";
$commands[] = $backupCommand;
$this->backup_output = instant_remote_process($commands, $this->server);
$this->backup_output = trim($this->backup_output);
if ($this->backup_output === '') {
@@ -452,7 +468,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
if ($this->backup->number_of_backups_locally === 0) {
$deletable = $this->backup->executions()->where('status', 'success');
} else {
$deletable = $this->backup->executions()->where('status', 'success')->orderByDesc('created_at')->skip($this->backup->number_of_backups_locally - 1);
$deletable = $this->backup->executions()->where('status', 'success')->skip($this->backup->number_of_backups_locally - 1);
}
foreach ($deletable->get() as $execution) {
delete_backup_locally($execution->filename, $this->server);

13
app/Jobs/DockerCleanupJob.php
View File

@@ -19,7 +19,9 @@ class DockerCleanupJob implements ShouldBeEncrypted, ShouldQueue
public $timeout = 300;
public int|string|null $usageBefore = null;
public $tries = 2;
public ?string $usageBefore = null;
public function __construct(public Server $server) {}
@@ -36,6 +38,10 @@ class DockerCleanupJob implements ShouldBeEncrypted, ShouldQueue
return;
}
return;
}
try {
$this->usageBefore = $this->server->getDiskUsage();
if (str($this->usageBefore)->isEmpty() || $this->usageBefore === null || $this->usageBefore === 0) {
Log::info('DockerCleanupJob force cleanup on ' . $this->server->name);
@@ -44,7 +50,7 @@ class DockerCleanupJob implements ShouldBeEncrypted, ShouldQueue
return;
}
if ($this->usageBefore >= $this->server->settings->cleanup_after_percentage) {
CleanupDocker::run(server: $this->server, force: false);
CleanupDocker::run(server: $this->server);
$usageAfter = $this->server->getDiskUsage();
if ($usageAfter < $this->usageBefore) {
$this->server->team?->notify(new DockerCleanup($this->server, 'Saved ' . ($this->usageBefore - $usageAfter) . '% disk space.'));
@@ -56,7 +62,8 @@ class DockerCleanupJob implements ShouldBeEncrypted, ShouldQueue
Log::info('No need to clean up ' . $this->server->name);
}
} catch (\Throwable $e) {
ray($e->getMessage());
CleanupDocker::run(server: $this->server);
Log::error('DockerCleanupJob failed: '.$e->getMessage());
throw $e;
}
}