Merge branch 'coollabsio:main' into fix-redis-db-ui

This commit is contained in:
peaklabs-dev
2024-09-26 20:02:05 +02:00
committed by GitHub
364 changed files with 13628 additions and 4875 deletions

View File

@@ -6,7 +6,9 @@ use App\Enums\ApplicationDeploymentStatus;
use Illuminate\Database\Eloquent\Casts\Attribute;
use Illuminate\Database\Eloquent\Relations\HasMany;
use Illuminate\Database\Eloquent\SoftDeletes;
use Illuminate\Process\InvokedProcess;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Process;
use Illuminate\Support\Str;
use OpenApi\Attributes as OA;
use RuntimeException;
@@ -102,6 +104,8 @@ class Application extends BaseModel
{
use SoftDeletes;
private static $parserVersion = '3';
protected $guarded = [];
protected $appends = ['server_status'];
@@ -125,7 +129,7 @@ class Application extends BaseModel
ApplicationSetting::create([
'application_id' => $application->id,
]);
$application->compose_parsing_version = '2';
$application->compose_parsing_version = self::$parserVersion;
$application->save();
});
static::forceDeleting(function ($application) {
@@ -138,6 +142,7 @@ class Application extends BaseModel
$task->delete();
}
$application->tags()->detach();
$application->previews()->delete();
});
}
@@ -146,12 +151,64 @@ class Application extends BaseModel
return Application::whereRelation('environment.project.team', 'id', $teamId)->orderBy('name');
}
public function getContainersToStop(bool $previewDeployments = false): array
{
$containers = $previewDeployments
? getCurrentApplicationContainerStatus($this->destination->server, $this->id, includePullrequests: true)
: getCurrentApplicationContainerStatus($this->destination->server, $this->id, 0);
return $containers->pluck('Names')->toArray();
}
public function stopContainers(array $containerNames, $server, int $timeout = 600)
{
$processes = [];
foreach ($containerNames as $containerName) {
$processes[$containerName] = $this->stopContainer($containerName, $server, $timeout);
}
$startTime = time();
while (count($processes) > 0) {
$finishedProcesses = array_filter($processes, function ($process) {
return ! $process->running();
});
foreach ($finishedProcesses as $containerName => $process) {
unset($processes[$containerName]);
$this->removeContainer($containerName, $server);
}
if (time() - $startTime >= $timeout) {
$this->forceStopRemainingContainers(array_keys($processes), $server);
break;
}
usleep(100000);
}
}
public function stopContainer(string $containerName, $server, int $timeout): InvokedProcess
{
return Process::timeout($timeout)->start("docker stop --time=$timeout $containerName");
}
public function removeContainer(string $containerName, $server)
{
instant_remote_process(command: ["docker rm -f $containerName"], server: $server, throwError: false);
}
public function forceStopRemainingContainers(array $containerNames, $server)
{
foreach ($containerNames as $containerName) {
instant_remote_process(command: ["docker kill $containerName"], server: $server, throwError: false);
$this->removeContainer($containerName, $server);
}
}
public function delete_configurations()
{
$server = data_get($this, 'destination.server');
$workdir = $this->workdir();
if (str($workdir)->endsWith($this->uuid)) {
ray('Deleting workdir');
instant_remote_process(['rm -rf '.$this->workdir()], $server, false);
}
}
@@ -173,6 +230,13 @@ class Application extends BaseModel
}
}
public function delete_connected_networks($uuid)
{
$server = data_get($this, 'destination.server');
instant_remote_process(["docker network disconnect {$uuid} coolify-proxy"], $server, false);
instant_remote_process(["docker network rm {$uuid}"], $server, false);
}
public function additional_servers()
{
return $this->belongsToMany(Server::class, 'additional_destinations')
@@ -412,23 +476,6 @@ class Application extends BaseModel
);
}
public function dockerComposePrLocation(): Attribute
{
return Attribute::make(
set: function ($value) {
if (is_null($value) || $value === '') {
return '/docker-compose.yaml';
} else {
if ($value !== '/') {
return Str::start(Str::replaceEnd('/', '', $value), '/');
}
return Str::start($value, '/');
}
}
);
}
public function baseDirectory(): Attribute
{
return Attribute::make(
@@ -479,12 +526,12 @@ class Application extends BaseModel
$main_server_status = $this->destination->server->isFunctional();
foreach ($additional_servers_status as $status) {
$server_status = str($status)->before(':')->value();
if ($main_server_status !== $server_status) {
if ($server_status !== 'running') {
return false;
}
}
return true;
return $main_server_status;
}
}
);
@@ -1040,7 +1087,7 @@ class Application extends BaseModel
}
}
public function parseRawCompose()
public function oldRawParser()
{
try {
$yaml = Yaml::parse($this->docker_compose_raw);
@@ -1048,6 +1095,7 @@ class Application extends BaseModel
throw new \Exception($e->getMessage());
}
$services = data_get($yaml, 'services');
$commands = collect([]);
$services = collect($services)->map(function ($service) use ($commands) {
$serviceVolumes = collect(data_get($service, 'volumes', []));
@@ -1100,9 +1148,11 @@ class Application extends BaseModel
instant_remote_process($commands, $this->destination->server, false);
}
public function parseCompose(int $pull_request_id = 0, ?int $preview_id = null)
public function parse(int $pull_request_id = 0, ?int $preview_id = null)
{
if ($this->docker_compose_raw) {
if ($this->compose_parsing_version === '3') {
return newParser($this, $pull_request_id, $preview_id);
} elseif ($this->docker_compose_raw) {
return parseDockerComposeFile(resource: $this, isNew: false, pull_request_id: $pull_request_id, preview_id: $preview_id);
} else {
return collect([]);
@@ -1154,7 +1204,7 @@ class Application extends BaseModel
if ($composeFileContent) {
$this->docker_compose_raw = $composeFileContent;
$this->save();
$parsedServices = $this->parseCompose();
$parsedServices = $this->parse();
if ($this->docker_compose_domains) {
$json = collect(json_decode($this->docker_compose_domains));
$names = collect(data_get($parsedServices, 'services'))->keys()->toArray();
@@ -1178,7 +1228,6 @@ class Application extends BaseModel
} else {
throw new \RuntimeException("Docker Compose file not found at: $workdir$composeFile<br><br>Check if you used the right extension (.yaml or .yml) in the compose file name.");
}
}
public function parseContainerLabels(?ApplicationPreview $preview = null)

View File

@@ -12,9 +12,9 @@ class ApplicationPreview extends BaseModel
protected static function booted()
{
static::deleting(function ($preview) {
if ($preview->application->build_pack === 'dockercompose') {
if (data_get($preview, 'application.build_pack') === 'dockercompose') {
$server = $preview->application->destination->server;
$composeFile = $preview->application->parseCompose(pull_request_id: $preview->pull_request_id);
$composeFile = $preview->application->parse(pull_request_id: $preview->pull_request_id);
$volumes = data_get($composeFile, 'volumes');
$networks = data_get($composeFile, 'networks');
$networkKeys = collect($networks)->keys();

View File

@@ -6,7 +6,6 @@ use App\Models\EnvironmentVariable as ModelsEnvironmentVariable;
use Illuminate\Database\Eloquent\Casts\Attribute;
use Illuminate\Database\Eloquent\Model;
use OpenApi\Attributes as OA;
use Symfony\Component\Yaml\Yaml;
use Visus\Cuid2\Cuid2;
#[OA\Schema(
@@ -97,8 +96,22 @@ class EnvironmentVariable extends Model
$resource = Application::find($this->application_id);
} elseif ($this->service_id) {
$resource = Service::find($this->service_id);
} elseif ($this->database_id) {
$resource = getResourceByUuid($this->parameters['database_uuid'], data_get(auth()->user()->currentTeam(), 'id'));
} elseif ($this->standalone_postgresql_id) {
$resource = StandalonePostgresql::find($this->standalone_postgresql_id);
} elseif ($this->standalone_redis_id) {
$resource = StandaloneRedis::find($this->standalone_redis_id);
} elseif ($this->standalone_mongodb_id) {
$resource = StandaloneMongodb::find($this->standalone_mongodb_id);
} elseif ($this->standalone_mysql_id) {
$resource = StandaloneMysql::find($this->standalone_mysql_id);
} elseif ($this->standalone_mariadb_id) {
$resource = StandaloneMariadb::find($this->standalone_mariadb_id);
} elseif ($this->standalone_keydb_id) {
$resource = StandaloneKeydb::find($this->standalone_keydb_id);
} elseif ($this->standalone_dragonfly_id) {
$resource = StandaloneDragonfly::find($this->standalone_dragonfly_id);
} elseif ($this->standalone_clickhouse_id) {
$resource = StandaloneClickhouse::find($this->standalone_clickhouse_id);
}
return $resource;
@@ -122,63 +135,6 @@ class EnvironmentVariable extends Model
);
}
protected function isFoundInCompose(): Attribute
{
return Attribute::make(
get: function () {
if (! $this->application_id) {
return true;
}
$found_in_compose = false;
$found_in_args = false;
$resource = $this->resource();
$compose = data_get($resource, 'docker_compose_raw');
if (! $compose) {
return true;
}
$yaml = Yaml::parse($compose);
$services = collect(data_get($yaml, 'services'));
if ($services->isEmpty()) {
return false;
}
foreach ($services as $service) {
$environments = collect(data_get($service, 'environment'));
$args = collect(data_get($service, 'build.args'));
if ($environments->isEmpty() && $args->isEmpty()) {
$found_in_compose = false;
break;
}
$found_in_compose = $environments->contains(function ($item) {
if (str($item)->contains('=')) {
$item = str($item)->before('=');
}
return strpos($item, $this->key) !== false;
});
if ($found_in_compose) {
break;
}
$found_in_args = $args->contains(function ($item) {
if (str($item)->contains('=')) {
$item = str($item)->before('=');
}
return strpos($item, $this->key) !== false;
});
if ($found_in_args) {
break;
}
}
return $found_in_compose || $found_in_args;
}
);
}
protected function isShared(): Attribute
{
return Attribute::make(
@@ -201,8 +157,10 @@ class EnvironmentVariable extends Model
$environment_variable = trim($environment_variable);
$sharedEnvsFound = str($environment_variable)->matchAll('/{{(.*?)}}/');
if ($sharedEnvsFound->isEmpty()) {
return $environment_variable;
}
foreach ($sharedEnvsFound as $sharedEnv) {
$type = str($sharedEnv)->match('/(.*?)\./');
if (! collect(SHARED_VARIABLE_TYPES)->contains($type)) {

View File

@@ -37,6 +37,30 @@ class InstanceSettings extends Model implements SendsEmail
);
}
public function updateCheckFrequency(): Attribute
{
return Attribute::make(
set: function ($value) {
return translate_cron_expression($value);
},
get: function ($value) {
return translate_cron_expression($value);
}
);
}
public function autoUpdateFrequency(): Attribute
{
return Attribute::make(
set: function ($value) {
return translate_cron_expression($value);
},
get: function ($value) {
return translate_cron_expression($value);
}
);
}
public static function get()
{
return InstanceSettings::findOrFail(0);

View File

@@ -24,8 +24,9 @@ class LocalFileVolume extends BaseModel
return $this->morphTo('resource');
}
public function deleteStorageOnServer()
public function loadStorageOnServer()
{
$this->load(['service']);
$isService = data_get($this->resource, 'service');
if ($isService) {
$workdir = $this->resource->service->workdir();
@@ -35,17 +36,46 @@ class LocalFileVolume extends BaseModel
$server = $this->resource->destination->server;
}
$commands = collect([]);
$fs_path = data_get($this, 'fs_path');
$isFile = instant_remote_process(["test -f $fs_path && echo OK || echo NOK"], $server);
$isDir = instant_remote_process(["test -d $fs_path && echo OK || echo NOK"], $server);
if ($fs_path && $fs_path != '/' && $fs_path != '.' && $fs_path != '..') {
ray($isFile, $isDir);
$path = data_get_str($this, 'fs_path');
if ($path->startsWith('.')) {
$path = $path->after('.');
$path = $workdir.$path;
}
$isFile = instant_remote_process(["test -f $path && echo OK || echo NOK"], $server);
if ($isFile === 'OK') {
$content = instant_remote_process(["cat $path"], $server, false);
$this->content = $content;
$this->is_directory = false;
$this->save();
}
}
public function deleteStorageOnServer()
{
$this->load(['service']);
$isService = data_get($this->resource, 'service');
if ($isService) {
$workdir = $this->resource->service->workdir();
$server = $this->resource->service->server;
} else {
$workdir = $this->resource->workdir();
$server = $this->resource->destination->server;
}
$commands = collect([]);
$path = data_get_str($this, 'fs_path');
if ($path->startsWith('.')) {
$path = $path->after('.');
$path = $workdir.$path;
}
$isFile = instant_remote_process(["test -f $path && echo OK || echo NOK"], $server);
$isDir = instant_remote_process(["test -d $path && echo OK || echo NOK"], $server);
if ($path && $path != '/' && $path != '.' && $path != '..') {
if ($isFile === 'OK') {
$commands->push("rm -rf $fs_path > /dev/null 2>&1 || true");
$commands->push("rm -rf $path > /dev/null 2>&1 || true");
} elseif ($isDir === 'OK') {
$commands->push("rm -rf $fs_path > /dev/null 2>&1 || true");
$commands->push("rmdir $fs_path > /dev/null 2>&1 || true");
$commands->push("rm -rf $path > /dev/null 2>&1 || true");
$commands->push("rmdir $path > /dev/null 2>&1 || true");
}
}
if ($commands->count() > 0) {
@@ -55,6 +85,7 @@ class LocalFileVolume extends BaseModel
public function saveStorageOnServer()
{
$this->load(['service']);
$isService = data_get($this->resource, 'service');
if ($isService) {
$workdir = $this->resource->service->workdir();
@@ -74,30 +105,36 @@ class LocalFileVolume extends BaseModel
$commands->push("mkdir -p $parent_dir > /dev/null 2>&1 || true");
}
}
$fileVolume = $this;
$path = str(data_get($fileVolume, 'fs_path'));
$content = data_get($fileVolume, 'content');
$path = data_get_str($this, 'fs_path');
$content = data_get($this, 'content');
if ($path->startsWith('.')) {
$path = $path->after('.');
$path = $workdir.$path;
}
$isFile = instant_remote_process(["test -f $path && echo OK || echo NOK"], $server);
$isDir = instant_remote_process(["test -d $path && echo OK || echo NOK"], $server);
if ($isFile == 'OK' && $fileVolume->is_directory) {
if ($isFile == 'OK' && $this->is_directory) {
$content = instant_remote_process(["cat $path"], $server, false);
$fileVolume->is_directory = false;
$fileVolume->content = $content;
$fileVolume->save();
$this->is_directory = false;
$this->content = $content;
$this->save();
FileStorageChanged::dispatch(data_get($server, 'team_id'));
throw new \Exception('The following file is a file on the server, but you are trying to mark it as a directory. Please delete the file on the server or mark it as directory.');
} elseif ($isDir == 'OK' && ! $fileVolume->is_directory) {
$fileVolume->is_directory = true;
$fileVolume->save();
throw new \Exception('The following file is a directory on the server, but you are trying to mark it as a file. <br><br>Please delete the directory on the server or mark it as directory.');
} elseif ($isDir == 'OK' && ! $this->is_directory) {
if ($path == '/' || $path == '.' || $path == '..' || $path == '' || str($path)->isEmpty() || is_null($path)) {
$this->is_directory = true;
$this->save();
throw new \Exception('The following file is a directory on the server, but you are trying to mark it as a file. <br><br>Please delete the directory on the server or mark it as directory.');
}
instant_remote_process([
"rm -fr $path",
"touch $path",
], $server, false);
FileStorageChanged::dispatch(data_get($server, 'team_id'));
}
if ($isDir == 'NOK' && ! $fileVolume->is_directory) {
$chmod = data_get($fileVolume, 'chmod');
$chown = data_get($fileVolume, 'chown');
if ($isDir == 'NOK' && ! $this->is_directory) {
$chmod = data_get($this, 'chmod');
$chown = data_get($this, 'chown');
if ($content) {
$content = base64_encode($content);
$commands->push("echo '$content' | base64 -d | tee $path > /dev/null");
@@ -111,7 +148,7 @@ class LocalFileVolume extends BaseModel
if ($chmod) {
$commands->push("chmod $chmod $path");
}
} elseif ($isDir == 'NOK' && $fileVolume->is_directory) {
} elseif ($isDir == 'NOK' && $this->is_directory) {
$commands->push("mkdir -p $path > /dev/null 2>&1 || true");
}

View File

@@ -2,6 +2,9 @@
namespace App\Models;
use DanHarrin\LivewireRateLimiting\WithRateLimiting;
use Illuminate\Support\Facades\Storage;
use Illuminate\Validation\ValidationException;
use OpenApi\Attributes as OA;
use phpseclib3\Crypt\PublicKeyLoader;
@@ -22,48 +25,144 @@ use phpseclib3\Crypt\PublicKeyLoader;
)]
class PrivateKey extends BaseModel
{
use WithRateLimiting;
protected $fillable = [
'name',
'description',
'private_key',
'is_git_related',
'team_id',
'fingerprint',
];
protected $casts = [
'private_key' => 'encrypted',
];
protected static function booted()
{
static::saving(function ($key) {
$privateKey = data_get($key, 'private_key');
if (substr($privateKey, -1) !== "\n") {
$key->private_key = $privateKey."\n";
$key->private_key = formatPrivateKey($key->private_key);
if (! self::validatePrivateKey($key->private_key)) {
throw ValidationException::withMessages([
'private_key' => ['The private key is invalid.'],
]);
}
$key->fingerprint = self::generateFingerprint($key->private_key);
if (self::fingerprintExists($key->fingerprint, $key->id)) {
throw ValidationException::withMessages([
'private_key' => ['This private key already exists.'],
]);
}
});
static::deleted(function ($key) {
self::deleteFromStorage($key);
});
}
public function getPublicKey()
{
return self::extractPublicKeyFromPrivate($this->private_key) ?? 'Error loading private key';
}
public static function ownedByCurrentTeam(array $select = ['*'])
{
$selectArray = collect($select)->concat(['id']);
return PrivateKey::whereTeamId(currentTeam()->id)->select($selectArray->all());
return self::whereTeamId(currentTeam()->id)->select($selectArray->all());
}
public function publicKey()
public static function validatePrivateKey($privateKey)
{
try {
return PublicKeyLoader::load($this->private_key)->getPublicKey()->toString('OpenSSH', ['comment' => '']);
PublicKeyLoader::load($privateKey);
return true;
} catch (\Throwable $e) {
return 'Error loading private key';
return false;
}
}
public function isEmpty()
public static function createAndStore(array $data)
{
if ($this->servers()->count() === 0 && $this->applications()->count() === 0 && $this->githubApps()->count() === 0 && $this->gitlabApps()->count() === 0) {
return true;
}
$privateKey = new self($data);
$privateKey->save();
$privateKey->storeInFileSystem();
return false;
return $privateKey;
}
public static function generateNewKeyPair($type = 'rsa')
{
try {
$instance = new self;
$instance->rateLimit(10);
$name = generate_random_name();
$description = 'Created by Coolify';
$keyPair = generateSSHKey($type === 'ed25519' ? 'ed25519' : 'rsa');
return [
'name' => $name,
'description' => $description,
'private_key' => $keyPair['private'],
'public_key' => $keyPair['public'],
];
} catch (\Throwable $e) {
throw new \Exception("Failed to generate new {$type} key: ".$e->getMessage());
}
}
public static function extractPublicKeyFromPrivate($privateKey)
{
try {
$key = PublicKeyLoader::load($privateKey);
return $key->getPublicKey()->toString('OpenSSH', ['comment' => '']);
} catch (\Throwable $e) {
return null;
}
}
public static function validateAndExtractPublicKey($privateKey)
{
$isValid = self::validatePrivateKey($privateKey);
$publicKey = $isValid ? self::extractPublicKeyFromPrivate($privateKey) : '';
return [
'isValid' => $isValid,
'publicKey' => $publicKey,
];
}
public function storeInFileSystem()
{
$filename = "ssh_key@{$this->uuid}";
Storage::disk('ssh-keys')->put($filename, $this->private_key);
return "/var/www/html/storage/app/ssh/keys/{$filename}";
}
public static function deleteFromStorage(self $privateKey)
{
$filename = "ssh_key@{$privateKey->uuid}";
Storage::disk('ssh-keys')->delete($filename);
}
public function getKeyLocation()
{
return "/var/www/html/storage/app/ssh/keys/ssh_key@{$this->uuid}";
}
public function updatePrivateKey(array $data)
{
$this->update($data);
$this->storeInFileSystem();
return $this;
}
public function servers()
@@ -85,4 +184,53 @@ class PrivateKey extends BaseModel
{
return $this->hasMany(GitlabApp::class);
}
public function isInUse()
{
return $this->servers()->exists()
|| $this->applications()->exists()
|| $this->githubApps()->exists()
|| $this->gitlabApps()->exists();
}
public function safeDelete()
{
if (! $this->isInUse()) {
$this->delete();
return true;
}
return false;
}
public static function generateFingerprint($privateKey)
{
try {
$key = PublicKeyLoader::load($privateKey);
$publicKey = $key->getPublicKey();
return $publicKey->getFingerprint('sha256');
} catch (\Throwable $e) {
return null;
}
}
private static function fingerprintExists($fingerprint, $excludeId = null)
{
$query = self::where('fingerprint', $fingerprint);
if (! is_null($excludeId)) {
$query->where('id', '!=', $excludeId);
}
return $query->exists();
}
public static function cleanupUnusedKeys()
{
self::ownedByCurrentTeam()->each(function ($privateKey) {
$privateKey->safeDelete();
});
}
}

View File

@@ -11,6 +11,7 @@ use OpenApi\Attributes as OA;
'id' => ['type' => 'integer'],
'uuid' => ['type' => 'string'],
'name' => ['type' => 'string'],
'description' => ['type' => 'string'],
'environments' => new OA\Property(
property: 'environments',
type: 'array',

View File

@@ -22,7 +22,8 @@ class ScheduledDatabaseBackup extends BaseModel
public function executions(): HasMany
{
return $this->hasMany(ScheduledDatabaseBackupExecution::class);
// Last execution first
return $this->hasMany(ScheduledDatabaseBackupExecution::class)->orderBy('created_at', 'desc');
}
public function s3()
@@ -34,4 +35,17 @@ class ScheduledDatabaseBackup extends BaseModel
{
return $this->hasMany(ScheduledDatabaseBackupExecution::class)->where('created_at', '>=', now()->subDays($days))->get();
}
public function server()
{
if ($this->database) {
if ($this->database->destination && $this->database->destination->server) {
$server = $this->database->destination->server;
return $server;
}
}
return null;
}
}

View File

@@ -26,6 +26,32 @@ class ScheduledTask extends BaseModel
public function executions(): HasMany
{
return $this->hasMany(ScheduledTaskExecution::class);
// Last execution first
return $this->hasMany(ScheduledTaskExecution::class)->orderBy('created_at', 'desc');
}
public function server()
{
if ($this->application) {
if ($this->application->destination && $this->application->destination->server) {
$server = $this->application->destination->server;
return $server;
}
} elseif ($this->service) {
if ($this->service->destination && $this->service->destination->server) {
$server = $this->service->destination->server;
return $server;
}
} elseif ($this->database) {
if ($this->database->destination && $this->database->destination->server) {
$server = $this->database->destination->server;
return $server;
}
}
return null;
}
}

View File

@@ -5,7 +5,6 @@ namespace App\Models;
use App\Actions\Server\InstallDocker;
use App\Enums\ProxyTypes;
use App\Jobs\PullSentinelImageJob;
use App\Notifications\Server\Revived;
use Illuminate\Database\Eloquent\Builder;
use Illuminate\Database\Eloquent\Casts\Attribute;
use Illuminate\Support\Collection;
@@ -112,6 +111,16 @@ class Server extends BaseModel
'proxy',
];
protected $fillable = [
'name',
'ip',
'port',
'user',
'description',
'private_key_id',
'team_id',
];
protected $guarded = [];
public static function isReachable()
@@ -146,6 +155,11 @@ class Server extends BaseModel
return $this->hasOne(ServerSetting::class);
}
public function proxySet()
{
return $this->proxyType() && $this->proxyType() !== 'NONE' && $this->isFunctional() && ! $this->isSwarmWorker() && ! $this->settings->is_build_server;
}
public function setupDefault404Redirect()
{
$dynamic_conf_path = $this->proxyPath().'/dynamic';
@@ -153,11 +167,11 @@ class Server extends BaseModel
$redirect_url = $this->proxy->redirect_url;
if ($proxy_type === ProxyTypes::TRAEFIK->value) {
$default_redirect_file = "$dynamic_conf_path/default_redirect_404.yaml";
} elseif ($proxy_type === 'CADDY') {
} elseif ($proxy_type === ProxyTypes::CADDY->value) {
$default_redirect_file = "$dynamic_conf_path/default_redirect_404.caddy";
}
if (empty($redirect_url)) {
if ($proxy_type === 'CADDY') {
if ($proxy_type === ProxyTypes::CADDY->value) {
$conf = ':80, :443 {
respond 404
}';
@@ -227,7 +241,7 @@ respond 404
$conf;
$base64 = base64_encode($conf);
} elseif ($proxy_type === 'CADDY') {
} elseif ($proxy_type === ProxyTypes::CADDY->value) {
$conf = ":80, :443 {
redir $redirect_url
}";
@@ -243,9 +257,6 @@ respond 404
"echo '$base64' | base64 -d | tee $default_redirect_file > /dev/null",
], $this);
if (config('app.env') == 'local') {
ray($conf);
}
if ($proxy_type === 'CADDY') {
$this->reloadCaddy();
}
@@ -295,6 +306,13 @@ respond 404
'service' => 'coolify-realtime',
'rule' => "Host(`{$host}`) && PathPrefix(`/app`)",
],
'coolify-terminal-ws' => [
'entryPoints' => [
0 => 'http',
],
'service' => 'coolify-terminal',
'rule' => "Host(`{$host}`) && PathPrefix(`/terminal/ws`)",
],
],
'services' => [
'coolify' => [
@@ -315,6 +333,15 @@ respond 404
],
],
],
'coolify-terminal' => [
'loadBalancer' => [
'servers' => [
0 => [
'url' => 'http://coolify-realtime:6002',
],
],
],
],
],
],
];
@@ -344,6 +371,16 @@ respond 404
'certresolver' => 'letsencrypt',
],
];
$traefik_dynamic_conf['http']['routers']['coolify-terminal-wss'] = [
'entryPoints' => [
0 => 'https',
],
'service' => 'coolify-terminal',
'rule' => "Host(`{$host}`) && PathPrefix(`/terminal/ws`)",
'tls' => [
'certresolver' => 'letsencrypt',
],
];
}
$yaml = Yaml::dump($traefik_dynamic_conf, 12, 2);
$yaml =
@@ -377,6 +414,9 @@ $schema://$host {
handle /app/* {
reverse_proxy coolify-realtime:6001
}
handle /terminal/ws {
reverse_proxy coolify-realtime:6002
}
reverse_proxy coolify:80
}";
$base64 = base64_encode($caddy_file);
@@ -649,7 +689,7 @@ $schema://$host {
}
}
public function getDiskUsage()
public function getDiskUsage(): ?string
{
return instant_remote_process(["df /| tail -1 | awk '{ print $5}' | sed 's/%//g'"], $this, false);
}
@@ -736,6 +776,18 @@ $schema://$host {
}
}
public function loadAllContainers(): Collection
{
if ($this->isFunctional()) {
$containers = instant_remote_process(["docker ps -a --format '{{json .}}'"], $this);
$containers = format_docker_command_output_to_json($containers);
return collect($containers);
}
return collect([]);
}
public function loadUnmanagedContainers(): Collection
{
if ($this->isFunctional()) {
@@ -782,9 +834,9 @@ $schema://$host {
$clickhouses = data_get($standaloneDocker, 'clickhouses', collect([]));
return $postgresqls->concat($redis)->concat($mongodbs)->concat($mysqls)->concat($mariadbs)->concat($keydbs)->concat($dragonflies)->concat($clickhouses);
})->filter(function ($item) {
})->flatten()->filter(function ($item) {
return data_get($item, 'name') !== 'coolify-db';
})->flatten();
});
}
public function applications()
@@ -828,6 +880,35 @@ $schema://$host {
return $this->hasMany(Service::class);
}
public function port(): Attribute
{
return Attribute::make(
get: function ($value) {
return preg_replace('/[^0-9]/', '', $value);
}
);
}
public function user(): Attribute
{
return Attribute::make(
get: function ($value) {
$sanitizedValue = preg_replace('/[^A-Za-z0-9\-_]/', '', $value);
return $sanitizedValue;
}
);
}
public function ip(): Attribute
{
return Attribute::make(
get: function ($value) {
return preg_replace('/[^0-9a-zA-Z.:%-]/', '', $value);
}
);
}
public function getIp(): Attribute
{
return Attribute::make(
@@ -880,7 +961,7 @@ $schema://$host {
public function muxFilename()
{
return "{$this->ip}_{$this->port}_{$this->user}";
return $this->uuid;
}
public function team()
@@ -900,10 +981,9 @@ $schema://$host {
public function isFunctional()
{
$isFunctional = $this->settings->is_reachable && $this->settings->is_usable && ! $this->settings->force_disabled;
['private_key_filename' => $private_key_filename, 'mux_filename' => $mux_filename] = server_ssh_configuration($this);
if (! $isFunctional) {
Storage::disk('ssh-keys')->delete($private_key_filename);
Storage::disk('ssh-mux')->delete($mux_filename);
Storage::disk('ssh-mux')->delete($this->muxFilename());
}
return $isFunctional;
@@ -955,9 +1035,10 @@ $schema://$host {
return data_get($this, 'settings.is_swarm_worker');
}
public function validateConnection()
public function validateConnection($isManualCheck = true)
{
config()->set('coolify.mux_enabled', false);
config()->set('constants.ssh.mux_enabled', ! $isManualCheck);
// ray('Manual Check: ' . ($isManualCheck ? 'true' : 'false'));
$server = Server::find($this->id);
if (! $server) {
@@ -967,7 +1048,6 @@ $schema://$host {
return ['uptime' => false, 'error' => 'Server skipped.'];
}
try {
// EC2 does not have `uptime` command, lol
instant_remote_process(['ls /'], $server);
$server->settings()->update([
'is_reachable' => true,
@@ -976,7 +1056,6 @@ $schema://$host {
'unreachable_count' => 0,
]);
if (data_get($server, 'unreachable_notification_sent') === true) {
// $server->team?->notify(new Revived($server));
$server->update(['unreachable_notification_sent' => false]);
}
@@ -1105,4 +1184,24 @@ $schema://$host {
{
return $this->settings->is_build_server;
}
public static function createWithPrivateKey(array $data, PrivateKey $privateKey)
{
$server = new self($data);
$server->privateKey()->associate($privateKey);
$server->save();
return $server;
}
public function updateWithPrivateKey(array $data, ?PrivateKey $privateKey = null)
{
$this->update($data);
if ($privateKey) {
$this->privateKey()->associate($privateKey);
$this->save();
}
return $this;
}
}

View File

@@ -2,6 +2,7 @@
namespace App\Models;
use Illuminate\Database\Eloquent\Casts\Attribute;
use Illuminate\Database\Eloquent\Model;
use OpenApi\Attributes as OA;
@@ -10,10 +11,10 @@ use OpenApi\Attributes as OA;
type: 'object',
properties: [
'id' => ['type' => 'integer'],
'cleanup_after_percentage' => ['type' => 'integer'],
'concurrent_builds' => ['type' => 'integer'],
'dynamic_timeout' => ['type' => 'integer'],
'force_disabled' => ['type' => 'boolean'],
'force_server_cleanup' => ['type' => 'boolean'],
'is_build_server' => ['type' => 'boolean'],
'is_cloudflare_tunnel' => ['type' => 'boolean'],
'is_jump_server' => ['type' => 'boolean'],
@@ -37,6 +38,8 @@ use OpenApi\Attributes as OA;
'metrics_history_days' => ['type' => 'integer'],
'metrics_refresh_rate_seconds' => ['type' => 'integer'],
'metrics_token' => ['type' => 'string'],
'docker_cleanup_frequency' => ['type' => 'string'],
'docker_cleanup_threshold' => ['type' => 'integer'],
'server_id' => ['type' => 'integer'],
'wildcard_domain' => ['type' => 'string'],
'created_at' => ['type' => 'string'],
@@ -47,8 +50,25 @@ class ServerSetting extends Model
{
protected $guarded = [];
protected $casts = [
'force_docker_cleanup' => 'boolean',
'docker_cleanup_threshold' => 'integer',
];
public function server()
{
return $this->belongsTo(Server::class);
}
public function dockerCleanupFrequency(): Attribute
{
return Attribute::make(
set: function ($value) {
return translate_cron_expression($value);
},
get: function ($value) {
return translate_cron_expression($value);
}
);
}
}

View File

@@ -6,10 +6,13 @@ use Illuminate\Database\Eloquent\Casts\Attribute;
use Illuminate\Database\Eloquent\Factories\HasFactory;
use Illuminate\Database\Eloquent\Relations\HasMany;
use Illuminate\Database\Eloquent\SoftDeletes;
use Illuminate\Process\InvokedProcess;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Process;
use Illuminate\Support\Facades\Storage;
use OpenApi\Attributes as OA;
use Spatie\Url\Url;
use Symfony\Component\Yaml\Yaml;
use Visus\Cuid2\Cuid2;
#[OA\Schema(
description: 'Service model',
@@ -23,6 +26,7 @@ use Symfony\Component\Yaml\Yaml;
'description' => ['type' => 'string', 'description' => 'The description of the service.'],
'docker_compose_raw' => ['type' => 'string', 'description' => 'The raw docker-compose.yml file of the service.'],
'docker_compose' => ['type' => 'string', 'description' => 'The docker-compose.yml file that is parsed and modified by Coolify.'],
'destination_type' => ['type' => 'string', 'description' => 'Destination type.'],
'destination_id' => ['type' => 'integer', 'description' => 'The unique identifier of the destination where the service is running.'],
'connect_to_docker_network' => ['type' => 'boolean', 'description' => 'The flag to connect the service to the predefined Docker network.'],
'is_container_label_escape_enabled' => ['type' => 'boolean', 'description' => 'The flag to enable the container label escape.'],
@@ -38,10 +42,20 @@ class Service extends BaseModel
{
use HasFactory, SoftDeletes;
private static $parserVersion = '3';
protected $guarded = [];
protected $appends = ['server_status'];
protected static function booted()
{
static::created(function ($service) {
$service->compose_parsing_version = self::$parserVersion;
$service->save();
});
}
public function isConfigurationChanged(bool $save = false)
{
$domains = $this->applications()->get()->pluck('fqdn')->sort()->toArray();
@@ -119,15 +133,81 @@ class Service extends BaseModel
return $this->morphToMany(Tag::class, 'taggable');
}
public function getContainersToStop(): array
{
$containersToStop = [];
$applications = $this->applications()->get();
foreach ($applications as $application) {
$containersToStop[] = "{$application->name}-{$this->uuid}";
}
$dbs = $this->databases()->get();
foreach ($dbs as $db) {
$containersToStop[] = "{$db->name}-{$this->uuid}";
}
return $containersToStop;
}
public function stopContainers(array $containerNames, $server, int $timeout = 300)
{
$processes = [];
foreach ($containerNames as $containerName) {
$processes[$containerName] = $this->stopContainer($containerName, $timeout);
}
$startTime = time();
while (count($processes) > 0) {
$finishedProcesses = array_filter($processes, function ($process) {
return ! $process->running();
});
foreach (array_keys($finishedProcesses) as $containerName) {
unset($processes[$containerName]);
$this->removeContainer($containerName, $server);
}
if (time() - $startTime >= $timeout) {
$this->forceStopRemainingContainers(array_keys($processes), $server);
break;
}
usleep(100000);
}
}
public function stopContainer(string $containerName, int $timeout): InvokedProcess
{
return Process::timeout($timeout)->start("docker stop --time=$timeout $containerName");
}
public function removeContainer(string $containerName, $server)
{
instant_remote_process(command: ["docker rm -f $containerName"], server: $server, throwError: false);
}
public function forceStopRemainingContainers(array $containerNames, $server)
{
foreach ($containerNames as $containerName) {
instant_remote_process(command: ["docker kill $containerName"], server: $server, throwError: false);
$this->removeContainer($containerName, $server);
}
}
public function delete_configurations()
{
$server = data_get($this, 'server');
$server = data_get($this, 'destination.server');
$workdir = $this->workdir();
if (str($workdir)->endsWith($this->uuid)) {
instant_remote_process(['rm -rf '.$this->workdir()], $server, false);
}
}
public function delete_connected_networks($uuid)
{
$server = data_get($this, 'destination.server');
instant_remote_process(["docker network disconnect {$uuid} coolify-proxy"], $server, false);
instant_remote_process(["docker network rm {$uuid}"], $server, false);
}
public function status()
{
$applications = $this->applications;
@@ -205,6 +285,41 @@ class Service extends BaseModel
foreach ($applications as $application) {
$image = str($application->image)->before(':')->value();
switch ($image) {
case str($image)?->contains('rabbitmq'):
$data = collect([]);
$host_port = $this->environment_variables()->where('key', 'PORT')->first();
$username = $this->environment_variables()->where('key', 'SERVICE_USER_RABBITMQ')->first();
$password = $this->environment_variables()->where('key', 'SERVICE_PASSWORD_RABBITMQ')->first();
if ($host_port) {
$data = $data->merge([
'Host Port Binding' => [
'key' => data_get($host_port, 'key'),
'value' => data_get($host_port, 'value'),
'rules' => 'required',
],
]);
}
if ($username) {
$data = $data->merge([
'Username' => [
'key' => data_get($username, 'key'),
'value' => data_get($username, 'value'),
'rules' => 'required',
],
]);
}
if ($password) {
$data = $data->merge([
'Password' => [
'key' => data_get($password, 'key'),
'value' => data_get($password, 'value'),
'rules' => 'required',
'isPassword' => true,
],
]);
}
$fields->put('RabbitMQ', $data->toArray());
break;
case str($image)?->contains('tolgee'):
$data = collect([]);
$admin_password = $this->environment_variables()->where('key', 'SERVICE_PASSWORD_TOLGEE')->first();
@@ -504,6 +619,9 @@ class Service extends BaseModel
default:
$data = collect([]);
$admin_user = $this->environment_variables()->where('key', 'SERVICE_USER_ADMIN')->first();
// Chaskiq
$admin_email = $this->environment_variables()->where('key', 'ADMIN_EMAIL')->first();
$admin_password = $this->environment_variables()->where('key', 'SERVICE_PASSWORD_ADMIN')->first();
if ($admin_user) {
$data = $data->merge([
@@ -525,6 +643,15 @@ class Service extends BaseModel
],
]);
}
if ($admin_email) {
$data = $data->merge([
'Email' => [
'key' => 'ADMIN_EMAIL',
'value' => data_get($admin_email, 'value'),
'rules' => 'required|email',
],
]);
}
$fields->put('Admin', $data->toArray());
break;
case str($image)?->contains('vaultwarden'):
@@ -608,7 +735,7 @@ class Service extends BaseModel
}
$data = $data->merge([
'Root User' => [
'key' => 'N/A',
'key' => 'GITLAB_ROOT_USER',
'value' => 'root',
'rules' => 'required',
'isPassword' => true,
@@ -617,6 +744,32 @@ class Service extends BaseModel
$fields->put('GitLab', $data->toArray());
break;
case str($image)->contains('code-server'):
$data = collect([]);
$password = $this->environment_variables()->where('key', 'SERVICE_PASSWORD_64_PASSWORDCODESERVER')->first();
if ($password) {
$data = $data->merge([
'Password' => [
'key' => data_get($password, 'key'),
'value' => data_get($password, 'value'),
'rules' => 'required',
'isPassword' => true,
],
]);
}
$sudoPassword = $this->environment_variables()->where('key', 'SERVICE_PASSWORD_SUDOCODESERVER')->first();
if ($sudoPassword) {
$data = $data->merge([
'Sudo Password' => [
'key' => data_get($sudoPassword, 'key'),
'value' => data_get($sudoPassword, 'value'),
'rules' => 'required',
'isPassword' => true,
],
]);
}
$fields->put('Code Server', $data->toArray());
break;
}
}
$databases = $this->databases()->get();
@@ -663,8 +816,8 @@ class Service extends BaseModel
$fields->put('PostgreSQL', $data->toArray());
break;
case str($image)->contains('mysql'):
$userVariables = ['SERVICE_USER_MYSQL', 'SERVICE_USER_WORDPRESS'];
$passwordVariables = ['SERVICE_PASSWORD_MYSQL', 'SERVICE_PASSWORD_WORDPRESS'];
$userVariables = ['SERVICE_USER_MYSQL', 'SERVICE_USER_WORDPRESS', 'MYSQL_USER'];
$passwordVariables = ['SERVICE_PASSWORD_MYSQL', 'SERVICE_PASSWORD_WORDPRESS', 'MYSQL_PASSWORD'];
$rootPasswordVariables = ['SERVICE_PASSWORD_MYSQLROOT', 'SERVICE_PASSWORD_ROOT'];
$dbNameVariables = ['MYSQL_DATABASE'];
$mysql_user = $this->environment_variables()->whereIn('key', $userVariables)->first();
@@ -713,10 +866,10 @@ class Service extends BaseModel
$fields->put('MySQL', $data->toArray());
break;
case str($image)->contains('mariadb'):
$userVariables = ['SERVICE_USER_MARIADB', 'SERVICE_USER_WORDPRESS', '_APP_DB_USER'];
$passwordVariables = ['SERVICE_PASSWORD_MARIADB', 'SERVICE_PASSWORD_WORDPRESS', '_APP_DB_PASS'];
$rootPasswordVariables = ['SERVICE_PASSWORD_MARIADBROOT', 'SERVICE_PASSWORD_ROOT', '_APP_DB_ROOT_PASS'];
$dbNameVariables = ['SERVICE_DATABASE_MARIADB', 'SERVICE_DATABASE_WORDPRESS', '_APP_DB_SCHEMA'];
$userVariables = ['SERVICE_USER_MARIADB', 'SERVICE_USER_WORDPRESS', '_APP_DB_USER', 'SERVICE_USER_MYSQL', 'MYSQL_USER'];
$passwordVariables = ['SERVICE_PASSWORD_MARIADB', 'SERVICE_PASSWORD_WORDPRESS', '_APP_DB_PASS', 'MYSQL_PASSWORD'];
$rootPasswordVariables = ['SERVICE_PASSWORD_MARIADBROOT', 'SERVICE_PASSWORD_ROOT', '_APP_DB_ROOT_PASS', 'MYSQL_ROOT_PASSWORD'];
$dbNameVariables = ['SERVICE_DATABASE_MARIADB', 'SERVICE_DATABASE_WORDPRESS', '_APP_DB_SCHEMA', 'MYSQL_DATABASE'];
$mariadb_user = $this->environment_variables()->whereIn('key', $userVariables)->first();
$mariadb_password = $this->environment_variables()->whereIn('key', $passwordVariables)->first();
$mariadb_root_password = $this->environment_variables()->whereIn('key', $rootPasswordVariables)->first();
@@ -763,6 +916,7 @@ class Service extends BaseModel
}
$fields->put('MariaDB', $data->toArray());
break;
}
}
@@ -897,7 +1051,8 @@ class Service extends BaseModel
public function environment_variables(): HasMany
{
return $this->hasMany(EnvironmentVariable::class)->orderBy('key', 'asc');
return $this->hasMany(EnvironmentVariable::class)->orderByRaw("key LIKE 'SERVICE%' DESC, value ASC");
}
public function environment_variables_preview(): HasMany
@@ -913,21 +1068,36 @@ class Service extends BaseModel
public function saveComposeConfigs()
{
$workdir = $this->workdir();
$commands[] = "mkdir -p $workdir";
instant_remote_process([
"mkdir -p $workdir",
"cd $workdir",
], $this->server);
$filename = new Cuid2.'-docker-compose.yml';
Storage::disk('local')->put("tmp/{$filename}", $this->docker_compose);
$path = Storage::path("tmp/{$filename}");
instant_scp($path, "{$workdir}/docker-compose.yml", $this->server);
Storage::disk('local')->delete("tmp/{$filename}");
$commands[] = "cd $workdir";
$json = Yaml::parse($this->docker_compose);
$this->docker_compose = Yaml::dump($json, 10, 2, Yaml::DUMP_MULTI_LINE_LITERAL_BLOCK);
$docker_compose_base64 = base64_encode($this->docker_compose);
$commands[] = "echo $docker_compose_base64 | base64 -d | tee docker-compose.yml > /dev/null";
$commands[] = 'rm -f .env || true';
$envs_from_coolify = $this->environment_variables()->get();
foreach ($envs_from_coolify as $env) {
$sorted = $envs_from_coolify->sortBy(function ($env) {
if (str($env->key)->startsWith('SERVICE_')) {
return 1;
}
if (str($env->value)->startsWith('$SERVICE_') || str($env->value)->startsWith('${SERVICE_')) {
return 2;
}
return 3;
});
foreach ($sorted as $env) {
$commands[] = "echo '{$env->key}={$env->real_value}' >> .env";
}
if ($envs_from_coolify->count() === 0) {
if ($sorted->count() === 0) {
$commands[] = 'touch .env';
}
instant_remote_process($commands, $this->server);
@@ -935,7 +1105,14 @@ class Service extends BaseModel
public function parse(bool $isNew = false): Collection
{
return parseDockerComposeFile($this, $isNew);
if ($this->compose_parsing_version === '3') {
return newParser($this);
} elseif ($this->docker_compose_raw) {
return parseDockerComposeFile($this, $isNew);
} else {
return collect([]);
}
}
public function networks()

View File

@@ -32,6 +32,16 @@ class ServiceApplication extends BaseModel
return ServiceApplication::whereRelation('service.environment.project.team', 'id', $teamId)->orderBy('name');
}
public function isRunning()
{
return str($this->status)->contains('running');
}
public function isExited()
{
return str($this->status)->contains('exited');
}
public function isLogDrainEnabled()
{
return data_get($this, 'is_log_drain_enabled', false);

View File

@@ -25,6 +25,16 @@ class ServiceDatabase extends BaseModel
remote_process(["docker restart {$container_id}"], $this->service->server);
}
public function isRunning()
{
return str($this->status)->contains('running');
}
public function isExited()
{
return str($this->status)->contains('exited');
}
public function isLogDrainEnabled()
{
return data_get($this, 'is_log_drain_enabled', false);

View File

@@ -75,6 +75,11 @@ class StandaloneClickhouse extends BaseModel
}
}
public function isRunning()
{
return (bool) str($this->status)->contains('running');
}
public function isExited()
{
return (bool) str($this->status)->startsWith('exited');

View File

@@ -75,6 +75,11 @@ class StandaloneDragonfly extends BaseModel
}
}
public function isRunning()
{
return (bool) str($this->status)->contains('running');
}
public function isExited()
{
return (bool) str($this->status)->startsWith('exited');

View File

@@ -75,6 +75,11 @@ class StandaloneKeydb extends BaseModel
}
}
public function isRunning()
{
return (bool) str($this->status)->contains('running');
}
public function isExited()
{
return (bool) str($this->status)->startsWith('exited');
@@ -209,7 +214,7 @@ class StandaloneKeydb extends BaseModel
protected function internalDbUrl(): Attribute
{
return new Attribute(
get: fn () => "redis://{$this->keydb_password}@{$this->uuid}:6379/0",
get: fn () => "redis://:{$this->keydb_password}@{$this->uuid}:6379/0",
);
}
@@ -218,7 +223,7 @@ class StandaloneKeydb extends BaseModel
return new Attribute(
get: function () {
if ($this->is_public && $this->public_port) {
return "redis://{$this->keydb_password}@{$this->destination->server->getIp}:{$this->public_port}/0";
return "redis://:{$this->keydb_password}@{$this->destination->server->getIp}:{$this->public_port}/0";
}
return null;

View File

@@ -75,6 +75,11 @@ class StandaloneMariadb extends BaseModel
}
}
public function isRunning()
{
return (bool) str($this->status)->contains('running');
}
public function isExited()
{
return (bool) str($this->status)->startsWith('exited');

View File

@@ -79,6 +79,11 @@ class StandaloneMongodb extends BaseModel
}
}
public function isRunning()
{
return (bool) str($this->status)->contains('running');
}
public function isExited()
{
return (bool) str($this->status)->startsWith('exited');

View File

@@ -76,6 +76,11 @@ class StandaloneMysql extends BaseModel
}
}
public function isRunning()
{
return (bool) str($this->status)->contains('running');
}
public function isExited()
{
return (bool) str($this->status)->startsWith('exited');

View File

@@ -102,6 +102,11 @@ class StandalonePostgresql extends BaseModel
}
}
public function isRunning()
{
return (bool) str($this->status)->contains('running');
}
public function isExited()
{
return (bool) str($this->status)->startsWith('exited');

View File

@@ -79,6 +79,11 @@ class StandaloneRedis extends BaseModel
}
}
public function isRunning()
{
return (bool) str($this->status)->contains('running');
}
public function isExited()
{
return (bool) str($this->status)->startsWith('exited');