dave/coolify
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

fix: appwrite template + parser

Browse Source
This commit is contained in:
Andras Bacsai
2024-09-09 15:04:51 +02:00
parent c2f7e85022
commit 1d72f76072
12 changed files with 661 additions and 717 deletions
Download Patch File Download Diff File Expand all files Collapse all files

62
bootstrap/helpers/remoteProcess.php
View File

@@ -147,16 +147,16 @@ function generateSshCommand(Server $server, string $command)
// Check if multiplexing is enabled
$muxEnabled = config('constants.ssh.mux_enabled', true);
ray('SSH Multiplexing Enabled:', $muxEnabled)->blue();
// ray('SSH Multiplexing Enabled:', $muxEnabled)->blue();
if ($muxEnabled) {
// Always use multiplexing when enabled
$muxSocket = "/var/www/html/storage/app/ssh/mux/{$server->muxFilename()}";
$ssh_command .= "-o ControlMaster=auto -o ControlPath=$muxSocket -o ControlPersist={$muxPersistTime} ";
ensureMultiplexedConnection($server);
ray('Using SSH Multiplexing')->green();
// ray('Using SSH Multiplexing')->green();
} else {
ray('Not using SSH Multiplexing')->red();
// ray('Not using SSH Multiplexing')->red();
}
if (data_get($server, 'settings.is_cloudflare_tunnel')) {
@@ -184,50 +184,52 @@ function generateSshCommand(Server $server, string $command)
function ensureMultiplexedConnection(Server $server)
{
static $ensuredConnections = [];
if (isset($ensuredConnections[$server->id])) {
if (!shouldResetMultiplexedConnection($server)) {
ray('Using Existing Multiplexed Connection')->green();
if (! shouldResetMultiplexedConnection($server)) {
// ray('Using Existing Multiplexed Connection')->green();
return;
}
}
$muxSocket = "/var/www/html/storage/app/ssh/mux/{$server->muxFilename()}";
$checkCommand = "ssh -O check -o ControlPath=$muxSocket {$server->user}@{$server->ip} 2>/dev/null";
$process = Process::run($checkCommand);
if ($process->exitCode() === 0) {
ray('Existing Multiplexed Connection is Valid')->green();
// ray('Existing Multiplexed Connection is Valid')->green();
$ensuredConnections[$server->id] = [
'timestamp' => now(),
'muxSocket' => $muxSocket,
];
return;
}
ray('Establishing New Multiplexed Connection')->orange();
// ray('Establishing New Multiplexed Connection')->orange();
$privateKeyLocation = savePrivateKeyToFs($server);
$connectionTimeout = config('constants.ssh.connection_timeout');
$serverInterval = config('constants.ssh.server_interval');
$muxPersistTime = config('constants.ssh.mux_persist_time');
$establishCommand = "ssh -fNM -o ControlMaster=auto -o ControlPath=$muxSocket -o ControlPersist={$muxPersistTime} "
. "-i {$privateKeyLocation} "
. "-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null "
. "-o PasswordAuthentication=no "
. "-o ConnectTimeout=$connectionTimeout "
. "-o ServerAliveInterval=$serverInterval "
. "-o RequestTTY=no "
. "-o LogLevel=ERROR "
. "-p {$server->port} "
. "{$server->user}@{$server->ip}";
."-i {$privateKeyLocation} "
.'-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null '
.'-o PasswordAuthentication=no '
."-o ConnectTimeout=$connectionTimeout "
."-o ServerAliveInterval=$serverInterval "
.'-o RequestTTY=no '
.'-o LogLevel=ERROR '
."-p {$server->port} "
."{$server->user}@{$server->ip}";
$establishProcess = Process::run($establishCommand);
if ($establishProcess->exitCode() !== 0) {
throw new \RuntimeException("Failed to establish multiplexed connection: " . $establishProcess->errorOutput());
throw new \RuntimeException('Failed to establish multiplexed connection: '.$establishProcess->errorOutput());
}
$ensuredConnections[$server->id] = [
@@ -235,14 +237,14 @@ function ensureMultiplexedConnection(Server $server)
'muxSocket' => $muxSocket,
];
ray('Established New Multiplexed Connection')->green();
// ray('Established New Multiplexed Connection')->green();
}
function shouldResetMultiplexedConnection(Server $server)
{
static $ensuredConnections = [];
if (!isset($ensuredConnections[$server->id])) {
if (! isset($ensuredConnections[$server->id])) {
return true;
}
@@ -278,18 +280,18 @@ function instant_remote_process(Collection|array $command, Server $server, bool
$command = parseCommandsByLineForSudo(collect($command), $server);
}
$command_string = implode("\n", $command);
$start_time = microtime(true);
$sshCommand = generateSshCommand($server, $command_string);
$process = Process::timeout($timeout)->run($sshCommand);
$end_time = microtime(true);
$execution_time = ($end_time - $start_time) * 1000; // Convert to milliseconds
ray('SSH command execution time:', $execution_time . ' ms')->orange();
// ray('SSH command execution time:', $execution_time.' ms')->orange();
$output = trim($process->output());
$exitCode = $process->exitCode();
if ($exitCode !== 0) {
if (! $throwError) {
return null;
@@ -398,10 +400,10 @@ function remove_mux_and_private_key(Server $server)
{
$muxFilename = $server->muxFilename();
$privateKeyLocation = savePrivateKeyToFs($server);
$closeCommand = "ssh -O exit -o ControlPath=/var/www/html/storage/app/ssh/mux/{$muxFilename} {$server->user}@{$server->ip}";
Process::run($closeCommand);
Storage::disk('ssh-mux')->delete($muxFilename);
Storage::disk('ssh-keys')->delete($privateKeyLocation);
}

5
bootstrap/helpers/services.php
View File

@@ -4,6 +4,7 @@ use App\Models\Application;
use App\Models\EnvironmentVariable;
use App\Models\ServiceApplication;
use App\Models\ServiceDatabase;
use Illuminate\Support\Stringable;
use Spatie\Url\Url;
use Symfony\Component\Yaml\Yaml;
@@ -15,9 +16,9 @@ function collectRegex(string $name)
{
return "/{$name}\w+/";
}
function replaceVariables($variable)
function replaceVariables(string $variable): Stringable
{
return $variable->before('}')->replaceFirst('$', '')->replaceFirst('{', '');
return str($variable)->before('}')->replaceFirst('$', '')->replaceFirst('{', '');
}
function getFilesystemVolumesFromServer(ServiceApplication|ServiceDatabase|Application $oneService, bool $isInit = false)

224
bootstrap/helpers/shared.php
View File

@@ -1866,7 +1866,7 @@ function parseDockerComposeFile(Service|Application $resource, bool $isNew = fal
'key' => $key,
'service_id' => $resource->id,
])->first();
$value = str(replaceVariables($value));
$value = replaceVariables($value);
$key = $value;
if ($value->startsWith('SERVICE_')) {
$foundEnv = EnvironmentVariable::where([
@@ -2627,7 +2627,7 @@ function parseDockerComposeFile(Service|Application $resource, bool $isNew = fal
'application_id' => $resource->id,
'is_preview' => false,
])->first();
$value = str(replaceVariables($value));
$value = replaceVariables($value);
$key = $value;
if ($value->startsWith('SERVICE_')) {
$foundEnv = EnvironmentVariable::where([
@@ -2921,8 +2921,8 @@ function newParser(Application|Service $resource, int $pull_request_id = 0, ?int
}
$parsedServices = collect([]);
ray()->clearAll();
// parse environments first
$allMagicEnvironments = collect([]);
foreach ($services as $serviceName => $service) {
$magicEnvironments = collect([]);
@@ -2959,132 +2959,138 @@ function newParser(Application|Service $resource, int $pull_request_id = 0, ?int
$allEnvironments = $allEnvironments->mapWithKeys(function ($item) {
return [$item['key'] => $item['value']];
});
// filter magic environments
// filter and add magic environments
foreach ($environment as $key => $value) {
// Get all SERVICE_ variables from keys and values
$key = str($key);
$value = str($value);
$regex = '/\$(\{?([a-zA-Z_\x80-\xff][a-zA-Z0-9_\x80-\xff]*)\}?)/';
preg_match_all($regex, $value, $valueMatches);
preg_match_all($regex, $key, $keyMatches);
if (count($valueMatches[1]) > 0) {
foreach ($valueMatches[1] as $match) {
if (str($match)->startsWith('SERVICE_')) {
if ($magicEnvironments->has($match)) {
$match = replaceVariables($match);
if ($match->startsWith('SERVICE_')) {
if ($magicEnvironments->has($match->value())) {
continue;
}
$magicEnvironments->put($match, '');
$magicEnvironments->put($match->value(), '');
}
}
}
if (count($keyMatches[1]) > 0) {
foreach ($keyMatches[1] as $match) {
if (str($match)->startsWith('SERVICE_')) {
if ($magicEnvironments->has($match)) {
continue;
}
$magicEnvironments->put($match, '');
// Get magic environments where we need to preset the FQDN
if ($key->startsWith('SERVICE_FQDN_')) {
// SERVICE_FQDN_APP or SERVICE_FQDN_APP_3000
$fqdnFor = $key->after('SERVICE_FQDN_')->lower()->value();
if ($isApplication) {
$fqdn = generateFqdn($server, "{$resource->name}-$uuid");
} elseif ($isService) {
if ($fqdnFor) {
$fqdn = generateFqdn($server, "$fqdnFor-$uuid");
} else {
$fqdn = generateFqdn($server, "{$savedService->name}-$uuid");
}
}
}
if (str($key)->startsWith('SERVICE_')) {
$magicEnvironments->put($key, $value);
if ($value && get_class($value) === 'Illuminate\Support\Stringable' && $value->startsWith('/')) {
$path = $value->value();
if ($path !== '/') {
$fqdn = "$fqdn$path";
}
}
if ($isApplication && is_null($resource->fqdn)) {
data_forget($resource, 'environment_variables');
data_forget($resource, 'environment_variables_preview');
$resource->fqdn = $fqdn;
$resource->save();
} elseif ($isService && is_null($savedService->fqdn)) {
$savedService->fqdn = $fqdn;
$savedService->save();
}
if (substr_count(str($key)->value(), '_') === 2) {
$resource->environment_variables()->where('key', $key->value())->where($nameOfId, $resource->id)->firstOrCreate([
'key' => $key->value(),
$nameOfId => $resource->id,
], [
'value' => $fqdn,
'is_build_time' => false,
'is_preview' => false,
]);
}
if (substr_count(str($key)->value(), '_') === 3) {
$newKey = str($key)->beforeLast('_')->value();
if ($newKey) {
$magicEnvironments->put($newKey, $value);
}
$newKey = str($key)->beforeLast('_');
$resource->environment_variables()->where('key', $newKey->value())->where($nameOfId, $resource->id)->firstOrCreate([
'key' => $newKey->value(),
$nameOfId => $resource->id,
], [
'value' => $fqdn,
'is_build_time' => false,
'is_preview' => false,
]);
}
}
}
$allMagicEnvironments = $allMagicEnvironments->merge($magicEnvironments);
if ($magicEnvironments->count() > 0) {
foreach ($magicEnvironments as $key => $value) {
$key = str($key);
$value = str(replaceVariables(str($value)));
$originalValue = $value;
$keyCommand = $key->after('SERVICE_')->before('_');
$valueCommand = $value->after('SERVICE_')->before('_');
if ($key->startsWith('SERVICE_FQDN_')) {
$value = replaceVariables($value);
$command = $key->after('SERVICE_')->before('_');
$found = $resource->environment_variables()->where('key', $key->value())->where($nameOfId, $resource->id)->first();
if ($found) {
continue;
}
if ($command->value() === 'FQDN') {
$fqdnFor = $key->after('SERVICE_FQDN_')->lower()->value();
if (str($fqdnFor)->contains('_')) {
$fqdnFor = str($fqdnFor)->before('_');
}
} elseif ($value->startsWith('SERVICE_FQDN_')) {
$fqdnFor = $value->after('SERVICE_FQDN_')->lower()->value();
if (str($fqdnFor)->contains('_')) {
$fqdnFor = str($fqdnFor)->before('_');
}
} else {
$fqdnFor = null;
}
if ($keyCommand->value() === 'FQDN' || $valueCommand->value() === 'FQDN') {
if ($isApplication) {
$fqdn = generateFqdn($server, "{$resource->name}-$uuid");
} elseif ($isService) {
if ($fqdnFor) {
$fqdn = generateFqdn($server, "$fqdnFor-$uuid");
} else {
$fqdn = generateFqdn($server, "{$savedService->name}-$uuid");
}
$fqdn = generateFqdn($server, "$fqdnFor-$uuid");
}
if ($value && get_class($value) === 'Illuminate\Support\Stringable' && $value->startsWith('/')) {
$path = $value->value();
if ($value === '/') {
$value = "$fqdn";
} else {
$value = "$fqdn$path";
}
} else {
$value = $fqdn;
$resource->environment_variables()->where('key', $key->value())->where($nameOfId, $resource->id)->firstOrCreate([
'key' => $key->value(),
$nameOfId => $resource->id,
], [
'value' => $fqdn,
'is_build_time' => false,
'is_preview' => false,
]);
} elseif ($command->value() === 'URL') {
$fqdnFor = $key->after('SERVICE_URL_')->lower()->value();
if (str($fqdnFor)->contains('_')) {
$fqdnFor = str($fqdnFor)->before('_');
}
if (! $isDatabase) {
if ($key->startsWith('SERVICE_FQDN_') && ($originalValue->value() === '' || $originalValue->startsWith('/'))) {
if ($isApplication && is_null($resource->fqdn)) {
data_forget($resource, 'environment_variables');
data_forget($resource, 'environment_variables_preview');
$resource->fqdn = $value;
$resource->save();
} elseif ($isService && is_null($savedService->fqdn)) {
if ($key->startsWith('SERVICE_FQDN_')) {
$savedService->fqdn = $value;
$savedService->save();
}
}
}
}
} elseif ($keyCommand->value() === 'URL' || $valueCommand->value() === 'URL') {
if ($isApplication) {
$fqdn = generateFqdn($server, "{$resource->name}-{$uuid}");
$fqdn = generateFqdn($server, "{$resource->name}-$uuid");
} elseif ($isService) {
$fqdn = generateFqdn($server, "{$savedService->name}-{$uuid}");
$fqdn = generateFqdn($server, "$fqdnFor-$uuid");
}
if ($value && get_class($value) === 'Illuminate\Support\Stringable' && $value->startsWith('/')) {
$path = $value->value();
$value = "$fqdn$path";
} else {
$value = $fqdn;
}
$value = str($fqdn)->replace('http://', '')->replace('https://', '');
$resource->environment_variables()->where('key', $key->value())->where($nameOfId, $resource->id)->firstOrCreate([
'key' => $key->value(),
$nameOfId => $resource->id,
], [
'value' => $fqdn,
'is_build_time' => false,
'is_preview' => false,
]);
} else {
$generatedValue = generateEnvValue($keyCommand, $resource);
if ($generatedValue) {
$value = $generatedValue;
}
$value = generateEnvValue($command, $resource);
$resource->environment_variables()->where('key', $key->value())->where($nameOfId, $resource->id)->firstOrCreate([
'key' => $key->value(),
$nameOfId => $resource->id,
], [
'value' => $value,
'is_build_time' => false,
'is_preview' => false,
]);
}
if (str($fqdnFor)->startsWith('/')) {
$fqdnFor = null;
}
// Lets save the magic value to the environment variables
if ($key->startsWith('SERVICE_')) {
$originalValue = $key;
}
$resource->environment_variables()->where('key', $originalValue->value())->where($nameOfId, $resource->id)->firstOrCreate([
'key' => $originalValue->value(),
$nameOfId => $resource->id,
], [
'value' => $value,
'is_build_time' => false,
'is_preview' => false,
]);
}
}
}
@@ -3388,8 +3394,23 @@ function newParser(Application|Service $resource, int $pull_request_id = 0, ?int
foreach ($normalEnvironments as $key => $value) {
$key = str($key);
$value = str($value);
$parsedValue = str(replaceVariables(str($value)));
$originalValue = $value;
$parsedValue = replaceVariables($value);
if ($value->startsWith('$SERVICE_')) {
$resource->environment_variables()->where('key', $key)->where($nameOfId, $resource->id)->firstOrCreate([
'key' => $key,
$nameOfId => $resource->id,
], [
'value' => $value,
'is_build_time' => false,
'is_preview' => false,
]);
continue;
}
if (! $value->startsWith('$')) {
continue;
}
if ($key->value() === $parsedValue->value()) {
$value = null;
$resource->environment_variables()->where('key', $key)->where($nameOfId, $resource->id)->firstOrCreate([
@@ -3403,25 +3424,28 @@ function newParser(Application|Service $resource, int $pull_request_id = 0, ?int
} else {
if ($value->startsWith('$')) {
if ($value->contains(':-')) {
$value = str(replaceVariables(str($value)));
$value = replaceVariables($value);
$key = $value->before(':');
$value = $value->after(':-');
} elseif ($value->contains('-')) {
$value = str(replaceVariables(str($value)));
$value = replaceVariables($value);
$key = $value->before('-');
$value = $value->after('-');
} elseif ($value->contains(':?')) {
$value = str(replaceVariables(str($value)));
$value = replaceVariables($value);
$key = $value->before(':');
$value = $value->after(':?');
} elseif ($value->contains('?')) {
$value = str(replaceVariables(str($value)));
$value = replaceVariables($value);
$key = $value->before('?');
$value = $value->after('?');
}
if ($originalValue->value() === $value->value()) {
continue;
}
$resource->environment_variables()->where('key', $key)->where($nameOfId, $resource->id)->firstOrCreate([
'key' => $key,
$nameOfId => $resource->id,