dave/coolify
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Fix: SSH multiplexing

Browse Source
This commit is contained in:
peaklabs-dev
2024-09-17 12:26:11 +02:00
parent f9375f91ec
commit 144508218e
7 changed files with 92 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
app/Actions/CoolifyTask/RunRemoteProcess.php
View File

@@ -10,6 +10,7 @@ use Illuminate\Process\ProcessResult;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Process;
use Spatie\Activitylog\Models\Activity;
use App\Helpers\SshMultiplexingHelper;
class RunRemoteProcess
{
@@ -137,7 +138,7 @@ class RunRemoteProcess
$command = $this->activity->getExtraProperty('command');
$server = Server::whereUuid($server_uuid)->firstOrFail();
return generateSshCommand($server, $command);
return SshMultiplexingHelper::generateSshCommand($server, $command);
}
protected function handleOutput(string $type, string $output)

126
app/Helpers/SshMultiplexingHelper.php
View File

@@ -3,11 +3,10 @@
namespace App\Helpers;
use App\Models\Server;
use App\Models\PrivateKey;
use Illuminate\Support\Facades\Process;
use Illuminate\Support\Facades\Storage;
use Illuminate\Support\Facades\Hash;
use Illuminate\Support\Str;
use App\Models\PrivateKey;
class SshMultiplexingHelper
{
@@ -15,7 +14,8 @@ class SshMultiplexingHelper
public static function serverSshConfiguration(Server $server)
{
$sshKeyLocation = $server->privateKey->getKeyLocation();
$privateKey = PrivateKey::findOrFail($server->private_key_id);
$sshKeyLocation = $privateKey->getKeyLocation();
$muxFilename = '/var/www/html/storage/app/ssh/mux/' . $server->muxFilename();
return [
@@ -26,15 +26,21 @@ class SshMultiplexingHelper
public static function ensureMultiplexedConnection(Server $server)
{
if (!self::isMultiplexingEnabled()) {
ray('Multiplexing is disabled');
return;
}
ray('Ensuring multiplexed connection for server: ' . $server->id);
$sshConfig = self::serverSshConfiguration($server);
$muxSocket = $sshConfig['muxFilename'];
$sshKeyLocation = $sshConfig['sshKeyLocation'];
if (!file_exists($sshKeyLocation)) {
throw new \RuntimeException("SSH key file not accessible: $sshKeyLocation");
}
self::validateSshKey($sshKeyLocation);
if (isset(self::$ensuredConnections[$server->id]) && !self::shouldResetMultiplexedConnection($server)) {
ray('Existing connection is still valid');
return;
}
@@ -42,6 +48,7 @@ class SshMultiplexingHelper
$fileCheckProcess = Process::run($checkFileCommand);
if ($fileCheckProcess->exitCode() !== 0) {
ray('Mux socket file not found, establishing new connection');
self::establishNewMultiplexedConnection($server);
return;
}
@@ -49,19 +56,22 @@ class SshMultiplexingHelper
$checkCommand = "ssh -O check -o ControlPath=$muxSocket {$server->user}@{$server->ip}";
$process = Process::run($checkCommand);
if ($process->exitCode() === 0) {
if ($process->exitCode() !== 0) {
ray('Existing connection check failed, establishing new connection');
self::establishNewMultiplexedConnection($server);
} else {
ray('Existing connection is valid');
self::$ensuredConnections[$server->id] = [
'timestamp' => now(),
'muxSocket' => $muxSocket,
];
return;
}
self::establishNewMultiplexedConnection($server);
}
public static function establishNewMultiplexedConnection(Server $server)
{
ray('Establishing new multiplexed connection for server: ' . $server->id);
$sshConfig = self::serverSshConfiguration($server);
$sshKeyLocation = $sshConfig['sshKeyLocation'];
$muxSocket = $sshConfig['muxFilename'];
@@ -84,9 +94,12 @@ class SshMultiplexingHelper
$establishProcess = Process::run($establishCommand);
if ($establishProcess->exitCode() !== 0) {
ray('Failed to establish multiplexed connection', $establishProcess->errorOutput());
throw new \RuntimeException('Failed to establish multiplexed connection: ' . $establishProcess->errorOutput());
}
ray('Multiplexed connection established successfully');
$muxContent = "Multiplexed connection established at " . now()->toDateTimeString();
Storage::disk('ssh-mux')->put(basename($muxSocket), $muxContent);
@@ -99,6 +112,7 @@ class SshMultiplexingHelper
public static function shouldResetMultiplexedConnection(Server $server)
{
if (!(config('constants.ssh.mux_enabled') && config('coolify.is_windows_docker_desktop') == false)) {
ray('Multiplexing is disabled or running on Windows Docker Desktop');
return false;
}
@@ -110,7 +124,9 @@ class SshMultiplexingHelper
$muxPersistTime = config('constants.ssh.mux_persist_time');
$resetInterval = strtotime($muxPersistTime) - time();
return $lastEnsured->addSeconds($resetInterval)->isPast();
$shouldReset = $lastEnsured->addSeconds($resetInterval)->isPast();
ray('Should reset multiplexed connection', ['server_id' => $server->id, 'should_reset' => $shouldReset]);
return $shouldReset;
}
public static function removeMuxFile(Server $server)
@@ -130,38 +146,22 @@ class SshMultiplexingHelper
$sshKeyLocation = $sshConfig['sshKeyLocation'];
$muxSocket = $sshConfig['muxFilename'];
$user = $server->user;
$port = $server->port;
$timeout = config('constants.ssh.command_timeout');
$connectionTimeout = config('constants.ssh.connection_timeout');
$serverInterval = config('constants.ssh.server_interval');
$muxPersistTime = config('constants.ssh.mux_persist_time');
$scp_command = "timeout $timeout scp ";
$muxEnabled = config('constants.ssh.mux_enabled', true) && config('coolify.is_windows_docker_desktop') == false;
ray('SSH Multiplexing Enabled:', $muxEnabled)->blue();
if ($muxEnabled) {
if (self::isMultiplexingEnabled()) {
$muxPersistTime = config('constants.ssh.mux_persist_time');
$scp_command .= "-o ControlMaster=auto -o ControlPath=$muxSocket -o ControlPersist={$muxPersistTime} ";
self::ensureMultiplexedConnection($server);
ray('Using SSH Multiplexing')->green();
} else {
ray('Not using SSH Multiplexing')->red();
}
if (data_get($server, 'settings.is_cloudflare_tunnel')) {
$scp_command .= '-o ProxyCommand="/usr/local/bin/cloudflared access ssh --hostname %h" ';
}
$scp_command .= "-i {$sshKeyLocation} "
.'-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null '
.'-o PasswordAuthentication=no '
."-o ConnectTimeout=$connectionTimeout "
."-o ServerAliveInterval=$serverInterval "
.'-o RequestTTY=no '
.'-o LogLevel=ERROR '
."-P {$port} "
."{$source} "
."{$user}@{$server->ip}:{$dest}";
self::addCloudflareProxyCommand($scp_command, $server);
$scp_command .= self::getCommonSshOptions($server, $sshKeyLocation, $connectionTimeout, $serverInterval);
$scp_command .= "{$source} {$server->user}@{$server->ip}:{$dest}";
return $scp_command;
}
@@ -179,45 +179,61 @@ class SshMultiplexingHelper
$timeout = config('constants.ssh.command_timeout');
$connectionTimeout = config('constants.ssh.connection_timeout');
$serverInterval = config('constants.ssh.server_interval');
$muxPersistTime = config('constants.ssh.mux_persist_time');
$muxEnabled = config('constants.ssh.mux_enabled') && !config('coolify.is_windows_docker_desktop');
ray('Config MUX Enabled:', config('constants.ssh.mux_enabled'));
ray('Config Windows Docker Desktop:', config('coolify.is_windows_docker_desktop'));
ray('MUX Enabled:', $muxEnabled);
$ssh_command = "timeout $timeout ssh ";
ray('SSH Multiplexing Enabled:', $muxEnabled)->blue();
if ($muxEnabled) {
if (self::isMultiplexingEnabled()) {
$muxPersistTime = config('constants.ssh.mux_persist_time');
$ssh_command .= "-o ControlMaster=auto -o ControlPath=$muxSocket -o ControlPersist={$muxPersistTime} ";
self::ensureMultiplexedConnection($server);
ray('Using SSH Multiplexing')->green();
} else {
ray('Not using SSH Multiplexing')->red();
}
if (data_get($server, 'settings.is_cloudflare_tunnel')) {
$ssh_command .= '-o ProxyCommand="/usr/local/bin/cloudflared access ssh --hostname %h" ';
}
self::addCloudflareProxyCommand($ssh_command, $server);
$ssh_command .= self::getCommonSshOptions($server, $sshKeyLocation, $connectionTimeout, $serverInterval);
$command = "PATH=\$PATH:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/host/usr/local/sbin:/host/usr/local/bin:/host/usr/sbin:/host/usr/bin:/host/sbin:/host/bin && $command";
$delimiter = Hash::make($command);
$command = str_replace($delimiter, '', $command);
$ssh_command .= "-i {$sshKeyLocation} "
$ssh_command .= "{$server->user}@{$server->ip} 'bash -se' << \\$delimiter".PHP_EOL
.$command.PHP_EOL
.$delimiter;
return $ssh_command;
}
private static function isMultiplexingEnabled(): bool
{
return config('constants.ssh.mux_enabled') && !config('coolify.is_windows_docker_desktop');
}
private static function validateSshKey(string $sshKeyLocation): void
{
$checkKeyCommand = "ls $sshKeyLocation 2>/dev/null";
$keyCheckProcess = Process::run($checkKeyCommand);
if ($keyCheckProcess->exitCode() !== 0) {
throw new \RuntimeException("SSH key file not accessible: $sshKeyLocation");
}
}
private static function addCloudflareProxyCommand(string &$command, Server $server): void
{
if (data_get($server, 'settings.is_cloudflare_tunnel')) {
$command .= '-o ProxyCommand="/usr/local/bin/cloudflared access ssh --hostname %h" ';
}
}
private static function getCommonSshOptions(Server $server, string $sshKeyLocation, int $connectionTimeout, int $serverInterval): string
{
return "-i {$sshKeyLocation} "
.'-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null '
.'-o PasswordAuthentication=no '
."-o ConnectTimeout=$connectionTimeout "
."-o ServerAliveInterval=$serverInterval "
.'-o RequestTTY=no '
.'-o LogLevel=ERROR '
."-p {$server->port} "
."{$server->user}@{$server->ip} "
." 'bash -se' << \\$delimiter".PHP_EOL
.$command.PHP_EOL
.$delimiter;
return $ssh_command;
."-p {$server->port} ";
}
}

7
app/Jobs/ServerCheckJob.php
View File

@@ -43,7 +43,7 @@ class ServerCheckJob implements ShouldBeEncrypted, ShouldQueue
return isDev() ? 1 : 3;
}
public function __construct(public Server $server) {}
public function __construct(public Server $server, public bool $isManualCheck = false) {}
public function middleware(): array
{
@@ -58,6 +58,9 @@ class ServerCheckJob implements ShouldBeEncrypted, ShouldQueue
public function handle()
{
try {
// Enable SSH multiplexing for autonomous checks, disable for manual checks
config()->set('constants.ssh.mux_enabled', !$this->isManualCheck);
$this->applications = $this->server->applications();
$this->databases = $this->server->databases();
$this->services = $this->server->services()->get();
@@ -93,7 +96,7 @@ class ServerCheckJob implements ShouldBeEncrypted, ShouldQueue
private function serverStatus()
{
['uptime' => $uptime] = $this->server->validateConnection();
['uptime' => $uptime] = $this->server->validateConnection($this->isManualCheck);
if ($uptime) {
if ($this->server->unreachable_notification_sent === true) {
$this->server->update(['unreachable_notification_sent' => false]);

9
app/Livewire/Project/Shared/GetLogs.php
View File

@@ -17,6 +17,7 @@ use App\Models\StandalonePostgresql;
use App\Models\StandaloneRedis;
use Illuminate\Support\Facades\Process;
use Livewire\Component;
use App\Helpers\SshMultiplexingHelper;
class GetLogs extends Component
{
@@ -108,14 +109,14 @@ class GetLogs extends Component
$command = parseCommandsByLineForSudo(collect($command), $this->server);
$command = $command[0];
}
$sshCommand = generateSshCommand($this->server, $command);
$sshCommand = SshMultiplexingHelper::generateSshCommand($this->server, $command);
} else {
$command = "docker logs -n {$this->numberOfLines} -t {$this->container}";
if ($this->server->isNonRoot()) {
$command = parseCommandsByLineForSudo(collect($command), $this->server);
$command = $command[0];
}
$sshCommand = generateSshCommand($this->server, $command);
$sshCommand = SshMultiplexingHelper::generateSshCommand($this->server, $command);
}
} else {
if ($this->server->isSwarm()) {
@@ -124,14 +125,14 @@ class GetLogs extends Component
$command = parseCommandsByLineForSudo(collect($command), $this->server);
$command = $command[0];
}
$sshCommand = generateSshCommand($this->server, $command);
$sshCommand = SshMultiplexingHelper::generateSshCommand($this->server, $command);
} else {
$command = "docker logs -n {$this->numberOfLines} {$this->container}";
if ($this->server->isNonRoot()) {
$command = parseCommandsByLineForSudo(collect($command), $this->server);
$command = $command[0];
}
$sshCommand = generateSshCommand($this->server, $command);
$sshCommand = SshMultiplexingHelper::generateSshCommand($this->server, $command);
}
}
if ($refresh) {

7
app/Models/Server.php
View File

@@ -967,9 +967,10 @@ $schema://$host {
return data_get($this, 'settings.is_swarm_worker');
}
public function validateConnection()
public function validateConnection($isManualCheck = true)
{
config()->set('constants.ssh.mux_enabled', false);
// Set mux_enabled to true for automatic checks, false for manual checks
config()->set('constants.ssh.mux_enabled', !$isManualCheck);
$server = Server::find($this->id);
if (! $server) {
@@ -979,7 +980,6 @@ $schema://$host {
return ['uptime' => false, 'error' => 'Server skipped.'];
}
try {
// EC2 does not have `uptime` command, lol
instant_remote_process(['ls /'], $server);
$server->settings()->update([
'is_reachable' => true,
@@ -988,7 +988,6 @@ $schema://$host {
'unreachable_count' => 0,
]);
if (data_get($server, 'unreachable_notification_sent') === true) {
// $server->team?->notify(new Revived($server));
$server->update(['unreachable_notification_sent' => false]);
}

3
app/Traits/ExecuteRemoteCommand.php
View File

@@ -7,6 +7,7 @@ use App\Models\Server;
use Carbon\Carbon;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Process;
use App\Helpers\SshMultiplexingHelper;
trait ExecuteRemoteCommand
{
@@ -42,7 +43,7 @@ trait ExecuteRemoteCommand
$command = parseLineForSudo($command, $this->server);
}
}
$remote_command = generateSshCommand($this->server, $command);
$remote_command = SshMultiplexingHelper::generateSshCommand($this->server, $command);
$process = Process::timeout(3600)->idleTimeout(3600)->start($remote_command, function (string $type, string $output) use ($command, $hidden, $customType, $append) {
$output = str($output)->trim();
if ($output->startsWith('╔')) {

26
bootstrap/helpers/remoteProcess.php
View File

@@ -35,8 +35,8 @@ function remote_process(
$command_string = implode("\n", $command);
if (auth()->user()) {
$teams = auth()->user()->teams->pluck('id');
if (Auth::check()) {
$teams = Auth::user()->teams->pluck('id');
if (!$teams->contains($server->team_id) && !$teams->contains(0)) {
throw new \Exception('User is not part of the team that owns this server');
}
@@ -58,15 +58,10 @@ function remote_process(
])();
}
function generateScpCommand(Server $server, string $source, string $dest)
{
return SshMultiplexingHelper::generateScpCommand($server, $source, $dest);
}
function instant_scp(string $source, string $dest, Server $server, $throwError = true)
{
$timeout = config('constants.ssh.command_timeout');
$scp_command = generateScpCommand($server, $source, $dest);
$scp_command = SshMultiplexingHelper::generateScpCommand($server, $source, $dest);
$process = Process::timeout($timeout)->run($scp_command);
$output = trim($process->output());
$exitCode = $process->exitCode();
@@ -84,16 +79,8 @@ function instant_scp(string $source, string $dest, Server $server, $throwError =
return $output;
}
function generateSshCommand(Server $server, string $command)
{
return SshMultiplexingHelper::generateSshCommand($server, $command);
}
function instant_remote_process(Collection|array $command, Server $server, bool $throwError = true, bool $no_sudo = false): ?string
{
static $processCount = 0;
$processCount++;
$timeout = config('constants.ssh.command_timeout');
if ($command instanceof Collection) {
$command = $command->toArray();
@@ -104,7 +91,7 @@ function instant_remote_process(Collection|array $command, Server $server, bool
$command_string = implode("\n", $command);
$start_time = microtime(true);
$sshCommand = generateSshCommand($server, $command_string);
$sshCommand = SshMultiplexingHelper::generateSshCommand($server, $command_string);
$process = Process::timeout($timeout)->run($sshCommand);
$end_time = microtime(true);
@@ -222,11 +209,6 @@ function remove_iip($text)
return preg_replace('/\x1b\[[0-9;]*m/', '', $text);
}
function remove_mux_file(Server $server)
{
SshMultiplexingHelper::removeMuxFile($server);
}
function refresh_server_connection(?PrivateKey $private_key = null)
{
if (is_null($private_key)) {