From cc10d08a7c2b054ae0be217227f9930b65c4228d Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Sun, 8 Sep 2024 18:13:00 +0200 Subject: [PATCH 1/4] Feat: Implement SSH Multiplexing --- bootstrap/helpers/remoteProcess.php | 62 ++++++++++++++++++++++------- config/constants.php | 2 +- 2 files changed, 48 insertions(+), 16 deletions(-) diff --git a/bootstrap/helpers/remoteProcess.php b/bootstrap/helpers/remoteProcess.php index 3f5cdfae2..c51cbfd66 100644 --- a/bootstrap/helpers/remoteProcess.php +++ b/bootstrap/helpers/remoteProcess.php @@ -130,7 +130,7 @@ function instant_scp(string $source, string $dest, Server $server, $throwError = return $output; } -function generateSshCommand(Server $server, string $command) +function generateSshCommand(Server $server, string $command, bool $useMux = true) { if ($server->settings->force_disabled) { throw new \RuntimeException('Server is disabled.'); @@ -145,9 +145,12 @@ function generateSshCommand(Server $server, string $command) $ssh_command = "timeout $timeout ssh "; - if (config('coolify.mux_enabled') && config('coolify.is_windows_docker_desktop') == false) { - $ssh_command .= "-o ControlMaster=auto -o ControlPersist={$muxPersistTime} -o ControlPath=/var/www/html/storage/app/ssh/mux/{$server->muxFilename()} "; + if ($useMux && config('coolify.mux_enabled') && config('coolify.is_windows_docker_desktop') == false) { + $muxSocket = "/var/www/html/storage/app/ssh/mux/{$server->muxFilename()}"; + $ssh_command .= "-o ControlMaster=auto -o ControlPath=$muxSocket -o ControlPersist={$muxPersistTime} "; + ensureMultiplexedConnection($server); } + if (data_get($server, 'settings.is_cloudflare_tunnel')) { $ssh_command .= '-o ProxyCommand="/usr/local/bin/cloudflared access ssh --hostname %h" '; } @@ -169,6 +172,34 @@ function generateSshCommand(Server $server, string $command) return $ssh_command; } + +function ensureMultiplexedConnection(Server $server) +{ + $muxSocket = "/var/www/html/storage/app/ssh/mux/{$server->muxFilename()}"; + $privateKeyLocation = savePrivateKeyToFs($server); + $connectionTimeout = config('constants.ssh.connection_timeout'); + $serverInterval = config('constants.ssh.server_interval'); + $muxPersistTime = config('constants.ssh.mux_persist_time'); + + $checkCommand = "ssh -O check -o ControlPath=$muxSocket {$server->user}@{$server->ip} 2>/dev/null"; + $process = Process::run($checkCommand); + + if ($process->exitCode() !== 0) { + $establishCommand = "ssh -fNM -o ControlMaster=auto -o ControlPath=$muxSocket -o ControlPersist={$muxPersistTime} " + . "-i {$privateKeyLocation} " + . "-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null " + . "-o PasswordAuthentication=no " + . "-o ConnectTimeout=$connectionTimeout " + . "-o ServerAliveInterval=$serverInterval " + . "-o RequestTTY=no " + . "-o LogLevel=ERROR " + . "-p {$server->port} " + . "{$server->user}@{$server->ip}"; + + Process::run($establishCommand); + } +} + function instant_remote_process(Collection|array $command, Server $server, bool $throwError = true, bool $no_sudo = false): ?string { $timeout = config('constants.ssh.command_timeout'); @@ -179,10 +210,13 @@ function instant_remote_process(Collection|array $command, Server $server, bool $command = parseCommandsByLineForSudo(collect($command), $server); } $command_string = implode("\n", $command); - $ssh_command = generateSshCommand($server, $command_string, $no_sudo); - $process = Process::timeout($timeout)->run($ssh_command); + + $sshCommand = generateSshCommand($server, $command_string, true); + $process = Process::timeout($timeout)->run($sshCommand); + $output = trim($process->output()); $exitCode = $process->exitCode(); + if ($exitCode !== 0) { if (! $throwError) { return null; @@ -222,7 +256,6 @@ function decode_remote_command_output(?ApplicationDeploymentQueue $application_d if (is_null($application_deployment_queue)) { return collect([]); } - // ray(data_get($application_deployment_queue, 'logs')); try { $decoded = json_decode( data_get($application_deployment_queue, 'logs'), @@ -232,7 +265,6 @@ function decode_remote_command_output(?ApplicationDeploymentQueue $application_d } catch (\JsonException $exception) { return collect([]); } - // ray($decoded ); $seenCommands = collect(); $formatted = collect($decoded); if (! $is_debug_enabled) { @@ -293,6 +325,10 @@ function remove_mux_and_private_key(Server $server) { $muxFilename = $server->muxFilename(); $privateKeyLocation = savePrivateKeyToFs($server); + + $closeCommand = "ssh -O exit -o ControlPath=/var/www/html/storage/app/ssh/mux/{$muxFilename} {$server->user}@{$server->ip}"; + Process::run($closeCommand); + Storage::disk('ssh-mux')->delete($muxFilename); Storage::disk('ssh-keys')->delete($privateKeyLocation); } @@ -302,7 +338,10 @@ function refresh_server_connection(?PrivateKey $private_key = null) return; } foreach ($private_key->servers as $server) { - Storage::disk('ssh-mux')->delete($server->muxFilename()); + $muxFilename = $server->muxFilename(); + $closeCommand = "ssh -O exit -o ControlPath=/var/www/html/storage/app/ssh/mux/{$muxFilename} {$server->user}@{$server->ip}"; + Process::run($closeCommand); + Storage::disk('ssh-mux')->delete($muxFilename); } } @@ -312,24 +351,17 @@ function checkRequiredCommands(Server $server) foreach ($commands as $command) { $commandFound = instant_remote_process(["docker run --rm --privileged --net=host --pid=host --ipc=host --volume /:/host busybox chroot /host bash -c 'command -v {$command}'"], $server, false); if ($commandFound) { - ray($command.' found'); - continue; } try { instant_remote_process(["docker run --rm --privileged --net=host --pid=host --ipc=host --volume /:/host busybox chroot /host bash -c 'apt update && apt install -y {$command}'"], $server); } catch (\Throwable $e) { - ray('could not install '.$command); - ray($e); break; } $commandFound = instant_remote_process(["docker run --rm --privileged --net=host --pid=host --ipc=host --volume /:/host busybox chroot /host bash -c 'command -v {$command}'"], $server, false); if ($commandFound) { - ray($command.' found'); - continue; } - ray('could not install '.$command); break; } } diff --git a/config/constants.php b/config/constants.php index 861b645ed..082be6e9e 100644 --- a/config/constants.php +++ b/config/constants.php @@ -6,7 +6,7 @@ return [ 'contact' => 'https://coolify.io/docs/contact', ], 'ssh' => [ - 'mux_persist_time' => env('SSH_MUX_PERSIST_TIME', '1m'), + 'mux_persist_time' => env('SSH_MUX_PERSIST_TIME', '1h'), 'connection_timeout' => 10, 'server_interval' => 20, 'command_timeout' => 7200, From c8218e690194ef0d1dcaac46c25f0cdcd05326cc Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Sun, 8 Sep 2024 19:15:37 +0200 Subject: [PATCH 2/4] Fix: Enabel mux --- bootstrap/helpers/remoteProcess.php | 107 +++++++++++++++++++++++----- config/constants.php | 1 + 2 files changed, 91 insertions(+), 17 deletions(-) diff --git a/bootstrap/helpers/remoteProcess.php b/bootstrap/helpers/remoteProcess.php index c51cbfd66..342eafa6d 100644 --- a/bootstrap/helpers/remoteProcess.php +++ b/bootstrap/helpers/remoteProcess.php @@ -130,7 +130,7 @@ function instant_scp(string $source, string $dest, Server $server, $throwError = return $output; } -function generateSshCommand(Server $server, string $command, bool $useMux = true) +function generateSshCommand(Server $server, string $command) { if ($server->settings->force_disabled) { throw new \RuntimeException('Server is disabled.'); @@ -145,10 +145,18 @@ function generateSshCommand(Server $server, string $command, bool $useMux = true $ssh_command = "timeout $timeout ssh "; - if ($useMux && config('coolify.mux_enabled') && config('coolify.is_windows_docker_desktop') == false) { + // Check if multiplexing is enabled + $muxEnabled = config('constants.ssh.mux_enabled', true); + ray('SSH Multiplexing Enabled:', $muxEnabled)->blue(); + + if ($muxEnabled) { + // Always use multiplexing when enabled $muxSocket = "/var/www/html/storage/app/ssh/mux/{$server->muxFilename()}"; $ssh_command .= "-o ControlMaster=auto -o ControlPath=$muxSocket -o ControlPersist={$muxPersistTime} "; ensureMultiplexedConnection($server); + ray('Using SSH Multiplexing')->green(); + } else { + ray('Not using SSH Multiplexing')->red(); } if (data_get($server, 'settings.is_cloudflare_tunnel')) { @@ -175,33 +183,93 @@ function generateSshCommand(Server $server, string $command, bool $useMux = true function ensureMultiplexedConnection(Server $server) { + static $ensuredConnections = []; + + if (isset($ensuredConnections[$server->id])) { + if (!shouldResetMultiplexedConnection($server)) { + ray('Using Existing Multiplexed Connection')->green(); + return; + } + } + $muxSocket = "/var/www/html/storage/app/ssh/mux/{$server->muxFilename()}"; + $checkCommand = "ssh -O check -o ControlPath=$muxSocket {$server->user}@{$server->ip} 2>/dev/null"; + + $process = Process::run($checkCommand); + + if ($process->exitCode() === 0) { + ray('Existing Multiplexed Connection is Valid')->green(); + $ensuredConnections[$server->id] = [ + 'timestamp' => now(), + 'muxSocket' => $muxSocket, + ]; + return; + } + + ray('Establishing New Multiplexed Connection')->orange(); + $privateKeyLocation = savePrivateKeyToFs($server); $connectionTimeout = config('constants.ssh.connection_timeout'); $serverInterval = config('constants.ssh.server_interval'); $muxPersistTime = config('constants.ssh.mux_persist_time'); - $checkCommand = "ssh -O check -o ControlPath=$muxSocket {$server->user}@{$server->ip} 2>/dev/null"; - $process = Process::run($checkCommand); + $establishCommand = "ssh -fNM -o ControlMaster=auto -o ControlPath=$muxSocket -o ControlPersist={$muxPersistTime} " + . "-i {$privateKeyLocation} " + . "-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null " + . "-o PasswordAuthentication=no " + . "-o ConnectTimeout=$connectionTimeout " + . "-o ServerAliveInterval=$serverInterval " + . "-o RequestTTY=no " + . "-o LogLevel=ERROR " + . "-p {$server->port} " + . "{$server->user}@{$server->ip}"; + + $establishProcess = Process::run($establishCommand); - if ($process->exitCode() !== 0) { - $establishCommand = "ssh -fNM -o ControlMaster=auto -o ControlPath=$muxSocket -o ControlPersist={$muxPersistTime} " - . "-i {$privateKeyLocation} " - . "-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null " - . "-o PasswordAuthentication=no " - . "-o ConnectTimeout=$connectionTimeout " - . "-o ServerAliveInterval=$serverInterval " - . "-o RequestTTY=no " - . "-o LogLevel=ERROR " - . "-p {$server->port} " - . "{$server->user}@{$server->ip}"; + if ($establishProcess->exitCode() !== 0) { + throw new \RuntimeException("Failed to establish multiplexed connection: " . $establishProcess->errorOutput()); + } - Process::run($establishCommand); + $ensuredConnections[$server->id] = [ + 'timestamp' => now(), + 'muxSocket' => $muxSocket, + ]; + + ray('Established New Multiplexed Connection')->green(); +} + +function shouldResetMultiplexedConnection(Server $server) +{ + static $ensuredConnections = []; + + if (!isset($ensuredConnections[$server->id])) { + return true; + } + + $lastEnsured = $ensuredConnections[$server->id]['timestamp']; + $muxPersistTime = config('constants.ssh.mux_persist_time'); + $resetInterval = strtotime($muxPersistTime) - time(); + + return $lastEnsured->addSeconds($resetInterval)->isPast(); +} + +function resetMultiplexedConnection(Server $server) +{ + static $ensuredConnections = []; + + if (isset($ensuredConnections[$server->id])) { + $muxSocket = $ensuredConnections[$server->id]['muxSocket']; + $closeCommand = "ssh -O exit -o ControlPath=$muxSocket {$server->user}@{$server->ip}"; + Process::run($closeCommand); + unset($ensuredConnections[$server->id]); } } function instant_remote_process(Collection|array $command, Server $server, bool $throwError = true, bool $no_sudo = false): ?string { + static $processCount = 0; + $processCount++; + $timeout = config('constants.ssh.command_timeout'); if ($command instanceof Collection) { $command = $command->toArray(); @@ -211,8 +279,13 @@ function instant_remote_process(Collection|array $command, Server $server, bool } $command_string = implode("\n", $command); - $sshCommand = generateSshCommand($server, $command_string, true); + $start_time = microtime(true); + $sshCommand = generateSshCommand($server, $command_string); $process = Process::timeout($timeout)->run($sshCommand); + $end_time = microtime(true); + + $execution_time = ($end_time - $start_time) * 1000; // Convert to milliseconds + ray('SSH command execution time:', $execution_time . ' ms')->orange(); $output = trim($process->output()); $exitCode = $process->exitCode(); diff --git a/config/constants.php b/config/constants.php index 082be6e9e..c223e6418 100644 --- a/config/constants.php +++ b/config/constants.php @@ -6,6 +6,7 @@ return [ 'contact' => 'https://coolify.io/docs/contact', ], 'ssh' => [ + 'mux_enabled' => env('SSH_MUX_ENABLED', true), 'mux_persist_time' => env('SSH_MUX_PERSIST_TIME', '1h'), 'connection_timeout' => 10, 'server_interval' => 20, From 4f9e1a3e5e105984ddfed4b997084d6d90ffdeab Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Sun, 8 Sep 2024 19:37:00 +0200 Subject: [PATCH 3/4] Feat: Cleanup stale multiplexing connections --- app/Console/Kernel.php | 6 ++- .../CleanupStaleMultiplexedConnections.php | 43 +++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 app/Jobs/CleanupStaleMultiplexedConnections.php diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index 96740ab24..b960a4a8b 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -4,6 +4,7 @@ namespace App\Console; use App\Jobs\CheckForUpdatesJob; use App\Jobs\CleanupInstanceStuffsJob; +use App\Jobs\CleanupStaleMultiplexedConnections; use App\Jobs\DatabaseBackupJob; use App\Jobs\DockerCleanupJob; use App\Jobs\PullHelperImageJob; @@ -29,7 +30,8 @@ class Kernel extends ConsoleKernel $this->all_servers = Server::all(); $settings = InstanceSettings::get(); - $schedule->command('telescope:prune')->daily(); + $schedule->job(new CleanupStaleMultiplexedConnections)->hourly(); + if (isDev()) { // Instance Jobs $schedule->command('horizon:snapshot')->everyMinute(); @@ -39,6 +41,8 @@ class Kernel extends ConsoleKernel $this->check_resources($schedule); $this->check_scheduled_tasks($schedule); $schedule->command('uploads:clear')->everyTwoMinutes(); + + $schedule->command('telescope:prune')->daily(); } else { // Instance Jobs $schedule->command('horizon:snapshot')->everyFiveMinutes(); diff --git a/app/Jobs/CleanupStaleMultiplexedConnections.php b/app/Jobs/CleanupStaleMultiplexedConnections.php new file mode 100644 index 000000000..733adb8d4 --- /dev/null +++ b/app/Jobs/CleanupStaleMultiplexedConnections.php @@ -0,0 +1,43 @@ +cleanupStaleConnection($server); + } + }); + } + + private function cleanupStaleConnection(Server $server) + { + $cacheKey = "mux_connection_{$server->id}"; + $cachedConnection = cache()->get($cacheKey); + + if ($cachedConnection) { + $muxSocket = $cachedConnection['muxSocket']; + $checkCommand = "ssh -O check -o ControlPath=$muxSocket {$server->user}@{$server->ip} 2>/dev/null"; + $checkProcess = Process::run($checkCommand); + + if ($checkProcess->exitCode() !== 0) { + $closeCommand = "ssh -O exit -o ControlPath=$muxSocket {$server->user}@{$server->ip} 2>/dev/null"; + Process::run($closeCommand); + cache()->forget($cacheKey); + } + } + } +} From 28bcd0023cc442dcad604f25fa368ba0c7129ab6 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Mon, 9 Sep 2024 10:33:57 +0200 Subject: [PATCH 4/4] remove cache --- .../CleanupStaleMultiplexedConnections.php | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/app/Jobs/CleanupStaleMultiplexedConnections.php b/app/Jobs/CleanupStaleMultiplexedConnections.php index 733adb8d4..bcca77c18 100644 --- a/app/Jobs/CleanupStaleMultiplexedConnections.php +++ b/app/Jobs/CleanupStaleMultiplexedConnections.php @@ -25,19 +25,13 @@ class CleanupStaleMultiplexedConnections implements ShouldQueue private function cleanupStaleConnection(Server $server) { - $cacheKey = "mux_connection_{$server->id}"; - $cachedConnection = cache()->get($cacheKey); + $muxSocket = "/tmp/mux_{$server->id}"; + $checkCommand = "ssh -O check -o ControlPath=$muxSocket {$server->user}@{$server->ip} 2>/dev/null"; + $checkProcess = Process::run($checkCommand); - if ($cachedConnection) { - $muxSocket = $cachedConnection['muxSocket']; - $checkCommand = "ssh -O check -o ControlPath=$muxSocket {$server->user}@{$server->ip} 2>/dev/null"; - $checkProcess = Process::run($checkCommand); - - if ($checkProcess->exitCode() !== 0) { - $closeCommand = "ssh -O exit -o ControlPath=$muxSocket {$server->user}@{$server->ip} 2>/dev/null"; - Process::run($closeCommand); - cache()->forget($cacheKey); - } + if ($checkProcess->exitCode() !== 0) { + $closeCommand = "ssh -O exit -o ControlPath=$muxSocket {$server->user}@{$server->ip} 2>/dev/null"; + Process::run($closeCommand); } } }