Fix: SSH multiplexing

app/Actions/CoolifyTask/RunRemoteProcess.php

@@ -10,6 +10,7 @@ use Illuminate\Process\ProcessResult;
 use Illuminate\Support\Facades\DB;
 use Illuminate\Support\Facades\Process;
 use Spatie\Activitylog\Models\Activity;
+use App\Helpers\SshMultiplexingHelper;
 
 class RunRemoteProcess
 {
@@ -137,7 +138,7 @@ class RunRemoteProcess
         $command = $this->activity->getExtraProperty('command');
         $server = Server::whereUuid($server_uuid)->firstOrFail();
 
-        return generateSshCommand($server, $command);
+        return SshMultiplexingHelper::generateSshCommand($server, $command);
     }
 
     protected function handleOutput(string $type, string $output)

app/Helpers/SshMultiplexingHelper.php

@@ -3,11 +3,10 @@
 namespace App\Helpers;
 
 use App\Models\Server;
+use App\Models\PrivateKey;
 use Illuminate\Support\Facades\Process;
 use Illuminate\Support\Facades\Storage;
 use Illuminate\Support\Facades\Hash;
-use Illuminate\Support\Str;
-use App\Models\PrivateKey;
 
 class SshMultiplexingHelper
 {
@@ -15,7 +14,8 @@ class SshMultiplexingHelper
 
     public static function serverSshConfiguration(Server $server)
     {
-        $sshKeyLocation = $server->privateKey->getKeyLocation();
+        $privateKey = PrivateKey::findOrFail($server->private_key_id);
+        $sshKeyLocation = $privateKey->getKeyLocation();
         $muxFilename = '/var/www/html/storage/app/ssh/mux/' . $server->muxFilename();
 
         return [
@@ -26,15 +26,21 @@ class SshMultiplexingHelper
 
     public static function ensureMultiplexedConnection(Server $server)
     {
+        if (!self::isMultiplexingEnabled()) {
+            ray('Multiplexing is disabled');
+            return;
+        }
+
+        ray('Ensuring multiplexed connection for server: ' . $server->id);
+
         $sshConfig = self::serverSshConfiguration($server);
         $muxSocket = $sshConfig['muxFilename'];
         $sshKeyLocation = $sshConfig['sshKeyLocation'];
 
-        if (!file_exists($sshKeyLocation)) {
+        self::validateSshKey($sshKeyLocation);
-            throw new \RuntimeException("SSH key file not accessible: $sshKeyLocation");
-        }
 
         if (isset(self::$ensuredConnections[$server->id]) && !self::shouldResetMultiplexedConnection($server)) {
+            ray('Existing connection is still valid');
             return;
         }
 
@@ -42,6 +48,7 @@ class SshMultiplexingHelper
         $fileCheckProcess = Process::run($checkFileCommand);
 
         if ($fileCheckProcess->exitCode() !== 0) {
+            ray('Mux socket file not found, establishing new connection');
             self::establishNewMultiplexedConnection($server);
             return;
         }
@@ -49,19 +56,22 @@ class SshMultiplexingHelper
         $checkCommand = "ssh -O check -o ControlPath=$muxSocket {$server->user}@{$server->ip}";
         $process = Process::run($checkCommand);
 
-        if ($process->exitCode() === 0) {
+        if ($process->exitCode() !== 0) {
+            ray('Existing connection check failed, establishing new connection');
+            self::establishNewMultiplexedConnection($server);
+        } else {
+            ray('Existing connection is valid');
             self::$ensuredConnections[$server->id] = [
                 'timestamp' => now(),
                 'muxSocket' => $muxSocket,
             ];
-            return;
         }
-
-        self::establishNewMultiplexedConnection($server);
     }
 
     public static function establishNewMultiplexedConnection(Server $server)
     {
+        ray('Establishing new multiplexed connection for server: ' . $server->id);
+
         $sshConfig = self::serverSshConfiguration($server);
         $sshKeyLocation = $sshConfig['sshKeyLocation'];
         $muxSocket = $sshConfig['muxFilename'];
@@ -84,9 +94,12 @@ class SshMultiplexingHelper
         $establishProcess = Process::run($establishCommand);
 
         if ($establishProcess->exitCode() !== 0) {
+            ray('Failed to establish multiplexed connection', $establishProcess->errorOutput());
             throw new \RuntimeException('Failed to establish multiplexed connection: ' . $establishProcess->errorOutput());
         }
 
+        ray('Multiplexed connection established successfully');
+
         $muxContent = "Multiplexed connection established at " . now()->toDateTimeString();
         Storage::disk('ssh-mux')->put(basename($muxSocket), $muxContent);
 
@@ -99,6 +112,7 @@ class SshMultiplexingHelper
     public static function shouldResetMultiplexedConnection(Server $server)
     {
         if (!(config('constants.ssh.mux_enabled') && config('coolify.is_windows_docker_desktop') == false)) {
+            ray('Multiplexing is disabled or running on Windows Docker Desktop');
             return false;
         }
 
@@ -110,7 +124,9 @@ class SshMultiplexingHelper
         $muxPersistTime = config('constants.ssh.mux_persist_time');
         $resetInterval = strtotime($muxPersistTime) - time();
 
-        return $lastEnsured->addSeconds($resetInterval)->isPast();
+        $shouldReset = $lastEnsured->addSeconds($resetInterval)->isPast();
+        ray('Should reset multiplexed connection', ['server_id' => $server->id, 'should_reset' => $shouldReset]);
+        return $shouldReset;
     }
 
     public static function removeMuxFile(Server $server)
@@ -130,38 +146,22 @@ class SshMultiplexingHelper
         $sshKeyLocation = $sshConfig['sshKeyLocation'];
         $muxSocket = $sshConfig['muxFilename'];
 
-        $user = $server->user;
-        $port = $server->port;
         $timeout = config('constants.ssh.command_timeout');
         $connectionTimeout = config('constants.ssh.connection_timeout');
         $serverInterval = config('constants.ssh.server_interval');
-        $muxPersistTime = config('constants.ssh.mux_persist_time');
 
         $scp_command = "timeout $timeout scp ";
-        $muxEnabled = config('constants.ssh.mux_enabled', true) && config('coolify.is_windows_docker_desktop') == false;
-        ray('SSH Multiplexing Enabled:', $muxEnabled)->blue();
 
-        if ($muxEnabled) {
+        if (self::isMultiplexingEnabled()) {
+            $muxPersistTime = config('constants.ssh.mux_persist_time');
             $scp_command .= "-o ControlMaster=auto -o ControlPath=$muxSocket -o ControlPersist={$muxPersistTime} ";
             self::ensureMultiplexedConnection($server);
-            ray('Using SSH Multiplexing')->green();
-        } else {
-            ray('Not using SSH Multiplexing')->red();
         }
 
-        if (data_get($server, 'settings.is_cloudflare_tunnel')) {
+        self::addCloudflareProxyCommand($scp_command, $server);
-            $scp_command .= '-o ProxyCommand="/usr/local/bin/cloudflared access ssh --hostname %h" ';
+
-        }
+        $scp_command .= self::getCommonSshOptions($server, $sshKeyLocation, $connectionTimeout, $serverInterval);
-        $scp_command .= "-i {$sshKeyLocation} "
+        $scp_command .= "{$source} {$server->user}@{$server->ip}:{$dest}";
-            .'-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null '
-            .'-o PasswordAuthentication=no '
-            ."-o ConnectTimeout=$connectionTimeout "
-            ."-o ServerAliveInterval=$serverInterval "
-            .'-o RequestTTY=no '
-            .'-o LogLevel=ERROR '
-            ."-P {$port} "
-            ."{$source} "
-            ."{$user}@{$server->ip}:{$dest}";
 
         return $scp_command;
     }
@@ -179,45 +179,61 @@ class SshMultiplexingHelper
         $timeout = config('constants.ssh.command_timeout');
         $connectionTimeout = config('constants.ssh.connection_timeout');
         $serverInterval = config('constants.ssh.server_interval');
-        $muxPersistTime = config('constants.ssh.mux_persist_time');
-        $muxEnabled = config('constants.ssh.mux_enabled') && !config('coolify.is_windows_docker_desktop');
-        ray('Config MUX Enabled:', config('constants.ssh.mux_enabled'));
-        ray('Config Windows Docker Desktop:', config('coolify.is_windows_docker_desktop'));
-        ray('MUX Enabled:', $muxEnabled);
 
         $ssh_command = "timeout $timeout ssh ";
 
-        ray('SSH Multiplexing Enabled:', $muxEnabled)->blue();
+        if (self::isMultiplexingEnabled()) {
-
+            $muxPersistTime = config('constants.ssh.mux_persist_time');
-        if ($muxEnabled) {
             $ssh_command .= "-o ControlMaster=auto -o ControlPath=$muxSocket -o ControlPersist={$muxPersistTime} ";
             self::ensureMultiplexedConnection($server);
-            ray('Using SSH Multiplexing')->green();
-        } else {
-            ray('Not using SSH Multiplexing')->red();
         }
 
-        if (data_get($server, 'settings.is_cloudflare_tunnel')) {
+        self::addCloudflareProxyCommand($ssh_command, $server);
-            $ssh_command .= '-o ProxyCommand="/usr/local/bin/cloudflared access ssh --hostname %h" ';
+
-        }
+        $ssh_command .= self::getCommonSshOptions($server, $sshKeyLocation, $connectionTimeout, $serverInterval);
 
         $command = "PATH=\$PATH:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/host/usr/local/sbin:/host/usr/local/bin:/host/usr/sbin:/host/usr/bin:/host/sbin:/host/bin && $command";
         $delimiter = Hash::make($command);
         $command = str_replace($delimiter, '', $command);
 
-        $ssh_command .= "-i {$sshKeyLocation} "
+        $ssh_command .= "{$server->user}@{$server->ip} 'bash -se' << \\$delimiter".PHP_EOL
+            .$command.PHP_EOL
+            .$delimiter;
+
+        return $ssh_command;
+    }
+
+    private static function isMultiplexingEnabled(): bool
+    {
+        return config('constants.ssh.mux_enabled') && !config('coolify.is_windows_docker_desktop');
+    }
+
+    private static function validateSshKey(string $sshKeyLocation): void
+    {
+        $checkKeyCommand = "ls $sshKeyLocation 2>/dev/null";
+        $keyCheckProcess = Process::run($checkKeyCommand);
+
+        if ($keyCheckProcess->exitCode() !== 0) {
+            throw new \RuntimeException("SSH key file not accessible: $sshKeyLocation");
+        }
+    }
+
+    private static function addCloudflareProxyCommand(string &$command, Server $server): void
+    {
+        if (data_get($server, 'settings.is_cloudflare_tunnel')) {
+            $command .= '-o ProxyCommand="/usr/local/bin/cloudflared access ssh --hostname %h" ';
+        }
+    }
+
+    private static function getCommonSshOptions(Server $server, string $sshKeyLocation, int $connectionTimeout, int $serverInterval): string
+    {
+        return "-i {$sshKeyLocation} "
             .'-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null '
             .'-o PasswordAuthentication=no '
             ."-o ConnectTimeout=$connectionTimeout "
             ."-o ServerAliveInterval=$serverInterval "
             .'-o RequestTTY=no '
             .'-o LogLevel=ERROR '
-            ."-p {$server->port} "
+            ."-p {$server->port} ";
-            ."{$server->user}@{$server->ip} "
-            ." 'bash -se' << \\$delimiter".PHP_EOL
-            .$command.PHP_EOL
-            .$delimiter;
-
-        return $ssh_command;
     }
 }

app/Jobs/ServerCheckJob.php

@@ -43,7 +43,7 @@ class ServerCheckJob implements ShouldBeEncrypted, ShouldQueue
         return isDev() ? 1 : 3;
     }
 
-    public function __construct(public Server $server) {}
+    public function __construct(public Server $server, public bool $isManualCheck = false) {}
 
     public function middleware(): array
     {
@@ -58,6 +58,9 @@ class ServerCheckJob implements ShouldBeEncrypted, ShouldQueue
     public function handle()
     {
         try {
+            // Enable SSH multiplexing for autonomous checks, disable for manual checks
+            config()->set('constants.ssh.mux_enabled', !$this->isManualCheck);
+
             $this->applications = $this->server->applications();
             $this->databases = $this->server->databases();
             $this->services = $this->server->services()->get();
@@ -93,7 +96,7 @@ class ServerCheckJob implements ShouldBeEncrypted, ShouldQueue
 
     private function serverStatus()
     {
-        ['uptime' => $uptime] = $this->server->validateConnection();
+        ['uptime' => $uptime] = $this->server->validateConnection($this->isManualCheck);
         if ($uptime) {
             if ($this->server->unreachable_notification_sent === true) {
                 $this->server->update(['unreachable_notification_sent' => false]);

app/Livewire/Project/Shared/GetLogs.php

@@ -17,6 +17,7 @@ use App\Models\StandalonePostgresql;
 use App\Models\StandaloneRedis;
 use Illuminate\Support\Facades\Process;
 use Livewire\Component;
+use App\Helpers\SshMultiplexingHelper;
 
 class GetLogs extends Component
 {
@@ -108,14 +109,14 @@ class GetLogs extends Component
                         $command = parseCommandsByLineForSudo(collect($command), $this->server);
                         $command = $command[0];
                     }
-                    $sshCommand = generateSshCommand($this->server, $command);
+                    $sshCommand = SshMultiplexingHelper::generateSshCommand($this->server, $command);
                 } else {
                     $command = "docker logs -n {$this->numberOfLines} -t {$this->container}";
                     if ($this->server->isNonRoot()) {
                         $command = parseCommandsByLineForSudo(collect($command), $this->server);
                         $command = $command[0];
                     }
-                    $sshCommand = generateSshCommand($this->server, $command);
+                    $sshCommand = SshMultiplexingHelper::generateSshCommand($this->server, $command);
                 }
             } else {
                 if ($this->server->isSwarm()) {
@@ -124,14 +125,14 @@ class GetLogs extends Component
                         $command = parseCommandsByLineForSudo(collect($command), $this->server);
                         $command = $command[0];
                     }
-                    $sshCommand = generateSshCommand($this->server, $command);
+                    $sshCommand = SshMultiplexingHelper::generateSshCommand($this->server, $command);
                 } else {
                     $command = "docker logs -n {$this->numberOfLines} {$this->container}";
                     if ($this->server->isNonRoot()) {
                         $command = parseCommandsByLineForSudo(collect($command), $this->server);
                         $command = $command[0];
                     }
-                    $sshCommand = generateSshCommand($this->server, $command);
+                    $sshCommand = SshMultiplexingHelper::generateSshCommand($this->server, $command);
                 }
             }
             if ($refresh) {

app/Models/Server.php

@@ -967,9 +967,10 @@ $schema://$host {
         return data_get($this, 'settings.is_swarm_worker');
     }
 
-    public function validateConnection()
+    public function validateConnection($isManualCheck = true)
     {
-        config()->set('constants.ssh.mux_enabled', false);
+        // Set mux_enabled to true for automatic checks, false for manual checks
+        config()->set('constants.ssh.mux_enabled', !$isManualCheck);
 
         $server = Server::find($this->id);
         if (! $server) {
@@ -979,7 +980,6 @@ $schema://$host {
             return ['uptime' => false, 'error' => 'Server skipped.'];
         }
         try {
-            // EC2 does not have `uptime` command, lol
             instant_remote_process(['ls /'], $server);
             $server->settings()->update([
                 'is_reachable' => true,
@@ -988,7 +988,6 @@ $schema://$host {
                 'unreachable_count' => 0,
             ]);
             if (data_get($server, 'unreachable_notification_sent') === true) {
-                // $server->team?->notify(new Revived($server));
                 $server->update(['unreachable_notification_sent' => false]);
             }
 

app/Traits/ExecuteRemoteCommand.php

@@ -7,6 +7,7 @@ use App\Models\Server;
 use Carbon\Carbon;
 use Illuminate\Support\Collection;
 use Illuminate\Support\Facades\Process;
+use App\Helpers\SshMultiplexingHelper;
 
 trait ExecuteRemoteCommand
 {
@@ -42,7 +43,7 @@ trait ExecuteRemoteCommand
                     $command = parseLineForSudo($command, $this->server);
                 }
             }
-            $remote_command = generateSshCommand($this->server, $command);
+            $remote_command = SshMultiplexingHelper::generateSshCommand($this->server, $command);
             $process = Process::timeout(3600)->idleTimeout(3600)->start($remote_command, function (string $type, string $output) use ($command, $hidden, $customType, $append) {
                 $output = str($output)->trim();
                 if ($output->startsWith('╔')) {

bootstrap/helpers/remoteProcess.php

@@ -35,8 +35,8 @@ function remote_process(
     
     $command_string = implode("\n", $command);
     
-    if (auth()->user()) {
+    if (Auth::check()) {
-        $teams = auth()->user()->teams->pluck('id');
+        $teams = Auth::user()->teams->pluck('id');
         if (!$teams->contains($server->team_id) && !$teams->contains(0)) {
             throw new \Exception('User is not part of the team that owns this server');
         }
@@ -58,15 +58,10 @@ function remote_process(
     ])();
 }
 
-function generateScpCommand(Server $server, string $source, string $dest)
-{
-    return SshMultiplexingHelper::generateScpCommand($server, $source, $dest);
-}
-
 function instant_scp(string $source, string $dest, Server $server, $throwError = true)
 {
     $timeout = config('constants.ssh.command_timeout');
-    $scp_command = generateScpCommand($server, $source, $dest);
+    $scp_command = SshMultiplexingHelper::generateScpCommand($server, $source, $dest);
     $process = Process::timeout($timeout)->run($scp_command);
     $output = trim($process->output());
     $exitCode = $process->exitCode();
@@ -84,16 +79,8 @@ function instant_scp(string $source, string $dest, Server $server, $throwError =
     return $output;
 }
 
-function generateSshCommand(Server $server, string $command)
-{
-    return SshMultiplexingHelper::generateSshCommand($server, $command);
-}
-
 function instant_remote_process(Collection|array $command, Server $server, bool $throwError = true, bool $no_sudo = false): ?string
 {
-    static $processCount = 0;
-    $processCount++;
-
     $timeout = config('constants.ssh.command_timeout');
     if ($command instanceof Collection) {
         $command = $command->toArray();
@@ -104,7 +91,7 @@ function instant_remote_process(Collection|array $command, Server $server, bool
     $command_string = implode("\n", $command);
 
     $start_time = microtime(true);
-    $sshCommand = generateSshCommand($server, $command_string);
+    $sshCommand = SshMultiplexingHelper::generateSshCommand($server, $command_string);
     $process = Process::timeout($timeout)->run($sshCommand);
     $end_time = microtime(true);
 
@@ -222,11 +209,6 @@ function remove_iip($text)
     return preg_replace('/\x1b\[[0-9;]*m/', '', $text);
 }
 
-function remove_mux_file(Server $server)
-{
-    SshMultiplexingHelper::removeMuxFile($server);
-}
-
 function refresh_server_connection(?PrivateKey $private_key = null)
 {
     if (is_null($private_key)) {