From b433f17dac4f651391ca65ef8a30a5ca49af766b Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Wed, 10 Sep 2025 08:19:38 +0200 Subject: [PATCH] feat(ssh-multiplexing): enhance multiplexed connection management with health checks and metadata caching --- app/Helpers/SshMultiplexingHelper.php | 129 +++++++++++++++++++++++++- config/constants.php | 3 + 2 files changed, 128 insertions(+), 4 deletions(-) diff --git a/app/Helpers/SshMultiplexingHelper.php b/app/Helpers/SshMultiplexingHelper.php index 8caa2880a..bf9561f5a 100644 --- a/app/Helpers/SshMultiplexingHelper.php +++ b/app/Helpers/SshMultiplexingHelper.php @@ -4,7 +4,9 @@ namespace App\Helpers; use App\Models\PrivateKey; use App\Models\Server; +use Illuminate\Support\Facades\Cache; use Illuminate\Support\Facades\Hash; +use Illuminate\Support\Facades\Log; use Illuminate\Support\Facades\Process; class SshMultiplexingHelper @@ -30,6 +32,7 @@ class SshMultiplexingHelper $sshConfig = self::serverSshConfiguration($server); $muxSocket = $sshConfig['muxFilename']; + // Check if connection exists $checkCommand = "ssh -O check -o ControlPath=$muxSocket "; if (data_get($server, 'settings.is_cloudflare_tunnel')) { $checkCommand .= '-o ProxyCommand="cloudflared access ssh --hostname %h" '; @@ -41,6 +44,18 @@ class SshMultiplexingHelper return self::establishNewMultiplexedConnection($server); } + // Connection exists, check if it needs refresh due to age + if (self::isConnectionExpired($server)) { + return self::refreshMultiplexedConnection($server); + } + + // Perform health check if enabled + if (config('constants.ssh.mux_health_check_enabled')) { + if (! self::isConnectionHealthy($server)) { + return self::refreshMultiplexedConnection($server); + } + } + return true; } @@ -65,6 +80,9 @@ class SshMultiplexingHelper return false; } + // Store connection metadata for tracking + self::storeConnectionMetadata($server); + return true; } @@ -79,6 +97,9 @@ class SshMultiplexingHelper } $closeCommand .= "{$server->user}@{$server->ip}"; Process::run($closeCommand); + + // Clear connection metadata from cache + self::clearConnectionMetadata($server); } public static function generateScpCommand(Server $server, string $source, string $dest) @@ -94,8 +115,18 @@ class SshMultiplexingHelper if ($server->isIpv6()) { $scp_command .= '-6 '; } - if (self::isMultiplexingEnabled() && self::ensureMultiplexedConnection($server)) { - $scp_command .= "-o ControlMaster=auto -o ControlPath=$muxSocket -o ControlPersist={$muxPersistTime} "; + if (self::isMultiplexingEnabled()) { + try { + if (self::ensureMultiplexedConnection($server)) { + $scp_command .= "-o ControlMaster=auto -o ControlPath=$muxSocket -o ControlPersist={$muxPersistTime} "; + } + } catch (\Exception $e) { + Log::warning('SSH multiplexing failed for SCP, falling back to non-multiplexed connection', [ + 'server' => $server->name ?? $server->ip, + 'error' => $e->getMessage(), + ]); + // Continue without multiplexing + } } if (data_get($server, 'settings.is_cloudflare_tunnel')) { @@ -130,8 +161,16 @@ class SshMultiplexingHelper $ssh_command = "timeout $timeout ssh "; - if (self::isMultiplexingEnabled() && self::ensureMultiplexedConnection($server)) { - $ssh_command .= "-o ControlMaster=auto -o ControlPath=$muxSocket -o ControlPersist={$muxPersistTime} "; + $multiplexingSuccessful = false; + if (self::isMultiplexingEnabled()) { + try { + $multiplexingSuccessful = self::ensureMultiplexedConnection($server); + if ($multiplexingSuccessful) { + $ssh_command .= "-o ControlMaster=auto -o ControlPath=$muxSocket -o ControlPersist={$muxPersistTime} "; + } + } catch (\Exception $e) { + // Continue without multiplexing + } } if (data_get($server, 'settings.is_cloudflare_tunnel')) { @@ -186,4 +225,86 @@ class SshMultiplexingHelper return $options; } + + /** + * Check if the multiplexed connection is healthy by running a test command + */ + public static function isConnectionHealthy(Server $server): bool + { + $sshConfig = self::serverSshConfiguration($server); + $muxSocket = $sshConfig['muxFilename']; + $healthCheckTimeout = config('constants.ssh.mux_health_check_timeout'); + + $healthCommand = "timeout $healthCheckTimeout ssh -o ControlMaster=auto -o ControlPath=$muxSocket "; + if (data_get($server, 'settings.is_cloudflare_tunnel')) { + $healthCommand .= '-o ProxyCommand="cloudflared access ssh --hostname %h" '; + } + $healthCommand .= "{$server->user}@{$server->ip} 'echo \"health_check_ok\"'"; + + $process = Process::run($healthCommand); + $isHealthy = $process->exitCode() === 0 && str_contains($process->output(), 'health_check_ok'); + + return $isHealthy; + } + + /** + * Check if the connection has exceeded its maximum age + */ + public static function isConnectionExpired(Server $server): bool + { + $connectionAge = self::getConnectionAge($server); + $maxAge = config('constants.ssh.mux_max_age'); + + return $connectionAge !== null && $connectionAge > $maxAge; + } + + /** + * Get the age of the current connection in seconds + */ + public static function getConnectionAge(Server $server): ?int + { + $cacheKey = "ssh_mux_connection_time_{$server->uuid}"; + $connectionTime = Cache::get($cacheKey); + + if ($connectionTime === null) { + return null; + } + + return time() - $connectionTime; + } + + /** + * Refresh a multiplexed connection by closing and re-establishing it + */ + public static function refreshMultiplexedConnection(Server $server): bool + { + Log::debug('Refreshing SSH multiplexed connection', [ + 'server' => $server->name ?? $server->ip, + 'age' => self::getConnectionAge($server), + ]); + + // Close existing connection + self::removeMuxFile($server); + + // Establish new connection + return self::establishNewMultiplexedConnection($server); + } + + /** + * Store connection metadata when a new connection is established + */ + private static function storeConnectionMetadata(Server $server): void + { + $cacheKey = "ssh_mux_connection_time_{$server->uuid}"; + Cache::put($cacheKey, time(), config('constants.ssh.mux_persist_time') + 300); // Cache slightly longer than persist time + } + + /** + * Clear connection metadata from cache + */ + private static function clearConnectionMetadata(Server $server): void + { + $cacheKey = "ssh_mux_connection_time_{$server->uuid}"; + Cache::forget($cacheKey); + } } diff --git a/config/constants.php b/config/constants.php index 652af5ff4..0d29c997e 100644 --- a/config/constants.php +++ b/config/constants.php @@ -59,6 +59,9 @@ return [ 'ssh' => [ 'mux_enabled' => env('MUX_ENABLED', env('SSH_MUX_ENABLED', true)), 'mux_persist_time' => env('SSH_MUX_PERSIST_TIME', 3600), + 'mux_health_check_enabled' => env('SSH_MUX_HEALTH_CHECK_ENABLED', true), + 'mux_health_check_timeout' => env('SSH_MUX_HEALTH_CHECK_TIMEOUT', 5), + 'mux_max_age' => env('SSH_MUX_MAX_AGE', 1800), // 30 minutes 'connection_timeout' => 10, 'server_interval' => 20, 'command_timeout' => 7200,