Fix: SSH Multiplexing for Jobs

This commit is contained in:
peaklabs-dev
2024-09-17 15:31:05 +02:00
parent 175f4b9ae1
commit ea3501ada6
4 changed files with 34 additions and 15 deletions

View File

@@ -26,7 +26,7 @@ class CheckProxy
if (is_null($proxyType) || $proxyType === 'NONE' || $server->proxy->force_stop) { if (is_null($proxyType) || $proxyType === 'NONE' || $server->proxy->force_stop) {
return false; return false;
} }
['uptime' => $uptime, 'error' => $error] = $server->validateConnection(); ['uptime' => $uptime, 'error' => $error] = $server->validateConnection(false);
if (! $uptime) { if (! $uptime) {
throw new \Exception($error); throw new \Exception($error);
} }

View File

@@ -25,11 +25,12 @@ class SshMultiplexingHelper
public static function ensureMultiplexedConnection(Server $server) public static function ensureMultiplexedConnection(Server $server)
{ {
if (!self::isMultiplexingEnabled()) { if (!self::isMultiplexingEnabled()) {
ray('Multiplexing is disabled'); ray('SSH Multiplexing: DISABLED')->red();
return; return;
} }
ray('Ensuring multiplexed connection for server: ' . $server->id); ray('SSH Multiplexing: ENABLED')->green();
ray('Ensuring multiplexed connection for server:', $server->id);
$sshConfig = self::serverSshConfiguration($server); $sshConfig = self::serverSshConfiguration($server);
$muxSocket = $sshConfig['muxFilename']; $muxSocket = $sshConfig['muxFilename'];
@@ -41,16 +42,17 @@ class SshMultiplexingHelper
$process = Process::run($checkCommand); $process = Process::run($checkCommand);
if ($process->exitCode() !== 0) { if ($process->exitCode() !== 0) {
ray('Existing connection check failed or not found, establishing new connection'); ray('SSH Multiplexing: Existing connection check failed or not found')->orange();
ray('Establishing new connection');
self::establishNewMultiplexedConnection($server); self::establishNewMultiplexedConnection($server);
} else { } else {
ray('Existing connection is valid'); ray('SSH Multiplexing: Existing connection is valid')->green();
} }
} }
public static function establishNewMultiplexedConnection(Server $server) public static function establishNewMultiplexedConnection(Server $server)
{ {
ray('Establishing new multiplexed connection for server: ' . $server->id); ray('SSH Multiplexing: Establishing new connection for server:', $server->id);
$sshConfig = self::serverSshConfiguration($server); $sshConfig = self::serverSshConfiguration($server);
$sshKeyLocation = $sshConfig['sshKeyLocation']; $sshKeyLocation = $sshConfig['sshKeyLocation'];
@@ -74,11 +76,11 @@ class SshMultiplexingHelper
$establishProcess = Process::run($establishCommand); $establishProcess = Process::run($establishCommand);
if ($establishProcess->exitCode() !== 0) { if ($establishProcess->exitCode() !== 0) {
ray('Failed to establish multiplexed connection', $establishProcess->errorOutput()); ray('SSH Multiplexing: Failed to establish connection', $establishProcess->errorOutput())->red();
throw new \RuntimeException('Failed to establish multiplexed connection: ' . $establishProcess->errorOutput()); throw new \RuntimeException('Failed to establish multiplexed connection: ' . $establishProcess->errorOutput());
} }
ray('Multiplexed connection established successfully'); ray('SSH Multiplexing: Connection established successfully')->green();
$muxContent = "Multiplexed connection established at " . now()->toDateTimeString(); $muxContent = "Multiplexed connection established at " . now()->toDateTimeString();
Storage::disk('ssh-mux')->put(basename($muxSocket), $muxContent); Storage::disk('ssh-mux')->put(basename($muxSocket), $muxContent);
@@ -108,9 +110,18 @@ class SshMultiplexingHelper
$scp_command = "timeout $timeout scp "; $scp_command = "timeout $timeout scp ";
if (self::isMultiplexingEnabled()) { if (self::isMultiplexingEnabled()) {
ray('SSH Multiplexing: Enabled for SCP command')->green();
$muxPersistTime = config('constants.ssh.mux_persist_time'); $muxPersistTime = config('constants.ssh.mux_persist_time');
$scp_command .= "-o ControlMaster=auto -o ControlPath=$muxSocket -o ControlPersist={$muxPersistTime} "; $scp_command .= "-o ControlMaster=auto -o ControlPath=$muxSocket -o ControlPersist={$muxPersistTime} ";
self::ensureMultiplexedConnection($server); self::ensureMultiplexedConnection($server);
// Add this line to verify multiplexing is being used
ray('SSH Multiplexing: Verifying usage')->blue();
$checkCommand = "ssh -O check -o ControlPath=$muxSocket {$server->user}@{$server->ip}";
$checkProcess = Process::run($checkCommand);
ray('SSH Multiplexing: ' . ($checkProcess->exitCode() === 0 ? 'Active' : 'Not Active'))->color($checkProcess->exitCode() === 0 ? 'green' : 'red');
} else {
ray('SSH Multiplexing: Disabled for SCP command')->orange();
} }
self::addCloudflareProxyCommand($scp_command, $server); self::addCloudflareProxyCommand($scp_command, $server);
@@ -138,9 +149,18 @@ class SshMultiplexingHelper
$ssh_command = "timeout $timeout ssh "; $ssh_command = "timeout $timeout ssh ";
if (self::isMultiplexingEnabled()) { if (self::isMultiplexingEnabled()) {
ray('SSH Multiplexing: Enabled for SSH command')->green();
$muxPersistTime = config('constants.ssh.mux_persist_time'); $muxPersistTime = config('constants.ssh.mux_persist_time');
$ssh_command .= "-o ControlMaster=auto -o ControlPath=$muxSocket -o ControlPersist={$muxPersistTime} "; $ssh_command .= "-o ControlMaster=auto -o ControlPath=$muxSocket -o ControlPersist={$muxPersistTime} ";
self::ensureMultiplexedConnection($server); self::ensureMultiplexedConnection($server);
// Add this line to verify multiplexing is being used
ray('SSH Multiplexing: Verifying usage')->blue();
$checkCommand = "ssh -O check -o ControlPath=$muxSocket {$server->user}@{$server->ip}";
$checkProcess = Process::run($checkCommand);
ray('SSH Multiplexing: ' . ($checkProcess->exitCode() === 0 ? 'Active' : 'Not Active'))->color($checkProcess->exitCode() === 0 ? 'green' : 'red');
} else {
ray('SSH Multiplexing: Disabled for SSH command')->orange();
} }
self::addCloudflareProxyCommand($ssh_command, $server); self::addCloudflareProxyCommand($ssh_command, $server);
@@ -160,7 +180,9 @@ class SshMultiplexingHelper
private static function isMultiplexingEnabled(): bool private static function isMultiplexingEnabled(): bool
{ {
return config('constants.ssh.mux_enabled') && !config('coolify.is_windows_docker_desktop'); $isEnabled = config('constants.ssh.mux_enabled') && !config('coolify.is_windows_docker_desktop');
ray('SSH Multiplexing Status:', $isEnabled ? 'ENABLED' : 'DISABLED')->color($isEnabled ? 'green' : 'red');
return $isEnabled;
} }
private static function validateSshKey(string $sshKeyLocation): void private static function validateSshKey(string $sshKeyLocation): void

View File

@@ -43,7 +43,7 @@ class ServerCheckJob implements ShouldBeEncrypted, ShouldQueue
return isDev() ? 1 : 3; return isDev() ? 1 : 3;
} }
public function __construct(public Server $server, public bool $isManualCheck = false) {} public function __construct(public Server $server) {}
public function middleware(): array public function middleware(): array
{ {
@@ -58,9 +58,6 @@ class ServerCheckJob implements ShouldBeEncrypted, ShouldQueue
public function handle() public function handle()
{ {
try { try {
// Enable SSH multiplexing for autonomous checks, disable for manual checks
config()->set('constants.ssh.mux_enabled', !$this->isManualCheck);
$this->applications = $this->server->applications(); $this->applications = $this->server->applications();
$this->databases = $this->server->databases(); $this->databases = $this->server->databases();
$this->services = $this->server->services()->get(); $this->services = $this->server->services()->get();
@@ -96,7 +93,7 @@ class ServerCheckJob implements ShouldBeEncrypted, ShouldQueue
private function serverStatus() private function serverStatus()
{ {
['uptime' => $uptime] = $this->server->validateConnection($this->isManualCheck); ['uptime' => $uptime] = $this->server->validateConnection(false);
if ($uptime) { if ($uptime) {
if ($this->server->unreachable_notification_sent === true) { if ($this->server->unreachable_notification_sent === true) {
$this->server->update(['unreachable_notification_sent' => false]); $this->server->update(['unreachable_notification_sent' => false]);

View File

@@ -966,8 +966,8 @@ $schema://$host {
public function validateConnection($isManualCheck = true) public function validateConnection($isManualCheck = true)
{ {
// Set mux_enabled to true for automatic checks, false for manual checks
config()->set('constants.ssh.mux_enabled', !$isManualCheck); config()->set('constants.ssh.mux_enabled', !$isManualCheck);
ray('Manual Check: ' . ($isManualCheck ? 'true' : 'false'));
$server = Server::find($this->id); $server = Server::find($this->id);
if (! $server) { if (! $server) {