Fix: SSH Multiplexing for Jobs
This commit is contained in:
@@ -26,7 +26,7 @@ class CheckProxy
|
|||||||
if (is_null($proxyType) || $proxyType === 'NONE' || $server->proxy->force_stop) {
|
if (is_null($proxyType) || $proxyType === 'NONE' || $server->proxy->force_stop) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
['uptime' => $uptime, 'error' => $error] = $server->validateConnection();
|
['uptime' => $uptime, 'error' => $error] = $server->validateConnection(false);
|
||||||
if (! $uptime) {
|
if (! $uptime) {
|
||||||
throw new \Exception($error);
|
throw new \Exception($error);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -25,11 +25,12 @@ class SshMultiplexingHelper
|
|||||||
public static function ensureMultiplexedConnection(Server $server)
|
public static function ensureMultiplexedConnection(Server $server)
|
||||||
{
|
{
|
||||||
if (!self::isMultiplexingEnabled()) {
|
if (!self::isMultiplexingEnabled()) {
|
||||||
ray('Multiplexing is disabled');
|
ray('SSH Multiplexing: DISABLED')->red();
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
ray('Ensuring multiplexed connection for server: ' . $server->id);
|
ray('SSH Multiplexing: ENABLED')->green();
|
||||||
|
ray('Ensuring multiplexed connection for server:', $server->id);
|
||||||
|
|
||||||
$sshConfig = self::serverSshConfiguration($server);
|
$sshConfig = self::serverSshConfiguration($server);
|
||||||
$muxSocket = $sshConfig['muxFilename'];
|
$muxSocket = $sshConfig['muxFilename'];
|
||||||
@@ -41,16 +42,17 @@ class SshMultiplexingHelper
|
|||||||
$process = Process::run($checkCommand);
|
$process = Process::run($checkCommand);
|
||||||
|
|
||||||
if ($process->exitCode() !== 0) {
|
if ($process->exitCode() !== 0) {
|
||||||
ray('Existing connection check failed or not found, establishing new connection');
|
ray('SSH Multiplexing: Existing connection check failed or not found')->orange();
|
||||||
|
ray('Establishing new connection');
|
||||||
self::establishNewMultiplexedConnection($server);
|
self::establishNewMultiplexedConnection($server);
|
||||||
} else {
|
} else {
|
||||||
ray('Existing connection is valid');
|
ray('SSH Multiplexing: Existing connection is valid')->green();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public static function establishNewMultiplexedConnection(Server $server)
|
public static function establishNewMultiplexedConnection(Server $server)
|
||||||
{
|
{
|
||||||
ray('Establishing new multiplexed connection for server: ' . $server->id);
|
ray('SSH Multiplexing: Establishing new connection for server:', $server->id);
|
||||||
|
|
||||||
$sshConfig = self::serverSshConfiguration($server);
|
$sshConfig = self::serverSshConfiguration($server);
|
||||||
$sshKeyLocation = $sshConfig['sshKeyLocation'];
|
$sshKeyLocation = $sshConfig['sshKeyLocation'];
|
||||||
@@ -74,11 +76,11 @@ class SshMultiplexingHelper
|
|||||||
$establishProcess = Process::run($establishCommand);
|
$establishProcess = Process::run($establishCommand);
|
||||||
|
|
||||||
if ($establishProcess->exitCode() !== 0) {
|
if ($establishProcess->exitCode() !== 0) {
|
||||||
ray('Failed to establish multiplexed connection', $establishProcess->errorOutput());
|
ray('SSH Multiplexing: Failed to establish connection', $establishProcess->errorOutput())->red();
|
||||||
throw new \RuntimeException('Failed to establish multiplexed connection: ' . $establishProcess->errorOutput());
|
throw new \RuntimeException('Failed to establish multiplexed connection: ' . $establishProcess->errorOutput());
|
||||||
}
|
}
|
||||||
|
|
||||||
ray('Multiplexed connection established successfully');
|
ray('SSH Multiplexing: Connection established successfully')->green();
|
||||||
|
|
||||||
$muxContent = "Multiplexed connection established at " . now()->toDateTimeString();
|
$muxContent = "Multiplexed connection established at " . now()->toDateTimeString();
|
||||||
Storage::disk('ssh-mux')->put(basename($muxSocket), $muxContent);
|
Storage::disk('ssh-mux')->put(basename($muxSocket), $muxContent);
|
||||||
@@ -108,9 +110,18 @@ class SshMultiplexingHelper
|
|||||||
$scp_command = "timeout $timeout scp ";
|
$scp_command = "timeout $timeout scp ";
|
||||||
|
|
||||||
if (self::isMultiplexingEnabled()) {
|
if (self::isMultiplexingEnabled()) {
|
||||||
|
ray('SSH Multiplexing: Enabled for SCP command')->green();
|
||||||
$muxPersistTime = config('constants.ssh.mux_persist_time');
|
$muxPersistTime = config('constants.ssh.mux_persist_time');
|
||||||
$scp_command .= "-o ControlMaster=auto -o ControlPath=$muxSocket -o ControlPersist={$muxPersistTime} ";
|
$scp_command .= "-o ControlMaster=auto -o ControlPath=$muxSocket -o ControlPersist={$muxPersistTime} ";
|
||||||
self::ensureMultiplexedConnection($server);
|
self::ensureMultiplexedConnection($server);
|
||||||
|
|
||||||
|
// Add this line to verify multiplexing is being used
|
||||||
|
ray('SSH Multiplexing: Verifying usage')->blue();
|
||||||
|
$checkCommand = "ssh -O check -o ControlPath=$muxSocket {$server->user}@{$server->ip}";
|
||||||
|
$checkProcess = Process::run($checkCommand);
|
||||||
|
ray('SSH Multiplexing: ' . ($checkProcess->exitCode() === 0 ? 'Active' : 'Not Active'))->color($checkProcess->exitCode() === 0 ? 'green' : 'red');
|
||||||
|
} else {
|
||||||
|
ray('SSH Multiplexing: Disabled for SCP command')->orange();
|
||||||
}
|
}
|
||||||
|
|
||||||
self::addCloudflareProxyCommand($scp_command, $server);
|
self::addCloudflareProxyCommand($scp_command, $server);
|
||||||
@@ -138,9 +149,18 @@ class SshMultiplexingHelper
|
|||||||
$ssh_command = "timeout $timeout ssh ";
|
$ssh_command = "timeout $timeout ssh ";
|
||||||
|
|
||||||
if (self::isMultiplexingEnabled()) {
|
if (self::isMultiplexingEnabled()) {
|
||||||
|
ray('SSH Multiplexing: Enabled for SSH command')->green();
|
||||||
$muxPersistTime = config('constants.ssh.mux_persist_time');
|
$muxPersistTime = config('constants.ssh.mux_persist_time');
|
||||||
$ssh_command .= "-o ControlMaster=auto -o ControlPath=$muxSocket -o ControlPersist={$muxPersistTime} ";
|
$ssh_command .= "-o ControlMaster=auto -o ControlPath=$muxSocket -o ControlPersist={$muxPersistTime} ";
|
||||||
self::ensureMultiplexedConnection($server);
|
self::ensureMultiplexedConnection($server);
|
||||||
|
|
||||||
|
// Add this line to verify multiplexing is being used
|
||||||
|
ray('SSH Multiplexing: Verifying usage')->blue();
|
||||||
|
$checkCommand = "ssh -O check -o ControlPath=$muxSocket {$server->user}@{$server->ip}";
|
||||||
|
$checkProcess = Process::run($checkCommand);
|
||||||
|
ray('SSH Multiplexing: ' . ($checkProcess->exitCode() === 0 ? 'Active' : 'Not Active'))->color($checkProcess->exitCode() === 0 ? 'green' : 'red');
|
||||||
|
} else {
|
||||||
|
ray('SSH Multiplexing: Disabled for SSH command')->orange();
|
||||||
}
|
}
|
||||||
|
|
||||||
self::addCloudflareProxyCommand($ssh_command, $server);
|
self::addCloudflareProxyCommand($ssh_command, $server);
|
||||||
@@ -160,7 +180,9 @@ class SshMultiplexingHelper
|
|||||||
|
|
||||||
private static function isMultiplexingEnabled(): bool
|
private static function isMultiplexingEnabled(): bool
|
||||||
{
|
{
|
||||||
return config('constants.ssh.mux_enabled') && !config('coolify.is_windows_docker_desktop');
|
$isEnabled = config('constants.ssh.mux_enabled') && !config('coolify.is_windows_docker_desktop');
|
||||||
|
ray('SSH Multiplexing Status:', $isEnabled ? 'ENABLED' : 'DISABLED')->color($isEnabled ? 'green' : 'red');
|
||||||
|
return $isEnabled;
|
||||||
}
|
}
|
||||||
|
|
||||||
private static function validateSshKey(string $sshKeyLocation): void
|
private static function validateSshKey(string $sshKeyLocation): void
|
||||||
|
|||||||
@@ -43,7 +43,7 @@ class ServerCheckJob implements ShouldBeEncrypted, ShouldQueue
|
|||||||
return isDev() ? 1 : 3;
|
return isDev() ? 1 : 3;
|
||||||
}
|
}
|
||||||
|
|
||||||
public function __construct(public Server $server, public bool $isManualCheck = false) {}
|
public function __construct(public Server $server) {}
|
||||||
|
|
||||||
public function middleware(): array
|
public function middleware(): array
|
||||||
{
|
{
|
||||||
@@ -58,9 +58,6 @@ class ServerCheckJob implements ShouldBeEncrypted, ShouldQueue
|
|||||||
public function handle()
|
public function handle()
|
||||||
{
|
{
|
||||||
try {
|
try {
|
||||||
// Enable SSH multiplexing for autonomous checks, disable for manual checks
|
|
||||||
config()->set('constants.ssh.mux_enabled', !$this->isManualCheck);
|
|
||||||
|
|
||||||
$this->applications = $this->server->applications();
|
$this->applications = $this->server->applications();
|
||||||
$this->databases = $this->server->databases();
|
$this->databases = $this->server->databases();
|
||||||
$this->services = $this->server->services()->get();
|
$this->services = $this->server->services()->get();
|
||||||
@@ -96,7 +93,7 @@ class ServerCheckJob implements ShouldBeEncrypted, ShouldQueue
|
|||||||
|
|
||||||
private function serverStatus()
|
private function serverStatus()
|
||||||
{
|
{
|
||||||
['uptime' => $uptime] = $this->server->validateConnection($this->isManualCheck);
|
['uptime' => $uptime] = $this->server->validateConnection(false);
|
||||||
if ($uptime) {
|
if ($uptime) {
|
||||||
if ($this->server->unreachable_notification_sent === true) {
|
if ($this->server->unreachable_notification_sent === true) {
|
||||||
$this->server->update(['unreachable_notification_sent' => false]);
|
$this->server->update(['unreachable_notification_sent' => false]);
|
||||||
|
|||||||
@@ -966,8 +966,8 @@ $schema://$host {
|
|||||||
|
|
||||||
public function validateConnection($isManualCheck = true)
|
public function validateConnection($isManualCheck = true)
|
||||||
{
|
{
|
||||||
// Set mux_enabled to true for automatic checks, false for manual checks
|
|
||||||
config()->set('constants.ssh.mux_enabled', !$isManualCheck);
|
config()->set('constants.ssh.mux_enabled', !$isManualCheck);
|
||||||
|
ray('Manual Check: ' . ($isManualCheck ? 'true' : 'false'));
|
||||||
|
|
||||||
$server = Server::find($this->id);
|
$server = Server::find($this->id);
|
||||||
if (! $server) {
|
if (! $server) {
|
||||||
|
|||||||
Reference in New Issue
Block a user