diff --git a/app/Models/Server.php b/app/Models/Server.php index 0f92bd390..736a59be4 100644 --- a/app/Models/Server.php +++ b/app/Models/Server.php @@ -1082,6 +1082,7 @@ $schema://$host { public function validateConnection(bool $justCheckingNewKey = false) { + ray('validateConnection', $this->id); $this->disableSshMux(); if ($this->skipServer()) { diff --git a/app/Traits/ExecuteRemoteCommand.php b/app/Traits/ExecuteRemoteCommand.php index 0b770a6e0..398f05bc9 100644 --- a/app/Traits/ExecuteRemoteCommand.php +++ b/app/Traits/ExecuteRemoteCommand.php @@ -88,10 +88,6 @@ trait ExecuteRemoteCommand private function executeCommandWithProcess($command, $hidden, $customType, $append, $ignore_errors) { $remote_command = SshMultiplexingHelper::generateSshCommand($this->server, $command); - // Randomly fail the command with a key exchange error for testing - // if (random_int(1, 20) === 1) { // 5% chance to fail - // throw new \RuntimeException('SSH key exchange failed: kex_exchange_identification: read: Connection reset by peer'); - // } $process = Process::timeout(3600)->idleTimeout(3600)->start($remote_command, function (string $type, string $output) use ($command, $hidden, $customType, $append) { $output = str($output)->trim(); if ($output->startsWith('╔')) { diff --git a/app/Traits/SshRetryable.php b/app/Traits/SshRetryable.php index c2756c2ea..2092dc5f3 100644 --- a/app/Traits/SshRetryable.php +++ b/app/Traits/SshRetryable.php @@ -57,9 +57,9 @@ trait SshRetryable */ protected function calculateRetryDelay(int $attempt): int { - $baseDelay = config('constants.ssh.retry_base_delay', 2); - $maxDelay = config('constants.ssh.retry_max_delay', 30); - $multiplier = config('constants.ssh.retry_multiplier', 2); + $baseDelay = config('constants.ssh.retry_base_delay'); + $maxDelay = config('constants.ssh.retry_max_delay'); + $multiplier = config('constants.ssh.retry_multiplier'); $delay = min($baseDelay * pow($multiplier, $attempt), $maxDelay); @@ -76,23 +76,17 @@ trait SshRetryable */ protected function executeWithSshRetry(callable $callback, array $context = [], bool $throwError = true) { - $maxRetries = config('constants.ssh.max_retries', 3); + $maxRetries = config('constants.ssh.max_retries'); $lastError = null; $lastErrorMessage = ''; + // Randomly fail the command with a key exchange error for testing + // if (random_int(1, 10) === 1) { // 10% chance to fail + // ray('SSH key exchange failed: kex_exchange_identification: read: Connection reset by peer'); + // throw new \RuntimeException('SSH key exchange failed: kex_exchange_identification: read: Connection reset by peer'); + // } for ($attempt = 0; $attempt < $maxRetries; $attempt++) { try { - // Execute the callback - $result = $callback(); - - // If we get here, it succeeded - if ($attempt > 0) { - Log::info('SSH operation succeeded after retry', array_merge($context, [ - 'attempt' => $attempt + 1, - ])); - } - - return $result; - + return $callback(); } catch (\Throwable $e) { $lastError = $e; $lastErrorMessage = $e->getMessage(); @@ -125,6 +119,12 @@ trait SshRetryable } if ($throwError && $lastError) { + // If the error message is empty, provide a more meaningful one + if (empty($lastErrorMessage) || trim($lastErrorMessage) === '') { + $contextInfo = isset($context['server']) ? " to server {$context['server']}" : ''; + $attemptInfo = $attempt > 1 ? " after {$attempt} attempts" : ''; + throw new \RuntimeException("SSH connection failed{$contextInfo}{$attemptInfo}", $lastError->getCode()); + } throw $lastError; } diff --git a/bootstrap/helpers/remoteProcess.php b/bootstrap/helpers/remoteProcess.php index 6efe4a405..b5bdeff49 100644 --- a/bootstrap/helpers/remoteProcess.php +++ b/bootstrap/helpers/remoteProcess.php @@ -159,11 +159,18 @@ function excludeCertainErrors(string $errorOutput, ?int $exitCode = null) 'Could not resolve hostname', ]); $ignored = $ignoredErrors->contains(fn ($error) => Str::contains($errorOutput, $error)); + + // Ensure we always have a meaningful error message + $errorMessage = trim($errorOutput); + if (empty($errorMessage)) { + $errorMessage = "SSH command failed with exit code: $exitCode"; + } + if ($ignored) { // TODO: Create new exception and disable in sentry - throw new \RuntimeException($errorOutput, $exitCode); + throw new \RuntimeException($errorMessage, $exitCode); } - throw new \RuntimeException($errorOutput, $exitCode); + throw new \RuntimeException($errorMessage, $exitCode); } function decode_remote_command_output(?ApplicationDeploymentQueue $application_deployment_queue = null): Collection