refactor(ssh): enhance error handling in SSH command execution and improve connection validation logging

app/Models/Server.php

@@ -1082,6 +1082,7 @@ $schema://$host {
 
     public function validateConnection(bool $justCheckingNewKey = false)
     {
+        ray('validateConnection', $this->id);
         $this->disableSshMux();
 
         if ($this->skipServer()) {

app/Traits/ExecuteRemoteCommand.php

@@ -88,10 +88,6 @@ trait ExecuteRemoteCommand
     private function executeCommandWithProcess($command, $hidden, $customType, $append, $ignore_errors)
     {
         $remote_command = SshMultiplexingHelper::generateSshCommand($this->server, $command);
-        // Randomly fail the command with a key exchange error for testing
-        // if (random_int(1, 20) === 1) { // 5% chance to fail
-        //     throw new \RuntimeException('SSH key exchange failed: kex_exchange_identification: read: Connection reset by peer');
-        // }
         $process = Process::timeout(3600)->idleTimeout(3600)->start($remote_command, function (string $type, string $output) use ($command, $hidden, $customType, $append) {
             $output = str($output)->trim();
             if ($output->startsWith('╔')) {

app/Traits/SshRetryable.php

@@ -57,9 +57,9 @@ trait SshRetryable
      */
     protected function calculateRetryDelay(int $attempt): int
     {
-        $baseDelay = config('constants.ssh.retry_base_delay', 2);
+        $baseDelay = config('constants.ssh.retry_base_delay');
-        $maxDelay = config('constants.ssh.retry_max_delay', 30);
+        $maxDelay = config('constants.ssh.retry_max_delay');
-        $multiplier = config('constants.ssh.retry_multiplier', 2);
+        $multiplier = config('constants.ssh.retry_multiplier');
 
         $delay = min($baseDelay * pow($multiplier, $attempt), $maxDelay);
 
@@ -76,23 +76,17 @@ trait SshRetryable
      */
     protected function executeWithSshRetry(callable $callback, array $context = [], bool $throwError = true)
     {
-        $maxRetries = config('constants.ssh.max_retries', 3);
+        $maxRetries = config('constants.ssh.max_retries');
         $lastError = null;
         $lastErrorMessage = '';
+        // Randomly fail the command with a key exchange error for testing
+        // if (random_int(1, 10) === 1) { // 10% chance to fail
+        //     ray('SSH key exchange failed: kex_exchange_identification: read: Connection reset by peer');
+        //     throw new \RuntimeException('SSH key exchange failed: kex_exchange_identification: read: Connection reset by peer');
+        // }
         for ($attempt = 0; $attempt < $maxRetries; $attempt++) {
             try {
-                // Execute the callback
+                return $callback();
-                $result = $callback();
-
-                // If we get here, it succeeded
-                if ($attempt > 0) {
-                    Log::info('SSH operation succeeded after retry', array_merge($context, [
-                        'attempt' => $attempt + 1,
-                    ]));
-                }
-
-                return $result;
-
             } catch (\Throwable $e) {
                 $lastError = $e;
                 $lastErrorMessage = $e->getMessage();
@@ -125,6 +119,12 @@ trait SshRetryable
         }
 
         if ($throwError && $lastError) {
+            // If the error message is empty, provide a more meaningful one
+            if (empty($lastErrorMessage) || trim($lastErrorMessage) === '') {
+                $contextInfo = isset($context['server']) ? " to server {$context['server']}" : '';
+                $attemptInfo = $attempt > 1 ? " after {$attempt} attempts" : '';
+                throw new \RuntimeException("SSH connection failed{$contextInfo}{$attemptInfo}", $lastError->getCode());
+            }
             throw $lastError;
         }
 

bootstrap/helpers/remoteProcess.php

@@ -159,11 +159,18 @@ function excludeCertainErrors(string $errorOutput, ?int $exitCode = null)
         'Could not resolve hostname',
     ]);
     $ignored = $ignoredErrors->contains(fn ($error) => Str::contains($errorOutput, $error));
+
+    // Ensure we always have a meaningful error message
+    $errorMessage = trim($errorOutput);
+    if (empty($errorMessage)) {
+        $errorMessage = "SSH command failed with exit code: $exitCode";
+    }
+
     if ($ignored) {
         // TODO: Create new exception and disable in sentry
-        throw new \RuntimeException($errorOutput, $exitCode);
+        throw new \RuntimeException($errorMessage, $exitCode);
     }
-    throw new \RuntimeException($errorOutput, $exitCode);
+    throw new \RuntimeException($errorMessage, $exitCode);
 }
 
 function decode_remote_command_output(?ApplicationDeploymentQueue $application_deployment_queue = null): Collection