dave/coolify
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

fix(ssh): introduce SshRetryHandler and SshRetryable trait for enhanced SSH command retry logic with exponential backoff and error handling

Browse Source
This commit is contained in:
Andras Bacsai
2025-09-07 17:17:35 +02:00
parent b847740924
commit 579cc25898
5 changed files with 420 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files

99
bootstrap/helpers/remoteProcess.php
View File

@@ -60,15 +60,28 @@ function remote_process(
function instant_scp(string $source, string $dest, Server $server, $throwError = true)
{
$scp_command = SshMultiplexingHelper::generateScpCommand($server, $source, $dest);
$process = Process::timeout(config('constants.ssh.command_timeout'))->run($scp_command);
$output = trim($process->output());
$exitCode = $process->exitCode();
if ($exitCode !== 0) {
return $throwError ? excludeCertainErrors($process->errorOutput(), $exitCode) : null;
}
return \App\Helpers\SshRetryHandler::retry(
function () use ($source, $dest, $server) {
$scp_command = SshMultiplexingHelper::generateScpCommand($server, $source, $dest);
$process = Process::timeout(config('constants.ssh.command_timeout'))->run($scp_command);
return $output === 'null' ? null : $output;
$output = trim($process->output());
$exitCode = $process->exitCode();
if ($exitCode !== 0) {
excludeCertainErrors($process->errorOutput(), $exitCode);
}
return $output === 'null' ? null : $output;
},
[
'server' => $server->ip,
'source' => $source,
'dest' => $dest,
'function' => 'instant_scp',
],
$throwError
);
}
function instant_remote_process_with_timeout(Collection|array $command, Server $server, bool $throwError = true, bool $no_sudo = false): ?string
@@ -79,25 +92,30 @@ function instant_remote_process_with_timeout(Collection|array $command, Server $
}
$command_string = implode("\n", $command);
// $start_time = microtime(true);
$sshCommand = SshMultiplexingHelper::generateSshCommand($server, $command_string);
$process = Process::timeout(30)->run($sshCommand);
// $end_time = microtime(true);
return \App\Helpers\SshRetryHandler::retry(
function () use ($server, $command_string) {
$sshCommand = SshMultiplexingHelper::generateSshCommand($server, $command_string);
$process = Process::timeout(30)->run($sshCommand);
// $execution_time = ($end_time - $start_time) * 1000; // Convert to milliseconds
// ray('SSH command execution time:', $execution_time.' ms')->orange();
$output = trim($process->output());
$exitCode = $process->exitCode();
$output = trim($process->output());
$exitCode = $process->exitCode();
if ($exitCode !== 0) {
excludeCertainErrors($process->errorOutput(), $exitCode);
}
if ($exitCode !== 0) {
return $throwError ? excludeCertainErrors($process->errorOutput(), $exitCode) : null;
}
// Sanitize output to ensure valid UTF-8 encoding
$output = $output === 'null' ? null : sanitize_utf8_text($output);
// Sanitize output to ensure valid UTF-8 encoding
$output = $output === 'null' ? null : sanitize_utf8_text($output);
return $output;
return $output;
},
[
'server' => $server->ip,
'command_preview' => substr($command_string, 0, 100),
'function' => 'instant_remote_process_with_timeout',
],
$throwError
);
}
function instant_remote_process(Collection|array $command, Server $server, bool $throwError = true, bool $no_sudo = false): ?string
@@ -108,25 +126,30 @@ function instant_remote_process(Collection|array $command, Server $server, bool
}
$command_string = implode("\n", $command);
// $start_time = microtime(true);
$sshCommand = SshMultiplexingHelper::generateSshCommand($server, $command_string);
$process = Process::timeout(config('constants.ssh.command_timeout'))->run($sshCommand);
// $end_time = microtime(true);
return \App\Helpers\SshRetryHandler::retry(
function () use ($server, $command_string) {
$sshCommand = SshMultiplexingHelper::generateSshCommand($server, $command_string);
$process = Process::timeout(config('constants.ssh.command_timeout'))->run($sshCommand);
// $execution_time = ($end_time - $start_time) * 1000; // Convert to milliseconds
// ray('SSH command execution time:', $execution_time.' ms')->orange();
$output = trim($process->output());
$exitCode = $process->exitCode();
$output = trim($process->output());
$exitCode = $process->exitCode();
if ($exitCode !== 0) {
excludeCertainErrors($process->errorOutput(), $exitCode);
}
if ($exitCode !== 0) {
return $throwError ? excludeCertainErrors($process->errorOutput(), $exitCode) : null;
}
// Sanitize output to ensure valid UTF-8 encoding
$output = $output === 'null' ? null : sanitize_utf8_text($output);
// Sanitize output to ensure valid UTF-8 encoding
$output = $output === 'null' ? null : sanitize_utf8_text($output);
return $output;
return $output;
},
[
'server' => $server->ip,
'command_preview' => substr($command_string, 0, 100),
'function' => 'instant_remote_process',
],
$throwError
);
}
function excludeCertainErrors(string $errorOutput, ?int $exitCode = null)