feat(deployment): implement cancellation checks during deployment process to enhance user control and prevent unnecessary execution

This commit is contained in:
Andras Bacsai
2025-09-16 13:40:51 +02:00
parent f9ed02a0b7
commit 9e8fb36bc8
3 changed files with 84 additions and 6 deletions

View File

@@ -250,6 +250,14 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
public function handle(): void public function handle(): void
{ {
// Check if deployment was cancelled before we even started
$this->application_deployment_queue->refresh();
if ($this->application_deployment_queue->status === ApplicationDeploymentStatus::CANCELLED_BY_USER->value) {
$this->application_deployment_queue->addLogEntry('Deployment was cancelled before starting.');
return;
}
$this->application_deployment_queue->update([ $this->application_deployment_queue->update([
'status' => ApplicationDeploymentStatus::IN_PROGRESS->value, 'status' => ApplicationDeploymentStatus::IN_PROGRESS->value,
'horizon_job_worker' => gethostname(), 'horizon_job_worker' => gethostname(),
@@ -1146,6 +1154,7 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
private function rolling_update() private function rolling_update()
{ {
$this->checkForCancellation();
if ($this->server->isSwarm()) { if ($this->server->isSwarm()) {
$this->application_deployment_queue->addLogEntry('Rolling update started.'); $this->application_deployment_queue->addLogEntry('Rolling update started.');
$this->execute_remote_command( $this->execute_remote_command(
@@ -1342,6 +1351,7 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
private function prepare_builder_image() private function prepare_builder_image()
{ {
$this->checkForCancellation();
$settings = instanceSettings(); $settings = instanceSettings();
$helperImage = config('constants.coolify.helper_image'); $helperImage = config('constants.coolify.helper_image');
$helperImage = "{$helperImage}:{$settings->helper_version}"; $helperImage = "{$helperImage}:{$settings->helper_version}";
@@ -1813,6 +1823,7 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
private function generate_compose_file() private function generate_compose_file()
{ {
$this->checkForCancellation();
$this->create_workdir(); $this->create_workdir();
$ports = $this->application->main_port(); $ports = $this->application->main_port();
$persistent_storages = $this->generate_local_persistent_volumes(); $persistent_storages = $this->generate_local_persistent_volumes();
@@ -2546,8 +2557,23 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf");
throw new RuntimeException('Post-deployment command: Could not find a valid container. Is the container name correct?'); throw new RuntimeException('Post-deployment command: Could not find a valid container. Is the container name correct?');
} }
/**
* Check if the deployment was cancelled and abort if it was
*/
private function checkForCancellation(): void
{
$this->application_deployment_queue->refresh();
if ($this->application_deployment_queue->status === ApplicationDeploymentStatus::CANCELLED_BY_USER->value) {
$this->application_deployment_queue->addLogEntry('Deployment cancelled by user, stopping execution.');
throw new \RuntimeException('Deployment cancelled by user', 69420);
}
}
private function next(string $status) private function next(string $status)
{ {
// Refresh to get latest status
$this->application_deployment_queue->refresh();
// Never allow changing status from FAILED or CANCELLED_BY_USER to anything else // Never allow changing status from FAILED or CANCELLED_BY_USER to anything else
if ($this->application_deployment_queue->status === ApplicationDeploymentStatus::FAILED->value) { if ($this->application_deployment_queue->status === ApplicationDeploymentStatus::FAILED->value) {
$this->application->environment->project->team?->notify(new DeploymentFailed($this->application, $this->deployment_uuid, $this->preview)); $this->application->environment->project->team?->notify(new DeploymentFailed($this->application, $this->deployment_uuid, $this->preview));
@@ -2555,7 +2581,9 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf");
return; return;
} }
if ($this->application_deployment_queue->status === ApplicationDeploymentStatus::CANCELLED_BY_USER->value) { if ($this->application_deployment_queue->status === ApplicationDeploymentStatus::CANCELLED_BY_USER->value) {
return; // Job was cancelled, stop execution
$this->application_deployment_queue->addLogEntry('Deployment cancelled by user, stopping execution.');
throw new \RuntimeException('Deployment cancelled by user', 69420);
} }
$this->application_deployment_queue->update([ $this->application_deployment_queue->update([

View File

@@ -52,15 +52,24 @@ class DeploymentNavbar extends Component
public function cancel() public function cancel()
{ {
$kill_command = "docker rm -f {$this->application_deployment_queue->deployment_uuid}"; $deployment_uuid = $this->application_deployment_queue->deployment_uuid;
$kill_command = "docker rm -f {$deployment_uuid}";
$build_server_id = $this->application_deployment_queue->build_server_id ?? $this->application->destination->server_id; $build_server_id = $this->application_deployment_queue->build_server_id ?? $this->application->destination->server_id;
$server_id = $this->application_deployment_queue->server_id ?? $this->application->destination->server_id; $server_id = $this->application_deployment_queue->server_id ?? $this->application->destination->server_id;
// First, mark the deployment as cancelled to prevent further processing
$this->application_deployment_queue->update([
'status' => ApplicationDeploymentStatus::CANCELLED_BY_USER->value,
]);
try { try {
if ($this->application->settings->is_build_server_enabled) { if ($this->application->settings->is_build_server_enabled) {
$server = Server::ownedByCurrentTeam()->find($build_server_id); $server = Server::ownedByCurrentTeam()->find($build_server_id);
} else { } else {
$server = Server::ownedByCurrentTeam()->find($server_id); $server = Server::ownedByCurrentTeam()->find($server_id);
} }
// Add cancellation log entry
if ($this->application_deployment_queue->logs) { if ($this->application_deployment_queue->logs) {
$previous_logs = json_decode($this->application_deployment_queue->logs, associative: true, flags: JSON_THROW_ON_ERROR); $previous_logs = json_decode($this->application_deployment_queue->logs, associative: true, flags: JSON_THROW_ON_ERROR);
@@ -77,13 +86,35 @@ class DeploymentNavbar extends Component
'logs' => json_encode($previous_logs, flags: JSON_THROW_ON_ERROR), 'logs' => json_encode($previous_logs, flags: JSON_THROW_ON_ERROR),
]); ]);
} }
instant_remote_process([$kill_command], $server);
// Try to stop the helper container if it exists
// Check if container exists first
$checkCommand = "docker ps -a --filter name={$deployment_uuid} --format '{{.Names}}'";
$containerExists = instant_remote_process([$checkCommand], $server);
if ($containerExists && str($containerExists)->trim()->isNotEmpty()) {
// Container exists, kill it
instant_remote_process([$kill_command], $server);
} else {
// Container hasn't started yet
$this->application_deployment_queue->addLogEntry('Helper container not yet started. Deployment will be cancelled when job checks status.');
}
// Also try to kill any running process if we have a process ID
if ($this->application_deployment_queue->current_process_id) {
try {
$processKillCommand = "kill -9 {$this->application_deployment_queue->current_process_id}";
instant_remote_process([$processKillCommand], $server);
} catch (\Throwable $e) {
// Process might already be gone, that's ok
}
}
} catch (\Throwable $e) { } catch (\Throwable $e) {
// Still mark as cancelled even if cleanup fails
return handleError($e, $this); return handleError($e, $this);
} finally { } finally {
$this->application_deployment_queue->update([ $this->application_deployment_queue->update([
'current_process_id' => null, 'current_process_id' => null,
'status' => ApplicationDeploymentStatus::CANCELLED_BY_USER->value,
]); ]);
next_after_cancel($server); next_after_cancel($server);
} }

View File

@@ -46,6 +46,14 @@ trait ExecuteRemoteCommand
} }
} }
// Check for cancellation before executing commands
if (isset($this->application_deployment_queue)) {
$this->application_deployment_queue->refresh();
if ($this->application_deployment_queue->status === \App\Enums\ApplicationDeploymentStatus::CANCELLED_BY_USER->value) {
throw new \RuntimeException('Deployment cancelled by user', 69420);
}
}
$maxRetries = config('constants.ssh.max_retries'); $maxRetries = config('constants.ssh.max_retries');
$attempt = 0; $attempt = 0;
$lastError = null; $lastError = null;
@@ -73,6 +81,12 @@ trait ExecuteRemoteCommand
// Add log entry for the retry // Add log entry for the retry
if (isset($this->application_deployment_queue)) { if (isset($this->application_deployment_queue)) {
$this->addRetryLogEntry($attempt, $maxRetries, $delay, $errorMessage); $this->addRetryLogEntry($attempt, $maxRetries, $delay, $errorMessage);
// Check for cancellation during retry wait
$this->application_deployment_queue->refresh();
if ($this->application_deployment_queue->status === \App\Enums\ApplicationDeploymentStatus::CANCELLED_BY_USER->value) {
throw new \RuntimeException('Deployment cancelled by user during retry', 69420);
}
} }
sleep($delay); sleep($delay);
@@ -85,6 +99,11 @@ trait ExecuteRemoteCommand
// If we exhausted all retries and still failed // If we exhausted all retries and still failed
if (! $commandExecuted && $lastError) { if (! $commandExecuted && $lastError) {
// Now we can set the status to FAILED since all retries have been exhausted
if (isset($this->application_deployment_queue)) {
$this->application_deployment_queue->status = ApplicationDeploymentStatus::FAILED->value;
$this->application_deployment_queue->save();
}
throw $lastError; throw $lastError;
} }
}); });
@@ -160,8 +179,8 @@ trait ExecuteRemoteCommand
$process_result = $process->wait(); $process_result = $process->wait();
if ($process_result->exitCode() !== 0) { if ($process_result->exitCode() !== 0) {
if (! $ignore_errors) { if (! $ignore_errors) {
$this->application_deployment_queue->status = ApplicationDeploymentStatus::FAILED->value; // Don't immediately set to FAILED - let the retry logic handle it
$this->application_deployment_queue->save(); // This prevents premature status changes during retryable SSH errors
throw new \RuntimeException($process_result->errorOutput()); throw new \RuntimeException($process_result->errorOutput());
} }
} }