dave/coolify
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

feat(database-backup): enhance error handling and output management in DatabaseBackupJob

Browse Source
This commit is contained in:
Andras Bacsai
2025-09-10 16:13:08 +02:00
parent 1c08d32b85
commit 2c8f5415f1
1 changed files with 47 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

62
app/Jobs/DatabaseBackupJob.php
View File

@@ -54,6 +54,10 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
public ?string $backup_output = null; public ?string $backup_output = null;
public ?string $error_output = null;
public bool $s3_uploaded = false;
public ?string $postgres_password = null; public ?string $postgres_password = null;
public ?string $mongo_root_username = null; public ?string $mongo_root_username = null;
@@ -355,7 +359,6 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
// If local backup is disabled, delete the local file immediately after S3 upload // If local backup is disabled, delete the local file immediately after S3 upload
if ($this->backup->disable_local_backup) { if ($this->backup->disable_local_backup) {
deleteBackupsLocally($this->backup_location, $this->server); deleteBackupsLocally($this->backup_location, $this->server);
$this->add_to_backup_output('Local backup file deleted after S3 upload (disable_local_backup enabled).');
} }
} }
@@ -367,15 +370,34 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
'size' => $size, 'size' => $size,
]); ]);
} catch (\Throwable $e) { } catch (\Throwable $e) {
if ($this->backup_log) { // Check if backup actually failed or if it's just a post-backup issue
$this->backup_log->update([ $actualBackupFailed = ! $this->s3_uploaded && $this->backup->save_s3;
'status' => 'failed',
'message' => $this->backup_output, if ($actualBackupFailed || $size === 0) {
'size' => $size, // Real backup failure
'filename' => null, if ($this->backup_log) {
]); $this->backup_log->update([
'status' => 'failed',
'message' => $this->error_output ?? $this->backup_output ?? $e->getMessage(),
'size' => $size,
'filename' => null,
]);
}
$this->team?->notify(new BackupFailed($this->backup, $this->database, $this->error_output ?? $this->backup_output ?? $e->getMessage(), $database));
} else {
// Backup succeeded but post-processing failed (cleanup, notification, etc.)
if ($this->backup_log) {
$this->backup_log->update([
'status' => 'success',
'message' => $this->backup_output ? $this->backup_output."\nWarning: Post-backup cleanup encountered an issue: ".$e->getMessage() : 'Warning: '.$e->getMessage(),
'size' => $size,
]);
}
// Send success notification since the backup itself succeeded
$this->team->notify(new BackupSuccess($this->backup, $this->database, $database));
// Log the post-backup issue
ray('Post-backup operation failed but backup was successful: '.$e->getMessage());
} }
$this->team?->notify(new BackupFailed($this->backup, $this->database, $this->backup_output, $database));
} }
} }
if ($this->backup_log && $this->backup_log->status === 'success') { if ($this->backup_log && $this->backup_log->status === 'success') {
@@ -446,7 +468,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
$this->backup_output = null; $this->backup_output = null;
} }
} catch (\Throwable $e) { } catch (\Throwable $e) {
$this->add_to_backup_output($e->getMessage()); $this->add_to_error_output($e->getMessage());
throw $e; throw $e;
} }
} }
@@ -472,7 +494,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
$this->backup_output = null; $this->backup_output = null;
} }
} catch (\Throwable $e) { } catch (\Throwable $e) {
$this->add_to_backup_output($e->getMessage()); $this->add_to_error_output($e->getMessage());
throw $e; throw $e;
} }
} }
@@ -492,7 +514,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
$this->backup_output = null; $this->backup_output = null;
} }
} catch (\Throwable $e) { } catch (\Throwable $e) {
$this->add_to_backup_output($e->getMessage()); $this->add_to_error_output($e->getMessage());
throw $e; throw $e;
} }
} }
@@ -512,7 +534,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
$this->backup_output = null; $this->backup_output = null;
} }
} catch (\Throwable $e) { } catch (\Throwable $e) {
$this->add_to_backup_output($e->getMessage()); $this->add_to_error_output($e->getMessage());
throw $e; throw $e;
} }
} }
@@ -526,6 +548,15 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
} }
} }
private function add_to_error_output($output): void
{
if ($this->error_output) {
$this->error_output = $this->error_output."\n".$output;
} else {
$this->error_output = $output;
}
}
private function calculate_size() private function calculate_size()
{ {
return instant_remote_process(["du -b $this->backup_location | cut -f1"], $this->server, false); return instant_remote_process(["du -b $this->backup_location | cut -f1"], $this->server, false);
@@ -571,9 +602,10 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
$commands[] = "docker exec backup-of-{$this->backup->uuid} mc cp $this->backup_location temporary/$bucket{$this->backup_dir}/"; $commands[] = "docker exec backup-of-{$this->backup->uuid} mc cp $this->backup_location temporary/$bucket{$this->backup_dir}/";
instant_remote_process($commands, $this->server); instant_remote_process($commands, $this->server);
$this->add_to_backup_output('Uploaded to S3.'); $this->s3_uploaded = true;
} catch (\Throwable $e) { } catch (\Throwable $e) {
$this->add_to_backup_output($e->getMessage()); $this->s3_uploaded = false;
$this->add_to_error_output($e->getMessage());
throw $e; throw $e;
} finally { } finally {
$command = "docker rm -f backup-of-{$this->backup->uuid}"; $command = "docker rm -f backup-of-{$this->backup->uuid}";