dave/coolify
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Merge pull request #4818 from peaklabs-dev/feat-backup-retention

Browse Source 
Feat: Improve backup retention (for database backups)
This commit is contained in:
Andras Bacsai
2025-01-14 09:10:33 +01:00
committed by GitHub
parent 57c998d35d 8709a7749b
commit b01ea8b46d
7 changed files with 425 additions and 176 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
app/Jobs/DatabaseBackupJob.php
View File

@@ -299,7 +299,6 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
throw new \Exception('Unsupported database type');
}
$size = $this->calculate_size();
$this->remove_old_backups();
if ($this->backup->save_s3) {
$this->upload_to_s3();
}
@@ -323,6 +322,9 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
$this->team?->notify(new BackupFailed($this->backup, $this->database, $this->backup_output, $database));
}
}
if ($this->backup_log && $this->backup_log->status === 'success') {
removeOldBackups($this->backup);
}
} catch (\Throwable $e) {
throw $e;
} finally {
@@ -457,19 +459,6 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
return instant_remote_process(["du -b $this->backup_location | cut -f1"], $this->server, false);
}
private function remove_old_backups(): void
{
if ($this->backup->number_of_backups_locally === 0) {
$deletable = $this->backup->executions()->where('status', 'success');
} else {
$deletable = $this->backup->executions()->where('status', 'success')->skip($this->backup->number_of_backups_locally - 1);
}
foreach ($deletable->get() as $execution) {
delete_backup_locally($execution->filename, $this->server);
$execution->delete();
}
}
private function upload_to_s3(): void
{
try {

116
app/Livewire/Project/Database/BackupEdit.php
View File

@@ -43,8 +43,23 @@ class BackupEdit extends Component
#[Validate(['string'])]
public string $timezone = '';
#[Validate(['required', 'integer', 'min:1'])]
public int $numberOfBackupsLocally = 1;
#[Validate(['required', 'integer'])]
public int $databaseBackupRetentionAmountLocally = 0;
#[Validate(['required', 'integer'])]
public ?int $databaseBackupRetentionDaysLocally = 0;
#[Validate(['required', 'numeric', 'min:0'])]
public ?float $databaseBackupRetentionMaxStorageLocally = 0;
#[Validate(['required', 'integer'])]
public ?int $databaseBackupRetentionAmountS3 = 0;
#[Validate(['required', 'integer'])]
public ?int $databaseBackupRetentionDaysS3 = 0;
#[Validate(['required', 'numeric', 'min:0'])]
public ?float $databaseBackupRetentionMaxStorageS3 = 0;
#[Validate(['required', 'boolean'])]
public bool $saveS3 = false;
@@ -73,7 +88,12 @@ class BackupEdit extends Component
if ($toModel) {
$this->backup->enabled = $this->backupEnabled;
$this->backup->frequency = $this->frequency;
$this->backup->number_of_backups_locally = $this->numberOfBackupsLocally;
$this->backup->database_backup_retention_amount_locally = $this->databaseBackupRetentionAmountLocally;
$this->backup->database_backup_retention_days_locally = $this->databaseBackupRetentionDaysLocally;
$this->backup->database_backup_retention_max_storage_locally = $this->databaseBackupRetentionMaxStorageLocally;
$this->backup->database_backup_retention_amount_s3 = $this->databaseBackupRetentionAmountS3;
$this->backup->database_backup_retention_days_s3 = $this->databaseBackupRetentionDaysS3;
$this->backup->database_backup_retention_max_storage_s3 = $this->databaseBackupRetentionMaxStorageS3;
$this->backup->save_s3 = $this->saveS3;
$this->backup->s3_storage_id = $this->s3StorageId;
$this->backup->databases_to_backup = $this->databasesToBackup;
@@ -84,7 +104,12 @@ class BackupEdit extends Component
$this->backupEnabled = $this->backup->enabled;
$this->frequency = $this->backup->frequency;
$this->timezone = data_get($this->backup->server(), 'settings.server_timezone', 'Instance timezone');
$this->numberOfBackupsLocally = $this->backup->number_of_backups_locally;
$this->databaseBackupRetentionAmountLocally = $this->backup->database_backup_retention_amount_locally;
$this->databaseBackupRetentionDaysLocally = $this->backup->database_backup_retention_days_locally;
$this->databaseBackupRetentionMaxStorageLocally = $this->backup->database_backup_retention_max_storage_locally;
$this->databaseBackupRetentionAmountS3 = $this->backup->database_backup_retention_amount_s3;
$this->databaseBackupRetentionDaysS3 = $this->backup->database_backup_retention_days_s3;
$this->databaseBackupRetentionMaxStorageS3 = $this->backup->database_backup_retention_max_storage_s3;
$this->saveS3 = $this->backup->save_s3;
$this->s3StorageId = $this->backup->s3_storage_id;
$this->databasesToBackup = $this->backup->databases_to_backup;
@@ -103,11 +128,29 @@ class BackupEdit extends Component
}
try {
if ($this->delete_associated_backups_locally) {
$this->deleteAssociatedBackupsLocally();
$server = null;
if ($this->backup->database instanceof \App\Models\ServiceDatabase) {
$server = $this->backup->database->service->destination->server;
} elseif ($this->backup->database->destination && $this->backup->database->destination->server) {
$server = $this->backup->database->destination->server;
}
if ($this->delete_associated_backups_s3) {
$this->deleteAssociatedBackupsS3();
$filenames = $this->backup->executions()
->whereNotNull('filename')
->where('filename', '!=', '')
->where('scheduled_database_backup_id', $this->backup->id)
->pluck('filename')
->filter()
->all();
if (! empty($filenames)) {
if ($this->delete_associated_backups_locally && $server) {
deleteBackupsLocally($filenames, $server);
}
if ($this->delete_associated_backups_s3 && $this->backup->s3) {
deleteBackupsS3($filenames, $this->backup->s3);
}
}
$this->backup->delete();
@@ -123,7 +166,9 @@ class BackupEdit extends Component
} else {
return redirect()->route('project.database.backup.index', $this->parameters);
}
} catch (\Throwable $e) {
} catch (\Exception $e) {
$this->dispatch('error', 'Failed to delete backup: '.$e->getMessage());
return handleError($e, $this);
}
}
@@ -160,63 +205,12 @@ class BackupEdit extends Component
}
}
private function deleteAssociatedBackupsLocally()
{
$executions = $this->backup->executions;
$backupFolder = null;
foreach ($executions as $execution) {
if ($this->backup->database->getMorphClass() === \App\Models\ServiceDatabase::class) {
$server = $this->backup->database->service->destination->server;
} else {
$server = $this->backup->database->destination->server;
}
if (! $backupFolder) {
$backupFolder = dirname($execution->filename);
}
delete_backup_locally($execution->filename, $server);
$execution->delete();
}
if (str($backupFolder)->isNotEmpty()) {
$this->deleteEmptyBackupFolder($backupFolder, $server);
}
}
private function deleteAssociatedBackupsS3()
{
// Add function to delete backups from S3
}
private function deleteAssociatedBackupsSftp()
{
// Add function to delete backups from SFTP
}
private function deleteEmptyBackupFolder($folderPath, $server)
{
$checkEmpty = instant_remote_process(["[ -z \"$(ls -A '$folderPath')\" ] && echo 'empty' || echo 'not empty'"], $server);
if (trim($checkEmpty) === 'empty') {
instant_remote_process(["rmdir '$folderPath'"], $server);
$parentFolder = dirname($folderPath);
$checkParentEmpty = instant_remote_process(["[ -z \"$(ls -A '$parentFolder')\" ] && echo 'empty' || echo 'not empty'"], $server);
if (trim($checkParentEmpty) === 'empty') {
instant_remote_process(["rmdir '$parentFolder'"], $server);
}
}
}
public function render()
{
return view('livewire.project.database.backup-edit', [
'checkboxes' => [
['id' => 'delete_associated_backups_locally', 'label' => __('database.delete_backups_locally')],
// ['id' => 'delete_associated_backups_s3', 'label' => 'All backups associated with this backup job from this database will be permanently deleted from the selected S3 Storage.']
['id' => 'delete_associated_backups_s3', 'label' => 'All backups associated with this backup job for this database will be permanently deleted from the selected S3 Storage.'],
// ['id' => 'delete_associated_backups_sftp', 'label' => 'All backups associated with this backup job from this database will be permanently deleted from the selected SFTP Storage.']
],
]);

36
app/Livewire/Project/Database/BackupExecutions.php
View File

@@ -18,9 +18,9 @@ class BackupExecutions extends Component
public $setDeletableBackup;
public $delete_backup_s3 = true;
public $delete_backup_s3 = false;
public $delete_backup_sftp = true;
public $delete_backup_sftp = false;
public function getListeners()
{
@@ -57,23 +57,25 @@ class BackupExecutions extends Component
return;
}
if ($execution->scheduledDatabaseBackup->database->getMorphClass() === \App\Models\ServiceDatabase::class) {
delete_backup_locally($execution->filename, $execution->scheduledDatabaseBackup->database->service->destination->server);
} else {
delete_backup_locally($execution->filename, $execution->scheduledDatabaseBackup->database->destination->server);
}
$server = $execution->scheduledDatabaseBackup->database->getMorphClass() === \App\Models\ServiceDatabase::class
? $execution->scheduledDatabaseBackup->database->service->destination->server
: $execution->scheduledDatabaseBackup->database->destination->server;
if ($this->delete_backup_s3) {
// Add logic to delete from S3
}
try {
if ($execution->filename) {
deleteBackupsLocally($execution->filename, $server);
if ($this->delete_backup_sftp) {
// Add logic to delete from SFTP
}
if ($this->delete_backup_s3 && $execution->scheduledDatabaseBackup->s3) {
deleteBackupsS3($execution->filename, $execution->scheduledDatabaseBackup->s3);
}
}
$execution->delete();
$this->dispatch('success', 'Backup deleted.');
$this->refreshBackupExecutions();
$execution->delete();
$this->dispatch('success', 'Backup deleted.');
$this->refreshBackupExecutions();
} catch (\Exception $e) {
$this->dispatch('error', 'Failed to delete backup: '.$e->getMessage());
}
}
public function download_file($exeuctionId)
@@ -143,7 +145,7 @@ class BackupExecutions extends Component
return view('livewire.project.database.backup-executions', [
'checkboxes' => [
['id' => 'delete_backup_s3', 'label' => 'Delete the selected backup permanently form S3 Storage'],
['id' => 'delete_backup_sftp', 'label' => 'Delete the selected backup permanently form SFTP Storage'],
// ['id' => 'delete_backup_sftp', 'label' => 'Delete the selected backup permanently form SFTP Storage'],
],
]);
}