dave/coolify
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Refactor + package updates + improve local backups

Browse Source
This commit is contained in:
Andras Bacsai
2023-08-10 15:52:54 +02:00
parent d2a4dbf283
commit e17f1935d2
30 changed files with 757 additions and 366 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
app/Console/Kernel.php
View File

@@ -2,9 +2,9 @@
namespace App\Console;
use App\Jobs\BackupDatabaseJob;
use App\Jobs\CheckResaleLicenseJob;
use App\Jobs\CheckResaleLicenseKeys;
use App\Jobs\DatabaseBackupJob;
use App\Jobs\DockerCleanupJob;
use App\Jobs\InstanceApplicationsStatusJob;
use App\Jobs\InstanceAutoUpdateJob;
@@ -50,7 +50,7 @@ class Kernel extends ConsoleKernel
if (isset(VALID_CRON_STRINGS[$scheduled_backup->frequency])) {
$scheduled_backup->frequency = VALID_CRON_STRINGS[$scheduled_backup->frequency];
}
$schedule->job(new BackupDatabaseJob(
$schedule->job(new DatabaseBackupJob(
backup: $scheduled_backup
))->cron($scheduled_backup->frequency);
}

25
app/Http/Controllers/DatabaseController.php
View File

@@ -26,6 +26,29 @@ class DatabaseController extends Controller
return view('project.database.configuration', ['database' => $database]);
}
public function backup_logs()
{
$backup_uuid = request()->route('backup_uuid');
$project = session('currentTeam')->load(['projects'])->projects->where('uuid', request()->route('project_uuid'))->first();
if (!$project) {
return redirect()->route('dashboard');
}
$environment = $project->load(['environments'])->environments->where('name', request()->route('environment_name'))->first()->load(['applications']);
if (!$environment) {
return redirect()->route('dashboard');
}
$database = $environment->databases->where('uuid', request()->route('database_uuid'))->first();
if (!$database) {
return redirect()->route('dashboard');
}
$backup = $database->scheduledBackups->where('uuid', $backup_uuid)->first();
if (!$backup) {
return redirect()->route('dashboard');
}
$backup_executions = collect($backup->executions)->sortByDesc('created_at');
return view('project.database.backups.logs', ['database' => $database, 'backup' => $backup, 'backup_executions' => $backup_executions]);
}
public function backups()
{
$project = session('currentTeam')->load(['projects'])->projects->where('uuid', request()->route('project_uuid'))->first();
@@ -40,6 +63,6 @@ class DatabaseController extends Controller
if (!$database) {
return redirect()->route('dashboard');
}
return view('project.database.backups', ['database' => $database]);
return view('project.database.backups.all', ['database' => $database]);
}
}

41
app/Http/Livewire/Project/Database/BackupEdit.php Normal file
View File

@@ -0,0 +1,41 @@
<?php
namespace App\Http\Livewire\Project\Database;
use Livewire\Component;
class BackupEdit extends Component
{
public $backup;
protected $rules = [
'backup.enabled' => 'required|boolean',
'backup.frequency' => 'required|string',
'backup.number_of_backups_locally' => 'required|integer|min:1',
];
protected $validationAttributes = [
'backup.enabled' => 'Enabled',
'backup.frequency' => 'Frequency',
'backup.number_of_backups_locally' => 'Number of Backups Locally',
];
public function instantSave()
{
$this->backup->save();
$this->backup->refresh();
$this->emit('success', 'Backup updated successfully');
}
public function submit()
{
$isValid = validate_cron_expression($this->backup->frequency);
if (!$isValid) {
$this->emit('error', 'Invalid Cron / Human expression');
return;
}
$this->validate();
$this->backup->save();
$this->backup->refresh();
$this->emit('success', 'Backup updated successfully');
}
}

24
app/Http/Livewire/Project/Database/BackupExecution.php Normal file
View File

@@ -0,0 +1,24 @@
<?php
namespace App\Http\Livewire\Project\Database;
use App\Models\ScheduledDatabaseBackupExecution;
use Livewire\Component;
class BackupExecution extends Component
{
public ScheduledDatabaseBackupExecution $execution;
public function download()
{
}
public function delete(): void
{
delete_backup_locally($this->execution->filename, $this->execution->scheduledDatabaseBackup->database->destination->server);
$this->execution->delete();
$this->emit('success', 'Backup execution deleted successfully.');
$this->emit('refreshBackupExecutions');
}
}

17
app/Http/Livewire/Project/Database/BackupExecutions.php Normal file
View File

@@ -0,0 +1,17 @@
<?php
namespace App\Http\Livewire\Project\Database;
use Livewire\Component;
class BackupExecutions extends Component
{
public $backup;
public $executions;
protected $listeners = ['refreshBackupExecutions'];
public function refreshBackupExecutions(): void
{
$this->executions = collect($this->backup->executions)->sortByDesc('created_at');
}
}

14
app/Http/Livewire/Project/Database/CreateScheduledBackup.php
View File

@@ -4,24 +4,20 @@ namespace App\Http\Livewire\Project\Database;
use App\Models\ScheduledDatabaseBackup;
use Livewire\Component;
use Poliander\Cron\CronExpression;
class CreateScheduledBackup extends Component
{
public $database;
public $frequency;
public bool $enabled = true;
public bool $keep_locally = true;
public bool $save_s3 = true;
protected $rules = [
'frequency' => 'required|string',
'keep_locally' => 'required|boolean',
'save_s3' => 'required|boolean',
];
protected $validationAttributes = [
'frequency' => 'Backup Frequency',
'keep_locally' => 'Keep Locally',
'save_s3' => 'Save to S3',
];
@@ -29,13 +25,7 @@ class CreateScheduledBackup extends Component
{
try {
$this->validate();
$expression = new CronExpression($this->frequency);
$isValid = $expression->isValid();
if (isset(VALID_CRON_STRINGS[$this->frequency])) {
$isValid = true;
}
$isValid = validate_cron_expression($this->frequency);
if (!$isValid) {
$this->emit('error', 'Invalid Cron / Human expression');
return;
@@ -43,7 +33,6 @@ class CreateScheduledBackup extends Component
ScheduledDatabaseBackup::create([
'enabled' => true,
'frequency' => $this->frequency,
'keep_locally' => $this->keep_locally,
'save_s3' => $this->save_s3,
'database_id' => $this->database->id,
'database_type' => $this->database->getMorphClass(),
@@ -54,7 +43,6 @@ class CreateScheduledBackup extends Component
general_error_handler($e, $this);
} finally {
$this->frequency = '';
$this->keep_locally = true;
$this->save_s3 = true;
}
}

15
app/Http/Livewire/Project/Database/ScheduledBackups.php
View File

@@ -7,9 +7,22 @@ use Livewire\Component;
class ScheduledBackups extends Component
{
public $database;
public $parameters;
protected $listeners = ['refreshScheduledBackups'];
public function refreshScheduledBackups()
public function mount(): void
{
$this->parameters = get_route_parameters();
}
public function delete($scheduled_backup_id): void
{
$this->database->scheduledBackups->find($scheduled_backup_id)->delete();
$this->emit('success', 'Scheduled backup deleted successfully.');
$this->refreshScheduledBackups();
}
public function refreshScheduledBackups(): void
{
ray('refreshScheduledBackups');
$this->database->refresh();

78
app/Jobs/BackupDatabaseJob.php
View File

@@ -1,78 +0,0 @@
<?php
namespace App\Jobs;
use App\Models\ScheduledDatabaseBackup;
use App\Models\Server;
use App\Models\StandalonePostgresql;
use App\Models\Team;
use Carbon\Carbon;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Queue\SerializesModels;
use Throwable;
class BackupDatabaseJob implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public Team|null $team = null;
public Server $server;
public ScheduledDatabaseBackup|null $backup;
public string $database_type;
public StandalonePostgresql $database;
public string $status;
public function __construct($backup)
{
$this->backup = $backup;
$this->team = Team::find($backup->team_id);
$this->database = $this->backup->database->first();
$this->database_type = $this->database->type();
$this->server = $this->database->destination->server;
$this->status = $this->database->status;
}
public function middleware(): array
{
return [new WithoutOverlapping($this->backup->id)];
}
public function uniqueId(): int
{
return $this->backup->id;
}
public function handle()
{
if ($this->status !== 'running') {
ray('database not running');
return;
}
if ($this->database_type === 'standalone-postgresql') {
$this->backup_standalone_postgresql();
}
}
private function backup_standalone_postgresql()
{
try {
$backup_filename = backup_dir() . "/{$this->database->uuid}/dumpall-" . Carbon::now()->timestamp . ".sql";
$commands[] = "mkdir -p " . backup_dir();
$commands[] = "mkdir -p " . backup_dir() . "/{$this->database->uuid}";
$commands[] = "docker exec {$this->database->uuid} pg_dumpall -U {$this->database->postgres_user} > $backup_filename";
instant_remote_process($commands, $this->server);
ray('Backup done for ' . $this->database->uuid . ' at ' . $this->server->name . ':' . $backup_filename);
if (!$this->backup->keep_locally) {
$commands[] = "rm -rf $backup_filename";
instant_remote_process($commands, $this->server);
}
} catch (Throwable $th) {
ray($th);
//throw $th;
}
}
}

142
app/Jobs/DatabaseBackupJob.php Normal file
View File

@@ -0,0 +1,142 @@
<?php
namespace App\Jobs;
use App\Models\ScheduledDatabaseBackup;
use App\Models\ScheduledDatabaseBackupExecution;
use App\Models\Server;
use App\Models\StandalonePostgresql;
use App\Models\Team;
use Carbon\Carbon;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Queue\SerializesModels;
use Throwable;
class DatabaseBackupJob implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public Team|null $team = null;
public Server $server;
public ScheduledDatabaseBackup|null $backup;
public string $database_type;
public StandalonePostgresql $database;
public string $database_status;
public ScheduledDatabaseBackupExecution|null $backup_log = null;
public string $backup_status;
public string|null $backup_filename = null;
public int $size = 0;
public string|null $backup_output = null;
public function __construct($backup)
{
$this->backup = $backup;
$this->team = Team::find($backup->team_id);
$this->database = $this->backup->database->first();
$this->database_type = $this->database->type();
$this->server = $this->database->destination->server;
$this->database_status = $this->database->status;
}
public function middleware(): array
{
return [new WithoutOverlapping($this->backup->id)];
}
public function uniqueId(): int
{
return $this->backup->id;
}
public function handle()
{
if ($this->database_status !== 'running') {
ray('database not running');
return;
}
$this->backup_filename = backup_dir() . "/{$this->database->uuid}/dumpall-" . Carbon::now()->timestamp . ".sql";
$this->backup_log = ScheduledDatabaseBackupExecution::create([
'filename' => $this->backup_filename,
'scheduled_database_backup_id' => $this->backup->id,
]);
if ($this->database_type === 'standalone-postgresql') {
$this->backup_standalone_postgresql();
}
$this->calculate_size();
$this->remove_old_backups();
$this->save_backup_logs();
}
private function backup_standalone_postgresql()
{
try {
$commands[] = "mkdir -p " . backup_dir();
$commands[] = "mkdir -p " . backup_dir() . "/{$this->database->uuid}";
$commands[] = "docker exec {$this->database->uuid} pg_dumpall -U {$this->database->postgres_user} > $this->backup_filename";
$this->backup_output = instant_remote_process($commands, $this->server);
$this->backup_output = trim($this->backup_output);
if ($this->backup_output === '') {
$this->backup_output = null;
}
ray('Backup done for ' . $this->database->uuid . ' at ' . $this->server->name . ':' . $this->backup_filename);
$this->backup_status = 'success';
} catch (Throwable $th) {
$this->backup_status = 'failed';
$this->add_to_backup_output($th->getMessage());
ray('Backup failed for ' . $this->database->uuid . ' at ' . $this->server->name . ':' . $this->backup_filename . '\n\nError:' . $th->getMessage());
} finally {
$this->backup_log->update([
'status' => $this->backup_status,
]);
}
}
private function add_to_backup_output($output)
{
if ($this->backup_output) {
$this->backup_output = $this->backup_output . "\n" . $output;
} else {
$this->backup_output = $output;
}
}
private function calculate_size()
{
$this->size = instant_remote_process(["du -b $this->backup_filename | cut -f1"], $this->server);
}
private function remove_old_backups()
{
if ($this->backup->number_of_backups_locally === 0) {
$deletable = $this->backup->executions()->where('status', 'success');
} else {
$deletable = $this->backup->executions()->where('status', 'success')->orderByDesc('created_at')->skip($this->backup->number_of_backups_locally);
}
ray($deletable->get());
foreach ($deletable->get() as $execution) {
delete_backup_locally($execution->filename, $this->server);
$execution->delete();
}
}
private function save_backup_logs()
{
$this->backup_log->update([
'status' => $this->backup_status,
'message' => $this->backup_output,
'size' => $this->size,
]);
}
}

16
app/Models/ScheduledDatabaseBackup.php
View File

@@ -3,12 +3,26 @@
namespace App\Models;
use Illuminate\Database\Eloquent\Relations\HasMany;
use Illuminate\Database\Eloquent\Relations\HasOne;
use Illuminate\Database\Eloquent\Relations\MorphTo;
class ScheduledDatabaseBackup extends BaseModel
{
protected $guarded = [];
public function database()
public function database(): MorphTo
{
return $this->morphTo();
}
public function latest_log(): HasOne
{
return $this->hasOne(ScheduledDatabaseBackupExecution::class)->latest();
}
public function executions(): HasMany
{
return $this->hasMany(ScheduledDatabaseBackupExecution::class);
}
}

15
app/Models/ScheduledDatabaseBackupExecution.php Normal file
View File

@@ -0,0 +1,15 @@
<?php
namespace App\Models;
use Illuminate\Database\Eloquent\Relations\BelongsTo;
class ScheduledDatabaseBackupExecution extends BaseModel
{
protected $guarded = [];
public function scheduledDatabaseBackup(): BelongsTo
{
return $this->belongsTo(ScheduledDatabaseBackup::class);
}
}

5
app/Models/StandalonePostgresql.php
View File

@@ -64,11 +64,6 @@ class StandalonePostgresql extends BaseModel
return $this->morphTo();
}
public function scheduled_database_backups()
{
return $this->morphMany(ScheduledDatabaseBackup::class, 'database');
}
public function environment_variables(): HasMany
{
return $this->hasMany(EnvironmentVariable::class);