dave/coolify
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Able to backup Coolify itself

Browse Source
This commit is contained in:
Andras Bacsai
2023-08-11 16:13:53 +02:00
parent b7c9810461
commit 61864970c1
52 changed files with 353 additions and 409 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
app/Http/Controllers/Controller.php
View File

@@ -6,6 +6,7 @@ use App\Models\InstanceSettings;
use App\Models\Project;
use App\Models\S3Storage;
use App\Models\Server;
use App\Models\StandalonePostgresql;
use App\Models\TeamInvitation;
use App\Models\User;
use Illuminate\Foundation\Auth\Access\AuthorizesRequests;
@@ -60,20 +61,16 @@ class Controller extends BaseController
{
if (auth()->user()->isInstanceAdmin()) {
$settings = InstanceSettings::get();
$database = StandalonePostgresql::whereName('coolify-db')->first();
if ($database) {
$backup = $database->scheduledBackups->first();
$s3s = S3Storage::whereTeamId(0)->get();
}
return view('settings.configuration', [
'settings' => $settings
]);
} else {
return redirect()->route('dashboard');
}
}
public function emails()
{
if (auth()->user()->isInstanceAdmin()) {
$settings = InstanceSettings::get();
return view('settings.emails', [
'settings' => $settings
'settings' => $settings,
'database' => $database,
'backup' => $backup ?? null,
's3s' => $s3s ?? [],
]);
} else {
return redirect()->route('dashboard');

12
app/Http/Controllers/DatabaseController.php
View File

@@ -46,7 +46,12 @@ class DatabaseController extends Controller
return redirect()->route('dashboard');
}
$executions = collect($backup->executions)->sortByDesc('created_at');
return view('project.database.backups.executions', ['database' => $database, 'backup' => $backup, 'executions' => $executions]);
return view('project.database.backups.executions', [
'database' => $database,
'backup' => $backup,
'executions' => $executions,
's3s' => auth()->user()->currentTeam()->s3s,
]);
}
public function backups()
@@ -63,6 +68,9 @@ class DatabaseController extends Controller
if (!$database) {
return redirect()->route('dashboard');
}
return view('project.database.backups.all', ['database' => $database]);
return view('project.database.backups.all', [
'database' => $database,
's3s' => auth()->user()->currentTeam()->s3s,
]);
}
}

53
app/Http/Livewire/Project/Database/BackupEdit.php
View File

@@ -7,6 +7,7 @@ use Livewire\Component;
class BackupEdit extends Component
{
public $backup;
public $s3s;
public array $parameters;
protected $rules = [
@@ -14,17 +15,25 @@ class BackupEdit extends Component
'backup.frequency' => 'required|string',
'backup.number_of_backups_locally' => 'required|integer|min:1',
'backup.save_s3' => 'required|boolean',
'backup.s3_storage_id' => 'nullable|integer',
];
protected $validationAttributes = [
'backup.enabled' => 'Enabled',
'backup.frequency' => 'Frequency',
'backup.number_of_backups_locally' => 'Number of Backups Locally',
'backup.save_s3' => 'Save to S3',
'backup.s3_storage_id' => 'S3 Storage',
];
protected $messages = [
'backup.s3_storage_id' => 'Select a S3 Storage',
];
public function mount()
{
$this->parameters = get_route_parameters();
if (is_null($this->backup->s3_storage_id)) {
$this->backup->s3_storage_id = 'default';
}
}
@@ -37,21 +46,43 @@ class BackupEdit extends Component
public function instantSave()
{
$this->backup->save();
$this->backup->refresh();
$this->emit('success', 'Backup updated successfully');
try {
$this->custom_validate();
$this->backup->save();
$this->backup->refresh();
$this->emit('success', 'Backup updated successfully');
} catch (\Exception $e) {
$this->emit('error', $e->getMessage());
}
}
private function custom_validate()
{
// if ($this->backup->save_s3) {
// if (!is_numeric($this->selected_storage_id)) {
// throw new \Exception('Invalid S3 Storage');
// } else {
// $this->backup->s3_storage_id = $this->selected_storage_id;
// }
// }
$isValid = validate_cron_expression($this->backup->frequency);
if (!$isValid) {
throw new \Exception('Invalid Cron / Human expression');
}
$this->validate();
}
public function submit()
{
$isValid = validate_cron_expression($this->backup->frequency);
if (!$isValid) {
$this->emit('error', 'Invalid Cron / Human expression');
return;
ray($this->backup->s3_storage_id);
try {
$this->custom_validate();
$this->backup->save();
$this->backup->refresh();
$this->emit('success', 'Backup updated successfully');
} catch (\Exception $e) {
$this->emit('error', $e->getMessage());
}
$this->validate();
$this->backup->save();
$this->backup->refresh();
$this->emit('success', 'Backup updated successfully');
}
}

5
app/Http/Livewire/Project/Database/CreateScheduledBackup.php
View File

@@ -11,6 +11,8 @@ class CreateScheduledBackup extends Component
public $frequency;
public bool $enabled = true;
public bool $save_s3 = true;
public $s3_storage_id;
public $s3s;
protected $rules = [
'frequency' => 'required|string',
@@ -27,13 +29,14 @@ class CreateScheduledBackup extends Component
$this->validate();
$isValid = validate_cron_expression($this->frequency);
if (!$isValid) {
$this->emit('error', 'Invalid Cron / Human expression');
$this->emit('error', 'Invalid Cron / Human expression.');
return;
}
ScheduledDatabaseBackup::create([
'enabled' => true,
'frequency' => $this->frequency,
'save_s3' => $this->save_s3,
's3_storage_id' => $this->s3_storage_id,
'database_id' => $this->database->id,
'database_type' => $this->database->getMorphClass(),
'team_id' => auth()->user()->currentTeam()->id,

75
app/Http/Livewire/Settings/Backup.php Normal file
View File

@@ -0,0 +1,75 @@
<?php
namespace App\Http\Livewire\Settings;
use App\Models\InstanceSettings;
use App\Models\S3Storage;
use App\Models\ScheduledDatabaseBackup;
use App\Models\Server;
use App\Models\StandalonePostgresql;
use Livewire\Component;
class Backup extends Component
{
public InstanceSettings $settings;
public $s3s;
public StandalonePostgresql|null $database = null;
public ScheduledDatabaseBackup|null $backup = null;
protected $rules = [
'database.uuid' => 'required',
'database.name' => 'required',
'database.description' => 'nullable',
'database.postgres_user' => 'required',
'database.postgres_password' => 'required',
];
protected $validationAttributes = [
'database.uuid' => 'uuid',
'database.name' => 'name',
'database.description' => 'description',
'database.postgres_user' => 'postgres user',
'database.postgres_password' => 'postgres password',
];
public function add_coolify_database()
{
ray('add_coolify_database');
$server = Server::find(0);
$out = instant_remote_process(['docker inspect coolify-db'], $server);
$envs = format_docker_envs_to_json($out);
$postgres_password = $envs['POSTGRES_PASSWORD'];
$postgres_user = $envs['POSTGRES_USER'];
$postgres_db = $envs['POSTGRES_DB'];
$this->database = StandalonePostgresql::create([
'id' => 0,
'name' => 'coolify-db',
'description' => 'Coolify database',
'postgres_user' => $postgres_user,
'postgres_password' => $postgres_password,
'postgres_db' => $postgres_db,
'status' => 'running',
'destination_type' => 'App\Models\StandaloneDocker',
'destination_id' => 0,
]);
$this->backup = ScheduledDatabaseBackup::create([
'id' => 0,
'enabled' => true,
'save_s3' => false,
'frequency' => '0 0 * * *',
'database_id' => $this->database->id,
'database_type' => 'App\Models\StandalonePostgresql',
'team_id' => auth()->user()->currentTeam()->id,
]);
$this->database->refresh();
$this->backup->refresh();
ray($this->backup);
$this->s3s = S3Storage::whereTeamId(0)->get();
}
public function submit()
{
$this->emit('success', 'Backup updated successfully');
}
}

4
app/Jobs/ApplicationDeploymentJob.php
View File

@@ -176,10 +176,10 @@ class ApplicationDeploymentJob implements ShouldQueue
{
$this->execute_remote_command(
[
"echo -n 'Pulling latest version of the builder image (ghcr.io/coollabsio/coolify-builder).'",
"echo -n 'Pulling latest version of the builder image (ghcr.io/coollabsio/coolify-helper).'",
],
[
"docker run --pull=always -d --name {$this->deployment_uuid} --rm -v /var/run/docker.sock:/var/run/docker.sock ghcr.io/coollabsio/coolify-builder",
"docker run --pull=always -d --name {$this->deployment_uuid} --rm -v /var/run/docker.sock:/var/run/docker.sock ghcr.io/coollabsio/coolify-helper",
"hidden" => true,
],
[

80
app/Jobs/DatabaseBackupJob.php
View File

@@ -30,22 +30,25 @@ class DatabaseBackupJob implements ShouldQueue
public StandalonePostgresql $database;
public string $database_status;
public string|null $container_name = null;
public ScheduledDatabaseBackupExecution|null $backup_log = null;
public string $backup_status;
public string|null $backup_filename = null;
public string|null $backup_location = null;
public string $backup_dir;
public string $backup_file;
public int $size = 0;
public string|null $backup_output = null;
public S3Storage $s3;
public S3Storage|null $s3 = null;
public function __construct($backup)
{
$this->backup = $backup;
$this->team = Team::find($backup->team_id);
$this->database = $this->backup->database->first();
$this->database = $this->backup->database;
$this->database_type = $this->database->type();
$this->server = $this->database->destination->server;
$this->database_status = $this->database->status;
$this->s3 = $this->team->s3;
$this->s3 = $this->backup->s3;
}
public function middleware(): array
@@ -58,16 +61,23 @@ class DatabaseBackupJob implements ShouldQueue
return $this->backup->id;
}
public function handle()
public function handle(): void
{
if ($this->database_status !== 'running') {
ray('database not running');
return;
}
$this->backup_filename = backup_dir() . "/{$this->database->uuid}/dumpall-" . Carbon::now()->timestamp . ".sql";
$this->container_name = $this->database->uuid;
if ($this->database->name === 'coolify-db') {
$this->container_name = "coolify-db";
}
$this->backup_dir = backup_dir() . "/" . $this->container_name;
$this->backup_file = "/dumpall-" . Carbon::now()->timestamp . ".sql";
$this->backup_location = $this->backup_dir . $this->backup_file;
$this->backup_log = ScheduledDatabaseBackupExecution::create([
'filename' => $this->backup_filename,
'filename' => $this->backup_location,
'scheduled_database_backup_id' => $this->backup->id,
]);
if ($this->database_type === 'standalone-postgresql') {
@@ -76,18 +86,17 @@ class DatabaseBackupJob implements ShouldQueue
$this->calculate_size();
$this->remove_old_backups();
if ($this->backup->save_s3) {
// $this->upload_to_s3();
$this->upload_to_s3();
}
$this->save_backup_logs();
// TODO: Notify user
}
private function backup_standalone_postgresql()
private function backup_standalone_postgresql(): void
{
try {
$commands[] = "mkdir -p " . backup_dir();
$commands[] = "mkdir -p " . backup_dir() . "/{$this->database->uuid}";
$commands[] = "docker exec {$this->database->uuid} pg_dumpall -U {$this->database->postgres_user} > $this->backup_filename";
$commands[] = "mkdir -p " . $this->backup_dir;
$commands[] = "docker exec $this->container_name pg_dumpall -U {$this->database->postgres_user} > $this->backup_location";
$this->backup_output = instant_remote_process($commands, $this->server);
@@ -97,14 +106,14 @@ class DatabaseBackupJob implements ShouldQueue
$this->backup_output = null;
}
ray('Backup done for ' . $this->database->uuid . ' at ' . $this->server->name . ':' . $this->backup_filename);
ray('Backup done for ' . $this->container_name . ' at ' . $this->server->name . ':' . $this->backup_location);
$this->backup_status = 'success';
$this->team->notify(new BackupSuccess($this->backup, $this->database));
} catch (Throwable $th) {
$this->backup_status = 'failed';
$this->add_to_backup_output($th->getMessage());
ray('Backup failed for ' . $this->database->uuid . ' at ' . $this->server->name . ':' . $this->backup_filename . '\n\nError:' . $th->getMessage());
ray('Backup failed for ' . $this->container_name . ' at ' . $this->server->name . ':' . $this->backup_location . '\n\nError:' . $th->getMessage());
$this->team->notify(new BackupFailed($this->backup, $this->database, $this->backup_output));
} finally {
$this->backup_log->update([
@@ -113,7 +122,7 @@ class DatabaseBackupJob implements ShouldQueue
}
}
private function add_to_backup_output($output)
private function add_to_backup_output($output): void
{
if ($this->backup_output) {
$this->backup_output = $this->backup_output . "\n" . $output;
@@ -122,12 +131,12 @@ class DatabaseBackupJob implements ShouldQueue
}
}
private function calculate_size()
private function calculate_size(): void
{
$this->size = instant_remote_process(["du -b $this->backup_filename | cut -f1"], $this->server);
$this->size = instant_remote_process(["du -b $this->backup_location | cut -f1"], $this->server);
}
private function remove_old_backups()
private function remove_old_backups(): void
{
if ($this->backup->number_of_backups_locally === 0) {
$deletable = $this->backup->executions()->where('status', 'success');
@@ -140,17 +149,7 @@ class DatabaseBackupJob implements ShouldQueue
}
}
private function save_backup_logs()
{
$this->backup_log->update([
'status' => $this->backup_status,
'message' => $this->backup_output,
'size' => $this->size,
]);
}
private function upload_to_s3()
private function upload_to_s3(): void
{
try {
if (is_null($this->s3)) {
@@ -161,16 +160,29 @@ class DatabaseBackupJob implements ShouldQueue
// $region = $this->s3->region;
$bucket = $this->s3->bucket;
$endpoint = $this->s3->endpoint;
$backup_dir = backup_dir() . "/{$this->database->uuid}";
$base_command = "docker run -t --network {$this->database->destination->network} -v {$this->backup_filename}:{$this->backup_filename}:ro --rm --entrypoint=/bin/sh minio/mc -c \"mc config host add temporary {$endpoint} $key $secret && mc cp $this->backup_filename temporary/$bucket/$backup_dir/ \"";
instant_remote_process([$base_command], $this->server);
$commands[] = "docker run --pull=always -d --network {$this->database->destination->network} --name backup-of-{$this->backup->uuid} --rm -v $this->backup_location:$this->backup_location:ro ghcr.io/coollabsio/coolify-helper";
$commands[] = "docker exec backup-of-{$this->backup->uuid} mc config host add temporary {$endpoint} $key $secret";
$commands[] = "docker exec backup-of-{$this->backup->uuid} mc cp $this->backup_location temporary/$bucket{$this->backup_dir}/";
instant_remote_process($commands, $this->server);
$this->add_to_backup_output('Uploaded to S3.');
ray('Uploaded to S3. ' . $this->backup_location . ' to s3://' . $bucket . $this->backup_dir);
} catch (\Throwable $th) {
$this->add_to_backup_output($th->getMessage());
ray($th->getMessage());
} finally {
$command = "docker rm -f backup-of-{$this->backup->uuid}";
instant_remote_process([$command], $this->server);
}
}
private function save_backup_logs(): void
{
$this->backup_log->update([
'status' => $this->backup_status,
'message' => $this->backup_output,
'size' => $this->size,
]);
}
}

20
app/Models/Application.php
View File

@@ -8,25 +8,7 @@ use Spatie\Activitylog\Models\Activity;
class Application extends BaseModel
{
protected $fillable = [
'name',
'repository_project_id',
'project_id',
'description',
'git_repository',
'git_branch',
'git_full_url',
'build_pack',
'environment_id',
'destination_id',
'destination_type',
'source_id',
'source_type',
'ports_mappings',
'ports_exposes',
'publish_directory',
'private_key_id'
];
protected $guarded = [];
protected static function booted()
{

16
app/Models/Database.php
View File

@@ -1,16 +0,0 @@
<?php
namespace App\Models;
class Database extends BaseModel
{
public function environment()
{
return $this->belongsTo(Environment::class);
}
public function destination()
{
return $this->morphTo();
}
}

5
app/Models/ScheduledDatabaseBackup.php
View File

@@ -25,4 +25,9 @@ class ScheduledDatabaseBackup extends BaseModel
{
return $this->hasMany(ScheduledDatabaseBackupExecution::class);
}
public function s3()
{
return $this->belongsTo(S3Storage::class, 's3_storage_id');
}
}

17
app/Models/Service.php
View File

@@ -1,17 +0,0 @@
<?php
namespace App\Models;
class Service extends BaseModel
{
public function environment()
{
return $this->belongsTo(Environment::class);
}
public function destination()
{
return $this->morphTo();
}
}

6
app/Models/StandaloneDocker.php
View File

@@ -4,11 +4,7 @@ namespace App\Models;
class StandaloneDocker extends BaseModel
{
protected $fillable = [
'name',
'network',
'server_id',
];
protected $guarded = [];
public function applications()
{

4
app/Models/Team.php
View File

@@ -84,8 +84,8 @@ class Team extends Model implements SendsDiscord, SendsEmail
return $sources;
}
public function s3()
public function s3s()
{
return $this->hasOne(S3Storage::class);
return $this->hasMany(S3Storage::class);
}
}

1
app/Notifications/Database/BackupSuccess.php
View File

@@ -36,7 +36,6 @@ class BackupSuccess extends Notification implements ShouldQueue
if ($isDiscordEnabled && $isSubscribedToDiscordEvent) {
$channels[] = DiscordChannel::class;
}
ray($channels);
return $channels;
}