dave/coolify
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

rector: arrrrr

Browse Source
This commit is contained in:
Andras Bacsai
2025-01-07 14:52:08 +01:00
parent c702ebff6d
commit 16c0cd10d8
349 changed files with 4204 additions and 3712 deletions
Download Patch File Download Diff File Expand all files Collapse all files

202
app/Jobs/DatabaseBackupJob.php
View File

@@ -16,6 +16,7 @@ use App\Models\Team;
use App\Notifications\Database\BackupFailed;
use App\Notifications\Database\BackupSuccess;
use Carbon\Carbon;
use Exception;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
use Illuminate\Contracts\Queue\ShouldQueue;
@@ -23,6 +24,7 @@ use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Str;
use Throwable;
class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
{
@@ -56,7 +58,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
public ?S3Storage $s3 = null;
public function __construct(public ScheduledDatabaseBackup $backup)
public function __construct(public ScheduledDatabaseBackup $scheduledDatabaseBackup)
{
$this->onQueue('high');
}
@@ -66,26 +68,26 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
try {
$databasesToBackup = null;
$this->team = Team::find($this->backup->team_id);
$this->team = Team::query()->find($this->scheduledDatabaseBackup->team_id);
if (! $this->team) {
$this->backup->delete();
$this->scheduledDatabaseBackup->delete();
return;
}
if (data_get($this->backup, 'database_type') === \App\Models\ServiceDatabase::class) {
$this->database = data_get($this->backup, 'database');
if (data_get($this->scheduledDatabaseBackup, 'database_type') === ServiceDatabase::class) {
$this->database = data_get($this->scheduledDatabaseBackup, 'database');
$this->server = $this->database->service->server;
$this->s3 = $this->backup->s3;
$this->s3 = $this->scheduledDatabaseBackup->s3;
} else {
$this->database = data_get($this->backup, 'database');
$this->database = data_get($this->scheduledDatabaseBackup, 'database');
$this->server = $this->database->destination->server;
$this->s3 = $this->backup->s3;
$this->s3 = $this->scheduledDatabaseBackup->s3;
}
if (is_null($this->server)) {
throw new \Exception('Server not found?!');
throw new Exception('Server not found?!');
}
if (is_null($this->database)) {
throw new \Exception('Database not found?!');
throw new Exception('Database not found?!');
}
BackupCreated::dispatch($this->team->id);
@@ -94,7 +96,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
if (! $status->startsWith('running') && $this->database->id !== 0) {
return;
}
if (data_get($this->backup, 'database_type') === \App\Models\ServiceDatabase::class) {
if (data_get($this->scheduledDatabaseBackup, 'database_type') === ServiceDatabase::class) {
$databaseType = $this->database->databaseType();
$serviceUuid = $this->database->service->uuid;
$serviceName = str($this->database->service->name)->slug();
@@ -108,21 +110,13 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
$user = $envs->filter(function ($env) {
return str($env)->startsWith('POSTGRES_USER=');
})->first();
if ($user) {
$this->database->postgres_user = str($user)->after('POSTGRES_USER=')->value();
} else {
$this->database->postgres_user = 'postgres';
}
$this->database->postgres_user = $user ? str($user)->after('POSTGRES_USER=')->value() : 'postgres';
$db = $envs->filter(function ($env) {
return str($env)->startsWith('POSTGRES_DB=');
})->first();
if ($db) {
$databasesToBackup = str($db)->after('POSTGRES_DB=')->value();
} else {
$databasesToBackup = $this->database->postgres_user;
}
$databasesToBackup = $db ? str($db)->after('POSTGRES_DB=')->value() : $this->database->postgres_user;
$this->postgres_password = $envs->filter(function ($env) {
return str($env)->startsWith('POSTGRES_PASSWORD=');
})->first();
@@ -150,7 +144,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
if ($db) {
$databasesToBackup = str($db)->after('MYSQL_DATABASE=')->value();
} else {
throw new \Exception('MYSQL_DATABASE not found');
throw new Exception('MYSQL_DATABASE not found');
}
} elseif (str($databaseType)->contains('mariadb')) {
$this->container_name = "{$this->database->name}-$serviceUuid";
@@ -186,7 +180,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
if ($db) {
$databasesToBackup = str($db)->after('MYSQL_DATABASE=')->value();
} else {
throw new \Exception('MARIADB_DATABASE or MYSQL_DATABASE not found');
throw new Exception('MARIADB_DATABASE or MYSQL_DATABASE not found');
}
}
}
@@ -195,7 +189,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
$this->container_name = $this->database->uuid;
$this->directory_name = $databaseName.'-'.$this->container_name;
$databaseType = $this->database->type();
$databasesToBackup = data_get($this->backup, 'databases_to_backup');
$databasesToBackup = data_get($this->scheduledDatabaseBackup, 'databases_to_backup');
}
if (blank($databasesToBackup)) {
if (str($databaseType)->contains('postgres')) {
@@ -209,26 +203,24 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
} else {
return;
}
} elseif (str($databaseType)->contains('postgres')) {
// Format: db1,db2,db3
$databasesToBackup = explode(',', $databasesToBackup);
$databasesToBackup = array_map('trim', $databasesToBackup);
} elseif (str($databaseType)->contains('mongodb')) {
// Format: db1:collection1,collection2|db2:collection3,collection4
$databasesToBackup = explode('|', $databasesToBackup);
$databasesToBackup = array_map('trim', $databasesToBackup);
} elseif (str($databaseType)->contains('mysql')) {
// Format: db1,db2,db3
$databasesToBackup = explode(',', $databasesToBackup);
$databasesToBackup = array_map('trim', $databasesToBackup);
} elseif (str($databaseType)->contains('mariadb')) {
// Format: db1,db2,db3
$databasesToBackup = explode(',', $databasesToBackup);
$databasesToBackup = array_map('trim', $databasesToBackup);
} else {
if (str($databaseType)->contains('postgres')) {
// Format: db1,db2,db3
$databasesToBackup = explode(',', $databasesToBackup);
$databasesToBackup = array_map('trim', $databasesToBackup);
} elseif (str($databaseType)->contains('mongodb')) {
// Format: db1:collection1,collection2|db2:collection3,collection4
$databasesToBackup = explode('|', $databasesToBackup);
$databasesToBackup = array_map('trim', $databasesToBackup);
} elseif (str($databaseType)->contains('mysql')) {
// Format: db1,db2,db3
$databasesToBackup = explode(',', $databasesToBackup);
$databasesToBackup = array_map('trim', $databasesToBackup);
} elseif (str($databaseType)->contains('mariadb')) {
// Format: db1,db2,db3
$databasesToBackup = explode(',', $databasesToBackup);
$databasesToBackup = array_map('trim', $databasesToBackup);
} else {
return;
}
return;
}
$this->backup_dir = backup_dir().'/databases/'.str($this->team->name)->slug().'-'.$this->team->id.'/'.$this->directory_name;
if ($this->database->name === 'coolify-db') {
@@ -237,82 +229,80 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
$ip = Str::slug($this->server->ip);
$this->backup_dir = backup_dir().'/coolify'."/coolify-db-$ip";
}
foreach ($databasesToBackup as $database) {
foreach ($databasesToBackup as $databaseToBackup) {
$size = 0;
try {
if (str($databaseType)->contains('postgres')) {
$this->backup_file = "/pg-dump-$database-".Carbon::now()->timestamp.'.dmp';
if ($this->backup->dump_all) {
$this->backup_file = "/pg-dump-{$databaseToBackup}-".Carbon::now()->timestamp.'.dmp';
if ($this->scheduledDatabaseBackup->dump_all) {
$this->backup_file = '/pg-dump-all-'.Carbon::now()->timestamp.'.gz';
}
$this->backup_location = $this->backup_dir.$this->backup_file;
$this->backup_log = ScheduledDatabaseBackupExecution::create([
'database_name' => $database,
$this->backup_log = ScheduledDatabaseBackupExecution::query()->create([
'database_name' => $databaseToBackup,
'filename' => $this->backup_location,
'scheduled_database_backup_id' => $this->backup->id,
'scheduled_database_backup_id' => $this->scheduledDatabaseBackup->id,
]);
$this->backup_standalone_postgresql($database);
$this->backup_standalone_postgresql($databaseToBackup);
} elseif (str($databaseType)->contains('mongodb')) {
if ($database === '*') {
$database = 'all';
if ($databaseToBackup === '*') {
$databaseToBackup = 'all';
$databaseName = 'all';
} elseif (str($databaseToBackup)->contains(':')) {
$databaseName = str($databaseToBackup)->before(':');
} else {
if (str($database)->contains(':')) {
$databaseName = str($database)->before(':');
} else {
$databaseName = $database;
}
$databaseName = $databaseToBackup;
}
$this->backup_file = "/mongo-dump-$databaseName-".Carbon::now()->timestamp.'.tar.gz';
$this->backup_location = $this->backup_dir.$this->backup_file;
$this->backup_log = ScheduledDatabaseBackupExecution::create([
$this->backup_log = ScheduledDatabaseBackupExecution::query()->create([
'database_name' => $databaseName,
'filename' => $this->backup_location,
'scheduled_database_backup_id' => $this->backup->id,
'scheduled_database_backup_id' => $this->scheduledDatabaseBackup->id,
]);
$this->backup_standalone_mongodb($database);
$this->backup_standalone_mongodb($databaseToBackup);
} elseif (str($databaseType)->contains('mysql')) {
$this->backup_file = "/mysql-dump-$database-".Carbon::now()->timestamp.'.dmp';
if ($this->backup->dump_all) {
$this->backup_file = "/mysql-dump-{$databaseToBackup}-".Carbon::now()->timestamp.'.dmp';
if ($this->scheduledDatabaseBackup->dump_all) {
$this->backup_file = '/mysql-dump-all-'.Carbon::now()->timestamp.'.gz';
}
$this->backup_location = $this->backup_dir.$this->backup_file;
$this->backup_log = ScheduledDatabaseBackupExecution::create([
'database_name' => $database,
$this->backup_log = ScheduledDatabaseBackupExecution::query()->create([
'database_name' => $databaseToBackup,
'filename' => $this->backup_location,
'scheduled_database_backup_id' => $this->backup->id,
'scheduled_database_backup_id' => $this->scheduledDatabaseBackup->id,
]);
$this->backup_standalone_mysql($database);
$this->backup_standalone_mysql($databaseToBackup);
} elseif (str($databaseType)->contains('mariadb')) {
$this->backup_file = "/mariadb-dump-$database-".Carbon::now()->timestamp.'.dmp';
if ($this->backup->dump_all) {
$this->backup_file = "/mariadb-dump-{$databaseToBackup}-".Carbon::now()->timestamp.'.dmp';
if ($this->scheduledDatabaseBackup->dump_all) {
$this->backup_file = '/mariadb-dump-all-'.Carbon::now()->timestamp.'.gz';
}
$this->backup_location = $this->backup_dir.$this->backup_file;
$this->backup_log = ScheduledDatabaseBackupExecution::create([
'database_name' => $database,
$this->backup_log = ScheduledDatabaseBackupExecution::query()->create([
'database_name' => $databaseToBackup,
'filename' => $this->backup_location,
'scheduled_database_backup_id' => $this->backup->id,
'scheduled_database_backup_id' => $this->scheduledDatabaseBackup->id,
]);
$this->backup_standalone_mariadb($database);
$this->backup_standalone_mariadb($databaseToBackup);
} else {
throw new \Exception('Unsupported database type');
throw new Exception('Unsupported database type');
}
$size = $this->calculate_size();
$this->remove_old_backups();
if ($this->backup->save_s3) {
if ($this->scheduledDatabaseBackup->save_s3) {
$this->upload_to_s3();
}
$this->team->notify(new BackupSuccess($this->backup, $this->database, $database));
$this->team->notify(new BackupSuccess($this->scheduledDatabaseBackup, $this->database, $databaseToBackup));
$this->backup_log->update([
'status' => 'success',
'message' => $this->backup_output,
'size' => $size,
]);
} catch (\Throwable $e) {
if ($this->backup_log) {
} catch (Throwable $e) {
if ($this->backup_log instanceof ScheduledDatabaseBackupExecution) {
$this->backup_log->update([
'status' => 'failed',
'message' => $this->backup_output,
@@ -320,13 +310,13 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
'filename' => null,
]);
}
$this->team?->notify(new BackupFailed($this->backup, $this->database, $this->backup_output, $database));
$this->team?->notify(new BackupFailed($this->scheduledDatabaseBackup, $this->database, $this->backup_output, $databaseToBackup));
}
}
} catch (\Throwable $e) {
} catch (Throwable $e) {
throw $e;
} finally {
if ($this->team) {
if ($this->team instanceof Team) {
BackupCreated::dispatch($this->team->id);
}
}
@@ -358,12 +348,10 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
} else {
$commands[] = "docker exec $this->container_name mongodump --authenticationDatabase=admin --uri=$url --db $databaseName --gzip --archive > $this->backup_location";
}
} elseif (str($this->database->image)->startsWith('mongo:4')) {
$commands[] = "docker exec $this->container_name mongodump --uri=$url --gzip --excludeCollection ".$collectionsToExclude->implode(' --excludeCollection ')." --archive > $this->backup_location";
} else {
if (str($this->database->image)->startsWith('mongo:4')) {
$commands[] = "docker exec $this->container_name mongodump --uri=$url --gzip --excludeCollection ".$collectionsToExclude->implode(' --excludeCollection ')." --archive > $this->backup_location";
} else {
$commands[] = "docker exec $this->container_name mongodump --authenticationDatabase=admin --uri=$url --db $databaseName --gzip --excludeCollection ".$collectionsToExclude->implode(' --excludeCollection ')." --archive > $this->backup_location";
}
$commands[] = "docker exec $this->container_name mongodump --authenticationDatabase=admin --uri=$url --db $databaseName --gzip --excludeCollection ".$collectionsToExclude->implode(' --excludeCollection ')." --archive > $this->backup_location";
}
}
$this->backup_output = instant_remote_process($commands, $this->server);
@@ -371,7 +359,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
if ($this->backup_output === '') {
$this->backup_output = null;
}
} catch (\Throwable $e) {
} catch (Throwable $e) {
$this->add_to_backup_output($e->getMessage());
throw $e;
}
@@ -385,7 +373,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
if ($this->postgres_password) {
$backupCommand .= " -e PGPASSWORD=$this->postgres_password";
}
if ($this->backup->dump_all) {
if ($this->scheduledDatabaseBackup->dump_all) {
$backupCommand .= " $this->container_name pg_dumpall --username {$this->database->postgres_user} | gzip > $this->backup_location";
} else {
$backupCommand .= " $this->container_name pg_dump --format=custom --no-acl --no-owner --username {$this->database->postgres_user} $database > $this->backup_location";
@@ -397,7 +385,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
if ($this->backup_output === '') {
$this->backup_output = null;
}
} catch (\Throwable $e) {
} catch (Throwable $e) {
$this->add_to_backup_output($e->getMessage());
throw $e;
}
@@ -407,7 +395,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
{
try {
$commands[] = 'mkdir -p '.$this->backup_dir;
if ($this->backup->dump_all) {
if ($this->scheduledDatabaseBackup->dump_all) {
$commands[] = "docker exec $this->container_name mysqldump -u root -p{$this->database->mysql_root_password} --all-databases --single-transaction --quick --lock-tables=false --compress | gzip > $this->backup_location";
} else {
$commands[] = "docker exec $this->container_name mysqldump -u root -p{$this->database->mysql_root_password} $database > $this->backup_location";
@@ -417,7 +405,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
if ($this->backup_output === '') {
$this->backup_output = null;
}
} catch (\Throwable $e) {
} catch (Throwable $e) {
$this->add_to_backup_output($e->getMessage());
throw $e;
}
@@ -427,7 +415,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
{
try {
$commands[] = 'mkdir -p '.$this->backup_dir;
if ($this->backup->dump_all) {
if ($this->scheduledDatabaseBackup->dump_all) {
$commands[] = "docker exec $this->container_name mariadb-dump -u root -p{$this->database->mariadb_root_password} --all-databases --single-transaction --quick --lock-tables=false --compress > $this->backup_location";
} else {
$commands[] = "docker exec $this->container_name mariadb-dump -u root -p{$this->database->mariadb_root_password} $database > $this->backup_location";
@@ -437,7 +425,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
if ($this->backup_output === '') {
$this->backup_output = null;
}
} catch (\Throwable $e) {
} catch (Throwable $e) {
$this->add_to_backup_output($e->getMessage());
throw $e;
}
@@ -445,11 +433,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
private function add_to_backup_output($output): void
{
if ($this->backup_output) {
$this->backup_output = $this->backup_output."\n".$output;
} else {
$this->backup_output = $output;
}
$this->backup_output = $this->backup_output ? $this->backup_output."\n".$output : $output;
}
private function calculate_size()
@@ -459,10 +443,10 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
private function remove_old_backups(): void
{
if ($this->backup->number_of_backups_locally === 0) {
$deletable = $this->backup->executions()->where('status', 'success');
if ($this->scheduledDatabaseBackup->number_of_backups_locally === 0) {
$deletable = $this->scheduledDatabaseBackup->executions()->where('status', 'success');
} else {
$deletable = $this->backup->executions()->where('status', 'success')->skip($this->backup->number_of_backups_locally - 1);
$deletable = $this->scheduledDatabaseBackup->executions()->where('status', 'success')->skip($this->scheduledDatabaseBackup->number_of_backups_locally - 1);
}
foreach ($deletable->get() as $execution) {
delete_backup_locally($execution->filename, $this->server);
@@ -482,7 +466,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
$bucket = $this->s3->bucket;
$endpoint = $this->s3->endpoint;
$this->s3->testConnection(shouldSave: true);
if (data_get($this->backup, 'database_type') === \App\Models\ServiceDatabase::class) {
if (data_get($this->scheduledDatabaseBackup, 'database_type') === ServiceDatabase::class) {
$network = $this->database->service->destination->network;
} else {
$network = $this->database->destination->network;
@@ -493,29 +477,29 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
if (isDev()) {
if ($this->database->name === 'coolify-db') {
$backup_location_from = '/var/lib/docker/volumes/coolify_dev_backups_data/_data/coolify/coolify-db-'.$this->server->ip.$this->backup_file;
$commands[] = "docker run -d --network {$network} --name backup-of-{$this->backup->uuid} --rm -v $backup_location_from:$this->backup_location:ro {$fullImageName}";
$commands[] = "docker run -d --network {$network} --name backup-of-{$this->scheduledDatabaseBackup->uuid} --rm -v $backup_location_from:$this->backup_location:ro {$fullImageName}";
} else {
$backup_location_from = '/var/lib/docker/volumes/coolify_dev_backups_data/_data/databases/'.str($this->team->name)->slug().'-'.$this->team->id.'/'.$this->directory_name.$this->backup_file;
$commands[] = "docker run -d --network {$network} --name backup-of-{$this->backup->uuid} --rm -v $backup_location_from:$this->backup_location:ro {$fullImageName}";
$commands[] = "docker run -d --network {$network} --name backup-of-{$this->scheduledDatabaseBackup->uuid} --rm -v $backup_location_from:$this->backup_location:ro {$fullImageName}";
}
} else {
$commands[] = "docker run -d --network {$network} --name backup-of-{$this->backup->uuid} --rm -v $this->backup_location:$this->backup_location:ro {$fullImageName}";
$commands[] = "docker run -d --network {$network} --name backup-of-{$this->scheduledDatabaseBackup->uuid} --rm -v $this->backup_location:$this->backup_location:ro {$fullImageName}";
}
if ($this->s3->isHetzner()) {
$endpointWithoutBucket = 'https://'.str($endpoint)->after('https://')->after('.')->value();
$commands[] = "docker exec backup-of-{$this->backup->uuid} mc alias set --path=off --api=S3v4 temporary {$endpointWithoutBucket} $key $secret";
$commands[] = "docker exec backup-of-{$this->scheduledDatabaseBackup->uuid} mc alias set --path=off --api=S3v4 temporary {$endpointWithoutBucket} $key $secret";
} else {
$commands[] = "docker exec backup-of-{$this->backup->uuid} mc config host add temporary {$endpoint} $key $secret";
$commands[] = "docker exec backup-of-{$this->scheduledDatabaseBackup->uuid} mc config host add temporary {$endpoint} $key $secret";
}
$commands[] = "docker exec backup-of-{$this->backup->uuid} mc cp $this->backup_location temporary/$bucket{$this->backup_dir}/";
$commands[] = "docker exec backup-of-{$this->scheduledDatabaseBackup->uuid} mc cp $this->backup_location temporary/$bucket{$this->backup_dir}/";
instant_remote_process($commands, $this->server);
$this->add_to_backup_output('Uploaded to S3.');
} catch (\Throwable $e) {
} catch (Throwable $e) {
$this->add_to_backup_output($e->getMessage());
throw $e;
} finally {
$command = "docker rm -f backup-of-{$this->backup->uuid}";
$command = "docker rm -f backup-of-{$this->scheduledDatabaseBackup->uuid}";
instant_remote_process([$command], $this->server);
}
}