dave/coolify
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

wip: scheduled backups

Browse Source 
fix: file locations vendor unlocking
This commit is contained in:
Andras Bacsai
2023-08-09 14:44:36 +02:00
parent 46909dca85
commit d18de24cf9
18 changed files with 116 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
app/Actions/Database/StartPostgresql.php
View File

@@ -12,17 +12,17 @@ class StartPostgresql
public StandalonePostgresql $database;
public array $commands = [];
public array $init_scripts = [];
public string $base_dir;
public string $configuration_dir;
public function __invoke(Server $server, StandalonePostgresql $database)
{
$this->database = $database;
$container_name = generate_container_name($this->database->uuid);
$this->base_dir = '/data/coolify/databases/' . $container_name;
$this->configuration_dir = database_configuration_dir() . '/' . $container_name;
$this->commands = [
"mkdir -p $this->base_dir",
"mkdir -p $this->base_dir/docker-entrypoint-initdb.d/"
"mkdir -p $this->configuration_dir",
"mkdir -p $this->configuration_dir/docker-entrypoint-initdb.d/"
];
$persistent_storages = $this->generate_local_persistent_volumes();
@@ -92,8 +92,10 @@ class StartPostgresql
}
$docker_compose = Yaml::dump($docker_compose, 10);
$docker_compose_base64 = base64_encode($docker_compose);
$this->commands[] = "echo '{$docker_compose_base64}' | base64 -d > $this->base_dir/docker-compose.yml";
$this->commands[] = "docker compose -f $this->base_dir/docker-compose.yml up -d";
$this->commands[] = "echo '{$docker_compose_base64}' | base64 -d > $this->configuration_dir/docker-compose.yml";
$readme = generate_readme_file($this->database->name, now());
$this->commands[] = "echo '{$readme}' > $this->configuration_dir/README.md";
$this->commands[] = "docker compose -f $this->configuration_dir/docker-compose.yml up -d";
return remote_process($this->commands, $server);
}
@@ -155,8 +157,8 @@ class StartPostgresql
$filename = data_get($init_script, 'filename');
$content = data_get($init_script, 'content');
$content_base64 = base64_encode($content);
$this->commands[] = "echo '{$content_base64}' | base64 -d > $this->base_dir/docker-entrypoint-initdb.d/{$filename}";
$this->init_scripts[] = "$this->base_dir/docker-entrypoint-initdb.d/{$filename}";
$this->commands[] = "echo '{$content_base64}' | base64 -d > $this->configuration_dir/docker-entrypoint-initdb.d/{$filename}";
$this->init_scripts[] = "$this->configuration_dir/docker-entrypoint-initdb.d/{$filename}";
}
}
}

1
app/Http/Controllers/ApplicationController.php
View File

@@ -24,7 +24,6 @@ class ApplicationController extends Controller
if (!$application) {
return redirect()->route('dashboard');
}
ray($application->persistentStorages()->get());
return view('project.application.configuration', ['application' => $application]);
}

3
app/Http/Livewire/Project/Database/Postgresql/General.php
View File

@@ -78,6 +78,9 @@ class General extends Component
$this->emit('error', 'Filename already exists.');
return;
}
if (!isset($this->database->init_scripts)) {
$this->database->init_scripts = [];
}
$this->database->init_scripts = array_merge($this->database->init_scripts, [
[
'index' => count($this->database->init_scripts),

30
app/Jobs/ApplicationDeploymentJob.php
View File

@@ -15,13 +15,13 @@ use App\Models\SwarmDocker;
use App\Notifications\Application\DeploymentFailed;
use App\Notifications\Application\DeploymentSuccess;
use App\Traits\ExecuteRemoteCommand;
use Exception;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Storage;
use Illuminate\Support\Str;
use Spatie\Url\Url;
use Symfony\Component\Yaml\Yaml;
@@ -51,6 +51,7 @@ class ApplicationDeploymentJob implements ShouldQueue
private string $container_name;
private string $workdir;
private string $configuration_dir;
private string $build_workdir;
private string $build_image_name;
private string $production_image_name;
@@ -58,6 +59,7 @@ class ApplicationDeploymentJob implements ShouldQueue
private $build_args;
private $env_args;
private $docker_compose;
private $docker_compose_base64;
private $log_model;
private Collection $saved_outputs;
@@ -78,9 +80,9 @@ class ApplicationDeploymentJob implements ShouldQueue
$this->source = $this->application->source->getMorphClass()::where('id', $this->application->source->id)->first();
$this->destination = $this->application->destination->getMorphClass()::where('id', $this->application->destination->id)->first();
$this->server = $this->destination->server;
$this->private_key_location = save_private_key_for_server($this->server);
$this->workdir = "/artifacts/{$this->deployment_uuid}";
$this->configuration_dir = application_configuration_dir() . "/{$this->application->uuid}";
$this->build_workdir = "{$this->workdir}" . rtrim($this->application->base_directory, '/');
$this->is_debug_enabled = $this->application->settings->is_debug_enabled;
@@ -122,12 +124,23 @@ class ApplicationDeploymentJob implements ShouldQueue
}
if ($this->application->fqdn) dispatch(new ProxyStartJob($this->server));
$this->next(ApplicationDeploymentStatus::FINISHED->value);
} catch (\Exception $e) {
} catch (Exception $e) {
ray($e);
$this->fail($e);
} finally {
if (isset($this->docker_compose)) {
Storage::disk('deployments')->put(Str::kebab($this->application->name) . '/docker-compose.yml', $this->docker_compose);
if (isset($this->docker_compose_base64)) {
$readme = generate_readme_file($this->application->name, $this->application_deployment_queue->updated_at);
$this->execute_remote_command(
[
"mkdir -p $this->configuration_dir"
],
[
"echo '{$this->docker_compose_base64}' | base64 -d > $this->configuration_dir/docker-compose.yml",
],
[
"echo '{$readme}' > $this->configuration_dir/README.md",
]
);
}
$this->execute_remote_command(
[
@@ -135,7 +148,6 @@ class ApplicationDeploymentJob implements ShouldQueue
"hidden" => true,
]
);
// ray()->measure();
}
}
@@ -368,8 +380,8 @@ class ApplicationDeploymentJob implements ShouldQueue
$docker_compose['volumes'] = $volume_names;
}
$this->docker_compose = Yaml::dump($docker_compose, 10);
$docker_compose_base64 = base64_encode($this->docker_compose);
$this->execute_remote_command([$this->execute_in_builder("echo '{$docker_compose_base64}' | base64 -d > {$this->workdir}/docker-compose.yml"), "hidden" => true]);
$this->docker_compose_base64 = base64_encode($this->docker_compose);
$this->execute_remote_command([$this->execute_in_builder("echo '{$this->docker_compose_base64}' | base64 -d > {$this->workdir}/docker-compose.yml"), "hidden" => true]);
}
private function generate_local_persistent_volumes()
@@ -591,7 +603,7 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf");
$tag = Str::of("{$this->commit}-{$this->application->id}-{$this->pull_request_id}");
if (strlen($tag) > 128) {
$tag = $tag->substr(0, 128);
};
}
$this->build_image_name = "{$this->application->git_repository}:{$tag}-build";
$this->production_image_name = "{$this->application->uuid}:{$tag}";

49
app/Jobs/BackupDatabaseJob.php
View File

@@ -2,23 +2,66 @@
namespace App\Jobs;
use App\Models\ScheduledDatabaseBackup;
use App\Models\Server;
use App\Models\StandalonePostgresql;
use App\Models\Team;
use Carbon\Carbon;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Throwable;
class BackupDatabaseJob implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public function __construct(public $backup)
public Team|null $team = null;
public Server $server;
public ScheduledDatabaseBackup|null $backup;
public string $database_type;
public StandalonePostgresql $database;
public string $status;
public function __construct($backup)
{
$this->backup = $backup;
$this->team = Team::find($backup->team_id);
$this->database = $this->backup->database->first();
$this->database_type = $this->database->type();
$this->server = $this->database->destination->server;
$this->status = $this->database->status;
}
public function handle()
{
ray('BackupDatabaseJob');
ray($this->backup);
if ($this->status !== 'running') {
ray('database not running');
return;
}
if ($this->database_type === 'standalone-postgresql') {
$this->backup_standalone_postgresql();
}
}
private function backup_standalone_postgresql()
{
try {
$backup_filename = backup_dir() . "/{$this->database->uuid}/dumpall-" . Carbon::now()->timestamp . ".sql";
$commands[] = "mkdir -p " . backup_dir();
$commands[] = "mkdir -p " . backup_dir() . "/{$this->database->uuid}";
$commands[] = "docker exec {$this->database->uuid} pg_dumpall -U {$this->database->postgres_user} > $backup_filename";
instant_remote_process($commands, $this->server);
ray('Backup done for ' . $this->database->uuid . ' at ' . $this->server->name . ':' . $backup_filename);
if (!$this->backup->keep_locally) {
$commands[] = "rm -rf $backup_filename";
instant_remote_process($commands, $this->server);
}
} catch (Throwable $th) {
ray($th);
//throw $th;
}
}
}

1
app/Models/S3Storage.php
View File

@@ -30,5 +30,4 @@ class S3Storage extends BaseModel
set_s3_target($this);
return \Storage::disk('custom-s3')->files();
}
}

2
app/Models/StandalonePostgresql.php
View File

@@ -49,7 +49,7 @@ class StandalonePostgresql extends BaseModel
);
}
public function type()
public function type(): string
{
return 'standalone-postgresql';
}

5
app/Models/Team.php
View File

@@ -83,4 +83,9 @@ class Team extends Model implements SendsDiscord, SendsEmail
$sources = $sources->merge($github_apps)->merge($gitlab_apps);
return $sources;
}
public function s3()
{
return $this->hasOne(S3Storage::class);
}
}