From d816863d31f07cc8fee56711ea041481626cc5c5 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Sat, 7 Sep 2024 11:48:20 +0200 Subject: [PATCH 1/4] Fix: Pull helper image if not available otherwise s3 backup upload fails --- app/Jobs/DatabaseBackupJob.php | 50 +++++++++++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/app/Jobs/DatabaseBackupJob.php b/app/Jobs/DatabaseBackupJob.php index 0666f78c3..7c42b2cd8 100644 --- a/app/Jobs/DatabaseBackupJob.php +++ b/app/Jobs/DatabaseBackupJob.php @@ -25,6 +25,7 @@ use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\Middleware\WithoutOverlapping; use Illuminate\Queue\SerializesModels; use Illuminate\Support\Str; +use App\Models\InstanceSettings; class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue { @@ -493,7 +494,16 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue } else { $network = $this->database->destination->network; } - $commands[] = "docker run -d --network {$network} --name backup-of-{$this->backup->uuid} --rm -v $this->backup_location:$this->backup_location:ro ghcr.io/coollabsio/coolify-helper"; + + // Ensure helper image is available before using it + $this->ensureHelperImageAvailable(); + + $settings = InstanceSettings::get(); + $helperImage = config('coolify.helper_image'); + $helperImageTag = $settings->helper_version; + $fullImageName = "{$helperImage}:{$helperImageTag}"; + + $commands[] = "docker run -d --network {$network} --name backup-of-{$this->backup->uuid} --rm -v $this->backup_location:$this->backup_location:ro {$fullImageName}"; $commands[] = "docker exec backup-of-{$this->backup->uuid} mc config host add temporary {$endpoint} $key $secret"; $commands[] = "docker exec backup-of-{$this->backup->uuid} mc cp $this->backup_location temporary/$bucket{$this->backup_dir}/"; instant_remote_process($commands, $this->server); @@ -507,4 +517,42 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue instant_remote_process([$command], $this->server); } } + + private function ensureHelperImageAvailable(): void + { + $settings = InstanceSettings::get(); + ray($settings); + + $helperImage = config('coolify.helper_image'); + ray('Helper Image:', $helperImage); + + $helperImageTag = $settings->helper_version; + ray('Helper Image Tag:', $helperImageTag); + + $fullImageName = "{$helperImage}:{$helperImageTag}"; + ray('Full Image Name:', $fullImageName); + + $imageExists = instant_remote_process(["docker image inspect {$fullImageName}"], $this->server, false); + ray('Image Exists:', $imageExists); + + if (empty($imageExists)) { + $this->add_to_backup_output("Helper image not found. Pulling {$fullImageName}."); + ray('Helper image not found. Attempting to pull.'); + try { + $pullResult = instant_remote_process(["docker pull {$fullImageName}"], $this->server); + ray('Pull Result:', $pullResult); + $this->add_to_backup_output("Helper image pulled successfully."); + ray('Helper image pulled successfully.'); + } catch (\Exception $e) { + $errorMessage = "Failed to pull helper image: " . $e->getMessage(); + ray('Error:', $errorMessage); + $this->add_to_backup_output($errorMessage); + throw new \RuntimeException($errorMessage); + } + } else { + $message = "Helper image {$fullImageName} is available."; + ray($message); + $this->add_to_backup_output($message); + } + } } From 504133232a2a961123005177ecb9dd1d27d708bc Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Sat, 7 Sep 2024 12:19:40 +0200 Subject: [PATCH 2/4] Improve logic, made it simpler --- app/Jobs/DatabaseBackupJob.php | 53 +++++++++++++++------------------- 1 file changed, 23 insertions(+), 30 deletions(-) diff --git a/app/Jobs/DatabaseBackupJob.php b/app/Jobs/DatabaseBackupJob.php index 7c42b2cd8..b4e6bfa53 100644 --- a/app/Jobs/DatabaseBackupJob.php +++ b/app/Jobs/DatabaseBackupJob.php @@ -495,7 +495,6 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue $network = $this->database->destination->network; } - // Ensure helper image is available before using it $this->ensureHelperImageAvailable(); $settings = InstanceSettings::get(); @@ -508,7 +507,6 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue $commands[] = "docker exec backup-of-{$this->backup->uuid} mc cp $this->backup_location temporary/$bucket{$this->backup_dir}/"; instant_remote_process($commands, $this->server); $this->add_to_backup_output('Uploaded to S3.'); - ray('Uploaded to S3. '.$this->backup_location.' to s3://'.$bucket.$this->backup_dir); } catch (\Throwable $e) { $this->add_to_backup_output($e->getMessage()); throw $e; @@ -521,38 +519,33 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue private function ensureHelperImageAvailable(): void { $settings = InstanceSettings::get(); - ray($settings); - $helperImage = config('coolify.helper_image'); - ray('Helper Image:', $helperImage); + $latestVersion = $settings->helper_version; + $fullImageName = "{$helperImage}:{$latestVersion}"; - $helperImageTag = $settings->helper_version; - ray('Helper Image Tag:', $helperImageTag); + $imageExists = $this->checkImageExists($fullImageName); - $fullImageName = "{$helperImage}:{$helperImageTag}"; - ray('Full Image Name:', $fullImageName); - - $imageExists = instant_remote_process(["docker image inspect {$fullImageName}"], $this->server, false); - ray('Image Exists:', $imageExists); - - if (empty($imageExists)) { - $this->add_to_backup_output("Helper image not found. Pulling {$fullImageName}."); - ray('Helper image not found. Attempting to pull.'); - try { - $pullResult = instant_remote_process(["docker pull {$fullImageName}"], $this->server); - ray('Pull Result:', $pullResult); - $this->add_to_backup_output("Helper image pulled successfully."); - ray('Helper image pulled successfully.'); - } catch (\Exception $e) { - $errorMessage = "Failed to pull helper image: " . $e->getMessage(); - ray('Error:', $errorMessage); - $this->add_to_backup_output($errorMessage); - throw new \RuntimeException($errorMessage); - } + if (!$imageExists) { + $this->pullHelperImage($fullImageName); } else { - $message = "Helper image {$fullImageName} is available."; - ray($message); - $this->add_to_backup_output($message); + return; + } + } + + private function checkImageExists(string $fullImageName): bool + { + $result = instant_remote_process(["docker image inspect {$fullImageName} >/dev/null 2>&1 && echo 'exists' || echo 'not exists'"], $this->server, false); + return trim($result) === 'exists'; + } + + private function pullHelperImage(string $fullImageName): void + { + try { + instant_remote_process(["docker pull {$fullImageName}"], $this->server); + } catch (\Exception $e) { + $errorMessage = "Failed to pull helper image: " . $e->getMessage(); + $this->add_to_backup_output($errorMessage); + throw new \RuntimeException($errorMessage); } } } From 6010b3209d213eb513261a57f055b2daea243c6e Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Sat, 7 Sep 2024 12:23:15 +0200 Subject: [PATCH 3/4] remove duplicated code --- app/Jobs/DatabaseBackupJob.php | 5 ----- 1 file changed, 5 deletions(-) diff --git a/app/Jobs/DatabaseBackupJob.php b/app/Jobs/DatabaseBackupJob.php index b4e6bfa53..59a1af104 100644 --- a/app/Jobs/DatabaseBackupJob.php +++ b/app/Jobs/DatabaseBackupJob.php @@ -497,11 +497,6 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue $this->ensureHelperImageAvailable(); - $settings = InstanceSettings::get(); - $helperImage = config('coolify.helper_image'); - $helperImageTag = $settings->helper_version; - $fullImageName = "{$helperImage}:{$helperImageTag}"; - $commands[] = "docker run -d --network {$network} --name backup-of-{$this->backup->uuid} --rm -v $this->backup_location:$this->backup_location:ro {$fullImageName}"; $commands[] = "docker exec backup-of-{$this->backup->uuid} mc config host add temporary {$endpoint} $key $secret"; $commands[] = "docker exec backup-of-{$this->backup->uuid} mc cp $this->backup_location temporary/$bucket{$this->backup_dir}/"; From a882d3c713eed7b94fc214f6094cb3d71ae68951 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Sat, 7 Sep 2024 12:25:50 +0200 Subject: [PATCH 4/4] code cleanup --- app/Jobs/DatabaseBackupJob.php | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/app/Jobs/DatabaseBackupJob.php b/app/Jobs/DatabaseBackupJob.php index 59a1af104..5d481199b 100644 --- a/app/Jobs/DatabaseBackupJob.php +++ b/app/Jobs/DatabaseBackupJob.php @@ -497,6 +497,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue $this->ensureHelperImageAvailable(); + $fullImageName = $this->getFullImageName(); $commands[] = "docker run -d --network {$network} --name backup-of-{$this->backup->uuid} --rm -v $this->backup_location:$this->backup_location:ro {$fullImageName}"; $commands[] = "docker exec backup-of-{$this->backup->uuid} mc config host add temporary {$endpoint} $key $secret"; $commands[] = "docker exec backup-of-{$this->backup->uuid} mc cp $this->backup_location temporary/$bucket{$this->backup_dir}/"; @@ -513,17 +514,12 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue private function ensureHelperImageAvailable(): void { - $settings = InstanceSettings::get(); - $helperImage = config('coolify.helper_image'); - $latestVersion = $settings->helper_version; - $fullImageName = "{$helperImage}:{$latestVersion}"; + $fullImageName = $this->getFullImageName(); $imageExists = $this->checkImageExists($fullImageName); if (!$imageExists) { $this->pullHelperImage($fullImageName); - } else { - return; } } @@ -543,4 +539,12 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue throw new \RuntimeException($errorMessage); } } + + private function getFullImageName(): string + { + $settings = InstanceSettings::get(); + $helperImage = config('coolify.helper_image'); + $latestVersion = $settings->helper_version; + return "{$helperImage}:{$latestVersion}"; + } }