From 2cd1785a929632832b81e7e3a6df19985f7f491d Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Fri, 29 Sep 2023 10:53:28 +0200 Subject: [PATCH 01/13] version++ --- config/sentry.php | 2 +- config/version.php | 2 +- versions.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/config/sentry.php b/config/sentry.php index 86047a0ef..469ea55d5 100644 --- a/config/sentry.php +++ b/config/sentry.php @@ -7,7 +7,7 @@ return [ // The release version of your application // Example with dynamic git hash: trim(exec('git --git-dir ' . base_path('.git') . ' log --pretty="%h" -n1 HEAD')) - 'release' => '4.0.0-beta.50', + 'release' => '4.0.0-beta.51', // When left empty or `null` the Laravel environment will be used 'environment' => config('app.env'), diff --git a/config/version.php b/config/version.php index e7e4e3810..9724bda6a 100644 --- a/config/version.php +++ b/config/version.php @@ -1,3 +1,3 @@ Date: Fri, 29 Sep 2023 10:53:33 +0200 Subject: [PATCH 02/13] remove plausible for now --- templates/deprecated.json | 7 +++++++ templates/service-templates.json | 5 ----- 2 files changed, 7 insertions(+), 5 deletions(-) create mode 100644 templates/deprecated.json diff --git a/templates/deprecated.json b/templates/deprecated.json new file mode 100644 index 000000000..595c5ec6f --- /dev/null +++ b/templates/deprecated.json @@ -0,0 +1,7 @@ +{ + "plausible-analytics": { + "documentation": "https://plausible.io/docs", + "slogan": "A lighweight and open-source website analytics tool.", + "compose": "dmVyc2lvbjogIjMuMyIKc2VydmljZXM6CiAgcGxhdXNpYmxlLWFuYWx5dGljczoKICAgIGltYWdlOiBwbGF1c2libGUvYW5hbHl0aWNzOnYyLjAKICAgIGNvbW1hbmQ6IHNoIC1jICJzbGVlcCAxMCAmJiAvZW50cnlwb2ludC5zaCBkYiBjcmVhdGVkYiAmJiAvZW50cnlwb2ludC5zaCBkYiBtaWdyYXRlICYmIC9lbnRyeXBvaW50LnNoIHJ1biIKICAgIGVudmlyb25tZW50OgogICAgICAtIERBVEFCQVNFX1VSTD1wb3N0Z3JlczovL3Bvc3RncmVzOiRTRVJWSUNFX1BBU1NXT1JEX1BPU1RHUkVTQHBsYXVzaWJsZV9kYi9wbGF1c2libGUKICAgICAgLSBCQVNFX1VSTD0kU0VSVklDRV9GUUROX1BMQVVTSUJMRQogICAgICAtIFNFQ1JFVF9LRVlfQkFTRT0kU0VSVklDRV9CQVNFNjRfNjRfUExBVVNJQkxFCiAgICBkZXBlbmRzX29uOgogICAgICAtIHBsYXVzaWJsZV9kYgogICAgICAtIHBsYXVzaWJsZV9ldmVudHNfZGIKICAgICAgLSBtYWlsCgogIG1haWw6CiAgICBpbWFnZTogYnl0ZW1hcmsvc210cAoKICBwbGF1c2libGVfZGI6CiAgICBpbWFnZTogcG9zdGdyZXM6MTQtYWxwaW5lCiAgICB2b2x1bWVzOgogICAgICAtIGRiLWRhdGE6L3Zhci9saWIvcG9zdGdyZXNxbC9kYXRhCiAgICBlbnZpcm9ubWVudDoKICAgICAgLSBQT1NUR1JFU19EQj1wbGF1c2libGUKICAgICAgLSBQT1NUR1JFU19QQVNTV09SRD0kU0VSVklDRV9QQVNTV09SRF9QT1NUR1JFUwoKICBwbGF1c2libGVfZXZlbnRzX2RiOgogICAgaW1hZ2U6IGNsaWNraG91c2UvY2xpY2tob3VzZS1zZXJ2ZXI6MjMuMy43LjUtYWxwaW5lCiAgICB2b2x1bWVzOgogICAgICAtIHR5cGU6IHZvbHVtZQogICAgICAgIHNvdXJjZTogZXZlbnQtZGF0YQogICAgICAgIHRhcmdldDogL3Zhci9saWIvY2xpY2tob3VzZQogICAgICAtIHR5cGU6IGJpbmQKICAgICAgICBzb3VyY2U6IC4vY2xpY2tob3VzZS9jbGlja2hvdXNlLWNvbmZpZy54bWwKICAgICAgICB0YXJnZXQ6IC9ldGMvY2xpY2tob3VzZS1zZXJ2ZXIvY29uZmlnLmQvbG9nZ2luZy54bWwKICAgICAgICByZWFkX29ubHk6IHRydWUKICAgICAgICBjb250ZW50OiA+LQogICAgICAgICAgPGNsaWNraG91c2U+PHByb2ZpbGVzPjxkZWZhdWx0Pjxsb2dfcXVlcmllcz4wPC9sb2dfcXVlcmllcz48bG9nX3F1ZXJ5X3RocmVhZHM+MDwvbG9nX3F1ZXJ5X3RocmVhZHM+PC9kZWZhdWx0PjwvcHJvZmlsZXM+PC9jbGlja2hvdXNlPgogICAgICAtIHR5cGU6IGJpbmQKICAgICAgICBzb3VyY2U6IC4vY2xpY2tob3VzZS9jbGlja2hvdXNlLXVzZXItY29uZmlnLnhtbAogICAgICAgIHRhcmdldDogL2V0Yy9jbGlja2hvdXNlLXNlcnZlci91c2Vycy5kL2xvZ2dpbmcueG1sCiAgICAgICAgcmVhZF9vbmx5OiB0cnVlCiAgICAgICAgY29udGVudDogPi0KICAgICAgICAgIDxjbGlja2hvdXNlPjxsb2dnZXI+PGxldmVsPndhcm5pbmc8L2xldmVsPjxjb25zb2xlPnRydWU8L2NvbnNvbGU+PC9sb2dnZXI+PHF1ZXJ5X3RocmVhZF9sb2cKICAgICAgICAgIHJlbW92ZT0icmVtb3ZlIi8+PHF1ZXJ5X2xvZyByZW1vdmU9InJlbW92ZSIvPjx0ZXh0X2xvZwogICAgICAgICAgcmVtb3ZlPSJyZW1vdmUiLz48dHJhY2VfbG9nIHJlbW92ZT0icmVtb3ZlIi8+PG1ldHJpY19sb2cKICAgICAgICAgIHJlbW92ZT0icmVtb3ZlIi8+PGFzeW5jaHJvbm91c19tZXRyaWNfbG9nCiAgICAgICAgICByZW1vdmU9InJlbW92ZSIvPjxzZXNzaW9uX2xvZyByZW1vdmU9InJlbW92ZSIvPjxwYXJ0X2xvZwogICAgICAgICAgcmVtb3ZlPSJyZW1vdmUiLz48L2NsaWNraG91c2U+CiAgICB1bGltaXRzOgogICAgICBub2ZpbGU6CiAgICAgICAgc29mdDogMjYyMTQ0CiAgICAgICAgaGFyZDogMjYyMTQ0Cg==" + }, +} diff --git a/templates/service-templates.json b/templates/service-templates.json index caa4bddca..2b06a382a 100644 --- a/templates/service-templates.json +++ b/templates/service-templates.json @@ -1,9 +1,4 @@ { - "plausible-analytics": { - "documentation": "https://plausible.io/docs", - "slogan": "A lighweight and open-source website analytics tool.", - "compose": "dmVyc2lvbjogIjMuMyIKc2VydmljZXM6CiAgcGxhdXNpYmxlLWFuYWx5dGljczoKICAgIGltYWdlOiBwbGF1c2libGUvYW5hbHl0aWNzOnYyLjAKICAgIGNvbW1hbmQ6IHNoIC1jICJzbGVlcCAxMCAmJiAvZW50cnlwb2ludC5zaCBkYiBjcmVhdGVkYiAmJiAvZW50cnlwb2ludC5zaCBkYiBtaWdyYXRlICYmIC9lbnRyeXBvaW50LnNoIHJ1biIKICAgIGVudmlyb25tZW50OgogICAgICAtIERBVEFCQVNFX1VSTD1wb3N0Z3JlczovL3Bvc3RncmVzOiRTRVJWSUNFX1BBU1NXT1JEX1BPU1RHUkVTQHBsYXVzaWJsZV9kYi9wbGF1c2libGUKICAgICAgLSBCQVNFX1VSTD0kU0VSVklDRV9GUUROX1BMQVVTSUJMRQogICAgICAtIFNFQ1JFVF9LRVlfQkFTRT0kU0VSVklDRV9CQVNFNjRfNjRfUExBVVNJQkxFCiAgICBkZXBlbmRzX29uOgogICAgICAtIHBsYXVzaWJsZV9kYgogICAgICAtIHBsYXVzaWJsZV9ldmVudHNfZGIKICAgICAgLSBtYWlsCgogIG1haWw6CiAgICBpbWFnZTogYnl0ZW1hcmsvc210cAoKICBwbGF1c2libGVfZGI6CiAgICBpbWFnZTogcG9zdGdyZXM6MTQtYWxwaW5lCiAgICB2b2x1bWVzOgogICAgICAtIGRiLWRhdGE6L3Zhci9saWIvcG9zdGdyZXNxbC9kYXRhCiAgICBlbnZpcm9ubWVudDoKICAgICAgLSBQT1NUR1JFU19EQj1wbGF1c2libGUKICAgICAgLSBQT1NUR1JFU19QQVNTV09SRD0kU0VSVklDRV9QQVNTV09SRF9QT1NUR1JFUwoKICBwbGF1c2libGVfZXZlbnRzX2RiOgogICAgaW1hZ2U6IGNsaWNraG91c2UvY2xpY2tob3VzZS1zZXJ2ZXI6MjMuMy43LjUtYWxwaW5lCiAgICB2b2x1bWVzOgogICAgICAtIHR5cGU6IHZvbHVtZQogICAgICAgIHNvdXJjZTogZXZlbnQtZGF0YQogICAgICAgIHRhcmdldDogL3Zhci9saWIvY2xpY2tob3VzZQogICAgICAtIHR5cGU6IGJpbmQKICAgICAgICBzb3VyY2U6IC4vY2xpY2tob3VzZS9jbGlja2hvdXNlLWNvbmZpZy54bWwKICAgICAgICB0YXJnZXQ6IC9ldGMvY2xpY2tob3VzZS1zZXJ2ZXIvY29uZmlnLmQvbG9nZ2luZy54bWwKICAgICAgICByZWFkX29ubHk6IHRydWUKICAgICAgICBjb250ZW50OiA+LQogICAgICAgICAgPGNsaWNraG91c2U+PHByb2ZpbGVzPjxkZWZhdWx0Pjxsb2dfcXVlcmllcz4wPC9sb2dfcXVlcmllcz48bG9nX3F1ZXJ5X3RocmVhZHM+MDwvbG9nX3F1ZXJ5X3RocmVhZHM+PC9kZWZhdWx0PjwvcHJvZmlsZXM+PC9jbGlja2hvdXNlPgogICAgICAtIHR5cGU6IGJpbmQKICAgICAgICBzb3VyY2U6IC4vY2xpY2tob3VzZS9jbGlja2hvdXNlLXVzZXItY29uZmlnLnhtbAogICAgICAgIHRhcmdldDogL2V0Yy9jbGlja2hvdXNlLXNlcnZlci91c2Vycy5kL2xvZ2dpbmcueG1sCiAgICAgICAgcmVhZF9vbmx5OiB0cnVlCiAgICAgICAgY29udGVudDogPi0KICAgICAgICAgIDxjbGlja2hvdXNlPjxsb2dnZXI+PGxldmVsPndhcm5pbmc8L2xldmVsPjxjb25zb2xlPnRydWU8L2NvbnNvbGU+PC9sb2dnZXI+PHF1ZXJ5X3RocmVhZF9sb2cKICAgICAgICAgIHJlbW92ZT0icmVtb3ZlIi8+PHF1ZXJ5X2xvZyByZW1vdmU9InJlbW92ZSIvPjx0ZXh0X2xvZwogICAgICAgICAgcmVtb3ZlPSJyZW1vdmUiLz48dHJhY2VfbG9nIHJlbW92ZT0icmVtb3ZlIi8+PG1ldHJpY19sb2cKICAgICAgICAgIHJlbW92ZT0icmVtb3ZlIi8+PGFzeW5jaHJvbm91c19tZXRyaWNfbG9nCiAgICAgICAgICByZW1vdmU9InJlbW92ZSIvPjxzZXNzaW9uX2xvZyByZW1vdmU9InJlbW92ZSIvPjxwYXJ0X2xvZwogICAgICAgICAgcmVtb3ZlPSJyZW1vdmUiLz48L2NsaWNraG91c2U+CiAgICB1bGltaXRzOgogICAgICBub2ZpbGU6CiAgICAgICAgc29mdDogMjYyMTQ0CiAgICAgICAgaGFyZDogMjYyMTQ0Cg==" - }, "umami": { "documentation": "https://umami.is/docs/getting-started", "slogan": "Umami makes it easy to collect, analyze, and understand your web data — while maintaining visitor privacy and data ownership.", From ac133875fa3f2221f821387e8e50b63ec9317e36 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Fri, 29 Sep 2023 11:01:40 +0200 Subject: [PATCH 03/13] fix: remove private key in case you removed a github app --- app/Models/GithubApp.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/Models/GithubApp.php b/app/Models/GithubApp.php index c4cd3568d..758bf35c5 100644 --- a/app/Models/GithubApp.php +++ b/app/Models/GithubApp.php @@ -6,6 +6,7 @@ use Illuminate\Database\Eloquent\Casts\Attribute; class GithubApp extends BaseModel { + protected $guarded = []; protected $appends = ['type']; protected $casts = [ @@ -17,6 +18,7 @@ class GithubApp extends BaseModel 'webhook_secret', ]; + static public function public() { return GithubApp::whereTeamId(currentTeam()->id)->whereisPublic(true)->whereNotNull('app_id')->get(); @@ -34,6 +36,7 @@ class GithubApp extends BaseModel if ($applications_count > 0) { throw new \Exception('You cannot delete this GitHub App because it is in use by ' . $applications_count . ' application(s). Delete them first.'); } + $github_app->privateKey()->delete(); }); } From bdcc0c8de567219bdcb7a8cb8a86c8836227066e Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Fri, 29 Sep 2023 11:01:58 +0200 Subject: [PATCH 04/13] fix: only show manually added private keys on server view --- routes/web.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/routes/web.php b/routes/web.php index ada529100..34adf01b2 100644 --- a/routes/web.php +++ b/routes/web.php @@ -99,7 +99,7 @@ Route::middleware(['auth'])->group(function () { ]))->name('server.proxy'); Route::get('/server/{server_uuid}/private-key', fn () => view('server.private-key', [ 'server' => Server::ownedByCurrentTeam()->whereUuid(request()->server_uuid)->firstOrFail(), - 'privateKeys' => PrivateKey::ownedByCurrentTeam()->get(), + 'privateKeys' => PrivateKey::ownedByCurrentTeam()->get()->where('is_git_related', false), ]))->name('server.private-key'); Route::get('/server/{server_uuid}/destinations', fn () => view('server.destinations', [ 'server' => Server::ownedByCurrentTeam(['name', 'proxy'])->whereUuid(request()->server_uuid)->firstOrFail() @@ -133,7 +133,7 @@ Route::middleware(['auth'])->group(function () { Route::middleware(['auth'])->group(function () { Route::get('/security', fn () => view('security.index'))->name('security.index'); Route::get('/security/private-key', fn () => view('security.private-key.index', [ - 'privateKeys' => PrivateKey::ownedByCurrentTeam(['name', 'uuid', 'is_git_related'])->where('is_git_related', false)->get() + 'privateKeys' => PrivateKey::ownedByCurrentTeam(['name', 'uuid', 'is_git_related'])->get() ]))->name('security.private-key.index'); Route::get('/security/private-key/new', fn () => view('security.private-key.new'))->name('security.private-key.new'); Route::get('/security/private-key/{private_key_uuid}', fn () => view('security.private-key.show', [ From 77037f893320e766abc3ace12db737de1b981db4 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Fri, 29 Sep 2023 14:26:19 +0200 Subject: [PATCH 05/13] ui: fix previews to preview --- resources/views/livewire/project/application/general.blade.php | 2 +- .../views/livewire/project/application/preview/form.blade.php | 2 +- resources/views/project/application/configuration.blade.php | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/resources/views/livewire/project/application/general.blade.php b/resources/views/livewire/project/application/general.blade.php index ca3fb7a6e..c901da0e6 100644 --- a/resources/views/livewire/project/application/general.blade.php +++ b/resources/views/livewire/project/application/general.blade.php @@ -89,7 +89,7 @@ id="is_auto_deploy_enabled" label="Auto Deploy" /> + instantSave id="is_preview_deployments_enabled" label="Preview Deployments" /> diff --git a/resources/views/livewire/project/application/preview/form.blade.php b/resources/views/livewire/project/application/preview/form.blade.php index e93d2f97c..c466c6af1 100644 --- a/resources/views/livewire/project/application/preview/form.blade.php +++ b/resources/views/livewire/project/application/preview/form.blade.php @@ -1,6 +1,6 @@
-

Previews Deployments

+

Preview Deployments

Save Reset template to default
diff --git a/resources/views/project/application/configuration.blade.php b/resources/views/project/application/configuration.blade.php index 91dcb57e6..6d8971c8f 100644 --- a/resources/views/project/application/configuration.blade.php +++ b/resources/views/project/application/configuration.blade.php @@ -22,7 +22,7 @@ @if ($application->git_based()) Previews + @click.prevent="activeTab = 'previews'; window.location.hash = 'previews'" href="#">Preview Deployments @endif From 4161ea7eb631db572c97ae8ae42c2f1623d03154 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Fri, 29 Sep 2023 14:26:31 +0200 Subject: [PATCH 06/13] fix: show source on all type of applications --- app/Models/Application.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/Models/Application.php b/app/Models/Application.php index b08561ef7..59b67fa16 100644 --- a/app/Models/Application.php +++ b/app/Models/Application.php @@ -226,7 +226,7 @@ class Application extends BaseModel } public function git_based(): bool { - if ($this->dockerfile || $this->build_pack === 'dockerfile' || $this->dockercompose || $this->build_pack === 'dockercompose') { + if ($this->dockerfile) { return false; } return true; From 23e205b6cd476a723d38598bb44bc56de99bd459 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Fri, 29 Sep 2023 14:26:42 +0200 Subject: [PATCH 07/13] fix: docker cleanup should be a job by server --- app/Console/Kernel.php | 11 ++++- app/Jobs/DockerCleanupJob.php | 91 ++++++++++++++++++----------------- 2 files changed, 56 insertions(+), 46 deletions(-) diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index 0531899fe..ee9f67ad7 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -27,14 +27,23 @@ class Kernel extends ConsoleKernel // $this->instance_auto_update($schedule); // $this->check_scheduled_backups($schedule); $this->check_resources($schedule); + $this->cleanup_servers($schedule); } else { $schedule->command('horizon:snapshot')->everyFiveMinutes(); $schedule->job(new CleanupInstanceStuffsJob)->everyTwoMinutes()->onOneServer(); $schedule->job(new CheckResaleLicenseJob)->hourly()->onOneServer(); - $schedule->job(new DockerCleanupJob)->everyTenMinutes()->onOneServer(); + // $schedule->job(new DockerCleanupJob)->everyTenMinutes()->onOneServer(); $this->instance_auto_update($schedule); $this->check_scheduled_backups($schedule); $this->check_resources($schedule); + $this->cleanup_servers($schedule); + } + } + private function cleanup_servers($schedule) + { + $servers = Server::all()->where('settings.is_usable', true)->where('settings.is_reachable', true); + foreach ($servers as $server) { + $schedule->job(new DockerCleanupJob($server))->everyTenMinutes()->onOneServer(); } } private function check_resources($schedule) diff --git a/app/Jobs/DockerCleanupJob.php b/app/Jobs/DockerCleanupJob.php index 80c76e067..08cb16959 100644 --- a/app/Jobs/DockerCleanupJob.php +++ b/app/Jobs/DockerCleanupJob.php @@ -16,65 +16,66 @@ class DockerCleanupJob implements ShouldQueue, ShouldBeEncrypted { use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; - public $timeout = 500; + public $timeout = 1000; public ?string $dockerRootFilesystem = null; public ?int $usageBefore = null; public function middleware(): array { - return [ - (new WithoutOverlapping("dockerimagejobs"))->shared(), - ]; + return [(new WithoutOverlapping($this->server->uuid))->dontRelease()]; } - public function __construct() + + public function uniqueId(): string + { + return $this->server->uuid; + } + public function __construct(public Server $server) { } public function handle(): void { - $queue = ApplicationDeploymentQueue::where('status', '==', 'in_progress')->get(); - if ($queue->count() > 0) { + $queuedCount = 0; + $this->server->applications()->each(function ($application) use ($queuedCount) { + $count = data_get($application->deployments(), 'count', 0); + $queuedCount += $count; + }); + if ($queuedCount > 0) { ray('DockerCleanupJob: ApplicationDeploymentQueue is not empty, skipping')->color('orange'); return; } try { - // ray()->showQueries()->color('orange'); - $servers = Server::all(); - foreach ($servers as $server) { - if ( - !$server->isFunctional() - ) { - continue; - } - if (isDev()) { - $this->dockerRootFilesystem = "/"; + if (!$this->server->isFunctional()) { + return; + } + if (isDev()) { + $this->dockerRootFilesystem = "/"; + } else { + $this->dockerRootFilesystem = instant_remote_process( + [ + "stat --printf=%m $(docker info --format '{{json .DockerRootDir}}'' |sed 's/\"//g')" + ], + $this->server, + false + ); + } + if (!$this->dockerRootFilesystem) { + return; + } + $this->usageBefore = $this->getFilesystemUsage(); + if ($this->usageBefore >= $this->server->settings->cleanup_after_percentage) { + ray('Cleaning up ' . $this->server->name)->color('orange'); + instant_remote_process(['docker image prune -af'], $this->server); + instant_remote_process(['docker container prune -f --filter "label=coolify.managed=true"'], $this->server); + instant_remote_process(['docker builder prune -af'], $this->server); + $usageAfter = $this->getFilesystemUsage(); + if ($usageAfter < $this->usageBefore) { + ray('Saved ' . ($this->usageBefore - $usageAfter) . '% disk space on ' . $this->server->name)->color('orange'); + send_internal_notification('DockerCleanupJob done: Saved ' . ($this->usageBefore - $usageAfter) . '% disk space on ' . $this->server->name); } else { - $this->dockerRootFilesystem = instant_remote_process( - [ - "stat --printf=%m $(docker info --format '{{json .DockerRootDir}}'' |sed 's/\"//g')" - ], - $server, - false - ); - } - if (!$this->dockerRootFilesystem) { - continue; - } - $this->usageBefore = $this->getFilesystemUsage($server); - if ($this->usageBefore >= $server->settings->cleanup_after_percentage) { - ray('Cleaning up ' . $server->name)->color('orange'); - instant_remote_process(['docker image prune -af'], $server); - instant_remote_process(['docker container prune -f --filter "label=coolify.managed=true"'], $server); - instant_remote_process(['docker builder prune -af'], $server); - $usageAfter = $this->getFilesystemUsage($server); - if ($usageAfter < $this->usageBefore) { - ray('Saved ' . ($this->usageBefore - $usageAfter) . '% disk space on ' . $server->name)->color('orange'); - send_internal_notification('DockerCleanupJob done: Saved ' . ($this->usageBefore - $usageAfter) . '% disk space on ' . $server->name); - } else { - ray('DockerCleanupJob failed to save disk space on ' . $server->name)->color('orange'); - } - } else { - ray('No need to clean up ' . $server->name)->color('orange'); + ray('DockerCleanupJob failed to save disk space on ' . $this->server->name)->color('orange'); } + } else { + ray('No need to clean up ' . $this->server->name)->color('orange'); } } catch (\Throwable $e) { send_internal_notification('DockerCleanupJob failed with: ' . $e->getMessage()); @@ -83,8 +84,8 @@ class DockerCleanupJob implements ShouldQueue, ShouldBeEncrypted } } - private function getFilesystemUsage(Server $server) + private function getFilesystemUsage() { - return instant_remote_process(["df '{$this->dockerRootFilesystem}'| tail -1 | awk '{ print $5}' | sed 's/%//g'"], $server, false); + return instant_remote_process(["df '{$this->dockerRootFilesystem}'| tail -1 | awk '{ print $5}' | sed 's/%//g'"], $this->server, false); } } From 64ce41df0f55958ea9c66fea124b020be82e06f7 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Fri, 29 Sep 2023 21:38:11 +0200 Subject: [PATCH 08/13] fix: file/dir based volumes are now read from the server --- .../Livewire/Project/Service/FileStorage.php | 10 +++- app/Http/Livewire/Project/Service/Show.php | 2 + app/Models/LocalFileVolume.php | 37 ++++++++++++++ app/Models/Service.php | 2 +- app/Models/ServiceApplication.php | 4 +- app/Models/ServiceDatabase.php | 4 +- bootstrap/helpers/services.php | 50 +++++++++++-------- .../project/service/file-storage.blade.php | 12 +++-- .../livewire/project/service/show.blade.php | 4 +- 9 files changed, 91 insertions(+), 34 deletions(-) diff --git a/app/Http/Livewire/Project/Service/FileStorage.php b/app/Http/Livewire/Project/Service/FileStorage.php index 4341c6f51..e4103f600 100644 --- a/app/Http/Livewire/Project/Service/FileStorage.php +++ b/app/Http/Livewire/Project/Service/FileStorage.php @@ -33,12 +33,20 @@ class FileStorage extends Component } public function submit() { + $original = $this->fileStorage->getOriginal(); try { $this->validate(); + if ($this->fileStorage->is_directory) { + $this->fileStorage->content = null; + } $this->fileStorage->save(); - $this->service->saveFileVolumes(); + $this->fileStorage->saveStorageOnServer($this->service); + // ray($this->fileStorage); + // $this->service->saveFileVolumes(); $this->emit('success', 'File updated successfully.'); } catch (\Throwable $e) { + $this->fileStorage->setRawAttributes($original); + $this->fileStorage->save(); return handleError($e, $this); } } diff --git a/app/Http/Livewire/Project/Service/Show.php b/app/Http/Livewire/Project/Service/Show.php index 656adff0a..6f48a1389 100644 --- a/app/Http/Livewire/Project/Service/Show.php +++ b/app/Http/Livewire/Project/Service/Show.php @@ -27,8 +27,10 @@ class Show extends Component $service = $this->service->applications()->whereName($this->parameters['service_name'])->first(); if ($service) { $this->serviceApplication = $service; + $this->serviceApplication->getFilesFromServer(); } else { $this->serviceDatabase = $this->service->databases()->whereName($this->parameters['service_name'])->first(); + $this->serviceDatabase->getFilesFromServer(); } } public function generateDockerCompose() diff --git a/app/Models/LocalFileVolume.php b/app/Models/LocalFileVolume.php index 9077a36d5..ebd64c6a2 100644 --- a/app/Models/LocalFileVolume.php +++ b/app/Models/LocalFileVolume.php @@ -3,6 +3,7 @@ namespace App\Models; use Illuminate\Database\Eloquent\Factories\HasFactory; +use Illuminate\Support\Str; class LocalFileVolume extends BaseModel { @@ -13,4 +14,40 @@ class LocalFileVolume extends BaseModel { return $this->morphTo('resource'); } + public function saveStorageOnServer(ServiceApplication|ServiceDatabase $service) + { + $workdir = $service->service->workdir(); + $server = $service->service->server; + $commands = collect([ + "mkdir -p $workdir > /dev/null 2>&1 || true", + "cd $workdir" + ]); + $fileVolume = $this; + $path = Str::of(data_get($fileVolume, 'fs_path')); + $content = data_get($fileVolume, 'content'); + if ($path->startsWith('.')) { + $path = $path->after('.'); + $path = $workdir . $path; + } + $isFile = instant_remote_process(["test -f $path && echo OK || echo NOK"], $server); + $isDir = instant_remote_process(["test -d $path && echo OK || echo NOK"], $server); + ray($path); + if ($isFile == 'OK' && $fileVolume->is_directory) { + throw new \Exception("File $path is a file on the server, but you are trying to mark it as a directory. Please delete the file on the server or mark it as directory."); + } else if ($isDir == 'OK' && !$fileVolume->is_directory) { + throw new \Exception("File $path is a directory on the server, but you are trying to mark it as a file. Please delete the directory on the server or mark it as directory."); + } + if (($isFile == 'NOK' && !$fileVolume->is_directory) || $isFile == 'OK') { + $rootDir = Str::of($path)->dirname(); + $commands->push("mkdir -p $rootDir > /dev/null 2>&1 || true"); + $commands->push("touch $path > /dev/null 2>&1 || true"); + if ($content) { + $content = base64_encode($content); + $commands->push("echo '$content' | base64 -d > $path"); + } + } else if ($isDir == 'NOK' && $fileVolume->is_directory) { + $commands->push("mkdir -p $path > /dev/null 2>&1 || true"); + } + return instant_remote_process($commands, $server); + } } diff --git a/app/Models/Service.php b/app/Models/Service.php index cc60a8cad..364e59248 100644 --- a/app/Models/Service.php +++ b/app/Models/Service.php @@ -306,7 +306,7 @@ class Service extends BaseModel ] ); } - $savedService->saveFileVolumes(); + $savedService->getFilesFromServer(); } } diff --git a/app/Models/ServiceApplication.php b/app/Models/ServiceApplication.php index f835ab753..58569b6bd 100644 --- a/app/Models/ServiceApplication.php +++ b/app/Models/ServiceApplication.php @@ -36,8 +36,8 @@ class ServiceApplication extends BaseModel ); } - public function saveFileVolumes() + public function getFilesFromServer() { - saveFileVolumesHelper($this); + getFilesystemVolumesFromServer($this); } } diff --git a/app/Models/ServiceDatabase.php b/app/Models/ServiceDatabase.php index 09b216219..a94bca40a 100644 --- a/app/Models/ServiceDatabase.php +++ b/app/Models/ServiceDatabase.php @@ -26,8 +26,8 @@ class ServiceDatabase extends BaseModel { return $this->morphMany(LocalFileVolume::class, 'resource'); } - public function saveFileVolumes() + public function getFilesFromServer() { - saveFileVolumesHelper($this); + getFilesystemVolumesFromServer($this); } } diff --git a/bootstrap/helpers/services.php b/bootstrap/helpers/services.php index 353e6853b..56a4445e7 100644 --- a/bootstrap/helpers/services.php +++ b/bootstrap/helpers/services.php @@ -64,40 +64,48 @@ function serviceStatus(Service $service) } return 'exited'; } -function saveFileVolumesHelper(ServiceApplication|ServiceDatabase $oneService) +function getFilesystemVolumesFromServer(ServiceApplication|ServiceDatabase $oneService) { + // TODO: make this async try { $workdir = $oneService->service->workdir(); $server = $oneService->service->server; - $applicationFileVolume = $oneService->fileStorages()->get(); + $fileVolumes = $oneService->fileStorages()->get(); $commands = collect([ "mkdir -p $workdir > /dev/null 2>&1 || true", "cd $workdir" ]); - foreach ($applicationFileVolume as $fileVolume) { - $path = Str::of($fileVolume->fs_path); - if ($fileVolume->is_directory) { - $commands->push("test -f $path && rm -f $path > /dev/null 2>&1 || true"); - $commands->push("mkdir -p $path > /dev/null 2>&1 || true"); - continue; + instant_remote_process($commands, $server); + foreach ($fileVolumes as $fileVolume) { + $path = Str::of(data_get($fileVolume, 'fs_path')); + $content = data_get($fileVolume, 'content'); + $isFile = instant_remote_process(["test -f $path && echo OK || echo NOK"], $server); + $isDir = instant_remote_process(["test -d $path && echo OK || echo NOK"], $server); + + if ($isFile == 'OK') { + $filesystemContent = instant_remote_process(["cat $path"], $server); + if (base64_encode($filesystemContent) != base64_encode($content)) { + $fileVolume->content = $filesystemContent; + $fileVolume->save(); + } + } else { + if ($isDir == 'OK') { + $fileVolume->content = null; + $fileVolume->is_directory = true; + $fileVolume->save(); + } else { + $fileVolume->content = null; + $fileVolume->is_directory = false; + $fileVolume->save(); + } } - $content = $fileVolume->content; - $dir = $path->beforeLast('/'); - if ($dir->startsWith('.')) { - $dir = $dir->after('.'); - $dir = $workdir . $dir; - } - $content = base64_encode($content); - $commands->push("test -d $path && rm -rf $path > /dev/null 2>&1 || true"); - $commands->push("mkdir -p $dir > /dev/null 2>&1 || true"); - $commands->push("echo '$content' | base64 -d > $path"); } - return instant_remote_process($commands, $server); } catch (\Throwable $e) { return handleError($e); } } -function updateCompose($resource) { +function updateCompose($resource) +{ try { $name = data_get($resource, 'name'); $dockerComposeRaw = data_get($resource, 'service.docker_compose_raw'); @@ -111,7 +119,7 @@ function updateCompose($resource) { $variableName = "SERVICE_FQDN_" . Str::of($resource->name)->upper(); ray($variableName); $generatedEnv = EnvironmentVariable::where('service_id', $resource->service_id)->where('key', $variableName)->first(); - if ($generatedEnv){ + if ($generatedEnv) { $generatedEnv->value = $resource->fqdn; $generatedEnv->save(); } diff --git a/resources/views/livewire/project/service/file-storage.blade.php b/resources/views/livewire/project/service/file-storage.blade.php index e38ab5f05..43f55bbf6 100644 --- a/resources/views/livewire/project/service/file-storage.blade.php +++ b/resources/views/livewire/project/service/file-storage.blade.php @@ -1,23 +1,25 @@ -
{{ $fileStorage->mount_path }}
+
{{ $fileStorage->fs_path }} -> {{ $fileStorage->mount_path }}
- @if ($fileStorage->is_directory) + {{-- @if ($fileStorage->is_directory) - @else -
+ @else --}} + {{--
- + --}} + @if (!$fileStorage->is_directory) Save @endif + {{-- @endif --}} diff --git a/resources/views/livewire/project/service/show.blade.php b/resources/views/livewire/project/service/show.blade.php index 7db827fcc..e0ab934a2 100644 --- a/resources/views/livewire/project/service/show.blade.php +++ b/resources/views/livewire/project/service/show.blade.php @@ -22,7 +22,7 @@ @if ($serviceApplication->fileStorages()->get()->count() > 0)

Mounted Files (binds)

- @foreach ($serviceApplication->fileStorages()->get() as $fileStorage) + @foreach ($serviceApplication->fileStorages()->get()->sort() as $fileStorage) @endforeach
@@ -39,7 +39,7 @@ @if ($serviceDatabase->fileStorages()->get()->count() > 0)

Mounted Files (binds)

- @foreach ($serviceDatabase->fileStorages()->get() as $fileStorage) + @foreach ($serviceDatabase->fileStorages()->get()->sort() as $fileStorage) @endforeach
From 79fde593a946ed68484cc9be9d0cd2128a87405c Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Sat, 30 Sep 2023 15:08:40 +0200 Subject: [PATCH 09/13] fix: service volume read from filesystem fix: edit compose moved to dialog --- .../Livewire/Project/Service/ComposeModal.php | 19 +++++++++++ .../Livewire/Project/Service/FileStorage.php | 17 +++++----- app/Http/Livewire/Project/Service/Index.php | 34 ++++++++++++------- app/Http/Livewire/Project/Service/Show.php | 30 ++++++++++------ app/Models/LocalFileVolume.php | 2 +- app/Models/Service.php | 6 ++++ bootstrap/helpers/services.php | 17 ++++++---- config/toaster.php | 2 +- .../components/applications/links.blade.php | 2 +- .../views/components/collapsible.blade.php | 2 +- .../views/components/services/links.blade.php | 2 +- .../project/service/compose-modal.blade.php | 26 ++++++++++++++ .../project/service/file-storage.blade.php | 2 +- .../livewire/project/service/index.blade.php | 29 ++-------------- 14 files changed, 118 insertions(+), 72 deletions(-) create mode 100644 app/Http/Livewire/Project/Service/ComposeModal.php create mode 100644 resources/views/livewire/project/service/compose-modal.blade.php diff --git a/app/Http/Livewire/Project/Service/ComposeModal.php b/app/Http/Livewire/Project/Service/ComposeModal.php new file mode 100644 index 000000000..0c9f5e98f --- /dev/null +++ b/app/Http/Livewire/Project/Service/ComposeModal.php @@ -0,0 +1,19 @@ +emit('warning', "Saving new docker compose..."); + $this->emit('saveCompose', $this->raw); + } +} diff --git a/app/Http/Livewire/Project/Service/FileStorage.php b/app/Http/Livewire/Project/Service/FileStorage.php index e4103f600..91ab8e659 100644 --- a/app/Http/Livewire/Project/Service/FileStorage.php +++ b/app/Http/Livewire/Project/Service/FileStorage.php @@ -13,6 +13,7 @@ class FileStorage extends Component public LocalFileVolume $fileStorage; public ServiceApplication|ServiceDatabase $service; public string $fs_path; + public ?string $workdir = null; protected $rules = [ 'fileStorage.is_directory' => 'required', @@ -23,13 +24,13 @@ class FileStorage extends Component public function mount() { $this->service = $this->fileStorage->service; - $this->fs_path = Str::of($this->fileStorage->fs_path)->beforeLast('/'); - $file = Str::of($this->fileStorage->fs_path)->afterLast('/'); - if (Str::of($this->fs_path)->startsWith('.')) { - $this->fs_path = Str::of($this->fs_path)->after('.'); - $this->fs_path = $this->service->service->workdir() . $this->fs_path . "/" . $file; - } - + if (Str::of($this->fileStorage->fs_path)->startsWith('.')) { + $this->workdir = $this->service->service->workdir(); + $this->fs_path = Str::of($this->fileStorage->fs_path)->after('.'); + } else { + $this->workdir = null; + $this->fs_path = $this->fileStorage->fs_path; + } } public function submit() { @@ -41,8 +42,6 @@ class FileStorage extends Component } $this->fileStorage->save(); $this->fileStorage->saveStorageOnServer($this->service); - // ray($this->fileStorage); - // $this->service->saveFileVolumes(); $this->emit('success', 'File updated successfully.'); } catch (\Throwable $e) { $this->fileStorage->setRawAttributes($original); diff --git a/app/Http/Livewire/Project/Service/Index.php b/app/Http/Livewire/Project/Service/Index.php index c3c6c57c8..18029fb73 100644 --- a/app/Http/Livewire/Project/Service/Index.php +++ b/app/Http/Livewire/Project/Service/Index.php @@ -4,7 +4,6 @@ namespace App\Http\Livewire\Project\Service; use App\Jobs\ContainerStatusJob; use App\Models\Service; -use DanHarrin\LivewireRateLimiting\WithRateLimiting; use Livewire\Component; class Index extends Component @@ -20,7 +19,25 @@ class Index extends Component 'service.name' => 'required', 'service.description' => 'nullable', ]; - public function checkStatus() { + protected $listeners = ["saveCompose"]; + public function render() + { + return view('livewire.project.service.index'); + } + public function mount() + { + $this->parameters = get_route_parameters(); + $this->query = request()->query(); + $this->service = Service::whereUuid($this->parameters['service_uuid'])->firstOrFail(); + $this->refreshStack(); + } + public function saveCompose($raw) + { + $this->service->docker_compose_raw = $raw; + $this->submit(); + } + public function checkStatus() + { dispatch_sync(new ContainerStatusJob($this->service->server)); $this->refreshStack(); } @@ -35,17 +52,8 @@ class Index extends Component $database->refresh(); }); } - public function mount() - { - $this->parameters = get_route_parameters(); - $this->query = request()->query(); - $this->service = Service::whereUuid($this->parameters['service_uuid'])->firstOrFail(); - $this->refreshStack(); - } - public function render() - { - return view('livewire.project.service.index'); - } + + public function submit() { try { diff --git a/app/Http/Livewire/Project/Service/Show.php b/app/Http/Livewire/Project/Service/Show.php index 6f48a1389..ebeb24ebc 100644 --- a/app/Http/Livewire/Project/Service/Show.php +++ b/app/Http/Livewire/Project/Service/Show.php @@ -20,18 +20,26 @@ class Show extends Component public function mount() { - $this->services = collect([]); - $this->parameters = get_route_parameters(); - $this->query = request()->query(); - $this->service = Service::whereUuid($this->parameters['service_uuid'])->firstOrFail(); - $service = $this->service->applications()->whereName($this->parameters['service_name'])->first(); - if ($service) { - $this->serviceApplication = $service; - $this->serviceApplication->getFilesFromServer(); - } else { - $this->serviceDatabase = $this->service->databases()->whereName($this->parameters['service_name'])->first(); - $this->serviceDatabase->getFilesFromServer(); + try { + $this->services = collect([]); + $this->parameters = get_route_parameters(); + $this->query = request()->query(); + $this->service = Service::whereUuid($this->parameters['service_uuid'])->firstOrFail(); + $service = $this->service->applications()->whereName($this->parameters['service_name'])->first(); + if ($service) { + $this->serviceApplication = $service; + $this->serviceApplication->getFilesFromServer(); + } else { + $this->serviceDatabase = $this->service->databases()->whereName($this->parameters['service_name'])->first(); + $this->serviceDatabase->getFilesFromServer(); + } + if (is_null($service)) { + throw new \Exception("Service not found."); + } + } catch(\Throwable $e) { + return handleError($e, $this); } + } public function generateDockerCompose() { diff --git a/app/Models/LocalFileVolume.php b/app/Models/LocalFileVolume.php index ebd64c6a2..b45a868aa 100644 --- a/app/Models/LocalFileVolume.php +++ b/app/Models/LocalFileVolume.php @@ -31,7 +31,7 @@ class LocalFileVolume extends BaseModel } $isFile = instant_remote_process(["test -f $path && echo OK || echo NOK"], $server); $isDir = instant_remote_process(["test -d $path && echo OK || echo NOK"], $server); - ray($path); + ray($isFile); if ($isFile == 'OK' && $fileVolume->is_directory) { throw new \Exception("File $path is a file on the server, but you are trying to mark it as a directory. Please delete the file on the server or mark it as directory."); } else if ($isDir == 'OK' && !$fileVolume->is_directory) { diff --git a/app/Models/Service.php b/app/Models/Service.php index 364e59248..f58529c48 100644 --- a/app/Models/Service.php +++ b/app/Models/Service.php @@ -196,6 +196,12 @@ class Service extends BaseModel } } + // Check if image changed + if ($savedService->image !== $image) { + $savedService->image = $image; + $savedService->save(); + } + // Collect/create/update networks if ($serviceNetworks->count() > 0) { foreach ($serviceNetworks as $networkName => $networkDetails) { diff --git a/bootstrap/helpers/services.php b/bootstrap/helpers/services.php index 56a4445e7..7977d4d26 100644 --- a/bootstrap/helpers/services.php +++ b/bootstrap/helpers/services.php @@ -73,17 +73,22 @@ function getFilesystemVolumesFromServer(ServiceApplication|ServiceDatabase $oneS $fileVolumes = $oneService->fileStorages()->get(); $commands = collect([ "mkdir -p $workdir > /dev/null 2>&1 || true", - "cd $workdir" + "cd " ]); instant_remote_process($commands, $server); foreach ($fileVolumes as $fileVolume) { $path = Str::of(data_get($fileVolume, 'fs_path')); $content = data_get($fileVolume, 'content'); - $isFile = instant_remote_process(["test -f $path && echo OK || echo NOK"], $server); - $isDir = instant_remote_process(["test -d $path && echo OK || echo NOK"], $server); - - if ($isFile == 'OK') { - $filesystemContent = instant_remote_process(["cat $path"], $server); + if ($path->startsWith('.')) { + $path = $path->after('.'); + $fileLocation = $workdir . $path; + } else { + $fileLocation = $path; + } + $isFile = instant_remote_process(["test -f $fileLocation && echo OK || echo NOK"], $server); + $isDir = instant_remote_process(["test -d $fileLocation && echo OK || echo NOK"], $server); + if ($isFile == 'OK' && !$fileVolume->is_directory) { + $filesystemContent = instant_remote_process(["cat $fileLocation"], $server); if (base64_encode($filesystemContent) != base64_encode($content)) { $fileVolume->content = $filesystemContent; $fileVolume->save(); diff --git a/config/toaster.php b/config/toaster.php index 13c9200a8..43565a9c6 100644 --- a/config/toaster.php +++ b/config/toaster.php @@ -30,7 +30,7 @@ return [ * * Minimum: 3000 (in milliseconds) */ - 'duration' => 3000, + 'duration' => 5000, /** * The horizontal position of each toast. diff --git a/resources/views/components/applications/links.blade.php b/resources/views/components/applications/links.blade.php index c9f9b0de6..1c716a3af 100644 --- a/resources/views/components/applications/links.blade.php +++ b/resources/views/components/applications/links.blade.php @@ -3,7 +3,7 @@ -