From e971094508c865faffc19dd543173d621e56797c Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Mon, 14 Oct 2024 20:32:24 +0200 Subject: [PATCH 001/126] new much larger default docker address pool --- other/nightly/install.sh | 10 ++++++++-- scripts/install.sh | 10 ++++++++-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/other/nightly/install.sh b/other/nightly/install.sh index 04faf50ea..c4bcb761b 100755 --- a/other/nightly/install.sh +++ b/other/nightly/install.sh @@ -287,7 +287,10 @@ test -s /etc/docker/daemon.json && cp /etc/docker/daemon.json /etc/docker/daemon "log-opts": { "max-size": "10m", "max-file": "3" - } + }, + "default-address-pools": [ + {"base":"10.0.0.0/8","size":24} + ] } EOL cat >/etc/docker/daemon.json.coolify </etc/docker/daemon.json.coolify </etc/docker/daemon.json.coolify </etc/docker/daemon.json.coolify < Date: Fri, 18 Oct 2024 13:48:41 +0200 Subject: [PATCH 002/126] fix project deletion and refactor some code --- app/Http/Controllers/Api/ProjectController.php | 2 +- app/Livewire/Project/DeleteProject.php | 11 ++++++----- app/Models/Project.php | 13 +++++++++++-- resources/views/livewire/project/edit.blade.php | 2 +- resources/views/livewire/project/show.blade.php | 2 +- 5 files changed, 20 insertions(+), 10 deletions(-) diff --git a/app/Http/Controllers/Api/ProjectController.php b/app/Http/Controllers/Api/ProjectController.php index f1958de2c..066d74e76 100644 --- a/app/Http/Controllers/Api/ProjectController.php +++ b/app/Http/Controllers/Api/ProjectController.php @@ -423,7 +423,7 @@ class ProjectController extends Controller if (! $project) { return response()->json(['message' => 'Project not found.'], 404); } - if ($project->resource_count() > 0) { + if (! $project->isEmpty()) { return response()->json(['message' => 'Project has resources, so it cannot be deleted.'], 400); } diff --git a/app/Livewire/Project/DeleteProject.php b/app/Livewire/Project/DeleteProject.php index 360fad10a..f320a19b0 100644 --- a/app/Livewire/Project/DeleteProject.php +++ b/app/Livewire/Project/DeleteProject.php @@ -27,11 +27,12 @@ class DeleteProject extends Component 'project_id' => 'required|int', ]); $project = Project::findOrFail($this->project_id); - if ($project->applications->count() > 0) { - return $this->dispatch('error', 'Project has resources defined, please delete them first.'); - } - $project->delete(); + if ($project->isEmpty()) { + $project->delete(); - return redirect()->route('project.index'); + return redirect()->route('project.index'); + } + + return $this->dispatch('error', "Project {$project->name} has resources defined, please delete them first."); } } diff --git a/app/Models/Project.php b/app/Models/Project.php index 5a9dd964a..16ce403eb 100644 --- a/app/Models/Project.php +++ b/app/Models/Project.php @@ -123,9 +123,18 @@ class Project extends BaseModel return $this->hasManyThrough(StandaloneMariadb::class, Environment::class); } - public function resource_count() + public function isEmpty() { - return $this->applications()->count() + $this->postgresqls()->count() + $this->redis()->count() + $this->mongodbs()->count() + $this->mysqls()->count() + $this->mariadbs()->count() + $this->keydbs()->count() + $this->dragonflies()->count() + $this->clickhouses()->count() + $this->services()->count(); + return $this->applications()->count() == 0 && + $this->redis()->count() == 0 && + $this->postgresqls()->count() == 0 && + $this->mysqls()->count() == 0 && + $this->keydbs()->count() == 0 && + $this->dragonflies()->count() == 0 && + $this->clickhouses()->count() == 0 && + $this->mariadbs()->count() == 0 && + $this->mongodbs()->count() == 0 && + $this->services()->count() == 0; } public function databases() diff --git a/resources/views/livewire/project/edit.blade.php b/resources/views/livewire/project/edit.blade.php index ec9304da9..de837eb65 100644 --- a/resources/views/livewire/project/edit.blade.php +++ b/resources/views/livewire/project/edit.blade.php @@ -7,7 +7,7 @@

Project: {{ data_get($project, 'name') }}

Save - +
Edit project details here.
diff --git a/resources/views/livewire/project/show.blade.php b/resources/views/livewire/project/show.blade.php index f4f5299f9..014a34faf 100644 --- a/resources/views/livewire/project/show.blade.php +++ b/resources/views/livewire/project/show.blade.php @@ -7,7 +7,7 @@ - +
{{ $project->name }}.
From e52139f4368b0e15ec275733094b4d4fb39e94a2 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Fri, 18 Oct 2024 13:49:17 +0200 Subject: [PATCH 003/126] chore better error message when deleting an environment --- app/Livewire/Project/DeleteEnvironment.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/Livewire/Project/DeleteEnvironment.php b/app/Livewire/Project/DeleteEnvironment.php index e01741770..9c1c358b8 100644 --- a/app/Livewire/Project/DeleteEnvironment.php +++ b/app/Livewire/Project/DeleteEnvironment.php @@ -33,6 +33,6 @@ class DeleteEnvironment extends Component return redirect()->route('project.show', ['project_uuid' => $this->parameters['project_uuid']]); } - return $this->dispatch('error', 'Environment has defined resources, please delete them first.'); + return $this->dispatch('error', "Environment {$environment->name} has defined resources, please delete them first."); } } From fe81c3e2947add5b4726a56b28df869957385da3 Mon Sep 17 00:00:00 2001 From: RayBB Date: Fri, 1 Nov 2024 18:26:33 -0500 Subject: [PATCH 004/126] add readeck service template --- public/svgs/readeck.svg | 9 +++++++++ templates/compose/readeck.yaml | 24 ++++++++++++++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 public/svgs/readeck.svg create mode 100644 templates/compose/readeck.yaml diff --git a/public/svgs/readeck.svg b/public/svgs/readeck.svg new file mode 100644 index 000000000..07f6e6157 --- /dev/null +++ b/public/svgs/readeck.svg @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/templates/compose/readeck.yaml b/templates/compose/readeck.yaml new file mode 100644 index 000000000..2a4907974 --- /dev/null +++ b/templates/compose/readeck.yaml @@ -0,0 +1,24 @@ +# documentation: https://readeck.org/en/docs/ +# slogan: Simple web application that lets you save the precious readable content of web pages you like and want to keep forever. +# tags: read-it-later,pocket-alternative,omnivore-alternative,instapaper-alternative +# logo: svgs/readeck.svg +# port: 8000 + +services: + readeck: + image: 'codeberg.org/readeck/readeck:latest' + environment: + - SERVICE_FQDN_READECK + - READECK_USE_X_FORWARDED=true + volumes: + - 'readeck-data:/readeck' + healthcheck: + test: + - CMD + - /bin/readeck + - healthcheck + - '-config' + - config.toml + interval: 30s + timeout: 2s + retries: 3 From c2e076be54cd3cf337e30ed4281323fa42d6be7c Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Sat, 2 Nov 2024 18:32:45 +0100 Subject: [PATCH 005/126] add min process for horizon --- config/horizon.php | 2 ++ config/telescope.php | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/config/horizon.php b/config/horizon.php index 939d74883..6086b30da 100644 --- a/config/horizon.php +++ b/config/horizon.php @@ -197,6 +197,7 @@ return [ 'production' => [ 's6' => [ 'autoScalingStrategy' => 'size', + 'minProcesses' => env('HORIZON_MIN_PROCESSES', 1), 'maxProcesses' => env('HORIZON_MAX_PROCESSES', 6), 'balanceMaxShift' => env('HORIZON_BALANCE_MAX_SHIFT', 1), 'balanceCooldown' => env('HORIZON_BALANCE_COOLDOWN', 1), @@ -206,6 +207,7 @@ return [ 'local' => [ 's6' => [ 'autoScalingStrategy' => 'size', + 'minProcesses' => env('HORIZON_MIN_PROCESSES', 1), 'maxProcesses' => env('HORIZON_MAX_PROCESSES', 6), 'balanceMaxShift' => env('HORIZON_BALANCE_MAX_SHIFT', 1), 'balanceCooldown' => env('HORIZON_BALANCE_COOLDOWN', 1), diff --git a/config/telescope.php b/config/telescope.php index 2444ee8cf..c940bec8a 100644 --- a/config/telescope.php +++ b/config/telescope.php @@ -76,8 +76,8 @@ return [ */ 'queue' => [ - 'connection' => env('TELESCOPE_QUEUE_CONNECTION', null), - 'queue' => env('TELESCOPE_QUEUE', null), + 'connection' => env('TELESCOPE_QUEUE_CONNECTION', 'redis'), + 'queue' => env('TELESCOPE_QUEUE', 'default'), ], /* From 2eab9296cf31b566c0fad814c6959f27d4ad66ad Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Sat, 2 Nov 2024 18:33:23 +0100 Subject: [PATCH 006/126] check server functionality before status --- app/Models/Server.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/Models/Server.php b/app/Models/Server.php index 9527e8820..87800bbe9 100644 --- a/app/Models/Server.php +++ b/app/Models/Server.php @@ -976,10 +976,10 @@ $schema://$host { public function serverStatus(): bool { - if ($this->status() === false) { + if ($this->isFunctional() === false) { return false; } - if ($this->isFunctional() === false) { + if ($this->status() === false) { return false; } From ca7c214775931c872b6daf01dfe0a3d645ffba27 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Sun, 3 Nov 2024 09:02:14 +0100 Subject: [PATCH 007/126] fix: new way to update container statuses --- app/Actions/Docker/GetContainersStatus.php | 2 +- app/Actions/Server/ResourcesCheck.php | 41 +++ app/Actions/Server/ServerCheck.php | 269 ++++++++++++++++++ app/Console/Commands/Weird.php | 58 ++++ app/Console/Kernel.php | 19 +- app/Jobs/ApplicationDeploymentJob.php | 2 +- app/Jobs/ServerCheckNewJob.php | 32 +++ app/Jobs/ServerStorageCheckJob.php | 4 +- app/Models/Application.php | 33 ++- app/Models/ApplicationPreview.php | 5 + app/Models/Server.php | 24 +- app/Models/ServiceApplication.php | 5 + app/Models/ServiceDatabase.php | 5 + app/Models/StandaloneClickhouse.php | 5 + app/Models/StandaloneDragonfly.php | 5 + app/Models/StandaloneKeydb.php | 5 + app/Models/StandaloneMariadb.php | 5 + app/Models/StandaloneMongodb.php | 5 + app/Models/StandaloneMysql.php | 5 + app/Models/StandalonePostgresql.php | 5 + app/Models/StandaloneRedis.php | 5 + ...213214_add_last_online_at_to_resources.php | 96 +++++++ routes/api.php | 2 + 23 files changed, 602 insertions(+), 35 deletions(-) create mode 100644 app/Actions/Server/ResourcesCheck.php create mode 100644 app/Actions/Server/ServerCheck.php create mode 100644 app/Console/Commands/Weird.php create mode 100644 app/Jobs/ServerCheckNewJob.php create mode 100644 database/migrations/2024_11_02_213214_add_last_online_at_to_resources.php diff --git a/app/Actions/Docker/GetContainersStatus.php b/app/Actions/Docker/GetContainersStatus.php index 6c1a53f3a..95c22efc1 100644 --- a/app/Actions/Docker/GetContainersStatus.php +++ b/app/Actions/Docker/GetContainersStatus.php @@ -30,7 +30,7 @@ class GetContainersStatus $this->containerReplicates = $containerReplicates; $this->server = $server; if (! $this->server->isFunctional()) { - return 'Server is not ready.'; + return 'Server is not functional.'; } $this->applications = $this->server->applications(); $skip_these_applications = collect([]); diff --git a/app/Actions/Server/ResourcesCheck.php b/app/Actions/Server/ResourcesCheck.php new file mode 100644 index 000000000..e6b90ba38 --- /dev/null +++ b/app/Actions/Server/ResourcesCheck.php @@ -0,0 +1,41 @@ +subSeconds($seconds))->update(['status' => 'exited']); + ServiceApplication::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']); + ServiceDatabase::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']); + StandalonePostgresql::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']); + StandaloneRedis::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']); + StandaloneMongodb::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']); + StandaloneMysql::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']); + StandaloneMariadb::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']); + StandaloneKeydb::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']); + StandaloneDragonfly::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']); + StandaloneClickhouse::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']); + } catch (\Throwable $e) { + return handleError($e); + } + } +} diff --git a/app/Actions/Server/ServerCheck.php b/app/Actions/Server/ServerCheck.php new file mode 100644 index 000000000..51f6511c8 --- /dev/null +++ b/app/Actions/Server/ServerCheck.php @@ -0,0 +1,269 @@ +server = $server; + try { + if ($this->server->isFunctional() === false) { + return 'Server is not functional.'; + } + + if (! $this->server->isSwarmWorker() && ! $this->server->isBuildServer()) { + + if (isset($data)) { + $data = collect($data); + + $this->server->sentinelHeartbeat(); + + $this->containers = collect(data_get($data, 'containers')); + + $filesystemUsageRoot = data_get($data, 'filesystem_usage_root.used_percentage'); + ServerStorageCheckJob::dispatch($this->server, $filesystemUsageRoot); + + $containerReplicates = null; + $this->isSentinel = true; + + } else { + ['containers' => $this->containers, 'containerReplicates' => $containerReplicates] = $this->server->getContainers(); + ServerStorageCheckJob::dispatch($this->server); + } + + if (is_null($this->containers)) { + return 'No containers found.'; + } + + if (isset($containerReplicates)) { + foreach ($containerReplicates as $containerReplica) { + $name = data_get($containerReplica, 'Name'); + $this->containers = $this->containers->map(function ($container) use ($name, $containerReplica) { + if (data_get($container, 'Spec.Name') === $name) { + $replicas = data_get($containerReplica, 'Replicas'); + $running = str($replicas)->explode('/')[0]; + $total = str($replicas)->explode('/')[1]; + if ($running === $total) { + data_set($container, 'State.Status', 'running'); + data_set($container, 'State.Health.Status', 'healthy'); + } else { + data_set($container, 'State.Status', 'starting'); + data_set($container, 'State.Health.Status', 'unhealthy'); + } + } + + return $container; + }); + } + } + $this->checkContainers(); + + if ($this->server->isSentinelEnabled() && $this->isSentinel === false) { + CheckAndStartSentinelJob::dispatch($this->server); + } + + if ($this->server->isLogDrainEnabled()) { + $this->checkLogDrainContainer(); + } + + if ($this->server->proxySet() && ! $this->server->proxy->force_stop) { + $foundProxyContainer = $this->containers->filter(function ($value, $key) { + if ($this->server->isSwarm()) { + return data_get($value, 'Spec.Name') === 'coolify-proxy_traefik'; + } else { + return data_get($value, 'Name') === '/coolify-proxy'; + } + })->first(); + if (! $foundProxyContainer) { + try { + $shouldStart = CheckProxy::run($this->server); + if ($shouldStart) { + StartProxy::run($this->server, false); + $this->server->team?->notify(new ContainerRestarted('coolify-proxy', $this->server)); + } + } catch (\Throwable $e) { + } + } else { + $this->server->proxy->status = data_get($foundProxyContainer, 'State.Status'); + $this->server->save(); + $connectProxyToDockerNetworks = connectProxyToNetworks($this->server); + instant_remote_process($connectProxyToDockerNetworks, $this->server, false); + } + } + } + } catch (\Throwable $e) { + return handleError($e); + } + } + + private function checkLogDrainContainer() + { + $foundLogDrainContainer = $this->containers->filter(function ($value, $key) { + return data_get($value, 'Name') === '/coolify-log-drain'; + })->first(); + if ($foundLogDrainContainer) { + $status = data_get($foundLogDrainContainer, 'State.Status'); + if ($status !== 'running') { + StartLogDrain::dispatch($this->server); + } + } else { + StartLogDrain::dispatch($this->server); + } + } + + private function checkContainers() + { + foreach ($this->containers as $container) { + if ($this->isSentinel) { + $labels = Arr::undot(data_get($container, 'labels')); + } else { + if ($this->server->isSwarm()) { + $labels = Arr::undot(data_get($container, 'Spec.Labels')); + } else { + $labels = Arr::undot(data_get($container, 'Config.Labels')); + } + + } + $managed = data_get($labels, 'coolify.managed'); + if (! $managed) { + continue; + } + $uuid = data_get($labels, 'coolify.name'); + if (! $uuid) { + $uuid = data_get($labels, 'com.docker.compose.service'); + } + + if ($this->isSentinel) { + $containerStatus = data_get($container, 'state'); + $containerHealth = data_get($container, 'health_status'); + } else { + $containerStatus = data_get($container, 'State.Status'); + $containerHealth = data_get($container, 'State.Health.Status', 'unhealthy'); + } + $containerStatus = "$containerStatus ($containerHealth)"; + + $applicationId = data_get($labels, 'coolify.applicationId'); + $serviceId = data_get($labels, 'coolify.serviceId'); + $databaseId = data_get($labels, 'coolify.databaseId'); + $pullRequestId = data_get($labels, 'coolify.pullRequestId'); + + if ($applicationId) { + // Application + if ($pullRequestId != 0) { + if (str($applicationId)->contains('-')) { + $applicationId = str($applicationId)->before('-'); + } + $preview = ApplicationPreview::where('application_id', $applicationId)->where('pull_request_id', $pullRequestId)->first(); + if ($preview) { + $preview->update(['status' => $containerStatus]); + } + } else { + $application = Application::where('id', $applicationId)->first(); + if ($application) { + $application->update([ + 'status' => $containerStatus, + 'last_online_at' => now(), + ]); + } + } + } elseif (isset($serviceId)) { + // Service + $subType = data_get($labels, 'coolify.service.subType'); + $subId = data_get($labels, 'coolify.service.subId'); + $service = Service::where('id', $serviceId)->first(); + if (! $service) { + continue; + } + if ($subType === 'application') { + $service = ServiceApplication::where('id', $subId)->first(); + } else { + $service = ServiceDatabase::where('id', $subId)->first(); + } + if ($service) { + $service->update([ + 'status' => $containerStatus, + 'last_online_at' => now(), + ]); + if ($subType === 'database') { + $isPublic = data_get($service, 'is_public'); + if ($isPublic) { + $foundTcpProxy = $this->containers->filter(function ($value, $key) use ($uuid) { + if ($this->isSentinel) { + return data_get($value, 'name') === $uuid.'-proxy'; + } else { + + if ($this->server->isSwarm()) { + return data_get($value, 'Spec.Name') === "coolify-proxy_$uuid"; + } else { + return data_get($value, 'Name') === "/$uuid-proxy"; + } + } + })->first(); + if (! $foundTcpProxy) { + StartDatabaseProxy::run($service); + } + } + } + } + } else { + // Database + if (is_null($this->databases)) { + $this->databases = $this->server->databases(); + } + $database = $this->databases->where('uuid', $uuid)->first(); + if ($database) { + $database->update([ + 'status' => $containerStatus, + 'last_online_at' => now(), + ]); + + $isPublic = data_get($database, 'is_public'); + if ($isPublic) { + $foundTcpProxy = $this->containers->filter(function ($value, $key) use ($uuid) { + if ($this->isSentinel) { + return data_get($value, 'name') === $uuid.'-proxy'; + } else { + if ($this->server->isSwarm()) { + return data_get($value, 'Spec.Name') === "coolify-proxy_$uuid"; + } else { + + return data_get($value, 'Name') === "/$uuid-proxy"; + } + } + })->first(); + if (! $foundTcpProxy) { + StartDatabaseProxy::run($database); + $this->server->team?->notify(new ContainerRestarted("TCP Proxy for {$database->name}", $this->server)); + } + } + } + } + } + } +} diff --git a/app/Console/Commands/Weird.php b/app/Console/Commands/Weird.php new file mode 100644 index 000000000..e471a5f96 --- /dev/null +++ b/app/Console/Commands/Weird.php @@ -0,0 +1,58 @@ +error('This command can only be run in development mode'); + + return; + } + $run = $this->option('run'); + if ($run) { + $servers = Server::all(); + foreach ($servers as $server) { + ServerCheck::dispatch($server); + } + + return; + } + $number = $this->option('number'); + for ($i = 0; $i < $number; $i++) { + $uuid = Str::uuid(); + $server = Server::create([ + 'name' => 'localhost-'.$uuid, + 'description' => 'This is a test docker container in development mode', + 'ip' => 'coolify-testing-host', + 'team_id' => 0, + 'private_key_id' => 1, + 'proxy' => [ + 'type' => ProxyTypes::NONE->value, + 'status' => ProxyStatus::EXITED->value, + ], + ]); + $server->settings->update([ + 'is_usable' => true, + 'is_reachable' => true, + ]); + } + } catch (\Exception $e) { + $this->error($e->getMessage()); + } + } +} diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index 7533e8932..9d1a2e40f 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -2,6 +2,7 @@ namespace App\Console; +use App\Actions\Server\ResourcesCheck; use App\Jobs\CheckAndStartSentinelJob; use App\Jobs\CheckForUpdatesJob; use App\Jobs\CheckHelperImageJob; @@ -41,13 +42,16 @@ class Kernel extends ConsoleKernel // Instance Jobs $schedule->command('horizon:snapshot')->everyMinute(); $schedule->job(new CleanupInstanceStuffsJob)->everyMinute()->onOneServer(); + $schedule->job(new CheckHelperImageJob)->everyFiveMinutes()->onOneServer(); + // Server Jobs - $this->checkScheduledBackups($schedule); $this->checkResources($schedule); + + $this->checkScheduledBackups($schedule); $this->checkScheduledTasks($schedule); + $schedule->command('uploads:clear')->everyTwoMinutes(); - $schedule->job(new CheckHelperImageJob)->everyFiveMinutes()->onOneServer(); } else { // Instance Jobs $schedule->command('horizon:snapshot')->everyFiveMinutes(); @@ -57,9 +61,11 @@ class Kernel extends ConsoleKernel $this->scheduleUpdates($schedule); // Server Jobs - $this->checkScheduledBackups($schedule); $this->checkResources($schedule); + $this->pullImages($schedule); + + $this->checkScheduledBackups($schedule); $this->checkScheduledTasks($schedule); $schedule->command('cleanup:database --yes')->daily(); @@ -109,12 +115,17 @@ class Kernel extends ConsoleKernel } else { $servers = $this->allServers; } + // $schedule->job(new ResourcesCheck)->everyMinute()->onOneServer(); + foreach ($servers as $server) { - $lastSentinelUpdate = $server->sentinel_updated_at; $serverTimezone = $server->settings->server_timezone; + + // Sentinel check + $lastSentinelUpdate = $server->sentinel_updated_at; if (Carbon::parse($lastSentinelUpdate)->isBefore(now()->subSeconds($server->waitBeforeDoingSshCheck()))) { $schedule->job(new ServerCheckJob($server))->everyMinute()->onOneServer(); } + if ($server->settings->force_docker_cleanup) { $schedule->job(new DockerCleanupJob($server))->cron($server->settings->docker_cleanup_frequency)->timezone($serverTimezone)->onOneServer(); } else { diff --git a/app/Jobs/ApplicationDeploymentJob.php b/app/Jobs/ApplicationDeploymentJob.php index c80b4a7db..a0fdd0e97 100644 --- a/app/Jobs/ApplicationDeploymentJob.php +++ b/app/Jobs/ApplicationDeploymentJob.php @@ -230,7 +230,7 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue $this->application_deployment_queue->update([ 'status' => ApplicationDeploymentStatus::IN_PROGRESS->value, ]); - if (! $this->server->isFunctional()) { + if ($this->server->isFunctional() === false) { $this->application_deployment_queue->addLogEntry('Server is not functional.'); $this->fail('Server is not functional.'); diff --git a/app/Jobs/ServerCheckNewJob.php b/app/Jobs/ServerCheckNewJob.php new file mode 100644 index 000000000..8455c9617 --- /dev/null +++ b/app/Jobs/ServerCheckNewJob.php @@ -0,0 +1,32 @@ +server); + } catch (\Throwable $e) { + return handleError($e); + } + } +} diff --git a/app/Jobs/ServerStorageCheckJob.php b/app/Jobs/ServerStorageCheckJob.php index cc838c77f..0723ffcee 100644 --- a/app/Jobs/ServerStorageCheckJob.php +++ b/app/Jobs/ServerStorageCheckJob.php @@ -30,8 +30,8 @@ class ServerStorageCheckJob implements ShouldBeEncrypted, ShouldQueue public function handle() { try { - if (! $this->server->isFunctional()) { - return 'Server is not ready.'; + if ($this->server->isFunctional() === false) { + return 'Server is not functional.'; } $team = data_get($this->server, 'team'); $serverDiskUsageNotificationThreshold = data_get($this->server, 'settings.server_disk_usage_notification_threshold'); diff --git a/app/Models/Application.php b/app/Models/Application.php index cd7bb533e..91abf2e3a 100644 --- a/app/Models/Application.php +++ b/app/Models/Application.php @@ -117,14 +117,31 @@ class Application extends BaseModel if ($application->fqdn === '') { $application->fqdn = null; } - $application->forceFill([ - 'fqdn' => $application->fqdn, - 'install_command' => str($application->install_command)->trim(), - 'build_command' => str($application->build_command)->trim(), - 'start_command' => str($application->start_command)->trim(), - 'base_directory' => str($application->base_directory)->trim(), - 'publish_directory' => str($application->publish_directory)->trim(), - ]); + $payload = []; + if ($application->isDirty('fqdn')) { + $payload['fqdn'] = $application->fqdn; + } + if ($application->isDirty('install_command')) { + $payload['install_command'] = str($application->install_command)->trim(); + } + if ($application->isDirty('build_command')) { + $payload['build_command'] = str($application->build_command)->trim(); + } + if ($application->isDirty('start_command')) { + $payload['start_command'] = str($application->start_command)->trim(); + } + if ($application->isDirty('base_directory')) { + $payload['base_directory'] = str($application->base_directory)->trim(); + } + if ($application->isDirty('publish_directory')) { + $payload['publish_directory'] = str($application->publish_directory)->trim(); + } + if ($application->isDirty('status')) { + $payload['last_online_at'] = now(); + } + if (count($payload) > 0) { + $application->forceFill($payload); + } }); static::created(function ($application) { ApplicationSetting::create([ diff --git a/app/Models/ApplicationPreview.php b/app/Models/ApplicationPreview.php index 04a0ab27e..bf2bf05bf 100644 --- a/app/Models/ApplicationPreview.php +++ b/app/Models/ApplicationPreview.php @@ -28,6 +28,11 @@ class ApplicationPreview extends BaseModel }); } }); + static::saving(function ($preview) { + if ($preview->isDirty('status')) { + $preview->forceFill(['last_online_at' => now()]); + } + }); } public static function findPreviewByApplicationAndPullId(int $application_id, int $pull_request_id) diff --git a/app/Models/Server.php b/app/Models/Server.php index 87800bbe9..1c6651017 100644 --- a/app/Models/Server.php +++ b/app/Models/Server.php @@ -507,20 +507,6 @@ $schema://$host { return Server::whereTeamId($teamId)->whereRelation('settings', 'is_reachable', true)->whereRelation('settings', 'is_build_server', true); } - public function skipServer() - { - if ($this->ip === '1.2.3.4') { - // ray('skipping 1.2.3.4'); - return true; - } - if ($this->settings->force_disabled === true) { - // ray('force_disabled'); - return true; - } - - return false; - } - public function isForceDisabled() { return $this->settings->force_disabled; @@ -691,7 +677,7 @@ $schema://$host { } } } else { - $containers = instant_remote_process(["docker container inspect $(docker container ls -q) --format '{{json .}}'"], $this, false); + $containers = instant_remote_process(["docker container inspect $(docker container ls -aq) --format '{{json .}}'"], $this, false); $containers = format_docker_command_output_to_json($containers); $containerReplicates = collect([]); } @@ -919,9 +905,9 @@ $schema://$host { public function isFunctional() { - $isFunctional = $this->settings->is_reachable && $this->settings->is_usable && ! $this->settings->force_disabled; + $isFunctional = $this->settings->is_reachable && $this->settings->is_usable && $this->settings->force_disabled === false && $this->ip !== '1.2.3.4'; - if (! $isFunctional) { + if ($isFunctional === false) { Storage::disk('ssh-mux')->delete($this->muxFilename()); } @@ -988,7 +974,7 @@ $schema://$host { public function status(): bool { - if ($this->skipServer()) { + if ($this->isFunctional() === false) { return false; } ['uptime' => $uptime] = $this->validateConnection(false); @@ -1058,7 +1044,7 @@ $schema://$host { { config()->set('constants.ssh.mux_enabled', ! $isManualCheck); - if ($this->skipServer()) { + if ($this->isFunctional() === false) { return ['uptime' => false, 'error' => 'Server skipped.']; } try { diff --git a/app/Models/ServiceApplication.php b/app/Models/ServiceApplication.php index 0e79e1e2e..305913068 100644 --- a/app/Models/ServiceApplication.php +++ b/app/Models/ServiceApplication.php @@ -19,6 +19,11 @@ class ServiceApplication extends BaseModel $service->persistentStorages()->delete(); $service->fileStorages()->delete(); }); + static::saving(function ($service) { + if ($service->isDirty('status')) { + $service->forceFill(['last_online_at' => now()]); + } + }); } public function restart() diff --git a/app/Models/ServiceDatabase.php b/app/Models/ServiceDatabase.php index 927527118..6641509dd 100644 --- a/app/Models/ServiceDatabase.php +++ b/app/Models/ServiceDatabase.php @@ -17,6 +17,11 @@ class ServiceDatabase extends BaseModel $service->persistentStorages()->delete(); $service->fileStorages()->delete(); }); + static::saving(function ($service) { + if ($service->isDirty('status')) { + $service->forceFill(['last_online_at' => now()]); + } + }); } public function restart() diff --git a/app/Models/StandaloneClickhouse.php b/app/Models/StandaloneClickhouse.php index fa27d50d8..6d66c6854 100644 --- a/app/Models/StandaloneClickhouse.php +++ b/app/Models/StandaloneClickhouse.php @@ -38,6 +38,11 @@ class StandaloneClickhouse extends BaseModel $database->environment_variables()->delete(); $database->tags()->detach(); }); + static::saving(function ($database) { + if ($database->isDirty('status')) { + $database->forceFill(['last_online_at' => now()]); + } + }); } protected function serverStatus(): Attribute diff --git a/app/Models/StandaloneDragonfly.php b/app/Models/StandaloneDragonfly.php index fa0bd4b44..f7d83f0a3 100644 --- a/app/Models/StandaloneDragonfly.php +++ b/app/Models/StandaloneDragonfly.php @@ -38,6 +38,11 @@ class StandaloneDragonfly extends BaseModel $database->environment_variables()->delete(); $database->tags()->detach(); }); + static::saving(function ($database) { + if ($database->isDirty('status')) { + $database->forceFill(['last_online_at' => now()]); + } + }); } protected function serverStatus(): Attribute diff --git a/app/Models/StandaloneKeydb.php b/app/Models/StandaloneKeydb.php index 56ee4d4a2..083c743d9 100644 --- a/app/Models/StandaloneKeydb.php +++ b/app/Models/StandaloneKeydb.php @@ -38,6 +38,11 @@ class StandaloneKeydb extends BaseModel $database->environment_variables()->delete(); $database->tags()->detach(); }); + static::saving(function ($database) { + if ($database->isDirty('status')) { + $database->forceFill(['last_online_at' => now()]); + } + }); } protected function serverStatus(): Attribute diff --git a/app/Models/StandaloneMariadb.php b/app/Models/StandaloneMariadb.php index c55a76af7..833dad6c4 100644 --- a/app/Models/StandaloneMariadb.php +++ b/app/Models/StandaloneMariadb.php @@ -38,6 +38,11 @@ class StandaloneMariadb extends BaseModel $database->environment_variables()->delete(); $database->tags()->detach(); }); + static::saving(function ($database) { + if ($database->isDirty('status')) { + $database->forceFill(['last_online_at' => now()]); + } + }); } protected function serverStatus(): Attribute diff --git a/app/Models/StandaloneMongodb.php b/app/Models/StandaloneMongodb.php index 3f12a8557..dd8893180 100644 --- a/app/Models/StandaloneMongodb.php +++ b/app/Models/StandaloneMongodb.php @@ -42,6 +42,11 @@ class StandaloneMongodb extends BaseModel $database->environment_variables()->delete(); $database->tags()->detach(); }); + static::saving(function ($database) { + if ($database->isDirty('status')) { + $database->forceFill(['last_online_at' => now()]); + } + }); } protected function serverStatus(): Attribute diff --git a/app/Models/StandaloneMysql.php b/app/Models/StandaloneMysql.php index 0d8359f88..710fea1bc 100644 --- a/app/Models/StandaloneMysql.php +++ b/app/Models/StandaloneMysql.php @@ -39,6 +39,11 @@ class StandaloneMysql extends BaseModel $database->environment_variables()->delete(); $database->tags()->detach(); }); + static::saving(function ($database) { + if ($database->isDirty('status')) { + $database->forceFill(['last_online_at' => now()]); + } + }); } protected function serverStatus(): Attribute diff --git a/app/Models/StandalonePostgresql.php b/app/Models/StandalonePostgresql.php index f31582c35..4a457a6cf 100644 --- a/app/Models/StandalonePostgresql.php +++ b/app/Models/StandalonePostgresql.php @@ -39,6 +39,11 @@ class StandalonePostgresql extends BaseModel $database->environment_variables()->delete(); $database->tags()->detach(); }); + static::saving(function ($database) { + if ($database->isDirty('status')) { + $database->forceFill(['last_online_at' => now()]); + } + }); } public function workdir() diff --git a/app/Models/StandaloneRedis.php b/app/Models/StandaloneRedis.php index 119978530..826bb951c 100644 --- a/app/Models/StandaloneRedis.php +++ b/app/Models/StandaloneRedis.php @@ -34,6 +34,11 @@ class StandaloneRedis extends BaseModel $database->environment_variables()->delete(); $database->tags()->detach(); }); + static::saving(function ($database) { + if ($database->isDirty('status')) { + $database->forceFill(['last_online_at' => now()]); + } + }); } protected function serverStatus(): Attribute diff --git a/database/migrations/2024_11_02_213214_add_last_online_at_to_resources.php b/database/migrations/2024_11_02_213214_add_last_online_at_to_resources.php new file mode 100644 index 000000000..51b8fb3ba --- /dev/null +++ b/database/migrations/2024_11_02_213214_add_last_online_at_to_resources.php @@ -0,0 +1,96 @@ +timestamp('last_online_at')->default(now())->after('updated_at'); + }); + Schema::table('application_previews', function (Blueprint $table) { + $table->timestamp('last_online_at')->default(now())->after('updated_at'); + }); + Schema::table('service_applications', function (Blueprint $table) { + $table->timestamp('last_online_at')->default(now())->after('updated_at'); + }); + Schema::table('service_databases', function (Blueprint $table) { + $table->timestamp('last_online_at')->default(now())->after('updated_at'); + }); + Schema::table('standalone_postgresqls', function (Blueprint $table) { + $table->timestamp('last_online_at')->default(now())->after('updated_at'); + }); + Schema::table('standalone_redis', function (Blueprint $table) { + $table->timestamp('last_online_at')->default(now())->after('updated_at'); + }); + Schema::table('standalone_mongodbs', function (Blueprint $table) { + $table->timestamp('last_online_at')->default(now())->after('updated_at'); + }); + Schema::table('standalone_mysqls', function (Blueprint $table) { + $table->timestamp('last_online_at')->default(now())->after('updated_at'); + }); + Schema::table('standalone_mariadbs', function (Blueprint $table) { + $table->timestamp('last_online_at')->default(now())->after('updated_at'); + }); + Schema::table('standalone_keydbs', function (Blueprint $table) { + $table->timestamp('last_online_at')->default(now())->after('updated_at'); + }); + Schema::table('standalone_dragonflies', function (Blueprint $table) { + $table->timestamp('last_online_at')->default(now())->after('updated_at'); + }); + Schema::table('standalone_clickhouses', function (Blueprint $table) { + $table->timestamp('last_online_at')->default(now())->after('updated_at'); + }); + + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('applications', function (Blueprint $table) { + $table->dropColumn('last_online_at'); + }); + Schema::table('application_previews', function (Blueprint $table) { + $table->dropColumn('last_online_at'); + }); + Schema::table('service_applications', function (Blueprint $table) { + $table->dropColumn('last_online_at'); + }); + Schema::table('service_databases', function (Blueprint $table) { + $table->dropColumn('last_online_at'); + }); + Schema::table('standalone_postgresqls', function (Blueprint $table) { + $table->dropColumn('last_online_at'); + }); + Schema::table('standalone_redis', function (Blueprint $table) { + $table->dropColumn('last_online_at'); + }); + Schema::table('standalone_mongodbs', function (Blueprint $table) { + $table->dropColumn('last_online_at'); + }); + Schema::table('standalone_mysqls', function (Blueprint $table) { + $table->dropColumn('last_online_at'); + }); + Schema::table('standalone_mariadbs', function (Blueprint $table) { + $table->dropColumn('last_online_at'); + }); + Schema::table('standalone_keydbs', function (Blueprint $table) { + $table->dropColumn('last_online_at'); + }); + Schema::table('standalone_dragonflies', function (Blueprint $table) { + $table->dropColumn('last_online_at'); + }); + Schema::table('standalone_clickhouses', function (Blueprint $table) { + $table->dropColumn('last_online_at'); + }); + + } +}; diff --git a/routes/api.php b/routes/api.php index 572325810..1942d3312 100644 --- a/routes/api.php +++ b/routes/api.php @@ -14,6 +14,7 @@ use App\Http\Middleware\ApiAllowed; use App\Http\Middleware\IgnoreReadOnlyApiToken; use App\Http\Middleware\OnlyRootApiToken; use App\Jobs\PushServerUpdateJob; +use App\Jobs\ServerCheckNewJob; use App\Models\Server; use Illuminate\Support\Facades\Route; @@ -151,6 +152,7 @@ Route::group([ } $data = request()->all(); + // ServerCheckNewJob::dispatch($server, $data); PushServerUpdateJob::dispatch($server, $data); return response()->json(['message' => 'ok'], 200); From ed03cd33b0a7fd8d931d1760aca0bd93729bd243 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Sun, 3 Nov 2024 09:10:36 +0100 Subject: [PATCH 008/126] fix: only run server storage every 10 mins if sentinel is not active --- app/Actions/Server/ServerCheck.php | 2 +- app/Console/Kernel.php | 7 ++++++- app/Jobs/ServerCheckJob.php | 1 - 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/app/Actions/Server/ServerCheck.php b/app/Actions/Server/ServerCheck.php index 51f6511c8..f61422807 100644 --- a/app/Actions/Server/ServerCheck.php +++ b/app/Actions/Server/ServerCheck.php @@ -54,7 +54,7 @@ class ServerCheck } else { ['containers' => $this->containers, 'containerReplicates' => $containerReplicates] = $this->server->getContainers(); - ServerStorageCheckJob::dispatch($this->server); + // ServerStorageCheckJob::dispatch($this->server); } if (is_null($this->containers)) { diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index 9d1a2e40f..8d709c4dc 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -14,6 +14,7 @@ use App\Jobs\PullTemplatesFromCDN; use App\Jobs\ScheduledTaskJob; use App\Jobs\ServerCheckJob; use App\Jobs\ServerCleanupMux; +use App\Jobs\ServerStorageCheckJob; use App\Jobs\UpdateCoolifyJob; use App\Models\InstanceSettings; use App\Models\ScheduledDatabaseBackup; @@ -123,14 +124,18 @@ class Kernel extends ConsoleKernel // Sentinel check $lastSentinelUpdate = $server->sentinel_updated_at; if (Carbon::parse($lastSentinelUpdate)->isBefore(now()->subSeconds($server->waitBeforeDoingSshCheck()))) { + // Check container status every minute if Sentinel does not activated $schedule->job(new ServerCheckJob($server))->everyMinute()->onOneServer(); - } + // Check storage usage every 10 minutes if Sentinel does not activated + $schedule->job(new ServerStorageCheckJob($server))->everyTenMinutes()->onOneServer(); + } if ($server->settings->force_docker_cleanup) { $schedule->job(new DockerCleanupJob($server))->cron($server->settings->docker_cleanup_frequency)->timezone($serverTimezone)->onOneServer(); } else { $schedule->job(new DockerCleanupJob($server))->everyTenMinutes()->timezone($serverTimezone)->onOneServer(); } + // Cleanup multiplexed connections every hour $schedule->job(new ServerCleanupMux($server))->hourly()->onOneServer(); diff --git a/app/Jobs/ServerCheckJob.php b/app/Jobs/ServerCheckJob.php index f949f4ec0..449a2da14 100644 --- a/app/Jobs/ServerCheckJob.php +++ b/app/Jobs/ServerCheckJob.php @@ -39,7 +39,6 @@ class ServerCheckJob implements ShouldBeEncrypted, ShouldQueue if (is_null($this->containers)) { return 'No containers found.'; } - ServerStorageCheckJob::dispatch($this->server); GetContainersStatus::run($this->server, $this->containers, $containerReplicates); if ($this->server->isSentinelEnabled()) { From 52b4e6216cdf0b680974df36972f5162ad967114 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Sun, 3 Nov 2024 13:43:28 +0100 Subject: [PATCH 009/126] fix: cloud admin view --- app/Livewire/Admin/Index.php | 39 ++++++------------ app/Livewire/Team/AdminView.php | 3 ++ .../views/livewire/admin/index.blade.php | 41 +++++++++---------- 3 files changed, 35 insertions(+), 48 deletions(-) diff --git a/app/Livewire/Admin/Index.php b/app/Livewire/Admin/Index.php index 16cd9152e..7078a21e9 100644 --- a/app/Livewire/Admin/Index.php +++ b/app/Livewire/Admin/Index.php @@ -3,16 +3,19 @@ namespace App\Livewire\Admin; use App\Models\User; +use Illuminate\Support\Collection; use Illuminate\Support\Facades\Cache; use Livewire\Component; class Index extends Component { - public $active_subscribers = []; + public int $activeSubscribers; - public $inactive_subscribers = []; + public int $inactiveSubscribers; - public $search = ''; + public Collection $foundUsers; + + public string $search = ''; public function mount() { @@ -29,39 +32,21 @@ class Index extends Component public function submitSearch() { if ($this->search !== '') { - $this->inactive_subscribers = User::whereDoesntHave('teams', function ($query) { - $query->whereRelation('subscription', 'stripe_subscription_id', '!=', null); - })->where(function ($query) { + $this->foundUsers = User::where(function ($query) { $query->where('name', 'like', "%{$this->search}%") ->orWhere('email', 'like', "%{$this->search}%"); - })->get()->filter(function ($user) { - return $user->id !== 0; - }); - $this->active_subscribers = User::whereHas('teams', function ($query) { - $query->whereRelation('subscription', 'stripe_subscription_id', '!=', null); - })->where(function ($query) { - $query->where('name', 'like', "%{$this->search}%") - ->orWhere('email', 'like', "%{$this->search}%"); - })->get()->filter(function ($user) { - return $user->id !== 0; - }); - } else { - $this->getSubscribers(); + })->get(); } } public function getSubscribers() { - $this->inactive_subscribers = User::whereDoesntHave('teams', function ($query) { + $this->inactiveSubscribers = User::whereDoesntHave('teams', function ($query) { $query->whereRelation('subscription', 'stripe_subscription_id', '!=', null); - })->get()->filter(function ($user) { - return $user->id !== 0; - }); - $this->active_subscribers = User::whereHas('teams', function ($query) { + })->count(); + $this->activeSubscribers = User::whereHas('teams', function ($query) { $query->whereRelation('subscription', 'stripe_subscription_id', '!=', null); - })->get()->filter(function ($user) { - return $user->id !== 0; - }); + })->count(); } public function switchUser(int $user_id) diff --git a/app/Livewire/Team/AdminView.php b/app/Livewire/Team/AdminView.php index c9dabcb5c..cfb47d9d8 100644 --- a/app/Livewire/Team/AdminView.php +++ b/app/Livewire/Team/AdminView.php @@ -74,6 +74,9 @@ class AdminView extends Component public function delete($id, $password) { + if (! isInstanceAdmin()) { + return redirect()->route('dashboard'); + } if (! data_get(InstanceSettings::get(), 'disable_two_step_confirmation')) { if (! Hash::check($password, Auth::user()->password)) { $this->addError('password', 'The provided password is incorrect.'); diff --git a/resources/views/livewire/admin/index.blade.php b/resources/views/livewire/admin/index.blade.php index 99abdf0e5..cceb71bb2 100644 --- a/resources/views/livewire/admin/index.blade.php +++ b/resources/views/livewire/admin/index.blade.php @@ -6,26 +6,25 @@ Search -

Active Subscribers

-
- @forelse ($active_subscribers as $user) -
-

{{ $user->name }}

-

{{ $user->email }}

+
Active Subscribers : {{ $activeSubscribers }}
+
Inactive Subscribers : {{ $inactiveSubscribers }}
+ @if ($search) + @if ($foundUsers->count() > 0) +
+ @foreach ($foundUsers as $user) +
+
+
{{ $user->name }}
+
{{ $user->email }}
+
Active: + {{ $user->teams()->whereRelation('subscription', 'stripe_subscription_id', '!=', null)->exists() ? 'Yes' : 'No' }} +
+
+
+ @endforeach
- @empty -

No active subscribers

- @endforelse -
-

Inactive Subscribers

-
- @forelse ($inactive_subscribers as $user) -
-

{{ $user->name }}

-

{{ $user->email }}

-
- @empty -

No inactive subscribers

- @endforelse -
+ @else +
No users found with {{ $search }}
+ @endif + @endif
From 9ff73d62e18ccff0a66af4de37c5232ba4d87e7b Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Sun, 3 Nov 2024 14:18:43 +0100 Subject: [PATCH 010/126] fix: queries in kernel.php --- app/Console/Kernel.php | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index 8d709c4dc..0b4b2e185 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -33,7 +33,7 @@ class Kernel extends ConsoleKernel protected function schedule(Schedule $schedule): void { - $this->allServers = Server::where('ip', '!=', '1.2.3.4')->get(); + $this->allServers = Server::where('ip', '!=', '1.2.3.4'); $this->settings = instanceSettings(); @@ -76,7 +76,7 @@ class Kernel extends ConsoleKernel private function pullImages($schedule): void { - $servers = $this->allServers->whereRelation('settings', 'is_usable', true)->whereRelation('settings', 'is_reachable', true); + $servers = $this->allServers->whereRelation('settings', 'is_usable', true)->whereRelation('settings', 'is_reachable', true)->get(); foreach ($servers as $server) { if ($server->isSentinelEnabled()) { $schedule->job(function () use ($server) { @@ -110,11 +110,11 @@ class Kernel extends ConsoleKernel private function checkResources($schedule): void { if (isCloud()) { - $servers = $this->allServers->whereNotNull('team.subscription')->where('team.subscription.stripe_trial_already_ended', false); + $servers = $this->allServers->whereHas('team.subscription')->get(); $own = Team::find(0)->servers; $servers = $servers->merge($own); } else { - $servers = $this->allServers; + $servers = $this->allServers->get(); } // $schedule->job(new ResourcesCheck)->everyMinute()->onOneServer(); @@ -150,14 +150,11 @@ class Kernel extends ConsoleKernel private function checkScheduledBackups($schedule): void { - $scheduled_backups = ScheduledDatabaseBackup::all(); + $scheduled_backups = ScheduledDatabaseBackup::where('enabled', true)->get(); if ($scheduled_backups->isEmpty()) { return; } foreach ($scheduled_backups as $scheduled_backup) { - if (! $scheduled_backup->enabled) { - continue; - } if (is_null(data_get($scheduled_backup, 'database'))) { $scheduled_backup->delete(); @@ -166,7 +163,7 @@ class Kernel extends ConsoleKernel $server = $scheduled_backup->server(); - if (! $server) { + if (is_null($server)) { continue; } $serverTimezone = $server->settings->server_timezone; From 6987951efea2a092ab77f3c43082c9d5ace446cf Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Sun, 3 Nov 2024 14:53:44 +0100 Subject: [PATCH 011/126] add new resource check to kernel.php --- app/Console/Kernel.php | 4 ++-- app/Jobs/ServerCheckNewJob.php | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index 0b4b2e185..8b60c694b 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -2,7 +2,6 @@ namespace App\Console; -use App\Actions\Server\ResourcesCheck; use App\Jobs\CheckAndStartSentinelJob; use App\Jobs\CheckForUpdatesJob; use App\Jobs\CheckHelperImageJob; @@ -116,7 +115,7 @@ class Kernel extends ConsoleKernel } else { $servers = $this->allServers->get(); } - // $schedule->job(new ResourcesCheck)->everyMinute()->onOneServer(); + // $schedule->job(new \App\Jobs\ResourcesCheck)->everyMinute()->onOneServer(); foreach ($servers as $server) { $serverTimezone = $server->settings->server_timezone; @@ -126,6 +125,7 @@ class Kernel extends ConsoleKernel if (Carbon::parse($lastSentinelUpdate)->isBefore(now()->subSeconds($server->waitBeforeDoingSshCheck()))) { // Check container status every minute if Sentinel does not activated $schedule->job(new ServerCheckJob($server))->everyMinute()->onOneServer(); + // $schedule->job(new \App\Jobs\ServerCheckNewJob($server))->everyMinute()->onOneServer(); // Check storage usage every 10 minutes if Sentinel does not activated $schedule->job(new ServerStorageCheckJob($server))->everyTenMinutes()->onOneServer(); diff --git a/app/Jobs/ServerCheckNewJob.php b/app/Jobs/ServerCheckNewJob.php index 8455c9617..9ce52759c 100644 --- a/app/Jobs/ServerCheckNewJob.php +++ b/app/Jobs/ServerCheckNewJob.php @@ -24,7 +24,7 @@ class ServerCheckNewJob implements ShouldBeEncrypted, ShouldQueue public function handle() { try { - ServerCheck::dispatch($this->server); + ServerCheck::run($this->server); } catch (\Throwable $e) { return handleError($e); } From 38fad7e6fd9599e6e0e7d661a634db33de0ac960 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Sun, 3 Nov 2024 14:59:21 +0100 Subject: [PATCH 012/126] asd --- routes/api.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/routes/api.php b/routes/api.php index 1942d3312..e8425aeb1 100644 --- a/routes/api.php +++ b/routes/api.php @@ -14,7 +14,6 @@ use App\Http\Middleware\ApiAllowed; use App\Http\Middleware\IgnoreReadOnlyApiToken; use App\Http\Middleware\OnlyRootApiToken; use App\Jobs\PushServerUpdateJob; -use App\Jobs\ServerCheckNewJob; use App\Models\Server; use Illuminate\Support\Facades\Route; @@ -152,7 +151,7 @@ Route::group([ } $data = request()->all(); - // ServerCheckNewJob::dispatch($server, $data); + // \App\Jobs\ServerCheckNewJob::dispatch($server, $data); PushServerUpdateJob::dispatch($server, $data); return response()->json(['message' => 'ok'], 200); From 46c218d773230aaf1d2ca78299c05cc266143be8 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Sun, 3 Nov 2024 15:07:16 +0100 Subject: [PATCH 013/126] fix ui --- .../views/livewire/server/advanced.blade.php | 3 + .../views/livewire/server/charts.blade.php | 284 +++++++++--------- .../server/destination/show.blade.php | 2 +- .../livewire/server/log-drains.blade.php | 2 +- .../server/private-key/show.blade.php | 2 +- .../views/livewire/server/show.blade.php | 2 +- 6 files changed, 151 insertions(+), 144 deletions(-) diff --git a/resources/views/livewire/server/advanced.blade.php b/resources/views/livewire/server/advanced.blade.php index fc1be78d8..4d32d711c 100644 --- a/resources/views/livewire/server/advanced.blade.php +++ b/resources/views/livewire/server/advanced.blade.php @@ -1,4 +1,7 @@
+ + {{ data_get_str($server, 'name')->limit(10) }} > Advanced | Coolify +
diff --git a/resources/views/livewire/server/charts.blade.php b/resources/views/livewire/server/charts.blade.php index 64a5b5fbb..27ea34af2 100644 --- a/resources/views/livewire/server/charts.blade.php +++ b/resources/views/livewire/server/charts.blade.php @@ -1,6 +1,6 @@
- {{ data_get_str($server, 'name')->limit(10) }} > Server Destinations | Coolify + {{ data_get_str($server, 'name')->limit(10) }} > Metrics | Coolify
@@ -8,139 +8,29 @@

Metrics

Basic metrics for your container.
-
- - - - - - - - - -

CPU (%)

-
- - - -
-

Memory (%)

-
+ @if ($server->isMetricsEnabled()) +
+ + + + + + + + + +

CPU (%)

+
+
+

Memory (%)

+
+ + + +
-
+ @else +
Metrics are disabled for this server.
+ @endif
diff --git a/resources/views/livewire/server/destination/show.blade.php b/resources/views/livewire/server/destination/show.blade.php index 9dba9c0d6..88503f62d 100644 --- a/resources/views/livewire/server/destination/show.blade.php +++ b/resources/views/livewire/server/destination/show.blade.php @@ -1,6 +1,6 @@
- {{ data_get_str($server, 'name')->limit(10) }} > Server Destinations | Coolify + {{ data_get_str($server, 'name')->limit(10) }} > Destinations | Coolify
diff --git a/resources/views/livewire/server/log-drains.blade.php b/resources/views/livewire/server/log-drains.blade.php index 2ababba93..19c33ef5d 100644 --- a/resources/views/livewire/server/log-drains.blade.php +++ b/resources/views/livewire/server/log-drains.blade.php @@ -1,6 +1,6 @@
- {{ data_get_str($server, 'name')->limit(10) }} > Server Log Drains | Coolify + {{ data_get_str($server, 'name')->limit(10) }} > Log Drains | Coolify
diff --git a/resources/views/livewire/server/private-key/show.blade.php b/resources/views/livewire/server/private-key/show.blade.php index 725d10f8e..53e9ed002 100644 --- a/resources/views/livewire/server/private-key/show.blade.php +++ b/resources/views/livewire/server/private-key/show.blade.php @@ -1,6 +1,6 @@
- {{ data_get_str($server, 'name')->limit(10) }} > Server Connection | Coolify + {{ data_get_str($server, 'name')->limit(10) }} > Private Key | Coolify
diff --git a/resources/views/livewire/server/show.blade.php b/resources/views/livewire/server/show.blade.php index 8861d12bc..de12d0e5b 100644 --- a/resources/views/livewire/server/show.blade.php +++ b/resources/views/livewire/server/show.blade.php @@ -1,6 +1,6 @@
- {{ data_get_str($server, 'name')->limit(10) }} > Server Configurations | Coolify + {{ data_get_str($server, 'name')->limit(10) }} > General | Coolify
From 63cff9e399a0ea453939ebd993c6f26b245690a0 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Sun, 3 Nov 2024 15:30:45 +0100 Subject: [PATCH 014/126] fix: lower case emails only --- app/Actions/Fortify/CreateNewUser.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/Actions/Fortify/CreateNewUser.php b/app/Actions/Fortify/CreateNewUser.php index 9f97dd0d4..ea2befd3a 100644 --- a/app/Actions/Fortify/CreateNewUser.php +++ b/app/Actions/Fortify/CreateNewUser.php @@ -40,7 +40,7 @@ class CreateNewUser implements CreatesNewUsers $user = User::create([ 'id' => 0, 'name' => $input['name'], - 'email' => $input['email'], + 'email' => strtolower($input['email']), 'password' => Hash::make($input['password']), ]); $team = $user->teams()->first(); @@ -52,7 +52,7 @@ class CreateNewUser implements CreatesNewUsers } else { $user = User::create([ 'name' => $input['name'], - 'email' => $input['email'], + 'email' => strtolower($input['email']), 'password' => Hash::make($input['password']), ]); $team = $user->teams()->first(); From f408d603cff02a7ac063d92d2e54cb332915c56e Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Sun, 3 Nov 2024 15:35:17 +0100 Subject: [PATCH 015/126] fix: change emails to lowercase on init --- app/Console/Commands/Init.php | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/app/Console/Commands/Init.php b/app/Console/Commands/Init.php index ccb864e1f..f90afd219 100644 --- a/app/Console/Commands/Init.php +++ b/app/Console/Commands/Init.php @@ -10,6 +10,7 @@ use App\Models\Environment; use App\Models\ScheduledDatabaseBackup; use App\Models\Server; use App\Models\StandalonePostgresql; +use App\Models\User; use Illuminate\Console\Command; use Illuminate\Support\Facades\File; use Illuminate\Support\Facades\Http; @@ -41,6 +42,7 @@ class Init extends Command $this->disable_metrics(); $this->replace_slash_in_environment_name(); $this->restore_coolify_db_backup(); + $this->update_user_emails(); // $this->update_traefik_labels(); if (! isCloud() || $this->option('force-cloud')) { @@ -92,6 +94,11 @@ class Init extends Command } } + private function update_user_emails() + { + User::whereRaw('email ~ \'[A-Z]\'')->get()->each(fn (User $user) => $user->update(['email' => strtolower($user->email)])); + } + private function update_traefik_labels() { try { From d3f422a7fb587ea47391b9842adf6b7af4c35591 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Sun, 3 Nov 2024 16:14:12 +0100 Subject: [PATCH 016/126] fix: do not error on update email --- app/Console/Commands/Init.php | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/app/Console/Commands/Init.php b/app/Console/Commands/Init.php index f90afd219..8d4f51d1c 100644 --- a/app/Console/Commands/Init.php +++ b/app/Console/Commands/Init.php @@ -96,7 +96,11 @@ class Init extends Command private function update_user_emails() { - User::whereRaw('email ~ \'[A-Z]\'')->get()->each(fn (User $user) => $user->update(['email' => strtolower($user->email)])); + try { + User::whereRaw('email ~ \'[A-Z]\'')->get()->each(fn (User $user) => $user->update(['email' => strtolower($user->email)])); + } catch (\Throwable $e) { + echo "Error in updating user emails: {$e->getMessage()}\n"; + } } private function update_traefik_labels() From b8a35be095d63d43d0fc80d442ae77bbaf9b0629 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Sun, 3 Nov 2024 16:14:25 +0100 Subject: [PATCH 017/126] fix: always authenticate with lowercase emails --- app/Providers/FortifyServiceProvider.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/Providers/FortifyServiceProvider.php b/app/Providers/FortifyServiceProvider.php index b916b6234..e8784bab3 100644 --- a/app/Providers/FortifyServiceProvider.php +++ b/app/Providers/FortifyServiceProvider.php @@ -75,7 +75,8 @@ class FortifyServiceProvider extends ServiceProvider }); Fortify::authenticateUsing(function (Request $request) { - $user = User::where('email', $request->email)->with('teams')->first(); + $email = strtolower($request->email); + $user = User::where('email', $email)->with('teams')->first(); if ( $user && Hash::check($request->password, $user->password) From bd8218932e61258f34fa6cd723d3861e8e422f8a Mon Sep 17 00:00:00 2001 From: RayBB Date: Sun, 3 Nov 2024 14:27:40 -0500 Subject: [PATCH 018/126] add hoarder service --- public/svgs/hoarder.svg | 1 + templates/compose/hoarder.yaml | 40 ++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) create mode 100644 public/svgs/hoarder.svg create mode 100644 templates/compose/hoarder.yaml diff --git a/public/svgs/hoarder.svg b/public/svgs/hoarder.svg new file mode 100644 index 000000000..6215461d2 --- /dev/null +++ b/public/svgs/hoarder.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/templates/compose/hoarder.yaml b/templates/compose/hoarder.yaml new file mode 100644 index 000000000..c58b8a82e --- /dev/null +++ b/templates/compose/hoarder.yaml @@ -0,0 +1,40 @@ +# documentation: https://docs.hoarder.app/ +# slogan: an open source "Bookmark Everything" app that uses AI for automatically tagging the content you throw at it. +# tags: media,read-it-later,pocket-alternative,omnivore-alternative,instapaper-alternative +# logo: svgs/hoarder.svg +# port: 3000 + +services: + web: + image: 'ghcr.io/hoarder-app/hoarder:release' + restart: unless-stopped + volumes: + - 'data:/data' + environment: + - SERVICE_FQDN_HOARDER + - NEXTAUTH_SECRET=${SERVICE_PASSWORD_HOARDERNEXTAUTH} + - MEILI_MASTER_KEY=${SERVICE_PASSWORD_MEILI} + - NEXTAUTH_URL=${SERVICE_FQDN_HOARDER} + - MEILI_ADDR=http://meilisearch:7700 + - BROWSER_WEB_URL=http://chrome:9222 + - DATA_DIR=/data + chrome: + image: 'gcr.io/zenika-hub/alpine-chrome:124' + restart: unless-stopped + command: + - '--no-sandbox' + - '--disable-gpu' + - '--disable-dev-shm-usage' + - '--remote-debugging-address=0.0.0.0' + - '--remote-debugging-port=9222' + - '--hide-scrollbars' + meilisearch: + image: 'getmeili/meilisearch:v1.6' + restart: unless-stopped + environment: + - MEILI_NO_ANALYTICS=true + - NEXTAUTH_SECRET=${SERVICE_PASSWORD_HOARDERNEXTAUTH} + - MEILI_MASTER_KEY=${SERVICE_PASSWORD_MEILI} + - NEXTAUTH_URL=${SERVICE_FQDN_HOARDER} + volumes: + - 'meilisearch:/meili_data' From cc7c25aafda93385887c7e7349e28c6a6e669392 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Sun, 3 Nov 2024 21:11:35 +0100 Subject: [PATCH 019/126] fix: dashboard refactor --- app/Livewire/Dashboard.php | 14 ++++----- resources/views/livewire/dashboard.blade.php | 30 ++++++-------------- 2 files changed, 16 insertions(+), 28 deletions(-) diff --git a/app/Livewire/Dashboard.php b/app/Livewire/Dashboard.php index d18a7689e..69ba19e40 100644 --- a/app/Livewire/Dashboard.php +++ b/app/Livewire/Dashboard.php @@ -16,28 +16,28 @@ class Dashboard extends Component public Collection $servers; - public Collection $private_keys; + public Collection $privateKeys; - public $deployments_per_server; + public array $deploymentsPerServer = []; public function mount() { - $this->private_keys = PrivateKey::ownedByCurrentTeam()->get(); + $this->privateKeys = PrivateKey::ownedByCurrentTeam()->get(); $this->servers = Server::ownedByCurrentTeam()->get(); $this->projects = Project::ownedByCurrentTeam()->get(); - $this->get_deployments(); + $this->loadDeployments(); } - public function cleanup_queue() + public function cleanupQueue() { Artisan::queue('cleanup:deployment-queue', [ '--team-id' => currentTeam()->id, ]); } - public function get_deployments() + public function loadDeployments() { - $this->deployments_per_server = ApplicationDeploymentQueue::whereIn('status', ['in_progress', 'queued'])->whereIn('server_id', $this->servers->pluck('id'))->get([ + $this->deploymentsPerServer = ApplicationDeploymentQueue::whereIn('status', ['in_progress', 'queued'])->whereIn('server_id', $this->servers->pluck('id'))->get([ 'id', 'application_id', 'application_name', diff --git a/resources/views/livewire/dashboard.blade.php b/resources/views/livewire/dashboard.blade.php index 2b9620e30..decd75c46 100644 --- a/resources/views/livewire/dashboard.blade.php +++ b/resources/views/livewire/dashboard.blade.php @@ -94,7 +94,7 @@ @endforeach
@else - @if ($private_keys->count() === 0) + @if ($privateKeys->count() === 0)
No private keys found.
Before you can add your server, first

Deployments

- @if (count($deployments_per_server) > 0) + @if (count($deploymentsPerServer) > 0) @endif - +
-
- @forelse ($deployments_per_server as $server_name => $deployments) -

{{ $server_name }}

+
+ @forelse ($deploymentsPerServer as $serverName => $deployments) +

{{ $serverName }}

@foreach ($deployments as $deployment) - {{-- Get IPTABLES --}} - -
From c211227141739f8249a882251ae78da9fabc7500 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Sun, 3 Nov 2024 21:27:02 +0100 Subject: [PATCH 020/126] fix: add min/max length to input/texarea --- app/View/Components/Forms/Textarea.php | 4 +++- resources/views/components/forms/input.blade.php | 5 +++-- resources/views/components/forms/textarea.blade.php | 7 ++++--- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/app/View/Components/Forms/Textarea.php b/app/View/Components/Forms/Textarea.php index 3f887877c..6081c2a8a 100644 --- a/app/View/Components/Forms/Textarea.php +++ b/app/View/Components/Forms/Textarea.php @@ -30,7 +30,9 @@ class Textarea extends Component public bool $realtimeValidation = false, public bool $allowToPeak = true, public string $defaultClass = 'input scrollbar font-mono', - public string $defaultClassInput = 'input' + public string $defaultClassInput = 'input', + public ?int $minlength = null, + public ?int $maxlength = null, ) { // } diff --git a/resources/views/components/forms/input.blade.php b/resources/views/components/forms/input.blade.php index fb206fac4..d832cb30d 100644 --- a/resources/views/components/forms/input.blade.php +++ b/resources/views/components/forms/input.blade.php @@ -41,8 +41,9 @@ @if ($id !== 'null') wire:model={{ $id }} @endif wire:dirty.class.remove='dark:focus:ring-coolgray-300 dark:ring-coolgray-300' wire:dirty.class="dark:focus:ring-warning dark:ring-warning" wire:loading.attr="disabled" - type="{{ $type }}" @disabled($disabled) - min="{{ $attributes->get('min') }}" max="{{ $attributes->get('max') }}" + type="{{ $type }}" @disabled($disabled) min="{{ $attributes->get('min') }}" + max="{{ $attributes->get('max') }}" minlength="{{ $attributes->get('minlength') }}" + maxlength="{{ $attributes->get('maxlength') }}" @if ($id !== 'null') id={{ $id }} @endif name="{{ $name }}" placeholder="{{ $attributes->get('placeholder') }}"> @endif diff --git a/resources/views/components/forms/textarea.blade.php b/resources/views/components/forms/textarea.blade.php index 24226ecdb..b3669e43d 100644 --- a/resources/views/components/forms/textarea.blade.php +++ b/resources/views/components/forms/textarea.blade.php @@ -51,8 +51,8 @@ type="{{ $type }}" @readonly($readonly) @disabled($disabled) id="{{ $id }}" name="{{ $name }}" placeholder="{{ $attributes->get('placeholder') }}" aria-placeholder="{{ $attributes->get('placeholder') }}"> -