From 8a2c9f3d443820541fbf14682efa0dd5b2fc319c Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Mon, 14 Oct 2024 17:54:29 +0200 Subject: [PATCH] updates sentinel --- app/Jobs/PushServerUpdateJob.php | 312 +++++++++++++++++++++++++------ 1 file changed, 259 insertions(+), 53 deletions(-) diff --git a/app/Jobs/PushServerUpdateJob.php b/app/Jobs/PushServerUpdateJob.php index 226cf9392..a426e8532 100644 --- a/app/Jobs/PushServerUpdateJob.php +++ b/app/Jobs/PushServerUpdateJob.php @@ -2,17 +2,20 @@ namespace App\Jobs; +use App\Actions\Database\StartDatabaseProxy; use App\Actions\Proxy\StartProxy; +use App\Actions\Shared\ComplexStatusCheck; use App\Models\Application; use App\Models\ApplicationPreview; use App\Models\Server; +use App\Models\ServiceApplication; +use App\Models\ServiceDatabase; use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; use Illuminate\Support\Collection; -use Illuminate\Support\Facades\Log; class PushServerUpdateJob implements ShouldQueue { @@ -20,7 +23,33 @@ class PushServerUpdateJob implements ShouldQueue public $tries = 1; - public $timeout = 60; + public $timeout = 30; + + public Collection $containers; + + public Collection $allApplicationIds; + + public Collection $allDatabaseUuids; + + public Collection $allServiceApplicationIds; + + public Collection $allApplicationPreviewsIds; + + public Collection $allServiceDatabaseIds; + + public Collection $allApplicationsWithAdditionalServers; + + public Collection $foundApplicationIds; + + public Collection $foundDatabaseUuids; + + public Collection $foundServiceApplicationIds; + + public Collection $foundServiceDatabaseIds; + + public Collection $foundApplicationPreviewsIds; + + public bool $foundProxy = false; public function backoff(): int { @@ -29,12 +58,19 @@ class PushServerUpdateJob implements ShouldQueue public function __construct(public Server $server, public $data) { - // TODO: Handle multiple servers - // TODO: Handle Preview deployments - // TODO: Handle DB TCP proxies - // TODO: Handle DBs - // TODO: Handle services - // TODO: Handle proxies + // TODO: Handle multiple servers - done - NOT TESTED + // TODO: Handle Preview deployments - done - NOT TESTED + $this->containers = collect(); + $this->foundApplicationIds = collect(); + $this->foundDatabaseUuids = collect(); + $this->foundServiceApplicationIds = collect(); + $this->foundApplicationPreviewsIds = collect(); + $this->foundServiceDatabaseIds = collect(); + $this->allApplicationIds = collect(); + $this->allDatabaseUuids = collect(); + $this->allTcpProxyUuids = collect(); + $this->allServiceApplicationIds = collect(); + $this->allServiceDatabaseIds = collect(); } public function handle() @@ -43,14 +79,34 @@ class PushServerUpdateJob implements ShouldQueue throw new \Exception('No data provided'); } $data = collect($this->data); - $containers = collect(data_get($data, 'containers')); - if ($containers->isEmpty()) { + $this->containers = collect(data_get($data, 'containers')); + if ($this->containers->isEmpty()) { return; } - $foundApplicationIds = collect(); - $foundServiceIds = collect(); - $foundProxy = false; - foreach ($containers as $container) { + $this->allApplicationIds = $this->server->applications() + ->filter(function ($application) { + return $application->additional_servers->count() === 0; + }) + ->pluck('id'); + $this->allApplicationsWithAdditionalServers = $this->server->applications() + ->filter(function ($application) { + return $application->additional_servers->count() > 0; + }); + $this->allApplicationPreviewsIds = $this->server->previews()->pluck('id'); + $this->allDatabaseUuids = $this->server->databases()->pluck('uuid'); + $this->allTcpProxyUuids = $this->server->databases()->where('is_public', true)->pluck('uuid'); + $this->server->services()->each(function ($service) { + $service->applications()->pluck('id')->each(function ($applicationId) { + $this->allServiceApplicationIds->push($applicationId); + }); + $service->databases()->pluck('id')->each(function ($databaseId) { + $this->allServiceDatabaseIds->push($databaseId); + }); + }); + + logger('allServiceApplicationIds', ['allServiceApplicationIds' => $this->allServiceApplicationIds]); + + foreach ($this->containers as $container) { $containerStatus = data_get($container, 'state', 'exited'); $containerHealth = data_get($container, 'health_status', 'unhealthy'); $containerStatus = "$containerStatus ($containerHealth)"; @@ -60,77 +116,227 @@ class PushServerUpdateJob implements ShouldQueue if ($labels->has('coolify.applicationId')) { $applicationId = $labels->get('coolify.applicationId'); $pullRequestId = data_get($labels, 'coolify.pullRequestId', '0'); - $foundApplicationIds->push($applicationId); try { - $this->updateApplicationStatus($applicationId, $pullRequestId, $containerStatus); + if ($pullRequestId === '0') { + if ($this->allApplicationIds->contains($applicationId)) { + $this->foundApplicationIds->push($applicationId); + } + $this->updateApplicationStatus($applicationId, $containerStatus); + } else { + if ($this->allApplicationPreviewsIds->contains($applicationId)) { + $this->foundApplicationPreviewsIds->push($applicationId); + } + $this->updateApplicationPreviewStatus($applicationId, $containerStatus); + } } catch (\Exception $e) { - Log::error($e); + logger()->error($e); } } elseif ($labels->has('coolify.serviceId')) { $serviceId = $labels->get('coolify.serviceId'); - $foundServiceIds->push($serviceId); - Log::info("Service: $serviceId, $containerStatus"); + $subType = $labels->get('coolify.service.subType'); + $subId = $labels->get('coolify.service.subId'); + if ($subType === 'application') { + $this->foundServiceApplicationIds->push($subId); + $this->updateServiceSubStatus($serviceId, $subType, $subId, $containerStatus); + } elseif ($subType === 'database') { + $this->foundServiceDatabaseIds->push($subId); + $this->updateServiceSubStatus($serviceId, $subType, $subId, $containerStatus); + } + } else { $name = data_get($container, 'name'); $uuid = $labels->get('com.docker.compose.service'); $type = $labels->get('coolify.type'); if ($name === 'coolify-proxy') { - $foundProxy = true; - Log::info("Proxy: $uuid, $containerStatus"); + $this->foundProxy = true; } elseif ($type === 'service') { - Log::info("Service: $uuid, $containerStatus"); + logger("Service: $uuid, $containerStatus"); } else { - Log::info("Database: $uuid, $containerStatus"); + if ($this->allDatabaseUuids->contains($uuid)) { + $this->foundDatabaseUuids->push($uuid); + if ($this->allTcpProxyUuids->contains($uuid)) { + $this->updateDatabaseStatus($uuid, $containerStatus, tcpProxy: true); + } else { + $this->updateDatabaseStatus($uuid, $containerStatus, tcpProxy: false); + } + } } } } } + $this->updateProxyStatus(); + + $this->updateNotFoundApplicationStatus(); + $this->updateNotFoundApplicationPreviewStatus(); + $this->updateNotFoundDatabaseStatus(); + $this->updateNotFoundServiceStatus(); + + $this->updateAdditionalServersStatus(); + } + + private function updateApplicationStatus(string $applicationId, string $containerStatus) + { + $application = $this->server->applications()->where('id', $applicationId)->first(); + if (! $application) { + return; + } + $application->status = $containerStatus; + $application->save(); + logger('Application updated', ['application_id' => $applicationId, 'status' => $containerStatus]); + } + + private function updateApplicationPreviewStatus(string $applicationId, string $containerStatus) + { + $application = $this->server->previews()->where('id', $applicationId)->first(); + if (! $application) { + return; + } + $application->status = $containerStatus; + $application->save(); + logger('Application preview updated', ['application_id' => $applicationId, 'status' => $containerStatus]); + } + + private function updateNotFoundApplicationStatus() + { + $notFoundApplicationIds = $this->allApplicationIds->diff($this->foundApplicationIds); + if ($notFoundApplicationIds->isNotEmpty()) { + logger('Not found application ids', ['application_ids' => $notFoundApplicationIds]); + $notFoundApplicationIds->each(function ($applicationId) { + logger('Updating application status', ['application_id' => $applicationId, 'status' => 'exited']); + $application = Application::find($applicationId); + if ($application) { + $application->status = 'exited'; + $application->save(); + logger('Application status updated', ['application_id' => $applicationId, 'status' => 'exited']); + } + }); + } + } + + private function updateNotFoundApplicationPreviewStatus() + { + $notFoundApplicationPreviewsIds = $this->allApplicationPreviewsIds->diff($this->foundApplicationPreviewsIds); + if ($notFoundApplicationPreviewsIds->isNotEmpty()) { + logger('Not found application previews ids', ['application_previews_ids' => $notFoundApplicationPreviewsIds]); + $notFoundApplicationPreviewsIds->each(function ($applicationPreviewId) { + logger('Updating application preview status', ['application_preview_id' => $applicationPreviewId, 'status' => 'exited']); + $applicationPreview = ApplicationPreview::find($applicationPreviewId); + if ($applicationPreview) { + $applicationPreview->status = 'exited'; + $applicationPreview->save(); + logger('Application preview status updated', ['application_preview_id' => $applicationPreviewId, 'status' => 'exited']); + } + }); + } + } + + private function updateProxyStatus() + { // If proxy is not found, start it - if (! $foundProxy && $this->server->isProxyShouldRun()) { - Log::info('Proxy not found, starting it'); + if (! $this->foundProxy && $this->server->isProxyShouldRun()) { + logger('Proxy not found, starting it.'); StartProxy::dispatch($this->server); } - // Update not found applications - $allApplicationIds = $this->server->applications()->pluck('id'); - $notFoundApplicationIds = $allApplicationIds->diff($foundApplicationIds); - if ($notFoundApplicationIds->isNotEmpty()) { - Log::info('Not found application ids', ['application_ids' => $notFoundApplicationIds]); - $this->updateNotFoundApplications($notFoundApplicationIds); + } + + private function updateDatabaseStatus(string $databaseUuid, string $containerStatus, bool $tcpProxy = false) + { + $database = $this->server->databases()->where('uuid', $databaseUuid)->first(); + if (! $database) { + return; + } + $database->status = $containerStatus; + $database->save(); + + if (str($containerStatus)->contains('running') && $tcpProxy) { + $tcpProxyContainerFound = $this->containers->filter(function ($value, $key) use ($databaseUuid) { + return data_get($value, 'name') === "$databaseUuid-proxy"; + })->first(); + + if (! $tcpProxyContainerFound) { + logger('Starting TCP proxy for database', ['database_uuid' => $databaseUuid]); + StartDatabaseProxy::dispatch($database); + } else { + logger('TCP proxy for database found in containers', ['database_uuid' => $databaseUuid]); + } } } - private function updateApplicationStatus(string $applicationId, string $pullRequestId, string $containerStatus) + private function updateNotFoundDatabaseStatus() { - if ($pullRequestId === '0') { - $application = Application::find($applicationId); - if (! $application) { - return; - } + $notFoundDatabaseUuids = $this->allDatabaseUuids->diff($this->foundDatabaseUuids); + if ($notFoundDatabaseUuids->isNotEmpty()) { + logger('Not found database uuids', ['database_uuids' => $notFoundDatabaseUuids]); + $notFoundDatabaseUuids->each(function ($databaseUuid) { + logger('Updating database status', ['database_uuid' => $databaseUuid, 'status' => 'exited']); + $database = $this->server->databases()->where('uuid', $databaseUuid)->first(); + if ($database) { + $database->status = 'exited'; + $database->save(); + logger('Database status updated', ['database_uuid' => $databaseUuid, 'status' => 'exited']); + } + }); + } + } + + private function updateServiceSubStatus(string $serviceId, string $subType, string $subId, string $containerStatus) + { + $service = $this->server->services()->where('id', $serviceId)->first(); + if (! $service) { + return; + } + if ($subType === 'application') { + $application = $service->applications()->where('id', $subId)->first(); $application->status = $containerStatus; $application->save(); - Log::info('Application updated', ['application_id' => $applicationId, 'status' => $containerStatus]); + logger('Service application updated', ['service_id' => $serviceId, 'sub_type' => $subType, 'sub_id' => $subId, 'status' => $containerStatus]); + } elseif ($subType === 'database') { + $database = $service->databases()->where('id', $subId)->first(); + $database->status = $containerStatus; + $database->save(); + logger('Service database updated', ['service_id' => $serviceId, 'sub_type' => $subType, 'sub_id' => $subId, 'status' => $containerStatus]); } else { - $application = ApplicationPreview::where('application_id', $applicationId)->where('pull_request_id', $pullRequestId)->first(); - if (! $application) { - return; - } - $application->status = $containerStatus; - $application->save(); + logger()->warning('Unknown sub type', ['service_id' => $serviceId, 'sub_type' => $subType, 'sub_id' => $subId, 'status' => $containerStatus]); } } - private function updateNotFoundApplications(Collection $applicationIds) + private function updateNotFoundServiceStatus() { - $applicationIds->each(function ($applicationId) { - Log::info('Updating application status', ['application_id' => $applicationId, 'status' => 'exited']); - $application = Application::find($applicationId); - if ($application) { - $application->status = 'exited'; - $application->save(); - Log::info('Application status updated', ['application_id' => $applicationId, 'status' => 'exited']); - } + $notFoundServiceApplicationIds = $this->allServiceApplicationIds->diff($this->foundServiceApplicationIds); + $notFoundServiceDatabaseIds = $this->allServiceDatabaseIds->diff($this->foundServiceDatabaseIds); + if ($notFoundServiceApplicationIds->isNotEmpty()) { + logger('Not found service application ids', ['service_application_ids' => $notFoundServiceApplicationIds]); + $notFoundServiceApplicationIds->each(function ($serviceApplicationId) { + logger('Updating service application status', ['service_application_id' => $serviceApplicationId, 'status' => 'exited']); + $application = ServiceApplication::find($serviceApplicationId); + if ($application) { + $application->status = 'exited'; + $application->save(); + logger('Service application status updated', ['service_application_id' => $serviceApplicationId, 'status' => 'exited']); + } + }); + } + if ($notFoundServiceDatabaseIds->isNotEmpty()) { + logger('Not found service database ids', ['service_database_ids' => $notFoundServiceDatabaseIds]); + $notFoundServiceDatabaseIds->each(function ($serviceDatabaseId) { + logger('Updating service database status', ['service_database_id' => $serviceDatabaseId, 'status' => 'exited']); + $database = ServiceDatabase::find($serviceDatabaseId); + if ($database) { + $database->status = 'exited'; + $database->save(); + logger('Service database status updated', ['service_database_id' => $serviceDatabaseId, 'status' => 'exited']); + } + }); + } + } + + private function updateAdditionalServersStatus() + { + $this->allApplicationsWithAdditionalServers->each(function ($application) { + logger('Updating additional servers status for application', ['application_id' => $application->id]); + ComplexStatusCheck::run($application); }); } }