From 3dedf5548f693a3994f46a3f2c148922414af74b Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Thu, 25 Jul 2024 20:30:22 +0200 Subject: [PATCH] feat: new server check job --- app/Jobs/ServerCheckJob.php | 486 ++++++++++++++++++++++++++++++++++++ 1 file changed, 486 insertions(+) create mode 100644 app/Jobs/ServerCheckJob.php diff --git a/app/Jobs/ServerCheckJob.php b/app/Jobs/ServerCheckJob.php new file mode 100644 index 000000000..adade3194 --- /dev/null +++ b/app/Jobs/ServerCheckJob.php @@ -0,0 +1,486 @@ +server->uuid))]; + } + + public function uniqueId(): int + { + return $this->server->uuid; + } + + public function handle() + { + + try { + $up = $this->serverStatus(); + if (! $up) { + ray('Server is not reachable.'); + + return 'Server is not reachable.'; + } + if (! $this->server->isFunctional()) { + ray('Server is not ready.'); + + return 'Server is not ready.'; + } + $this->checkSentinel(); + $this->getContainers(); + + if (is_null($this->containers)) { + return 'No containers found.'; + } + $this->checkLogDrainContainer(); + $this->containerStatus(); + + } catch (\Throwable $e) { + ray($e->getMessage()); + + return handleError($e); + } + + } + + private function checkSentinel() + { + if ($this->server->isSentinelEnabled()) { + $sentinelContainerFound = $this->containers->filter(function ($value, $key) { + return data_get($value, 'Name') === '/coolify-sentinel'; + })->first(); + if ($sentinelContainerFound) { + $status = data_get($sentinelContainerFound, 'State.Status'); + if ($status !== 'running') { + PullSentinelImageJob::dispatch($this); + } + } + } + } + + private function serverStatus() + { + $this->removeUnnevessaryCoolifyYaml(); + ['uptime' => $uptime] = $this->server->validateConnection(); + if ($uptime) { + if ($this->server->unreachable_notification_sent === true) { + $this->server->update(['unreachable_notification_sent' => false]); + } + } else { + foreach ($this->applications as $application) { + $application->update(['status' => 'exited']); + } + foreach ($this->databases as $database) { + $database->update(['status' => 'exited']); + } + foreach ($this->services as $service) { + $apps = $service->applications()->get(); + $dbs = $service->databases()->get(); + foreach ($apps as $app) { + $app->update(['status' => 'exited']); + } + foreach ($dbs as $db) { + $db->update(['status' => 'exited']); + } + } + + return false; + } + + return true; + + } + + private function removeUnnevessaryCoolifyYaml() + { + // This will remote the coolify.yaml file from the server as it is not needed on cloud servers + if (isCloud() && $this->server->id !== 0) { + $file = $this->server->proxyPath().'/dynamic/coolify.yaml'; + + return instant_remote_process([ + "rm -f $file", + ], $this->server, false); + } + } + + private function checkLogDrainContainer() + { + $foundLogDrainContainer = $this->containers->filter(function ($value, $key) { + return data_get($value, 'Name') === '/coolify-log-drain'; + })->first(); + if ($foundLogDrainContainer) { + $status = data_get($foundLogDrainContainer, 'State.Status'); + if ($status !== 'running') { + InstallLogDrain::dispatch($this->server); + } + } else { + InstallLogDrain::dispatch($this->server); + } + } + + private function getContainers() + { + if ($this->server->isSwarm()) { + $this->containers = instant_remote_process(["docker service inspect $(docker service ls -q) --format '{{json .}}'"], $this->server, false); + $this->containers = format_docker_command_output_to_json($this->containers); + $containerReplicates = instant_remote_process(["docker service ls --format '{{json .}}'"], $this->server, false); + if ($containerReplicates) { + $containerReplicates = format_docker_command_output_to_json($containerReplicates); + foreach ($containerReplicates as $containerReplica) { + $name = data_get($containerReplica, 'Name'); + $this->containers = $this->containers->map(function ($container) use ($name, $containerReplica) { + if (data_get($container, 'Spec.Name') === $name) { + $replicas = data_get($containerReplica, 'Replicas'); + $running = str($replicas)->explode('/')[0]; + $total = str($replicas)->explode('/')[1]; + if ($running === $total) { + data_set($container, 'State.Status', 'running'); + data_set($container, 'State.Health.Status', 'healthy'); + } else { + data_set($container, 'State.Status', 'starting'); + data_set($container, 'State.Health.Status', 'unhealthy'); + } + } + + return $container; + }); + } + } + } else { + $this->containers = instant_remote_process(["docker container inspect $(docker container ls -q) --format '{{json .}}'"], $this->server, false); + $this->containers = format_docker_command_output_to_json($this->containers); + } + + } + + private function containerStatus() + { + + $this->applications = $this->server->applications(); + $this->databases = $this->server->databases(); + $this->services = $this->server->services()->get(); + $this->previews = $this->server->previews(); + + $foundApplications = []; + $foundApplicationPreviews = []; + $foundDatabases = []; + $foundServices = []; + + foreach ($this->containers as $container) { + if ($this->server->isSwarm()) { + $labels = data_get($container, 'Spec.Labels'); + $uuid = data_get($labels, 'coolify.name'); + } else { + $labels = data_get($container, 'Config.Labels'); + } + $containerStatus = data_get($container, 'State.Status'); + $containerHealth = data_get($container, 'State.Health.Status', 'unhealthy'); + $containerStatus = "$containerStatus ($containerHealth)"; + $labels = Arr::undot(format_docker_labels_to_json($labels)); + $applicationId = data_get($labels, 'coolify.applicationId'); + if ($applicationId) { + $pullRequestId = data_get($labels, 'coolify.pullRequestId'); + if ($pullRequestId) { + if (str($applicationId)->contains('-')) { + $applicationId = str($applicationId)->before('-'); + } + $preview = ApplicationPreview::where('application_id', $applicationId)->where('pull_request_id', $pullRequestId)->first(); + if ($preview) { + $foundApplicationPreviews[] = $preview->id; + $statusFromDb = $preview->status; + if ($statusFromDb !== $containerStatus) { + $preview->update(['status' => $containerStatus]); + } + } else { + //Notify user that this container should not be there. + } + } else { + $application = $this->applications->where('id', $applicationId)->first(); + if ($application) { + $foundApplications[] = $application->id; + $statusFromDb = $application->status; + if ($statusFromDb !== $containerStatus) { + $application->update(['status' => $containerStatus]); + } + } else { + //Notify user that this container should not be there. + } + } + } else { + $uuid = data_get($labels, 'com.docker.compose.service'); + $type = data_get($labels, 'coolify.type'); + + if ($uuid) { + if ($type === 'service') { + $database_id = data_get($labels, 'coolify.service.subId'); + if ($database_id) { + $service_db = ServiceDatabase::where('id', $database_id)->first(); + if ($service_db) { + $uuid = data_get($service_db, 'service.uuid'); + if ($uuid) { + $isPublic = data_get($service_db, 'is_public'); + if ($isPublic) { + $foundTcpProxy = $this->containers->filter(function ($value, $key) use ($uuid) { + if ($this->server->isSwarm()) { + return data_get($value, 'Spec.Name') === "coolify-proxy_$uuid"; + } else { + return data_get($value, 'Name') === "/$uuid-proxy"; + } + })->first(); + if (! $foundTcpProxy) { + StartDatabaseProxy::run($service_db); + // $this->server->team?->notify(new ContainerRestarted("TCP Proxy for {$service_db->service->name}", $this->server)); + } + } + } + } + } + } else { + $database = $this->databases->where('uuid', $uuid)->first(); + if ($database) { + $isPublic = data_get($database, 'is_public'); + $foundDatabases[] = $database->id; + $statusFromDb = $database->status; + if ($statusFromDb !== $containerStatus) { + $database->update(['status' => $containerStatus]); + } + if ($isPublic) { + $foundTcpProxy = $this->containers->filter(function ($value, $key) use ($uuid) { + if ($this->server->isSwarm()) { + return data_get($value, 'Spec.Name') === "coolify-proxy_$uuid"; + } else { + return data_get($value, 'Name') === "/$uuid-proxy"; + } + })->first(); + if (! $foundTcpProxy) { + StartDatabaseProxy::run($database); + $this->server->team?->notify(new ContainerRestarted("TCP Proxy for {$database->name}", $this->server)); + } + } + } else { + // Notify user that this container should not be there. + } + } + } + if (data_get($container, 'Name') === '/coolify-db') { + $foundDatabases[] = 0; + } + } + $serviceLabelId = data_get($labels, 'coolify.serviceId'); + if ($serviceLabelId) { + $subType = data_get($labels, 'coolify.service.subType'); + $subId = data_get($labels, 'coolify.service.subId'); + $service = $this->services->where('id', $serviceLabelId)->first(); + if (! $service) { + continue; + } + if ($subType === 'application') { + $service = $service->applications()->where('id', $subId)->first(); + } else { + $service = $service->databases()->where('id', $subId)->first(); + } + if ($service) { + $foundServices[] = "$service->id-$service->name"; + $statusFromDb = $service->status; + if ($statusFromDb !== $containerStatus) { + // ray('Updating status: ' . $containerStatus); + $service->update(['status' => $containerStatus]); + } + } + } + } + $exitedServices = collect([]); + foreach ($this->services as $service) { + $apps = $service->applications()->get(); + $dbs = $service->databases()->get(); + foreach ($apps as $app) { + if (in_array("$app->id-$app->name", $foundServices)) { + continue; + } else { + $exitedServices->push($app); + } + } + foreach ($dbs as $db) { + if (in_array("$db->id-$db->name", $foundServices)) { + continue; + } else { + $exitedServices->push($db); + } + } + } + $exitedServices = $exitedServices->unique('id'); + foreach ($exitedServices as $exitedService) { + if (str($exitedService->status)->startsWith('exited')) { + continue; + } + $name = data_get($exitedService, 'name'); + $fqdn = data_get($exitedService, 'fqdn'); + if ($name) { + if ($fqdn) { + $containerName = "$name, available at $fqdn"; + } else { + $containerName = $name; + } + } else { + if ($fqdn) { + $containerName = $fqdn; + } else { + $containerName = null; + } + } + $projectUuid = data_get($service, 'environment.project.uuid'); + $serviceUuid = data_get($service, 'uuid'); + $environmentName = data_get($service, 'environment.name'); + + if ($projectUuid && $serviceUuid && $environmentName) { + $url = base_url().'/project/'.$projectUuid.'/'.$environmentName.'/service/'.$serviceUuid; + } else { + $url = null; + } + // $this->server->team?->notify(new ContainerStopped($containerName, $this->server, $url)); + $exitedService->update(['status' => 'exited']); + } + + $notRunningApplications = $this->applications->pluck('id')->diff($foundApplications); + foreach ($notRunningApplications as $applicationId) { + $application = $this->applications->where('id', $applicationId)->first(); + if (str($application->status)->startsWith('exited')) { + continue; + } + $application->update(['status' => 'exited']); + + $name = data_get($application, 'name'); + $fqdn = data_get($application, 'fqdn'); + + $containerName = $name ? "$name ($fqdn)" : $fqdn; + + $projectUuid = data_get($application, 'environment.project.uuid'); + $applicationUuid = data_get($application, 'uuid'); + $environment = data_get($application, 'environment.name'); + + if ($projectUuid && $applicationUuid && $environment) { + $url = base_url().'/project/'.$projectUuid.'/'.$environment.'/application/'.$applicationUuid; + } else { + $url = null; + } + + // $this->server->team?->notify(new ContainerStopped($containerName, $this->server, $url)); + } + $notRunningApplicationPreviews = $this->previews->pluck('id')->diff($foundApplicationPreviews); + foreach ($notRunningApplicationPreviews as $previewId) { + $preview = $this->previews->where('id', $previewId)->first(); + if (str($preview->status)->startsWith('exited')) { + continue; + } + $preview->update(['status' => 'exited']); + + $name = data_get($preview, 'name'); + $fqdn = data_get($preview, 'fqdn'); + + $containerName = $name ? "$name ($fqdn)" : $fqdn; + + $projectUuid = data_get($preview, 'application.environment.project.uuid'); + $environmentName = data_get($preview, 'application.environment.name'); + $applicationUuid = data_get($preview, 'application.uuid'); + + if ($projectUuid && $applicationUuid && $environmentName) { + $url = base_url().'/project/'.$projectUuid.'/'.$environmentName.'/application/'.$applicationUuid; + } else { + $url = null; + } + + // $this->server->team?->notify(new ContainerStopped($containerName, $this->server, $url)); + } + $notRunningDatabases = $this->databases->pluck('id')->diff($foundDatabases); + foreach ($notRunningDatabases as $database) { + $database = $this->databases->where('id', $database)->first(); + if (str($database->status)->startsWith('exited')) { + continue; + } + $database->update(['status' => 'exited']); + + $name = data_get($database, 'name'); + $fqdn = data_get($database, 'fqdn'); + + $containerName = $name; + + $projectUuid = data_get($database, 'environment.project.uuid'); + $environmentName = data_get($database, 'environment.name'); + $databaseUuid = data_get($database, 'uuid'); + + if ($projectUuid && $databaseUuid && $environmentName) { + $url = base_url().'/project/'.$projectUuid.'/'.$environmentName.'/database/'.$databaseUuid; + } else { + $url = null; + } + // $this->server->team?->notify(new ContainerStopped($containerName, $this->server, $url)); + } + + // Check if proxy is running + $this->server->proxyType(); + $foundProxyContainer = $this->containers->filter(function ($value, $key) { + if ($this->server->isSwarm()) { + return data_get($value, 'Spec.Name') === 'coolify-proxy_traefik'; + } else { + return data_get($value, 'Name') === '/coolify-proxy'; + } + })->first(); + if (! $foundProxyContainer) { + try { + $shouldStart = CheckProxy::run($this->server); + if ($shouldStart) { + StartProxy::run($this->server, false); + $this->server->team?->notify(new ContainerRestarted('coolify-proxy', $this->server)); + } + } catch (\Throwable $e) { + ray($e); + } + } else { + $this->server->proxy->status = data_get($foundProxyContainer, 'State.Status'); + $this->server->save(); + $connectProxyToDockerNetworks = connectProxyToNetworks($this->server); + instant_remote_process($connectProxyToDockerNetworks, $this->server, false); + } + } +}