diff --git a/app/Actions/Docker/GetContainersStatus.php b/app/Actions/Docker/GetContainersStatus.php index 9b32e89f3..fdaa88ebf 100644 --- a/app/Actions/Docker/GetContainersStatus.php +++ b/app/Actions/Docker/GetContainersStatus.php @@ -12,6 +12,7 @@ use App\Models\ServiceDatabase; use App\Notifications\Container\ContainerRestarted; use App\Notifications\Container\ContainerStopped; use Illuminate\Support\Arr; +use Illuminate\Support\Collection; use Lorisleiva\Actions\Concerns\AsAction; class GetContainersStatus @@ -20,13 +21,16 @@ class GetContainersStatus public $applications; + public ?Collection $containers; + + public ?Collection $containerReplicates; + public $server; - public function handle(Server $server) + public function handle(Server $server, ?Collection $containers = null, ?Collection $containerReplicates = null) { - // if (isDev()) { - // $server = Server::find(0); - // } + $this->containers = $containers; + $this->containerReplicates = $containerReplicates; $this->server = $server; if (! $this->server->isFunctional()) { return 'Server is not ready.'; @@ -66,322 +70,312 @@ class GetContainersStatus // } } - private function sentinel() - { - try { - $containers = $this->server->getContainers(); - if ($containers->count() === 0) { - return; - } - $databases = $this->server->databases(); - $services = $this->server->services()->get(); - $previews = $this->server->previews(); - $foundApplications = []; - $foundApplicationPreviews = []; - $foundDatabases = []; - $foundServices = []; + // private function sentinel() + // { + // try { + // $this->containers = $this->server->getContainersWithSentinel(); + // if ($this->containers->count() === 0) { + // return; + // } + // $databases = $this->server->databases(); + // $services = $this->server->services()->get(); + // $previews = $this->server->previews(); + // $foundApplications = []; + // $foundApplicationPreviews = []; + // $foundDatabases = []; + // $foundServices = []; - foreach ($containers as $container) { - $labels = Arr::undot(data_get($container, 'labels')); - $containerStatus = data_get($container, 'state'); - $containerHealth = data_get($container, 'health_status', 'unhealthy'); - $containerStatus = "$containerStatus ($containerHealth)"; - $applicationId = data_get($labels, 'coolify.applicationId'); - if ($applicationId) { - $pullRequestId = data_get($labels, 'coolify.pullRequestId'); - if ($pullRequestId) { - if (str($applicationId)->contains('-')) { - $applicationId = str($applicationId)->before('-'); - } - $preview = ApplicationPreview::where('application_id', $applicationId)->where('pull_request_id', $pullRequestId)->first(); - if ($preview) { - $foundApplicationPreviews[] = $preview->id; - $statusFromDb = $preview->status; - if ($statusFromDb !== $containerStatus) { - $preview->update(['status' => $containerStatus]); - } - } else { - //Notify user that this container should not be there. - } - } else { - $application = $this->applications->where('id', $applicationId)->first(); - if ($application) { - $foundApplications[] = $application->id; - $statusFromDb = $application->status; - if ($statusFromDb !== $containerStatus) { - $application->update(['status' => $containerStatus]); - } - } else { - //Notify user that this container should not be there. - } - } - } else { - $uuid = data_get($labels, 'com.docker.compose.service'); - $type = data_get($labels, 'coolify.type'); - if ($uuid) { - if ($type === 'service') { - $database_id = data_get($labels, 'coolify.service.subId'); - if ($database_id) { - $service_db = ServiceDatabase::where('id', $database_id)->first(); - if ($service_db) { - $uuid = $service_db->service->uuid; - $isPublic = data_get($service_db, 'is_public'); - if ($isPublic) { - $foundTcpProxy = $containers->filter(function ($value, $key) use ($uuid) { - if ($this->server->isSwarm()) { - // TODO: fix this with sentinel - return data_get($value, 'Spec.Name') === "coolify-proxy_$uuid"; - } else { - return data_get($value, 'name') === "$uuid-proxy"; - } - })->first(); - if (! $foundTcpProxy) { - StartDatabaseProxy::run($service_db); - // $this->server->team?->notify(new ContainerRestarted("TCP Proxy for {$service_db->service->name}", $this->server)); - } - } - } - } - } else { - $database = $databases->where('uuid', $uuid)->first(); - if ($database) { - $isPublic = data_get($database, 'is_public'); - $foundDatabases[] = $database->id; - $statusFromDb = $database->status; - if ($statusFromDb !== $containerStatus) { - $database->update(['status' => $containerStatus]); - } - if ($isPublic) { - $foundTcpProxy = $containers->filter(function ($value, $key) use ($uuid) { - if ($this->server->isSwarm()) { - // TODO: fix this with sentinel - return data_get($value, 'Spec.Name') === "coolify-proxy_$uuid"; - } else { - return data_get($value, 'name') === "$uuid-proxy"; - } - })->first(); - if (! $foundTcpProxy) { - StartDatabaseProxy::run($database); - $this->server->team?->notify(new ContainerRestarted("TCP Proxy for {$database->name}", $this->server)); - } - } - } else { - // Notify user that this container should not be there. - } - } - } - if (data_get($container, 'name') === 'coolify-db') { - $foundDatabases[] = 0; - } - } - $serviceLabelId = data_get($labels, 'coolify.serviceId'); - if ($serviceLabelId) { - $subType = data_get($labels, 'coolify.service.subType'); - $subId = data_get($labels, 'coolify.service.subId'); - $service = $services->where('id', $serviceLabelId)->first(); - if (! $service) { - continue; - } - if ($subType === 'application') { - $service = $service->applications()->where('id', $subId)->first(); - } else { - $service = $service->databases()->where('id', $subId)->first(); - } - if ($service) { - $foundServices[] = "$service->id-$service->name"; - $statusFromDb = $service->status; - if ($statusFromDb !== $containerStatus) { - // ray('Updating status: ' . $containerStatus); - $service->update(['status' => $containerStatus]); - } - } - } - } - $exitedServices = collect([]); - foreach ($services as $service) { - $apps = $service->applications()->get(); - $dbs = $service->databases()->get(); - foreach ($apps as $app) { - if (in_array("$app->id-$app->name", $foundServices)) { - continue; - } else { - $exitedServices->push($app); - } - } - foreach ($dbs as $db) { - if (in_array("$db->id-$db->name", $foundServices)) { - continue; - } else { - $exitedServices->push($db); - } - } - } - $exitedServices = $exitedServices->unique('id'); - foreach ($exitedServices as $exitedService) { - if (str($exitedService->status)->startsWith('exited')) { - continue; - } - $name = data_get($exitedService, 'name'); - $fqdn = data_get($exitedService, 'fqdn'); - if ($name) { - if ($fqdn) { - $containerName = "$name, available at $fqdn"; - } else { - $containerName = $name; - } - } else { - if ($fqdn) { - $containerName = $fqdn; - } else { - $containerName = null; - } - } - $projectUuid = data_get($service, 'environment.project.uuid'); - $serviceUuid = data_get($service, 'uuid'); - $environmentName = data_get($service, 'environment.name'); + // foreach ($this->containers as $container) { + // $labels = Arr::undot(data_get($container, 'labels')); + // $containerStatus = data_get($container, 'state'); + // $containerHealth = data_get($container, 'health_status', 'unhealthy'); + // $containerStatus = "$containerStatus ($containerHealth)"; + // $applicationId = data_get($labels, 'coolify.applicationId'); + // if ($applicationId) { + // $pullRequestId = data_get($labels, 'coolify.pullRequestId'); + // if ($pullRequestId) { + // if (str($applicationId)->contains('-')) { + // $applicationId = str($applicationId)->before('-'); + // } + // $preview = ApplicationPreview::where('application_id', $applicationId)->where('pull_request_id', $pullRequestId)->first(); + // if ($preview) { + // $foundApplicationPreviews[] = $preview->id; + // $statusFromDb = $preview->status; + // if ($statusFromDb !== $containerStatus) { + // $preview->update(['status' => $containerStatus]); + // } + // } else { + // //Notify user that this container should not be there. + // } + // } else { + // $application = $this->applications->where('id', $applicationId)->first(); + // if ($application) { + // $foundApplications[] = $application->id; + // $statusFromDb = $application->status; + // if ($statusFromDb !== $containerStatus) { + // $application->update(['status' => $containerStatus]); + // } + // } else { + // //Notify user that this container should not be there. + // } + // } + // } else { + // $uuid = data_get($labels, 'com.docker.compose.service'); + // $type = data_get($labels, 'coolify.type'); + // if ($uuid) { + // if ($type === 'service') { + // $database_id = data_get($labels, 'coolify.service.subId'); + // if ($database_id) { + // $service_db = ServiceDatabase::where('id', $database_id)->first(); + // if ($service_db) { + // $uuid = $service_db->service->uuid; + // $isPublic = data_get($service_db, 'is_public'); + // if ($isPublic) { + // $foundTcpProxy = $this->containers->filter(function ($value, $key) use ($uuid) { + // if ($this->server->isSwarm()) { + // // TODO: fix this with sentinel + // return data_get($value, 'Spec.Name') === "coolify-proxy_$uuid"; + // } else { + // return data_get($value, 'name') === "$uuid-proxy"; + // } + // })->first(); + // if (! $foundTcpProxy) { + // StartDatabaseProxy::run($service_db); + // // $this->server->team?->notify(new ContainerRestarted("TCP Proxy for {$service_db->service->name}", $this->server)); + // } + // } + // } + // } + // } else { + // $database = $databases->where('uuid', $uuid)->first(); + // if ($database) { + // $isPublic = data_get($database, 'is_public'); + // $foundDatabases[] = $database->id; + // $statusFromDb = $database->status; + // if ($statusFromDb !== $containerStatus) { + // $database->update(['status' => $containerStatus]); + // } + // if ($isPublic) { + // $foundTcpProxy = $this->containers->filter(function ($value, $key) use ($uuid) { + // if ($this->server->isSwarm()) { + // // TODO: fix this with sentinel + // return data_get($value, 'Spec.Name') === "coolify-proxy_$uuid"; + // } else { + // return data_get($value, 'name') === "$uuid-proxy"; + // } + // })->first(); + // if (! $foundTcpProxy) { + // StartDatabaseProxy::run($database); + // $this->server->team?->notify(new ContainerRestarted("TCP Proxy for {$database->name}", $this->server)); + // } + // } + // } else { + // // Notify user that this container should not be there. + // } + // } + // } + // if (data_get($container, 'name') === 'coolify-db') { + // $foundDatabases[] = 0; + // } + // } + // $serviceLabelId = data_get($labels, 'coolify.serviceId'); + // if ($serviceLabelId) { + // $subType = data_get($labels, 'coolify.service.subType'); + // $subId = data_get($labels, 'coolify.service.subId'); + // $service = $services->where('id', $serviceLabelId)->first(); + // if (! $service) { + // continue; + // } + // if ($subType === 'application') { + // $service = $service->applications()->where('id', $subId)->first(); + // } else { + // $service = $service->databases()->where('id', $subId)->first(); + // } + // if ($service) { + // $foundServices[] = "$service->id-$service->name"; + // $statusFromDb = $service->status; + // if ($statusFromDb !== $containerStatus) { + // // ray('Updating status: ' . $containerStatus); + // $service->update(['status' => $containerStatus]); + // } + // } + // } + // } + // $exitedServices = collect([]); + // foreach ($services as $service) { + // $apps = $service->applications()->get(); + // $dbs = $service->databases()->get(); + // foreach ($apps as $app) { + // if (in_array("$app->id-$app->name", $foundServices)) { + // continue; + // } else { + // $exitedServices->push($app); + // } + // } + // foreach ($dbs as $db) { + // if (in_array("$db->id-$db->name", $foundServices)) { + // continue; + // } else { + // $exitedServices->push($db); + // } + // } + // } + // $exitedServices = $exitedServices->unique('id'); + // foreach ($exitedServices as $exitedService) { + // if (str($exitedService->status)->startsWith('exited')) { + // continue; + // } + // $name = data_get($exitedService, 'name'); + // $fqdn = data_get($exitedService, 'fqdn'); + // if ($name) { + // if ($fqdn) { + // $containerName = "$name, available at $fqdn"; + // } else { + // $containerName = $name; + // } + // } else { + // if ($fqdn) { + // $containerName = $fqdn; + // } else { + // $containerName = null; + // } + // } + // $projectUuid = data_get($service, 'environment.project.uuid'); + // $serviceUuid = data_get($service, 'uuid'); + // $environmentName = data_get($service, 'environment.name'); - if ($projectUuid && $serviceUuid && $environmentName) { - $url = base_url().'/project/'.$projectUuid.'/'.$environmentName.'/service/'.$serviceUuid; - } else { - $url = null; - } - // $this->server->team?->notify(new ContainerStopped($containerName, $this->server, $url)); - $exitedService->update(['status' => 'exited']); - } + // if ($projectUuid && $serviceUuid && $environmentName) { + // $url = base_url().'/project/'.$projectUuid.'/'.$environmentName.'/service/'.$serviceUuid; + // } else { + // $url = null; + // } + // // $this->server->team?->notify(new ContainerStopped($containerName, $this->server, $url)); + // $exitedService->update(['status' => 'exited']); + // } - $notRunningApplications = $this->applications->pluck('id')->diff($foundApplications); - foreach ($notRunningApplications as $applicationId) { - $application = $this->applications->where('id', $applicationId)->first(); - if (str($application->status)->startsWith('exited')) { - continue; - } - $application->update(['status' => 'exited']); + // $notRunningApplications = $this->applications->pluck('id')->diff($foundApplications); + // foreach ($notRunningApplications as $applicationId) { + // $application = $this->applications->where('id', $applicationId)->first(); + // if (str($application->status)->startsWith('exited')) { + // continue; + // } + // $application->update(['status' => 'exited']); - $name = data_get($application, 'name'); - $fqdn = data_get($application, 'fqdn'); + // $name = data_get($application, 'name'); + // $fqdn = data_get($application, 'fqdn'); - $containerName = $name ? "$name ($fqdn)" : $fqdn; + // $containerName = $name ? "$name ($fqdn)" : $fqdn; - $projectUuid = data_get($application, 'environment.project.uuid'); - $applicationUuid = data_get($application, 'uuid'); - $environment = data_get($application, 'environment.name'); + // $projectUuid = data_get($application, 'environment.project.uuid'); + // $applicationUuid = data_get($application, 'uuid'); + // $environment = data_get($application, 'environment.name'); - if ($projectUuid && $applicationUuid && $environment) { - $url = base_url().'/project/'.$projectUuid.'/'.$environment.'/application/'.$applicationUuid; - } else { - $url = null; - } + // if ($projectUuid && $applicationUuid && $environment) { + // $url = base_url().'/project/'.$projectUuid.'/'.$environment.'/application/'.$applicationUuid; + // } else { + // $url = null; + // } - // $this->server->team?->notify(new ContainerStopped($containerName, $this->server, $url)); - } - $notRunningApplicationPreviews = $previews->pluck('id')->diff($foundApplicationPreviews); - foreach ($notRunningApplicationPreviews as $previewId) { - $preview = $previews->where('id', $previewId)->first(); - if (str($preview->status)->startsWith('exited')) { - continue; - } - $preview->update(['status' => 'exited']); + // // $this->server->team?->notify(new ContainerStopped($containerName, $this->server, $url)); + // } + // $notRunningApplicationPreviews = $previews->pluck('id')->diff($foundApplicationPreviews); + // foreach ($notRunningApplicationPreviews as $previewId) { + // $preview = $previews->where('id', $previewId)->first(); + // if (str($preview->status)->startsWith('exited')) { + // continue; + // } + // $preview->update(['status' => 'exited']); - $name = data_get($preview, 'name'); - $fqdn = data_get($preview, 'fqdn'); + // $name = data_get($preview, 'name'); + // $fqdn = data_get($preview, 'fqdn'); - $containerName = $name ? "$name ($fqdn)" : $fqdn; + // $containerName = $name ? "$name ($fqdn)" : $fqdn; - $projectUuid = data_get($preview, 'application.environment.project.uuid'); - $environmentName = data_get($preview, 'application.environment.name'); - $applicationUuid = data_get($preview, 'application.uuid'); + // $projectUuid = data_get($preview, 'application.environment.project.uuid'); + // $environmentName = data_get($preview, 'application.environment.name'); + // $applicationUuid = data_get($preview, 'application.uuid'); - if ($projectUuid && $applicationUuid && $environmentName) { - $url = base_url().'/project/'.$projectUuid.'/'.$environmentName.'/application/'.$applicationUuid; - } else { - $url = null; - } + // if ($projectUuid && $applicationUuid && $environmentName) { + // $url = base_url().'/project/'.$projectUuid.'/'.$environmentName.'/application/'.$applicationUuid; + // } else { + // $url = null; + // } - // $this->server->team?->notify(new ContainerStopped($containerName, $this->server, $url)); - } - $notRunningDatabases = $databases->pluck('id')->diff($foundDatabases); - foreach ($notRunningDatabases as $database) { - $database = $databases->where('id', $database)->first(); - if (str($database->status)->startsWith('exited')) { - continue; - } - $database->update(['status' => 'exited']); + // // $this->server->team?->notify(new ContainerStopped($containerName, $this->server, $url)); + // } + // $notRunningDatabases = $databases->pluck('id')->diff($foundDatabases); + // foreach ($notRunningDatabases as $database) { + // $database = $databases->where('id', $database)->first(); + // if (str($database->status)->startsWith('exited')) { + // continue; + // } + // $database->update(['status' => 'exited']); - $name = data_get($database, 'name'); - $fqdn = data_get($database, 'fqdn'); + // $name = data_get($database, 'name'); + // $fqdn = data_get($database, 'fqdn'); - $containerName = $name; + // $containerName = $name; - $projectUuid = data_get($database, 'environment.project.uuid'); - $environmentName = data_get($database, 'environment.name'); - $databaseUuid = data_get($database, 'uuid'); + // $projectUuid = data_get($database, 'environment.project.uuid'); + // $environmentName = data_get($database, 'environment.name'); + // $databaseUuid = data_get($database, 'uuid'); - if ($projectUuid && $databaseUuid && $environmentName) { - $url = base_url().'/project/'.$projectUuid.'/'.$environmentName.'/database/'.$databaseUuid; - } else { - $url = null; - } - // $this->server->team?->notify(new ContainerStopped($containerName, $this->server, $url)); - } + // if ($projectUuid && $databaseUuid && $environmentName) { + // $url = base_url().'/project/'.$projectUuid.'/'.$environmentName.'/database/'.$databaseUuid; + // } else { + // $url = null; + // } + // // $this->server->team?->notify(new ContainerStopped($containerName, $this->server, $url)); + // } - // Check if proxy is running - $this->server->proxyType(); - $foundProxyContainer = $containers->filter(function ($value, $key) { - if ($this->server->isSwarm()) { - // TODO: fix this with sentinel - return data_get($value, 'Spec.Name') === 'coolify-proxy_traefik'; - } else { - return data_get($value, 'name') === 'coolify-proxy'; - } - })->first(); - if (! $foundProxyContainer) { - try { - $shouldStart = CheckProxy::run($this->server); - if ($shouldStart) { - StartProxy::run($this->server, false); - $this->server->team?->notify(new ContainerRestarted('coolify-proxy', $this->server)); - } - } catch (\Throwable $e) { - ray($e); - } - } else { - $this->server->proxy->status = data_get($foundProxyContainer, 'state'); - $this->server->save(); - $connectProxyToDockerNetworks = connectProxyToNetworks($this->server); - instant_remote_process($connectProxyToDockerNetworks, $this->server, false); - } - } catch (\Exception $e) { - // send_internal_notification("ContainerStatusJob failed on ({$this->server->id}) with: " . $e->getMessage()); - ray($e->getMessage()); + // // Check if proxy is running + // $this->server->proxyType(); + // $foundProxyContainer = $this->containers->filter(function ($value, $key) { + // if ($this->server->isSwarm()) { + // // TODO: fix this with sentinel + // return data_get($value, 'Spec.Name') === 'coolify-proxy_traefik'; + // } else { + // return data_get($value, 'name') === 'coolify-proxy'; + // } + // })->first(); + // if (! $foundProxyContainer) { + // try { + // $shouldStart = CheckProxy::run($this->server); + // if ($shouldStart) { + // StartProxy::run($this->server, false); + // $this->server->team?->notify(new ContainerRestarted('coolify-proxy', $this->server)); + // } + // } catch (\Throwable $e) { + // ray($e); + // } + // } else { + // $this->server->proxy->status = data_get($foundProxyContainer, 'state'); + // $this->server->save(); + // $connectProxyToDockerNetworks = connectProxyToNetworks($this->server); + // instant_remote_process($connectProxyToDockerNetworks, $this->server, false); + // } + // } catch (\Exception $e) { + // // send_internal_notification("ContainerStatusJob failed on ({$this->server->id}) with: " . $e->getMessage()); + // ray($e->getMessage()); - return handleError($e); - } - } + // return handleError($e); + // } + // } private function old_way() { - if ($this->server->isSwarm()) { - $containers = instant_remote_process(["docker service inspect $(docker service ls -q) --format '{{json .}}'"], $this->server, false); - $containerReplicates = instant_remote_process(["docker service ls --format '{{json .}}'"], $this->server, false); - } else { - // Precheck for containers - $containers = instant_remote_process(['docker container ls -q'], $this->server, false); - if (! $containers) { - return; - } - $containers = instant_remote_process(["docker container inspect $(docker container ls -q) --format '{{json .}}'"], $this->server, false); - $containerReplicates = null; + if ($this->containers === null) { + ['containers' => $this->containers,'containerReplicates' => $this->containerReplicates] = $this->server->getContainers(); } - if (is_null($containers)) { + + if (is_null($this->containers)) { return; } - $containers = format_docker_command_output_to_json($containers); - if ($containerReplicates) { - $containerReplicates = format_docker_command_output_to_json($containerReplicates); - foreach ($containerReplicates as $containerReplica) { + if ($this->containerReplicates) { + foreach ($this->containerReplicates as $containerReplica) { $name = data_get($containerReplica, 'Name'); - $containers = $containers->map(function ($container) use ($name, $containerReplica) { + $this->containers = $this->containers->map(function ($container) use ($name, $containerReplica) { if (data_get($container, 'Spec.Name') === $name) { $replicas = data_get($containerReplica, 'Replicas'); $running = str($replicas)->explode('/')[0]; @@ -407,7 +401,7 @@ class GetContainersStatus $foundDatabases = []; $foundServices = []; - foreach ($containers as $container) { + foreach ($this->containers as $container) { if ($this->server->isSwarm()) { $labels = data_get($container, 'Spec.Labels'); $uuid = data_get($labels, 'coolify.name'); @@ -461,7 +455,7 @@ class GetContainersStatus if ($uuid) { $isPublic = data_get($service_db, 'is_public'); if ($isPublic) { - $foundTcpProxy = $containers->filter(function ($value, $key) use ($uuid) { + $foundTcpProxy = $this->containers->filter(function ($value, $key) use ($uuid) { if ($this->server->isSwarm()) { return data_get($value, 'Spec.Name') === "coolify-proxy_$uuid"; } else { @@ -486,7 +480,7 @@ class GetContainersStatus $database->update(['status' => $containerStatus]); } if ($isPublic) { - $foundTcpProxy = $containers->filter(function ($value, $key) use ($uuid) { + $foundTcpProxy = $this->containers->filter(function ($value, $key) use ($uuid) { if ($this->server->isSwarm()) { return data_get($value, 'Spec.Name') === "coolify-proxy_$uuid"; } else { @@ -659,7 +653,7 @@ class GetContainersStatus // Check if proxy is running $this->server->proxyType(); - $foundProxyContainer = $containers->filter(function ($value, $key) { + $foundProxyContainer = $this->containers->filter(function ($value, $key) { if ($this->server->isSwarm()) { return data_get($value, 'Spec.Name') === 'coolify-proxy_traefik'; } else { diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index c6ee39524..34761763c 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -12,6 +12,7 @@ use App\Jobs\PullHelperImageJob; use App\Jobs\PullSentinelImageJob; use App\Jobs\PullTemplatesFromCDN; use App\Jobs\ScheduledTaskJob; +use App\Jobs\ServerCheckJob; use App\Jobs\ServerStatusJob; use App\Models\ScheduledDatabaseBackup; use App\Models\ScheduledTask; @@ -34,7 +35,8 @@ class Kernel extends ConsoleKernel $schedule->job(new PullTemplatesFromCDN)->everyTwoHours()->onOneServer(); // Server Jobs $this->check_scheduled_backups($schedule); - $this->check_resources($schedule); + $this->checkResourcesNew($schedule); + // $this->check_resources($schedule); $this->check_scheduled_backups($schedule); $this->check_scheduled_tasks($schedule); $schedule->command('uploads:clear')->everyTwoMinutes(); @@ -49,7 +51,8 @@ class Kernel extends ConsoleKernel // Server Jobs $this->check_scheduled_backups($schedule); - $this->check_resources($schedule); + $this->checkResourcesNew($schedule); + // $this->check_resources($schedule); $this->pull_images($schedule); $this->check_scheduled_tasks($schedule); @@ -69,6 +72,21 @@ class Kernel extends ConsoleKernel } } + private function checkResourcesNew($schedule) + { + if (isCloud()) { + $servers = $this->all_servers->whereNotNull('team.subscription')->where('team.subscription.stripe_trial_already_ended', false)->where('ip', '!=', '1.2.3.4'); + $own = Team::find(0)->servers; + $servers = $servers->merge($own); + } else { + $servers = $this->all_servers->where('ip', '!=', '1.2.3.4'); + } + foreach ($servers as $server) { + $schedule->job(new ServerCheckJob($server))->everyMinute()->onOneServer(); + $schedule->job(new DockerCleanupJob($server))->everyTenMinutes()->onOneServer(); + } + } + private function check_resources($schedule) { if (isCloud()) { diff --git a/app/Jobs/ServerCheckJob.php b/app/Jobs/ServerCheckJob.php index adade3194..6b90c976c 100644 --- a/app/Jobs/ServerCheckJob.php +++ b/app/Jobs/ServerCheckJob.php @@ -3,6 +3,7 @@ namespace App\Jobs; use App\Actions\Database\StartDatabaseProxy; +use App\Actions\Docker\GetContainersStatus; use App\Actions\Proxy\CheckProxy; use App\Actions\Proxy\StartProxy; use App\Actions\Server\InstallLogDrain; @@ -15,7 +16,6 @@ use Illuminate\Contracts\Queue\ShouldBeEncrypted; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; -use Illuminate\Queue\Middleware\WithoutOverlapping; use Illuminate\Queue\SerializesModels; use Illuminate\Support\Arr; @@ -42,20 +42,24 @@ class ServerCheckJob implements ShouldBeEncrypted, ShouldQueue public function __construct(public Server $server) {} - public function middleware(): array - { - return [(new WithoutOverlapping($this->server->uuid))]; - } + // public function middleware(): array + // { + // return [(new WithoutOverlapping($this->server->uuid))]; + // } - public function uniqueId(): int - { - return $this->server->uuid; - } + // public function uniqueId(): int + // { + // return $this->server->uuid; + // } public function handle() { - try { + $this->applications = $this->server->applications(); + $this->databases = $this->server->databases(); + $this->services = $this->server->services()->get(); + $this->previews = $this->server->previews(); + $up = $this->serverStatus(); if (! $up) { ray('Server is not reachable.'); @@ -67,14 +71,17 @@ class ServerCheckJob implements ShouldBeEncrypted, ShouldQueue return 'Server is not ready.'; } - $this->checkSentinel(); - $this->getContainers(); - - if (is_null($this->containers)) { - return 'No containers found.'; + if (! $this->server->isSwarmWorker() && ! $this->server->isBuildServer()) { + ray('Server is not a worker or build server.'); + ['containers' => $this->containers, 'containerReplicates' => $containerReplicates] = $this->server->getContainers(); + if (is_null($this->containers)) { + return 'No containers found.'; + } + GetContainersStatus::run($this->server, $this->containers, $containerReplicates); + // $this->containerStatus(); + $this->checkLogDrainContainer(); + $this->checkSentinel(); } - $this->checkLogDrainContainer(); - $this->containerStatus(); } catch (\Throwable $e) { ray($e->getMessage()); @@ -108,6 +115,7 @@ class ServerCheckJob implements ShouldBeEncrypted, ShouldQueue $this->server->update(['unreachable_notification_sent' => false]); } } else { + // $this->server->team?->notify(new Unreachable($this->server)); foreach ($this->applications as $application) { $application->update(['status' => 'exited']); } @@ -159,49 +167,9 @@ class ServerCheckJob implements ShouldBeEncrypted, ShouldQueue } } - private function getContainers() - { - if ($this->server->isSwarm()) { - $this->containers = instant_remote_process(["docker service inspect $(docker service ls -q) --format '{{json .}}'"], $this->server, false); - $this->containers = format_docker_command_output_to_json($this->containers); - $containerReplicates = instant_remote_process(["docker service ls --format '{{json .}}'"], $this->server, false); - if ($containerReplicates) { - $containerReplicates = format_docker_command_output_to_json($containerReplicates); - foreach ($containerReplicates as $containerReplica) { - $name = data_get($containerReplica, 'Name'); - $this->containers = $this->containers->map(function ($container) use ($name, $containerReplica) { - if (data_get($container, 'Spec.Name') === $name) { - $replicas = data_get($containerReplica, 'Replicas'); - $running = str($replicas)->explode('/')[0]; - $total = str($replicas)->explode('/')[1]; - if ($running === $total) { - data_set($container, 'State.Status', 'running'); - data_set($container, 'State.Health.Status', 'healthy'); - } else { - data_set($container, 'State.Status', 'starting'); - data_set($container, 'State.Health.Status', 'unhealthy'); - } - } - - return $container; - }); - } - } - } else { - $this->containers = instant_remote_process(["docker container inspect $(docker container ls -q) --format '{{json .}}'"], $this->server, false); - $this->containers = format_docker_command_output_to_json($this->containers); - } - - } - private function containerStatus() { - $this->applications = $this->server->applications(); - $this->databases = $this->server->databases(); - $this->services = $this->server->services()->get(); - $this->previews = $this->server->previews(); - $foundApplications = []; $foundApplicationPreviews = []; $foundDatabases = []; diff --git a/app/Models/Application.php b/app/Models/Application.php index 8ee78bb73..e2871da4b 100644 --- a/app/Models/Application.php +++ b/app/Models/Application.php @@ -104,6 +104,8 @@ class Application extends BaseModel protected $guarded = []; + protected $appends = ['server_status']; + protected static function booted() { static::saving(function ($application) { @@ -466,6 +468,28 @@ class Application extends BaseModel return $this->getRawOriginal('status'); } + protected function serverStatus(): Attribute + { + return Attribute::make( + get: function () { + if ($this->additional_servers->count() === 0) { + return $this->destination->server->isFunctional(); + } else { + $additional_servers_status = $this->additional_servers->pluck('pivot.status'); + $main_server_status = $this->destination->server->isFunctional(); + foreach ($additional_servers_status as $status) { + $server_status = str($status)->before(':')->value(); + if ($main_server_status !== $server_status) { + return false; + } + } + + return true; + } + } + ); + } + public function status(): Attribute { return Attribute::make( diff --git a/app/Models/Server.php b/app/Models/Server.php index 9166f5a0f..a53de14e3 100644 --- a/app/Models/Server.php +++ b/app/Models/Server.php @@ -5,6 +5,7 @@ namespace App\Models; use App\Actions\Server\InstallDocker; use App\Enums\ProxyTypes; use App\Jobs\PullSentinelImageJob; +use App\Notifications\Server\Revived; use Illuminate\Database\Eloquent\Builder; use Illuminate\Database\Eloquent\Casts\Attribute; use Illuminate\Support\Collection; @@ -677,7 +678,49 @@ $schema://$host { return instant_remote_process(["docker start $id"], $this); } - public function getContainers(): Collection + public function getContainers() + { + $containers = collect([]); + $containerReplicates = collect([]); + if ($this->isSwarm()) { + $containers = instant_remote_process(["docker service inspect $(docker service ls -q) --format '{{json .}}'"], $this, false); + $containers = format_docker_command_output_to_json($containers); + $containerReplicates = instant_remote_process(["docker service ls --format '{{json .}}'"], $this, false); + if ($containerReplicates) { + $containerReplicates = format_docker_command_output_to_json($containerReplicates); + foreach ($containerReplicates as $containerReplica) { + $name = data_get($containerReplica, 'Name'); + $containers = $containers->map(function ($container) use ($name, $containerReplica) { + if (data_get($container, 'Spec.Name') === $name) { + $replicas = data_get($containerReplica, 'Replicas'); + $running = str($replicas)->explode('/')[0]; + $total = str($replicas)->explode('/')[1]; + if ($running === $total) { + data_set($container, 'State.Status', 'running'); + data_set($container, 'State.Health.Status', 'healthy'); + } else { + data_set($container, 'State.Status', 'starting'); + data_set($container, 'State.Health.Status', 'unhealthy'); + } + } + + return $container; + }); + } + } + } else { + $containers = instant_remote_process(["docker container inspect $(docker container ls -q) --format '{{json .}}'"], $this, false); + $containers = format_docker_command_output_to_json($containers); + $containerReplicates = collect([]); + } + + return [ + 'containers' => $containers ?? collect([]), + 'containerReplicates' => $containerReplicates ?? collect([]), + ]; + } + + public function getContainersWithSentinel(): Collection { $sentinel_found = instant_remote_process(['docker inspect coolify-sentinel'], $this, false); $sentinel_found = json_decode($sentinel_found, true); @@ -690,21 +733,6 @@ $schema://$host { $containers = data_get(json_decode($containers, true), 'containers', []); return collect($containers); - } else { - if ($this->isSwarm()) { - $containers = instant_remote_process(["docker service inspect $(docker service ls -q) --format '{{json .}}'"], $this, false); - } else { - $containers = instant_remote_process(['docker container ls -q'], $this, false); - if (! $containers) { - return collect([]); - } - $containers = instant_remote_process(["docker container inspect $(docker container ls -q) --format '{{json .}}'"], $this, false); - } - if (is_null($containers)) { - return collect([]); - } - - return format_docker_command_output_to_json($containers); } } diff --git a/app/Models/Service.php b/app/Models/Service.php index 1aa88c8ec..33238281e 100644 --- a/app/Models/Service.php +++ b/app/Models/Service.php @@ -2,6 +2,7 @@ namespace App\Models; +use Illuminate\Database\Eloquent\Casts\Attribute; use Illuminate\Database\Eloquent\Factories\HasFactory; use Illuminate\Database\Eloquent\Relations\HasMany; use Illuminate\Database\Eloquent\SoftDeletes; @@ -39,6 +40,8 @@ class Service extends BaseModel protected $guarded = []; + protected $appends = ['server_status']; + public function isConfigurationChanged(bool $save = false) { $domains = $this->applications()->get()->pluck('fqdn')->sort()->toArray(); @@ -77,6 +80,15 @@ class Service extends BaseModel } } + protected function serverStatus(): Attribute + { + return Attribute::make( + get: function () { + return $this->server->isFunctional(); + } + ); + } + public function isRunning() { return (bool) str($this->status())->contains('running'); diff --git a/app/Models/StandaloneClickhouse.php b/app/Models/StandaloneClickhouse.php index f930226da..4cd194cd8 100644 --- a/app/Models/StandaloneClickhouse.php +++ b/app/Models/StandaloneClickhouse.php @@ -14,7 +14,7 @@ class StandaloneClickhouse extends BaseModel protected $guarded = []; - protected $appends = ['internal_db_url', 'external_db_url', 'database_type']; + protected $appends = ['internal_db_url', 'external_db_url', 'database_type', 'server_status']; protected $casts = [ 'clickhouse_password' => 'encrypted', @@ -40,6 +40,15 @@ class StandaloneClickhouse extends BaseModel }); } + protected function serverStatus(): Attribute + { + return Attribute::make( + get: function () { + return $this->destination->server->isFunctional(); + } + ); + } + public function isConfigurationChanged(bool $save = false) { $newConfigHash = $this->image.$this->ports_mappings; diff --git a/app/Models/StandaloneDragonfly.php b/app/Models/StandaloneDragonfly.php index be04e2d83..8726b2546 100644 --- a/app/Models/StandaloneDragonfly.php +++ b/app/Models/StandaloneDragonfly.php @@ -14,7 +14,7 @@ class StandaloneDragonfly extends BaseModel protected $guarded = []; - protected $appends = ['internal_db_url', 'external_db_url', 'database_type']; + protected $appends = ['internal_db_url', 'external_db_url', 'database_type', 'server_status']; protected $casts = [ 'dragonfly_password' => 'encrypted', @@ -40,6 +40,15 @@ class StandaloneDragonfly extends BaseModel }); } + protected function serverStatus(): Attribute + { + return Attribute::make( + get: function () { + return $this->destination->server->isFunctional(); + } + ); + } + public function isConfigurationChanged(bool $save = false) { $newConfigHash = $this->image.$this->ports_mappings; diff --git a/app/Models/StandaloneKeydb.php b/app/Models/StandaloneKeydb.php index 1b7d5a958..607cacade 100644 --- a/app/Models/StandaloneKeydb.php +++ b/app/Models/StandaloneKeydb.php @@ -14,7 +14,7 @@ class StandaloneKeydb extends BaseModel protected $guarded = []; - protected $appends = ['internal_db_url', 'external_db_url']; + protected $appends = ['internal_db_url', 'external_db_url', 'server_status']; protected $casts = [ 'keydb_password' => 'encrypted', @@ -40,6 +40,15 @@ class StandaloneKeydb extends BaseModel }); } + protected function serverStatus(): Attribute + { + return Attribute::make( + get: function () { + return $this->destination->server->isFunctional(); + } + ); + } + public function isConfigurationChanged(bool $save = false) { $newConfigHash = $this->image.$this->ports_mappings.$this->keydb_conf; diff --git a/app/Models/StandaloneMariadb.php b/app/Models/StandaloneMariadb.php index 2081d9c89..d88653e41 100644 --- a/app/Models/StandaloneMariadb.php +++ b/app/Models/StandaloneMariadb.php @@ -14,7 +14,7 @@ class StandaloneMariadb extends BaseModel protected $guarded = []; - protected $appends = ['internal_db_url', 'external_db_url', 'database_type']; + protected $appends = ['internal_db_url', 'external_db_url', 'database_type', 'server_status']; protected $casts = [ 'mariadb_password' => 'encrypted', @@ -40,6 +40,15 @@ class StandaloneMariadb extends BaseModel }); } + protected function serverStatus(): Attribute + { + return Attribute::make( + get: function () { + return $this->destination->server->isFunctional(); + } + ); + } + public function isConfigurationChanged(bool $save = false) { $newConfigHash = $this->image.$this->ports_mappings.$this->mariadb_conf; diff --git a/app/Models/StandaloneMongodb.php b/app/Models/StandaloneMongodb.php index 066b34ab7..f09e932bf 100644 --- a/app/Models/StandaloneMongodb.php +++ b/app/Models/StandaloneMongodb.php @@ -14,7 +14,7 @@ class StandaloneMongodb extends BaseModel protected $guarded = []; - protected $appends = ['internal_db_url', 'external_db_url', 'database_type']; + protected $appends = ['internal_db_url', 'external_db_url', 'database_type', 'server_status']; protected static function booted() { @@ -44,6 +44,15 @@ class StandaloneMongodb extends BaseModel }); } + protected function serverStatus(): Attribute + { + return Attribute::make( + get: function () { + return $this->destination->server->isFunctional(); + } + ); + } + public function isConfigurationChanged(bool $save = false) { $newConfigHash = $this->image.$this->ports_mappings.$this->mongo_conf; diff --git a/app/Models/StandaloneMysql.php b/app/Models/StandaloneMysql.php index 375b56133..f4e56fab2 100644 --- a/app/Models/StandaloneMysql.php +++ b/app/Models/StandaloneMysql.php @@ -14,7 +14,7 @@ class StandaloneMysql extends BaseModel protected $guarded = []; - protected $appends = ['internal_db_url', 'external_db_url', 'database_type']; + protected $appends = ['internal_db_url', 'external_db_url', 'database_type', 'server_status']; protected $casts = [ 'mysql_password' => 'encrypted', @@ -41,6 +41,15 @@ class StandaloneMysql extends BaseModel }); } + protected function serverStatus(): Attribute + { + return Attribute::make( + get: function () { + return $this->destination->server->isFunctional(); + } + ); + } + public function isConfigurationChanged(bool $save = false) { $newConfigHash = $this->image.$this->ports_mappings.$this->mysql_conf; diff --git a/app/Models/StandalonePostgresql.php b/app/Models/StandalonePostgresql.php index ab6bcc626..311c09c36 100644 --- a/app/Models/StandalonePostgresql.php +++ b/app/Models/StandalonePostgresql.php @@ -14,7 +14,7 @@ class StandalonePostgresql extends BaseModel protected $guarded = []; - protected $appends = ['internal_db_url', 'external_db_url', 'database_type']; + protected $appends = ['internal_db_url', 'external_db_url', 'database_type', 'server_status']; protected $casts = [ 'init_scripts' => 'array', @@ -46,6 +46,15 @@ class StandalonePostgresql extends BaseModel return database_configuration_dir()."/{$this->uuid}"; } + protected function serverStatus(): Attribute + { + return Attribute::make( + get: function () { + return $this->destination->server->isFunctional(); + } + ); + } + public function delete_configurations() { $server = data_get($this, 'destination.server'); diff --git a/app/Models/StandaloneRedis.php b/app/Models/StandaloneRedis.php index df6cc8aeb..8a202ea9e 100644 --- a/app/Models/StandaloneRedis.php +++ b/app/Models/StandaloneRedis.php @@ -14,7 +14,7 @@ class StandaloneRedis extends BaseModel protected $guarded = []; - protected $appends = ['internal_db_url', 'external_db_url', 'database_type']; + protected $appends = ['internal_db_url', 'external_db_url', 'database_type', 'server_status']; protected static function booted() { @@ -36,6 +36,15 @@ class StandaloneRedis extends BaseModel }); } + protected function serverStatus(): Attribute + { + return Attribute::make( + get: function () { + return $this->destination->server->isFunctional(); + } + ); + } + public function isConfigurationChanged(bool $save = false) { $newConfigHash = $this->image.$this->ports_mappings.$this->redis_conf; diff --git a/app/Notifications/Server/Revived.php b/app/Notifications/Server/Revived.php index e05e13e9b..3f2b3b696 100644 --- a/app/Notifications/Server/Revived.php +++ b/app/Notifications/Server/Revived.php @@ -12,6 +12,7 @@ use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Notifications\Messages\MailMessage; use Illuminate\Notifications\Notification; +use Illuminate\Support\Facades\RateLimiter; class Revived extends Notification implements ShouldQueue { @@ -44,8 +45,20 @@ class Revived extends Notification implements ShouldQueue if ($isTelegramEnabled) { $channels[] = TelegramChannel::class; } + $executed = RateLimiter::attempt( + 'notification-server-revived-'.$this->server->uuid, + 1, + function () use ($channels) { + return $channels; + }, + 7200, + ); - return $channels; + if (! $executed) { + return []; + } + + return $executed; } public function toMail(): MailMessage diff --git a/app/Notifications/Server/Unreachable.php b/app/Notifications/Server/Unreachable.php index f178c9be3..2fb83559a 100644 --- a/app/Notifications/Server/Unreachable.php +++ b/app/Notifications/Server/Unreachable.php @@ -10,6 +10,7 @@ use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Notifications\Messages\MailMessage; use Illuminate\Notifications\Notification; +use Illuminate\Support\Facades\RateLimiter; class Unreachable extends Notification implements ShouldQueue { @@ -35,8 +36,20 @@ class Unreachable extends Notification implements ShouldQueue if ($isTelegramEnabled) { $channels[] = TelegramChannel::class; } + $executed = RateLimiter::attempt( + 'notification-server-unreachable-'.$this->server->uuid, + 1, + function () use ($channels) { + return $channels; + }, + 7200, + ); - return $channels; + if (! $executed) { + return []; + } + + return $executed; } public function toMail(): MailMessage diff --git a/resources/views/livewire/project/application/configuration.blade.php b/resources/views/livewire/project/application/configuration.blade.php index ada74260f..b5ba6c822 100644 --- a/resources/views/livewire/project/application/configuration.blade.php +++ b/resources/views/livewire/project/application/configuration.blade.php @@ -41,6 +41,14 @@ @endif + @if ($application->server_status == false) + + + + + + @endif
+
Network: {{ data_get($resource, 'destination.network') }}
+ @if ($resource->server_status == false) +
This server has connection problems.
+ @endif @if ($resource?->additional_networks?->count() > 0)
@@ -76,7 +79,6 @@ @if ($resource->getMorphClass() === 'App\Models\Application' && data_get($resource, 'build_pack') !== 'dockercompose') @if (count($networks) > 0)

Choose another server

-
(experimental)
@foreach ($networks as $network)