sentinel updates

This commit is contained in:
Andras Bacsai
2024-10-15 13:39:19 +02:00
parent 81db57002b
commit d446cd4f31
19 changed files with 293 additions and 145 deletions

View File

@@ -7,6 +7,7 @@ use App\Enums\ProxyTypes;
use App\Jobs\PullSentinelImageJob;
use Illuminate\Database\Eloquent\Builder;
use Illuminate\Database\Eloquent\Casts\Attribute;
use Illuminate\Support\Carbon;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Process;
@@ -16,6 +17,7 @@ use OpenApi\Attributes as OA;
use Spatie\SchemalessAttributes\Casts\SchemalessAttributes;
use Spatie\SchemalessAttributes\SchemalessAttributesTrait;
use Spatie\Url\Url;
use Illuminate\Support\Str;
use Symfony\Component\Yaml\Yaml;
#[OA\Schema(
@@ -166,7 +168,7 @@ class Server extends BaseModel
public function setupDefault404Redirect()
{
$dynamic_conf_path = $this->proxyPath().'/dynamic';
$dynamic_conf_path = $this->proxyPath() . '/dynamic';
$proxy_type = $this->proxyType();
$redirect_url = $this->proxy->redirect_url;
if ($proxy_type === ProxyTypes::TRAEFIK->value) {
@@ -180,8 +182,8 @@ class Server extends BaseModel
respond 404
}';
$conf =
"# This file is automatically generated by Coolify.\n".
"# Do not edit it manually (only if you know what are you doing).\n\n".
"# This file is automatically generated by Coolify.\n" .
"# Do not edit it manually (only if you know what are you doing).\n\n" .
$conf;
$base64 = base64_encode($conf);
instant_remote_process([
@@ -243,8 +245,8 @@ respond 404
];
$conf = Yaml::dump($dynamic_conf, 12, 2);
$conf =
"# This file is automatically generated by Coolify.\n".
"# Do not edit it manually (only if you know what are you doing).\n\n".
"# This file is automatically generated by Coolify.\n" .
"# Do not edit it manually (only if you know what are you doing).\n\n" .
$conf;
$base64 = base64_encode($conf);
@@ -253,8 +255,8 @@ respond 404
redir $redirect_url
}";
$conf =
"# This file is automatically generated by Coolify.\n".
"# Do not edit it manually (only if you know what are you doing).\n\n".
"# This file is automatically generated by Coolify.\n" .
"# Do not edit it manually (only if you know what are you doing).\n\n" .
$conf;
$base64 = base64_encode($conf);
}
@@ -272,7 +274,7 @@ respond 404
public function setupDynamicProxyConfiguration()
{
$settings = instanceSettings();
$dynamic_config_path = $this->proxyPath().'/dynamic';
$dynamic_config_path = $this->proxyPath() . '/dynamic';
if ($this->proxyType() === ProxyTypes::TRAEFIK->value) {
$file = "$dynamic_config_path/coolify.yaml";
if (empty($settings->fqdn) || (isCloud() && $this->id !== 0) || ! $this->isLocalhost()) {
@@ -391,8 +393,8 @@ respond 404
}
$yaml = Yaml::dump($traefik_dynamic_conf, 12, 2);
$yaml =
"# This file is automatically generated by Coolify.\n".
"# Do not edit it manually (only if you know what are you doing).\n\n".
"# This file is automatically generated by Coolify.\n" .
"# Do not edit it manually (only if you know what are you doing).\n\n" .
$yaml;
$base64 = base64_encode($yaml);
@@ -456,13 +458,13 @@ $schema://$host {
if (isDev()) {
$proxy_path = '/var/lib/docker/volumes/coolify_dev_coolify_data/_data/proxy/caddy';
} else {
$proxy_path = $proxy_path.'/caddy';
$proxy_path = $proxy_path . '/caddy';
}
} elseif ($proxyType === ProxyTypes::NGINX->value) {
if (isDev()) {
$proxy_path = '/var/lib/docker/volumes/coolify_dev_coolify_data/_data/proxy/nginx';
} else {
$proxy_path = $proxy_path.'/nginx';
$proxy_path = $proxy_path . '/nginx';
}
}
@@ -538,6 +540,16 @@ $schema://$host {
return $encrypted;
}
public function sentinelUpdateAt(bool $isReset = false)
{
$this->sentinel_updated_at = $isReset ? now()->subMinutes(6000) : now();
$this->save();
}
public function isSentinelLive()
{
return Carbon::parse($this->sentinel_updated_at)->isAfter(now()->subMinutes(4));
}
public function isSentinelEnabled()
{
return $this->isMetricsEnabled() || $this->isServerApiEnabled();
@@ -550,7 +562,7 @@ $schema://$host {
public function isServerApiEnabled()
{
return $this->settings->is_server_api_enabled;
return $this->settings->is_sentinel_enabled;
}
public function checkServerApi()
@@ -591,7 +603,15 @@ $schema://$host {
{
if ($this->isMetricsEnabled()) {
$from = now()->subMinutes($mins)->toIso8601ZuluString();
$cpu = instant_remote_process(["docker exec coolify-sentinel sh -c 'curl -H \"Authorization: Bearer {$this->settings->sentinel_token}\" http://localhost:8888/api/cpu/history?from=$from'"], $this, false);
if (isDev() && $this->id === 0) {
$process = Process::run("curl -H \"Authorization: Bearer {$this->settings->sentinel_token}\" http://host.docker.internal:8888/api/cpu/history?from=$from");
if ($process->failed()) {
throw new \Exception($process->errorOutput());
}
$cpu = $process->output();
} else {
$cpu = instant_remote_process(["docker exec coolify-sentinel sh -c 'curl -H \"Authorization: Bearer {$this->settings->sentinel_token}\" http://localhost:8888/api/cpu/history?from=$from'"], $this, false);
}
if (str($cpu)->contains('error')) {
$error = json_decode($cpu, true);
$error = data_get($error, 'error', 'Something is not okay, are you okay?');
@@ -600,17 +620,12 @@ $schema://$host {
}
throw new \Exception($error);
}
$cpu = str($cpu)->explode("\n")->skip(1)->all();
$parsedCollection = collect($cpu)->flatMap(function ($item) {
return collect(explode("\n", trim($item)))->map(function ($line) {
[$time, $cpu_usage_percent] = explode(',', trim($line));
$cpu_usage_percent = number_format($cpu_usage_percent, 0);
return [(int) $time, (float) $cpu_usage_percent];
});
$cpu = json_decode($cpu, true);
$parsedCollection = collect($cpu)->map(function ($metric) {
return [(int)$metric['time'], (float)$metric['percent']];
});
return $parsedCollection;
return $parsedCollection->toArray();
}
}
@@ -618,7 +633,15 @@ $schema://$host {
{
if ($this->isMetricsEnabled()) {
$from = now()->subMinutes($mins)->toIso8601ZuluString();
$memory = instant_remote_process(["docker exec coolify-sentinel sh -c 'curl -H \"Authorization: Bearer {$this->settings->sentinel_token}\" http://localhost:8888/api/memory/history?from=$from'"], $this, false);
if (isDev() && $this->id === 0) {
$process = Process::run("curl -H \"Authorization: Bearer {$this->settings->sentinel_token}\" http://host.docker.internal:8888/api/memory/history?from=$from");
if ($process->failed()) {
throw new \Exception($process->errorOutput());
}
$memory = $process->output();
} else {
$memory = instant_remote_process(["docker exec coolify-sentinel sh -c 'curl -H \"Authorization: Bearer {$this->settings->sentinel_token}\" http://localhost:8888/api/memory/history?from=$from'"], $this, false);
}
if (str($memory)->contains('error')) {
$error = json_decode($memory, true);
$error = data_get($error, 'error', 'Something is not okay, are you okay?');
@@ -627,14 +650,9 @@ $schema://$host {
}
throw new \Exception($error);
}
$memory = str($memory)->explode("\n")->skip(1)->all();
$parsedCollection = collect($memory)->flatMap(function ($item) {
return collect(explode("\n", trim($item)))->map(function ($line) {
[$time, $used, $free, $usedPercent] = explode(',', trim($line));
$usedPercent = number_format($usedPercent, 0);
return [(int) $time, (float) $usedPercent];
});
$memory = json_decode($memory, true);
$parsedCollection = collect($memory)->map(function ($metric) {
return [(int)$metric['time'], (float)$metric['usedPercent']];
});
return $parsedCollection->toArray();
@@ -1054,6 +1072,37 @@ $schema://$host {
return data_get($this, 'settings.is_swarm_worker');
}
public function status(): bool
{
['uptime' => $uptime] = $this->validateConnection(false);
if ($uptime) {
if ($this->unreachable_notification_sent === true) {
$this->update(['unreachable_notification_sent' => false]);
}
} else {
// $this->server->team?->notify(new Unreachable($this->server));
foreach ($this->applications as $application) {
$application->update(['status' => 'exited']);
}
foreach ($this->databases as $database) {
$database->update(['status' => 'exited']);
}
foreach ($this->services as $service) {
$apps = $service->applications()->get();
$dbs = $service->databases()->get();
foreach ($apps as $app) {
$app->update(['status' => 'exited']);
}
foreach ($dbs as $db) {
$db->update(['status' => 'exited']);
}
}
return false;
}
return true;
}
public function validateConnection($isManualCheck = true)
{
config()->set('constants.ssh.mux_enabled', ! $isManualCheck);