dave/coolify
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

feat: sentinel + charts

Browse Source
This commit is contained in:
Andras Bacsai
2024-06-18 16:42:42 +02:00
parent 83983bbb32
commit 23ed697b98
24 changed files with 557 additions and 193 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
app/Actions/Server/StartSentinel.php
View File

@@ -12,10 +12,13 @@ class StartSentinel
public function handle(Server $server, $version = 'latest', bool $restart = false)
{
if ($restart) {
instant_remote_process(['docker rm -f coolify-sentinel'], $server, false);
StopSentinel::run($server);
}
$metrics_history = $server->settings->metrics_history_days;
$refresh_rate = $server->settings->metrics_refresh_rate_seconds;
$token = $server->settings->metrics_token;
instant_remote_process([
"docker run --rm --pull always -d -e \"SCHEDULER=true\" -e \"METRICS_HISTORY=10\" -e \"REFRESH_RATE=5\" --name coolify-sentinel -v /var/run/docker.sock:/var/run/docker.sock -v /data/coolify/metrics:/app/metrics -v /data/coolify/logs:/app/logs --pid host --health-cmd \"curl --fail http://127.0.0.1:8888/api/health || exit 1\" --health-interval 10s --health-retries 3 ghcr.io/coollabsio/sentinel:$version",
"docker run --rm --pull always -d -e \"TOKEN={$token}\" -e \"SCHEDULER=true\" -e \"METRICS_HISTORY={$metrics_history}\" -e \"REFRESH_RATE={$refresh_rate}\" --name coolify-sentinel -v /var/run/docker.sock:/var/run/docker.sock -v /data/coolify/metrics:/app/metrics -v /data/coolify/logs:/app/logs --pid host --health-cmd \"curl --fail http://127.0.0.1:8888/api/health || exit 1\" --health-interval 10s --health-retries 3 ghcr.io/coollabsio/sentinel:$version",
'chown -R 9999:root /data/coolify/metrics /data/coolify/logs',
'chmod -R 700 /data/coolify/metrics /data/coolify/logs',
], $server, false);

16
app/Actions/Server/StopSentinel.php Normal file
View File

@@ -0,0 +1,16 @@
<?php
namespace App\Actions\Server;
use App\Models\Server;
use Lorisleiva\Actions\Concerns\AsAction;
class StopSentinel
{
use AsAction;
public function handle(Server $server)
{
instant_remote_process(['docker rm -f coolify-sentinel'], $server, false);
}
}

2
app/Console/Kernel.php
View File

@@ -61,7 +61,7 @@ class Kernel extends ConsoleKernel
{
$servers = $this->all_servers->where('settings.is_usable', true)->where('settings.is_reachable', true)->where('ip', '!=', '1.2.3.4');
foreach ($servers as $server) {
if ($server->is_metrics_enabled) {
if ($server->isMetricsEnabled()) {
$schedule->job(new PullSentinelImageJob($server))->everyFiveMinutes()->onOneServer();
}
$schedule->job(new PullHelperImageJob($server))->everyFiveMinutes()->onOneServer();

7
app/Jobs/PullSentinelImageJob.php
View File

@@ -36,10 +36,7 @@ class PullSentinelImageJob implements ShouldBeEncrypted, ShouldQueue
{
try {
$version = get_latest_sentinel_version();
if (isDev()) {
$version = '0.0.5';
}
if (! $version) {
if (!$version) {
ray('Failed to get latest Sentinel version');
return;
@@ -54,7 +51,7 @@ class PullSentinelImageJob implements ShouldBeEncrypted, ShouldQueue
}
ray('Sentinel image is up to date');
} catch (\Throwable $e) {
send_internal_notification('PullSentinelImageJob failed with: '.$e->getMessage());
send_internal_notification('PullSentinelImageJob failed with: ' . $e->getMessage());
ray($e->getMessage());
throw $e;
}

2
app/Jobs/ServerStatusJob.php
View File

@@ -48,7 +48,7 @@ class ServerStatusJob implements ShouldBeEncrypted, ShouldQueue
if ($this->server->isFunctional()) {
$this->cleanup(notify: false);
$this->remove_unnecessary_coolify_yaml();
if ($this->server->is_metrics_enabled) {
if ($this->server->isMetricsEnabled()) {
$this->server->checkSentinel();
}
}

2
app/Livewire/Boarding/Index.php
View File

@@ -12,7 +12,7 @@ use Livewire\Component;
class Index extends Component
{
protected $listeners = ['serverInstalled' => 'validateServer'];
protected $listeners = ['refreshBoardingIndex' => 'validateServer'];
public string $currentState = 'welcome';

38
app/Livewire/Charts/Server.php
View File

@@ -1,38 +0,0 @@
<?php
namespace App\Livewire\Charts;
use App\Models\Server as ModelsServer;
use Livewire\Component;
class Server extends Component
{
public ModelsServer $server;
public $chartId = 'server';
public $data;
public $categories;
public function render()
{
return view('livewire.charts.server');
}
public function mount()
{
$this->loadData();
}
public function loadData()
{
$metrics = $this->server->getMetrics();
$metrics = collect($metrics)->map(function ($metric) {
return [$metric[0], $metric[1]];
});
$this->dispatch("refreshChartData-{$this->chartId}", [
'seriesData' => $metrics,
]);
}
}

48
app/Livewire/Charts/ServerCpu.php Normal file
View File

@@ -0,0 +1,48 @@
<?php
namespace App\Livewire\Charts;
use App\Models\Server as ModelsServer;
use Livewire\Component;
class ServerCpu extends Component
{
public ModelsServer $server;
public $chartId = 'server-cpu';
public $data;
public $categories;
public $interval = 5;
public function render()
{
return view('livewire.charts.server-cpu');
}
public function mount()
{
$this->loadData();
}
public function loadData()
{
try {
$metrics = $this->server->getCpuMetrics($this->interval);
$metrics = collect($metrics)->map(function ($metric) {
return [$metric[0], $metric[1]];
});
$this->dispatch("refreshChartData-{$this->chartId}", [
'seriesData' => $metrics,
]);
} catch (\Throwable $e) {
return handleError($e, $this);
}
}
public function setInterval()
{
$this->loadData();
}
}

45
app/Livewire/Charts/ServerMemory.php Normal file
View File

@@ -0,0 +1,45 @@
<?php
namespace App\Livewire\Charts;
use App\Models\Server;
use Livewire\Component;
class ServerMemory extends Component
{
public Server $server;
public $chartId = 'server-memory';
public $data;
public $categories;
public $interval = 5;
public function render()
{
return view('livewire.charts.server-memory');
}
public function mount()
{
$this->loadData();
}
public function loadData()
{
try {
$metrics = $this->server->getMemoryMetrics($this->interval);
$metrics = collect($metrics)->map(function ($metric) {
return [$metric[0], $metric[1]];
});
$this->dispatch("refreshChartData-{$this->chartId}", [
'seriesData' => $metrics,
]);
} catch (\Throwable $e) {
return handleError($e, $this);
}
}
public function setInterval() {
$this->loadData();
}
}

2
app/Livewire/Project/Shared/Logs.php
View File

@@ -64,7 +64,7 @@ class Logs extends Component
return;
$server = data_get($this->resource, 'destination.server');
if ($server->isFunctional()) {
$this->cpu = $server->getMetrics();
$this->cpu = $server->getCpuMetrics();
}
}

4
app/Livewire/Server/ConfigureCloudflareTunnels.php
View File

@@ -21,7 +21,7 @@ class ConfigureCloudflareTunnels extends Component
$server->settings->is_cloudflare_tunnel = true;
$server->settings->save();
$this->dispatch('success', 'Cloudflare Tunnels configured successfully.');
$this->dispatch('serverInstalled');
$this->dispatch('refreshServerShow');
} catch (\Throwable $e) {
return handleError($e, $this);
}
@@ -37,7 +37,7 @@ class ConfigureCloudflareTunnels extends Component
$server->save();
$server->settings->save();
$this->dispatch('success', 'Cloudflare Tunnels configured successfully.');
$this->dispatch('serverInstalled');
$this->dispatch('refreshServerShow');
} catch (\Throwable $e) {
return handleError($e, $this);
}

30
app/Livewire/Server/Form.php
View File

@@ -2,6 +2,9 @@
namespace App\Livewire\Server;
use App\Actions\Server\StartSentinel;
use App\Actions\Server\StopSentinel;
use App\Jobs\PullSentinelImageJob;
use App\Models\Server;
use Livewire\Component;
@@ -36,6 +39,10 @@ class Form extends Component
'server.settings.is_build_server' => 'required|boolean',
'server.settings.concurrent_builds' => 'required|integer|min:1',
'server.settings.dynamic_timeout' => 'required|integer|min:1',
'server.settings.is_metrics_enabled' => 'required|boolean',
'server.settings.metrics_token' => 'required',
'server.settings.metrics_refresh_rate_seconds' => 'required|integer|min:1',
'server.settings.metrics_history_days' => 'required|integer|min:1',
'wildcard_domain' => 'nullable|url',
];
@@ -52,6 +59,10 @@ class Form extends Component
'server.settings.is_build_server' => 'Build Server',
'server.settings.concurrent_builds' => 'Concurrent Builds',
'server.settings.dynamic_timeout' => 'Dynamic Timeout',
'server.settings.is_metrics_enabled' => 'Metrics',
'server.settings.metrics_token' => 'Metrics Token',
'server.settings.metrics_refresh_rate_seconds' => 'Metrics Interval',
'server.settings.metrics_history_days' => 'Metrics History',
];
@@ -69,7 +80,7 @@ class Form extends Component
public function updatedServerSettingsIsBuildServer()
{
$this->dispatch('serverInstalled');
$this->dispatch('refreshServerShow');
$this->dispatch('serverRefresh');
$this->dispatch('proxyStatusUpdated');
}
@@ -80,7 +91,24 @@ class Form extends Component
refresh_server_connection($this->server->privateKey);
$this->validateServer(false);
$this->server->settings->save();
$this->server->save();
$this->dispatch('success', 'Server updated.');
$this->dispatch('refreshServerShow');
if ($this->server->isMetricsEnabled()) {
PullSentinelImageJob::dispatchSync($this->server);
$this->dispatch('reloadWindow');
} else {
StopSentinel::dispatch($this->server);
}
} catch (\Throwable $e) {
return handleError($e, $this);
}
}
public function restartSentinel() {
try {
$version = get_latest_sentinel_version();
StartSentinel::run($this->server, $version, true);
$this->dispatch('success', 'Sentinel restarted.');
} catch (\Throwable $e) {
return handleError($e, $this);
}

2
app/Livewire/Server/Show.php
View File

@@ -14,7 +14,7 @@ class Show extends Component
public $parameters = [];
protected $listeners = ['serverInstalled' => '$refresh'];
protected $listeners = ['refreshServerShow' => '$refresh'];
public function mount()
{

3
app/Livewire/Server/ValidateAndInstall.php
View File

@@ -143,7 +143,8 @@ class ValidateAndInstall extends Component
} else {
$this->docker_version = $this->server->validateDockerEngineVersion();
if ($this->docker_version) {
$this->dispatch('serverInstalled');
$this->dispatch('refreshServerShow');
$this->dispatch('refreshBoardingIndex');
$this->dispatch('success', 'Server validated.');
} else {
$this->error = 'Docker Engine version is not 22+. Please install Docker manually before continuing: <a target="_blank" class="underline" href="https://docs.docker.com/engine/install/#server">documentation</a>.';

83
app/Models/Server.php
View File

@@ -131,7 +131,7 @@ class Server extends BaseModel
public function setupDefault404Redirect()
{
$dynamic_conf_path = $this->proxyPath().'/dynamic';
$dynamic_conf_path = $this->proxyPath() . '/dynamic';
$proxy_type = $this->proxyType();
$redirect_url = $this->proxy->redirect_url;
if ($proxy_type === 'TRAEFIK_V2') {
@@ -145,8 +145,8 @@ class Server extends BaseModel
respond 404
}';
$conf =
"# This file is automatically generated by Coolify.\n".
"# Do not edit it manually (only if you know what are you doing).\n\n".
"# This file is automatically generated by Coolify.\n" .
"# Do not edit it manually (only if you know what are you doing).\n\n" .
$conf;
$base64 = base64_encode($conf);
instant_remote_process([
@@ -205,8 +205,8 @@ respond 404
];
$conf = Yaml::dump($dynamic_conf, 12, 2);
$conf =
"# This file is automatically generated by Coolify.\n".
"# Do not edit it manually (only if you know what are you doing).\n\n".
"# This file is automatically generated by Coolify.\n" .
"# Do not edit it manually (only if you know what are you doing).\n\n" .
$conf;
$base64 = base64_encode($conf);
@@ -215,8 +215,8 @@ respond 404
redir $redirect_url
}";
$conf =
"# This file is automatically generated by Coolify.\n".
"# Do not edit it manually (only if you know what are you doing).\n\n".
"# This file is automatically generated by Coolify.\n" .
"# Do not edit it manually (only if you know what are you doing).\n\n" .
$conf;
$base64 = base64_encode($conf);
}
@@ -237,7 +237,7 @@ respond 404
public function setupDynamicProxyConfiguration()
{
$settings = InstanceSettings::get();
$dynamic_config_path = $this->proxyPath().'/dynamic';
$dynamic_config_path = $this->proxyPath() . '/dynamic';
if ($this->proxyType() === 'TRAEFIK_V2') {
$file = "$dynamic_config_path/coolify.yaml";
if (empty($settings->fqdn) || (isCloud() && $this->id !== 0)) {
@@ -330,8 +330,8 @@ respond 404
}
$yaml = Yaml::dump($traefik_dynamic_conf, 12, 2);
$yaml =
"# This file is automatically generated by Coolify.\n".
"# Do not edit it manually (only if you know what are you doing).\n\n".
"# This file is automatically generated by Coolify.\n" .
"# Do not edit it manually (only if you know what are you doing).\n\n" .
$yaml;
$base64 = base64_encode($yaml);
@@ -389,9 +389,9 @@ $schema://$host {
if ($proxyType === ProxyTypes::TRAEFIK_V2->value) {
$proxy_path = $proxy_path;
} elseif ($proxyType === ProxyTypes::CADDY->value) {
$proxy_path = $proxy_path.'/caddy';
$proxy_path = $proxy_path . '/caddy';
} elseif ($proxyType === ProxyTypes::NGINX->value) {
$proxy_path = $proxy_path.'/nginx';
$proxy_path = $proxy_path . '/nginx';
}
return $proxy_path;
@@ -462,10 +462,14 @@ $schema://$host {
Storage::disk('ssh-mux')->delete($this->muxFilename());
}
public function isMetricsEnabled()
{
return $this->settings->is_metrics_enabled;
}
public function checkSentinel()
{
ray("Checking sentinel on server: {$this->name}");
if ($this->is_metrics_enabled) {
if ($this->isMetricsEnabled()) {
$sentinel_found = instant_remote_process(['docker inspect coolify-sentinel'], $this, false);
$sentinel_found = json_decode($sentinel_found, true);
$status = data_get($sentinel_found, '0.State.Status', 'exited');
@@ -478,11 +482,19 @@ $schema://$host {
}
}
public function getMetrics()
public function getCpuMetrics(int $mins = 5)
{
if ($this->is_metrics_enabled) {
$from = now()->subMinutes(5)->toIso8601ZuluString();
$cpu = instant_remote_process(["docker exec coolify-sentinel sh -c 'curl http://localhost:8888/api/cpu/history?from=$from'"], $this, false);
if ($this->isMetricsEnabled()) {
$from = now()->subMinutes($mins)->toIso8601ZuluString();
$cpu = instant_remote_process(["docker exec coolify-sentinel sh -c 'curl -H \"Authorization: Bearer {$this->settings->metrics_token}\" http://localhost:8888/api/cpu/history?from=$from'"], $this, false);
if (str($cpu)->contains('error')) {
$error = json_decode($cpu, true);
$error = data_get($error, 'error', 'Something is not okay, are you okay?');
if ($error == 'Unauthorized') {
$error = 'Unauthorized, please check your metrics token or restart Sentinel to set a new token.';
}
throw new \Exception($error);
}
$cpu = str($cpu)->explode("\n")->skip(1)->all();
$parsedCollection = collect($cpu)->flatMap(function ($item) {
return collect(explode("\n", trim($item)))->map(function ($line) {
@@ -495,6 +507,31 @@ $schema://$host {
return $parsedCollection;
}
}
public function getMemoryMetrics(int $mins = 5)
{
if ($this->isMetricsEnabled()) {
$from = now()->subMinutes($mins)->toIso8601ZuluString();
$memory = instant_remote_process(["docker exec coolify-sentinel sh -c 'curl -H \"Authorization: Bearer {$this->settings->metrics_token}\" http://localhost:8888/api/memory/history?from=$from'"], $this, false);
if (str($memory)->contains('error')) {
$error = json_decode($memory, true);
$error = data_get($error, 'error', 'Something is not okay, are you okay?');
if ($error == 'Unauthorized') {
$error = 'Unauthorized, please check your metrics token or restart Sentinel to set a new token.';
}
throw new \Exception($error);
}
$memory = str($memory)->explode("\n")->skip(1)->all();
$parsedCollection = collect($memory)->flatMap(function ($item) {
return collect(explode("\n", trim($item)))->map(function ($line) {
[$time, $used, $free, $usedPercent] = explode(',', trim($line));
return [(int) $time, (float) $usedPercent];
});
})->toArray();
return $parsedCollection;
}
}
public function isServerReady(int $tries = 3)
{
@@ -609,7 +646,7 @@ $schema://$host {
$containers = instant_remote_process(["docker service inspect $(docker service ls -q) --format '{{json .}}'"], $this, false);
} else {
$containers = instant_remote_process(['docker container ls -q'], $this, false);
if (! $containers) {
if (!$containers) {
return collect([]);
}
$containers = instant_remote_process(["docker container inspect $(docker container ls -q) --format '{{json .}}'"], $this, false);
@@ -629,7 +666,7 @@ $schema://$host {
$containers = format_docker_command_output_to_json($containers);
$containers = $containers->map(function ($container) {
$labels = data_get($container, 'Labels');
if (! str($labels)->contains('coolify.managed')) {
if (!str($labels)->contains('coolify.managed')) {
return $container;
}
@@ -701,7 +738,7 @@ $schema://$host {
return $this->previews()->filter(function ($preview) {
$applicationId = data_get($preview, 'application_id');
$application = Application::find($applicationId);
if (! $application) {
if (!$application) {
return false;
}
@@ -785,9 +822,9 @@ $schema://$host {
public function isFunctional()
{
$isFunctional = $this->settings->is_reachable && $this->settings->is_usable && ! $this->settings->force_disabled;
$isFunctional = $this->settings->is_reachable && $this->settings->is_usable && !$this->settings->force_disabled;
['private_key_filename' => $private_key_filename, 'mux_filename' => $mux_filename] = server_ssh_configuration($this);
if (! $isFunctional) {
if (!$isFunctional) {
Storage::disk('ssh-keys')->delete($private_key_filename);
Storage::disk('ssh-mux')->delete($mux_filename);
}
@@ -846,7 +883,7 @@ $schema://$host {
config()->set('coolify.mux_enabled', false);
$server = Server::find($this->id);
if (! $server) {
if (!$server) {
return ['uptime' => false, 'error' => 'Server not found.'];
}
if ($server->skipServer()) {