dave/coolify
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

feat: container metrics

Browse Source
This commit is contained in:
Andras Bacsai
2024-06-20 13:17:06 +02:00
parent 439bee1203
commit c81ad5cd03
27 changed files with 704 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
app/Actions/Server/StartSentinel.php
View File

@@ -21,6 +21,6 @@ class StartSentinel
"docker run --rm --pull always -d -e \"TOKEN={$token}\" -e \"SCHEDULER=true\" -e \"METRICS_HISTORY={$metrics_history}\" -e \"REFRESH_RATE={$refresh_rate}\" --name coolify-sentinel -v /var/run/docker.sock:/var/run/docker.sock -v /data/coolify/metrics:/app/metrics -v /data/coolify/logs:/app/logs --pid host --health-cmd \"curl --fail http://127.0.0.1:8888/api/health || exit 1\" --health-interval 10s --health-retries 3 ghcr.io/coollabsio/sentinel:$version",
'chown -R 9999:root /data/coolify/metrics /data/coolify/logs',
'chmod -R 700 /data/coolify/metrics /data/coolify/logs',
], $server, false);
], $server, true);
}
}

2
app/Console/Kernel.php
View File

@@ -61,7 +61,7 @@ class Kernel extends ConsoleKernel
{
$servers = $this->all_servers->where('settings.is_usable', true)->where('settings.is_reachable', true)->where('ip', '!=', '1.2.3.4');
foreach ($servers as $server) {
if ($server->isMetricsEnabled()) {
if ($server->isSentinelEnabled()) {
$schedule->job(new PullSentinelImageJob($server))->everyFiveMinutes()->onOneServer();
}
$schedule->job(new PullHelperImageJob($server))->everyFiveMinutes()->onOneServer();

4
app/Jobs/DockerCleanupJob.php
View File

@@ -22,7 +22,9 @@ class DockerCleanupJob implements ShouldBeEncrypted, ShouldQueue
public ?int $usageBefore = null;
public function __construct(public Server $server) {}
public function __construct(public Server $server)
{
}
public function handle(): void
{

6
app/Jobs/PullSentinelImageJob.php
View File

@@ -28,7 +28,9 @@ class PullSentinelImageJob implements ShouldBeEncrypted, ShouldQueue
return $this->server->uuid;
}
public function __construct(public Server $server) {}
public function __construct(public Server $server)
{
}
public function handle(): void
{
@@ -50,7 +52,7 @@ class PullSentinelImageJob implements ShouldBeEncrypted, ShouldQueue
}
ray('Sentinel image is up to date');
} catch (\Throwable $e) {
send_internal_notification('PullSentinelImageJob failed with: '.$e->getMessage());
// send_internal_notification('PullSentinelImageJob failed with: '.$e->getMessage());
ray($e->getMessage());
throw $e;
}

6
app/Jobs/ServerStatusJob.php
View File

@@ -25,7 +25,9 @@ class ServerStatusJob implements ShouldBeEncrypted, ShouldQueue
return isDev() ? 1 : 3;
}
public function __construct(public Server $server) {}
public function __construct(public Server $server)
{
}
public function middleware(): array
{
@@ -46,7 +48,7 @@ class ServerStatusJob implements ShouldBeEncrypted, ShouldQueue
if ($this->server->isFunctional()) {
$this->cleanup(notify: false);
$this->remove_unnecessary_coolify_yaml();
if ($this->server->isMetricsEnabled()) {
if ($this->server->isSentinelEnabled()) {
$this->server->checkSentinel();
}
}

64
app/Livewire/Project/Shared/Metrics.php Normal file
View File

@@ -0,0 +1,64 @@
<?php
namespace App\Livewire\Project\Shared;
use Livewire\Component;
class Metrics extends Component
{
public $resource;
public $chartId = 'container-cpu';
public $data;
public $categories;
public int $interval = 5;
public bool $poll = true;
public function pollData()
{
if ($this->poll || $this->interval <= 10) {
$this->loadData();
if ($this->interval > 10) {
$this->poll = false;
}
}
}
public function loadData()
{
try {
$metrics = $this->resource->getMetrics($this->interval);
$cpuMetrics = collect($metrics)->map(function ($metric) {
return [$metric[0], $metric[1]];
});
$memoryMetrics = collect($metrics)->map(function ($metric) {
return [$metric[0], $metric[2]];
});
$this->dispatch("refreshChartData-{$this->chartId}-cpu", [
'seriesData' => $cpuMetrics,
]);
$this->dispatch("refreshChartData-{$this->chartId}-memory", [
'seriesData' => $memoryMetrics,
]);
} catch (\Throwable $e) {
return handleError($e, $this);
}
}
public function setInterval()
{
if ($this->interval <= 10) {
$this->poll = true;
}
$this->loadData();
}
public function render()
{
return view('livewire.project.shared.metrics');
}
}

29
app/Livewire/Server/Form.php
View File

@@ -44,6 +44,7 @@ class Form extends Component
'server.settings.metrics_refresh_rate_seconds' => 'required|integer|min:1',
'server.settings.metrics_history_days' => 'required|integer|min:1',
'wildcard_domain' => 'nullable|url',
'server.settings.is_server_api_enabled' => 'required|boolean',
];
protected $validationAttributes = [
@@ -63,7 +64,7 @@ class Form extends Component
'server.settings.metrics_token' => 'Metrics Token',
'server.settings.metrics_refresh_rate_seconds' => 'Metrics Interval',
'server.settings.metrics_history_days' => 'Metrics History',
'server.settings.is_server_api_enabled' => 'Server API',
];
public function mount()
@@ -85,6 +86,18 @@ class Form extends Component
$this->dispatch('proxyStatusUpdated');
}
public function checkPortForServerApi()
{
try {
if ($this->server->settings->is_server_api_enabled === true) {
$this->server->checkServerApi();
$this->dispatch('success', 'Server API is reachable.');
}
} catch (\Throwable $e) {
return handleError($e, $this);
}
}
public function instantSave()
{
try {
@@ -94,12 +107,22 @@ class Form extends Component
$this->server->save();
$this->dispatch('success', 'Server updated.');
$this->dispatch('refreshServerShow');
if ($this->server->isMetricsEnabled()) {
if ($this->server->isSentinelEnabled()) {
PullSentinelImageJob::dispatchSync($this->server);
$this->dispatch('reloadWindow');
ray('Sentinel is enabled');
if ($this->server->settings->isDirty('is_metrics_enabled')) {
$this->dispatch('reloadWindow');
}
if ($this->server->settings->isDirty('is_server_api_enabled') && $this->server->settings->is_server_api_enabled === true) {
ray('Starting sentinel');
}
} else {
ray('Sentinel is not enabled');
StopSentinel::dispatch($this->server);
}
// $this->checkPortForServerApi();
} catch (\Throwable $e) {
return handleError($e, $this);
}

29
app/Models/Application.php
View File

@@ -1167,4 +1167,33 @@ class Application extends BaseModel
return $preview;
}
public function getMetrics(int $mins = 5)
{
$server = $this->destination->server;
$container_name = generateApplicationContainerName($this);
if ($server->isMetricsEnabled()) {
$from = now()->subMinutes($mins)->toIso8601ZuluString();
$metrics = instant_remote_process(["docker exec coolify-sentinel sh -c 'curl -H \"Authorization: Bearer {$server->settings->metrics_token}\" http://localhost:8888/api/container/{$container_name}/metrics/history?from=$from'"], $server, false);
if (str($metrics)->contains('error')) {
$error = json_decode($metrics, true);
$error = data_get($error, 'error', 'Something is not okay, are you okay?');
if ($error == 'Unauthorized') {
$error = 'Unauthorized, please check your metrics token or restart Sentinel to set a new token.';
}
throw new \Exception($error);
}
$metrics = str($metrics)->explode("\n")->skip(1)->all();
$parsedCollection = collect($metrics)->flatMap(function ($item) {
return collect(explode("\n", trim($item)))->map(function ($line) {
[$time, $cpu_usage_percent, $memory_usage, $memory_usage_percent] = explode(',', trim($line));
$cpu_usage_percent = number_format($cpu_usage_percent, 2);
return [(int) $time, (float) $cpu_usage_percent, (int) $memory_usage];
});
});
return $parsedCollection->toArray();
}
}
}

38
app/Models/Server.php
View File

@@ -9,6 +9,7 @@ use Illuminate\Database\Eloquent\Builder;
use Illuminate\Database\Eloquent\Casts\Attribute;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Process;
use Illuminate\Support\Facades\Storage;
use Illuminate\Support\Str;
use Illuminate\Support\Stringable;
@@ -460,15 +461,44 @@ $schema://$host {
Storage::disk('ssh-mux')->delete($this->muxFilename());
}
public function isSentinelEnabled()
{
return $this->isMetricsEnabled() || $this->isServerApiEnabled();
}
public function isMetricsEnabled()
{
return $this->settings->is_metrics_enabled;
}
public function isServerApiEnabled()
{
return $this->settings->is_server_api_enabled;
}
public function checkServerApi()
{
if ($this->isServerApiEnabled()) {
$server_ip = $this->ip;
if (isDev()) {
if ($this->id === 0) {
$server_ip = 'localhost';
}
}
$command = "curl -s http://{$server_ip}:12172/api/health";
$process = Process::timeout(5)->run($command);
if ($process->failed()) {
ray($process->exitCode(), $process->output(), $process->errorOutput());
throw new \Exception("Server API is not reachable on http://{$server_ip}:12172");
}
}
}
public function checkSentinel()
{
ray("Checking sentinel on server: {$this->name}");
if ($this->isMetricsEnabled()) {
if ($this->isSentinelEnabled()) {
$sentinel_found = instant_remote_process(['docker inspect coolify-sentinel'], $this, false);
$sentinel_found = json_decode($sentinel_found, true);
$status = data_get($sentinel_found, '0.State.Status', 'exited');
@@ -497,10 +527,10 @@ $schema://$host {
$cpu = str($cpu)->explode("\n")->skip(1)->all();
$parsedCollection = collect($cpu)->flatMap(function ($item) {
return collect(explode("\n", trim($item)))->map(function ($line) {
[$time, $value] = explode(',', trim($line));
$value = number_format($value, 0);
[$time, $cpu_usage_percent] = explode(',', trim($line));
$cpu_usage_percent = number_format($cpu_usage_percent, 0);
return [(int) $time, (float) $value];
return [(int) $time, (float) $cpu_usage_percent];
});
});

29
app/Models/StandaloneClickhouse.php
View File

@@ -226,4 +226,33 @@ class StandaloneClickhouse extends BaseModel
{
return $this->morphMany(ScheduledDatabaseBackup::class, 'database');
}
public function getMetrics(int $mins = 5)
{
$server = $this->destination->server;
$container_name = $this->uuid;
if ($server->isMetricsEnabled()) {
$from = now()->subMinutes($mins)->toIso8601ZuluString();
$metrics = instant_remote_process(["docker exec coolify-sentinel sh -c 'curl -H \"Authorization: Bearer {$server->settings->metrics_token}\" http://localhost:8888/api/container/{$container_name}/metrics/history?from=$from'"], $server, false);
if (str($metrics)->contains('error')) {
$error = json_decode($metrics, true);
$error = data_get($error, 'error', 'Something is not okay, are you okay?');
if ($error == 'Unauthorized') {
$error = 'Unauthorized, please check your metrics token or restart Sentinel to set a new token.';
}
throw new \Exception($error);
}
$metrics = str($metrics)->explode("\n")->skip(1)->all();
$parsedCollection = collect($metrics)->flatMap(function ($item) {
return collect(explode("\n", trim($item)))->map(function ($line) {
[$time, $cpu_usage_percent, $memory_usage, $memory_usage_percent] = explode(',', trim($line));
$cpu_usage_percent = number_format($cpu_usage_percent, 2);
return [(int) $time, (float) $cpu_usage_percent, (int) $memory_usage];
});
});
return $parsedCollection->toArray();
}
}
}

29
app/Models/StandaloneDragonfly.php
View File

@@ -226,4 +226,33 @@ class StandaloneDragonfly extends BaseModel
{
return $this->morphMany(ScheduledDatabaseBackup::class, 'database');
}
public function getMetrics(int $mins = 5)
{
$server = $this->destination->server;
$container_name = $this->uuid;
if ($server->isMetricsEnabled()) {
$from = now()->subMinutes($mins)->toIso8601ZuluString();
$metrics = instant_remote_process(["docker exec coolify-sentinel sh -c 'curl -H \"Authorization: Bearer {$server->settings->metrics_token}\" http://localhost:8888/api/container/{$container_name}/metrics/history?from=$from'"], $server, false);
if (str($metrics)->contains('error')) {
$error = json_decode($metrics, true);
$error = data_get($error, 'error', 'Something is not okay, are you okay?');
if ($error == 'Unauthorized') {
$error = 'Unauthorized, please check your metrics token or restart Sentinel to set a new token.';
}
throw new \Exception($error);
}
$metrics = str($metrics)->explode("\n")->skip(1)->all();
$parsedCollection = collect($metrics)->flatMap(function ($item) {
return collect(explode("\n", trim($item)))->map(function ($line) {
[$time, $cpu_usage_percent, $memory_usage, $memory_usage_percent] = explode(',', trim($line));
$cpu_usage_percent = number_format($cpu_usage_percent, 2);
return [(int) $time, (float) $cpu_usage_percent, (int) $memory_usage];
});
});
return $parsedCollection->toArray();
}
}
}

29
app/Models/StandaloneKeydb.php
View File

@@ -226,4 +226,33 @@ class StandaloneKeydb extends BaseModel
{
return $this->morphMany(ScheduledDatabaseBackup::class, 'database');
}
public function getMetrics(int $mins = 5)
{
$server = $this->destination->server;
$container_name = $this->uuid;
if ($server->isMetricsEnabled()) {
$from = now()->subMinutes($mins)->toIso8601ZuluString();
$metrics = instant_remote_process(["docker exec coolify-sentinel sh -c 'curl -H \"Authorization: Bearer {$server->settings->metrics_token}\" http://localhost:8888/api/container/{$container_name}/metrics/history?from=$from'"], $server, false);
if (str($metrics)->contains('error')) {
$error = json_decode($metrics, true);
$error = data_get($error, 'error', 'Something is not okay, are you okay?');
if ($error == 'Unauthorized') {
$error = 'Unauthorized, please check your metrics token or restart Sentinel to set a new token.';
}
throw new \Exception($error);
}
$metrics = str($metrics)->explode("\n")->skip(1)->all();
$parsedCollection = collect($metrics)->flatMap(function ($item) {
return collect(explode("\n", trim($item)))->map(function ($line) {
[$time, $cpu_usage_percent, $memory_usage, $memory_usage_percent] = explode(',', trim($line));
$cpu_usage_percent = number_format($cpu_usage_percent, 2);
return [(int) $time, (float) $cpu_usage_percent, (int) $memory_usage];
});
});
return $parsedCollection->toArray();
}
}
}

29
app/Models/StandaloneMariadb.php
View File

@@ -226,4 +226,33 @@ class StandaloneMariadb extends BaseModel
{
return $this->morphMany(ScheduledDatabaseBackup::class, 'database');
}
public function getMetrics(int $mins = 5)
{
$server = $this->destination->server;
$container_name = $this->uuid;
if ($server->isMetricsEnabled()) {
$from = now()->subMinutes($mins)->toIso8601ZuluString();
$metrics = instant_remote_process(["docker exec coolify-sentinel sh -c 'curl -H \"Authorization: Bearer {$server->settings->metrics_token}\" http://localhost:8888/api/container/{$container_name}/metrics/history?from=$from'"], $server, false);
if (str($metrics)->contains('error')) {
$error = json_decode($metrics, true);
$error = data_get($error, 'error', 'Something is not okay, are you okay?');
if ($error == 'Unauthorized') {
$error = 'Unauthorized, please check your metrics token or restart Sentinel to set a new token.';
}
throw new \Exception($error);
}
$metrics = str($metrics)->explode("\n")->skip(1)->all();
$parsedCollection = collect($metrics)->flatMap(function ($item) {
return collect(explode("\n", trim($item)))->map(function ($line) {
[$time, $cpu_usage_percent, $memory_usage, $memory_usage_percent] = explode(',', trim($line));
$cpu_usage_percent = number_format($cpu_usage_percent, 2);
return [(int) $time, (float) $cpu_usage_percent, (int) $memory_usage];
});
});
return $parsedCollection->toArray();
}
}
}

29
app/Models/StandaloneMongodb.php
View File

@@ -246,4 +246,33 @@ class StandaloneMongodb extends BaseModel
{
return $this->morphMany(ScheduledDatabaseBackup::class, 'database');
}
public function getMetrics(int $mins = 5)
{
$server = $this->destination->server;
$container_name = $this->uuid;
if ($server->isMetricsEnabled()) {
$from = now()->subMinutes($mins)->toIso8601ZuluString();
$metrics = instant_remote_process(["docker exec coolify-sentinel sh -c 'curl -H \"Authorization: Bearer {$server->settings->metrics_token}\" http://localhost:8888/api/container/{$container_name}/metrics/history?from=$from'"], $server, false);
if (str($metrics)->contains('error')) {
$error = json_decode($metrics, true);
$error = data_get($error, 'error', 'Something is not okay, are you okay?');
if ($error == 'Unauthorized') {
$error = 'Unauthorized, please check your metrics token or restart Sentinel to set a new token.';
}
throw new \Exception($error);
}
$metrics = str($metrics)->explode("\n")->skip(1)->all();
$parsedCollection = collect($metrics)->flatMap(function ($item) {
return collect(explode("\n", trim($item)))->map(function ($line) {
[$time, $cpu_usage_percent, $memory_usage, $memory_usage_percent] = explode(',', trim($line));
$cpu_usage_percent = number_format($cpu_usage_percent, 2);
return [(int) $time, (float) $cpu_usage_percent, (int) $memory_usage];
});
});
return $parsedCollection->toArray();
}
}
}

29
app/Models/StandaloneMysql.php
View File

@@ -227,4 +227,33 @@ class StandaloneMysql extends BaseModel
{
return $this->morphMany(ScheduledDatabaseBackup::class, 'database');
}
public function getMetrics(int $mins = 5)
{
$server = $this->destination->server;
$container_name = $this->uuid;
if ($server->isMetricsEnabled()) {
$from = now()->subMinutes($mins)->toIso8601ZuluString();
$metrics = instant_remote_process(["docker exec coolify-sentinel sh -c 'curl -H \"Authorization: Bearer {$server->settings->metrics_token}\" http://localhost:8888/api/container/{$container_name}/metrics/history?from=$from'"], $server, false);
if (str($metrics)->contains('error')) {
$error = json_decode($metrics, true);
$error = data_get($error, 'error', 'Something is not okay, are you okay?');
if ($error == 'Unauthorized') {
$error = 'Unauthorized, please check your metrics token or restart Sentinel to set a new token.';
}
throw new \Exception($error);
}
$metrics = str($metrics)->explode("\n")->skip(1)->all();
$parsedCollection = collect($metrics)->flatMap(function ($item) {
return collect(explode("\n", trim($item)))->map(function ($line) {
[$time, $cpu_usage_percent, $memory_usage, $memory_usage_percent] = explode(',', trim($line));
$cpu_usage_percent = number_format($cpu_usage_percent, 2);
return [(int) $time, (float) $cpu_usage_percent, (int) $memory_usage];
});
});
return $parsedCollection->toArray();
}
}
}

29
app/Models/StandalonePostgresql.php
View File

@@ -227,4 +227,33 @@ class StandalonePostgresql extends BaseModel
{
return $this->morphMany(ScheduledDatabaseBackup::class, 'database');
}
public function getMetrics(int $mins = 5)
{
$server = $this->destination->server;
$container_name = $this->uuid;
if ($server->isMetricsEnabled()) {
$from = now()->subMinutes($mins)->toIso8601ZuluString();
$metrics = instant_remote_process(["docker exec coolify-sentinel sh -c 'curl -H \"Authorization: Bearer {$server->settings->metrics_token}\" http://localhost:8888/api/container/{$container_name}/metrics/history?from=$from'"], $server, false);
if (str($metrics)->contains('error')) {
$error = json_decode($metrics, true);
$error = data_get($error, 'error', 'Something is not okay, are you okay?');
if ($error == 'Unauthorized') {
$error = 'Unauthorized, please check your metrics token or restart Sentinel to set a new token.';
}
throw new \Exception($error);
}
$metrics = str($metrics)->explode("\n")->skip(1)->all();
$parsedCollection = collect($metrics)->flatMap(function ($item) {
return collect(explode("\n", trim($item)))->map(function ($line) {
[$time, $cpu_usage_percent, $memory_usage, $memory_usage_percent] = explode(',', trim($line));
$cpu_usage_percent = number_format($cpu_usage_percent, 2);
return [(int) $time, (float) $cpu_usage_percent, (int) $memory_usage];
});
});
return $parsedCollection->toArray();
}
}
}

29
app/Models/StandaloneRedis.php
View File

@@ -222,4 +222,33 @@ class StandaloneRedis extends BaseModel
{
return $this->morphMany(ScheduledDatabaseBackup::class, 'database');
}
public function getMetrics(int $mins = 5)
{
$server = $this->destination->server;
$container_name = $this->uuid;
if ($server->isMetricsEnabled()) {
$from = now()->subMinutes($mins)->toIso8601ZuluString();
$metrics = instant_remote_process(["docker exec coolify-sentinel sh -c 'curl -H \"Authorization: Bearer {$server->settings->metrics_token}\" http://localhost:8888/api/container/{$container_name}/metrics/history?from=$from'"], $server, false);
if (str($metrics)->contains('error')) {
$error = json_decode($metrics, true);
$error = data_get($error, 'error', 'Something is not okay, are you okay?');
if ($error == 'Unauthorized') {
$error = 'Unauthorized, please check your metrics token or restart Sentinel to set a new token.';
}
throw new \Exception($error);
}
$metrics = str($metrics)->explode("\n")->skip(1)->all();
$parsedCollection = collect($metrics)->flatMap(function ($item) {
return collect(explode("\n", trim($item)))->map(function ($line) {
[$time, $cpu_usage_percent, $memory_usage, $memory_usage_percent] = explode(',', trim($line));
$cpu_usage_percent = number_format($cpu_usage_percent, 2);
return [(int) $time, (float) $cpu_usage_percent, (int) $memory_usage];
});
});
return $parsedCollection->toArray();
}
}
}