dave/coolify
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

sentinel updates

Browse Source
This commit is contained in:
Andras Bacsai
2024-10-14 12:07:37 +02:00
parent c137620b81
commit 1f193d465d
23 changed files with 293 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
app/Actions/Server/StartSentinel.php
View File

@@ -2,6 +2,7 @@
namespace App\Actions\Server;
use App\Models\InstanceSettings;
use App\Models\Server;
use Lorisleiva\Actions\Concerns\AsAction;
@@ -16,11 +17,25 @@ class StartSentinel
}
$metrics_history = $server->settings->metrics_history_days;
$refresh_rate = $server->settings->metrics_refresh_rate_seconds;
$token = $server->settings->metrics_token;
instant_remote_process([
"docker run --rm --pull always -d -e \"TOKEN={$token}\" -e \"SCHEDULER=true\" -e \"METRICS_HISTORY={$metrics_history}\" -e \"REFRESH_RATE={$refresh_rate}\" --name coolify-sentinel -v /var/run/docker.sock:/var/run/docker.sock -v /data/coolify/metrics:/app/metrics -v /data/coolify/logs:/app/logs --pid host --health-cmd \"curl --fail http://127.0.0.1:8888/api/health || exit 1\" --health-interval 10s --health-retries 3 ghcr.io/coollabsio/sentinel:$version",
'chown -R 9999:root /data/coolify/metrics /data/coolify/logs',
'chmod -R 700 /data/coolify/metrics /data/coolify/logs',
], $server, true);
$token = $server->settings->sentinel_token;
$fqdn = InstanceSettings::get()->fqdn;
if (str($fqdn)->startsWith('http')) {
throw new \Exception('You should use https to run Sentinel.');
}
$environments = [
'TOKEN' => $token,
'ENDPOINT' => InstanceSettings::get()->fqdn,
'COLLECTOR_ENABLED' => 'true',
'COLLECTOR_REFRESH_RATE_SECONDS' => $refresh_rate,
'COLLECTOR_RETENTION_PERIOD_DAYS' => $metrics_history
];
$docker_environments = "-e \"" . implode("\" -e \"", array_map(fn($key, $value) => "$key=$value", array_keys($environments), $environments)) . "\"";
ray($docker_environments);
return true;
// instant_remote_process([
// "docker run --rm --pull always -d $docker_environments --name coolify-sentinel -v /var/run/docker.sock:/var/run/docker.sock -v /data/coolify/sentinel:/app/sentinel --pid host --health-cmd \"curl --fail http://127.0.0.1:8888/api/health || exit 1\" --health-interval 10s --health-retries 3 ghcr.io/coollabsio/sentinel:$version",
// 'chown -R 9999:root /data/coolify/sentinel',
// 'chmod -R 700 /data/coolify/sentinel',
// ], $server, true);
}
}

2
app/Http/Controllers/Api/ServersController.php
View File

@@ -23,7 +23,7 @@ class ServersController extends Controller
return serializeApiResponse($settings);
}
$settings = $settings->makeHidden([
'metrics_token',
'sentinel_token',
]);
return serializeApiResponse($settings);

53
app/Jobs/PushServerUpdateJob.php Normal file
View File

@@ -0,0 +1,53 @@
<?php
namespace App\Jobs;
use App\Models\Server;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Arr;
use Illuminate\Support\Facades\Log;
class PushServerUpdateJob implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public $tries = 1;
public $timeout = 60;
public function backoff(): int
{
return isDev() ? 1 : 3;
}
public function __construct(public Server $server, public $data) {}
public function handle()
{
if (!$this->data) {
throw new \Exception('No data provided');
}
$data = collect($this->data);
$containers = collect(data_get($data, 'containers'));
if ($containers->isEmpty()) {
return;
}
foreach ($containers as $container) {
$containerStatus = data_get($container, 'status', 'exited');
$containerHealth = data_get($container, 'health', 'unhealthy');
$containerStatus = "$containerStatus ($containerHealth)";
$labels = collect(data_get($container, 'labels'));
if ($labels->has('coolify.applicationId')) {
$applicationId = $labels->get('coolify.applicationId');
}
Log::info("$applicationId, $containerStatus");
}
}
}

45
app/Livewire/Server/Form.php
View File

@@ -7,6 +7,7 @@ use App\Actions\Server\StopSentinel;
use App\Jobs\DockerCleanupJob;
use App\Jobs\PullSentinelImageJob;
use App\Models\Server;
use Illuminate\Support\Facades\Http;
use Livewire\Component;
class Form extends Component
@@ -54,9 +55,9 @@ class Form extends Component
'server.settings.concurrent_builds' => 'required|integer|min:1',
'server.settings.dynamic_timeout' => 'required|integer|min:1',
'server.settings.is_metrics_enabled' => 'required|boolean',
'server.settings.metrics_token' => 'required',
'server.settings.metrics_refresh_rate_seconds' => 'required|integer|min:1',
'server.settings.metrics_history_days' => 'required|integer|min:1',
'server.settings.sentinel_token' => 'required',
'server.settings.sentinel_metrics_refresh_rate_seconds' => 'required|integer|min:1',
'server.settings.sentinel_metrics_history_days' => 'required|integer|min:1',
'wildcard_domain' => 'nullable|url',
'server.settings.is_server_api_enabled' => 'required|boolean',
'server.settings.server_timezone' => 'required|string|timezone',
@@ -81,9 +82,9 @@ class Form extends Component
'server.settings.concurrent_builds' => 'Concurrent Builds',
'server.settings.dynamic_timeout' => 'Dynamic Timeout',
'server.settings.is_metrics_enabled' => 'Metrics',
'server.settings.metrics_token' => 'Metrics Token',
'server.settings.metrics_refresh_rate_seconds' => 'Metrics Interval',
'server.settings.metrics_history_days' => 'Metrics History',
'server.settings.sentinel_token' => 'Metrics Token',
'server.settings.sentinel_metrics_refresh_rate_seconds' => 'Metrics Interval',
'server.settings.sentinel_metrics_history_days' => 'Metrics History',
'server.settings.is_server_api_enabled' => 'Server API',
'server.settings.server_timezone' => 'Server Timezone',
'server.settings.delete_unused_volumes' => 'Delete Unused Volumes',
@@ -100,7 +101,15 @@ class Form extends Component
$this->server->settings->delete_unused_volumes = $server->settings->delete_unused_volumes;
$this->server->settings->delete_unused_networks = $server->settings->delete_unused_networks;
}
public function regenerateSentinelToken() {
try {
$this->server->generateSentinelToken();
$this->server->settings->refresh();
$this->dispatch('success', 'Metrics token regenerated.');
} catch (\Throwable $e) {
return handleError($e, $this);
}
}
public function updated($field)
{
if ($field === 'server.settings.docker_cleanup_frequency') {
@@ -174,6 +183,28 @@ class Form extends Component
}
}
public function getPushData()
{
try {
if (!isDev()) {
throw new \Exception('This feature is only available in dev mode.');
}
$response = Http::withHeaders([
'Authorization' => 'Bearer ' . $this->server->settings->sentinel_token,
])->post('http://host.docker.internal:8888/api/push', [
'data' => 'test',
]);
if ($response->successful()) {
$this->dispatch('success', 'Push data sent.');
return;
}
$error = data_get($response->json(), 'error');
throw new \Exception($error);
} catch(\Throwable $e) {
return handleError($e, $this);
}
}
public function restartSentinel()
{
try {

2
app/Models/Application.php
View File

@@ -1406,7 +1406,7 @@ class Application extends BaseModel
$container_name = $this->uuid;
if ($server->isMetricsEnabled()) {
$from = now()->subMinutes($mins)->toIso8601ZuluString();
$metrics = instant_remote_process(["docker exec coolify-sentinel sh -c 'curl -H \"Authorization: Bearer {$server->settings->metrics_token}\" http://localhost:8888/api/container/{$container_name}/metrics/history?from=$from'"], $server, false);
$metrics = instant_remote_process(["docker exec coolify-sentinel sh -c 'curl -H \"Authorization: Bearer {$server->settings->sentinel_token}\" http://localhost:8888/api/container/{$container_name}/metrics/history?from=$from'"], $server, false);
if (str($metrics)->contains('error')) {
$error = json_decode($metrics, true);
$error = data_get($error, 'error', 'Something is not okay, are you okay?');

1
app/Models/InstanceSettings.php
View File

@@ -21,6 +21,7 @@ class InstanceSettings extends Model implements SendsEmail
'is_auto_update_enabled' => 'boolean',
'auto_update_frequency' => 'string',
'update_check_frequency' => 'string',
'sentinel_token' => 'encrypted',
];
public function fqdn(): Attribute

42
app/Models/Server.php
View File

@@ -17,6 +17,8 @@ use Spatie\SchemalessAttributes\Casts\SchemalessAttributes;
use Spatie\SchemalessAttributes\SchemalessAttributesTrait;
use Spatie\Url\Url;
use Symfony\Component\Yaml\Yaml;
use Illuminate\Support\Str;
#[OA\Schema(
description: 'Server model',
@@ -166,7 +168,7 @@ class Server extends BaseModel
public function setupDefault404Redirect()
{
$dynamic_conf_path = $this->proxyPath().'/dynamic';
$dynamic_conf_path = $this->proxyPath() . '/dynamic';
$proxy_type = $this->proxyType();
$redirect_url = $this->proxy->redirect_url;
if ($proxy_type === ProxyTypes::TRAEFIK->value) {
@@ -180,8 +182,8 @@ class Server extends BaseModel
respond 404
}';
$conf =
"# This file is automatically generated by Coolify.\n".
"# Do not edit it manually (only if you know what are you doing).\n\n".
"# This file is automatically generated by Coolify.\n" .
"# Do not edit it manually (only if you know what are you doing).\n\n" .
$conf;
$base64 = base64_encode($conf);
instant_remote_process([
@@ -243,8 +245,8 @@ respond 404
];
$conf = Yaml::dump($dynamic_conf, 12, 2);
$conf =
"# This file is automatically generated by Coolify.\n".
"# Do not edit it manually (only if you know what are you doing).\n\n".
"# This file is automatically generated by Coolify.\n" .
"# Do not edit it manually (only if you know what are you doing).\n\n" .
$conf;
$base64 = base64_encode($conf);
@@ -253,8 +255,8 @@ respond 404
redir $redirect_url
}";
$conf =
"# This file is automatically generated by Coolify.\n".
"# Do not edit it manually (only if you know what are you doing).\n\n".
"# This file is automatically generated by Coolify.\n" .
"# Do not edit it manually (only if you know what are you doing).\n\n" .
$conf;
$base64 = base64_encode($conf);
}
@@ -272,7 +274,7 @@ respond 404
public function setupDynamicProxyConfiguration()
{
$settings = instanceSettings();
$dynamic_config_path = $this->proxyPath().'/dynamic';
$dynamic_config_path = $this->proxyPath() . '/dynamic';
if ($this->proxyType() === ProxyTypes::TRAEFIK->value) {
$file = "$dynamic_config_path/coolify.yaml";
if (empty($settings->fqdn) || (isCloud() && $this->id !== 0) || ! $this->isLocalhost()) {
@@ -391,8 +393,8 @@ respond 404
}
$yaml = Yaml::dump($traefik_dynamic_conf, 12, 2);
$yaml =
"# This file is automatically generated by Coolify.\n".
"# Do not edit it manually (only if you know what are you doing).\n\n".
"# This file is automatically generated by Coolify.\n" .
"# Do not edit it manually (only if you know what are you doing).\n\n" .
$yaml;
$base64 = base64_encode($yaml);
@@ -456,13 +458,13 @@ $schema://$host {
if (isDev()) {
$proxy_path = '/var/lib/docker/volumes/coolify_dev_coolify_data/_data/proxy/caddy';
} else {
$proxy_path = $proxy_path.'/caddy';
$proxy_path = $proxy_path . '/caddy';
}
} elseif ($proxyType === ProxyTypes::NGINX->value) {
if (isDev()) {
$proxy_path = '/var/lib/docker/volumes/coolify_dev_coolify_data/_data/proxy/nginx';
} else {
$proxy_path = $proxy_path.'/nginx';
$proxy_path = $proxy_path . '/nginx';
}
}
@@ -525,6 +527,17 @@ $schema://$host {
Storage::disk('ssh-mux')->delete($this->muxFilename());
}
public function generateSentinelToken()
{
$data = [
'server_uuid' => $this->uuid,
];
$token = json_encode($data);
$encrypted = encrypt($token);
$this->settings->sentinel_token = $encrypted;
$this->settings->save();
return $encrypted;
}
public function isSentinelEnabled()
{
return $this->isMetricsEnabled() || $this->isServerApiEnabled();
@@ -555,7 +568,6 @@ $schema://$host {
ray($process->exitCode(), $process->output(), $process->errorOutput());
throw new \Exception("Server API is not reachable on http://{$server_ip}:12172");
}
}
}
@@ -579,7 +591,7 @@ $schema://$host {
{
if ($this->isMetricsEnabled()) {
$from = now()->subMinutes($mins)->toIso8601ZuluString();
$cpu = instant_remote_process(["docker exec coolify-sentinel sh -c 'curl -H \"Authorization: Bearer {$this->settings->metrics_token}\" http://localhost:8888/api/cpu/history?from=$from'"], $this, false);
$cpu = instant_remote_process(["docker exec coolify-sentinel sh -c 'curl -H \"Authorization: Bearer {$this->settings->sentinel_token}\" http://localhost:8888/api/cpu/history?from=$from'"], $this, false);
if (str($cpu)->contains('error')) {
$error = json_decode($cpu, true);
$error = data_get($error, 'error', 'Something is not okay, are you okay?');
@@ -606,7 +618,7 @@ $schema://$host {
{
if ($this->isMetricsEnabled()) {
$from = now()->subMinutes($mins)->toIso8601ZuluString();
$memory = instant_remote_process(["docker exec coolify-sentinel sh -c 'curl -H \"Authorization: Bearer {$this->settings->metrics_token}\" http://localhost:8888/api/memory/history?from=$from'"], $this, false);
$memory = instant_remote_process(["docker exec coolify-sentinel sh -c 'curl -H \"Authorization: Bearer {$this->settings->sentinel_token}\" http://localhost:8888/api/memory/history?from=$from'"], $this, false);
if (str($memory)->contains('error')) {
$error = json_decode($memory, true);
$error = data_get($error, 'error', 'Something is not okay, are you okay?');

7
app/Models/ServerSetting.php
View File

@@ -35,9 +35,9 @@ use OpenApi\Attributes as OA;
'logdrain_highlight_project_id' => ['type' => 'string'],
'logdrain_newrelic_base_uri' => ['type' => 'string'],
'logdrain_newrelic_license_key' => ['type' => 'string'],
'metrics_history_days' => ['type' => 'integer'],
'metrics_refresh_rate_seconds' => ['type' => 'integer'],
'metrics_token' => ['type' => 'string'],
'sentinel_metrics_history_days' => ['type' => 'integer'],
'sentinel_metrics_refresh_rate_seconds' => ['type' => 'integer'],
'sentinel_token' => ['type' => 'string'],
'docker_cleanup_frequency' => ['type' => 'string'],
'docker_cleanup_threshold' => ['type' => 'integer'],
'server_id' => ['type' => 'integer'],
@@ -53,6 +53,7 @@ class ServerSetting extends Model
protected $casts = [
'force_docker_cleanup' => 'boolean',
'docker_cleanup_threshold' => 'integer',
'sentinel_token' => 'encrypted',
];
public function server()

2
app/Models/StandaloneClickhouse.php
View File

@@ -272,7 +272,7 @@ class StandaloneClickhouse extends BaseModel
$container_name = $this->uuid;
if ($server->isMetricsEnabled()) {
$from = now()->subMinutes($mins)->toIso8601ZuluString();
$metrics = instant_remote_process(["docker exec coolify-sentinel sh -c 'curl -H \"Authorization: Bearer {$server->settings->metrics_token}\" http://localhost:8888/api/container/{$container_name}/metrics/history?from=$from'"], $server, false);
$metrics = instant_remote_process(["docker exec coolify-sentinel sh -c 'curl -H \"Authorization: Bearer {$server->settings->sentinel_token}\" http://localhost:8888/api/container/{$container_name}/metrics/history?from=$from'"], $server, false);
if (str($metrics)->contains('error')) {
$error = json_decode($metrics, true);
$error = data_get($error, 'error', 'Something is not okay, are you okay?');

2
app/Models/StandaloneDragonfly.php
View File

@@ -272,7 +272,7 @@ class StandaloneDragonfly extends BaseModel
$container_name = $this->uuid;
if ($server->isMetricsEnabled()) {
$from = now()->subMinutes($mins)->toIso8601ZuluString();
$metrics = instant_remote_process(["docker exec coolify-sentinel sh -c 'curl -H \"Authorization: Bearer {$server->settings->metrics_token}\" http://localhost:8888/api/container/{$container_name}/metrics/history?from=$from'"], $server, false);
$metrics = instant_remote_process(["docker exec coolify-sentinel sh -c 'curl -H \"Authorization: Bearer {$server->settings->sentinel_token}\" http://localhost:8888/api/container/{$container_name}/metrics/history?from=$from'"], $server, false);
if (str($metrics)->contains('error')) {
$error = json_decode($metrics, true);
$error = data_get($error, 'error', 'Something is not okay, are you okay?');

2
app/Models/StandaloneKeydb.php
View File

@@ -272,7 +272,7 @@ class StandaloneKeydb extends BaseModel
$container_name = $this->uuid;
if ($server->isMetricsEnabled()) {
$from = now()->subMinutes($mins)->toIso8601ZuluString();
$metrics = instant_remote_process(["docker exec coolify-sentinel sh -c 'curl -H \"Authorization: Bearer {$server->settings->metrics_token}\" http://localhost:8888/api/container/{$container_name}/metrics/history?from=$from'"], $server, false);
$metrics = instant_remote_process(["docker exec coolify-sentinel sh -c 'curl -H \"Authorization: Bearer {$server->settings->sentinel_token}\" http://localhost:8888/api/container/{$container_name}/metrics/history?from=$from'"], $server, false);
if (str($metrics)->contains('error')) {
$error = json_decode($metrics, true);
$error = data_get($error, 'error', 'Something is not okay, are you okay?');

2
app/Models/StandaloneMariadb.php
View File

@@ -272,7 +272,7 @@ class StandaloneMariadb extends BaseModel
$container_name = $this->uuid;
if ($server->isMetricsEnabled()) {
$from = now()->subMinutes($mins)->toIso8601ZuluString();
$metrics = instant_remote_process(["docker exec coolify-sentinel sh -c 'curl -H \"Authorization: Bearer {$server->settings->metrics_token}\" http://localhost:8888/api/container/{$container_name}/metrics/history?from=$from'"], $server, false);
$metrics = instant_remote_process(["docker exec coolify-sentinel sh -c 'curl -H \"Authorization: Bearer {$server->settings->sentinel_token}\" http://localhost:8888/api/container/{$container_name}/metrics/history?from=$from'"], $server, false);
if (str($metrics)->contains('error')) {
$error = json_decode($metrics, true);
$error = data_get($error, 'error', 'Something is not okay, are you okay?');

2
app/Models/StandaloneMongodb.php
View File

@@ -292,7 +292,7 @@ class StandaloneMongodb extends BaseModel
$container_name = $this->uuid;
if ($server->isMetricsEnabled()) {
$from = now()->subMinutes($mins)->toIso8601ZuluString();
$metrics = instant_remote_process(["docker exec coolify-sentinel sh -c 'curl -H \"Authorization: Bearer {$server->settings->metrics_token}\" http://localhost:8888/api/container/{$container_name}/metrics/history?from=$from'"], $server, false);
$metrics = instant_remote_process(["docker exec coolify-sentinel sh -c 'curl -H \"Authorization: Bearer {$server->settings->sentinel_token}\" http://localhost:8888/api/container/{$container_name}/metrics/history?from=$from'"], $server, false);
if (str($metrics)->contains('error')) {
$error = json_decode($metrics, true);
$error = data_get($error, 'error', 'Something is not okay, are you okay?');

2
app/Models/StandaloneMysql.php
View File

@@ -273,7 +273,7 @@ class StandaloneMysql extends BaseModel
$container_name = $this->uuid;
if ($server->isMetricsEnabled()) {
$from = now()->subMinutes($mins)->toIso8601ZuluString();
$metrics = instant_remote_process(["docker exec coolify-sentinel sh -c 'curl -H \"Authorization: Bearer {$server->settings->metrics_token}\" http://localhost:8888/api/container/{$container_name}/metrics/history?from=$from'"], $server, false);
$metrics = instant_remote_process(["docker exec coolify-sentinel sh -c 'curl -H \"Authorization: Bearer {$server->settings->sentinel_token}\" http://localhost:8888/api/container/{$container_name}/metrics/history?from=$from'"], $server, false);
if (str($metrics)->contains('error')) {
$error = json_decode($metrics, true);
$error = data_get($error, 'error', 'Something is not okay, are you okay?');

2
app/Models/StandalonePostgresql.php
View File

@@ -274,7 +274,7 @@ class StandalonePostgresql extends BaseModel
$container_name = $this->uuid;
if ($server->isMetricsEnabled()) {
$from = now()->subMinutes($mins)->toIso8601ZuluString();
$metrics = instant_remote_process(["docker exec coolify-sentinel sh -c 'curl -H \"Authorization: Bearer {$server->settings->metrics_token}\" http://localhost:8888/api/container/{$container_name}/metrics/history?from=$from'"], $server, false);
$metrics = instant_remote_process(["docker exec coolify-sentinel sh -c 'curl -H \"Authorization: Bearer {$server->settings->sentinel_token}\" http://localhost:8888/api/container/{$container_name}/metrics/history?from=$from'"], $server, false);
if (str($metrics)->contains('error')) {
$error = json_decode($metrics, true);
$error = data_get($error, 'error', 'Something is not okay, are you okay?');

2
app/Models/StandaloneRedis.php
View File

@@ -268,7 +268,7 @@ class StandaloneRedis extends BaseModel
$container_name = $this->uuid;
if ($server->isMetricsEnabled()) {
$from = now()->subMinutes($mins)->toIso8601ZuluString();
$metrics = instant_remote_process(["docker exec coolify-sentinel sh -c 'curl -H \"Authorization: Bearer {$server->settings->metrics_token}\" http://localhost:8888/api/container/{$container_name}/metrics/history?from=$from'"], $server, false);
$metrics = instant_remote_process(["docker exec coolify-sentinel sh -c 'curl -H \"Authorization: Bearer {$server->settings->sentinel_token}\" http://localhost:8888/api/container/{$container_name}/metrics/history?from=$from'"], $server, false);
if (str($metrics)->contains('error')) {
$error = json_decode($metrics, true);
$error = data_get($error, 'error', 'Something is not okay, are you okay?');