sentinel updates
This commit is contained in:
@@ -17,39 +17,46 @@ class StartSentinel
|
||||
}
|
||||
$metrics_history = $server->settings->sentinel_metrics_history_days;
|
||||
$refresh_rate = $server->settings->sentinel_metrics_refresh_rate_seconds;
|
||||
$push_interval = $server->settings->sentinel_push_interval_seconds;
|
||||
$token = $server->settings->sentinel_token;
|
||||
$endpoint = InstanceSettings::get()->fqdn;
|
||||
if (isDev()) {
|
||||
$mount_dir = '/data/coolify/sentinel';
|
||||
$image = "ghcr.io/coollabsio/sentinel:$version";
|
||||
|
||||
if ($server->isLocalhost()) {
|
||||
$endpoint = 'http://host.docker.internal:8000';
|
||||
}
|
||||
} else {
|
||||
if (! $endpoint) {
|
||||
throw new \Exception('You should set FQDN in Instance Settings.');
|
||||
}
|
||||
// Ensure the endpoint is using HTTPS
|
||||
$endpoint = str($endpoint)->replace('http://', 'https://')->value();
|
||||
}
|
||||
$environments = [
|
||||
'TOKEN' => $token,
|
||||
'ENDPOINT' => $endpoint,
|
||||
'COLLECTOR_ENABLED' => 'true',
|
||||
'PUSH_ENDPOINT' => $endpoint,
|
||||
'PUSH_INTERVAL_SECONDS' => $push_interval,
|
||||
'COLLECTOR_ENABLED' => $server->isMetricsEnabled() ? 'true' : 'false',
|
||||
'COLLECTOR_REFRESH_RATE_SECONDS' => $refresh_rate,
|
||||
'COLLECTOR_RETENTION_PERIOD_DAYS' => $metrics_history,
|
||||
];
|
||||
if (isDev()) {
|
||||
data_set($environments, 'GIN_MODE', 'debug');
|
||||
}
|
||||
$mount_dir = '/data/coolify/sentinel';
|
||||
if (isDev()) {
|
||||
data_set($environments, 'DEBUG', 'true');
|
||||
$mount_dir = '/var/lib/docker/volumes/coolify_dev_coolify_data/_data/sentinel';
|
||||
$image = 'sentinel';
|
||||
}
|
||||
$docker_environments = '-e "' . implode('" -e "', array_map(fn($key, $value) => "$key=$value", array_keys($environments), $environments)) . '"';
|
||||
$docker_command = "docker run --pull always --rm -d $docker_environments --name coolify-sentinel -v /var/run/docker.sock:/var/run/docker.sock -v $mount_dir:/app/db --pid host --health-cmd \"curl --fail http://127.0.0.1:8888/api/health || exit 1\" --health-interval 10s --health-retries 3 --add-host=host.docker.internal:host-gateway ghcr.io/coollabsio/sentinel:$version";
|
||||
|
||||
return instant_remote_process([
|
||||
$docker_command = "docker run -d $docker_environments --name coolify-sentinel -v /var/run/docker.sock:/var/run/docker.sock -v $mount_dir:/app/db --pid host --health-cmd \"curl --fail http://127.0.0.1:8888/api/health || exit 1\" --health-interval 10s --health-retries 3 --add-host=host.docker.internal:host-gateway $image";
|
||||
|
||||
instant_remote_process([
|
||||
'docker rm -f coolify-sentinel || true',
|
||||
"mkdir -p $mount_dir",
|
||||
$docker_command,
|
||||
"chown -R 9999:root $mount_dir",
|
||||
"chmod -R 700 $mount_dir",
|
||||
], $server, true);
|
||||
], $server);
|
||||
|
||||
$server->settings->is_sentinel_enabled = true;
|
||||
$server->settings->save();
|
||||
$server->sentinelUpdateAt();
|
||||
}
|
||||
}
|
||||
|
@@ -3,6 +3,7 @@
|
||||
namespace App\Actions\Server;
|
||||
|
||||
use App\Models\Server;
|
||||
use Carbon\Carbon;
|
||||
use Lorisleiva\Actions\Concerns\AsAction;
|
||||
|
||||
class StopSentinel
|
||||
@@ -12,5 +13,6 @@ class StopSentinel
|
||||
public function handle(Server $server)
|
||||
{
|
||||
instant_remote_process(['docker rm -f coolify-sentinel'], $server, false);
|
||||
$server->sentinelUpdateAt(isReset: true);
|
||||
}
|
||||
}
|
||||
|
@@ -12,7 +12,6 @@ use App\Jobs\PullSentinelImageJob;
|
||||
use App\Jobs\PullTemplatesFromCDN;
|
||||
use App\Jobs\ScheduledTaskJob;
|
||||
use App\Jobs\ServerCheckJob;
|
||||
use App\Jobs\ServerStorageCheckJob;
|
||||
use App\Jobs\UpdateCoolifyJob;
|
||||
use App\Models\ScheduledDatabaseBackup;
|
||||
use App\Models\ScheduledTask;
|
||||
@@ -20,6 +19,7 @@ use App\Models\Server;
|
||||
use App\Models\Team;
|
||||
use Illuminate\Console\Scheduling\Schedule;
|
||||
use Illuminate\Foundation\Console\Kernel as ConsoleKernel;
|
||||
use Illuminate\Support\Carbon;
|
||||
|
||||
class Kernel extends ConsoleKernel
|
||||
{
|
||||
@@ -38,7 +38,7 @@ class Kernel extends ConsoleKernel
|
||||
$schedule->job(new CleanupInstanceStuffsJob)->everyMinute()->onOneServer();
|
||||
// Server Jobs
|
||||
$this->check_scheduled_backups($schedule);
|
||||
// $this->check_resources($schedule);
|
||||
$this->check_resources($schedule);
|
||||
$this->check_scheduled_tasks($schedule);
|
||||
$schedule->command('uploads:clear')->everyTwoMinutes();
|
||||
|
||||
@@ -115,7 +115,10 @@ class Kernel extends ConsoleKernel
|
||||
$servers = $this->all_servers->where('ip', '!=', '1.2.3.4');
|
||||
}
|
||||
foreach ($servers as $server) {
|
||||
$last_sentinel_update = $server->sentinel_updated_at;
|
||||
if (Carbon::parse($last_sentinel_update)->isBefore(now()->subMinutes(4))) {
|
||||
$schedule->job(new ServerCheckJob($server))->everyMinute()->onOneServer();
|
||||
}
|
||||
// $schedule->job(new ServerStorageCheckJob($server))->everyMinute()->onOneServer();
|
||||
$serverTimezone = $server->settings->server_timezone;
|
||||
if ($server->settings->force_docker_cleanup) {
|
||||
|
@@ -4,7 +4,9 @@ namespace App\Jobs;
|
||||
|
||||
use App\Actions\Database\StartDatabaseProxy;
|
||||
use App\Actions\Database\StopDatabaseProxy;
|
||||
use App\Actions\Proxy\CheckProxy;
|
||||
use App\Actions\Proxy\StartProxy;
|
||||
use App\Actions\Server\InstallLogDrain;
|
||||
use App\Actions\Shared\ComplexStatusCheck;
|
||||
use App\Models\Application;
|
||||
use App\Models\ApplicationPreview;
|
||||
@@ -40,6 +42,8 @@ class PushServerUpdateJob implements ShouldQueue
|
||||
|
||||
public Collection $allDatabaseUuids;
|
||||
|
||||
public Collection $allTcpProxyUuids;
|
||||
|
||||
public Collection $allServiceApplicationIds;
|
||||
|
||||
public Collection $allApplicationPreviewsIds;
|
||||
@@ -59,6 +63,7 @@ class PushServerUpdateJob implements ShouldQueue
|
||||
public Collection $foundApplicationPreviewsIds;
|
||||
|
||||
public bool $foundProxy = false;
|
||||
public bool $foundLogDrainContainer = false;
|
||||
|
||||
public function backoff(): int
|
||||
{
|
||||
@@ -87,6 +92,11 @@ class PushServerUpdateJob implements ShouldQueue
|
||||
throw new \Exception('No data provided');
|
||||
}
|
||||
$data = collect($this->data);
|
||||
|
||||
$this->serverStatus();
|
||||
|
||||
$this->server->sentinelUpdateAt();
|
||||
|
||||
$this->containers = collect(data_get($data, 'containers'));
|
||||
if ($this->containers->isEmpty()) {
|
||||
return;
|
||||
@@ -122,6 +132,10 @@ class PushServerUpdateJob implements ShouldQueue
|
||||
$labels = collect(data_get($container, 'labels'));
|
||||
$coolify_managed = $labels->has('coolify.managed');
|
||||
if ($coolify_managed) {
|
||||
$name = data_get($container, 'name');
|
||||
if ($name === 'coolify-log-drain' && $this->isRunning($containerStatus)) {
|
||||
$this->foundLogDrainContainer = true;
|
||||
}
|
||||
if ($labels->has('coolify.applicationId')) {
|
||||
$applicationId = $labels->get('coolify.applicationId');
|
||||
$pullRequestId = data_get($labels, 'coolify.pullRequestId', '0');
|
||||
@@ -153,7 +167,6 @@ class PushServerUpdateJob implements ShouldQueue
|
||||
}
|
||||
|
||||
} else {
|
||||
$name = data_get($container, 'name');
|
||||
$uuid = $labels->get('com.docker.compose.service');
|
||||
$type = $labels->get('coolify.type');
|
||||
if ($name === 'coolify-proxy' && $this->isRunning($containerStatus)) {
|
||||
@@ -182,12 +195,23 @@ class PushServerUpdateJob implements ShouldQueue
|
||||
$this->updateNotFoundServiceStatus();
|
||||
|
||||
$this->updateAdditionalServersStatus();
|
||||
|
||||
$this->checkLogDrainContainer();
|
||||
|
||||
} catch (\Exception $e) {
|
||||
throw $e;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private function serverStatus(){
|
||||
if ($this->server->isFunctional() === false) {
|
||||
throw new \Exception('Server is not ready.');
|
||||
}
|
||||
if ($this->server->status() === false) {
|
||||
throw new \Exception('Server is not reachable.');
|
||||
}
|
||||
}
|
||||
private function updateApplicationStatus(string $applicationId, string $containerStatus)
|
||||
{
|
||||
$application = $this->applications->where('id', $applicationId)->first();
|
||||
@@ -247,9 +271,19 @@ class PushServerUpdateJob implements ShouldQueue
|
||||
private function updateProxyStatus()
|
||||
{
|
||||
// If proxy is not found, start it
|
||||
if (! $this->foundProxy && $this->server->isProxyShouldRun()) {
|
||||
ray('Proxy not found, starting it.');
|
||||
StartProxy::dispatch($this->server);
|
||||
if ($this->server->isProxyShouldRun()) {
|
||||
if ($this->foundProxy === false) {
|
||||
try {
|
||||
if (CheckProxy::run($this->server)) {
|
||||
StartProxy::run($this->server, false);
|
||||
}
|
||||
} catch (\Throwable $e) {
|
||||
logger()->error($e);
|
||||
}
|
||||
} else {
|
||||
$connectProxyToDockerNetworks = connectProxyToNetworks($this->server);
|
||||
instant_remote_process($connectProxyToDockerNetworks, $this->server, false);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -361,4 +395,10 @@ class PushServerUpdateJob implements ShouldQueue
|
||||
{
|
||||
return str($containerStatus)->contains('running');
|
||||
}
|
||||
|
||||
private function checkLogDrainContainer(){
|
||||
if ($this->server->isLogDrainEnabled() && $this->foundLogDrainContainer === false) {
|
||||
InstallLogDrain::dispatch($this->server);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@@ -34,12 +34,12 @@ class Charts extends Component
|
||||
try {
|
||||
$cpuMetrics = $this->server->getCpuMetrics($this->interval);
|
||||
$memoryMetrics = $this->server->getMemoryMetrics($this->interval);
|
||||
$cpuMetrics = collect($cpuMetrics)->map(function ($metric) {
|
||||
return [$metric[0], $metric[1]];
|
||||
});
|
||||
$memoryMetrics = collect($memoryMetrics)->map(function ($metric) {
|
||||
return [$metric[0], $metric[1]];
|
||||
});
|
||||
// $cpuMetrics = collect($cpuMetrics)->map(function ($metric) {
|
||||
// return [$metric[0], $metric[1]];
|
||||
// });
|
||||
// $memoryMetrics = collect($memoryMetrics)->map(function ($metric) {
|
||||
// return [$metric[0], $metric[1]];
|
||||
// });
|
||||
$this->dispatch("refreshChartData-{$this->chartId}-cpu", [
|
||||
'seriesData' => $cpuMetrics,
|
||||
]);
|
||||
|
@@ -58,8 +58,9 @@ class Form extends Component
|
||||
'server.settings.sentinel_token' => 'required',
|
||||
'server.settings.sentinel_metrics_refresh_rate_seconds' => 'required|integer|min:1',
|
||||
'server.settings.sentinel_metrics_history_days' => 'required|integer|min:1',
|
||||
'server.settings.sentinel_push_interval_seconds' => 'required|integer|min:10',
|
||||
'wildcard_domain' => 'nullable|url',
|
||||
'server.settings.is_server_api_enabled' => 'required|boolean',
|
||||
'server.settings.is_sentinel_enabled' => 'required|boolean',
|
||||
'server.settings.server_timezone' => 'required|string|timezone',
|
||||
'server.settings.force_docker_cleanup' => 'required|boolean',
|
||||
'server.settings.docker_cleanup_frequency' => 'required_if:server.settings.force_docker_cleanup,true|string',
|
||||
@@ -85,7 +86,8 @@ class Form extends Component
|
||||
'server.settings.sentinel_token' => 'Metrics Token',
|
||||
'server.settings.sentinel_metrics_refresh_rate_seconds' => 'Metrics Interval',
|
||||
'server.settings.sentinel_metrics_history_days' => 'Metrics History',
|
||||
'server.settings.is_server_api_enabled' => 'Server API',
|
||||
'server.settings.sentinel_push_interval_seconds' => 'Push Interval',
|
||||
'server.settings.is_sentinel_enabled' => 'Server API',
|
||||
'server.settings.server_timezone' => 'Server Timezone',
|
||||
'server.settings.delete_unused_volumes' => 'Delete Unused Volumes',
|
||||
'server.settings.delete_unused_networks' => 'Delete Unused Networks',
|
||||
@@ -102,12 +104,17 @@ class Form extends Component
|
||||
$this->server->settings->delete_unused_networks = $server->settings->delete_unused_networks;
|
||||
}
|
||||
|
||||
public function checkSyncStatus(){
|
||||
$this->server->refresh();
|
||||
$this->server->settings->refresh();
|
||||
}
|
||||
|
||||
public function regenerateSentinelToken()
|
||||
{
|
||||
try {
|
||||
$this->server->generateSentinelToken();
|
||||
$this->server->settings->refresh();
|
||||
$this->dispatch('success', 'Metrics token regenerated.');
|
||||
$this->dispatch('success', 'Sentinel token regenerated. Please restart your Sentinel.');
|
||||
} catch (\Throwable $e) {
|
||||
return handleError($e, $this);
|
||||
}
|
||||
@@ -143,18 +150,22 @@ class Form extends Component
|
||||
$this->dispatch('proxyStatusUpdated');
|
||||
}
|
||||
|
||||
public function checkPortForServerApi()
|
||||
{
|
||||
try {
|
||||
if ($this->server->settings->is_server_api_enabled === true) {
|
||||
$this->server->checkServerApi();
|
||||
$this->dispatch('success', 'Server API is reachable.');
|
||||
}
|
||||
} catch (\Throwable $e) {
|
||||
return handleError($e, $this);
|
||||
public function updatedServerSettingsIsSentinelEnabled($value){
|
||||
if($value === false){
|
||||
StopSentinel::dispatch($this->server);
|
||||
$this->server->settings->is_metrics_enabled = false;
|
||||
$this->server->settings->save();
|
||||
$this->server->sentinelUpdateAt(isReset: true);
|
||||
} else {
|
||||
StartSentinel::run($this->server);
|
||||
}
|
||||
}
|
||||
|
||||
public function updatedServerSettingsIsMetricsEnabled(){
|
||||
$this->restartSentinel();
|
||||
}
|
||||
|
||||
|
||||
public function instantSave()
|
||||
{
|
||||
try {
|
||||
@@ -165,19 +176,20 @@ class Form extends Component
|
||||
$this->server->save();
|
||||
$this->dispatch('success', 'Server updated.');
|
||||
$this->dispatch('refreshServerShow');
|
||||
if ($this->server->isSentinelEnabled()) {
|
||||
PullSentinelImageJob::dispatchSync($this->server);
|
||||
ray('Sentinel is enabled');
|
||||
if ($this->server->settings->isDirty('is_metrics_enabled')) {
|
||||
$this->dispatch('reloadWindow');
|
||||
}
|
||||
if ($this->server->settings->isDirty('is_server_api_enabled') && $this->server->settings->is_server_api_enabled === true) {
|
||||
ray('Starting sentinel');
|
||||
}
|
||||
} else {
|
||||
ray('Sentinel is not enabled');
|
||||
StopSentinel::dispatch($this->server);
|
||||
}
|
||||
|
||||
// if ($this->server->isSentinelEnabled()) {
|
||||
// PullSentinelImageJob::dispatchSync($this->server);
|
||||
// ray('Sentinel is enabled');
|
||||
// if ($this->server->settings->isDirty('is_metrics_enabled')) {
|
||||
// $this->dispatch('reloadWindow');
|
||||
// }
|
||||
// if ($this->server->settings->isDirty('is_sentinel_enabled') && $this->server->settings->is_sentinel_enabled === true) {
|
||||
// ray('Starting sentinel');
|
||||
// }
|
||||
// } else {
|
||||
// ray('Sentinel is not enabled');
|
||||
// StopSentinel::dispatch($this->server);
|
||||
// }
|
||||
$this->server->settings->save();
|
||||
// $this->checkPortForServerApi();
|
||||
|
||||
@@ -186,35 +198,12 @@ class Form extends Component
|
||||
}
|
||||
}
|
||||
|
||||
public function getPushData()
|
||||
{
|
||||
try {
|
||||
if (! isDev()) {
|
||||
throw new \Exception('This feature is only available in dev mode.');
|
||||
}
|
||||
$response = Http::withHeaders([
|
||||
'Authorization' => 'Bearer '.$this->server->settings->sentinel_token,
|
||||
])->post('http://host.docker.internal:8888/api/push', [
|
||||
'data' => 'test',
|
||||
]);
|
||||
if ($response->successful()) {
|
||||
$this->dispatch('success', 'Push data sent.');
|
||||
|
||||
return;
|
||||
}
|
||||
$error = data_get($response->json(), 'error');
|
||||
throw new \Exception($error);
|
||||
} catch (\Throwable $e) {
|
||||
return handleError($e, $this);
|
||||
}
|
||||
}
|
||||
|
||||
public function restartSentinel()
|
||||
{
|
||||
try {
|
||||
$version = get_latest_sentinel_version();
|
||||
StartSentinel::run($this->server, $version, true);
|
||||
$this->dispatch('success', 'Sentinel restarted.');
|
||||
$this->dispatch('success', 'Sentinel started.');
|
||||
} catch (\Throwable $e) {
|
||||
return handleError($e, $this);
|
||||
}
|
||||
@@ -307,10 +296,4 @@ class Form extends Component
|
||||
$this->server->refresh();
|
||||
$this->dispatch('success', 'Cloudflare Tunnels enabled.');
|
||||
}
|
||||
|
||||
public function startSentinel()
|
||||
{
|
||||
StartSentinel::run($this->server);
|
||||
$this->dispatch('success', 'Sentinel started.');
|
||||
}
|
||||
}
|
||||
|
@@ -7,6 +7,7 @@ use App\Enums\ProxyTypes;
|
||||
use App\Jobs\PullSentinelImageJob;
|
||||
use Illuminate\Database\Eloquent\Builder;
|
||||
use Illuminate\Database\Eloquent\Casts\Attribute;
|
||||
use Illuminate\Support\Carbon;
|
||||
use Illuminate\Support\Collection;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
use Illuminate\Support\Facades\Process;
|
||||
@@ -16,6 +17,7 @@ use OpenApi\Attributes as OA;
|
||||
use Spatie\SchemalessAttributes\Casts\SchemalessAttributes;
|
||||
use Spatie\SchemalessAttributes\SchemalessAttributesTrait;
|
||||
use Spatie\Url\Url;
|
||||
use Illuminate\Support\Str;
|
||||
use Symfony\Component\Yaml\Yaml;
|
||||
|
||||
#[OA\Schema(
|
||||
@@ -538,6 +540,16 @@ $schema://$host {
|
||||
return $encrypted;
|
||||
}
|
||||
|
||||
public function sentinelUpdateAt(bool $isReset = false)
|
||||
{
|
||||
$this->sentinel_updated_at = $isReset ? now()->subMinutes(6000) : now();
|
||||
$this->save();
|
||||
}
|
||||
public function isSentinelLive()
|
||||
{
|
||||
return Carbon::parse($this->sentinel_updated_at)->isAfter(now()->subMinutes(4));
|
||||
}
|
||||
|
||||
public function isSentinelEnabled()
|
||||
{
|
||||
return $this->isMetricsEnabled() || $this->isServerApiEnabled();
|
||||
@@ -550,7 +562,7 @@ $schema://$host {
|
||||
|
||||
public function isServerApiEnabled()
|
||||
{
|
||||
return $this->settings->is_server_api_enabled;
|
||||
return $this->settings->is_sentinel_enabled;
|
||||
}
|
||||
|
||||
public function checkServerApi()
|
||||
@@ -591,7 +603,15 @@ $schema://$host {
|
||||
{
|
||||
if ($this->isMetricsEnabled()) {
|
||||
$from = now()->subMinutes($mins)->toIso8601ZuluString();
|
||||
if (isDev() && $this->id === 0) {
|
||||
$process = Process::run("curl -H \"Authorization: Bearer {$this->settings->sentinel_token}\" http://host.docker.internal:8888/api/cpu/history?from=$from");
|
||||
if ($process->failed()) {
|
||||
throw new \Exception($process->errorOutput());
|
||||
}
|
||||
$cpu = $process->output();
|
||||
} else {
|
||||
$cpu = instant_remote_process(["docker exec coolify-sentinel sh -c 'curl -H \"Authorization: Bearer {$this->settings->sentinel_token}\" http://localhost:8888/api/cpu/history?from=$from'"], $this, false);
|
||||
}
|
||||
if (str($cpu)->contains('error')) {
|
||||
$error = json_decode($cpu, true);
|
||||
$error = data_get($error, 'error', 'Something is not okay, are you okay?');
|
||||
@@ -600,17 +620,12 @@ $schema://$host {
|
||||
}
|
||||
throw new \Exception($error);
|
||||
}
|
||||
$cpu = str($cpu)->explode("\n")->skip(1)->all();
|
||||
$parsedCollection = collect($cpu)->flatMap(function ($item) {
|
||||
return collect(explode("\n", trim($item)))->map(function ($line) {
|
||||
[$time, $cpu_usage_percent] = explode(',', trim($line));
|
||||
$cpu_usage_percent = number_format($cpu_usage_percent, 0);
|
||||
|
||||
return [(int) $time, (float) $cpu_usage_percent];
|
||||
});
|
||||
$cpu = json_decode($cpu, true);
|
||||
$parsedCollection = collect($cpu)->map(function ($metric) {
|
||||
return [(int)$metric['time'], (float)$metric['percent']];
|
||||
});
|
||||
return $parsedCollection;
|
||||
|
||||
return $parsedCollection->toArray();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -618,7 +633,15 @@ $schema://$host {
|
||||
{
|
||||
if ($this->isMetricsEnabled()) {
|
||||
$from = now()->subMinutes($mins)->toIso8601ZuluString();
|
||||
if (isDev() && $this->id === 0) {
|
||||
$process = Process::run("curl -H \"Authorization: Bearer {$this->settings->sentinel_token}\" http://host.docker.internal:8888/api/memory/history?from=$from");
|
||||
if ($process->failed()) {
|
||||
throw new \Exception($process->errorOutput());
|
||||
}
|
||||
$memory = $process->output();
|
||||
} else {
|
||||
$memory = instant_remote_process(["docker exec coolify-sentinel sh -c 'curl -H \"Authorization: Bearer {$this->settings->sentinel_token}\" http://localhost:8888/api/memory/history?from=$from'"], $this, false);
|
||||
}
|
||||
if (str($memory)->contains('error')) {
|
||||
$error = json_decode($memory, true);
|
||||
$error = data_get($error, 'error', 'Something is not okay, are you okay?');
|
||||
@@ -627,14 +650,9 @@ $schema://$host {
|
||||
}
|
||||
throw new \Exception($error);
|
||||
}
|
||||
$memory = str($memory)->explode("\n")->skip(1)->all();
|
||||
$parsedCollection = collect($memory)->flatMap(function ($item) {
|
||||
return collect(explode("\n", trim($item)))->map(function ($line) {
|
||||
[$time, $used, $free, $usedPercent] = explode(',', trim($line));
|
||||
$usedPercent = number_format($usedPercent, 0);
|
||||
|
||||
return [(int) $time, (float) $usedPercent];
|
||||
});
|
||||
$memory = json_decode($memory, true);
|
||||
$parsedCollection = collect($memory)->map(function ($metric) {
|
||||
return [(int)$metric['time'], (float)$metric['usedPercent']];
|
||||
});
|
||||
|
||||
return $parsedCollection->toArray();
|
||||
@@ -1054,6 +1072,37 @@ $schema://$host {
|
||||
return data_get($this, 'settings.is_swarm_worker');
|
||||
}
|
||||
|
||||
public function status(): bool
|
||||
{
|
||||
['uptime' => $uptime] = $this->validateConnection(false);
|
||||
if ($uptime) {
|
||||
if ($this->unreachable_notification_sent === true) {
|
||||
$this->update(['unreachable_notification_sent' => false]);
|
||||
}
|
||||
} else {
|
||||
// $this->server->team?->notify(new Unreachable($this->server));
|
||||
foreach ($this->applications as $application) {
|
||||
$application->update(['status' => 'exited']);
|
||||
}
|
||||
foreach ($this->databases as $database) {
|
||||
$database->update(['status' => 'exited']);
|
||||
}
|
||||
foreach ($this->services as $service) {
|
||||
$apps = $service->applications()->get();
|
||||
$dbs = $service->databases()->get();
|
||||
foreach ($apps as $app) {
|
||||
$app->update(['status' => 'exited']);
|
||||
}
|
||||
foreach ($dbs as $db) {
|
||||
$db->update(['status' => 'exited']);
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
public function validateConnection($isManualCheck = true)
|
||||
{
|
||||
config()->set('constants.ssh.mux_enabled', ! $isManualCheck);
|
||||
|
@@ -24,7 +24,7 @@ use OpenApi\Attributes as OA;
|
||||
'is_logdrain_newrelic_enabled' => ['type' => 'boolean'],
|
||||
'is_metrics_enabled' => ['type' => 'boolean'],
|
||||
'is_reachable' => ['type' => 'boolean'],
|
||||
'is_server_api_enabled' => ['type' => 'boolean'],
|
||||
'is_sentinel_enabled' => ['type' => 'boolean'],
|
||||
'is_swarm_manager' => ['type' => 'boolean'],
|
||||
'is_swarm_worker' => ['type' => 'boolean'],
|
||||
'is_usable' => ['type' => 'boolean'],
|
||||
@@ -55,7 +55,6 @@ class ServerSetting extends Model
|
||||
'docker_cleanup_threshold' => 'integer',
|
||||
'sentinel_token' => 'encrypted',
|
||||
];
|
||||
|
||||
public function server()
|
||||
{
|
||||
return $this->belongsTo(Server::class);
|
||||
|
@@ -164,7 +164,7 @@ function get_route_parameters(): array
|
||||
function get_latest_sentinel_version(): string
|
||||
{
|
||||
try {
|
||||
$response = Http::get('https://cdn.coollabs.io/sentinel/versions.json');
|
||||
$response = Http::get('https://cdn.coollabs.io/coolify/versions.json');
|
||||
$versions = $response->json();
|
||||
|
||||
return data_get($versions, 'sentinel.version');
|
||||
|
@@ -12,7 +12,7 @@ return new class extends Migration
|
||||
public function up(): void
|
||||
{
|
||||
Schema::table('server_settings', function (Blueprint $table) {
|
||||
$table->boolean('is_force_cleanup_enabled')->default(false)->after('is_sentinel_enabled');
|
||||
$table->boolean('is_force_cleanup_enabled')->default(false);
|
||||
});
|
||||
}
|
||||
|
||||
|
@@ -15,12 +15,16 @@ return new class extends Migration
|
||||
$table->dropColumn('metrics_token');
|
||||
$table->dropColumn('metrics_refresh_rate_seconds');
|
||||
$table->dropColumn('metrics_history_days');
|
||||
$table->dropColumn('is_server_api_enabled');
|
||||
|
||||
$table->boolean('is_sentinel_enabled')->default(true);
|
||||
$table->text('sentinel_token')->nullable();
|
||||
$table->integer('sentinel_metrics_refresh_rate_seconds')->default(5);
|
||||
$table->integer('sentinel_metrics_history_days')->default(30);
|
||||
$table->integer('sentinel_metrics_refresh_rate_seconds')->default(10);
|
||||
$table->integer('sentinel_metrics_history_days')->default(7);
|
||||
$table->integer('sentinel_push_interval_seconds')->default(60);
|
||||
});
|
||||
Schema::table('servers', function (Blueprint $table) {
|
||||
$table->dateTime('sentinel_update_at')->default(now());
|
||||
$table->dateTime('sentinel_updated_at')->default(now());
|
||||
});
|
||||
}
|
||||
|
||||
@@ -33,12 +37,16 @@ return new class extends Migration
|
||||
$table->string('metrics_token')->nullable();
|
||||
$table->integer('metrics_refresh_rate_seconds')->default(5);
|
||||
$table->integer('metrics_history_days')->default(30);
|
||||
$table->boolean('is_server_api_enabled')->default(false);
|
||||
|
||||
$table->dropColumn('sentinel_token');
|
||||
$table->dropColumn('sentinel_metrics_refresh_rate_seconds');
|
||||
$table->dropColumn('sentinel_metrics_history_days');
|
||||
$table->dropColumn('sentinel_push_interval_seconds');
|
||||
$table->dropColumn('is_sentinel_enabled');
|
||||
});
|
||||
Schema::table('servers', function (Blueprint $table) {
|
||||
$table->dropColumn('sentinel_update_at');
|
||||
$table->dropColumn('sentinel_updated_at');
|
||||
});
|
||||
}
|
||||
};
|
||||
|
@@ -26,6 +26,7 @@ class DatabaseSeeder extends Seeder
|
||||
S3StorageSeeder::class,
|
||||
StandalonePostgresqlSeeder::class,
|
||||
OauthSettingSeeder::class,
|
||||
GenerateSentinelTokenSeeder::class,
|
||||
]);
|
||||
}
|
||||
}
|
||||
|
25
database/seeders/GenerateSentinelTokenSeeder.php
Normal file
25
database/seeders/GenerateSentinelTokenSeeder.php
Normal file
@@ -0,0 +1,25 @@
|
||||
<?php
|
||||
|
||||
namespace Database\Seeders;
|
||||
|
||||
use App\Models\Server;
|
||||
use Illuminate\Database\Seeder;
|
||||
|
||||
class GenerateSentinelTokenSeeder extends Seeder
|
||||
{
|
||||
public function run()
|
||||
{
|
||||
try {
|
||||
Server::chunk(100, function ($servers) {
|
||||
foreach ($servers as $server) {
|
||||
if (str($server->settings->sentinel_token)->isEmpty()) {
|
||||
$server->generateSentinelToken();
|
||||
}
|
||||
}
|
||||
});
|
||||
} catch (\Throwable $e) {
|
||||
echo "Error: {$e->getMessage()}\n";
|
||||
ray($e->getMessage());
|
||||
}
|
||||
}
|
||||
}
|
@@ -186,6 +186,7 @@ uZx9iFkCELtxrh31QJ68AAAAEXNhaWxANzZmZjY2ZDJlMmRkAQIDBA==
|
||||
|
||||
$this->call(OauthSettingSeeder::class);
|
||||
$this->call(PopulateSshKeysDirectorySeeder::class);
|
||||
$this->call(GenerateSentinelTokenSeeder::class);
|
||||
|
||||
}
|
||||
}
|
||||
|
@@ -4959,7 +4959,7 @@ components:
|
||||
type: boolean
|
||||
is_reachable:
|
||||
type: boolean
|
||||
is_server_api_enabled:
|
||||
is_sentinel_enabled:
|
||||
type: boolean
|
||||
is_swarm_manager:
|
||||
type: boolean
|
||||
|
@@ -14,7 +14,10 @@
|
||||
'w-full' => $fullWidth,
|
||||
])>
|
||||
@if (!$hideLabel)
|
||||
<label class="flex gap-4 px-0 min-w-fit label">
|
||||
<label @class([
|
||||
"flex gap-4 px-0 min-w-fit label",
|
||||
'opacity-40' => $disabled,
|
||||
])>
|
||||
<span class="flex gap-2">
|
||||
@if ($label)
|
||||
{!! $label !!}
|
||||
|
@@ -277,21 +277,37 @@
|
||||
</div>
|
||||
<div class="flex gap-2 items-center pt-4 pb-2">
|
||||
<h3>Sentinel</h3>
|
||||
{{-- @if ($server->isSentinelEnabled()) --}}
|
||||
{{-- <x-forms.button wire:click='restartSentinel'>Restart</x-forms.button> --}}
|
||||
{{-- @endif --}}
|
||||
@if ($server->isSentinelEnabled())
|
||||
<div class="flex gap-2 items-center"
|
||||
wire:poll.{{ $server->settings->sentinel_push_interval_seconds }}s="checkSyncStatus">
|
||||
@if ($server->isSentinelLive())
|
||||
<x-status.running status="In-sync" noLoading />
|
||||
@else
|
||||
<x-status.stopped status="Out-of-sync" noLoading />
|
||||
@endif
|
||||
<x-forms.button wire:click='restartSentinel'>Restart</x-forms.button>
|
||||
</div>
|
||||
@endif
|
||||
</div>
|
||||
@if (isDev())
|
||||
<div class="w-64">
|
||||
<x-forms.checkbox instantSave id="server.settings.is_metrics_enabled" label="Enable Metrics" />
|
||||
<x-forms.button wire:click="startSentinel">Start Sentinel</x-forms.button>
|
||||
</div>
|
||||
<div class="flex flex-col gap-2">
|
||||
<div class="w-64">
|
||||
<x-forms.checkbox instantSave id="server.settings.is_sentinel_enabled"
|
||||
label="Enable Sentinel" />
|
||||
@if ($server->isSentinelEnabled())
|
||||
<x-forms.checkbox instantSave id="server.settings.is_metrics_enabled"
|
||||
label="Enable Metrics" />
|
||||
@else
|
||||
<x-forms.checkbox instantSave disabled id="server.settings.is_metrics_enabled"
|
||||
label="Enable Metrics" />
|
||||
@endif
|
||||
</div>
|
||||
<div class="flex flex-wrap gap-2 sm:flex-nowrap items-end">
|
||||
<x-forms.input type="password" id="server.settings.sentinel_token" label="Metrics token"
|
||||
required helper="Token for collector (Sentinel)." />
|
||||
<x-forms.input type="password" id="server.settings.sentinel_token" label="Sentinel token"
|
||||
required helper="Token for Sentinel." />
|
||||
<x-forms.button wire:click="regenerateSentinelToken">Regenerate</x-forms.button>
|
||||
</div>
|
||||
<div class="flex flex-col gap-2">
|
||||
<div class="flex flex-wrap gap-2 sm:flex-nowrap">
|
||||
<x-forms.input id="server.settings.sentinel_metrics_refresh_rate_seconds"
|
||||
label="Metrics rate (seconds)" required
|
||||
@@ -299,6 +315,10 @@
|
||||
<x-forms.input id="server.settings.sentinel_metrics_history_days"
|
||||
label="Metrics history (days)" required
|
||||
helper="How many days should the metrics data should be reserved." />
|
||||
<x-forms.input id="server.settings.sentinel_push_interval_seconds"
|
||||
label="Push interval (seconds)" required
|
||||
helper="How many seconds should the metrics data should be pushed to the collector." />
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@else
|
||||
|
@@ -147,10 +147,14 @@ Route::group([
|
||||
if (! $server) {
|
||||
return response()->json(['message' => 'Server not found'], 404);
|
||||
}
|
||||
if ($server->settings->sentinel_token !== $naked_token) {
|
||||
logger('Unauthorized');
|
||||
return response()->json(['message' => 'Unauthorized'], 401);
|
||||
}
|
||||
$data = request()->all();
|
||||
$server->update(['sentinel_update_at' => now()]);
|
||||
PushServerUpdateJob::dispatch($server, $data);
|
||||
|
||||
PushServerUpdateJob::dispatch($server, $data);
|
||||
logger('hello');
|
||||
return response()->json(['message' => 'ok'], 200);
|
||||
});
|
||||
});
|
||||
|
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"coolify": {
|
||||
"v4": {
|
||||
"version": "4.0.0-beta.361"
|
||||
"version": "4.0.0-beta.360"
|
||||
},
|
||||
"nightly": {
|
||||
"version": "4.0.0-beta.362"
|
||||
@@ -11,6 +11,9 @@
|
||||
},
|
||||
"realtime": {
|
||||
"version": "1.0.3"
|
||||
},
|
||||
"sentinel": {
|
||||
"version": "next"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
Reference in New Issue
Block a user