sentinel updates

app/Actions/Server/StartSentinel.php

@@ -17,39 +17,46 @@ class StartSentinel
         }
         $metrics_history = $server->settings->sentinel_metrics_history_days;
         $refresh_rate = $server->settings->sentinel_metrics_refresh_rate_seconds;
+        $push_interval = $server->settings->sentinel_push_interval_seconds;
         $token = $server->settings->sentinel_token;
         $endpoint = InstanceSettings::get()->fqdn;
-        if (isDev()) {
+        $mount_dir = '/data/coolify/sentinel';
+        $image = "ghcr.io/coollabsio/sentinel:$version";
+
+        if ($server->isLocalhost()) {
             $endpoint = 'http://host.docker.internal:8000';
-        }
+        } else {
             if (! $endpoint) {
                 throw new \Exception('You should set FQDN in Instance Settings.');
             }
-        // Ensure the endpoint is using HTTPS
+        }
-        $endpoint = str($endpoint)->replace('http://', 'https://')->value();
         $environments = [
             'TOKEN' => $token,
-            'ENDPOINT' => $endpoint,
+            'PUSH_ENDPOINT' => $endpoint,
-            'COLLECTOR_ENABLED' => 'true',
+            'PUSH_INTERVAL_SECONDS' => $push_interval,
+            'COLLECTOR_ENABLED' => $server->isMetricsEnabled() ? 'true' : 'false',
             'COLLECTOR_REFRESH_RATE_SECONDS' => $refresh_rate,
             'COLLECTOR_RETENTION_PERIOD_DAYS' => $metrics_history,
         ];
         if (isDev()) {
-            data_set($environments, 'GIN_MODE', 'debug');
+            data_set($environments, 'DEBUG', 'true');
-        }
-        $mount_dir = '/data/coolify/sentinel';
-        if (isDev()) {
             $mount_dir = '/var/lib/docker/volumes/coolify_dev_coolify_data/_data/sentinel';
+            $image = 'sentinel';
         }
         $docker_environments = '-e "' . implode('" -e "', array_map(fn($key, $value) => "$key=$value", array_keys($environments), $environments)) . '"';
-        $docker_command = "docker run --pull always --rm -d $docker_environments --name coolify-sentinel -v /var/run/docker.sock:/var/run/docker.sock -v $mount_dir:/app/db --pid host --health-cmd \"curl --fail http://127.0.0.1:8888/api/health || exit 1\" --health-interval 10s --health-retries 3 --add-host=host.docker.internal:host-gateway ghcr.io/coollabsio/sentinel:$version";
 
-        return instant_remote_process([
+        $docker_command = "docker run -d $docker_environments --name coolify-sentinel -v /var/run/docker.sock:/var/run/docker.sock -v $mount_dir:/app/db --pid host --health-cmd \"curl --fail http://127.0.0.1:8888/api/health || exit 1\" --health-interval 10s --health-retries 3 --add-host=host.docker.internal:host-gateway $image";
+
+        instant_remote_process([
             'docker rm -f coolify-sentinel || true',
             "mkdir -p $mount_dir",
             $docker_command,
             "chown -R 9999:root $mount_dir",
             "chmod -R 700 $mount_dir",
-        ], $server, true);
+        ], $server);
+
+        $server->settings->is_sentinel_enabled = true;
+        $server->settings->save();
+        $server->sentinelUpdateAt();
     }
 }

app/Actions/Server/StopSentinel.php

@@ -3,6 +3,7 @@
 namespace App\Actions\Server;
 
 use App\Models\Server;
+use Carbon\Carbon;
 use Lorisleiva\Actions\Concerns\AsAction;
 
 class StopSentinel
@@ -12,5 +13,6 @@ class StopSentinel
     public function handle(Server $server)
     {
         instant_remote_process(['docker rm -f coolify-sentinel'], $server, false);
+        $server->sentinelUpdateAt(isReset: true);
     }
 }

app/Console/Kernel.php

@@ -12,7 +12,6 @@ use App\Jobs\PullSentinelImageJob;
 use App\Jobs\PullTemplatesFromCDN;
 use App\Jobs\ScheduledTaskJob;
 use App\Jobs\ServerCheckJob;
-use App\Jobs\ServerStorageCheckJob;
 use App\Jobs\UpdateCoolifyJob;
 use App\Models\ScheduledDatabaseBackup;
 use App\Models\ScheduledTask;
@@ -20,6 +19,7 @@ use App\Models\Server;
 use App\Models\Team;
 use Illuminate\Console\Scheduling\Schedule;
 use Illuminate\Foundation\Console\Kernel as ConsoleKernel;
+use Illuminate\Support\Carbon;
 
 class Kernel extends ConsoleKernel
 {
@@ -38,7 +38,7 @@ class Kernel extends ConsoleKernel
             $schedule->job(new CleanupInstanceStuffsJob)->everyMinute()->onOneServer();
             // Server Jobs
             $this->check_scheduled_backups($schedule);
-            // $this->check_resources($schedule);
+            $this->check_resources($schedule);
             $this->check_scheduled_tasks($schedule);
             $schedule->command('uploads:clear')->everyTwoMinutes();
 
@@ -115,7 +115,10 @@ class Kernel extends ConsoleKernel
             $servers = $this->all_servers->where('ip', '!=', '1.2.3.4');
         }
         foreach ($servers as $server) {
+            $last_sentinel_update = $server->sentinel_updated_at;
+            if (Carbon::parse($last_sentinel_update)->isBefore(now()->subMinutes(4))) {
                 $schedule->job(new ServerCheckJob($server))->everyMinute()->onOneServer();
+            }
             // $schedule->job(new ServerStorageCheckJob($server))->everyMinute()->onOneServer();
             $serverTimezone = $server->settings->server_timezone;
             if ($server->settings->force_docker_cleanup) {

app/Jobs/PushServerUpdateJob.php

@@ -4,7 +4,9 @@ namespace App\Jobs;
 
 use App\Actions\Database\StartDatabaseProxy;
 use App\Actions\Database\StopDatabaseProxy;
+use App\Actions\Proxy\CheckProxy;
 use App\Actions\Proxy\StartProxy;
+use App\Actions\Server\InstallLogDrain;
 use App\Actions\Shared\ComplexStatusCheck;
 use App\Models\Application;
 use App\Models\ApplicationPreview;
@@ -40,6 +42,8 @@ class PushServerUpdateJob implements ShouldQueue
 
     public Collection $allDatabaseUuids;
 
+    public Collection $allTcpProxyUuids;
+
     public Collection $allServiceApplicationIds;
 
     public Collection $allApplicationPreviewsIds;
@@ -59,6 +63,7 @@ class PushServerUpdateJob implements ShouldQueue
     public Collection $foundApplicationPreviewsIds;
 
     public bool $foundProxy = false;
+    public bool $foundLogDrainContainer = false;
 
     public function backoff(): int
     {
@@ -87,6 +92,11 @@ class PushServerUpdateJob implements ShouldQueue
                 throw new \Exception('No data provided');
             }
             $data = collect($this->data);
+
+            $this->serverStatus();
+
+            $this->server->sentinelUpdateAt();
+
             $this->containers = collect(data_get($data, 'containers'));
             if ($this->containers->isEmpty()) {
                 return;
@@ -122,6 +132,10 @@ class PushServerUpdateJob implements ShouldQueue
                 $labels = collect(data_get($container, 'labels'));
                 $coolify_managed = $labels->has('coolify.managed');
                 if ($coolify_managed) {
+                    $name = data_get($container, 'name');
+                    if ($name === 'coolify-log-drain' && $this->isRunning($containerStatus)) {
+                        $this->foundLogDrainContainer = true;
+                    }
                     if ($labels->has('coolify.applicationId')) {
                         $applicationId = $labels->get('coolify.applicationId');
                         $pullRequestId = data_get($labels, 'coolify.pullRequestId', '0');
@@ -153,7 +167,6 @@ class PushServerUpdateJob implements ShouldQueue
                         }
 
                     } else {
-                        $name = data_get($container, 'name');
                         $uuid = $labels->get('com.docker.compose.service');
                         $type = $labels->get('coolify.type');
                         if ($name === 'coolify-proxy' && $this->isRunning($containerStatus)) {
@@ -182,12 +195,23 @@ class PushServerUpdateJob implements ShouldQueue
             $this->updateNotFoundServiceStatus();
 
             $this->updateAdditionalServersStatus();
+
+            $this->checkLogDrainContainer();
+
         } catch (\Exception $e) {
             throw $e;
         }
 
     }
 
+    private function serverStatus(){
+        if ($this->server->isFunctional() === false) {
+            throw new \Exception('Server is not ready.');
+        }
+        if ($this->server->status() === false) {
+            throw new \Exception('Server is not reachable.');
+        }
+    }
     private function updateApplicationStatus(string $applicationId, string $containerStatus)
     {
         $application = $this->applications->where('id', $applicationId)->first();
@@ -247,9 +271,19 @@ class PushServerUpdateJob implements ShouldQueue
     private function updateProxyStatus()
     {
         // If proxy is not found, start it
-        if (! $this->foundProxy && $this->server->isProxyShouldRun()) {
+        if ($this->server->isProxyShouldRun()) {
-            ray('Proxy not found, starting it.');
+            if ($this->foundProxy === false) {
-            StartProxy::dispatch($this->server);
+                try {
+                    if (CheckProxy::run($this->server)) {
+                        StartProxy::run($this->server, false);
+                    }
+                } catch (\Throwable $e) {
+                    logger()->error($e);
+                }
+            } else {
+                $connectProxyToDockerNetworks = connectProxyToNetworks($this->server);
+                instant_remote_process($connectProxyToDockerNetworks, $this->server, false);
+            }
         }
 
     }
@@ -361,4 +395,10 @@ class PushServerUpdateJob implements ShouldQueue
     {
         return str($containerStatus)->contains('running');
     }
+
+    private function checkLogDrainContainer(){
+        if ($this->server->isLogDrainEnabled() && $this->foundLogDrainContainer === false) {
+            InstallLogDrain::dispatch($this->server);
+        }
+    }
 }

app/Livewire/Server/Charts.php

@@ -34,12 +34,12 @@ class Charts extends Component
         try {
             $cpuMetrics = $this->server->getCpuMetrics($this->interval);
             $memoryMetrics = $this->server->getMemoryMetrics($this->interval);
-            $cpuMetrics = collect($cpuMetrics)->map(function ($metric) {
+            // $cpuMetrics = collect($cpuMetrics)->map(function ($metric) {
-                return [$metric[0], $metric[1]];
+            //     return [$metric[0], $metric[1]];
-            });
+            // });
-            $memoryMetrics = collect($memoryMetrics)->map(function ($metric) {
+            // $memoryMetrics = collect($memoryMetrics)->map(function ($metric) {
-                return [$metric[0], $metric[1]];
+            //     return [$metric[0], $metric[1]];
-            });
+            // });
             $this->dispatch("refreshChartData-{$this->chartId}-cpu", [
                 'seriesData' => $cpuMetrics,
             ]);

app/Livewire/Server/Form.php

@@ -58,8 +58,9 @@ class Form extends Component
         'server.settings.sentinel_token' => 'required',
         'server.settings.sentinel_metrics_refresh_rate_seconds' => 'required|integer|min:1',
         'server.settings.sentinel_metrics_history_days' => 'required|integer|min:1',
+        'server.settings.sentinel_push_interval_seconds' => 'required|integer|min:10',
         'wildcard_domain' => 'nullable|url',
-        'server.settings.is_server_api_enabled' => 'required|boolean',
+        'server.settings.is_sentinel_enabled' => 'required|boolean',
         'server.settings.server_timezone' => 'required|string|timezone',
         'server.settings.force_docker_cleanup' => 'required|boolean',
         'server.settings.docker_cleanup_frequency' => 'required_if:server.settings.force_docker_cleanup,true|string',
@@ -85,7 +86,8 @@ class Form extends Component
         'server.settings.sentinel_token' => 'Metrics Token',
         'server.settings.sentinel_metrics_refresh_rate_seconds' => 'Metrics Interval',
         'server.settings.sentinel_metrics_history_days' => 'Metrics History',
-        'server.settings.is_server_api_enabled' => 'Server API',
+        'server.settings.sentinel_push_interval_seconds' => 'Push Interval',
+        'server.settings.is_sentinel_enabled' => 'Server API',
         'server.settings.server_timezone' => 'Server Timezone',
         'server.settings.delete_unused_volumes' => 'Delete Unused Volumes',
         'server.settings.delete_unused_networks' => 'Delete Unused Networks',
@@ -102,12 +104,17 @@ class Form extends Component
         $this->server->settings->delete_unused_networks = $server->settings->delete_unused_networks;
     }
 
+    public function checkSyncStatus(){
+        $this->server->refresh();
+        $this->server->settings->refresh();
+    }
+
     public function regenerateSentinelToken()
     {
         try {
             $this->server->generateSentinelToken();
             $this->server->settings->refresh();
-            $this->dispatch('success', 'Metrics token regenerated.');
+            $this->dispatch('success', 'Sentinel token regenerated. Please restart your Sentinel.');
         } catch (\Throwable $e) {
             return handleError($e, $this);
         }
@@ -143,18 +150,22 @@ class Form extends Component
         $this->dispatch('proxyStatusUpdated');
     }
 
-    public function checkPortForServerApi()
+    public function updatedServerSettingsIsSentinelEnabled($value){
-    {
+        if($value === false){
-        try {
+            StopSentinel::dispatch($this->server);
-            if ($this->server->settings->is_server_api_enabled === true) {
+            $this->server->settings->is_metrics_enabled = false;
-                $this->server->checkServerApi();
+            $this->server->settings->save();
-                $this->dispatch('success', 'Server API is reachable.');
+            $this->server->sentinelUpdateAt(isReset: true);
-            }
+        } else {
-        } catch (\Throwable $e) {
+            StartSentinel::run($this->server);
-            return handleError($e, $this);
         }
     }
 
+    public function updatedServerSettingsIsMetricsEnabled(){
+        $this->restartSentinel();
+    }
+
+
     public function instantSave()
     {
         try {
@@ -165,19 +176,20 @@ class Form extends Component
             $this->server->save();
             $this->dispatch('success', 'Server updated.');
             $this->dispatch('refreshServerShow');
-            if ($this->server->isSentinelEnabled()) {
+
-                PullSentinelImageJob::dispatchSync($this->server);
+            // if ($this->server->isSentinelEnabled()) {
-                ray('Sentinel is enabled');
+            //     PullSentinelImageJob::dispatchSync($this->server);
-                if ($this->server->settings->isDirty('is_metrics_enabled')) {
+            //     ray('Sentinel is enabled');
-                    $this->dispatch('reloadWindow');
+            //     if ($this->server->settings->isDirty('is_metrics_enabled')) {
-                }
+            //         $this->dispatch('reloadWindow');
-                if ($this->server->settings->isDirty('is_server_api_enabled') && $this->server->settings->is_server_api_enabled === true) {
+            //     }
-                    ray('Starting sentinel');
+            //     if ($this->server->settings->isDirty('is_sentinel_enabled') && $this->server->settings->is_sentinel_enabled === true) {
-                }
+            //         ray('Starting sentinel');
-            } else {
+            //     }
-                ray('Sentinel is not enabled');
+            // } else {
-                StopSentinel::dispatch($this->server);
+            //     ray('Sentinel is not enabled');
-            }
+            //     StopSentinel::dispatch($this->server);
+            // }
             $this->server->settings->save();
             // $this->checkPortForServerApi();
 
@@ -186,35 +198,12 @@ class Form extends Component
         }
     }
 
-    public function getPushData()
-    {
-        try {
-            if (! isDev()) {
-                throw new \Exception('This feature is only available in dev mode.');
-            }
-            $response = Http::withHeaders([
-                'Authorization' => 'Bearer '.$this->server->settings->sentinel_token,
-            ])->post('http://host.docker.internal:8888/api/push', [
-                'data' => 'test',
-            ]);
-            if ($response->successful()) {
-                $this->dispatch('success', 'Push data sent.');
-
-                return;
-            }
-            $error = data_get($response->json(), 'error');
-            throw new \Exception($error);
-        } catch (\Throwable $e) {
-            return handleError($e, $this);
-        }
-    }
-
     public function restartSentinel()
     {
         try {
             $version = get_latest_sentinel_version();
             StartSentinel::run($this->server, $version, true);
-            $this->dispatch('success', 'Sentinel restarted.');
+            $this->dispatch('success', 'Sentinel started.');
         } catch (\Throwable $e) {
             return handleError($e, $this);
         }
@@ -307,10 +296,4 @@ class Form extends Component
         $this->server->refresh();
         $this->dispatch('success', 'Cloudflare Tunnels enabled.');
     }
-
-    public function startSentinel()
-    {
-        StartSentinel::run($this->server);
-        $this->dispatch('success', 'Sentinel started.');
-    }
 }

app/Models/Server.php

@@ -7,6 +7,7 @@ use App\Enums\ProxyTypes;
 use App\Jobs\PullSentinelImageJob;
 use Illuminate\Database\Eloquent\Builder;
 use Illuminate\Database\Eloquent\Casts\Attribute;
+use Illuminate\Support\Carbon;
 use Illuminate\Support\Collection;
 use Illuminate\Support\Facades\DB;
 use Illuminate\Support\Facades\Process;
@@ -16,6 +17,7 @@ use OpenApi\Attributes as OA;
 use Spatie\SchemalessAttributes\Casts\SchemalessAttributes;
 use Spatie\SchemalessAttributes\SchemalessAttributesTrait;
 use Spatie\Url\Url;
+use Illuminate\Support\Str;
 use Symfony\Component\Yaml\Yaml;
 
 #[OA\Schema(
@@ -538,6 +540,16 @@ $schema://$host {
         return $encrypted;
     }
 
+    public function sentinelUpdateAt(bool $isReset = false)
+    {
+        $this->sentinel_updated_at = $isReset ? now()->subMinutes(6000) :  now();
+        $this->save();
+    }
+    public function isSentinelLive()
+    {
+        return Carbon::parse($this->sentinel_updated_at)->isAfter(now()->subMinutes(4));
+    }
+
     public function isSentinelEnabled()
     {
         return $this->isMetricsEnabled() || $this->isServerApiEnabled();
@@ -550,7 +562,7 @@ $schema://$host {
 
     public function isServerApiEnabled()
     {
-        return $this->settings->is_server_api_enabled;
+        return $this->settings->is_sentinel_enabled;
     }
 
     public function checkServerApi()
@@ -591,7 +603,15 @@ $schema://$host {
     {
         if ($this->isMetricsEnabled()) {
             $from = now()->subMinutes($mins)->toIso8601ZuluString();
+            if (isDev() && $this->id === 0) {
+                $process = Process::run("curl -H \"Authorization: Bearer {$this->settings->sentinel_token}\" http://host.docker.internal:8888/api/cpu/history?from=$from");
+                if ($process->failed()) {
+                    throw new \Exception($process->errorOutput());
+                }
+                $cpu = $process->output();
+            } else {
                 $cpu = instant_remote_process(["docker exec coolify-sentinel sh -c 'curl -H \"Authorization: Bearer {$this->settings->sentinel_token}\" http://localhost:8888/api/cpu/history?from=$from'"], $this, false);
+            }
             if (str($cpu)->contains('error')) {
                 $error = json_decode($cpu, true);
                 $error = data_get($error, 'error', 'Something is not okay, are you okay?');
@@ -600,17 +620,12 @@ $schema://$host {
                 }
                 throw new \Exception($error);
             }
-            $cpu = str($cpu)->explode("\n")->skip(1)->all();
+            $cpu = json_decode($cpu, true);
-            $parsedCollection = collect($cpu)->flatMap(function ($item) {
+            $parsedCollection = collect($cpu)->map(function ($metric) {
-                return collect(explode("\n", trim($item)))->map(function ($line) {
+                return [(int)$metric['time'], (float)$metric['percent']];
-                    [$time, $cpu_usage_percent] = explode(',', trim($line));
-                    $cpu_usage_percent = number_format($cpu_usage_percent, 0);
-
-                    return [(int) $time, (float) $cpu_usage_percent];
-                });
             });
+            return $parsedCollection;
 
-            return $parsedCollection->toArray();
         }
     }
 
@@ -618,7 +633,15 @@ $schema://$host {
     {
         if ($this->isMetricsEnabled()) {
             $from = now()->subMinutes($mins)->toIso8601ZuluString();
+            if (isDev() && $this->id === 0) {
+                $process = Process::run("curl -H \"Authorization: Bearer {$this->settings->sentinel_token}\" http://host.docker.internal:8888/api/memory/history?from=$from");
+                if ($process->failed()) {
+                    throw new \Exception($process->errorOutput());
+                }
+                $memory = $process->output();
+            } else {
                 $memory = instant_remote_process(["docker exec coolify-sentinel sh -c 'curl -H \"Authorization: Bearer {$this->settings->sentinel_token}\" http://localhost:8888/api/memory/history?from=$from'"], $this, false);
+            }
             if (str($memory)->contains('error')) {
                 $error = json_decode($memory, true);
                 $error = data_get($error, 'error', 'Something is not okay, are you okay?');
@@ -627,14 +650,9 @@ $schema://$host {
                 }
                 throw new \Exception($error);
             }
-            $memory = str($memory)->explode("\n")->skip(1)->all();
+            $memory = json_decode($memory, true);
-            $parsedCollection = collect($memory)->flatMap(function ($item) {
+           $parsedCollection = collect($memory)->map(function ($metric) {
-                return collect(explode("\n", trim($item)))->map(function ($line) {
+                return [(int)$metric['time'], (float)$metric['usedPercent']];
-                    [$time, $used, $free, $usedPercent] = explode(',', trim($line));
-                    $usedPercent = number_format($usedPercent, 0);
-
-                    return [(int) $time, (float) $usedPercent];
-                });
             });
 
             return $parsedCollection->toArray();
@@ -1054,6 +1072,37 @@ $schema://$host {
         return data_get($this, 'settings.is_swarm_worker');
     }
 
+    public function status(): bool
+    {
+        ['uptime' => $uptime] = $this->validateConnection(false);
+        if ($uptime) {
+            if ($this->unreachable_notification_sent === true) {
+                $this->update(['unreachable_notification_sent' => false]);
+            }
+        } else {
+            // $this->server->team?->notify(new Unreachable($this->server));
+            foreach ($this->applications as $application) {
+                $application->update(['status' => 'exited']);
+            }
+            foreach ($this->databases as $database) {
+                $database->update(['status' => 'exited']);
+            }
+            foreach ($this->services as $service) {
+                $apps = $service->applications()->get();
+                $dbs = $service->databases()->get();
+                foreach ($apps as $app) {
+                    $app->update(['status' => 'exited']);
+                }
+                foreach ($dbs as $db) {
+                    $db->update(['status' => 'exited']);
+                }
+            }
+
+            return false;
+        }
+
+        return true;
+    }
     public function validateConnection($isManualCheck = true)
     {
         config()->set('constants.ssh.mux_enabled', ! $isManualCheck);

app/Models/ServerSetting.php

@@ -24,7 +24,7 @@ use OpenApi\Attributes as OA;
         'is_logdrain_newrelic_enabled' => ['type' => 'boolean'],
         'is_metrics_enabled' => ['type' => 'boolean'],
         'is_reachable' => ['type' => 'boolean'],
-        'is_server_api_enabled' => ['type' => 'boolean'],
+        'is_sentinel_enabled' => ['type' => 'boolean'],
         'is_swarm_manager' => ['type' => 'boolean'],
         'is_swarm_worker' => ['type' => 'boolean'],
         'is_usable' => ['type' => 'boolean'],
@@ -55,7 +55,6 @@ class ServerSetting extends Model
         'docker_cleanup_threshold' => 'integer',
         'sentinel_token' => 'encrypted',
     ];
-
     public function server()
     {
         return $this->belongsTo(Server::class);

bootstrap/helpers/shared.php

@@ -164,7 +164,7 @@ function get_route_parameters(): array
 function get_latest_sentinel_version(): string
 {
     try {
-        $response = Http::get('https://cdn.coollabs.io/sentinel/versions.json');
+        $response = Http::get('https://cdn.coollabs.io/coolify/versions.json');
         $versions = $response->json();
 
         return data_get($versions, 'sentinel.version');

database/migrations/2024_07_18_123458_add_force_cleanup_server.php

@@ -12,7 +12,7 @@ return new class extends Migration
     public function up(): void
     {
         Schema::table('server_settings', function (Blueprint $table) {
-            $table->boolean('is_force_cleanup_enabled')->default(false)->after('is_sentinel_enabled');
+            $table->boolean('is_force_cleanup_enabled')->default(false);
         });
     }
 

database/migrations/2024_10_14_090416_update_metrics_token_in_server_settings.php

@@ -15,12 +15,16 @@ return new class extends Migration
             $table->dropColumn('metrics_token');
             $table->dropColumn('metrics_refresh_rate_seconds');
             $table->dropColumn('metrics_history_days');
+            $table->dropColumn('is_server_api_enabled');
+
+            $table->boolean('is_sentinel_enabled')->default(true);
             $table->text('sentinel_token')->nullable();
-            $table->integer('sentinel_metrics_refresh_rate_seconds')->default(5);
+            $table->integer('sentinel_metrics_refresh_rate_seconds')->default(10);
-            $table->integer('sentinel_metrics_history_days')->default(30);
+            $table->integer('sentinel_metrics_history_days')->default(7);
+            $table->integer('sentinel_push_interval_seconds')->default(60);
         });
         Schema::table('servers', function (Blueprint $table) {
-            $table->dateTime('sentinel_update_at')->default(now());
+            $table->dateTime('sentinel_updated_at')->default(now());
         });
     }
 
@@ -33,12 +37,16 @@ return new class extends Migration
             $table->string('metrics_token')->nullable();
             $table->integer('metrics_refresh_rate_seconds')->default(5);
             $table->integer('metrics_history_days')->default(30);
+            $table->boolean('is_server_api_enabled')->default(false);
+
             $table->dropColumn('sentinel_token');
             $table->dropColumn('sentinel_metrics_refresh_rate_seconds');
             $table->dropColumn('sentinel_metrics_history_days');
+            $table->dropColumn('sentinel_push_interval_seconds');
+            $table->dropColumn('is_sentinel_enabled');
         });
         Schema::table('servers', function (Blueprint $table) {
-            $table->dropColumn('sentinel_update_at');
+            $table->dropColumn('sentinel_updated_at');
         });
     }
 };

database/seeders/DatabaseSeeder.php

@@ -26,6 +26,7 @@ class DatabaseSeeder extends Seeder
             S3StorageSeeder::class,
             StandalonePostgresqlSeeder::class,
             OauthSettingSeeder::class,
+            GenerateSentinelTokenSeeder::class,
         ]);
     }
 }

database/seeders/GenerateSentinelTokenSeeder.php

@@ -0,0 +1,25 @@
+<?php
+
+namespace Database\Seeders;
+
+use App\Models\Server;
+use Illuminate\Database\Seeder;
+
+class GenerateSentinelTokenSeeder extends Seeder
+{
+    public function run()
+    {
+        try {
+            Server::chunk(100, function ($servers) {
+                foreach ($servers as $server) {
+                    if (str($server->settings->sentinel_token)->isEmpty()) {
+                        $server->generateSentinelToken();
+                    }
+                }
+            });
+        } catch (\Throwable $e) {
+            echo "Error: {$e->getMessage()}\n";
+            ray($e->getMessage());
+        }
+    }
+}

database/seeders/ProductionSeeder.php

@@ -186,6 +186,7 @@ uZx9iFkCELtxrh31QJ68AAAAEXNhaWxANzZmZjY2ZDJlMmRkAQIDBA==
 
         $this->call(OauthSettingSeeder::class);
         $this->call(PopulateSshKeysDirectorySeeder::class);
+        $this->call(GenerateSentinelTokenSeeder::class);
 
     }
 }

openapi.yaml

@@ -4959,7 +4959,7 @@ components:
           type: boolean
         is_reachable:
           type: boolean
-        is_server_api_enabled:
+        is_sentinel_enabled:
           type: boolean
         is_swarm_manager:
           type: boolean

resources/views/components/forms/checkbox.blade.php

@@ -14,7 +14,10 @@
     'w-full' => $fullWidth,
 ])>
     @if (!$hideLabel)
-        <label class="flex gap-4 px-0 min-w-fit label">
+        <label  @class([
+            "flex gap-4 px-0 min-w-fit label",
+            'opacity-40' => $disabled,
+        ])>
             <span class="flex gap-2">
                 @if ($label)
                     {!! $label !!}

resources/views/livewire/server/form.blade.php

@@ -277,21 +277,37 @@
             </div>
             <div class="flex gap-2 items-center pt-4 pb-2">
                 <h3>Sentinel</h3>
-                {{-- @if ($server->isSentinelEnabled()) --}}
+                @if ($server->isSentinelEnabled())
-                {{-- <x-forms.button wire:click='restartSentinel'>Restart</x-forms.button> --}}
+                    <div class="flex gap-2 items-center"
-                {{-- @endif --}}
+                        wire:poll.{{ $server->settings->sentinel_push_interval_seconds }}s="checkSyncStatus">
+                        @if ($server->isSentinelLive())
+                            <x-status.running status="In-sync" noLoading />
+                        @else
+                            <x-status.stopped status="Out-of-sync" noLoading />
+                        @endif
+                        <x-forms.button wire:click='restartSentinel'>Restart</x-forms.button>
+                    </div>
+                @endif
             </div>
             @if (isDev())
-            <div class="w-64">
-                <x-forms.checkbox instantSave id="server.settings.is_metrics_enabled" label="Enable Metrics" />
-                <x-forms.button wire:click="startSentinel">Start Sentinel</x-forms.button>
-            </div>
                 <div class="flex flex-col gap-2">
+                    <div class="w-64">
+                        <x-forms.checkbox instantSave id="server.settings.is_sentinel_enabled"
+                            label="Enable Sentinel" />
+                        @if ($server->isSentinelEnabled())
+                            <x-forms.checkbox instantSave id="server.settings.is_metrics_enabled"
+                                label="Enable Metrics" />
+                        @else
+                            <x-forms.checkbox instantSave disabled id="server.settings.is_metrics_enabled"
+                                label="Enable Metrics" />
+                        @endif
+                    </div>
                     <div class="flex flex-wrap gap-2 sm:flex-nowrap items-end">
-                        <x-forms.input type="password" id="server.settings.sentinel_token" label="Metrics token"
+                        <x-forms.input type="password" id="server.settings.sentinel_token" label="Sentinel token"
-                            required helper="Token for collector (Sentinel)." />
+                            required helper="Token for Sentinel." />
                         <x-forms.button wire:click="regenerateSentinelToken">Regenerate</x-forms.button>
                     </div>
+                    <div class="flex flex-col gap-2">
                         <div class="flex flex-wrap gap-2 sm:flex-nowrap">
                             <x-forms.input id="server.settings.sentinel_metrics_refresh_rate_seconds"
                                 label="Metrics rate (seconds)" required
@@ -299,6 +315,10 @@
                             <x-forms.input id="server.settings.sentinel_metrics_history_days"
                                 label="Metrics history (days)" required
                                 helper="How many days should the metrics data should be reserved." />
+                            <x-forms.input id="server.settings.sentinel_push_interval_seconds"
+                                label="Push interval (seconds)" required
+                                helper="How many seconds should the metrics data should be pushed to the collector." />
+                        </div>
                     </div>
                 </div>
             @else

routes/api.php

@@ -147,10 +147,14 @@ Route::group([
         if (! $server) {
             return response()->json(['message' => 'Server not found'], 404);
         }
+        if ($server->settings->sentinel_token !== $naked_token) {
+            logger('Unauthorized');
+            return response()->json(['message' => 'Unauthorized'], 401);
+        }
         $data = request()->all();
-        $server->update(['sentinel_update_at' => now()]);
-        PushServerUpdateJob::dispatch($server, $data);
 
+        PushServerUpdateJob::dispatch($server, $data);
+        logger('hello');
         return response()->json(['message' => 'ok'], 200);
     });
 });

versions.json

@@ -1,7 +1,7 @@
 {
     "coolify": {
         "v4": {
-            "version": "4.0.0-beta.361"
+            "version": "4.0.0-beta.360"
         },
         "nightly": {
             "version": "4.0.0-beta.362"
@@ -11,6 +11,9 @@
         },
         "realtime": {
             "version": "1.0.3"
+        },
+        "sentinel": {
+            "version": "next"
         }
     }
 }