feat: log drainer container check

app/Actions/Server/InstallLogDrain.php

@@ -8,8 +8,17 @@ use App\Models\Server;
 class InstallLogDrain
 {
     use AsAction;
-    public function handle(Server $server, string $type)
+    public function handle(Server $server)
     {
+        if ($server->settings->is_logdrain_newrelic_enabled) {
+            $type = 'newrelic';
+        } else if ($server->settings->is_logdrain_highlight_enabled) {
+            $type = 'highlight';
+        } else if ($server->settings->is_logdrain_axiom_enabled) {
+            $type = 'axiom';
+        } else {
+            $type = 'none';
+        }
         try {
             if ($type === 'none') {
                 $command = [

app/Console/Kernel.php

@@ -2,6 +2,7 @@
 
 namespace App\Console;
 
+use App\Jobs\CheckLogDrainContainerJob;
 use App\Jobs\CleanupInstanceStuffsJob;
 use App\Jobs\DatabaseBackupJob;
 use App\Jobs\InstanceAutoUpdateJob;
@@ -61,6 +62,9 @@ class Kernel extends ConsoleKernel
         foreach ($servers as $server) {
             $schedule->job(new ServerStatusJob($server))->everyTenMinutes()->onOneServer();
             $schedule->job(new ContainerStatusJob($server))->everyMinute()->onOneServer();
+            if ($server->isLogDrainEnabled()) {
+                $schedule->job(new CheckLogDrainContainerJob($server))->everyMinute()->onOneServer();
+            }
         }
     }
     private function instance_auto_update($schedule)

app/Http/Livewire/Server/LogDrains.php

@@ -2,6 +2,7 @@
 
 namespace App\Http\Livewire\Server;
 
+use App\Actions\Server\InstallLogDrain;
 use App\Models\Server;
 use Livewire\Component;
 
@@ -46,14 +47,8 @@ class LogDrains extends Component
     public function configureLogDrain()
     {
         try {
-            if ($this->server->settings->is_logdrain_newrelic_enabled) {
-                $this->server->logDrain('newrelic');
-            } else if ($this->server->settings->is_logdrain_highlight_enabled) {
-                $this->server->logDrain('highlight');
-            } else if ($this->server->settings->is_logdrain_axiom_enabled) {
-                $this->server->logDrain('axiom');
-            } else {
-                $this->server->logDrain('none');
+            InstallLogDrain::run($this->server);
+            if (!$this->server->isLogDrainEnabled()) {
                 $this->emit('serverRefresh');
                 $this->emit('success', 'Log drain service stopped.');
                 return;

app/Jobs/CheckLogDrainContainerJob.php

@@ -0,0 +1,83 @@
+<?php
+
+namespace App\Jobs;
+
+use App\Actions\Server\InstallLogDrain;
+use App\Models\Server;
+use App\Notifications\Container\ContainerRestarted;
+use App\Notifications\Container\ContainerStopped;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeEncrypted;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Sleep;
+
+class CheckLogDrainContainerJob implements ShouldQueue, ShouldBeEncrypted
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    public function __construct(public Server $server)
+    {
+    }
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping($this->server->id))->dontRelease()];
+    }
+
+    public function uniqueId(): int
+    {
+        return $this->server->id;
+    }
+    public function healthcheck()
+    {
+        $status = instant_remote_process(["docker inspect --format='{{json .State.Status}}' coolify-log-drain"], $this->server, false);
+        if (str($status)->contains('running')) {
+            return true;
+        } else {
+            return false;
+        }
+    }
+    public function handle(): void
+    {
+        // ray("checking log drain statuses for {$this->server->id}");
+        try {
+            if (!$this->server->isServerReady()) {
+                return;
+            };
+            $containers = instant_remote_process(["docker container ls -q"], $this->server);
+            if (!$containers) {
+                return;
+            }
+            $containers = instant_remote_process(["docker container inspect $(docker container ls -q) --format '{{json .}}'"], $this->server);
+            $containers = format_docker_command_output_to_json($containers);
+
+            $foundLogDrainContainer = $containers->filter(function ($value, $key) {
+                return data_get($value, 'Name') === '/coolify-log-drain';
+            })->first();
+            if (!$foundLogDrainContainer || !$this->healthcheck()) {
+                ray('Log drain container not found or unhealthy. Restarting...');
+                InstallLogDrain::run($this->server);
+                Sleep::for(10)->seconds();
+                if ($this->healthcheck()) {
+                    if ($this->server->log_drain_notification_sent) {
+                        $this->server->team->notify(new ContainerRestarted('Coolify Log Drainer', $this->server));
+                        $this->server->update(['log_drain_notification_sent' => false]);
+                    }
+                    return;
+                }
+                if (!$this->server->log_drain_notification_sent) {
+                    ray('Log drain container still unhealthy. Sending notification...');
+                    $this->server->team->notify(new ContainerStopped('Coolify Log Drainer', $this->server, null));
+                    $this->server->update(['log_drain_notification_sent' => true]);
+                }
+            }
+        } catch (\Throwable $e) {
+            send_internal_notification("CheckLogDrainContainerJob failed on ({$this->server->id}) with: " . $e->getMessage());
+            ray($e->getMessage());
+            handleError($e);
+        }
+    }
+}

app/Jobs/ContainerStatusJob.php

@@ -52,29 +52,7 @@ class ContainerStatusJob implements ShouldQueue, ShouldBeEncrypted
             $databases = $this->server->databases();
             $services = $this->server->services()->get();
             $previews = $this->server->previews();
-            $this->server->proxyType();
-            /// Check if proxy is running
-            $foundProxyContainer = $containers->filter(function ($value, $key) {
-                return data_get($value, 'Name') === '/coolify-proxy';
-            })->first();
-            if (!$foundProxyContainer) {
-                try {
-                    $shouldStart = CheckProxy::run($this->server);
-                    if ($shouldStart) {
-                        StartProxy::run($this->server, false);
-                        $this->server->team->notify(new ContainerRestarted('coolify-proxy', $this->server));
-                    } else {
-                        ray('Proxy could not be started.');
-                    }
-                } catch (\Throwable $e) {
-                    ray($e);
-                }
-            } else {
-                $this->server->proxy->status = data_get($foundProxyContainer, 'State.Status');
-                $this->server->save();
-                $connectProxyToDockerNetworks = connectProxyToNetworks($this->server);
-                instant_remote_process($connectProxyToDockerNetworks, $this->server, false);
-            }
+
             $foundApplications = [];
             $foundApplicationPreviews = [];
             $foundDatabases = [];
@@ -267,6 +245,30 @@ class ContainerStatusJob implements ShouldQueue, ShouldBeEncrypted
                 }
                 $this->server->team->notify(new ContainerStopped($containerName, $this->server, $url));
             }
+
+            // Check if proxy is running
+            $this->server->proxyType();
+            $foundProxyContainer = $containers->filter(function ($value, $key) {
+                return data_get($value, 'Name') === '/coolify-proxy';
+            })->first();
+            if (!$foundProxyContainer) {
+                try {
+                    $shouldStart = CheckProxy::run($this->server);
+                    if ($shouldStart) {
+                        StartProxy::run($this->server, false);
+                        $this->server->team->notify(new ContainerRestarted('coolify-proxy', $this->server));
+                    } else {
+                        ray('Proxy could not be started.');
+                    }
+                } catch (\Throwable $e) {
+                    ray($e);
+                }
+            } else {
+                $this->server->proxy->status = data_get($foundProxyContainer, 'State.Status');
+                $this->server->save();
+                $connectProxyToDockerNetworks = connectProxyToNetworks($this->server);
+                instant_remote_process($connectProxyToDockerNetworks, $this->server, false);
+            }
         } catch (\Throwable $e) {
             send_internal_notification("ContainerStatusJob failed on ({$this->server->id}) with: " . $e->getMessage());
             ray($e->getMessage());

app/Models/Server.php

@@ -296,10 +296,6 @@ class Server extends BaseModel
         // }
         return true;
     }
-    public function logDrain($type)
-    {
-        InstallLogDrain::run($this, $type);
-    }
     public function isFunctional()
     {
         return $this->settings->is_reachable && $this->settings->is_usable;

app/Notifications/Container/ContainerRestarted.php

@@ -27,7 +27,7 @@ class ContainerRestarted extends Notification implements ShouldQueue
     public function toMail(): MailMessage
     {
         $mail = new MailMessage();
-        $mail->subject("Coolify: Container ({$this->name}) has been restarted automatically on {$this->server->name}");
+        $mail->subject("Coolify: A service ({$this->name}) has been restarted automatically on {$this->server->name}");
         $mail->view('emails.container-restarted', [
             'containerName' => $this->name,
             'serverName' => $this->server->name,
@@ -38,12 +38,12 @@ class ContainerRestarted extends Notification implements ShouldQueue
 
     public function toDiscord(): string
     {
-        $message = "Coolify: Container ({$this->name}) has been restarted automatically on {$this->server->name}";
+        $message = "Coolify: A service ({$this->name}) has been restarted automatically on {$this->server->name}";
         return $message;
     }
     public function toTelegram(): array
     {
-        $message = "Coolify:  Container ({$this->name}) has been restarted automatically on {$this->server->name}";
+        $message = "Coolify: A service ({$this->name}) has been restarted automatically on {$this->server->name}";
         $payload = [
             "message" => $message,
         ];

app/Notifications/Container/ContainerStopped.php

@@ -26,7 +26,7 @@ class ContainerStopped extends Notification implements ShouldQueue
     public function toMail(): MailMessage
     {
         $mail = new MailMessage();
-        $mail->subject("Coolify: Container ({$this->name}) has been stopped on {$this->server->name}");
+        $mail->subject("Coolify: A service ({$this->name}) has been stopped on {$this->server->name}");
         $mail->view('emails.container-stopped', [
             'containerName' => $this->name,
             'serverName' => $this->server->name,
@@ -37,12 +37,12 @@ class ContainerStopped extends Notification implements ShouldQueue
 
     public function toDiscord(): string
     {
-        $message = "Coolify:  Container ({$this->name}) has been stopped on {$this->server->name}";
+        $message = "Coolify: A service ({$this->name}) has been stopped on {$this->server->name}";
         return $message;
     }
     public function toTelegram(): array
     {
-        $message = "Coolify:  Container ($this->name} has been stopped on {$this->server->name}";
+        $message = "Coolify: A service ($this->name} has been stopped on {$this->server->name}";
         $payload = [
             "message" => $message,
         ];