feat: log drainer container check

app/Actions/Server/InstallLogDrain.php

@@ -8,8 +8,17 @@ use App\Models\Server;
 class InstallLogDrain
 {
     use AsAction;
-    public function handle(Server $server, string $type)
+    public function handle(Server $server)
     {
+        if ($server->settings->is_logdrain_newrelic_enabled) {
+            $type = 'newrelic';
+        } else if ($server->settings->is_logdrain_highlight_enabled) {
+            $type = 'highlight';
+        } else if ($server->settings->is_logdrain_axiom_enabled) {
+            $type = 'axiom';
+        } else {
+            $type = 'none';
+        }
         try {
             if ($type === 'none') {
                 $command = [

app/Console/Kernel.php

@@ -2,6 +2,7 @@
 
 namespace App\Console;
 
+use App\Jobs\CheckLogDrainContainerJob;
 use App\Jobs\CleanupInstanceStuffsJob;
 use App\Jobs\DatabaseBackupJob;
 use App\Jobs\InstanceAutoUpdateJob;
@@ -61,6 +62,9 @@ class Kernel extends ConsoleKernel
         foreach ($servers as $server) {
             $schedule->job(new ServerStatusJob($server))->everyTenMinutes()->onOneServer();
             $schedule->job(new ContainerStatusJob($server))->everyMinute()->onOneServer();
+            if ($server->isLogDrainEnabled()) {
+                $schedule->job(new CheckLogDrainContainerJob($server))->everyMinute()->onOneServer();
+            }
         }
     }
     private function instance_auto_update($schedule)

app/Http/Livewire/Server/LogDrains.php

@@ -2,6 +2,7 @@
 
 namespace App\Http\Livewire\Server;
 
+use App\Actions\Server\InstallLogDrain;
 use App\Models\Server;
 use Livewire\Component;
 
@@ -46,14 +47,8 @@ class LogDrains extends Component
     public function configureLogDrain()
     {
         try {
-            if ($this->server->settings->is_logdrain_newrelic_enabled) {
+            InstallLogDrain::run($this->server);
-                $this->server->logDrain('newrelic');
+            if (!$this->server->isLogDrainEnabled()) {
-            } else if ($this->server->settings->is_logdrain_highlight_enabled) {
-                $this->server->logDrain('highlight');
-            } else if ($this->server->settings->is_logdrain_axiom_enabled) {
-                $this->server->logDrain('axiom');
-            } else {
-                $this->server->logDrain('none');
                 $this->emit('serverRefresh');
                 $this->emit('success', 'Log drain service stopped.');
                 return;

app/Jobs/CheckLogDrainContainerJob.php

@@ -0,0 +1,83 @@
+<?php
+
+namespace App\Jobs;
+
+use App\Actions\Server\InstallLogDrain;
+use App\Models\Server;
+use App\Notifications\Container\ContainerRestarted;
+use App\Notifications\Container\ContainerStopped;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeEncrypted;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Sleep;
+
+class CheckLogDrainContainerJob implements ShouldQueue, ShouldBeEncrypted
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    public function __construct(public Server $server)
+    {
+    }
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping($this->server->id))->dontRelease()];
+    }
+
+    public function uniqueId(): int
+    {
+        return $this->server->id;
+    }
+    public function healthcheck()
+    {
+        $status = instant_remote_process(["docker inspect --format='{{json .State.Status}}' coolify-log-drain"], $this->server, false);
+        if (str($status)->contains('running')) {
+            return true;
+        } else {
+            return false;
+        }
+    }
+    public function handle(): void
+    {
+        // ray("checking log drain statuses for {$this->server->id}");
+        try {
+            if (!$this->server->isServerReady()) {
+                return;
+            };
+            $containers = instant_remote_process(["docker container ls -q"], $this->server);
+            if (!$containers) {
+                return;
+            }
+            $containers = instant_remote_process(["docker container inspect $(docker container ls -q) --format '{{json .}}'"], $this->server);
+            $containers = format_docker_command_output_to_json($containers);
+
+            $foundLogDrainContainer = $containers->filter(function ($value, $key) {
+                return data_get($value, 'Name') === '/coolify-log-drain';
+            })->first();
+            if (!$foundLogDrainContainer || !$this->healthcheck()) {
+                ray('Log drain container not found or unhealthy. Restarting...');
+                InstallLogDrain::run($this->server);
+                Sleep::for(10)->seconds();
+                if ($this->healthcheck()) {
+                    if ($this->server->log_drain_notification_sent) {
+                        $this->server->team->notify(new ContainerRestarted('Coolify Log Drainer', $this->server));
+                        $this->server->update(['log_drain_notification_sent' => false]);
+                    }
+                    return;
+                }
+                if (!$this->server->log_drain_notification_sent) {
+                    ray('Log drain container still unhealthy. Sending notification...');
+                    $this->server->team->notify(new ContainerStopped('Coolify Log Drainer', $this->server, null));
+                    $this->server->update(['log_drain_notification_sent' => true]);
+                }
+            }
+        } catch (\Throwable $e) {
+            send_internal_notification("CheckLogDrainContainerJob failed on ({$this->server->id}) with: " . $e->getMessage());
+            ray($e->getMessage());
+            handleError($e);
+        }
+    }
+}

app/Jobs/ContainerStatusJob.php

@@ -52,29 +52,7 @@ class ContainerStatusJob implements ShouldQueue, ShouldBeEncrypted
             $databases = $this->server->databases();
             $services = $this->server->services()->get();
             $previews = $this->server->previews();
-            $this->server->proxyType();
+
-            /// Check if proxy is running
-            $foundProxyContainer = $containers->filter(function ($value, $key) {
-                return data_get($value, 'Name') === '/coolify-proxy';
-            })->first();
-            if (!$foundProxyContainer) {
-                try {
-                    $shouldStart = CheckProxy::run($this->server);
-                    if ($shouldStart) {
-                        StartProxy::run($this->server, false);
-                        $this->server->team->notify(new ContainerRestarted('coolify-proxy', $this->server));
-                    } else {
-                        ray('Proxy could not be started.');
-                    }
-                } catch (\Throwable $e) {
-                    ray($e);
-                }
-            } else {
-                $this->server->proxy->status = data_get($foundProxyContainer, 'State.Status');
-                $this->server->save();
-                $connectProxyToDockerNetworks = connectProxyToNetworks($this->server);
-                instant_remote_process($connectProxyToDockerNetworks, $this->server, false);
-            }
             $foundApplications = [];
             $foundApplicationPreviews = [];
             $foundDatabases = [];
@@ -267,6 +245,30 @@ class ContainerStatusJob implements ShouldQueue, ShouldBeEncrypted
                 }
                 $this->server->team->notify(new ContainerStopped($containerName, $this->server, $url));
             }
+
+            // Check if proxy is running
+            $this->server->proxyType();
+            $foundProxyContainer = $containers->filter(function ($value, $key) {
+                return data_get($value, 'Name') === '/coolify-proxy';
+            })->first();
+            if (!$foundProxyContainer) {
+                try {
+                    $shouldStart = CheckProxy::run($this->server);
+                    if ($shouldStart) {
+                        StartProxy::run($this->server, false);
+                        $this->server->team->notify(new ContainerRestarted('coolify-proxy', $this->server));
+                    } else {
+                        ray('Proxy could not be started.');
+                    }
+                } catch (\Throwable $e) {
+                    ray($e);
+                }
+            } else {
+                $this->server->proxy->status = data_get($foundProxyContainer, 'State.Status');
+                $this->server->save();
+                $connectProxyToDockerNetworks = connectProxyToNetworks($this->server);
+                instant_remote_process($connectProxyToDockerNetworks, $this->server, false);
+            }
         } catch (\Throwable $e) {
             send_internal_notification("ContainerStatusJob failed on ({$this->server->id}) with: " . $e->getMessage());
             ray($e->getMessage());

app/Models/Server.php

@@ -296,10 +296,6 @@ class Server extends BaseModel
         // }
         return true;
     }
-    public function logDrain($type)
-    {
-        InstallLogDrain::run($this, $type);
-    }
     public function isFunctional()
     {
         return $this->settings->is_reachable && $this->settings->is_usable;

app/Notifications/Container/ContainerRestarted.php

@@ -27,7 +27,7 @@ class ContainerRestarted extends Notification implements ShouldQueue
     public function toMail(): MailMessage
     {
         $mail = new MailMessage();
-        $mail->subject("Coolify: Container ({$this->name}) has been restarted automatically on {$this->server->name}");
+        $mail->subject("Coolify: A service ({$this->name}) has been restarted automatically on {$this->server->name}");
         $mail->view('emails.container-restarted', [
             'containerName' => $this->name,
             'serverName' => $this->server->name,
@@ -38,12 +38,12 @@ class ContainerRestarted extends Notification implements ShouldQueue
 
     public function toDiscord(): string
     {
-        $message = "Coolify: Container ({$this->name}) has been restarted automatically on {$this->server->name}";
+        $message = "Coolify: A service ({$this->name}) has been restarted automatically on {$this->server->name}";
         return $message;
     }
     public function toTelegram(): array
     {
-        $message = "Coolify:  Container ({$this->name}) has been restarted automatically on {$this->server->name}";
+        $message = "Coolify: A service ({$this->name}) has been restarted automatically on {$this->server->name}";
         $payload = [
             "message" => $message,
         ];

app/Notifications/Container/ContainerStopped.php

@@ -26,7 +26,7 @@ class ContainerStopped extends Notification implements ShouldQueue
     public function toMail(): MailMessage
     {
         $mail = new MailMessage();
-        $mail->subject("Coolify: Container ({$this->name}) has been stopped on {$this->server->name}");
+        $mail->subject("Coolify: A service ({$this->name}) has been stopped on {$this->server->name}");
         $mail->view('emails.container-stopped', [
             'containerName' => $this->name,
             'serverName' => $this->server->name,
@@ -37,12 +37,12 @@ class ContainerStopped extends Notification implements ShouldQueue
 
     public function toDiscord(): string
     {
-        $message = "Coolify:  Container ({$this->name}) has been stopped on {$this->server->name}";
+        $message = "Coolify: A service ({$this->name}) has been stopped on {$this->server->name}";
         return $message;
     }
     public function toTelegram(): array
     {
-        $message = "Coolify:  Container ($this->name} has been stopped on {$this->server->name}";
+        $message = "Coolify: A service ($this->name} has been stopped on {$this->server->name}";
         $payload = [
             "message" => $message,
         ];

database/migrations/2023_11_17_160437_add_drain_log_enable_by_service.php

@@ -35,7 +35,9 @@ return new class extends Migration
         Schema::table('service_databases', function (Blueprint $table) {
             $table->boolean('is_log_drain_enabled')->default(false);
         });
-
+        Schema::table('servers', function (Blueprint $table) {
+            $table->boolean('log_drain_notification_sent')->default(false);
+        });
     }
 
     /**
@@ -67,6 +69,8 @@ return new class extends Migration
         Schema::table('service_databases', function (Blueprint $table) {
             $table->dropColumn('is_log_drain_enabled');
         });
-
+        Schema::table('servers', function (Blueprint $table) {
+            $table->dropColumn('log_drain_notification_sent');
+        });
     }
 };

resources/views/emails/container-restarted.blade.php

@@ -1,6 +1,6 @@
 <x-emails.layout>
 
-Container ({{ $containerName }}) has been restarted automatically on {{$serverName}}, because it was stopped unexpectedly.
+A service ({{ $containerName }}) has been restarted automatically on {{$serverName}}, because it was stopped unexpectedly.
 
 @if ($containerName === 'coolify-proxy')
 Coolify Proxy should run on your server as you have FQDNs set up in one of your resources.

resources/views/emails/container-stopped.blade.php

@@ -1,6 +1,6 @@
 <x-emails.layout>
 
-Container {{ $containerName }} has been stopped unexpectedly on {{$serverName}}.
+A service ({{ $containerName }}) has been stopped unexpectedly on {{$serverName}}.
 
 @if ($url)
 Please check what is going on [here]({{ $url }}).

resources/views/livewire/server/log-drains.blade.php

@@ -1,7 +1,7 @@
 <div>
     <x-server.navbar :server="$server" :parameters="$parameters" />
     <h2>Log Drains</h2>
-    <div class="pb-4">Sends resource logs to external services.</div>
+    <div class="pb-4">Sends service logs to 3rd party tools.</div>
     <div class="flex flex-col gap-4 pt-4">
         <div class="p-4 border border-coolgray-500">
             <form wire:submit.prevent='submit("newrelic")' class="flex flex-col">