Merge pull request #4586 from peaklabs-dev/fix-unreachable-notificiations

Fix: Unreachable Notifications

app/Console/Commands/CloudCleanupSubscriptions.php

@@ -2,6 +2,7 @@
 
 namespace App\Console\Commands;
 
+use App\Events\ServerReachabilityChanged;
 use App\Models\Team;
 use Illuminate\Console\Command;
 
@@ -92,6 +93,8 @@ class CloudCleanupSubscriptions extends Command
                 $server->update([
                     'ip' => '1.2.3.4',
                 ]);
+
+                ServerReachabilityChanged::dispatch($server);
             }
         }
     }

app/Events/ServerReachabilityChanged.php

@@ -0,0 +1,17 @@
+<?php
+
+namespace App\Events;
+
+use App\Models\Server;
+use Illuminate\Foundation\Events\Dispatchable;
+
+class ServerReachabilityChanged
+{
+    use Dispatchable;
+
+    public function __construct(
+        public readonly Server $server
+    ) {
+        $this->server->isReachableChanged();
+    }
+}

app/Jobs/ServerCheckJob.php

@@ -31,12 +31,7 @@ class ServerCheckJob implements ShouldBeEncrypted, ShouldQueue
         return [(new WithoutOverlapping($this->server->uuid))->dontRelease()];
     }
 
-    public function __construct(public Server $server)
-    {
-        if (isDev()) {
-            $this->handle();
-        }
-    }
+    public function __construct(public Server $server) {}
 
     public function handle()
     {

app/Livewire/Server/Show.php

@@ -4,6 +4,7 @@ namespace App\Livewire\Server;
 
 use App\Actions\Server\StartSentinel;
 use App\Actions\Server\StopSentinel;
+use App\Events\ServerReachabilityChanged;
 use App\Models\Server;
 use Livewire\Attributes\Computed;
 use Livewire\Attributes\Validate;
@@ -202,6 +203,7 @@ class Show extends Component
             $this->server->settings->is_reachable = $this->isReachable = true;
             $this->server->settings->is_usable = $this->isUsable = true;
             $this->server->settings->save();
+            ServerReachabilityChanged::dispatch($this->server);
             $this->dispatch('proxyStatusUpdated');
         } else {
             $this->dispatch('error', 'Server is not reachable.', 'Please validate your configuration and connection.<br><br>Check this <a target="_blank" class="underline" href="https://coolify.io/docs/knowledge-base/server/openssh">documentation</a> for further help. <br><br>Error: '.$error);

app/Models/Server.php

@@ -6,6 +6,7 @@ use App\Actions\Proxy\StartProxy;
 use App\Actions\Server\InstallDocker;
 use App\Actions\Server\StartSentinel;
 use App\Enums\ProxyTypes;
+use App\Events\ServerReachabilityChanged;
 use App\Jobs\CheckAndStartSentinelJob;
 use App\Notifications\Server\Reachable;
 use App\Notifications\Server\Unreachable;
@@ -1024,14 +1025,63 @@ $schema://$host {
         $this->refresh();
         $unreachableNotificationSent = (bool) $this->unreachable_notification_sent;
         $isReachable = (bool) $this->settings->is_reachable;
-        // If the server is reachable, send the reachable notification if it was sent before
+
+        \Log::debug('Server reachability check', [
+            'server_id' => $this->id,
+            'is_reachable' => $isReachable,
+            'notification_sent' => $unreachableNotificationSent,
+            'unreachable_count' => $this->unreachable_count,
+        ]);
+
         if ($isReachable === true) {
+            $this->unreachable_count = 0;
+            $this->save();
+
             if ($unreachableNotificationSent === true) {
+                \Log::debug('Server is now reachable, sending notification', [
+                    'server_id' => $this->id,
+                ]);
                 $this->sendReachableNotification();
             }
-        } else {
-            // If the server is unreachable, send the unreachable notification if it was not sent before
-            if ($unreachableNotificationSent === false) {
+
+            return;
+        }
+
+        $this->increment('unreachable_count');
+        \Log::debug('Incremented unreachable count', [
+            'server_id' => $this->id,
+            'new_count' => $this->unreachable_count,
+        ]);
+
+        if ($this->unreachable_count === 1) {
+            $this->settings->is_reachable = true;
+            $this->settings->save();
+            \Log::debug('First unreachable attempt, marking as reachable', [
+                'server_id' => $this->id,
+            ]);
+
+            return;
+        }
+
+        if ($this->unreachable_count >= 2 && ! $unreachableNotificationSent) {
+            $failedChecks = 0;
+            for ($i = 0; $i < 3; $i++) {
+                $status = $this->serverStatus();
+                \Log::debug('Additional reachability check', [
+                    'server_id' => $this->id,
+                    'attempt' => $i + 1,
+                    'status' => $status,
+                ]);
+                sleep(5);
+                if (! $status) {
+                    $failedChecks++;
+                }
+            }
+
+            if ($failedChecks === 3 && ! $unreachableNotificationSent) {
+                \Log::debug('Server confirmed unreachable after 3 attempts, sending notification', [
+                    'server_id' => $this->id,
+                ]);
                 $this->sendUnreachableNotification();
             }
         }
@@ -1065,6 +1115,7 @@ $schema://$host {
             if ($this->settings->is_reachable === false) {
                 $this->settings->is_reachable = true;
                 $this->settings->save();
+                ServerReachabilityChanged::dispatch($this);
             }
 
             return ['uptime' => true, 'error' => null];
@@ -1075,6 +1126,7 @@ $schema://$host {
             if ($this->settings->is_reachable === true) {
                 $this->settings->is_reachable = false;
                 $this->settings->save();
+                ServerReachabilityChanged::dispatch($this);
             }
 
             return ['uptime' => false, 'error' => $e->getMessage()];
@@ -1165,6 +1217,7 @@ $schema://$host {
         $this->settings->is_reachable = true;
         $this->settings->is_usable = true;
         $this->settings->save();
+        ServerReachabilityChanged::dispatch($this);
 
         return true;
     }

app/Models/ServerSetting.php

@@ -85,9 +85,6 @@ class ServerSetting extends Model
             ) {
                 $settings->server->restartSentinel();
             }
-            if ($settings->isDirty('is_reachable')) {
-                $settings->server->isReachableChanged();
-            }
         });
     }
 

app/Models/Team.php

@@ -2,6 +2,7 @@
 
 namespace App\Models;
 
+use App\Events\ServerReachabilityChanged;
 use App\Notifications\Channels\SendsDiscord;
 use App\Notifications\Channels\SendsEmail;
 use App\Notifications\Channels\SendsPushover;
@@ -202,6 +203,7 @@ class Team extends Model implements SendsDiscord, SendsEmail, SendsPushover, Sen
                 'is_usable' => false,
                 'is_reachable' => false,
             ]);
+            ServerReachabilityChanged::dispatch($server);
         }
     }