From fcdec3a80df96cd0f3793c3fcae2306d8d23d43e Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Sun, 15 Dec 2024 15:09:58 +0100 Subject: [PATCH 1/8] fix: unreachable notifications --- app/Models/Server.php | 52 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 48 insertions(+), 4 deletions(-) diff --git a/app/Models/Server.php b/app/Models/Server.php index cc8211789..1e4413d20 100644 --- a/app/Models/Server.php +++ b/app/Models/Server.php @@ -1024,14 +1024,58 @@ $schema://$host { $this->refresh(); $unreachableNotificationSent = (bool) $this->unreachable_notification_sent; $isReachable = (bool) $this->settings->is_reachable; - // If the server is reachable, send the reachable notification if it was sent before + + ray('Server Reachability Check', [ + 'server_id' => $this->id, + 'unreachable_notification_sent' => $unreachableNotificationSent, + 'is_reachable' => $isReachable, + 'unreachable_count' => $this->unreachable_count, + ]); + if ($isReachable === true) { + ray('Server is reachable - resetting counts'); + $this->unreachable_count = 0; + $this->save(); + if ($unreachableNotificationSent === true) { + ray('Sending reachable notification'); $this->sendReachableNotification(); } - } else { - // If the server is unreachable, send the unreachable notification if it was not sent before - if ($unreachableNotificationSent === false) { + + return; + } + + ray('Server is not reachable - incrementing count'); + $this->increment('unreachable_count'); + ray('Unreachable count: '.$this->unreachable_count); + if ($this->unreachable_count === 1) { + $this->settings->is_reachable = true; + $this->settings->saveQuietly(); + + return; + } + + if ($this->unreachable_count >= 2 && ! $unreachableNotificationSent) { + ray('Starting additional checks', [ + 'unreachable_count' => $this->unreachable_count, + ]); + $failedChecks = 0; + for ($i = 0; $i < 2; $i++) { + sleep(5); + $status = $this->status(); + ray('Check attempt '.($i + 1), ['status' => $status]); + if (! $status) { + $failedChecks++; + } + } + + ray('Additional checks complete', [ + 'failed_checks' => $failedChecks, + 'will_send_notification' => ($failedChecks === 2 && ! $unreachableNotificationSent), + ]); + + if ($failedChecks === 2 && ! $unreachableNotificationSent) { + ray('Sending unreachable notification'); $this->sendUnreachableNotification(); } } From 1da3febd852d219e6269fa4e6b3603eaeb8cf788 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Mon, 16 Dec 2024 14:04:22 +0100 Subject: [PATCH 2/8] fix: remove duplicated ServerCheckJob call --- app/Jobs/ServerCheckJob.php | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/app/Jobs/ServerCheckJob.php b/app/Jobs/ServerCheckJob.php index 49d8dfe08..9818d5c6a 100644 --- a/app/Jobs/ServerCheckJob.php +++ b/app/Jobs/ServerCheckJob.php @@ -31,12 +31,7 @@ class ServerCheckJob implements ShouldBeEncrypted, ShouldQueue return [(new WithoutOverlapping($this->server->uuid))->dontRelease()]; } - public function __construct(public Server $server) - { - if (isDev()) { - $this->handle(); - } - } + public function __construct(public Server $server) {} public function handle() { From e30495a59b1fa795ff5a793e70b88f08467f9be3 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Mon, 16 Dec 2024 14:05:17 +0100 Subject: [PATCH 3/8] feat: new ServerReachabilityChanged event --- app/Events/ServerReachabilityChanged.php | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 app/Events/ServerReachabilityChanged.php diff --git a/app/Events/ServerReachabilityChanged.php b/app/Events/ServerReachabilityChanged.php new file mode 100644 index 000000000..fb4680146 --- /dev/null +++ b/app/Events/ServerReachabilityChanged.php @@ -0,0 +1,17 @@ +server->isReachableChanged(); + } +} From 5571aa4530d4e3e6107f1ebf878e12bb9479a51a Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Mon, 16 Dec 2024 14:06:16 +0100 Subject: [PATCH 4/8] feat: use new ServerReachabilityChanged event instead of isDirty --- app/Console/Commands/CloudCleanupSubscriptions.php | 3 +++ app/Livewire/Server/Show.php | 2 ++ app/Models/ServerSetting.php | 3 --- app/Models/Team.php | 2 ++ 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/app/Console/Commands/CloudCleanupSubscriptions.php b/app/Console/Commands/CloudCleanupSubscriptions.php index 9198b003e..9dc6e24f5 100644 --- a/app/Console/Commands/CloudCleanupSubscriptions.php +++ b/app/Console/Commands/CloudCleanupSubscriptions.php @@ -2,6 +2,7 @@ namespace App\Console\Commands; +use App\Events\ServerReachabilityChanged; use App\Models\Team; use Illuminate\Console\Command; @@ -92,6 +93,8 @@ class CloudCleanupSubscriptions extends Command $server->update([ 'ip' => '1.2.3.4', ]); + + ServerReachabilityChanged::dispatch($server); } } } diff --git a/app/Livewire/Server/Show.php b/app/Livewire/Server/Show.php index ac5211c1b..6d267b9c8 100644 --- a/app/Livewire/Server/Show.php +++ b/app/Livewire/Server/Show.php @@ -4,6 +4,7 @@ namespace App\Livewire\Server; use App\Actions\Server\StartSentinel; use App\Actions\Server\StopSentinel; +use App\Events\ServerReachabilityChanged; use App\Models\Server; use Livewire\Attributes\Computed; use Livewire\Attributes\Validate; @@ -202,6 +203,7 @@ class Show extends Component $this->server->settings->is_reachable = $this->isReachable = true; $this->server->settings->is_usable = $this->isUsable = true; $this->server->settings->save(); + ServerReachabilityChanged::dispatch($this->server); $this->dispatch('proxyStatusUpdated'); } else { $this->dispatch('error', 'Server is not reachable.', 'Please validate your configuration and connection.

Check this documentation for further help.

Error: '.$error); diff --git a/app/Models/ServerSetting.php b/app/Models/ServerSetting.php index e078372e2..f4b776cca 100644 --- a/app/Models/ServerSetting.php +++ b/app/Models/ServerSetting.php @@ -85,9 +85,6 @@ class ServerSetting extends Model ) { $settings->server->restartSentinel(); } - if ($settings->isDirty('is_reachable')) { - $settings->server->isReachableChanged(); - } }); } diff --git a/app/Models/Team.php b/app/Models/Team.php index e55cb0d19..33847a3c8 100644 --- a/app/Models/Team.php +++ b/app/Models/Team.php @@ -2,6 +2,7 @@ namespace App\Models; +use App\Events\ServerReachabilityChanged; use App\Notifications\Channels\SendsDiscord; use App\Notifications\Channels\SendsEmail; use App\Notifications\Channels\SendsPushover; @@ -202,6 +203,7 @@ class Team extends Model implements SendsDiscord, SendsEmail, SendsPushover, Sen 'is_usable' => false, 'is_reachable' => false, ]); + ServerReachabilityChanged::dispatch($server); } } From 80425a16eec8882d90e102b00cc35eab32c42e0c Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Mon, 16 Dec 2024 14:06:53 +0100 Subject: [PATCH 5/8] fix: few fixes and use new ServerReachabilityChanged event --- app/Models/Server.php | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/app/Models/Server.php b/app/Models/Server.php index 1e4413d20..aa709bf6b 100644 --- a/app/Models/Server.php +++ b/app/Models/Server.php @@ -6,6 +6,7 @@ use App\Actions\Proxy\StartProxy; use App\Actions\Server\InstallDocker; use App\Actions\Server\StartSentinel; use App\Enums\ProxyTypes; +use App\Events\ServerReachabilityChanged; use App\Jobs\CheckAndStartSentinelJob; use App\Notifications\Server\Reachable; use App\Notifications\Server\Unreachable; @@ -1050,7 +1051,7 @@ $schema://$host { ray('Unreachable count: '.$this->unreachable_count); if ($this->unreachable_count === 1) { $this->settings->is_reachable = true; - $this->settings->saveQuietly(); + $this->settings->save(); return; } @@ -1060,21 +1061,22 @@ $schema://$host { 'unreachable_count' => $this->unreachable_count, ]); $failedChecks = 0; - for ($i = 0; $i < 2; $i++) { - sleep(5); + for ($i = 0; $i < 3; $i++) { $status = $this->status(); + sleep(5); ray('Check attempt '.($i + 1), ['status' => $status]); if (! $status) { $failedChecks++; + } } ray('Additional checks complete', [ 'failed_checks' => $failedChecks, - 'will_send_notification' => ($failedChecks === 2 && ! $unreachableNotificationSent), + 'will_send_notification' => ($failedChecks === 3 && ! $unreachableNotificationSent), ]); - if ($failedChecks === 2 && ! $unreachableNotificationSent) { + if ($failedChecks === 3 && ! $unreachableNotificationSent) { ray('Sending unreachable notification'); $this->sendUnreachableNotification(); } @@ -1109,6 +1111,7 @@ $schema://$host { if ($this->settings->is_reachable === false) { $this->settings->is_reachable = true; $this->settings->save(); + ServerReachabilityChanged::dispatch($this); } return ['uptime' => true, 'error' => null]; @@ -1119,6 +1122,7 @@ $schema://$host { if ($this->settings->is_reachable === true) { $this->settings->is_reachable = false; $this->settings->save(); + ServerReachabilityChanged::dispatch($this); } return ['uptime' => false, 'error' => $e->getMessage()]; @@ -1209,6 +1213,7 @@ $schema://$host { $this->settings->is_reachable = true; $this->settings->is_usable = true; $this->settings->save(); + ServerReachabilityChanged::dispatch($this); return true; } From 0957fe8bfc74414af444e48a3f5791c7c47ebcf5 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Mon, 16 Dec 2024 14:15:38 +0100 Subject: [PATCH 6/8] chore: remove ray debugging --- app/Models/Server.php | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/app/Models/Server.php b/app/Models/Server.php index aa709bf6b..902433530 100644 --- a/app/Models/Server.php +++ b/app/Models/Server.php @@ -1026,29 +1026,18 @@ $schema://$host { $unreachableNotificationSent = (bool) $this->unreachable_notification_sent; $isReachable = (bool) $this->settings->is_reachable; - ray('Server Reachability Check', [ - 'server_id' => $this->id, - 'unreachable_notification_sent' => $unreachableNotificationSent, - 'is_reachable' => $isReachable, - 'unreachable_count' => $this->unreachable_count, - ]); - if ($isReachable === true) { - ray('Server is reachable - resetting counts'); $this->unreachable_count = 0; $this->save(); if ($unreachableNotificationSent === true) { - ray('Sending reachable notification'); $this->sendReachableNotification(); } return; } - ray('Server is not reachable - incrementing count'); $this->increment('unreachable_count'); - ray('Unreachable count: '.$this->unreachable_count); if ($this->unreachable_count === 1) { $this->settings->is_reachable = true; $this->settings->save(); @@ -1057,27 +1046,16 @@ $schema://$host { } if ($this->unreachable_count >= 2 && ! $unreachableNotificationSent) { - ray('Starting additional checks', [ - 'unreachable_count' => $this->unreachable_count, - ]); $failedChecks = 0; for ($i = 0; $i < 3; $i++) { $status = $this->status(); sleep(5); - ray('Check attempt '.($i + 1), ['status' => $status]); if (! $status) { $failedChecks++; - } } - ray('Additional checks complete', [ - 'failed_checks' => $failedChecks, - 'will_send_notification' => ($failedChecks === 3 && ! $unreachableNotificationSent), - ]); - if ($failedChecks === 3 && ! $unreachableNotificationSent) { - ray('Sending unreachable notification'); $this->sendUnreachableNotification(); } } From 0cbb3a08c285d0052149535455e6003334ef4bdd Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Mon, 16 Dec 2024 14:27:17 +0100 Subject: [PATCH 7/8] fix: use serverStatus not just status --- app/Models/Server.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/Models/Server.php b/app/Models/Server.php index 902433530..6abc7cb43 100644 --- a/app/Models/Server.php +++ b/app/Models/Server.php @@ -1048,7 +1048,7 @@ $schema://$host { if ($this->unreachable_count >= 2 && ! $unreachableNotificationSent) { $failedChecks = 0; for ($i = 0; $i < 3; $i++) { - $status = $this->status(); + $status = $this->serverStatus(); sleep(5); if (! $status) { $failedChecks++; From a8401182afbcdd4dd7ebd5488956ac0d6ed5923c Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Tue, 17 Dec 2024 23:18:40 +0100 Subject: [PATCH 8/8] add debug logs to debug the logs - lol --- app/Models/Server.php | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/app/Models/Server.php b/app/Models/Server.php index 6abc7cb43..767327b8e 100644 --- a/app/Models/Server.php +++ b/app/Models/Server.php @@ -1026,11 +1026,21 @@ $schema://$host { $unreachableNotificationSent = (bool) $this->unreachable_notification_sent; $isReachable = (bool) $this->settings->is_reachable; + \Log::debug('Server reachability check', [ + 'server_id' => $this->id, + 'is_reachable' => $isReachable, + 'notification_sent' => $unreachableNotificationSent, + 'unreachable_count' => $this->unreachable_count, + ]); + if ($isReachable === true) { $this->unreachable_count = 0; $this->save(); if ($unreachableNotificationSent === true) { + \Log::debug('Server is now reachable, sending notification', [ + 'server_id' => $this->id, + ]); $this->sendReachableNotification(); } @@ -1038,9 +1048,17 @@ $schema://$host { } $this->increment('unreachable_count'); + \Log::debug('Incremented unreachable count', [ + 'server_id' => $this->id, + 'new_count' => $this->unreachable_count, + ]); + if ($this->unreachable_count === 1) { $this->settings->is_reachable = true; $this->settings->save(); + \Log::debug('First unreachable attempt, marking as reachable', [ + 'server_id' => $this->id, + ]); return; } @@ -1049,6 +1067,11 @@ $schema://$host { $failedChecks = 0; for ($i = 0; $i < 3; $i++) { $status = $this->serverStatus(); + \Log::debug('Additional reachability check', [ + 'server_id' => $this->id, + 'attempt' => $i + 1, + 'status' => $status, + ]); sleep(5); if (! $status) { $failedChecks++; @@ -1056,6 +1079,9 @@ $schema://$host { } if ($failedChecks === 3 && ! $unreachableNotificationSent) { + \Log::debug('Server confirmed unreachable after 3 attempts, sending notification', [ + 'server_id' => $this->id, + ]); $this->sendUnreachableNotification(); } }