diff --git a/app/Console/Commands/Cloud.php b/app/Console/Commands/Cloud.php new file mode 100644 index 000000000..1386b296c --- /dev/null +++ b/app/Console/Commands/Cloud.php @@ -0,0 +1,33 @@ +whereNotNull('team.subscription')->where('team.subscription.stripe_trial_already_ended',true)->each(function($server){ + $this->info($server->name); + }); + } +} diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index ee9f67ad7..d6ea360ae 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -48,7 +48,11 @@ class Kernel extends ConsoleKernel } private function check_resources($schedule) { - $servers = Server::all()->where('settings.is_usable', true)->where('settings.is_reachable', true); + if (isCloud()) { + $servers = Server::all()->whereNotNull('team.subscription')->where('team.subscription.stripe_trial_already_ended', false); + } else { + $servers = Server::all(); + } foreach ($servers as $server) { $schedule->job(new ContainerStatusJob($server))->everyMinute()->onOneServer(); } diff --git a/app/Jobs/ContainerStatusJob.php b/app/Jobs/ContainerStatusJob.php index 7f28954a0..a4e7688ef 100644 --- a/app/Jobs/ContainerStatusJob.php +++ b/app/Jobs/ContainerStatusJob.php @@ -7,6 +7,7 @@ use App\Models\ApplicationPreview; use App\Models\Server; use App\Notifications\Container\ContainerRestarted; use App\Notifications\Container\ContainerStopped; +use App\Notifications\Server\Revived; use App\Notifications\Server\Unreachable; use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldBeEncrypted; @@ -51,14 +52,20 @@ class ContainerStatusJob implements ShouldQueue, ShouldBeEncrypted public function handle(): void { try { + ray("checking server status for {$this->server->name}"); // ray()->clearAll(); $serverUptimeCheckNumber = 0; $serverUptimeCheckNumberMax = 3; while (true) { if ($serverUptimeCheckNumber >= $serverUptimeCheckNumberMax) { send_internal_notification('Server unreachable: ' . $this->server->name); - // $this->server->settings()->update(['is_reachable' => false]); - // $this->server->team->notify(new Unreachable($this->server)); + if ($this->server->unreachable_email_sent === false) { + $this->server->team->notify(new Unreachable($this->server)); + } + $this->server->settings()->update([ + 'is_reachable' => false, + ]); + $this->server->update(['unreachable_email_sent' => true]); return; } $result = $this->checkServerConnection(); @@ -68,6 +75,20 @@ class ContainerStatusJob implements ShouldQueue, ShouldBeEncrypted $serverUptimeCheckNumber++; sleep(5); } + if (data_get($this->server, 'unreachable_email_sent') === true) { + $this->server->team->notify(new Revived($this->server)); + $this->server->update(['unreachable_email_sent' => false]); + } + if ( + data_get($this->server, 'settings.is_reachable') === false || + data_get($this->server, 'settings.is_usable') === false + ) { + $this->server->settings()->update([ + 'is_reachable' => true, + 'is_usable' => true + ]); + } + $containers = instant_remote_process(["docker container ls -q"], $this->server); if (!$containers) { return; diff --git a/app/Models/Server.php b/app/Models/Server.php index f174222c0..b41720a06 100644 --- a/app/Models/Server.php +++ b/app/Models/Server.php @@ -8,6 +8,7 @@ use Illuminate\Database\Eloquent\Builder; use Illuminate\Database\Eloquent\Casts\Attribute; use Spatie\SchemalessAttributes\Casts\SchemalessAttributes; use Spatie\SchemalessAttributes\SchemalessAttributesTrait; +use Illuminate\Support\Str; class Server extends BaseModel { @@ -15,6 +16,11 @@ class Server extends BaseModel protected static function booted() { + static::saved(function ($server) { + $server->ip = Str::of($server->ip)->trim(); + $server->user = Str::of($server->user)->trim(); + }); + static::created(function ($server) { ServerSetting::create([ 'server_id' => $server->id, diff --git a/app/Notifications/Server/Revived.php b/app/Notifications/Server/Revived.php new file mode 100644 index 000000000..4c9d9c7d2 --- /dev/null +++ b/app/Notifications/Server/Revived.php @@ -0,0 +1,49 @@ +server->unreachable_email_sent === false) { + return; + } + } + + public function via(object $notifiable): array + { + return setNotificationChannels($notifiable, 'status_changes'); + } + + public function toMail(): MailMessage + { + $mail = new MailMessage(); + $mail->subject("✅ Server ({$this->server->name}) revived."); + $mail->view('emails.server-revived', [ + 'name' => $this->server->name, + ]); + return $mail; + } + + public function toDiscord(): string + { + $message = "✅ Server '{$this->server->name}' revived. All automations & integrations are turned on again!"; + return $message; + } + public function toTelegram(): array + { + return [ + "message" => "✅ Server '{$this->server->name}' revived. All automations & integrations are turned on again!" + ]; + } +} diff --git a/app/Notifications/Server/Unreachable.php b/app/Notifications/Server/Unreachable.php index ec9c11d11..b7cc7af51 100644 --- a/app/Notifications/Server/Unreachable.php +++ b/app/Notifications/Server/Unreachable.php @@ -35,13 +35,13 @@ class Unreachable extends Notification implements ShouldQueue public function toDiscord(): string { - $message = "⛔ Server '{$this->server->name}' is unreachable after trying to connect to it 5 times. All automations & integrations are turned off! Please check your server! IMPORTANT: You have to validate your server again after you fix the issue."; + $message = "⛔ Server '{$this->server->name}' is unreachable after trying to connect to it 5 times. All automations & integrations are turned off! Please check your server! IMPORTANT: We automatically try to revive your server. If your server is back online, we will automatically turn on all automations & integrations."; return $message; } public function toTelegram(): array { return [ - "message" => "⛔ Server '{$this->server->name}' is unreachable after trying to connect to it 5 times. All automations & integrations are turned off! Please check your server! IMPORTANT: You have to validate your server again after you fix the issue." + "message" => "⛔ Server '{$this->server->name}' is unreachable after trying to connect to it 5 times. All automations & integrations are turned off! Please check your server! IMPORTANT: We automatically try to revive your server. If your server is back online, we will automatically turn on all automations & integrations." ]; } } diff --git a/bootstrap/helpers/remoteProcess.php b/bootstrap/helpers/remoteProcess.php index e5a2a5863..8aa22d6f2 100644 --- a/bootstrap/helpers/remoteProcess.php +++ b/bootstrap/helpers/remoteProcess.php @@ -7,6 +7,8 @@ use App\Models\Application; use App\Models\ApplicationDeploymentQueue; use App\Models\PrivateKey; use App\Models\Server; +use App\Notifications\Server\Revived; +use App\Notifications\Server\Unreachable; use Carbon\Carbon; use Illuminate\Database\Eloquent\Model; use Illuminate\Support\Collection; @@ -85,7 +87,7 @@ function generateSshCommand(Server $server, string $command, bool $isMux = true) if ($isMux && config('coolify.mux_enabled')) { $ssh_command .= '-o ControlMaster=auto -o ControlPersist=1m -o ControlPath=/var/www/html/storage/app/ssh/mux/%h_%p_%r '; } - if (data_get($server,'settings.is_cloudflare_tunnel')) { + if (data_get($server, 'settings.is_cloudflare_tunnel')) { $ssh_command .= '-o ProxyCommand="/usr/local/bin/cloudflared access ssh --hostname %h" '; } $command = "PATH=\$PATH:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/host/usr/local/sbin:/host/usr/local/bin:/host/usr/sbin:/host/usr/bin:/host/sbin:/host/bin && $command"; @@ -122,13 +124,14 @@ function instant_remote_process(Collection|array $command, Server $server, $thro } return $output; } -function excludeCertainErrors(string $errorOutput, ?int $exitCode = null) { +function excludeCertainErrors(string $errorOutput, ?int $exitCode = null) +{ $ignoredErrors = collect([ 'Permission denied (publickey', 'Could not resolve hostname', ]); $ignored = false; - foreach ($ignoredErrors as $ignoredError) { + foreach ($ignoredErrors as $ignoredError) { if (Str::contains($errorOutput, $ignoredError)) { $ignored = true; break; @@ -183,6 +186,9 @@ function validateServer(Server $server, bool $throwError = false) $uptime = instant_remote_process(['uptime'], $server, $throwError); if (!$uptime) { $server->settings->is_reachable = false; + $server->team->notify(new Unreachable($server)); + $server->unreachable_email_sent = true; + $server->save(); return [ "uptime" => null, "dockerVersion" => null, @@ -203,6 +209,11 @@ function validateServer(Server $server, bool $throwError = false) $server->settings->is_usable = false; } else { $server->settings->is_usable = true; + if (data_get($server, 'unreachable_email_sent') === true) { + $server->team->notify(new Revived($server)); + $server->unreachable_email_sent = false; + $server->save(); + } } return [ "uptime" => $uptime, @@ -213,7 +224,9 @@ function validateServer(Server $server, bool $throwError = false) $server->settings->is_usable = false; throw $e; } finally { - if (data_get($server, 'settings')) $server->settings->save(); + if (data_get($server, 'settings')) { + $server->settings->save(); + } } } diff --git a/bootstrap/helpers/shared.php b/bootstrap/helpers/shared.php index a12d6b02d..25237abab 100644 --- a/bootstrap/helpers/shared.php +++ b/bootstrap/helpers/shared.php @@ -310,6 +310,7 @@ function send_internal_notification(string $message): void $baseUrl = config('app.name'); $team = Team::find(0); $team->notify(new GeneralNotification("👀 {$baseUrl}: " . $message)); + ray("👀 {$baseUrl}: " . $message); } catch (\Throwable $e) { ray($e->getMessage()); } diff --git a/database/migrations/2023_09_23_111819_add_server_emails.php b/database/migrations/2023_09_23_111819_add_server_emails.php new file mode 100644 index 000000000..03c1e6bd2 --- /dev/null +++ b/database/migrations/2023_09_23_111819_add_server_emails.php @@ -0,0 +1,31 @@ +boolean('unreachable_email_sent')->default(false); + $table->dropColumn('unreachable_count'); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('servers', function (Blueprint $table) { + $table->dropColumn('unreachable_email_sent'); + $table->integer('unreachable_count')->default(0); + }); + + } +}; diff --git a/resources/views/emails/server-lost-connection.blade.php b/resources/views/emails/server-lost-connection.blade.php index 83b5e2db5..6e3f28fd4 100644 --- a/resources/views/emails/server-lost-connection.blade.php +++ b/resources/views/emails/server-lost-connection.blade.php @@ -4,7 +4,7 @@ Coolify cannot connect to your server ({{$name}}). Please check your server and All automations & integrations are turned off! -IMPORTANT: You have to validate your server again after you fix the issue. +IMPORTANT: We automatically try to revive your server. If your server is back online, we will automatically turn on all automations & integrations. If you have any questions, please contact us. diff --git a/resources/views/emails/server-revived.blade.php b/resources/views/emails/server-revived.blade.php new file mode 100644 index 000000000..8d16fd2c8 --- /dev/null +++ b/resources/views/emails/server-revived.blade.php @@ -0,0 +1,6 @@ + + +Your server ({{$name}}) was offline for a while, but it is back online now. All automations & integrations are turned on again. + + +