fix: better unreachable/revived server statuses

This commit is contained in:
Andras Bacsai
2023-10-07 00:51:01 +02:00
parent c58706e3e4
commit 9e81416fef
11 changed files with 174 additions and 10 deletions

View File

@@ -0,0 +1,33 @@
<?php
namespace App\Console\Commands;
use App\Models\Server;
use Illuminate\Console\Command;
class Cloud extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'cloud:unused-servers';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Get Unused Servers from Cloud';
/**
* Execute the console command.
*/
public function handle()
{
Server::all()->whereNotNull('team.subscription')->where('team.subscription.stripe_trial_already_ended',true)->each(function($server){
$this->info($server->name);
});
}
}

View File

@@ -48,7 +48,11 @@ class Kernel extends ConsoleKernel
}
private function check_resources($schedule)
{
$servers = Server::all()->where('settings.is_usable', true)->where('settings.is_reachable', true);
if (isCloud()) {
$servers = Server::all()->whereNotNull('team.subscription')->where('team.subscription.stripe_trial_already_ended', false);
} else {
$servers = Server::all();
}
foreach ($servers as $server) {
$schedule->job(new ContainerStatusJob($server))->everyMinute()->onOneServer();
}

View File

@@ -7,6 +7,7 @@ use App\Models\ApplicationPreview;
use App\Models\Server;
use App\Notifications\Container\ContainerRestarted;
use App\Notifications\Container\ContainerStopped;
use App\Notifications\Server\Revived;
use App\Notifications\Server\Unreachable;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
@@ -51,14 +52,20 @@ class ContainerStatusJob implements ShouldQueue, ShouldBeEncrypted
public function handle(): void
{
try {
ray("checking server status for {$this->server->name}");
// ray()->clearAll();
$serverUptimeCheckNumber = 0;
$serverUptimeCheckNumberMax = 3;
while (true) {
if ($serverUptimeCheckNumber >= $serverUptimeCheckNumberMax) {
send_internal_notification('Server unreachable: ' . $this->server->name);
// $this->server->settings()->update(['is_reachable' => false]);
// $this->server->team->notify(new Unreachable($this->server));
if ($this->server->unreachable_email_sent === false) {
$this->server->team->notify(new Unreachable($this->server));
}
$this->server->settings()->update([
'is_reachable' => false,
]);
$this->server->update(['unreachable_email_sent' => true]);
return;
}
$result = $this->checkServerConnection();
@@ -68,6 +75,20 @@ class ContainerStatusJob implements ShouldQueue, ShouldBeEncrypted
$serverUptimeCheckNumber++;
sleep(5);
}
if (data_get($this->server, 'unreachable_email_sent') === true) {
$this->server->team->notify(new Revived($this->server));
$this->server->update(['unreachable_email_sent' => false]);
}
if (
data_get($this->server, 'settings.is_reachable') === false ||
data_get($this->server, 'settings.is_usable') === false
) {
$this->server->settings()->update([
'is_reachable' => true,
'is_usable' => true
]);
}
$containers = instant_remote_process(["docker container ls -q"], $this->server);
if (!$containers) {
return;

View File

@@ -8,6 +8,7 @@ use Illuminate\Database\Eloquent\Builder;
use Illuminate\Database\Eloquent\Casts\Attribute;
use Spatie\SchemalessAttributes\Casts\SchemalessAttributes;
use Spatie\SchemalessAttributes\SchemalessAttributesTrait;
use Illuminate\Support\Str;
class Server extends BaseModel
{
@@ -15,6 +16,11 @@ class Server extends BaseModel
protected static function booted()
{
static::saved(function ($server) {
$server->ip = Str::of($server->ip)->trim();
$server->user = Str::of($server->user)->trim();
});
static::created(function ($server) {
ServerSetting::create([
'server_id' => $server->id,

View File

@@ -0,0 +1,49 @@
<?php
namespace App\Notifications\Server;
use App\Models\Server;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Notifications\Messages\MailMessage;
use Illuminate\Notifications\Notification;
class Revived extends Notification implements ShouldQueue
{
use Queueable;
public $tries = 1;
public function __construct(public Server $server)
{
if ($this->server->unreachable_email_sent === false) {
return;
}
}
public function via(object $notifiable): array
{
return setNotificationChannels($notifiable, 'status_changes');
}
public function toMail(): MailMessage
{
$mail = new MailMessage();
$mail->subject("✅ Server ({$this->server->name}) revived.");
$mail->view('emails.server-revived', [
'name' => $this->server->name,
]);
return $mail;
}
public function toDiscord(): string
{
$message = "✅ Server '{$this->server->name}' revived. All automations & integrations are turned on again!";
return $message;
}
public function toTelegram(): array
{
return [
"message" => "✅ Server '{$this->server->name}' revived. All automations & integrations are turned on again!"
];
}
}

View File

@@ -35,13 +35,13 @@ class Unreachable extends Notification implements ShouldQueue
public function toDiscord(): string
{
$message = "⛔ Server '{$this->server->name}' is unreachable after trying to connect to it 5 times. All automations & integrations are turned off! Please check your server! IMPORTANT: You have to validate your server again after you fix the issue.";
$message = "⛔ Server '{$this->server->name}' is unreachable after trying to connect to it 5 times. All automations & integrations are turned off! Please check your server! IMPORTANT: We automatically try to revive your server. If your server is back online, we will automatically turn on all automations & integrations.";
return $message;
}
public function toTelegram(): array
{
return [
"message" => "⛔ Server '{$this->server->name}' is unreachable after trying to connect to it 5 times. All automations & integrations are turned off! Please check your server! IMPORTANT: You have to validate your server again after you fix the issue."
"message" => "⛔ Server '{$this->server->name}' is unreachable after trying to connect to it 5 times. All automations & integrations are turned off! Please check your server! IMPORTANT: We automatically try to revive your server. If your server is back online, we will automatically turn on all automations & integrations."
];
}
}