fix: better unreachable/revived server statuses
This commit is contained in:
33
app/Console/Commands/Cloud.php
Normal file
33
app/Console/Commands/Cloud.php
Normal file
@@ -0,0 +1,33 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
|
||||
use App\Models\Server;
|
||||
use Illuminate\Console\Command;
|
||||
|
||||
class Cloud extends Command
|
||||
{
|
||||
/**
|
||||
* The name and signature of the console command.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $signature = 'cloud:unused-servers';
|
||||
|
||||
/**
|
||||
* The console command description.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $description = 'Get Unused Servers from Cloud';
|
||||
|
||||
/**
|
||||
* Execute the console command.
|
||||
*/
|
||||
public function handle()
|
||||
{
|
||||
Server::all()->whereNotNull('team.subscription')->where('team.subscription.stripe_trial_already_ended',true)->each(function($server){
|
||||
$this->info($server->name);
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -48,7 +48,11 @@ class Kernel extends ConsoleKernel
|
||||
}
|
||||
private function check_resources($schedule)
|
||||
{
|
||||
$servers = Server::all()->where('settings.is_usable', true)->where('settings.is_reachable', true);
|
||||
if (isCloud()) {
|
||||
$servers = Server::all()->whereNotNull('team.subscription')->where('team.subscription.stripe_trial_already_ended', false);
|
||||
} else {
|
||||
$servers = Server::all();
|
||||
}
|
||||
foreach ($servers as $server) {
|
||||
$schedule->job(new ContainerStatusJob($server))->everyMinute()->onOneServer();
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ use App\Models\ApplicationPreview;
|
||||
use App\Models\Server;
|
||||
use App\Notifications\Container\ContainerRestarted;
|
||||
use App\Notifications\Container\ContainerStopped;
|
||||
use App\Notifications\Server\Revived;
|
||||
use App\Notifications\Server\Unreachable;
|
||||
use Illuminate\Bus\Queueable;
|
||||
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
|
||||
@@ -51,14 +52,20 @@ class ContainerStatusJob implements ShouldQueue, ShouldBeEncrypted
|
||||
public function handle(): void
|
||||
{
|
||||
try {
|
||||
ray("checking server status for {$this->server->name}");
|
||||
// ray()->clearAll();
|
||||
$serverUptimeCheckNumber = 0;
|
||||
$serverUptimeCheckNumberMax = 3;
|
||||
while (true) {
|
||||
if ($serverUptimeCheckNumber >= $serverUptimeCheckNumberMax) {
|
||||
send_internal_notification('Server unreachable: ' . $this->server->name);
|
||||
// $this->server->settings()->update(['is_reachable' => false]);
|
||||
// $this->server->team->notify(new Unreachable($this->server));
|
||||
if ($this->server->unreachable_email_sent === false) {
|
||||
$this->server->team->notify(new Unreachable($this->server));
|
||||
}
|
||||
$this->server->settings()->update([
|
||||
'is_reachable' => false,
|
||||
]);
|
||||
$this->server->update(['unreachable_email_sent' => true]);
|
||||
return;
|
||||
}
|
||||
$result = $this->checkServerConnection();
|
||||
@@ -68,6 +75,20 @@ class ContainerStatusJob implements ShouldQueue, ShouldBeEncrypted
|
||||
$serverUptimeCheckNumber++;
|
||||
sleep(5);
|
||||
}
|
||||
if (data_get($this->server, 'unreachable_email_sent') === true) {
|
||||
$this->server->team->notify(new Revived($this->server));
|
||||
$this->server->update(['unreachable_email_sent' => false]);
|
||||
}
|
||||
if (
|
||||
data_get($this->server, 'settings.is_reachable') === false ||
|
||||
data_get($this->server, 'settings.is_usable') === false
|
||||
) {
|
||||
$this->server->settings()->update([
|
||||
'is_reachable' => true,
|
||||
'is_usable' => true
|
||||
]);
|
||||
}
|
||||
|
||||
$containers = instant_remote_process(["docker container ls -q"], $this->server);
|
||||
if (!$containers) {
|
||||
return;
|
||||
|
||||
@@ -8,6 +8,7 @@ use Illuminate\Database\Eloquent\Builder;
|
||||
use Illuminate\Database\Eloquent\Casts\Attribute;
|
||||
use Spatie\SchemalessAttributes\Casts\SchemalessAttributes;
|
||||
use Spatie\SchemalessAttributes\SchemalessAttributesTrait;
|
||||
use Illuminate\Support\Str;
|
||||
|
||||
class Server extends BaseModel
|
||||
{
|
||||
@@ -15,6 +16,11 @@ class Server extends BaseModel
|
||||
|
||||
protected static function booted()
|
||||
{
|
||||
static::saved(function ($server) {
|
||||
$server->ip = Str::of($server->ip)->trim();
|
||||
$server->user = Str::of($server->user)->trim();
|
||||
});
|
||||
|
||||
static::created(function ($server) {
|
||||
ServerSetting::create([
|
||||
'server_id' => $server->id,
|
||||
|
||||
49
app/Notifications/Server/Revived.php
Normal file
49
app/Notifications/Server/Revived.php
Normal file
@@ -0,0 +1,49 @@
|
||||
<?php
|
||||
|
||||
namespace App\Notifications\Server;
|
||||
|
||||
use App\Models\Server;
|
||||
use Illuminate\Bus\Queueable;
|
||||
use Illuminate\Contracts\Queue\ShouldQueue;
|
||||
use Illuminate\Notifications\Messages\MailMessage;
|
||||
use Illuminate\Notifications\Notification;
|
||||
|
||||
class Revived extends Notification implements ShouldQueue
|
||||
{
|
||||
use Queueable;
|
||||
|
||||
public $tries = 1;
|
||||
public function __construct(public Server $server)
|
||||
{
|
||||
if ($this->server->unreachable_email_sent === false) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
public function via(object $notifiable): array
|
||||
{
|
||||
return setNotificationChannels($notifiable, 'status_changes');
|
||||
}
|
||||
|
||||
public function toMail(): MailMessage
|
||||
{
|
||||
$mail = new MailMessage();
|
||||
$mail->subject("✅ Server ({$this->server->name}) revived.");
|
||||
$mail->view('emails.server-revived', [
|
||||
'name' => $this->server->name,
|
||||
]);
|
||||
return $mail;
|
||||
}
|
||||
|
||||
public function toDiscord(): string
|
||||
{
|
||||
$message = "✅ Server '{$this->server->name}' revived. All automations & integrations are turned on again!";
|
||||
return $message;
|
||||
}
|
||||
public function toTelegram(): array
|
||||
{
|
||||
return [
|
||||
"message" => "✅ Server '{$this->server->name}' revived. All automations & integrations are turned on again!"
|
||||
];
|
||||
}
|
||||
}
|
||||
@@ -35,13 +35,13 @@ class Unreachable extends Notification implements ShouldQueue
|
||||
|
||||
public function toDiscord(): string
|
||||
{
|
||||
$message = "⛔ Server '{$this->server->name}' is unreachable after trying to connect to it 5 times. All automations & integrations are turned off! Please check your server! IMPORTANT: You have to validate your server again after you fix the issue.";
|
||||
$message = "⛔ Server '{$this->server->name}' is unreachable after trying to connect to it 5 times. All automations & integrations are turned off! Please check your server! IMPORTANT: We automatically try to revive your server. If your server is back online, we will automatically turn on all automations & integrations.";
|
||||
return $message;
|
||||
}
|
||||
public function toTelegram(): array
|
||||
{
|
||||
return [
|
||||
"message" => "⛔ Server '{$this->server->name}' is unreachable after trying to connect to it 5 times. All automations & integrations are turned off! Please check your server! IMPORTANT: You have to validate your server again after you fix the issue."
|
||||
"message" => "⛔ Server '{$this->server->name}' is unreachable after trying to connect to it 5 times. All automations & integrations are turned off! Please check your server! IMPORTANT: We automatically try to revive your server. If your server is back online, we will automatically turn on all automations & integrations."
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user