fix: force enable/disable server in case ultimate package quantity decreases

This commit is contained in:
Andras Bacsai
2024-02-26 10:25:21 +01:00
parent 453956172b
commit 678647f39a
25 changed files with 172 additions and 68 deletions

View File

@@ -14,7 +14,7 @@ use Illuminate\Http\Request;
use Illuminate\Support\Collection;
use Visus\Cuid2\Cuid2;
class Deploy extends Controller
class APIDeploy extends Controller
{
public function deploy(Request $request)
{

View File

@@ -6,7 +6,7 @@ use App\Http\Controllers\Controller;
use App\Models\Project as ModelsProject;
use Illuminate\Http\Request;
class Project extends Controller
class APIProject extends Controller
{
public function projects(Request $request)
{

View File

@@ -6,7 +6,7 @@ use App\Http\Controllers\Controller;
use App\Models\Server as ModelsServer;
use Illuminate\Http\Request;
class Server extends Controller
class APIServer extends Controller
{
public function servers(Request $request)
{

View File

@@ -1675,7 +1675,6 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf");
);
}
}
$this->next(ApplicationDeploymentStatus::FAILED->value);
}
}

View File

@@ -3,7 +3,8 @@
namespace App\Jobs;
use App\Models\Team;
use App\Notifications\Server\DisabledDueToOverflow;
use App\Notifications\Server\ForceDisabled;
use App\Notifications\Server\ForceEnabled;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
use Illuminate\Contracts\Queue\ShouldQueue;
@@ -12,7 +13,7 @@ use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Queue\SerializesModels;
class ServerOverflowJob implements ShouldQueue, ShouldBeEncrypted
class ServerLimitCheckJob implements ShouldQueue, ShouldBeEncrypted
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
@@ -37,26 +38,31 @@ class ServerOverflowJob implements ShouldQueue, ShouldBeEncrypted
public function handle()
{
try {
ray('ServerOverflowJob');
$servers = $this->team->servers;
$servers_count = $servers->count();
$limit = $this->team->limits['serverLimit'];
$number_of_servers_to_disable = $servers_count - $limit;
ray($number_of_servers_to_disable, $servers_count, $limit);
ray('ServerLimitCheckJob', $this->team->uuid, $servers_count, $limit, $number_of_servers_to_disable);
if ($number_of_servers_to_disable > 0) {
ray('Disabling servers');
$servers = $servers->sortBy('created_at');
$servers = $servers->sortbyDesc('created_at');
$servers_to_disable = $servers->take($number_of_servers_to_disable);
$servers_to_disable->each(function ($server) {
$server->disableServerDueToOverflow();
$this->team->notify(new DisabledDueToOverflow($server));
$server->forceDisableServer();
$this->team->notify(new ForceDisabled($server));
});
} else if ($number_of_servers_to_disable === 0) {
$servers->each(function ($server) {
if ($server->isForceDisabled()) {
$server->forceEnableServer();
$this->team->notify(new ForceEnabled($server));
}
});
}
} catch (\Throwable $e) {
send_internal_notification('ServerOverflowJob failed with: ' . $e->getMessage());
send_internal_notification('ServerLimitCheckJob failed with: ' . $e->getMessage());
ray($e->getMessage());
return handleError($e);
}
}
}

View File

@@ -3,6 +3,7 @@
namespace App\Livewire\Admin;
use App\Models\User;
use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\Crypt;
use Livewire\Component;
@@ -27,6 +28,7 @@ class Index extends Component
auth()->login($user);
if ($user_id === 0) {
Cache::forget('team:0');
session()->forget('adminToken');
} else {
$token_payload = [
@@ -35,6 +37,7 @@ class Index extends Component
$token = Crypt::encrypt($token_payload);
session(['adminToken' => $token]);
}
session()->regenerate();
return refreshSession();
}
public function render()

View File

@@ -2,7 +2,7 @@
namespace App\Livewire\Tags;
use App\Http\Controllers\Api\Deploy;
use App\Http\Controllers\Api\APIDeploy as Deploy;
use App\Models\ApplicationDeploymentQueue;
use App\Models\Tag;
use Livewire\Component;

View File

@@ -10,6 +10,7 @@ use App\Notifications\Server\Unreachable;
use Illuminate\Database\Eloquent\Builder;
use Illuminate\Database\Eloquent\Casts\Attribute;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Storage;
use Spatie\SchemalessAttributes\Casts\SchemalessAttributes;
use Spatie\SchemalessAttributes\SchemalessAttributesTrait;
use Illuminate\Support\Str;
@@ -69,7 +70,7 @@ class Server extends BaseModel
static public function isUsable()
{
return Server::ownedByCurrentTeam()->whereRelation('settings', 'is_reachable', true)->whereRelation('settings', 'is_usable', true)->whereRelation('settings', 'is_swarm_worker', false)->whereRelation('settings', 'is_build_server', false);
return Server::ownedByCurrentTeam()->whereRelation('settings', 'is_reachable', true)->whereRelation('settings', 'is_usable', true)->whereRelation('settings', 'is_swarm_worker', false)->whereRelation('settings', 'is_build_server', false)->whereRelation('settings', 'force_disabled', false);
}
static public function destinationsByServer(string $server_id)
@@ -149,13 +150,31 @@ class Server extends BaseModel
ray('skipping 1.2.3.4');
return true;
}
if ($this->settings->force_disabled === true) {
ray('force_disabled');
return true;
}
return false;
}
public function disableServerDueToOverflow() {
public function isForceDisabled()
{
return $this->settings->force_disabled;
}
public function forceEnableServer()
{
$this->settings->update([
'disabled_by_overflow' => true,
'force_disabled' => false,
]);
}
public function forceDisableServer()
{
$this->settings->update([
'force_disabled' => true,
]);
$sshKeyFileLocation = "id.root@{$this->uuid}";
Storage::disk('ssh-keys')->delete($sshKeyFileLocation);
Storage::disk('ssh-mux')->delete($this->muxFilename());
}
public function isServerReady(int $tries = 3)
{
if ($this->skipServer()) {
@@ -379,7 +398,7 @@ class Server extends BaseModel
}
public function isFunctional()
{
return $this->settings->is_reachable && $this->settings->is_usable;
return $this->settings->is_reachable && $this->settings->is_usable && !$this->settings->force_disabled;
}
public function isLogDrainEnabled()
{

View File

@@ -11,7 +11,7 @@ use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Notifications\Messages\MailMessage;
use Illuminate\Notifications\Notification;
class DisabledDueToOverflow extends Notification implements ShouldQueue
class ForceDisabled extends Notification implements ShouldQueue
{
use Queueable;
@@ -43,7 +43,7 @@ class DisabledDueToOverflow extends Notification implements ShouldQueue
{
$mail = new MailMessage();
$mail->subject("Coolify: Server ({$this->server->name}) disabled because it is not paid!");
$mail->view('emails.server-disabled-due-to-overflow', [
$mail->view('emails.server-force-disabled', [
'name' => $this->server->name,
]);
return $mail;

View File

@@ -0,0 +1,63 @@
<?php
namespace App\Notifications\Server;
use App\Models\Server;
use Illuminate\Bus\Queueable;
use App\Notifications\Channels\DiscordChannel;
use App\Notifications\Channels\EmailChannel;
use App\Notifications\Channels\TelegramChannel;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Notifications\Messages\MailMessage;
use Illuminate\Notifications\Notification;
class ForceEnabled extends Notification implements ShouldQueue
{
use Queueable;
public $tries = 1;
public function __construct(public Server $server)
{
}
public function via(object $notifiable): array
{
$channels = [];
$isEmailEnabled = isEmailEnabled($notifiable);
$isDiscordEnabled = data_get($notifiable, 'discord_enabled');
$isTelegramEnabled = data_get($notifiable, 'telegram_enabled');
if ($isDiscordEnabled) {
$channels[] = DiscordChannel::class;
}
if ($isEmailEnabled) {
$channels[] = EmailChannel::class;
}
if ($isTelegramEnabled) {
$channels[] = TelegramChannel::class;
}
return $channels;
}
public function toMail(): MailMessage
{
$mail = new MailMessage();
$mail->subject("Coolify: Server ({$this->server->name}) enabled again!");
$mail->view('emails.server-force-enabled', [
'name' => $this->server->name,
]);
return $mail;
}
public function toDiscord(): string
{
$message = "Coolify: Server ({$this->server->name}) enabled again!";
return $message;
}
public function toTelegram(): array
{
return [
"message" => "Coolify: Server ({$this->server->name}) enabled again!"
];
}
}

View File

@@ -24,6 +24,12 @@ trait ExecuteRemoteCommand
if ($this->server instanceof Server === false) {
throw new \RuntimeException('Server is not set or is not an instance of Server model');
}
if ($this->server->settings->force_disabled) {
$this->application_deployment_queue->update([
'status' => ApplicationDeploymentStatus::FAILED->value,
]);
throw new \RuntimeException('Server is disabled');
}
$commandsText->each(function ($single_command) {
$command = data_get($single_command, 'command') ?? $single_command[0] ?? null;
if ($command === null) {