@@ -2,13 +2,12 @@
|
|||||||
|
|
||||||
namespace App\Console;
|
namespace App\Console;
|
||||||
|
|
||||||
use App\Jobs\CheckResaleLicenseJob;
|
|
||||||
use App\Jobs\CleanupInstanceStuffsJob;
|
use App\Jobs\CleanupInstanceStuffsJob;
|
||||||
use App\Jobs\DatabaseBackupJob;
|
use App\Jobs\DatabaseBackupJob;
|
||||||
use App\Jobs\DockerCleanupJob;
|
|
||||||
use App\Jobs\InstanceAutoUpdateJob;
|
use App\Jobs\InstanceAutoUpdateJob;
|
||||||
use App\Jobs\ContainerStatusJob;
|
use App\Jobs\ContainerStatusJob;
|
||||||
use App\Jobs\PullHelperImageJob;
|
use App\Jobs\PullHelperImageJob;
|
||||||
|
use App\Jobs\ServerStatusJob;
|
||||||
use App\Models\InstanceSettings;
|
use App\Models\InstanceSettings;
|
||||||
use App\Models\ScheduledDatabaseBackup;
|
use App\Models\ScheduledDatabaseBackup;
|
||||||
use App\Models\Server;
|
use App\Models\Server;
|
||||||
@@ -27,7 +26,6 @@ class Kernel extends ConsoleKernel
|
|||||||
// Server Jobs
|
// Server Jobs
|
||||||
$this->check_scheduled_backups($schedule);
|
$this->check_scheduled_backups($schedule);
|
||||||
$this->check_resources($schedule);
|
$this->check_resources($schedule);
|
||||||
$this->cleanup_servers($schedule);
|
|
||||||
$this->check_scheduled_backups($schedule);
|
$this->check_scheduled_backups($schedule);
|
||||||
$this->pull_helper_image($schedule);
|
$this->pull_helper_image($schedule);
|
||||||
} else {
|
} else {
|
||||||
@@ -40,7 +38,6 @@ class Kernel extends ConsoleKernel
|
|||||||
$this->instance_auto_update($schedule);
|
$this->instance_auto_update($schedule);
|
||||||
$this->check_scheduled_backups($schedule);
|
$this->check_scheduled_backups($schedule);
|
||||||
$this->check_resources($schedule);
|
$this->check_resources($schedule);
|
||||||
$this->cleanup_servers($schedule);
|
|
||||||
$this->pull_helper_image($schedule);
|
$this->pull_helper_image($schedule);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -51,13 +48,6 @@ class Kernel extends ConsoleKernel
|
|||||||
$schedule->job(new PullHelperImageJob($server))->everyTenMinutes()->onOneServer();
|
$schedule->job(new PullHelperImageJob($server))->everyTenMinutes()->onOneServer();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
private function cleanup_servers($schedule)
|
|
||||||
{
|
|
||||||
$servers = Server::all()->where('settings.is_usable', true)->where('settings.is_reachable', true);
|
|
||||||
foreach ($servers as $server) {
|
|
||||||
$schedule->job(new DockerCleanupJob($server))->everyTenMinutes()->onOneServer();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
private function check_resources($schedule)
|
private function check_resources($schedule)
|
||||||
{
|
{
|
||||||
if (isCloud()) {
|
if (isCloud()) {
|
||||||
@@ -66,6 +56,7 @@ class Kernel extends ConsoleKernel
|
|||||||
$servers = Server::all();
|
$servers = Server::all();
|
||||||
}
|
}
|
||||||
foreach ($servers as $server) {
|
foreach ($servers as $server) {
|
||||||
|
$schedule->job(new ServerStatusJob($server))->everyTenMinutes()->onOneServer();
|
||||||
$schedule->job(new ContainerStatusJob($server))->everyMinute()->onOneServer();
|
$schedule->job(new ContainerStatusJob($server))->everyMinute()->onOneServer();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@@ -33,6 +33,8 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted
|
|||||||
{
|
{
|
||||||
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels, ExecuteRemoteCommand;
|
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels, ExecuteRemoteCommand;
|
||||||
|
|
||||||
|
public $timeout = 3600;
|
||||||
|
|
||||||
public static int $batch_counter = 0;
|
public static int $batch_counter = 0;
|
||||||
|
|
||||||
private int $application_deployment_queue_id;
|
private int $application_deployment_queue_id;
|
||||||
@@ -827,6 +829,13 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted
|
|||||||
'networks' => [
|
'networks' => [
|
||||||
$this->destination->network,
|
$this->destination->network,
|
||||||
],
|
],
|
||||||
|
// 'logging' => [
|
||||||
|
// 'driver' => 'fluentd',
|
||||||
|
// 'options' => [
|
||||||
|
// 'fluentd-async' => 'true',
|
||||||
|
// 'tag' => $this->application->name . '-' . $this->application->uuid
|
||||||
|
// ]
|
||||||
|
// ],
|
||||||
'healthcheck' => [
|
'healthcheck' => [
|
||||||
'test' => [
|
'test' => [
|
||||||
'CMD-SHELL',
|
'CMD-SHELL',
|
||||||
@@ -1089,21 +1098,19 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf");
|
|||||||
|
|
||||||
private function start_by_compose_file()
|
private function start_by_compose_file()
|
||||||
{
|
{
|
||||||
if (
|
if ($this->application->dockerfile || $this->application->build_pack === 'dockerfile') {
|
||||||
!$this->application->dockerfile &&
|
$this->execute_remote_command(
|
||||||
(
|
["echo -n 'Starting application (could take a while).'"],
|
||||||
$this->application->build_pack === 'dockerimage' ||
|
[executeInDocker($this->deployment_uuid, "docker compose --project-directory {$this->workdir} up --build -d"), "hidden" => true],
|
||||||
$this->application->build_pack === 'dockerfile')
|
);
|
||||||
) {
|
} else if ($this->application->build_pack === 'dockerimage') {
|
||||||
$this->execute_remote_command(
|
$this->execute_remote_command(
|
||||||
["echo -n 'Pulling latest images from the registry.'"],
|
["echo -n 'Pulling latest images from the registry.'"],
|
||||||
[executeInDocker($this->deployment_uuid, "docker compose --project-directory {$this->workdir}"), "hidden" => true],
|
[executeInDocker($this->deployment_uuid, "docker compose --project-directory {$this->workdir} pull"), "hidden" => true],
|
||||||
|
["echo -n 'Starting application (could take a while).'"],
|
||||||
|
[executeInDocker($this->deployment_uuid, "docker compose --project-directory {$this->workdir} up --build -d"), "hidden" => true],
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
$this->execute_remote_command(
|
|
||||||
["echo -n 'Starting application (could take a while).'"],
|
|
||||||
[executeInDocker($this->deployment_uuid, "docker compose --project-directory {$this->workdir} up --build -d"), "hidden" => true],
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private function generate_build_env_variables()
|
private function generate_build_env_variables()
|
||||||
|
@@ -8,8 +8,6 @@ use App\Models\ApplicationPreview;
|
|||||||
use App\Models\Server;
|
use App\Models\Server;
|
||||||
use App\Notifications\Container\ContainerRestarted;
|
use App\Notifications\Container\ContainerRestarted;
|
||||||
use App\Notifications\Container\ContainerStopped;
|
use App\Notifications\Container\ContainerStopped;
|
||||||
use App\Notifications\Server\Revived;
|
|
||||||
use App\Notifications\Server\Unreachable;
|
|
||||||
use Illuminate\Bus\Queueable;
|
use Illuminate\Bus\Queueable;
|
||||||
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
|
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
|
||||||
use Illuminate\Contracts\Queue\ShouldQueue;
|
use Illuminate\Contracts\Queue\ShouldQueue;
|
||||||
@@ -41,76 +39,7 @@ class ContainerStatusJob implements ShouldQueue, ShouldBeEncrypted
|
|||||||
{
|
{
|
||||||
// ray("checking server status for {$this->server->id}");
|
// ray("checking server status for {$this->server->id}");
|
||||||
try {
|
try {
|
||||||
// ray()->clearAll();
|
$this->server->checkServerRediness();
|
||||||
$serverUptimeCheckNumber = $this->server->unreachable_count;
|
|
||||||
$serverUptimeCheckNumberMax = 3;
|
|
||||||
|
|
||||||
// ray('checking # ' . $serverUptimeCheckNumber);
|
|
||||||
if ($serverUptimeCheckNumber >= $serverUptimeCheckNumberMax) {
|
|
||||||
if ($this->server->unreachable_email_sent === false) {
|
|
||||||
ray('Server unreachable, sending notification...');
|
|
||||||
$this->server->team->notify(new Unreachable($this->server));
|
|
||||||
$this->server->update(['unreachable_email_sent' => true]);
|
|
||||||
}
|
|
||||||
$this->server->settings()->update([
|
|
||||||
'is_reachable' => false,
|
|
||||||
]);
|
|
||||||
$this->server->update([
|
|
||||||
'unreachable_count' => 0,
|
|
||||||
]);
|
|
||||||
// Update all applications, databases and services to exited
|
|
||||||
foreach ($this->server->applications() as $application) {
|
|
||||||
$application->update(['status' => 'exited']);
|
|
||||||
}
|
|
||||||
foreach ($this->server->databases() as $database) {
|
|
||||||
$database->update(['status' => 'exited']);
|
|
||||||
}
|
|
||||||
foreach ($this->server->services() as $service) {
|
|
||||||
$apps = $service->applications()->get();
|
|
||||||
$dbs = $service->databases()->get();
|
|
||||||
foreach ($apps as $app) {
|
|
||||||
$app->update(['status' => 'exited']);
|
|
||||||
}
|
|
||||||
foreach ($dbs as $db) {
|
|
||||||
$db->update(['status' => 'exited']);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
$result = $this->server->validateConnection();
|
|
||||||
if ($result) {
|
|
||||||
$this->server->settings()->update([
|
|
||||||
'is_reachable' => true,
|
|
||||||
]);
|
|
||||||
$this->server->update([
|
|
||||||
'unreachable_count' => 0,
|
|
||||||
]);
|
|
||||||
} else {
|
|
||||||
$serverUptimeCheckNumber++;
|
|
||||||
$this->server->settings()->update([
|
|
||||||
'is_reachable' => false,
|
|
||||||
]);
|
|
||||||
$this->server->update([
|
|
||||||
'unreachable_count' => $serverUptimeCheckNumber,
|
|
||||||
]);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (data_get($this->server, 'unreachable_email_sent') === true) {
|
|
||||||
ray('Server is reachable again, sending notification...');
|
|
||||||
$this->server->team->notify(new Revived($this->server));
|
|
||||||
$this->server->update(['unreachable_email_sent' => false]);
|
|
||||||
}
|
|
||||||
if (
|
|
||||||
data_get($this->server, 'settings.is_reachable') === false ||
|
|
||||||
data_get($this->server, 'settings.is_usable') === false
|
|
||||||
) {
|
|
||||||
$this->server->settings()->update([
|
|
||||||
'is_reachable' => true,
|
|
||||||
'is_usable' => true
|
|
||||||
]);
|
|
||||||
}
|
|
||||||
// $this->server->validateDockerEngine(true);
|
|
||||||
$containers = instant_remote_process(["docker container ls -q"], $this->server);
|
$containers = instant_remote_process(["docker container ls -q"], $this->server);
|
||||||
if (!$containers) {
|
if (!$containers) {
|
||||||
return;
|
return;
|
||||||
|
@@ -3,6 +3,7 @@
|
|||||||
namespace App\Jobs;
|
namespace App\Jobs;
|
||||||
|
|
||||||
use App\Models\Server;
|
use App\Models\Server;
|
||||||
|
use App\Notifications\Server\HighDiskUsage;
|
||||||
use Exception;
|
use Exception;
|
||||||
use Illuminate\Bus\Queueable;
|
use Illuminate\Bus\Queueable;
|
||||||
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
|
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
|
||||||
@@ -18,7 +19,6 @@ class DockerCleanupJob implements ShouldQueue, ShouldBeEncrypted
|
|||||||
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
||||||
|
|
||||||
public $timeout = 300;
|
public $timeout = 300;
|
||||||
public ?string $dockerRootFilesystem = null;
|
|
||||||
public ?int $usageBefore = null;
|
public ?int $usageBefore = null;
|
||||||
|
|
||||||
public function __construct(public Server $server)
|
public function __construct(public Server $server)
|
||||||
@@ -26,28 +26,28 @@ class DockerCleanupJob implements ShouldQueue, ShouldBeEncrypted
|
|||||||
}
|
}
|
||||||
public function handle(): void
|
public function handle(): void
|
||||||
{
|
{
|
||||||
$isInprogress = false;
|
|
||||||
$this->server->applications()->each(function ($application) use (&$isInprogress) {
|
|
||||||
if ($application->isDeploymentInprogress()) {
|
|
||||||
$isInprogress = true;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
});
|
|
||||||
if ($isInprogress) {
|
|
||||||
throw new Exception('DockerCleanupJob: ApplicationDeploymentQueue is not empty, skipping...');
|
|
||||||
}
|
|
||||||
try {
|
try {
|
||||||
|
$isInprogress = false;
|
||||||
|
$this->server->applications()->each(function ($application) use (&$isInprogress) {
|
||||||
|
if ($application->isDeploymentInprogress()) {
|
||||||
|
$isInprogress = true;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
if ($isInprogress) {
|
||||||
|
throw new Exception('DockerCleanupJob: ApplicationDeploymentQueue is not empty, skipping...');
|
||||||
|
}
|
||||||
if (!$this->server->isFunctional()) {
|
if (!$this->server->isFunctional()) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
$this->dockerRootFilesystem = "/";
|
$this->usageBefore = $this->server->getDiskUsage();
|
||||||
$this->usageBefore = $this->getFilesystemUsage();
|
ray('Usage before: ' . $this->usageBefore);
|
||||||
if ($this->usageBefore >= $this->server->settings->cleanup_after_percentage) {
|
if ($this->usageBefore >= $this->server->settings->cleanup_after_percentage) {
|
||||||
ray('Cleaning up ' . $this->server->name);
|
ray('Cleaning up ' . $this->server->name);
|
||||||
instant_remote_process(['docker image prune -af'], $this->server);
|
instant_remote_process(['docker image prune -af'], $this->server, false);
|
||||||
instant_remote_process(['docker container prune -f --filter "label=coolify.managed=true"'], $this->server);
|
instant_remote_process(['docker container prune -f --filter "label=coolify.managed=true"'], $this->server, false);
|
||||||
instant_remote_process(['docker builder prune -af'], $this->server);
|
instant_remote_process(['docker builder prune -af'], $this->server, false);
|
||||||
$usageAfter = $this->getFilesystemUsage();
|
$usageAfter = $this->server->getDiskUsage();
|
||||||
if ($usageAfter < $this->usageBefore) {
|
if ($usageAfter < $this->usageBefore) {
|
||||||
ray('Saved ' . ($this->usageBefore - $usageAfter) . '% disk space on ' . $this->server->name);
|
ray('Saved ' . ($this->usageBefore - $usageAfter) . '% disk space on ' . $this->server->name);
|
||||||
send_internal_notification('DockerCleanupJob done: Saved ' . ($this->usageBefore - $usageAfter) . '% disk space on ' . $this->server->name);
|
send_internal_notification('DockerCleanupJob done: Saved ' . ($this->usageBefore - $usageAfter) . '% disk space on ' . $this->server->name);
|
||||||
@@ -65,9 +65,4 @@ class DockerCleanupJob implements ShouldQueue, ShouldBeEncrypted
|
|||||||
throw $e;
|
throw $e;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private function getFilesystemUsage()
|
|
||||||
{
|
|
||||||
return instant_remote_process(["df '{$this->dockerRootFilesystem}'| tail -1 | awk '{ print $5}' | sed 's/%//g'"], $this->server, false);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
67
app/Jobs/ServerStatusJob.php
Normal file
67
app/Jobs/ServerStatusJob.php
Normal file
@@ -0,0 +1,67 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
namespace App\Jobs;
|
||||||
|
|
||||||
|
use App\Models\Server;
|
||||||
|
use App\Notifications\Server\HighDiskUsage;
|
||||||
|
use Illuminate\Bus\Queueable;
|
||||||
|
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
|
||||||
|
use Illuminate\Contracts\Queue\ShouldQueue;
|
||||||
|
use Illuminate\Foundation\Bus\Dispatchable;
|
||||||
|
use Illuminate\Queue\InteractsWithQueue;
|
||||||
|
use Illuminate\Queue\Middleware\WithoutOverlapping;
|
||||||
|
use Illuminate\Queue\SerializesModels;
|
||||||
|
|
||||||
|
class ServerStatusJob implements ShouldQueue, ShouldBeEncrypted
|
||||||
|
{
|
||||||
|
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
||||||
|
|
||||||
|
public ?int $disk_usage = null;
|
||||||
|
public function __construct(public Server $server)
|
||||||
|
{
|
||||||
|
}
|
||||||
|
public function middleware(): array
|
||||||
|
{
|
||||||
|
return [(new WithoutOverlapping($this->server->id))->dontRelease()];
|
||||||
|
}
|
||||||
|
|
||||||
|
public function uniqueId(): int
|
||||||
|
{
|
||||||
|
return $this->server->id;
|
||||||
|
}
|
||||||
|
|
||||||
|
public function handle(): void
|
||||||
|
{
|
||||||
|
ray("checking server status for {$this->server->id}");
|
||||||
|
try {
|
||||||
|
$this->server->checkServerRediness();
|
||||||
|
$this->cleanup(notify: false);
|
||||||
|
} catch (\Throwable $e) {
|
||||||
|
send_internal_notification('ServerStatusJob failed with: ' . $e->getMessage());
|
||||||
|
ray($e->getMessage());
|
||||||
|
handleError($e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
public function cleanup(bool $notify = false): void
|
||||||
|
{
|
||||||
|
$this->disk_usage = $this->server->getDiskUsage();
|
||||||
|
if ($this->disk_usage >= $this->server->settings->cleanup_after_percentage) {
|
||||||
|
if ($notify) {
|
||||||
|
if ($this->server->high_disk_usage_notification_sent) {
|
||||||
|
ray('high disk usage notification already sent');
|
||||||
|
return;
|
||||||
|
} else {
|
||||||
|
$this->server->high_disk_usage_notification_sent = true;
|
||||||
|
$this->server->save();
|
||||||
|
$this->server->team->notify(new HighDiskUsage($this->server, $this->disk_usage, $this->server->settings->cleanup_after_percentage));
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
DockerCleanupJob::dispatchSync($this->server);
|
||||||
|
$this->cleanup(notify: true);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
$this->server->high_disk_usage_notification_sent = false;
|
||||||
|
$this->server->save();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
@@ -4,8 +4,11 @@ namespace App\Models;
|
|||||||
|
|
||||||
use App\Enums\ProxyStatus;
|
use App\Enums\ProxyStatus;
|
||||||
use App\Enums\ProxyTypes;
|
use App\Enums\ProxyTypes;
|
||||||
|
use App\Notifications\Server\Revived;
|
||||||
|
use App\Notifications\Server\Unreachable;
|
||||||
use Illuminate\Database\Eloquent\Builder;
|
use Illuminate\Database\Eloquent\Builder;
|
||||||
use Illuminate\Database\Eloquent\Casts\Attribute;
|
use Illuminate\Database\Eloquent\Casts\Attribute;
|
||||||
|
use Illuminate\Support\Sleep;
|
||||||
use Spatie\SchemalessAttributes\Casts\SchemalessAttributes;
|
use Spatie\SchemalessAttributes\Casts\SchemalessAttributes;
|
||||||
use Spatie\SchemalessAttributes\SchemalessAttributesTrait;
|
use Spatie\SchemalessAttributes\SchemalessAttributesTrait;
|
||||||
use Illuminate\Support\Str;
|
use Illuminate\Support\Str;
|
||||||
@@ -109,6 +112,83 @@ class Server extends BaseModel
|
|||||||
return $this->proxy->modelScope();
|
return $this->proxy->modelScope();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public function isLocalhost()
|
||||||
|
{
|
||||||
|
return $this->ip === 'host.docker.internal' || $this->id === 0;
|
||||||
|
}
|
||||||
|
public function checkServerRediness()
|
||||||
|
{
|
||||||
|
$serverUptimeCheckNumber = $this->unreachable_count;
|
||||||
|
$serverUptimeCheckNumberMax = 5;
|
||||||
|
|
||||||
|
$currentTime = now()->timestamp;
|
||||||
|
$runtime5Minutes = 1 * 60;
|
||||||
|
// Run for 1 minutes max and check every 5 seconds
|
||||||
|
while ($currentTime + $runtime5Minutes > now()->timestamp) {
|
||||||
|
if ($serverUptimeCheckNumber >= $serverUptimeCheckNumberMax) {
|
||||||
|
if ($this->unreachable_notification_sent === false) {
|
||||||
|
ray('Server unreachable, sending notification...');
|
||||||
|
$this->team->notify(new Unreachable($this));
|
||||||
|
$this->update(['unreachable_notification_sent' => true]);
|
||||||
|
}
|
||||||
|
$this->settings()->update([
|
||||||
|
'is_reachable' => false,
|
||||||
|
]);
|
||||||
|
$this->update([
|
||||||
|
'unreachable_count' => 0,
|
||||||
|
]);
|
||||||
|
foreach ($this->applications() as $application) {
|
||||||
|
$application->update(['status' => 'exited']);
|
||||||
|
}
|
||||||
|
foreach ($this->databases() as $database) {
|
||||||
|
$database->update(['status' => 'exited']);
|
||||||
|
}
|
||||||
|
foreach ($this->services() as $service) {
|
||||||
|
$apps = $service->applications()->get();
|
||||||
|
$dbs = $service->databases()->get();
|
||||||
|
foreach ($apps as $app) {
|
||||||
|
$app->update(['status' => 'exited']);
|
||||||
|
}
|
||||||
|
foreach ($dbs as $db) {
|
||||||
|
$db->update(['status' => 'exited']);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
throw new \Exception('Server is not reachable.');
|
||||||
|
}
|
||||||
|
$result = $this->validateConnection();
|
||||||
|
ray('validateConnection: ' . $result);
|
||||||
|
if (!$result) {
|
||||||
|
$serverUptimeCheckNumber++;
|
||||||
|
$this->update([
|
||||||
|
'unreachable_count' => $serverUptimeCheckNumber,
|
||||||
|
]);
|
||||||
|
Sleep::for(5)->seconds();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
$this->update([
|
||||||
|
'unreachable_count' => 0,
|
||||||
|
]);
|
||||||
|
if (data_get($this, 'unreachable_notification_sent') === true) {
|
||||||
|
ray('Server is reachable again, sending notification...');
|
||||||
|
$this->team->notify(new Revived($this));
|
||||||
|
$this->update(['unreachable_notification_sent' => false]);
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
data_get($this, 'settings.is_reachable') === false ||
|
||||||
|
data_get($this, 'settings.is_usable') === false
|
||||||
|
) {
|
||||||
|
$this->settings()->update([
|
||||||
|
'is_reachable' => true,
|
||||||
|
'is_usable' => true
|
||||||
|
]);
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
public function getDiskUsage()
|
||||||
|
{
|
||||||
|
return instant_remote_process(["df /| tail -1 | awk '{ print $5}' | sed 's/%//g'"], $this, false);
|
||||||
|
}
|
||||||
public function hasDefinedResources()
|
public function hasDefinedResources()
|
||||||
{
|
{
|
||||||
$applications = $this->applications()->count() > 0;
|
$applications = $this->applications()->count() > 0;
|
||||||
@@ -148,7 +228,7 @@ class Server extends BaseModel
|
|||||||
if (isDev()) {
|
if (isDev()) {
|
||||||
return '127.0.0.1';
|
return '127.0.0.1';
|
||||||
}
|
}
|
||||||
if ($this->ip === 'host.docker.internal') {
|
if ($this->isLocalhost()) {
|
||||||
return base_ip();
|
return base_ip();
|
||||||
}
|
}
|
||||||
return $this->ip;
|
return $this->ip;
|
||||||
|
@@ -36,7 +36,7 @@ class ServiceDatabase extends BaseModel
|
|||||||
{
|
{
|
||||||
$port = $this->public_port;
|
$port = $this->public_port;
|
||||||
$realIp = $this->service->server->ip;
|
$realIp = $this->service->server->ip;
|
||||||
if ($realIp === 'host.docker.internal' || isDev()) {
|
if ($this->service->server->isLocalhost() || isDev()) {
|
||||||
$realIp = base_ip();
|
$realIp = base_ip();
|
||||||
}
|
}
|
||||||
$url = "{$realIp}:{$port}";
|
$url = "{$realIp}:{$port}";
|
||||||
|
65
app/Notifications/Server/HighDiskUsage.php
Normal file
65
app/Notifications/Server/HighDiskUsage.php
Normal file
@@ -0,0 +1,65 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
namespace App\Notifications\Server;
|
||||||
|
|
||||||
|
use App\Models\Server;
|
||||||
|
use Illuminate\Bus\Queueable;
|
||||||
|
use App\Notifications\Channels\DiscordChannel;
|
||||||
|
use App\Notifications\Channels\EmailChannel;
|
||||||
|
use App\Notifications\Channels\TelegramChannel;
|
||||||
|
use Illuminate\Contracts\Queue\ShouldQueue;
|
||||||
|
use Illuminate\Notifications\Messages\MailMessage;
|
||||||
|
use Illuminate\Notifications\Notification;
|
||||||
|
|
||||||
|
class HighDiskUsage extends Notification implements ShouldQueue
|
||||||
|
{
|
||||||
|
use Queueable;
|
||||||
|
|
||||||
|
public $tries = 1;
|
||||||
|
public function __construct(public Server $server, public int $disk_usage, public int $cleanup_after_percentage)
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
|
public function via(object $notifiable): array
|
||||||
|
{
|
||||||
|
$channels = [];
|
||||||
|
$isEmailEnabled = isEmailEnabled($notifiable);
|
||||||
|
$isDiscordEnabled = data_get($notifiable, 'discord_enabled');
|
||||||
|
$isTelegramEnabled = data_get($notifiable, 'telegram_enabled');
|
||||||
|
|
||||||
|
if ($isDiscordEnabled) {
|
||||||
|
$channels[] = DiscordChannel::class;
|
||||||
|
}
|
||||||
|
if ($isEmailEnabled) {
|
||||||
|
$channels[] = EmailChannel::class;
|
||||||
|
}
|
||||||
|
if ($isTelegramEnabled) {
|
||||||
|
$channels[] = TelegramChannel::class;
|
||||||
|
}
|
||||||
|
return $channels;
|
||||||
|
}
|
||||||
|
|
||||||
|
public function toMail(): MailMessage
|
||||||
|
{
|
||||||
|
$mail = new MailMessage();
|
||||||
|
$mail->subject("Coolify: Server ({$this->server->name}) high disk usage detected!");
|
||||||
|
$mail->view('emails.high-disk-usage', [
|
||||||
|
'name' => $this->server->name,
|
||||||
|
'disk_usage' => $this->disk_usage,
|
||||||
|
'threshold' => $this->cleanup_after_percentage,
|
||||||
|
]);
|
||||||
|
return $mail;
|
||||||
|
}
|
||||||
|
|
||||||
|
public function toDiscord(): string
|
||||||
|
{
|
||||||
|
$message = "Coolify: Server '{$this->server->name}' high disk usage detected!\nDisk usage: {$this->disk_usage}%. Threshold: {$this->cleanup_after_percentage}%.\nPlease cleanup your disk to prevent data-loss.\nHere are some tips: https://coolify.io/docs/automated-cleanup.";
|
||||||
|
return $message;
|
||||||
|
}
|
||||||
|
public function toTelegram(): array
|
||||||
|
{
|
||||||
|
return [
|
||||||
|
"message" => "Coolify: Server '{$this->server->name}' high disk usage detected!\nDisk usage: {$this->disk_usage}%. Threshold: {$this->cleanup_after_percentage}%.\nPlease cleanup your disk to prevent data-loss.\nHere are some tips: https://coolify.io/docs/automated-cleanup."
|
||||||
|
];
|
||||||
|
}
|
||||||
|
}
|
@@ -18,7 +18,7 @@ class Revived extends Notification implements ShouldQueue
|
|||||||
public $tries = 1;
|
public $tries = 1;
|
||||||
public function __construct(public Server $server)
|
public function __construct(public Server $server)
|
||||||
{
|
{
|
||||||
if ($this->server->unreachable_email_sent === false) {
|
if ($this->server->unreachable_notification_sent === false) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@@ -191,7 +191,7 @@ function refresh_server_connection(?PrivateKey $private_key = null)
|
|||||||
// if (!$uptime) {
|
// if (!$uptime) {
|
||||||
// $server->settings->is_reachable = false;
|
// $server->settings->is_reachable = false;
|
||||||
// $server->team->notify(new Unreachable($server));
|
// $server->team->notify(new Unreachable($server));
|
||||||
// $server->unreachable_email_sent = true;
|
// $server->unreachable_notification_sent = true;
|
||||||
// $server->save();
|
// $server->save();
|
||||||
// return [
|
// return [
|
||||||
// "uptime" => null,
|
// "uptime" => null,
|
||||||
@@ -213,9 +213,9 @@ function refresh_server_connection(?PrivateKey $private_key = null)
|
|||||||
// $server->settings->is_usable = false;
|
// $server->settings->is_usable = false;
|
||||||
// } else {
|
// } else {
|
||||||
// $server->settings->is_usable = true;
|
// $server->settings->is_usable = true;
|
||||||
// if (data_get($server, 'unreachable_email_sent') === true) {
|
// if (data_get($server, 'unreachable_notification_sent') === true) {
|
||||||
// $server->team->notify(new Revived($server));
|
// $server->team->notify(new Revived($server));
|
||||||
// $server->unreachable_email_sent = false;
|
// $server->unreachable_notification_sent = false;
|
||||||
// $server->save();
|
// $server->save();
|
||||||
// }
|
// }
|
||||||
// }
|
// }
|
||||||
|
@@ -7,7 +7,7 @@ return [
|
|||||||
|
|
||||||
// The release version of your application
|
// The release version of your application
|
||||||
// Example with dynamic git hash: trim(exec('git --git-dir ' . base_path('.git') . ' log --pretty="%h" -n1 HEAD'))
|
// Example with dynamic git hash: trim(exec('git --git-dir ' . base_path('.git') . ' log --pretty="%h" -n1 HEAD'))
|
||||||
'release' => '4.0.0-beta.136',
|
'release' => '4.0.0-beta.137',
|
||||||
// When left empty or `null` the Laravel environment will be used
|
// When left empty or `null` the Laravel environment will be used
|
||||||
'environment' => config('app.env'),
|
'environment' => config('app.env'),
|
||||||
|
|
||||||
|
@@ -1,3 +1,3 @@
|
|||||||
<?php
|
<?php
|
||||||
|
|
||||||
return '4.0.0-beta.136';
|
return '4.0.0-beta.137';
|
||||||
|
@@ -0,0 +1,30 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
use Illuminate\Database\Migrations\Migration;
|
||||||
|
use Illuminate\Database\Schema\Blueprint;
|
||||||
|
use Illuminate\Support\Facades\Schema;
|
||||||
|
|
||||||
|
return new class extends Migration
|
||||||
|
{
|
||||||
|
/**
|
||||||
|
* Run the migrations.
|
||||||
|
*/
|
||||||
|
public function up(): void
|
||||||
|
{
|
||||||
|
Schema::table('servers', function (Blueprint $table) {
|
||||||
|
$table->boolean('high_disk_usage_notification_sent')->default(false);
|
||||||
|
$table->renameColumn('unreachable_email_sent', 'unreachable_notification_sent');
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reverse the migrations.
|
||||||
|
*/
|
||||||
|
public function down(): void
|
||||||
|
{
|
||||||
|
Schema::table('servers', function (Blueprint $table) {
|
||||||
|
$table->dropColumn('high_disk_usage_notification_sent');
|
||||||
|
$table->renameColumn('unreachable_notification_sent', 'unreachable_email_sent');
|
||||||
|
});
|
||||||
|
}
|
||||||
|
};
|
16
examples/fluent-bit/fluent-bit.conf
Normal file
16
examples/fluent-bit/fluent-bit.conf
Normal file
@@ -0,0 +1,16 @@
|
|||||||
|
[SERVICE]
|
||||||
|
Flush 1
|
||||||
|
Daemon off
|
||||||
|
[INPUT]
|
||||||
|
Name forward
|
||||||
|
Buffer_Chunk_Size 1M
|
||||||
|
Buffer_Max_Size 6M
|
||||||
|
# [OUTPUT]
|
||||||
|
# Name nrlogs
|
||||||
|
# Match *
|
||||||
|
# license_key ${LICENSE_KEY}
|
||||||
|
# base_uri https://log-api.eu.newrelic.com/log/v1
|
||||||
|
|
||||||
|
[OUTPUT]
|
||||||
|
Name stdout
|
||||||
|
Match *
|
9
examples/fluent-bit/fluent-bit.yaml
Normal file
9
examples/fluent-bit/fluent-bit.yaml
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
version: '3'
|
||||||
|
services:
|
||||||
|
coolify-fluent-bit:
|
||||||
|
image: cr.fluentbit.io/fluent/fluent-bit:2.0
|
||||||
|
command: -c /fluent-bit.conf
|
||||||
|
volumes:
|
||||||
|
- ./fluent-bit.conf:/fluent-bit.conf
|
||||||
|
ports:
|
||||||
|
- 24224:24224
|
21
examples/newrelic.yaml
Normal file
21
examples/newrelic.yaml
Normal file
@@ -0,0 +1,21 @@
|
|||||||
|
version: '3'
|
||||||
|
services:
|
||||||
|
newrelic-infra:
|
||||||
|
container_name: newrelic-infra
|
||||||
|
image: newrelic/infrastructure:latest
|
||||||
|
networks:
|
||||||
|
- coolify
|
||||||
|
cap_add:
|
||||||
|
- SYS_PTRACE
|
||||||
|
privileged: true
|
||||||
|
pid: host
|
||||||
|
volumes:
|
||||||
|
- "/:/host:ro"
|
||||||
|
- "/var/run/docker.sock:/var/run/docker.sock"
|
||||||
|
- "newrelic-infra:/etc/newrelic-infra"
|
||||||
|
environment:
|
||||||
|
- NRIA_LICENSE_KEY=${NRIA_LICENSE_KEY}
|
||||||
|
- NRIA_DISPLAY_NAME=${HOSTNAME}
|
||||||
|
|
||||||
|
networks:
|
||||||
|
coolify:
|
34
examples/otl/config.yaml
Normal file
34
examples/otl/config.yaml
Normal file
@@ -0,0 +1,34 @@
|
|||||||
|
receivers:
|
||||||
|
hostmetrics:
|
||||||
|
collection_interval: 5s
|
||||||
|
scrapers:
|
||||||
|
cpu:
|
||||||
|
metrics:
|
||||||
|
system.cpu.utilization:
|
||||||
|
enabled: true
|
||||||
|
processors:
|
||||||
|
resourcedetection:
|
||||||
|
detectors: [env, system]
|
||||||
|
system:
|
||||||
|
hostname_sources: ["os"]
|
||||||
|
resource_attributes:
|
||||||
|
host.id:
|
||||||
|
enabled: true
|
||||||
|
batch:
|
||||||
|
memory_limiter:
|
||||||
|
check_interval: 1s
|
||||||
|
limit_mib: 1000
|
||||||
|
spike_limit_mib: 200
|
||||||
|
exporters:
|
||||||
|
debug:
|
||||||
|
verbosity: detailed
|
||||||
|
otlp:
|
||||||
|
endpoint: ${OTLP_ENDPOINT}
|
||||||
|
headers:
|
||||||
|
api-key: ${OTLP_API_KEY}
|
||||||
|
service:
|
||||||
|
pipelines:
|
||||||
|
metrics:
|
||||||
|
receivers: [hostmetrics]
|
||||||
|
processors: [memory_limiter, resourcedetection, batch]
|
||||||
|
exporters: [debug, otlp]
|
9
resources/views/emails/high-disk-usage.blade.php
Normal file
9
resources/views/emails/high-disk-usage.blade.php
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
<x-emails.layout>
|
||||||
|
|
||||||
|
Your server ({{ $name }}) has high disk usage ({{ $disk_usage }}% used). Threshold is {{ $threshold }}%.
|
||||||
|
|
||||||
|
Please cleanup your disk to prevent data-loss. Here are some [tips](https://coolify.io/docs/automated-cleanup).
|
||||||
|
|
||||||
|
(You can change the threshold in the Server Settings menu.)
|
||||||
|
|
||||||
|
</x-emails.layout>
|
@@ -49,11 +49,13 @@
|
|||||||
<x-forms.input type="number" id="server.port" label="Port" required />
|
<x-forms.input type="number" id="server.port" label="Port" required />
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<div class="w-64">
|
@if (!$server->isLocalhost())
|
||||||
<x-forms.checkbox instantSave
|
<div class="w-64">
|
||||||
helper="If you are using Cloudflare Tunnels, enable this. It will proxy all ssh requests to your server through Cloudflare.<span class='text-warning'>Coolify does not install/setup Cloudflare (cloudflared) on your server.</span>"
|
<x-forms.checkbox instantSave
|
||||||
id="server.settings.is_cloudflare_tunnel" label="Cloudflare Tunnel" />
|
helper="If you are using Cloudflare Tunnels, enable this. It will proxy all ssh requests to your server through Cloudflare.<span class='text-warning'>Coolify does not install/setup Cloudflare (cloudflared) on your server.</span>"
|
||||||
</div>
|
id="server.settings.is_cloudflare_tunnel" label="Cloudflare Tunnel" />
|
||||||
|
</div>
|
||||||
|
@endif
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
@if ($server->isFunctional())
|
@if ($server->isFunctional())
|
||||||
|
@@ -4,7 +4,7 @@
|
|||||||
"version": "3.12.36"
|
"version": "3.12.36"
|
||||||
},
|
},
|
||||||
"v4": {
|
"v4": {
|
||||||
"version": "4.0.0-beta.136"
|
"version": "4.0.0-beta.137"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
Reference in New Issue
Block a user