dave/coolify
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

feat: new container status checks

Browse Source
This commit is contained in:
Andras Bacsai
2023-09-14 12:45:50 +02:00
parent 3c8c8e20b1
commit 53c20e1e99
18 changed files with 369 additions and 189 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
app/Actions/Proxy/StartProxy.php
View File

@@ -8,7 +8,7 @@ use Spatie\Activitylog\Models\Activity;
class StartProxy
{
public function __invoke(Server $server): Activity
public function __invoke(Server $server, bool $async = true): Activity|string
{
$proxy_path = get_proxy_path();
$networks = collect($server->standaloneDockers)->map(function ($docker) {
@@ -26,8 +26,7 @@ class StartProxy
$docker_compose_yml_base64 = base64_encode($configuration);
$server->proxy->last_applied_settings = Str::of($docker_compose_yml_base64)->pipe('md5')->value;
$server->save();
$activity = remote_process([
$commands = [
"echo '####### Creating required Docker networks...'",
...$create_networks_command,
"cd $proxy_path",
@@ -44,8 +43,13 @@ class StartProxy
"echo '####### Starting coolify-proxy...'",
'docker compose up -d --remove-orphans',
"echo '####### Proxy installed successfully...'"
], $server);
return $activity;
];
if (!$async) {
instant_remote_process($commands, $server);
return 'OK';
} else {
$activity = remote_process($commands, $server);
return $activity;
}
}
}

32
app/Console/Kernel.php
View File

@@ -2,21 +2,15 @@
namespace App\Console;
use App\Enums\ProxyTypes;
use App\Jobs\ApplicationContainerStatusJob;
use App\Jobs\CheckResaleLicenseJob;
use App\Jobs\CleanupInstanceStuffsJob;
use App\Jobs\DatabaseBackupJob;
use App\Jobs\DatabaseContainerStatusJob;
use App\Jobs\DockerCleanupJob;
use App\Jobs\InstanceAutoUpdateJob;
use App\Jobs\ProxyContainerStatusJob;
use App\Jobs\ServerDetailsCheckJob;
use App\Models\Application;
use App\Jobs\ContainerStatusJob;
use App\Models\InstanceSettings;
use App\Models\ScheduledDatabaseBackup;
use App\Models\Server;
use App\Models\StandalonePostgresql;
use Illuminate\Console\Scheduling\Schedule;
use Illuminate\Foundation\Console\Kernel as ConsoleKernel;
@@ -25,15 +19,14 @@ class Kernel extends ConsoleKernel
protected function schedule(Schedule $schedule): void
{
if (isDev()) {
$schedule->job(new ServerDetailsCheckJob(Server::find(0)))->everyTenMinutes()->onOneServer();
// $schedule->job(new ContainerStatusJob(Server::find(0)))->everyTenMinutes()->onOneServer();
// $schedule->command('horizon:snapshot')->everyMinute();
// $schedule->job(new CleanupInstanceStuffsJob)->everyMinute();
// $schedule->job(new CheckResaleLicenseJob)->hourly();
// $schedule->job(new DockerCleanupJob)->everyOddHour();
// $this->instance_auto_update($schedule);
// $this->check_scheduled_backups($schedule);
// $this->check_resources($schedule);
// $this->check_proxies($schedule);
$this->check_resources($schedule);
} else {
$schedule->command('horizon:snapshot')->everyFiveMinutes();
$schedule->job(new CleanupInstanceStuffsJob)->everyTenMinutes()->onOneServer();
@@ -42,26 +35,15 @@ class Kernel extends ConsoleKernel
$this->instance_auto_update($schedule);
$this->check_scheduled_backups($schedule);
$this->check_resources($schedule);
$this->check_proxies($schedule);
}
}
private function check_proxies($schedule)
{
$servers = Server::all()->where('settings.is_usable', true)->where('settings.is_reachable', true)->whereNotNull('proxy.type')->where('proxy.type', '!=', ProxyTypes::NONE->value);
foreach ($servers as $server) {
$schedule->job(new ProxyContainerStatusJob($server))->everyMinute()->onOneServer();
}
}
private function check_resources($schedule)
{
$applications = Application::all();
foreach ($applications as $application) {
$schedule->job(new ApplicationContainerStatusJob($application))->everyMinute()->onOneServer();
}
$servers = Server::all()->where('settings.is_usable', true)->where('settings.is_reachable', true);
ray($servers);
$postgresqls = StandalonePostgresql::all();
foreach ($postgresqls as $postgresql) {
$schedule->job(new DatabaseContainerStatusJob($postgresql))->everyMinute()->onOneServer();
foreach ($servers as $server) {
$schedule->job(new ContainerStatusJob($server))->everyMinute()->onOneServer();
}
}
private function instance_auto_update($schedule)

6
app/Http/Livewire/Project/Application/Heading.php
View File

@@ -3,6 +3,7 @@
namespace App\Http\Livewire\Project\Application;
use App\Jobs\ApplicationContainerStatusJob;
use App\Jobs\ContainerStatusJob;
use App\Models\Application;
use App\Notifications\Application\StatusChanged;
use Livewire\Component;
@@ -22,9 +23,8 @@ class Heading extends Component
public function check_status()
{
dispatch_sync(new ApplicationContainerStatusJob(
application: $this->application,
));
ray($this->application->destination->server);
dispatch_sync(new ContainerStatusJob($this->application->destination->server));
$this->application->refresh();
}

7
app/Http/Livewire/Project/Database/Heading.php
View File

@@ -3,8 +3,7 @@
namespace App\Http\Livewire\Project\Database;
use App\Actions\Database\StartPostgresql;
use App\Jobs\DatabaseContainerStatusJob;
use App\Notifications\Application\StatusChanged;
use App\Jobs\ContainerStatusJob;
use Livewire\Component;
class Heading extends Component
@@ -25,9 +24,7 @@ class Heading extends Component
public function check_status()
{
dispatch_sync(new DatabaseContainerStatusJob(
database: $this->database,
));
dispatch_sync(new ContainerStatusJob($this->database->destination->server));
$this->database->refresh();
}

16
app/Http/Livewire/Server/Form.php
View File

@@ -88,17 +88,11 @@ class Form extends Component
public function submit()
{
$this->validate();
// $validation = Validator::make($this->server->toArray(), [
// 'ip' => [
// 'ip'
// ],
// ]);
// if ($validation->fails()) {
// foreach ($validation->errors()->getMessages() as $key => $value) {
// $this->addError("server.{$key}", $value[0]);
// }
// return;
// }
$uniqueIPs = Server::all()->pluck('ip')->toArray();
if (in_array($this->server->ip, $uniqueIPs)) {
$this->emit('error', 'IP address is already in use by another team.');
return;
}
$this->server->settings->wildcard_domain = $this->wildcard_domain;
$this->server->settings->cleanup_after_percentage = $this->cleanup_after_percentage;
$this->server->settings->save();

4
app/Http/Livewire/Settings/Configuration.php
View File

@@ -2,7 +2,7 @@
namespace App\Http\Livewire\Settings;
use App\Jobs\ProxyContainerStatusJob;
use App\Jobs\ContainerStatusJob;
use App\Models\InstanceSettings as ModelsInstanceSettings;
use App\Models\Server;
use Livewire\Component;
@@ -124,7 +124,7 @@ class Configuration extends Component
];
}
$this->save_configuration_to_disk($traefik_dynamic_conf, $file);
dispatch(new ProxyContainerStatusJob($this->server));
dispatch(new ContainerStatusJob($this->server));
}
}

1
app/Jobs/ApplicationContainerStatusJob.php
View File

@@ -4,7 +4,6 @@ namespace App\Jobs;
use App\Models\Application;
use App\Models\ApplicationPreview;
use App\Notifications\Application\StatusChanged;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
use Illuminate\Contracts\Queue\ShouldBeUnique;

5
app/Jobs/ApplicationDeploymentJob.php
View File

@@ -132,7 +132,9 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted
$this->deploy();
}
}
if ($this->application->fqdn) dispatch(new ProxyContainerStatusJob($this->server));
if ($this->server->isProxyShouldRun()) {
dispatch(new ContainerStatusJob($this->server));
}
$this->next(ApplicationDeploymentStatus::FINISHED->value);
} catch (Exception $e) {
ray($e);
@@ -267,6 +269,7 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted
"echo 'Rolling update completed.'"
],
);
$this->application->update(['status' => 'running']);
break;
}
$counter++;

139
app/Jobs/ContainerStatusJob.php Normal file
View File

@@ -0,0 +1,139 @@
<?php
namespace App\Jobs;
use App\Actions\Proxy\StartProxy;
use App\Models\Server;
use App\Notifications\Container\ContainerRestarted;
use App\Notifications\Container\ContainerStopped;
use App\Notifications\Server\Unreachable;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
use Illuminate\Contracts\Queue\ShouldBeUnique;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Queue\SerializesModels;
use Str;
class ContainerStatusJob implements ShouldQueue, ShouldBeUnique, ShouldBeEncrypted
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public $tries = 1;
public $timeout = 120;
public function __construct(public Server $server)
{
}
public function middleware(): array
{
return [new WithoutOverlapping($this->server->uuid)];
}
public function uniqueId(): string
{
return $this->server->uuid;
}
private function checkServerConnection() {
ray("Checking server connection to {$this->server->ip}");
$uptime = instant_remote_process(['uptime'], $this->server, false);
if (!is_null($uptime)) {
ray('Server is up');
return true;
}
}
public function handle(): void
{
try {
ray()->clearAll();
$serverUptimeCheckNumber = 0;
$serverUptimeCheckNumberMax = 5;
while (true) {
if ($serverUptimeCheckNumber >= $serverUptimeCheckNumberMax) {
$this->server->settings()->update(['is_reachable' => false]);
$this->server->team->notify(new Unreachable($this->server));
return;
}
$result = $this->checkServerConnection();
if ($result) {
break;
}
$serverUptimeCheckNumber++;
sleep(5);
}
$containers = instant_remote_process(["docker container inspect $(docker container ls -q) --format '{{json .}}'"], $this->server);
$containers = format_docker_command_output_to_json($containers);
$applications = $this->server->applications();
$databases = $this->server->databases();
if ($this->server->isProxyShouldRun()) {
$foundProxyContainer = $containers->filter(function ($value, $key) {
return data_get($value, 'Name') === '/coolify-proxy';
})->first();
if (!$foundProxyContainer) {
resolve(StartProxy::class)($this->server, false);
$this->server->team->notify(new ContainerRestarted('coolify-proxy', $this->server));
}
}
foreach ($applications as $application) {
$uuid = data_get($application, 'uuid');
$foundContainer = $containers->filter(function ($value, $key) use ($uuid) {
return Str::startsWith(data_get($value, 'Name'), "/$uuid");
})->first();
if ($foundContainer) {
$containerStatus = data_get($foundContainer, 'State.Status');
$databaseStatus = data_get($application, 'status');
if ($containerStatus !== $databaseStatus) {
$application->update(['status' => $containerStatus]);
}
} else {
$databaseStatus = data_get($application, 'status');
if ($databaseStatus !== 'exited') {
$application->update(['status' => 'exited']);
$name = data_get($application, 'name');
$fqdn = data_get($application, 'fqdn');
$containerName = $name ? "$name ($fqdn)" : $fqdn;
$project = data_get($application, 'environment.project');
$environment = data_get($application, 'environment');
$url = base_url() . '/project/' . $project->uuid . "/" . $environment->name . "/application/" . $application->uuid;
$this->server->team->notify(new ContainerStopped($containerName, $this->server, $url));
}
}
}
foreach ($databases as $database) {
$uuid = data_get($database, 'uuid');
$foundContainer = $containers->filter(function ($value, $key) use ($uuid) {
return Str::startsWith(data_get($value, 'Name'), "/$uuid");
})->first();
if ($foundContainer) {
$containerStatus = data_get($foundContainer, 'State.Status');
$databaseStatus = data_get($database, 'status');
if ($containerStatus !== $databaseStatus) {
$database->update(['status' => $containerStatus]);
}
} else {
$databaseStatus = data_get($database, 'status');
if ($databaseStatus !== 'exited') {
$database->update(['status' => 'exited']);
$name = data_get($database, 'name');
$containerName = $name;
$project = data_get($database, 'environment.project');
$environment = data_get($database, 'environment');
$url = base_url() . '/project/' . $project->uuid . "/" . $environment->name . "/database/" . $database->uuid;
$this->server->team->notify(new ContainerStopped($containerName, $this->server, $url));
}
}
}
// TODO Monitor other containers not managed by Coolify
} catch (\Throwable $e) {
send_internal_notification('ContainerStatusJob failed with: ' . $e->getMessage());
ray($e->getMessage());
throw $e;
}
}
}

80
app/Jobs/ServerDetailsCheckJob.php
View File

@@ -1,80 +0,0 @@
<?php
namespace App\Jobs;
use App\Models\Server;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
use Illuminate\Contracts\Queue\ShouldBeUnique;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Queue\SerializesModels;
use Str;
class ServerDetailsCheckJob implements ShouldQueue, ShouldBeUnique, ShouldBeEncrypted
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public $tries = 1;
public $timeout = 120;
public function __construct(public Server $server)
{
}
public function middleware(): array
{
return [new WithoutOverlapping($this->server->uuid)];
}
public function uniqueId(): string
{
return $this->server->uuid;
}
public function handle(): void
{
try {
ray()->clearAll();
$containers = instant_remote_process(["docker container inspect $(docker container ls -q) --format '{{json .}}'"], $this->server);
$containers = format_docker_command_output_to_json($containers);
$applications = $this->server->applications();
// ray($applications);
// ray(format_docker_command_output_to_json($containers));
foreach ($applications as $application) {
$uuid = data_get($application, 'uuid');
$foundContainer = $containers->filter(function ($value, $key) use ($uuid) {
$image = data_get($value, 'Config.Image');
return Str::startsWith($image, $uuid);
})->first();
if ($foundContainer) {
$containerStatus = data_get($foundContainer, 'State.Status');
$databaseStatus = data_get($application, 'status');
ray($containerStatus, $databaseStatus);
if ($containerStatus !== $databaseStatus) {
// $application->update(['status' => $containerStatus]);
}
}
}
// foreach ($containers as $container) {
// $labels = format_docker_labels_to_json(data_get($container,'Config.Labels'));
// $foundLabel = $labels->filter(fn ($value, $key) => Str::startsWith($key, 'coolify.applicationId'));
// if ($foundLabel->count() > 0) {
// $appFound = $applications->where('id', $foundLabel['coolify.applicationId'])->first();
// if ($appFound) {
// $containerStatus = data_get($container, 'State.Status');
// $databaseStatus = data_get($appFound, 'status');
// ray($containerStatus, $databaseStatus);
// }
// }
// }
} catch (\Throwable $e) {
// send_internal_notification('ServerDetailsCheckJob failed with: ' . $e->getMessage());
ray($e->getMessage());
throw $e;
}
}
}

2
app/Notifications/Application/DeploymentSuccess.php
View File

@@ -53,7 +53,7 @@ class DeploymentSuccess extends Notification implements ShouldQueue
$pull_request_id = data_get($this->preview, 'pull_request_id', 0);
$fqdn = $this->fqdn;
if ($pull_request_id === 0) {
$mail->subject("✅New version is deployed of {$this->application_name}");
$mail->subject("✅ New version is deployed of {$this->application_name}");
} else {
$fqdn = $this->preview->fqdn;
$mail->subject("✅ Pull request #{$pull_request_id} of {$this->application_name} deployed successfully");

62
app/Notifications/Container/ContainerRestarted.php Normal file
View File

@@ -0,0 +1,62 @@
<?php
namespace App\Notifications\Container;
use App\Models\Server;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Notifications\Messages\MailMessage;
use Illuminate\Notifications\Notification;
class ContainerRestarted extends Notification implements ShouldQueue
{
use Queueable;
public $tries = 5;
public function __construct(public string $name, public Server $server, public ?string $url = null)
{
}
public function via(object $notifiable): array
{
return setNotificationChannels($notifiable, 'status_changes');
}
public function toMail(): MailMessage
{
$mail = new MailMessage();
$mail->subject("✅ Container ({$this->name}) has been restarted automatically on {$this->server->name}");
$mail->view('emails.container-restarted', [
'containerName' => $this->name,
'serverName' => $this->server->name,
'url' => $this->url ,
]);
return $mail;
}
public function toDiscord(): string
{
$message = "✅ Container ({$this->name}) has been restarted automatically on {$this->server->name}";
return $message;
}
public function toTelegram(): array
{
$message = "✅ Container ({$this->name}) has been restarted automatically on {$this->server->name}";
$payload = [
"message" => $message,
];
if ($this->url) {
$payload['buttons'] = [
[
[
"text" => "Check Proxy in Coolify",
"url" => $this->url
]
]
];
};
return $payload;
}
}

61
app/Notifications/Container/ContainerStopped.php Normal file
View File

@@ -0,0 +1,61 @@
<?php
namespace App\Notifications\Container;
use App\Models\Server;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Notifications\Messages\MailMessage;
use Illuminate\Notifications\Notification;
class ContainerStopped extends Notification implements ShouldQueue
{
use Queueable;
public $tries = 1;
public function __construct(public string $name, public Server $server, public ?string $url = null)
{
}
public function via(object $notifiable): array
{
return setNotificationChannels($notifiable, 'status_changes');
}
public function toMail(): MailMessage
{
$mail = new MailMessage();
$mail->subject("⛔ Container ({$this->name}) has been stopped on {$this->server->name}");
$mail->view('emails.container-stopped', [
'containerName' => $this->name,
'serverName' => $this->server->name,
'url' => $this->url,
]);
return $mail;
}
public function toDiscord(): string
{
$message = "⛔ Container ({$this->name}) has been stopped on {$this->server->name}";
return $message;
}
public function toTelegram(): array
{
$message = "⛔ Container ({$this->name}) has been stopped on {$this->server->name}";
$payload = [
"message" => $message,
];
if ($this->url) {
$payload['buttons'] = [
[
[
"text" => "Open Application in Coolify",
"url" => $this->url
]
]
];
}
return $payload;
}
}

53
app/Notifications/Server/NotReachable.php
View File

@@ -1,53 +0,0 @@
<?php
namespace App\Notifications\Server;
use App\Models\Server;
use App\Notifications\Channels\DiscordChannel;
use App\Notifications\Channels\EmailChannel;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Notifications\Messages\MailMessage;
use Illuminate\Notifications\Notification;
use Illuminate\Support\Str;
class NotReachable extends Notification implements ShouldQueue
{
use Queueable;
public $tries = 5;
public function __construct(public Server $server)
{
}
public function via(object $notifiable): array
{
return setNotificationChannels($notifiable, 'status_changes');
}
public function toMail(): MailMessage
{
$mail = new MailMessage();
// $fqdn = $this->fqdn;
$mail->subject("⛔ Server '{$this->server->name}' is unreachable");
// $mail->view('emails.application-status-changes', [
// 'name' => $this->application_name,
// 'fqdn' => $fqdn,
// 'application_url' => $this->application_url,
// ]);
return $mail;
}
public function toDiscord(): string
{
$message = '⛔ Server \'' . $this->server->name . '\' is unreachable (could be a temporary issue). If you receive this more than twice in a row, please check your server.';
return $message;
}
public function toTelegram(): array
{
return [
"message" => '⛔ Server \'' . $this->server->name . '\' is unreachable (could be a temporary issue). If you receive this more than twice in a row, please check your server.'
];
}
}

47
app/Notifications/Server/Unreachable.php Normal file
View File

@@ -0,0 +1,47 @@
<?php
namespace App\Notifications\Server;
use App\Models\Server;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Notifications\Messages\MailMessage;
use Illuminate\Notifications\Notification;
class Unreachable extends Notification implements ShouldQueue
{
use Queueable;
public $tries = 1;
public function __construct(public Server $server)
{
}
public function via(object $notifiable): array
{
return setNotificationChannels($notifiable, 'status_changes');
}
public function toMail(): MailMessage
{
$mail = new MailMessage();
$mail->subject("⛔ Server ({$this->server->name}) is unreachable after trying to connect to it 5 times");
$mail->view('emails.server-lost-connection', [
'name' => $this->server->name,
]);
return $mail;
}
public function toDiscord(): string
{
$message = "⛔ Server '{$this->server->name}' is unreachable after trying to connect to it 5 times. All automations & integrations are turned off! Please check your server! IMPORTANT: You have to validate your server again after you fix the issue.";
return $message;
}
public function toTelegram(): array
{
return [
"message" => "⛔ Server '{$this->server->name}' is unreachable after trying to connect to it 5 times. All automations & integrations are turned off! Please check your server! IMPORTANT: You have to validate your server again after you fix the issue."
];
}
}