dave/coolify
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

feat: monitor server connection

Browse Source
This commit is contained in:
Andras Bacsai
2023-08-16 17:18:50 +02:00
parent fd74e07fc8
commit b34ab8a128
11 changed files with 184 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
bootstrap/helpers/docker.php
View File

@@ -8,8 +8,8 @@ function format_docker_command_output_to_json($rawOutput): Collection
$outputLines = explode(PHP_EOL, $rawOutput);
return collect($outputLines)
->reject(fn ($line) => empty($line))
->map(fn ($outputLine) => json_decode($outputLine, true, flags: JSON_THROW_ON_ERROR));
->reject(fn($line) => empty($line))
->map(fn($outputLine) => json_decode($outputLine, true, flags: JSON_THROW_ON_ERROR));
}
function format_docker_labels_to_json($rawOutput): Collection
@@ -17,7 +17,7 @@ function format_docker_labels_to_json($rawOutput): Collection
$outputLines = explode(PHP_EOL, $rawOutput);
return collect($outputLines)
->reject(fn ($line) => empty($line))
->reject(fn($line) => empty($line))
->map(function ($outputLine) {
$outputArray = explode(',', $outputLine);
return collect($outputArray)
@@ -45,6 +45,7 @@ function format_docker_envs_to_json($rawOutput)
function get_container_status(Server $server, string $container_id, bool $all_data = false, bool $throwError = false)
{
check_server_connection($server);
$container = instant_remote_process(["docker inspect --format '{{json .}}' {$container_id}"], $server, $throwError);
if (!$container) {
return 'exited';

36
bootstrap/helpers/remoteProcess.php
View File

@@ -7,6 +7,7 @@ use App\Models\Application;
use App\Models\ApplicationDeploymentQueue;
use App\Models\PrivateKey;
use App\Models\Server;
use App\Notifications\Server\NotReachable;
use Carbon\Carbon;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Support\Collection;
@@ -109,8 +110,8 @@ function instant_remote_process(array $command, Server $server, $throwError = tr
$exitCode = $process->exitCode();
if ($exitCode !== 0) {
if ($repeat > 1) {
ray("repeat: ", $repeat);
Sleep::for(200)->milliseconds();
ray('executing again');
return instant_remote_process($command, $server, $throwError, $repeat - 1);
}
// ray('ERROR OCCURED: ' . $process->errorOutput());
@@ -152,21 +153,22 @@ function decode_remote_command_output(?ApplicationDeploymentQueue $application_d
return $formatted;
}
function refreshPrivateKey(PrivateKey $private_key)
function refresh_server_connection(PrivateKey $private_key)
{
foreach ($private_key->servers as $server) {
// Delete the old ssh mux file to force a new one to be created
Storage::disk('ssh-mux')->delete($server->muxFilename());
if (auth()->user()->currentTeam()->id) {
auth()->user()->currentTeam()->privateKeys = PrivateKey::where('team_id', auth()->user()->currentTeam()->id)->get();
}
// check if user is authenticated
if (auth()?->user()?->currentTeam()->id) {
auth()->user()->currentTeam()->privateKeys = PrivateKey::where('team_id', auth()->user()->currentTeam()->id)->get();
}
}
}
function validateServer(Server $server)
{
try {
refreshPrivateKey($server->privateKey);
refresh_server_connection($server->privateKey);
$uptime = instant_remote_process(['uptime'], $server);
if (!$uptime) {
$uptime = 'Server not reachable.';
@@ -192,3 +194,25 @@ function validateServer(Server $server)
$server->settings->save();
}
}
function check_server_connection(Server $server) {
try {
refresh_server_connection($server->privateKey);
instant_remote_process(['uptime'], $server);
$server->unreachable_count = 0;
$server->settings->is_reachable = true;
} catch (\Exception $e) {
if ($server->unreachable_count == 2) {
$server->team->notify(new NotReachable($server));
$server->settings->is_reachable = false;
$server->settings->save();
} else {
$server->unreachable_count += 1;
}
throw $e;
} finally {
$server->settings->save();
$server->save();
}
}