dave/coolify
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

wip: swarm

Browse Source
This commit is contained in:
Andras Bacsai
2023-12-18 14:01:25 +01:00
parent 27c36bec83
commit 62c38c9859
24 changed files with 387 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
app/Console/Kernel.php
View File

@@ -56,16 +56,20 @@ class Kernel extends ConsoleKernel
$servers = Server::all()->whereNotNull('team.subscription')->where('team.subscription.stripe_trial_already_ended', false)->where('ip', '!=', '1.2.3.4');
$own = Team::find(0)->servers;
$servers = $servers->merge($own);
$containerServers = $servers->where('settings.is_swarm_worker', false);
} else {
$servers = Server::all()->where('ip', '!=', '1.2.3.4');
$containerServers = $servers->where('settings.is_swarm_worker', false);
}
foreach ($servers as $server) {
$schedule->job(new ServerStatusJob($server))->everyTenMinutes()->onOneServer();
foreach ($containerServers as $server) {
$schedule->job(new ContainerStatusJob($server))->everyMinute()->onOneServer();
if ($server->isLogDrainEnabled()) {
$schedule->job(new CheckLogDrainContainerJob($server))->everyMinute()->onOneServer();
}
}
foreach ($servers as $server) {
$schedule->job(new ServerStatusJob($server))->everyTenMinutes()->onOneServer();
}
}
private function instance_auto_update($schedule)
{

60
app/Jobs/ApplicationDeploymentJob.php
View File

@@ -217,19 +217,8 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted
if ($this->server->isProxyShouldRun()) {
dispatch(new ContainerStatusJob($this->server));
}
if ($this->application->docker_registry_image_name && $this->application->build_pack !== 'dockerimage') {
if ($this->application->docker_registry_image_name && $this->application->build_pack !== 'dockerimage' && !$this->application->destination->server->isSwarm()) {
$this->push_to_docker_registry();
if ($this->server->isSwarm()) {
$this->application_deployment_queue->addLogEntry("Creating / updating stack.");
$this->execute_remote_command(
[
executeInDocker($this->deployment_uuid, "cd {$this->workdir} && docker stack deploy --with-registry-auth -c docker-compose.yml {$this->application->uuid}")
],
[
"echo 'Stack deployed. It may take a few minutes to fully available in your swarm.'"
]
);
}
}
$this->next(ApplicationDeploymentStatus::FINISHED->value);
$this->application->isConfigurationChanged(true);
@@ -301,6 +290,9 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted
"echo -n 'Image pushed to docker registry.'"
]);
} catch (Exception $e) {
if ($this->application->destination->server->isSwarm()) {
throw $e;
}
$this->execute_remote_command(
["echo -n 'Failed to push image to docker registry. Please check debug logs for more information.'"],
);
@@ -604,7 +596,14 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted
private function rolling_update()
{
if ($this->server->isSwarm()) {
// Skip this.
$this->push_to_docker_registry();
$this->application_deployment_queue->addLogEntry("Rolling update started.");
$this->execute_remote_command(
[
executeInDocker($this->deployment_uuid, "docker stack deploy --with-registry-auth -c {$this->workdir}{$this->docker_compose_location} {$this->application->uuid}")
],
);
$this->application_deployment_queue->addLogEntry("Rolling update completed.");
} else {
if (count($this->application->ports_mappings_array) > 0) {
$this->execute_remote_command(
@@ -703,10 +702,20 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted
$this->add_build_env_variables_to_dockerfile();
$this->build_image();
$this->stop_running_container();
$this->execute_remote_command(
["echo -n 'Starting preview deployment.'"],
[executeInDocker($this->deployment_uuid, "docker compose --project-directory {$this->workdir} up -d"), "hidden" => true],
);
if ($this->application->destination->server->isSwarm()) {
ray("{$this->workdir}{$this->docker_compose_location}");
$this->push_to_docker_registry();
$this->execute_remote_command(
[
executeInDocker($this->deployment_uuid, "docker stack deploy --with-registry-auth -c {$this->workdir}{$this->docker_compose_location} {$this->application->uuid}-{$this->pull_request_id}")
],
);
} else {
$this->execute_remote_command(
["echo -n 'Starting preview deployment.'"],
[executeInDocker($this->deployment_uuid, "docker compose --project-directory {$this->workdir} up -d"), "hidden" => true],
);
}
}
private function create_workdir()
{
@@ -970,13 +979,8 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted
data_forget($docker_compose, 'services.' . $this->container_name . '.cpu_shares');
$docker_compose['services'][$this->container_name]['deploy'] = [
'placement' => [
'constraints' => [
'node.role == worker'
]
],
'mode' => 'replicated',
'replicas' => 1,
'replicas' => data_get($this->application, 'swarm_replicas', 1),
'update_config' => [
'order' => 'start-first'
],
@@ -995,6 +999,16 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted
]
]
];
if (data_get($this->application, 'settings.is_swarm_only_worker_nodes')) {
$docker_compose['services'][$this->container_name]['deploy']['placement'] = [
'constraints' => [
'node.role == worker'
]
];
}
if ($this->pull_request_id !== 0) {
$docker_compose['services'][$this->container_name]['deploy']['replicas'] = 1;
}
} else {
$docker_compose['services'][$this->container_name]['labels'] = $labels;
}

10
app/Jobs/ContainerStatusJob.php
View File

@@ -46,7 +46,7 @@ class ContainerStatusJob implements ShouldQueue, ShouldBeEncrypted
};
if ($this->server->isSwarm()) {
$containers = instant_remote_process(["docker service inspect $(docker service ls -q) --format '{{json .}}'"], $this->server, false);
$containerReplicase = instant_remote_process(["docker service ls --format '{{json .}}'"], $this->server, false);
$containerReplicates = instant_remote_process(["docker service ls --format '{{json .}}'"], $this->server, false);
} else {
// Precheck for containers
$containers = instant_remote_process(["docker container ls -q"], $this->server, false);
@@ -54,15 +54,15 @@ class ContainerStatusJob implements ShouldQueue, ShouldBeEncrypted
return;
}
$containers = instant_remote_process(["docker container inspect $(docker container ls -q) --format '{{json .}}'"], $this->server, false);
$containerReplicase = null;
$containerReplicates = null;
}
if (is_null($containers)) {
return;
}
$containers = format_docker_command_output_to_json($containers);
if ($containerReplicase) {
$containerReplicase = format_docker_command_output_to_json($containerReplicase);
foreach ($containerReplicase as $containerReplica) {
if ($containerReplicates) {
$containerReplicates = format_docker_command_output_to_json($containerReplicates);
foreach ($containerReplicates as $containerReplica) {
$name = data_get($containerReplica, 'Name');
$containers = $containers->map(function ($container) use ($name, $containerReplica) {
if (data_get($container, 'Spec.Name') === $name) {

2
app/Livewire/Project/Application/General.php
View File

@@ -113,7 +113,7 @@ class General extends Component
$this->application->isConfigurationChanged(true);
}
$this->isConfigurationChanged = $this->application->isConfigurationChanged();
$this->customLabels = $this->application->parseContainerLabels();
$this->customLabels = $this->application->parseContainerLabels();
$this->initialDockerComposeLocation = $this->application->docker_compose_location;
$this->checkLabelUpdates();
}

4
app/Livewire/Project/Application/Heading.php
View File

@@ -73,6 +73,10 @@ class Heading extends Component
$this->dispatch('error', 'Please load a Compose file first.');
return;
}
if ($this->application->destination->server->isSwarm() && is_null($this->application->docker_registry_image_name)) {
$this->dispatch('error', 'Please set a Docker image name first.');
return;
}
$this->setDeploymentUuid();
queue_application_deployment(
application_id: $this->application->id,

12
app/Livewire/Project/Application/Previews.php
View File

@@ -72,10 +72,14 @@ class Previews extends Component
public function stop(int $pull_request_id)
{
try {
$containers = getCurrentApplicationContainerStatus($this->application->destination->server, $this->application->id, $pull_request_id);
foreach ($containers as $container) {
$name = str_replace('/', '', $container['Names']);
instant_remote_process(["docker rm -f $name"], $this->application->destination->server, throwError: false);
if ($this->application->destination->server->isSwarm()) {
instant_remote_process(["docker stack rm {$this->application->uuid}-{$pull_request_id}"], $this->application->destination->server);
} else {
$containers = getCurrentApplicationContainerStatus($this->application->destination->server, $this->application->id, $pull_request_id);
foreach ($containers as $container) {
$name = str_replace('/', '', $container['Names']);
instant_remote_process(["docker rm -f $name"], $this->application->destination->server, throwError: false);
}
}
ApplicationPreview::where('application_id', $this->application->id)->where('pull_request_id', $pull_request_id)->first()->delete();
$this->application->refresh();

51
app/Livewire/Project/Application/Swarm.php Normal file
View File

@@ -0,0 +1,51 @@
<?php
namespace App\Livewire\Project\Application;
use App\Models\Application;
use Livewire\Component;
class Swarm extends Component
{
public Application $application;
public string $swarm_placement_constraints = '';
protected $rules = [
'application.swarm_replicas' => 'required',
'application.swarm_placement_constraints' => 'nullable',
'application.settings.is_swarm_only_worker_nodes' => 'required',
];
public function mount() {
if ($this->application->swarm_placement_constraints) {
$this->swarm_placement_constraints = base64_decode($this->application->swarm_placement_constraints);
}
}
public function instantSave() {
try {
$this->validate();
$this->application->settings->save();
$this->dispatch('success', 'Swarm settings updated.');
} catch (\Throwable $e) {
return handleError($e, $this);
}
}
public function submit() {
try {
$this->validate();
if ($this->swarm_placement_constraints) {
$this->application->swarm_placement_constraints = base64_encode($this->swarm_placement_constraints);
} else {
$this->application->swarm_placement_constraints = null;
}
$this->application->save();
$this->dispatch('success', 'Swarm settings updated.');
} catch (\Throwable $e) {
return handleError($e, $this);
}
}
public function render()
{
return view('livewire.project.application.swarm');
}
}

38
app/Livewire/Project/New/Select.php
View File

@@ -15,12 +15,15 @@ class Select extends Component
public string $type;
public string $server_id;
public string $destination_uuid;
public Countable|array|Server $allServers = [];
public Countable|array|Server $servers = [];
public Collection|array $standaloneDockers = [];
public Collection|array $swarmDockers = [];
public array $parameters;
public Collection|array $services = [];
public Collection|array $allServices = [];
public bool $isDatabase = false;
public bool $includeSwarm = true;
public bool $loadingServices = true;
public bool $loading = false;
@@ -96,19 +99,43 @@ class Select extends Component
$this->loadingServices = false;
}
}
public function instantSave()
{
if ($this->includeSwarm) {
$this->servers = $this->allServers;
} else {
$this->servers = $this->allServers->where('settings.is_swarm_worker', false)->where('settings.is_swarm_manager', false);
}
}
public function setType(string $type)
{
$this->type = $type;
if ($this->loading) return;
$this->loading = true;
$this->type = $type;
switch ($type) {
case 'postgresql':
case 'mysql':
case 'mariadb':
case 'redis':
case 'mongodb':
$this->isDatabase = true;
$this->includeSwarm = false;
$this->servers = $this->allServers->where('settings.is_swarm_worker', false)->where('settings.is_swarm_manager', false);
break;
}
if (str($type)->startsWith('one-click-service')) {
$this->isDatabase = true;
$this->includeSwarm = false;
$this->servers = $this->allServers->where('settings.is_swarm_worker', false)->where('settings.is_swarm_manager', false);
}
if ($type === "existing-postgresql") {
$this->current_step = $type;
return;
}
if (count($this->servers) === 1) {
$server = $this->servers->first();
$this->setServer($server);
}
// if (count($this->servers) === 1) {
// $server = $this->servers->first();
// $this->setServer($server);
// }
if (!is_null($this->server)) {
$foundServer = $this->servers->where('id', $this->server->id)->first();
if ($foundServer) {
@@ -142,5 +169,6 @@ class Select extends Component
public function loadServers()
{
$this->servers = Server::isUsable()->get();
$this->allServers = $this->servers;
}
}

38
app/Livewire/Project/Shared/Storages/Add.php
View File

@@ -2,22 +2,26 @@
namespace App\Livewire\Project\Shared\Storages;
use App\Models\Application;
use Livewire\Component;
class Add extends Component
{
public $uuid;
public $parameters;
public $isSwarm = false;
public string $name;
public string $mount_path;
public string|null $host_path = null;
public ?string $host_path = null;
protected $listeners = ['clearAddStorage' => 'clear'];
protected $rules = [
public $rules = [
'name' => 'required|string',
'mount_path' => 'required|string',
'host_path' => 'string|nullable',
];
protected $listeners = ['clearAddStorage' => 'clear'];
protected $validationAttributes = [
'name' => 'name',
'mount_path' => 'mount',
@@ -27,17 +31,31 @@ class Add extends Component
public function mount()
{
$this->parameters = get_route_parameters();
$applicationUuid = $this->parameters['application_uuid'];
$application = Application::where('uuid', $applicationUuid)->first();
if (!$application) {
abort(404);
}
if ($application->destination->server->isSwarm()) {
$this->isSwarm = true;
$this->rules['host_path'] = 'required|string';
}
}
public function submit()
{
$this->validate();
$name = $this->uuid . '-' . $this->name;
$this->dispatch('addNewVolume', [
'name' => $name,
'mount_path' => $this->mount_path,
'host_path' => $this->host_path,
]);
try {
$this->validate($this->rules);
$name = $this->uuid . '-' . $this->name;
$this->dispatch('addNewVolume', [
'name' => $name,
'mount_path' => $this->mount_path,
'host_path' => $this->host_path,
]);
$this->dispatch('closeStorageModal');
} catch (\Throwable $e) {
return handleError($e, $this);
}
}
public function clear()

4
app/Livewire/Server/Form.php
View File

@@ -25,7 +25,7 @@ class Form extends Component
'server.settings.is_cloudflare_tunnel' => 'required|boolean',
'server.settings.is_reachable' => 'required',
'server.settings.is_swarm_manager' => 'required|boolean',
// 'server.settings.is_swarm_worker' => 'required|boolean',
'server.settings.is_swarm_worker' => 'required|boolean',
'wildcard_domain' => 'nullable|url',
];
protected $validationAttributes = [
@@ -37,7 +37,7 @@ class Form extends Component
'server.settings.is_cloudflare_tunnel' => 'Cloudflare Tunnel',
'server.settings.is_reachable' => 'Is reachable',
'server.settings.is_swarm_manager' => 'Swarm Manager',
// 'server.settings.is_swarm_worker' => 'Swarm Worker',
'server.settings.is_swarm_worker' => 'Swarm Worker',
];
public function mount()

5
app/Livewire/Server/New/ByIp.php
View File

@@ -22,6 +22,8 @@ class ByIp extends Component
public string $user = 'root';
public int $port = 22;
public bool $is_swarm_manager = false;
public bool $is_swarm_worker = false;
protected $rules = [
'name' => 'required|string',
@@ -30,6 +32,7 @@ class ByIp extends Component
'user' => 'required|string',
'port' => 'required|integer',
'is_swarm_manager' => 'required|boolean',
'is_swarm_worker' => 'required|boolean',
];
protected $validationAttributes = [
'name' => 'Name',
@@ -38,6 +41,7 @@ class ByIp extends Component
'user' => 'User',
'port' => 'Port',
'is_swarm_manager' => 'Swarm Manager',
'is_swarm_worker' => 'Swarm Worker',
];
public function mount()
@@ -77,6 +81,7 @@ class ByIp extends Component
],
]);
$server->settings->is_swarm_manager = $this->is_swarm_manager;
$server->settings->is_swarm_worker = $this->is_swarm_worker;
$server->settings->save();
$server->addInitialNetwork();
return $this->redirectRoute('server.show', $server->uuid, navigate: true);

10
app/Models/Server.php
View File

@@ -72,7 +72,7 @@ class Server extends BaseModel
static public function isUsable()
{
return Server::ownedByCurrentTeam()->whereRelation('settings', 'is_reachable', true)->whereRelation('settings', 'is_usable', true);
return Server::ownedByCurrentTeam()->whereRelation('settings', 'is_reachable', true)->whereRelation('settings', 'is_usable', true)->whereRelation('settings', 'is_swarm_worker', false);
}
static public function destinationsByServer(string $server_id)
@@ -380,6 +380,14 @@ class Server extends BaseModel
{
return data_get($this, 'settings.is_swarm_manager') || data_get($this, 'settings.is_swarm_worker');
}
public function isSwarmManager()
{
return data_get($this, 'settings.is_swarm_manager');
}
public function isSwarmWorker()
{
return data_get($this, 'settings.is_swarm_worker');
}
public function validateConnection()
{
$server = Server::find($this->id);