dave/coolify
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

fix: new logging for deployment jobs

Browse Source 
fix: git based docker compose files
This commit is contained in:
Andras Bacsai
2023-11-27 11:54:55 +01:00
parent fae97e4dee
commit 8d86d53292
21 changed files with 247 additions and 168 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
app/Console/Commands/ServicesGenerate.php
View File

@@ -26,7 +26,7 @@ class ServicesGenerate extends Command
*/
public function handle()
{
ray()->clearAll();
// ray()->clearAll();
$files = array_diff(scandir(base_path('templates/compose')), ['.', '..']);
$files = array_filter($files, function ($file) {
return strpos($file, '.yaml') !== false;

55
app/Http/Livewire/Project/Application/General.php
View File

@@ -26,6 +26,8 @@ class General extends Component
public bool $labelsChanged = false;
public bool $isConfigurationChanged = false;
public ?string $initialDockerComposeLocation = null;
public bool $is_static;
public $parsedServices = [];
@@ -109,6 +111,7 @@ class General extends Component
} else {
$this->customLabels = str($this->application->custom_labels)->replace(',', "\n");
}
$this->initialDockerComposeLocation = $this->application->docker_compose_location;
$this->checkLabelUpdates();
}
public function instantSave()
@@ -118,35 +121,30 @@ class General extends Component
}
public function loadComposeFile($isInit = false)
{
if ($isInit && $this->application->docker_compose_raw) {
return;
}
$uuid = new Cuid2();
['commands' => $cloneCommand] = $this->application->generateGitImportCommands(deployment_uuid: $uuid, only_checkout: true, exec_in_docker: false, custom_base_dir: '.');
$workdir = rtrim($this->application->base_directory, '/');
$composeFile = $this->application->docker_compose_location;
$commands = collect([
"mkdir -p /tmp/{$uuid} && cd /tmp/{$uuid}",
$cloneCommand,
"git sparse-checkout init --cone",
"git sparse-checkout set .$workdir$composeFile",
"git read-tree -mu HEAD",
"cat .$workdir$composeFile",
]);
$composeFileContent = instant_remote_process($commands, $this->application->destination->server, false);
if (!$composeFileContent) {
$this->emit('error', "Could not load compose file from $workdir$composeFile");
return;
} else {
$this->application->docker_compose_raw = $composeFileContent;
try {
if ($isInit && $this->application->docker_compose_raw) {
return;
}
['parsedServices' => $this->parsedServices, 'initialDockerComposeLocation' => $this->initialDockerComposeLocation] = $this->application->loadComposeFile($isInit);
$this->emit('success', 'Docker compose file loaded.');
} catch (\Throwable $e) {
$this->application->docker_compose_location = $this->initialDockerComposeLocation;
$this->application->save();
return handleError($e, $this);
}
$commands = collect([
"rm -rf /tmp/{$uuid}",
]);
instant_remote_process($commands, $this->application->destination->server, false);
$this->parsedServices = $this->application->parseCompose();
$this->emit('success', 'Compose file loaded.');
}
public function generateDomain(string $serviceName)
{
$domain = $this->parsedServiceDomains[$serviceName]['domain'] ?? null;
if (!$domain) {
$uuid = new Cuid2(7);
$domain = generateFqdn($this->application->destination->server, $uuid);
$this->parsedServiceDomains[$serviceName]['domain'] = $domain;
$this->application->docker_compose_domains = json_encode($this->parsedServiceDomains);
$this->application->save();
$this->emit('success', 'Domain generated.');
}
return $domain;
}
public function updatedApplicationBuildPack()
{
@@ -190,6 +188,9 @@ class General extends Component
public function submit($showToaster = true)
{
try {
if ($this->initialDockerComposeLocation !== $this->application->docker_compose_location) {
$this->loadComposeFile();
}
$this->validate();
if ($this->ports_exposes !== $this->application->ports_exposes) {
$this->resetDefaultLabels(false);

7
app/Http/Livewire/Project/Application/Heading.php
View File

@@ -41,6 +41,10 @@ class Heading extends Component
public function deploy(bool $force_rebuild = false)
{
if (!$this->application->deployableComposeBuildPack()) {
$this->emit('error', 'Please load a Compose file first.');
return;
}
$this->setDeploymentUuid();
queue_application_deployment(
application_id: $this->application->id,
@@ -68,7 +72,8 @@ class Heading extends Component
$this->application->save();
$this->application->refresh();
}
public function restart() {
public function restart()
{
$this->setDeploymentUuid();
queue_application_deployment(
application_id: $this->application->id,

1
app/Http/Livewire/Project/Shared/EnvironmentVariable/Show.php
View File

@@ -59,6 +59,7 @@ class Show extends Component
{
$this->validate();
$this->env->save();
ray($this->env);
$this->emit('success', 'Environment variable updated successfully.');
$this->emit('refreshEnvs');
}

8
app/Http/Livewire/Project/Shared/Logs.php
View File

@@ -17,13 +17,15 @@ class Logs extends Component
public ?string $type = null;
public Application|Service|StandalonePostgresql|StandaloneRedis|StandaloneMongodb|StandaloneMysql|StandaloneMariadb $resource;
public Server $server;
public ?string $container = null;
public $container = [];
public $containers;
public $parameters;
public $query;
public $status;
public function mount()
{
$this->containers = collect();
$this->parameters = get_route_parameters();
$this->query = request()->query();
if (data_get($this->parameters, 'application_uuid')) {
@@ -33,7 +35,9 @@ class Logs extends Component
$this->server = $this->resource->destination->server;
$containers = getCurrentApplicationContainerStatus($this->server, $this->resource->id, 0);
if ($containers->count() > 0) {
$this->container = data_get($containers[0], 'Names');
$containers->each(function ($container) {
$this->containers->push(str_replace('/', '', $container['Names']));
});
}
} else if (data_get($this->parameters, 'database_uuid')) {
$this->type = 'database';

2
app/Jobs/ApplicationDeployDockerImageJob.php
View File

@@ -30,7 +30,7 @@ class ApplicationDeployDockerImageJob implements ShouldQueue, ShouldBeEncrypted
}
public function handle()
{
ray()->clearAll();
// ray()->clearAll();
ray('Deploying Docker Image');
static::$batch_counter = 0;
try {

29
app/Jobs/ApplicationDeploymentJob.php
View File

@@ -76,7 +76,6 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted
private string $docker_compose_location = '/docker-compose.yml';
private ?string $addHosts = null;
private ?string $buildTarget = null;
private $log_model;
private Collection $saved_outputs;
private ?string $full_healthcheck_url = null;
@@ -93,9 +92,7 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted
public $tries = 1;
public function __construct(int $application_deployment_queue_id)
{
// ray()->clearScreen();
$this->application_deployment_queue = ApplicationDeploymentQueue::find($application_deployment_queue_id);
$this->log_model = $this->application_deployment_queue;
$this->application = Application::find($this->application_deployment_queue->application_id);
$this->build_pack = data_get($this->application, 'build_pack');
@@ -187,7 +184,6 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted
['repository' => $this->customRepository, 'port' => $this->customPort] = $this->application->customRepository();
try {
ray($this->application->build_pack);
if ($this->restart_only && $this->application->build_pack !== 'dockerimage') {
$this->just_restart();
if ($this->server->isProxyShouldRun()) {
@@ -451,7 +447,6 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted
$this->docker_compose_location = $this->application->docker_compose_location;
}
$this->application_deployment_queue->addLogEntry("Starting deployment of {$this->application->name}.");
$this->server->executeRemoteCommand(
commands: $this->application->prepareHelperImage($this->deployment_uuid),
loggingModel: $this->application_deployment_queue
@@ -474,6 +469,7 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted
$this->save_environment_variables();
$this->stop_running_container(force: true);
$this->start_by_compose_file();
$this->application->loadComposeFile(isInit: false);
}
private function deploy_dockerfile_buildpack()
{
@@ -752,14 +748,9 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted
private function clone_repository()
{
$importCommands = $this->generate_git_import_commands();
ray($importCommands);
$this->application_deployment_queue->addLogEntry("\n----------------------------------------");
$this->application_deployment_queue->addLogEntry("Importing {$this->customRepository}:{$this->application->git_branch} (commit sha {$this->application->git_commit_sha}) to {$this->basedir}.");
$this->execute_remote_command(
[
"echo '\n----------------------------------------'",
],
[
"echo -n 'Importing {$this->customRepository}:{$this->application->git_branch} (commit sha {$this->application->git_commit_sha}) to {$this->basedir}. '"
],
[
$importCommands, "hidden" => true
]
@@ -768,7 +759,6 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted
private function generate_git_import_commands()
{
['commands' => $commands, 'branch' => $this->branch, 'fullRepoUrl' => $this->fullRepoUrl] = $this->application->generateGitImportCommands($this->deployment_uuid, $this->pull_request_id, $this->git_type);
return $commands;
}
@@ -1178,10 +1168,9 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf");
private function stop_running_container(bool $force = false)
{
$this->execute_remote_command(["echo -n 'Removing old container.'"]);
$this->application_deployment_queue->addLogEntry("Removing old containers.");
if ($this->newVersionIsHealthy || $force) {
$containers = getCurrentApplicationContainerStatus($this->server, $this->application->id, $this->pull_request_id);
ray($containers);
if ($this->pull_request_id !== 0) {
$containers = $containers->filter(function ($container) {
return data_get($container, 'Names') === $this->container_name;
@@ -1197,14 +1186,10 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf");
[executeInDocker($this->deployment_uuid, "docker rm -f $containerName >/dev/null 2>&1"), "hidden" => true, "ignore_errors" => true],
);
});
$this->execute_remote_command(
[
"echo 'Rolling update completed.'"
],
);
$this->application_deployment_queue->addLogEntry("Rolling update completed.");
} else {
$this->application_deployment_queue->addLogEntry("New container is not healthy, rolling back to the old container.");
$this->execute_remote_command(
["echo -n 'New container is not healthy, rolling back to the old container.'"],
[executeInDocker($this->deployment_uuid, "docker rm -f $this->container_name >/dev/null 2>&1"), "hidden" => true, "ignore_errors" => true],
);
}
@@ -1213,8 +1198,8 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf");
private function start_by_compose_file()
{
if ($this->application->build_pack === 'dockerimage') {
$this->application_deployment_queue->addLogEntry("Pulling latest images from the registry.");
$this->execute_remote_command(
["echo -n 'Pulling latest images from the registry.'"],
[executeInDocker($this->deployment_uuid, "docker compose --project-directory {$this->workdir} pull"), "hidden" => true],
[executeInDocker($this->deployment_uuid, "docker compose --project-directory {$this->workdir} up --build -d"), "hidden" => true],
);

1
app/Jobs/MultipleApplicationDeploymentJob.php
View File

@@ -91,7 +91,6 @@ class MultipleApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted
public $tries = 1;
public function __construct(int $application_deployment_queue_id)
{
// ray()->clearScreen();
$this->application_deployment_queue = ApplicationDeploymentQueue::find($application_deployment_queue_id);
$this->log_model = $this->application_deployment_queue;
$this->application = Application::find($this->application_deployment_queue->application_id);

43
app/Models/Application.php
View File

@@ -8,6 +8,7 @@ use Spatie\Activitylog\Models\Activity;
use Illuminate\Support\Str;
use RuntimeException;
use Symfony\Component\Yaml\Yaml;
use Visus\Cuid2\Cuid2;
class Application extends BaseModel
{
@@ -47,6 +48,10 @@ class Application extends BaseModel
$application->environment_variables_preview()->delete();
});
}
public function deployableComposeBuildPack()
{
return $this->build_pack === 'dockercompose' && $this->docker_compose_raw;
}
public function link()
{
return route('project.application.configuration', [
@@ -592,4 +597,42 @@ class Application extends BaseModel
return collect([]);
}
}
function loadComposeFile($isInit = false)
{
$initialDockerComposeLocation = $this->docker_compose_location;
if ($this->build_pack === 'dockercompose') {
if ($isInit && $this->docker_compose_raw) {
return;
}
$uuid = new Cuid2();
['commands' => $cloneCommand] = $this->generateGitImportCommands(deployment_uuid: $uuid, only_checkout: true, exec_in_docker: false, custom_base_dir: '.');
$workdir = rtrim($this->base_directory, '/');
$composeFile = $this->docker_compose_location;
$commands = collect([
"mkdir -p /tmp/{$uuid} && cd /tmp/{$uuid}",
$cloneCommand,
"git sparse-checkout init --cone",
"git sparse-checkout set .$workdir$composeFile",
"git read-tree -mu HEAD",
"cat .$workdir$composeFile",
]);
$composeFileContent = instant_remote_process($commands, $this->destination->server, false);
if (!$composeFileContent) {
$this->docker_compose_location = $initialDockerComposeLocation;
$this->save();
throw new \Exception("Could not load compose file from $workdir$composeFile");
} else {
$this->docker_compose_raw = $composeFileContent;
$this->save();
}
$commands = collect([
"rm -rf /tmp/{$uuid}",
]);
instant_remote_process($commands, $this->destination->server, false);
return [
'parsedServices' => $this->parseCompose(),
'initialDockerComposeLocation' => $this->docker_compose_location
];
}
}
}

9
app/Models/ApplicationDeploymentQueue.php
View File

@@ -9,7 +9,8 @@ class ApplicationDeploymentQueue extends Model
{
protected $guarded = [];
public function getOutput($name) {
public function getOutput($name)
{
if (!$this->logs) {
return null;
}
@@ -21,9 +22,13 @@ class ApplicationDeploymentQueue extends Model
if ($type === 'error') {
$type = 'stderr';
}
$message = str($message)->trim();
if ($message->startsWith('╔')) {
$message = "\n" . $message;
}
$newLogEntry = [
'command' => null,
'output' => $message,
'output' => remove_iip($message),
'type' => $type,
'timestamp' => Carbon::now('UTC'),
'hidden' => $hidden,

18
app/Traits/ExecuteRemoteCommand.php
View File

@@ -24,7 +24,6 @@ trait ExecuteRemoteCommand
if ($this->server instanceof Server === false) {
throw new \RuntimeException('Server is not set or is not an instance of Server model');
}
$commandsText->each(function ($single_command) {
$command = data_get($single_command, 'command') ?? $single_command[0] ?? null;
if ($command === null) {
@@ -49,32 +48,29 @@ trait ExecuteRemoteCommand
'hidden' => $hidden,
'batch' => static::$batch_counter,
];
if (!$this->log_model->logs) {
if (!$this->application_deployment_queue->logs) {
$new_log_entry['order'] = 1;
} else {
$previous_logs = json_decode($this->log_model->logs, associative: true, flags: JSON_THROW_ON_ERROR);
$previous_logs = json_decode($this->application_deployment_queue->logs, associative: true, flags: JSON_THROW_ON_ERROR);
$new_log_entry['order'] = count($previous_logs) + 1;
}
$previous_logs[] = $new_log_entry;
$this->log_model->logs = json_encode($previous_logs, flags: JSON_THROW_ON_ERROR);
$this->log_model->save();
$this->application_deployment_queue->logs = json_encode($previous_logs, flags: JSON_THROW_ON_ERROR);
$this->application_deployment_queue->save();
if ($this->save) {
$this->saved_outputs[$this->save] = Str::of($output)->trim();
}
});
$this->log_model->update([
$this->application_deployment_queue->update([
'current_process_id' => $process->id(),
]);
$process_result = $process->wait();
if ($process_result->exitCode() !== 0) {
if (!$ignore_errors) {
$status = ApplicationDeploymentStatus::FAILED->value;
$this->log_model->status = $status;
$this->log_model->save();
$this->application_deployment_queue->status = ApplicationDeploymentStatus::FAILED->value;
$this->application_deployment_queue->save();
throw new \RuntimeException($process_result->errorOutput());
}
}