wip: new deployment jobs

app/Jobs/ApplicationDeployDockerImageJob.php

@@ -0,0 +1,111 @@
+<?php
+
+namespace App\Jobs;
+
+use App\Enums\ApplicationDeploymentStatus;
+use App\Models\Application;
+use App\Models\ApplicationDeploymentQueue;
+use App\Traits\ExecuteRemoteCommandNew;
+use Exception;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeEncrypted;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Throwable;
+
+class ApplicationDeployDockerImageJob implements ShouldQueue, ShouldBeEncrypted
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels, ExecuteRemoteCommandNew;
+
+    public $timeout = 3600;
+    public $tries = 1;
+    public string $applicationDeploymentQueueId;
+
+    public function __construct(string $applicationDeploymentQueueId)
+    {
+        $this->applicationDeploymentQueueId = $applicationDeploymentQueueId;
+    }
+    public function handle()
+    {
+        ray()->clearAll();
+        ray('Deploying Docker Image');
+        try {
+            $applicationDeploymentQueue = ApplicationDeploymentQueue::find($this->applicationDeploymentQueueId);
+            $application = Application::find($applicationDeploymentQueue->application_id);
+
+            $deploymentUuid = data_get($applicationDeploymentQueue, 'deployment_uuid');
+            $dockerImage = data_get($application, 'docker_registry_image_name');
+            $dockerImageTag = data_get($application, 'docker_registry_image_tag');
+            $productionImageName = str("{$dockerImage}:{$dockerImageTag}");
+            $destination = $application->destination->getMorphClass()::where('id', $application->destination->id)->first();
+            $pullRequestId = data_get($applicationDeploymentQueue, 'pull_request_id');
+
+            $server = data_get($destination, 'server');
+            $network = data_get($destination, 'network');
+
+            $containerName = generateApplicationContainerName($application, $pullRequestId);
+            savePrivateKeyToFs($server);
+
+            ray("echo 'Starting deployment of {$productionImageName}.'");
+
+            $applicationDeploymentQueue->update([
+                'status' => ApplicationDeploymentStatus::IN_PROGRESS->value,
+            ]);
+            $this->executeRemoteCommand(
+                server: $server,
+                logModel: $applicationDeploymentQueue,
+                commands: prepareHelperContainer($server, $network, $deploymentUuid)
+            );
+
+            $this->executeRemoteCommand(
+                server: $server,
+                logModel: $applicationDeploymentQueue,
+                commands: generateComposeFile(
+                    deploymentUuid: $deploymentUuid,
+                    server: $server,
+                    network: $network,
+                    application: $application,
+                    containerName: $containerName,
+                    imageName: $productionImageName,
+                    pullRequestId: $pullRequestId
+                )
+            );
+            $this->executeRemoteCommand(
+                server: $server,
+                logModel: $applicationDeploymentQueue,
+                commands: rollingUpdate(application: $application, deploymentUuid: $deploymentUuid)
+            );
+        } catch (Throwable $e) {
+            $this->executeRemoteCommand(
+                server: $server,
+                logModel: $applicationDeploymentQueue,
+                commands: [
+                    "echo 'Oops something is not okay, are you okay? 😢'",
+                    "echo '{$e->getMessage()}'",
+                    "echo -n 'Deployment failed. Removing the new version of your application.'",
+                    executeInDocker($deploymentUuid, "docker rm -f $containerName >/dev/null 2>&1"),
+                ]
+            );
+            // $this->next(ApplicationDeploymentStatus::FAILED->value);
+            throw $e;
+        }
+    }
+    // private function next(string $status)
+    // {
+    //     // If the deployment is cancelled by the user, don't update the status
+    //     if ($this->application_deployment_queue->status !== ApplicationDeploymentStatus::CANCELLED_BY_USER->value) {
+    //         $this->application_deployment_queue->update([
+    //             'status' => $status,
+    //         ]);
+    //     }
+    //     queue_next_deployment($this->application);
+    //     if ($status === ApplicationDeploymentStatus::FINISHED->value) {
+    //         $this->application->environment->project->team->notify(new DeploymentSuccess($this->application, $this->deployment_uuid, $this->preview));
+    //     }
+    //     if ($status === ApplicationDeploymentStatus::FAILED->value) {
+    //         $this->application->environment->project->team->notify(new DeploymentFailed($this->application, $this->deployment_uuid, $this->preview));
+    //     }
+    // }
+}

app/Jobs/ApplicationDeploySimpleDockerfileJob.php

@@ -0,0 +1,29 @@
+<?php
+
+namespace App\Jobs;
+
+use App\Traits\ExecuteRemoteCommand;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeEncrypted;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+
+
+class ApplicationDeploySimpleDockerfileJob implements ShouldQueue, ShouldBeEncrypted
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels, ExecuteRemoteCommand;
+
+    public $timeout = 3600;
+    public $tries = 1;
+    public string $applicationDeploymentQueueId;
+
+    public function __construct(string $applicationDeploymentQueueId)
+    {
+        $this->applicationDeploymentQueueId = $applicationDeploymentQueueId;
+    }
+    public function handle() {
+        ray('Deploying Simple Dockerfile');
+    }
+}

app/Jobs/ApplicationRestartJob.php

@@ -0,0 +1,28 @@
+<?php
+
+namespace App\Jobs;
+
+use App\Traits\ExecuteRemoteCommand;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeEncrypted;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+
+
+class ApplicationRestartJob implements ShouldQueue, ShouldBeEncrypted
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels, ExecuteRemoteCommand;
+
+    public $timeout = 3600;
+    public $tries = 1;
+    public string $applicationDeploymentQueueId;
+    public function __construct(string $applicationDeploymentQueueId)
+    {
+        $this->applicationDeploymentQueueId = $applicationDeploymentQueueId;
+    }
+    public function handle() {
+        ray('Restarting application');
+    }
+}

app/Models/Application.php

@@ -332,4 +332,14 @@ class Application extends BaseModel
             return true;
         }
     }
+    public function isMultipleServerDeployment()
+    {
+        if (isDev()) {
+            return true;
+        }
+        if (data_get($this, 'additional_destinations') && data_get($this, 'docker_registry_image_name')) {
+            return true;
+        }
+        return false;
+    }
 }

app/Models/Server.php

@@ -171,7 +171,7 @@ class Server extends BaseModel
                 break;
             }
             $result = $this->validateConnection();
-            ray('validateConnection: ' . $result);
+            // ray('validateConnection: ' . $result);
             if (!$result) {
                 $serverUptimeCheckNumber++;
                 $this->update([

app/Traits/ExecuteRemoteCommand.php

@@ -12,7 +12,6 @@ use Illuminate\Support\Str;
 trait ExecuteRemoteCommand
 {
     public ?string $save = null;
-
     public function execute_remote_command(...$commands)
     {
         static::$batch_counter++;

app/Traits/ExecuteRemoteCommandNew.php

@@ -0,0 +1,77 @@
+<?php
+
+namespace App\Traits;
+
+use App\Enums\ApplicationDeploymentStatus;
+use App\Models\Server;
+use Carbon\Carbon;
+use Illuminate\Support\Collection;
+use Illuminate\Support\Facades\Process;
+
+trait ExecuteRemoteCommandNew
+{
+    public static $batch_counter = 0;
+    public function executeRemoteCommand(Server $server, $logModel, $commands)
+    {
+        static::$batch_counter++;
+        if ($commands instanceof Collection) {
+            $commandsText = $commands;
+        } else {
+            $commandsText = collect($commands);
+        }
+        $commandsText->each(function ($singleCommand) use ($server, $logModel) {
+            $command = data_get($singleCommand, 'command') ?? $singleCommand[0] ?? null;
+            if ($command === null) {
+                throw new \RuntimeException('Command is not set');
+            }
+            $hidden = data_get($singleCommand, 'hidden', false);
+            $customType = data_get($singleCommand, 'type');
+            $ignoreErrors = data_get($singleCommand, 'ignore_errors', false);
+            $save = data_get($singleCommand, 'save');
+
+            $remote_command = generateSshCommand($server, $command);
+            $process = Process::timeout(3600)->idleTimeout(3600)->start($remote_command, function (string $type, string $output) use ($command, $hidden, $customType, $logModel, $save) {
+                $output = str($output)->trim();
+                if ($output->startsWith('╔')) {
+                    $output = "\n" . $output;
+                }
+                $newLogEntry = [
+                    'command' => remove_iip($command),
+                    'output' => remove_iip($output),
+                    'type' => $customType ?? $type === 'err' ? 'stderr' : 'stdout',
+                    'timestamp' => Carbon::now('UTC'),
+                    'hidden' => $hidden,
+                    'batch' => static::$batch_counter,
+                ];
+
+                if (!$logModel->logs) {
+                    $newLogEntry['order'] = 1;
+                } else {
+                    $previousLogs = json_decode($logModel->logs, associative: true, flags: JSON_THROW_ON_ERROR);
+                    $newLogEntry['order'] = count($previousLogs) + 1;
+                }
+
+                $previousLogs[] = $newLogEntry;
+                $logModel->logs = json_encode($previousLogs, flags: JSON_THROW_ON_ERROR);
+                $logModel->save();
+
+                if ($save) {
+                    $this->remoteCommandOutputs[$save] = str($output)->trim();
+                }
+            });
+            $logModel->update([
+                'current_process_id' => $process->id(),
+            ]);
+
+            $processResult = $process->wait();
+            if ($processResult->exitCode() !== 0) {
+                if (!$ignoreErrors) {
+                    $status = ApplicationDeploymentStatus::FAILED->value;
+                    $logModel->status = $status;
+                    $logModel->save();
+                    throw new \RuntimeException($processResult->errorOutput());
+                }
+            }
+        });
+    }
+}

bootstrap/helpers/applications.php

@@ -1,8 +1,15 @@
 <?php
 
+use App\Jobs\ApplicationDeployDockerImageJob;
 use App\Jobs\ApplicationDeploymentJob;
+use App\Jobs\ApplicationDeploySimpleDockerfileJob;
+use App\Jobs\ApplicationRestartJob;
+use App\Jobs\MultipleApplicationDeploymentJob;
 use App\Models\Application;
 use App\Models\ApplicationDeploymentQueue;
+use App\Models\ApplicationPreview;
+use App\Models\Server;
+use Symfony\Component\Yaml\Yaml;
 
 function queue_application_deployment(int $application_id, string $deployment_uuid, int | null $pull_request_id = 0, string $commit = 'HEAD', bool $force_rebuild = false, bool $is_webhook = false, bool $restart_only = false, ?string $git_type = null)
 {
@@ -29,17 +36,305 @@ function queue_application_deployment(int $application_id, string $deployment_uu
     if ($running_deployments->count() > 0) {
         return;
     }
+    // New deployment
+    // dispatchDeploymentJob($deployment->id);
     dispatch(new ApplicationDeploymentJob(
         application_deployment_queue_id: $deployment->id,
     ))->onConnection('long-running')->onQueue('long-running');
+
 }
 
 function queue_next_deployment(Application $application)
 {
     $next_found = ApplicationDeploymentQueue::where('application_id', $application->id)->where('status', 'queued')->first();
     if ($next_found) {
+         // New deployment
+        // dispatchDeploymentJob($next_found->id);
         dispatch(new ApplicationDeploymentJob(
             application_deployment_queue_id: $next_found->id,
         ))->onConnection('long-running')->onQueue('long-running');
+
+    }
+}
+function dispatchDeploymentJob($id)
+{
+    $applicationQueue = ApplicationDeploymentQueue::find($id);
+    $application = Application::find($applicationQueue->application_id);
+
+    $isRestartOnly = data_get($applicationQueue, 'restart_only');
+    $isSimpleDockerFile = data_get($application, 'dockerfile');
+    $isDockerImage = data_get($application, 'build_pack') === 'dockerimage';
+
+    if ($isRestartOnly) {
+        ApplicationRestartJob::dispatch(applicationDeploymentQueueId: $id)->onConnection('long-running')->onQueue('long-running');
+    } else if ($isSimpleDockerFile) {
+        ApplicationDeploySimpleDockerfileJob::dispatch(applicationDeploymentQueueId: $id)->onConnection('long-running')->onQueue('long-running');
+    } else if ($isDockerImage) {
+        ApplicationDeployDockerImageJob::dispatch(applicationDeploymentQueueId: $id)->onConnection('long-running')->onQueue('long-running');
+    } else {
+        throw new Exception('Unknown build pack');
+    }
+}
+
+// Deployment things
+function generateHostIpMapping(Server $server, string $network)
+{
+    // Generate custom host<->ip hostnames
+    $allContainers = instant_remote_process(["docker network inspect {$network} -f '{{json .Containers}}' "], $server);
+    $allContainers = format_docker_command_output_to_json($allContainers);
+    $ips = collect([]);
+    if (count($allContainers) > 0) {
+        $allContainers = $allContainers[0];
+        foreach ($allContainers as $container) {
+            $containerName = data_get($container, 'Name');
+            if ($containerName === 'coolify-proxy') {
+                continue;
+            }
+            $containerIp = data_get($container, 'IPv4Address');
+            if ($containerName && $containerIp) {
+                $containerIp = str($containerIp)->before('/');
+                $ips->put($containerName, $containerIp->value());
+            }
+        }
+    }
+    return $ips->map(function ($ip, $name) {
+        return "--add-host $name:$ip";
+    })->implode(' ');
+}
+
+function generateBaseDir(string $deplyomentUuid)
+{
+    return "/artifacts/$deplyomentUuid";
+}
+function generateWorkdir(string $deplyomentUuid, Application $application)
+{
+    return generateBaseDir($deplyomentUuid) . rtrim($application->base_directory, '/');
+}
+
+function prepareHelperContainer(Server $server, string $network, string $deploymentUuid)
+{
+    $basedir = generateBaseDir($deploymentUuid);
+    $helperImage = config('coolify.helper_image');
+
+    $serverUserHomeDir = instant_remote_process(["echo \$HOME"], $server);
+    $dockerConfigFileExists = instant_remote_process(["test -f {$serverUserHomeDir}/.docker/config.json && echo 'OK' || echo 'NOK'"], $server);
+
+    $commands = collect([]);
+    if ($dockerConfigFileExists === 'OK') {
+        $commands->push([
+            "command" => "docker run -d --network $network -v /:/host --name $deploymentUuid --rm -v {$serverUserHomeDir}/.docker/config.json:/root/.docker/config.json:ro -v /var/run/docker.sock:/var/run/docker.sock $helperImage",
+            "hidden" => true,
+        ]);
+    } else {
+        $commands->push([
+            "command" => "docker run -d --network {$network} -v /:/host --name {$deploymentUuid} --rm -v /var/run/docker.sock:/var/run/docker.sock {$helperImage}",
+            "hidden" => true,
+        ]);
+    }
+    $commands->push([
+        "command" => executeInDocker($deploymentUuid, "mkdir -p {$basedir}"),
+        "hidden" => true,
+    ]);
+    return $commands;
+}
+
+function generateComposeFile(string $deploymentUuid, Server $server, string $network, Application $application, string $containerName, string $imageName, ?ApplicationPreview $preview = null, int $pullRequestId = 0)
+{
+    $ports = $application->settings->is_static ? [80] : $application->ports_exposes_array;
+    $workDir = generateWorkdir($deploymentUuid, $application);
+    $persistent_storages = generateLocalPersistentVolumes($application, $pullRequestId);
+    $volume_names = generateLocalPersistentVolumesOnlyVolumeNames($application, $pullRequestId);
+    $environment_variables = generateEnvironmentVariables($application, $ports, $pullRequestId);
+
+    if (data_get($application, 'custom_labels')) {
+        $labels = collect(str($application->custom_labels)->explode(','));
+        $labels = $labels->filter(function ($value, $key) {
+            return !str($value)->startsWith('coolify.');
+        });
+        $application->custom_labels = $labels->implode(',');
+        $application->save();
+    } else {
+        $labels = collect(generateLabelsApplication($application, $preview));
+    }
+    if ($pullRequestId !== 0) {
+        $labels = collect(generateLabelsApplication($application, $preview));
+    }
+    $labels = $labels->merge(defaultLabels($application->id, $application->uuid, 0))->toArray();
+    $docker_compose = [
+        'version' => '3.8',
+        'services' => [
+            $containerName => [
+                'image' => $imageName,
+                'container_name' => $containerName,
+                'restart' => RESTART_MODE,
+                'environment' => $environment_variables,
+                'labels' => $labels,
+                'expose' => $ports,
+                'networks' => [
+                    $network,
+                ],
+                'mem_limit' => $application->limits_memory,
+                'memswap_limit' => $application->limits_memory_swap,
+                'mem_swappiness' => $application->limits_memory_swappiness,
+                'mem_reservation' => $application->limits_memory_reservation,
+                'cpus' => (int) $application->limits_cpus,
+                'cpuset' => $application->limits_cpuset,
+                'cpu_shares' => $application->limits_cpu_shares,
+            ]
+        ],
+        'networks' => [
+            $network => [
+                'external' => true,
+                'name' => $network,
+                'attachable' => true
+            ]
+        ]
+    ];
+    if ($server->isLogDrainEnabled() && $application->isLogDrainEnabled()) {
+        $docker_compose['services'][$containerName]['logging'] = [
+            'driver' => 'fluentd',
+            'options' => [
+                'fluentd-address' => "tcp://127.0.0.1:24224",
+                'fluentd-async' => "true",
+                'fluentd-sub-second-precision' => "true",
+            ]
+        ];
+    }
+    if ($application->settings->is_gpu_enabled) {
+        ray('asd');
+        $docker_compose['services'][$containerName]['deploy']['resources']['reservations']['devices'] = [
+            [
+                'driver' => data_get($application, 'settings.gpu_driver', 'nvidia'),
+                'capabilities' => ['gpu'],
+                'options' => data_get($application, 'settings.gpu_options', [])
+            ]
+        ];
+        if (data_get($application, 'settings.gpu_count')) {
+            $count = data_get($application, 'settings.gpu_count');
+            if ($count === 'all') {
+                $docker_compose['services'][$containerName]['deploy']['resources']['reservations']['devices'][0]['count'] = $count;
+            } else {
+                $docker_compose['services'][$containerName]['deploy']['resources']['reservations']['devices'][0]['count'] = (int) $count;
+            }
+        } else if (data_get($application, 'settings.gpu_device_ids')) {
+            $docker_compose['services'][$containerName]['deploy']['resources']['reservations']['devices'][0]['ids'] = data_get($application, 'settings.gpu_device_ids');
+        }
+    }
+    if ($application->isHealthcheckDisabled()) {
+        data_forget($docker_compose, 'services.' . $containerName . '.healthcheck');
+    }
+    if (count($application->ports_mappings_array) > 0 && $pullRequestId === 0) {
+        $docker_compose['services'][$containerName]['ports'] = $application->ports_mappings_array;
+    }
+    if (count($persistent_storages) > 0) {
+        $docker_compose['services'][$containerName]['volumes'] = $persistent_storages;
+    }
+    if (count($volume_names) > 0) {
+        $docker_compose['volumes'] = $volume_names;
+    }
+    $docker_compose = Yaml::dump($docker_compose, 10);
+    $docker_compose_base64 = base64_encode($docker_compose);
+    $commands = collect([]);
+    $commands->push([
+        "command" => executeInDocker($deploymentUuid, "echo '{$docker_compose_base64}' | base64 -d > {$workDir}/docker-compose.yml"),
+        "hidden" => true,
+    ]);
+    return $commands;
+}
+function generateLocalPersistentVolumes(Application $application, int $pullRequestId = 0)
+{
+    $local_persistent_volumes = [];
+    foreach ($application->persistentStorages as $persistentStorage) {
+        $volume_name = $persistentStorage->host_path ?? $persistentStorage->name;
+        if ($pullRequestId !== 0) {
+            $volume_name = $volume_name . '-pr-' . $pullRequestId;
+        }
+        $local_persistent_volumes[] = $volume_name . ':' . $persistentStorage->mount_path;
+    }
+    return $local_persistent_volumes;
+}
+
+function generateLocalPersistentVolumesOnlyVolumeNames(Application $application, int $pullRequestId = 0)
+{
+    $local_persistent_volumes_names = [];
+    foreach ($application->persistentStorages as $persistentStorage) {
+        if ($persistentStorage->host_path) {
+            continue;
+        }
+        $name = $persistentStorage->name;
+
+        if ($pullRequestId !== 0) {
+            $name = $name . '-pr-' . $pullRequestId;
+        }
+
+        $local_persistent_volumes_names[$name] = [
+            'name' => $name,
+            'external' => false,
+        ];
+    }
+    return $local_persistent_volumes_names;
+}
+function generateEnvironmentVariables(Application $application, $ports, int $pullRequestId = 0)
+{
+    $environment_variables = collect();
+    // ray('Generate Environment Variables')->green();
+    if ($pullRequestId === 0) {
+        // ray($this->application->runtime_environment_variables)->green();
+        foreach ($application->runtime_environment_variables as $env) {
+            $environment_variables->push("$env->key=$env->value");
+        }
+        foreach ($application->nixpacks_environment_variables as $env) {
+            $environment_variables->push("$env->key=$env->value");
+        }
+    } else {
+        // ray($this->application->runtime_environment_variables_preview)->green();
+        foreach ($application->runtime_environment_variables_preview as $env) {
+            $environment_variables->push("$env->key=$env->value");
+        }
+        foreach ($application->nixpacks_environment_variables_preview as $env) {
+            $environment_variables->push("$env->key=$env->value");
+        }
+    }
+    // Add PORT if not exists, use the first port as default
+    if ($environment_variables->filter(fn ($env) => str($env)->contains('PORT'))->isEmpty()) {
+        $environment_variables->push("PORT={$ports[0]}");
+    }
+    return $environment_variables->all();
+}
+
+function rollingUpdate(Application $application, string $deploymentUuid)
+{
+    $commands = collect([]);
+    $workDir = generateWorkdir($deploymentUuid, $application);
+    if (count($application->ports_mappings_array) > 0) {
+        // $this->execute_remote_command(
+        //     [
+        //         "echo '\n----------------------------------------'",
+        //     ],
+        //     ["echo -n 'Application has ports mapped to the host system, rolling update is not supported.'"],
+        // );
+        // $this->stop_running_container(force: true);
+        // $this->start_by_compose_file();
+    } else {
+        $commands->push(
+            [
+                "command" =>  "echo '\n----------------------------------------'"
+            ],
+            [
+                "command" =>  "echo -n 'Rolling update started.'"
+            ]
+        );
+        if ($application->build_pack === 'dockerimage') {
+            $commands->push(
+                ["echo -n 'Pulling latest images from the registry.'"],
+                [executeInDocker($deploymentUuid, "docker compose --project-directory {$workDir} pull"), "hidden" => true],
+                [executeInDocker($deploymentUuid, "docker compose --project-directory {$workDir} up --build -d"), "hidden" => true],
+            );
+        } else {
+            $commands->push(
+                [executeInDocker($deploymentUuid, "docker compose --project-directory {$workDir} up --build -d"), "hidden" => true],
+            );
+        }
+        return $commands;
     }
 }