Merge pull request #3454 from peaklabs-dev/fix-ssh-keys

Fix: SSH Keys, Multiplexing issues and a lot of other small things for dev and prod

.env.development.example

@@ -6,7 +6,7 @@ APP_KEY=
 APP_URL=http://localhost
 APP_PORT=8000
 APP_DEBUG=true
-SSH_MUX_ENABLED=false
+SSH_MUX_ENABLED=true
 
 # PostgreSQL Database Configuration
 DB_DATABASE=coolify
@@ -21,7 +21,7 @@ RAY_ENABLED=false
 # Set custom ray port
 RAY_PORT=
 
-# Clockwork Configuration
+# Clockwork Configuration (remove?)
 CLOCKWORK_ENABLED=false
 CLOCKWORK_QUEUE_COLLECT=true
 

app/Actions/CoolifyTask/RunRemoteProcess.php

@@ -10,6 +10,7 @@ use Illuminate\Process\ProcessResult;
 use Illuminate\Support\Facades\DB;
 use Illuminate\Support\Facades\Process;
 use Spatie\Activitylog\Models\Activity;
+use App\Helpers\SshMultiplexingHelper;
 
 class RunRemoteProcess
 {
@@ -137,7 +138,7 @@ class RunRemoteProcess
         $command = $this->activity->getExtraProperty('command');
         $server = Server::whereUuid($server_uuid)->firstOrFail();
 
-        return generateSshCommand($server, $command);
+        return SshMultiplexingHelper::generateSshCommand($server, $command);
     }
 
     protected function handleOutput(string $type, string $output)

app/Actions/Proxy/CheckProxy.php

@@ -26,7 +26,7 @@ class CheckProxy
         if (is_null($proxyType) || $proxyType === 'NONE' || $server->proxy->force_stop) {
             return false;
         }
-        ['uptime' => $uptime, 'error' => $error] = $server->validateConnection();
+        ['uptime' => $uptime, 'error' => $error] = $server->validateConnection(false);
         if (! $uptime) {
             throw new \Exception($error);
         }

app/Console/Kernel.php

@@ -7,6 +7,7 @@ use App\Jobs\CleanupInstanceStuffsJob;
 use App\Jobs\CleanupStaleMultiplexedConnections;
 use App\Jobs\DatabaseBackupJob;
 use App\Jobs\DockerCleanupJob;
+use App\Jobs\CleanupSshKeysJob;
 use App\Jobs\PullHelperImageJob;
 use App\Jobs\PullSentinelImageJob;
 use App\Jobs\PullTemplatesFromCDN;
@@ -43,6 +44,8 @@ class Kernel extends ConsoleKernel
             $schedule->command('uploads:clear')->everyTwoMinutes();
 
             $schedule->command('telescope:prune')->daily();
+
+            $schedule->job(new CleanupSshKeysJob)->weekly()->onOneServer();
         } else {
             // Instance Jobs
             $schedule->command('horizon:snapshot')->everyFiveMinutes();
@@ -59,6 +62,8 @@ class Kernel extends ConsoleKernel
 
             $schedule->command('cleanup:database --yes')->daily();
             $schedule->command('uploads:clear')->everyTwoMinutes();
+
+            $schedule->job(new CleanupSshKeysJob)->weekly()->onOneServer();
         }
     }
 

app/Helpers/SshMultiplexingHelper.php

@@ -0,0 +1,204 @@
+<?php
+
+namespace App\Helpers;
+
+use App\Models\Server;
+use App\Models\PrivateKey;
+use Illuminate\Support\Facades\Process;
+use Illuminate\Support\Facades\Hash;
+
+class SshMultiplexingHelper
+{
+    public static function serverSshConfiguration(Server $server)
+    {
+        $privateKey = PrivateKey::findOrFail($server->private_key_id);
+        $sshKeyLocation = $privateKey->getKeyLocation();
+        $muxFilename = '/var/www/html/storage/app/ssh/mux/mux_' . $server->uuid;
+
+        return [
+            'sshKeyLocation' => $sshKeyLocation,
+            'muxFilename' => $muxFilename,
+        ];
+    }
+
+    public static function ensureMultiplexedConnection(Server $server)
+    {
+        if (!self::isMultiplexingEnabled()) {
+            // ray('SSH Multiplexing: DISABLED')->red();
+            return;
+        }
+
+        // ray('SSH Multiplexing: ENABLED')->green();
+        // ray('Ensuring multiplexed connection for server:', $server);
+
+        $sshConfig = self::serverSshConfiguration($server);
+        $muxSocket = $sshConfig['muxFilename'];
+        $sshKeyLocation = $sshConfig['sshKeyLocation'];
+
+        self::validateSshKey($sshKeyLocation);
+
+        $checkCommand = "ssh -O check -o ControlPath=$muxSocket {$server->user}@{$server->ip}";
+        $process = Process::run($checkCommand);
+
+        if ($process->exitCode() !== 0) {
+            // ray('SSH Multiplexing: Existing connection check failed or not found')->orange();
+            // ray('Establishing new connection');
+            self::establishNewMultiplexedConnection($server);
+        } else {
+            // ray('SSH Multiplexing: Existing connection is valid')->green();
+        }
+    }
+
+    public static function establishNewMultiplexedConnection(Server $server)
+    {
+        $sshConfig = self::serverSshConfiguration($server);
+        $sshKeyLocation = $sshConfig['sshKeyLocation'];
+        $muxSocket = $sshConfig['muxFilename'];
+
+        // ray('Establishing new multiplexed connection')->blue();
+        // ray('SSH Key Location:', $sshKeyLocation);
+        // ray('Mux Socket:', $muxSocket);
+
+        $connectionTimeout = config('constants.ssh.connection_timeout');
+        $serverInterval = config('constants.ssh.server_interval');
+        $muxPersistTime = config('constants.ssh.mux_persist_time');
+
+        $establishCommand = "ssh -fNM -o ControlMaster=auto -o ControlPath=$muxSocket -o ControlPersist={$muxPersistTime} "
+            . self::getCommonSshOptions($server, $sshKeyLocation, $connectionTimeout, $serverInterval)
+            . "{$server->user}@{$server->ip}";
+
+        // ray('Establish Command:', $establishCommand);
+
+        $establishProcess = Process::run($establishCommand);
+
+        // ray('Establish Process Exit Code:', $establishProcess->exitCode());
+        // ray('Establish Process Output:', $establishProcess->output());
+        // ray('Establish Process Error Output:', $establishProcess->errorOutput());
+
+        if ($establishProcess->exitCode() !== 0) {
+            // ray('Failed to establish multiplexed connection')->red();
+            throw new \RuntimeException('Failed to establish multiplexed connection: ' . $establishProcess->errorOutput());
+        }
+
+        // ray('Successfully established multiplexed connection')->green();
+
+        // Check if the mux socket file was created
+        if (!file_exists($muxSocket)) {
+            // ray('Mux socket file not found after connection establishment')->orange();
+        }
+    }
+
+    public static function removeMuxFile(Server $server)
+    {
+        $sshConfig = self::serverSshConfiguration($server);
+        $muxSocket = $sshConfig['muxFilename'];
+        
+        $closeCommand = "ssh -O exit -o ControlPath=$muxSocket {$server->user}@{$server->ip}";
+        $process = Process::run($closeCommand);
+        
+        // ray('Closing multiplexed connection')->blue();
+        // ray('Close command:', $closeCommand);
+        // ray('Close process exit code:', $process->exitCode());
+        // ray('Close process output:', $process->output());
+        // ray('Close process error output:', $process->errorOutput());
+
+        if ($process->exitCode() !== 0) {
+            // ray('Failed to close multiplexed connection')->orange();
+        } else {
+            // ray('Successfully closed multiplexed connection')->green();
+        }
+    }
+
+    public static function generateScpCommand(Server $server, string $source, string $dest)
+    {
+        $sshConfig = self::serverSshConfiguration($server);
+        $sshKeyLocation = $sshConfig['sshKeyLocation'];
+        $muxSocket = $sshConfig['muxFilename'];
+
+        $timeout = config('constants.ssh.command_timeout');
+
+        $scp_command = "timeout $timeout scp ";
+
+        if (self::isMultiplexingEnabled()) {
+            $muxPersistTime = config('constants.ssh.mux_persist_time');
+            $scp_command .= "-o ControlMaster=auto -o ControlPath=$muxSocket -o ControlPersist={$muxPersistTime} ";
+            self::ensureMultiplexedConnection($server);
+        }
+
+        self::addCloudflareProxyCommand($scp_command, $server);
+
+        $scp_command .= self::getCommonSshOptions($server, $sshKeyLocation, config('constants.ssh.connection_timeout'), config('constants.ssh.server_interval'));
+        $scp_command .= "{$source} {$server->user}@{$server->ip}:{$dest}";
+
+        return $scp_command;
+    }
+
+    public static function generateSshCommand(Server $server, string $command)
+    {
+        if ($server->settings->force_disabled) {
+            throw new \RuntimeException('Server is disabled.');
+        }
+
+        $sshConfig = self::serverSshConfiguration($server);
+        $sshKeyLocation = $sshConfig['sshKeyLocation'];
+        $muxSocket = $sshConfig['muxFilename'];
+
+        $timeout = config('constants.ssh.command_timeout');
+
+        $ssh_command = "timeout $timeout ssh ";
+
+        if (self::isMultiplexingEnabled()) {
+            $muxPersistTime = config('constants.ssh.mux_persist_time');
+            $ssh_command .= "-o ControlMaster=auto -o ControlPath=$muxSocket -o ControlPersist={$muxPersistTime} ";
+            self::ensureMultiplexedConnection($server);
+        }
+
+        self::addCloudflareProxyCommand($ssh_command, $server);
+
+        $ssh_command .= self::getCommonSshOptions($server, $sshKeyLocation, config('constants.ssh.connection_timeout'), config('constants.ssh.server_interval'));
+
+        $command = "PATH=\$PATH:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/host/usr/local/sbin:/host/usr/local/bin:/host/usr/sbin:/host/usr/bin:/host/sbin:/host/bin && $command";
+        $delimiter = Hash::make($command);
+        $command = str_replace($delimiter, '', $command);
+
+        $ssh_command .= "{$server->user}@{$server->ip} 'bash -se' << \\$delimiter".PHP_EOL
+            .$command.PHP_EOL
+            .$delimiter;
+
+        return $ssh_command;
+    }
+
+    private static function isMultiplexingEnabled(): bool
+    {
+        return config('constants.ssh.mux_enabled') && !config('coolify.is_windows_docker_desktop');
+    }
+
+    private static function validateSshKey(string $sshKeyLocation): void
+    {
+        $checkKeyCommand = "ls $sshKeyLocation 2>/dev/null";
+        $keyCheckProcess = Process::run($checkKeyCommand);
+
+        if ($keyCheckProcess->exitCode() !== 0) {
+            throw new \RuntimeException("SSH key file not accessible: $sshKeyLocation");
+        }
+    }
+
+    private static function addCloudflareProxyCommand(string &$command, Server $server): void
+    {
+        if (data_get($server, 'settings.is_cloudflare_tunnel')) {
+            $command .= '-o ProxyCommand="/usr/local/bin/cloudflared access ssh --hostname %h" ';
+        }
+    }
+
+    private static function getCommonSshOptions(Server $server, string $sshKeyLocation, int $connectionTimeout, int $serverInterval): string
+    {
+        return "-i {$sshKeyLocation} "
+            .'-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null '
+            .'-o PasswordAuthentication=no '
+            ."-o ConnectTimeout=$connectionTimeout "
+            ."-o ServerAliveInterval=$serverInterval "
+            .'-o RequestTTY=no '
+            .'-o LogLevel=ERROR '
+            ."-p {$server->port} ";
+    }
+}

app/Jobs/ApplicationDeploymentJob.php

@@ -211,7 +211,6 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
         }
         ray('New container name: ', $this->container_name)->green();
 
-        savePrivateKeyToFs($this->server);
         $this->saved_outputs = collect();
 
         // Set preview fqdn
@@ -970,7 +969,7 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
                 }
             }
             if ($this->application->environment_variables->where('key', 'COOLIFY_URL')->isEmpty()) {
-                $url = str($this->application->fqdn)->replace('http://', '')->replace('https://', '');
+                $url = str($this->application->fqdn)->replace('http://', '').replace('https://', '');
                 if ($this->application->compose_parsing_version === '3') {
                     $envs->push("COOLIFY_FQDN={$url}");
                 } else {
@@ -1443,21 +1442,11 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
         if ($this->pull_request_id !== 0) {
             $local_branch = "pull/{$this->pull_request_id}/head";
         }
-        $private_key = data_get($this->application, 'private_key.private_key');
+        $private_key = $this->application->privateKey->getKeyLocation();
         if ($private_key) {
-            $private_key = base64_encode($private_key);
             $this->execute_remote_command(
                 [
-                    executeInDocker($this->deployment_uuid, 'mkdir -p /root/.ssh'),
+                    executeInDocker($this->deployment_uuid, "GIT_SSH_COMMAND=\"ssh -o ConnectTimeout=30 -p {$this->customPort} -o Port={$this->customPort} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i {$private_key}\" git ls-remote {$this->fullRepoUrl} {$local_branch}"),
-                ],
-                [
-                    executeInDocker($this->deployment_uuid, "echo '{$private_key}' | base64 -d | tee /root/.ssh/id_rsa > /dev/null"),
-                ],
-                [
-                    executeInDocker($this->deployment_uuid, 'chmod 600 /root/.ssh/id_rsa'),
-                ],
-                [
-                    executeInDocker($this->deployment_uuid, "GIT_SSH_COMMAND=\"ssh -o ConnectTimeout=30 -p {$this->customPort} -o Port={$this->customPort} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /root/.ssh/id_rsa\" git ls-remote {$this->fullRepoUrl} {$local_branch}"),
                     'hidden' => true,
                     'save' => 'git_commit_sha',
                 ],

app/Jobs/CleanupSshKeysJob.php

@@ -0,0 +1,27 @@
+<?php
+
+namespace App\Jobs;
+
+use App\Models\PrivateKey;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Carbon\Carbon;
+
+class CleanupSshKeysJob implements ShouldQueue
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    public function handle()
+    {
+        $oneWeekAgo = Carbon::now()->subWeek();
+
+        PrivateKey::where('created_at', '<', $oneWeekAgo)
+            ->get()
+            ->each(function ($privateKey) {
+                $privateKey->safeDelete();
+            });
+    }
+}

app/Jobs/CleanupStaleMultiplexedConnections.php

@@ -9,6 +9,8 @@ use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
 use Illuminate\Support\Facades\Process;
+use Illuminate\Support\Facades\Storage;
+use Carbon\Carbon;
 
 class CleanupStaleMultiplexedConnections implements ShouldQueue
 {
@@ -16,22 +18,64 @@ class CleanupStaleMultiplexedConnections implements ShouldQueue
 
     public function handle()
     {
-        Server::chunk(100, function ($servers) {
+        $this->cleanupStaleConnections();
-            foreach ($servers as $server) {
+        $this->cleanupNonExistentServerConnections();
-                $this->cleanupStaleConnection($server);
-            }
-        });
     }
 
-    private function cleanupStaleConnection(Server $server)
+    private function cleanupStaleConnections()
     {
-        $muxSocket = "/tmp/mux_{$server->id}";
+        $muxFiles = Storage::disk('ssh-mux')->files();
-        $checkCommand = "ssh -O check -o ControlPath=$muxSocket {$server->user}@{$server->ip} 2>/dev/null";
-        $checkProcess = Process::run($checkCommand);
 
-        if ($checkProcess->exitCode() !== 0) {
+        foreach ($muxFiles as $muxFile) {
-            $closeCommand = "ssh -O exit -o ControlPath=$muxSocket {$server->user}@{$server->ip} 2>/dev/null";
+            $serverUuid = $this->extractServerUuidFromMuxFile($muxFile);
-            Process::run($closeCommand);
+            $server = Server::where('uuid', $serverUuid)->first();
+
+            if (!$server) {
+                $this->removeMultiplexFile($muxFile);
+                continue;
+            }
+
+            $muxSocket = "/var/www/html/storage/app/ssh/mux/{$muxFile}";
+            $checkCommand = "ssh -O check -o ControlPath={$muxSocket} {$server->user}@{$server->ip} 2>/dev/null";
+            $checkProcess = Process::run($checkCommand);
+
+            if ($checkProcess->exitCode() !== 0) {
+                $this->removeMultiplexFile($muxFile);
+            } else {
+                $muxContent = Storage::disk('ssh-mux')->get($muxFile);
+                $establishedAt = Carbon::parse(substr($muxContent, 37));
+                $expirationTime = $establishedAt->addSeconds(config('constants.ssh.mux_persist_time'));
+
+                if (Carbon::now()->isAfter($expirationTime)) {
+                    $this->removeMultiplexFile($muxFile);
+                }
+            }
         }
     }
+
+    private function cleanupNonExistentServerConnections()
+    {
+        $muxFiles = Storage::disk('ssh-mux')->files();
+        $existingServerUuids = Server::pluck('uuid')->toArray();
+
+        foreach ($muxFiles as $muxFile) {
+            $serverUuid = $this->extractServerUuidFromMuxFile($muxFile);
+            if (!in_array($serverUuid, $existingServerUuids)) {
+                $this->removeMultiplexFile($muxFile);
+            }
+        }
+    }
+
+    private function extractServerUuidFromMuxFile($muxFile)
+    {
+        return substr($muxFile, 4);
+    }
+
+    private function removeMultiplexFile($muxFile)
+    {
+        $muxSocket = "/var/www/html/storage/app/ssh/mux/{$muxFile}";
+        $closeCommand = "ssh -O exit -o ControlPath={$muxSocket} localhost 2>/dev/null";
+        Process::run($closeCommand);
+        Storage::disk('ssh-mux')->delete($muxFile);
+    }
 }

app/Jobs/ServerCheckJob.php

@@ -93,7 +93,7 @@ class ServerCheckJob implements ShouldBeEncrypted, ShouldQueue
 
     private function serverStatus()
     {
-        ['uptime' => $uptime] = $this->server->validateConnection();
+        ['uptime' => $uptime] = $this->server->validateConnection(false);
         if ($uptime) {
             if ($this->server->unreachable_notification_sent === true) {
                 $this->server->update(['unreachable_notification_sent' => false]);

app/Livewire/Boarding/Index.php

@@ -141,7 +141,7 @@ uZx9iFkCELtxrh31QJ68AAAAEXNhaWxANzZmZjY2ZDJlMmRkAQIDBA==
             if (! $this->createdServer) {
                 return $this->dispatch('error', 'Localhost server is not found. Something went wrong during installation. Please try to reinstall or contact support.');
             }
-            $this->serverPublicKey = $this->createdServer->privateKey->publicKey();
+            $this->serverPublicKey = $this->createdServer->privateKey->getPublicKey();
 
             return $this->validateServer('localhost');
         } elseif ($this->selectedServerType === 'remote') {
@@ -175,7 +175,7 @@ uZx9iFkCELtxrh31QJ68AAAAEXNhaWxANzZmZjY2ZDJlMmRkAQIDBA==
             return;
         }
         $this->selectedExistingPrivateKey = $this->createdServer->privateKey->id;
-        $this->serverPublicKey = $this->createdServer->privateKey->publicKey();
+        $this->serverPublicKey = $this->createdServer->privateKey->getPublicKey();
         $this->updateServerDetails();
         $this->currentState = 'validate-server';
     }
@@ -231,17 +231,24 @@ uZx9iFkCELtxrh31QJ68AAAAEXNhaWxANzZmZjY2ZDJlMmRkAQIDBA==
     public function savePrivateKey()
     {
         $this->validate([
-            'privateKeyName' => 'required',
+            'privateKeyName' => 'required|string|max:255',
-            'privateKey' => 'required',
+            'privateKeyDescription' => 'nullable|string|max:255',
+            'privateKey' => 'required|string',
         ]);
-        $this->createdPrivateKey = PrivateKey::create([
+
-            'name' => $this->privateKeyName,
+        try {
-            'description' => $this->privateKeyDescription,
+            $privateKey = PrivateKey::createAndStore([
-            'private_key' => $this->privateKey,
+                'name' => $this->privateKeyName,
-            'team_id' => currentTeam()->id,
+                'description' => $this->privateKeyDescription,
-        ]);
+                'private_key' => $this->privateKey,
-        $this->createdPrivateKey->save();
+                'team_id' => currentTeam()->id,
-        $this->currentState = 'create-server';
+            ]);
+
+            $this->createdPrivateKey = $privateKey;
+            $this->currentState = 'create-server';
+        } catch (\Exception $e) {
+            $this->addError('privateKey', 'Failed to save private key: ' . $e->getMessage());
+        }
     }
 
     public function saveServer()

app/Livewire/Project/Shared/GetLogs.php

@@ -17,6 +17,7 @@ use App\Models\StandalonePostgresql;
 use App\Models\StandaloneRedis;
 use Illuminate\Support\Facades\Process;
 use Livewire\Component;
+use App\Helpers\SshMultiplexingHelper;
 
 class GetLogs extends Component
 {
@@ -108,14 +109,14 @@ class GetLogs extends Component
                         $command = parseCommandsByLineForSudo(collect($command), $this->server);
                         $command = $command[0];
                     }
-                    $sshCommand = generateSshCommand($this->server, $command);
+                    $sshCommand = SshMultiplexingHelper::generateSshCommand($this->server, $command);
                 } else {
                     $command = "docker logs -n {$this->numberOfLines} -t {$this->container}";
                     if ($this->server->isNonRoot()) {
                         $command = parseCommandsByLineForSudo(collect($command), $this->server);
                         $command = $command[0];
                     }
-                    $sshCommand = generateSshCommand($this->server, $command);
+                    $sshCommand = SshMultiplexingHelper::generateSshCommand($this->server, $command);
                 }
             } else {
                 if ($this->server->isSwarm()) {
@@ -124,14 +125,14 @@ class GetLogs extends Component
                         $command = parseCommandsByLineForSudo(collect($command), $this->server);
                         $command = $command[0];
                     }
-                    $sshCommand = generateSshCommand($this->server, $command);
+                    $sshCommand = SshMultiplexingHelper::generateSshCommand($this->server, $command);
                 } else {
                     $command = "docker logs -n {$this->numberOfLines} {$this->container}";
                     if ($this->server->isNonRoot()) {
                         $command = parseCommandsByLineForSudo(collect($command), $this->server);
                         $command = $command[0];
                     }
-                    $sshCommand = generateSshCommand($this->server, $command);
+                    $sshCommand = SshMultiplexingHelper::generateSshCommand($this->server, $command);
                 }
             }
             if ($refresh) {

app/Livewire/Security/PrivateKey/Create.php

@@ -3,22 +3,14 @@
 namespace App\Livewire\Security\PrivateKey;
 
 use App\Models\PrivateKey;
-use DanHarrin\LivewireRateLimiting\WithRateLimiting;
 use Livewire\Component;
-use phpseclib3\Crypt\PublicKeyLoader;
 
 class Create extends Component
 {
-    use WithRateLimiting;
+    public string $name = '';
-
+    public string $value = '';
-    public string $name;
-
-    public string $value;
-
     public ?string $from = null;
-
     public ?string $description = null;
-
     public ?string $publicKey = null;
 
     protected $rules = [
@@ -26,72 +18,69 @@ class Create extends Component
         'value' => 'required|string',
     ];
 
-    protected $validationAttributes = [
-        'name' => 'name',
-        'value' => 'private Key',
-    ];
-
     public function generateNewRSAKey()
     {
-        try {
+        $this->generateNewKey('rsa');
-            $this->rateLimit(10);
-            $this->name = generate_random_name();
-            $this->description = 'Created by Coolify';
-            ['private' => $this->value, 'public' => $this->publicKey] = generateSSHKey();
-        } catch (\Throwable $e) {
-            return handleError($e, $this);
-        }
     }
 
     public function generateNewEDKey()
     {
-        try {
+        $this->generateNewKey('ed25519');
-            $this->rateLimit(10);
-            $this->name = generate_random_name();
-            $this->description = 'Created by Coolify';
-            ['private' => $this->value, 'public' => $this->publicKey] = generateSSHKey('ed25519');
-        } catch (\Throwable $e) {
-            return handleError($e, $this);
-        }
     }
 
-    public function updated($updateProperty)
+    private function generateNewKey($type)
     {
-        if ($updateProperty === 'value') {
+        $keyData = PrivateKey::generateNewKeyPair($type);
-            try {
+        $this->setKeyData($keyData);
-                $this->publicKey = PublicKeyLoader::load($this->$updateProperty)->getPublicKey()->toString('OpenSSH', ['comment' => '']);
+    }
-            } catch (\Throwable $e) {
+
-                if ($this->$updateProperty === '') {
+    public function updated($property)
-                    $this->publicKey = '';
+    {
-                } else {
+        if ($property === 'value') {
-                    $this->publicKey = 'Invalid private key';
+            $this->validatePrivateKey();
-                }
-            }
         }
-        $this->validateOnly($updateProperty);
     }
 
     public function createPrivateKey()
     {
         $this->validate();
+
         try {
-            $this->value = trim($this->value);
+            $privateKey = PrivateKey::createAndStore([
-            if (! str_ends_with($this->value, "\n")) {
-                $this->value .= "\n";
-            }
-            $private_key = PrivateKey::create([
                 'name' => $this->name,
                 'description' => $this->description,
-                'private_key' => $this->value,
+                'private_key' => trim($this->value) . "\n",
                 'team_id' => currentTeam()->id,
             ]);
-            if ($this->from === 'server') {
-                return redirect()->route('dashboard');
-            }
 
-            return redirect()->route('security.private-key.show', ['private_key_uuid' => $private_key->uuid]);
+            return $this->redirectAfterCreation($privateKey);
         } catch (\Throwable $e) {
             return handleError($e, $this);
         }
     }
+
+    private function setKeyData(array $keyData)
+    {
+        $this->name = $keyData['name'];
+        $this->description = $keyData['description'];
+        $this->value = $keyData['private_key'];
+        $this->publicKey = $keyData['public_key'];
+    }
+
+    private function validatePrivateKey()
+    {
+        $validationResult = PrivateKey::validateAndExtractPublicKey($this->value);
+        $this->publicKey = $validationResult['publicKey'];
+
+        if (!$validationResult['isValid']) {
+            $this->addError('value', 'Invalid private key');
+        }
+    }
+
+    private function redirectAfterCreation(PrivateKey $privateKey)
+    {
+        return $this->from === 'server'
+            ? redirect()->route('dashboard')
+            : redirect()->route('security.private-key.show', ['private_key_uuid' => $privateKey->uuid]);
+    }
 }

app/Livewire/Security/PrivateKey/Show.php

@@ -35,19 +35,20 @@ class Show extends Component
 
     public function loadPublicKey()
     {
-        $this->public_key = $this->private_key->publicKey();
+        $this->public_key = $this->private_key->getPublicKey();
+        if ($this->public_key === 'Error loading private key') {
+            $this->dispatch('error', 'Failed to load public key. The private key may be invalid.');
+        }
     }
 
     public function delete()
     {
         try {
-            if ($this->private_key->isEmpty()) {
+            $this->private_key->safeDelete();
-                $this->private_key->delete();
+            currentTeam()->privateKeys = PrivateKey::where('team_id', currentTeam()->id)->get();
-                currentTeam()->privateKeys = PrivateKey::where('team_id', currentTeam()->id)->get();
+            return redirect()->route('security.private-key.index');
-
+        } catch (\Exception $e) {
-                return redirect()->route('security.private-key.index');
+            $this->dispatch('error', $e->getMessage());
-            }
-            $this->dispatch('error', 'This private key is in use and cannot be deleted. Please delete all servers, applications, and GitHub/GitLab apps that use this private key before deleting it.');
         } catch (\Throwable $e) {
             return handleError($e, $this);
         }
@@ -56,8 +57,9 @@ class Show extends Component
     public function changePrivateKey()
     {
         try {
-            $this->private_key->private_key = formatPrivateKey($this->private_key->private_key);
+            $this->private_key->updatePrivateKey([
-            $this->private_key->save();
+                'private_key' => formatPrivateKey($this->private_key->private_key)
+            ]);
             refresh_server_connection($this->private_key);
             $this->dispatch('success', 'Private key updated.');
         } catch (\Throwable $e) {

app/Livewire/Server/ShowPrivateKey.php

@@ -4,6 +4,7 @@ namespace App\Livewire\Server;
 
 use App\Models\Server;
 use Livewire\Component;
+use App\Models\PrivateKey;
 
 class ShowPrivateKey extends Component
 {
@@ -13,25 +14,15 @@ class ShowPrivateKey extends Component
 
     public $parameters;
 
-    public function setPrivateKey($newPrivateKeyId)
+    public function setPrivateKey($privateKeyId)
     {
         try {
-            $oldPrivateKeyId = $this->server->private_key_id;
+            $privateKey = PrivateKey::findOrFail($privateKeyId);
-            refresh_server_connection($this->server->privateKey);
+            $this->server->update(['private_key_id' => $privateKey->id]);
-            $this->server->update([
-                'private_key_id' => $newPrivateKeyId,
-            ]);
             $this->server->refresh();
-            refresh_server_connection($this->server->privateKey);
+            $this->dispatch('success', 'Private key updated successfully.');
-            $this->checkConnection();
+        } catch (\Exception $e) {
-        } catch (\Throwable $e) {
+            $this->dispatch('error', 'Failed to update private key: ' . $e->getMessage());
-            $this->server->update([
-                'private_key_id' => $oldPrivateKeyId,
-            ]);
-            $this->server->refresh();
-            refresh_server_connection($this->server->privateKey);
-
-            return handleError($e, $this);
         }
     }
 

app/Models/PrivateKey.php

@@ -3,7 +3,10 @@
 namespace App\Models;
 
 use OpenApi\Attributes as OA;
+use Illuminate\Support\Facades\Storage;
+use Illuminate\Validation\ValidationException;
 use phpseclib3\Crypt\PublicKeyLoader;
+use DanHarrin\LivewireRateLimiting\WithRateLimiting;
 
 #[OA\Schema(
     description: 'Private Key model',
@@ -22,48 +25,139 @@ use phpseclib3\Crypt\PublicKeyLoader;
 )]
 class PrivateKey extends BaseModel
 {
+    use WithRateLimiting;
+
     protected $fillable = [
         'name',
         'description',
         'private_key',
         'is_git_related',
         'team_id',
+        'fingerprint',
+    ];
+
+    protected $casts = [
+        'private_key' => 'encrypted',
     ];
 
     protected static function booted()
     {
         static::saving(function ($key) {
-            $privateKey = data_get($key, 'private_key');
+            $key->private_key = formatPrivateKey($key->private_key);
-            if (substr($privateKey, -1) !== "\n") {
+            
-                $key->private_key = $privateKey."\n";
+            if (!self::validatePrivateKey($key->private_key)) {
+                throw ValidationException::withMessages([
+                    'private_key' => ['The private key is invalid.'],
+                ]);
+            }
+
+            $key->fingerprint = self::generateFingerprint($key->private_key);
+
+            if (self::fingerprintExists($key->fingerprint, $key->id)) {
+                throw ValidationException::withMessages([
+                    'private_key' => ['This private key already exists.'],
+                ]);
             }
         });
 
+        static::deleted(function ($key) {
+            self::deleteFromStorage($key);
+        });
+    }
+
+    public function getPublicKey()
+    {
+        return self::extractPublicKeyFromPrivate($this->private_key) ?? 'Error loading private key';
     }
 
     public static function ownedByCurrentTeam(array $select = ['*'])
     {
         $selectArray = collect($select)->concat(['id']);
-
+        return self::whereTeamId(currentTeam()->id)->select($selectArray->all());
-        return PrivateKey::whereTeamId(currentTeam()->id)->select($selectArray->all());
     }
 
-    public function publicKey()
+    public static function validatePrivateKey($privateKey)
     {
         try {
-            return PublicKeyLoader::load($this->private_key)->getPublicKey()->toString('OpenSSH', ['comment' => '']);
+            PublicKeyLoader::load($privateKey);
+            return true;
         } catch (\Throwable $e) {
-            return 'Error loading private key';
+            return false;
         }
     }
 
-    public function isEmpty()
+    public static function createAndStore(array $data)
     {
-        if ($this->servers()->count() === 0 && $this->applications()->count() === 0 && $this->githubApps()->count() === 0 && $this->gitlabApps()->count() === 0) {
+        $privateKey = new self($data);
-            return true;
+        $privateKey->save();
-        }
+        $privateKey->storeInFileSystem();
+        return $privateKey;
+    }
 
-        return false;
+    public static function generateNewKeyPair($type = 'rsa')
+    {
+        try {
+            $instance = new self();
+            $instance->rateLimit(10);
+            $name = generate_random_name();
+            $description = 'Created by Coolify';
+            $keyPair = generateSSHKey($type === 'ed25519' ? 'ed25519' : 'rsa');
+            
+            return [
+                'name' => $name,
+                'description' => $description,
+                'private_key' => $keyPair['private'],
+                'public_key' => $keyPair['public'],
+            ];
+        } catch (\Throwable $e) {
+            throw new \Exception("Failed to generate new {$type} key: " . $e->getMessage());
+        }
+    }
+
+    public static function extractPublicKeyFromPrivate($privateKey)
+    {
+        try {
+            $key = PublicKeyLoader::load($privateKey);
+            return $key->getPublicKey()->toString('OpenSSH', ['comment' => '']);
+        } catch (\Throwable $e) {
+            return null;
+        }
+    }
+
+    public static function validateAndExtractPublicKey($privateKey)
+    {
+        $isValid = self::validatePrivateKey($privateKey);
+        $publicKey = $isValid ? self::extractPublicKeyFromPrivate($privateKey) : '';
+        
+        return [
+            'isValid' => $isValid,
+            'publicKey' => $publicKey,
+        ];
+    }
+
+    public function storeInFileSystem()
+    {
+        $filename = "ssh_key@{$this->uuid}";
+        Storage::disk('ssh-keys')->put($filename, $this->private_key);
+        return "/var/www/html/storage/app/ssh/keys/{$filename}";
+    }
+
+    public static function deleteFromStorage(self $privateKey)
+    {   
+        $filename = "ssh_key@{$privateKey->uuid}";
+        Storage::disk('ssh-keys')->delete($filename);
+    }
+
+    public function getKeyLocation()
+    {
+        return "/var/www/html/storage/app/ssh/keys/ssh_key@{$this->uuid}";
+    }
+
+    public function updatePrivateKey(array $data)
+    {
+        $this->update($data);
+        $this->storeInFileSystem();
+        return $this;
     }
 
     public function servers()
@@ -85,4 +179,43 @@ class PrivateKey extends BaseModel
     {
         return $this->hasMany(GitlabApp::class);
     }
+
+    public function isInUse()
+    {
+        return $this->servers()->exists() 
+            || $this->applications()->exists() 
+            || $this->githubApps()->exists() 
+            || $this->gitlabApps()->exists();
+    }
+
+    public function safeDelete()
+    {
+        if (!$this->isInUse()) {
+            $this->delete();
+            return true;
+        }
+        return false;
+    }
+
+    public static function generateFingerprint($privateKey)
+    {
+        try {
+            $key = PublicKeyLoader::load($privateKey);
+            $publicKey = $key->getPublicKey();
+            return $publicKey->getFingerprint('sha256');
+        } catch (\Throwable $e) {
+            return null;
+        }
+    }
+
+    private static function fingerprintExists($fingerprint, $excludeId = null)
+    {
+        $query = self::where('fingerprint', $fingerprint);
+        
+        if ($excludeId) {
+            $query->where('id', '!=', $excludeId);
+        }
+
+        return $query->exists();
+    }
 }

app/Models/Server.php

@@ -950,11 +950,10 @@ $schema://$host {
 
     public function isFunctional()
     {
-        $isFunctional = $this->settings->is_reachable && $this->settings->is_usable && ! $this->settings->force_disabled;
+        $isFunctional = $this->settings->is_reachable && $this->settings->is_usable && !$this->settings->force_disabled;
-        ['private_key_filename' => $private_key_filename, 'mux_filename' => $mux_filename] = server_ssh_configuration($this);
+        
-        if (! $isFunctional) {
+        if (!$isFunctional) {
-            Storage::disk('ssh-keys')->delete($private_key_filename);
+            Storage::disk('ssh-mux')->delete($this->muxFilename());
-            Storage::disk('ssh-mux')->delete($mux_filename);
         }
         
         return $isFunctional;
@@ -1006,9 +1005,10 @@ $schema://$host {
         return data_get($this, 'settings.is_swarm_worker');
     }
 
-    public function validateConnection()
+    public function validateConnection($isManualCheck = true)
     {
-        config()->set('constants.ssh.mux_enabled', false);
+        config()->set('constants.ssh.mux_enabled', !$isManualCheck);
+        // ray('Manual Check: ' . ($isManualCheck ? 'true' : 'false'));
 
         $server = Server::find($this->id);
         if (! $server) {
@@ -1018,7 +1018,6 @@ $schema://$host {
             return ['uptime' => false, 'error' => 'Server skipped.'];
         }
         try {
-            // EC2 does not have `uptime` command, lol
             instant_remote_process(['ls /'], $server);
             $server->settings()->update([
                 'is_reachable' => true,
@@ -1027,7 +1026,6 @@ $schema://$host {
                 'unreachable_count' => 0,
             ]);
             if (data_get($server, 'unreachable_notification_sent') === true) {
-                // $server->team?->notify(new Revived($server));
                 $server->update(['unreachable_notification_sent' => false]);
             }
 
@@ -1156,4 +1154,22 @@ $schema://$host {
     {
         return $this->settings->is_build_server;
     }
+
+    public static function createWithPrivateKey(array $data, PrivateKey $privateKey)
+    {
+        $server = new self($data);
+        $server->privateKey()->associate($privateKey);
+        $server->save();
+        return $server;
+    }
+
+    public function updateWithPrivateKey(array $data, PrivateKey $privateKey = null)
+    {
+        $this->update($data);
+        if ($privateKey) {
+            $this->privateKey()->associate($privateKey);
+            $this->save();
+        }
+        return $this;
+    }
 }

app/Traits/ExecuteRemoteCommand.php

@@ -7,6 +7,7 @@ use App\Models\Server;
 use Carbon\Carbon;
 use Illuminate\Support\Collection;
 use Illuminate\Support\Facades\Process;
+use App\Helpers\SshMultiplexingHelper;
 
 trait ExecuteRemoteCommand
 {
@@ -42,7 +43,7 @@ trait ExecuteRemoteCommand
                     $command = parseLineForSudo($command, $this->server);
                 }
             }
-            $remote_command = generateSshCommand($this->server, $command);
+            $remote_command = SshMultiplexingHelper::generateSshCommand($this->server, $command);
             $process = Process::timeout(3600)->idleTimeout(3600)->start($remote_command, function (string $type, string $output) use ($command, $hidden, $customType, $append) {
                 $output = str($output)->trim();
                 if ($output->startsWith('╔')) {

bootstrap/helpers/remoteProcess.php

@@ -3,6 +3,7 @@
 use App\Actions\CoolifyTask\PrepareCoolifyTask;
 use App\Data\CoolifyTaskArgs;
 use App\Enums\ActivityTypes;
+use App\Helpers\SshMultiplexingHelper;
 use App\Models\Application;
 use App\Models\ApplicationDeploymentQueue;
 use App\Models\PrivateKey;
@@ -10,9 +11,8 @@ use App\Models\Server;
 use Carbon\Carbon;
 use Illuminate\Database\Eloquent\Model;
 use Illuminate\Support\Collection;
-use Illuminate\Support\Facades\Hash;
 use Illuminate\Support\Facades\Process;
-use Illuminate\Support\Facades\Storage;
+use Illuminate\Support\Facades\Auth;
 use Illuminate\Support\Str;
 use Spatie\Activitylog\Contracts\Activity;
 
@@ -26,29 +26,28 @@ function remote_process(
     $callEventOnFinish = null,
     $callEventData = null
 ): Activity {
-    if (is_null($type)) {
+    $type = $type ?? ActivityTypes::INLINE->value;
-        $type = ActivityTypes::INLINE->value;
+    $command = $command instanceof Collection ? $command->toArray() : $command;
-    }
+    
-    if ($command instanceof Collection) {
-        $command = $command->toArray();
-    }
     if ($server->isNonRoot()) {
         $command = parseCommandsByLineForSudo(collect($command), $server);
     }
+    
     $command_string = implode("\n", $command);
-    if (auth()->user()) {
+    
-        $teams = auth()->user()->teams->pluck('id');
+    if (Auth::check()) {
-        if (! $teams->contains($server->team_id) && ! $teams->contains(0)) {
+        $teams = Auth::user()->teams->pluck('id');
+        if (!$teams->contains($server->team_id) && !$teams->contains(0)) {
             throw new \Exception('User is not part of the team that owns this server');
         }
     }
 
+    SshMultiplexingHelper::ensureMultiplexedConnection($server);
+
     return resolve(PrepareCoolifyTask::class, [
         'remoteProcessArgs' => new CoolifyTaskArgs(
             server_uuid: $server->uuid,
-            command: <<<EOT
+            command: $command_string,
-                {$command_string}
-                EOT,
             type: $type,
             type_uuid: $type_uuid,
             model: $model,
@@ -58,313 +57,65 @@ function remote_process(
         ),
     ])();
 }
-function server_ssh_configuration(Server $server)
-{
-    $uuid = data_get($server, 'uuid');
-    if (is_null($uuid)) {
-        throw new \Exception('Server does not have a uuid');
-    }
-    $private_key_filename = "id.root@{$server->uuid}";
-    $location = '/var/www/html/storage/app/ssh/keys/'.$private_key_filename;
-    $mux_filename = '/var/www/html/storage/app/ssh/mux/'.$server->muxFilename();
 
-    return [
-        'location' => $location,
-        'mux_filename' => $mux_filename,
-        'private_key_filename' => $private_key_filename,
-    ];
-}
-function savePrivateKeyToFs(Server $server)
-{
-    if (data_get($server, 'privateKey.private_key') === null) {
-        throw new \Exception("Server {$server->name} does not have a private key");
-    }
-    ['location' => $location, 'private_key_filename' => $private_key_filename] = server_ssh_configuration($server);
-    Storage::disk('ssh-keys')->makeDirectory('.');
-    Storage::disk('ssh-mux')->makeDirectory('.');
-    Storage::disk('ssh-keys')->put($private_key_filename, $server->privateKey->private_key);
-
-    return $location;
-}
-
-function generateScpCommand(Server $server, string $source, string $dest)
-{
-    $user = $server->user;
-    $port = $server->port;
-    $privateKeyLocation = savePrivateKeyToFs($server);
-    $timeout = config('constants.ssh.command_timeout');
-    $connectionTimeout = config('constants.ssh.connection_timeout');
-    $serverInterval = config('constants.ssh.server_interval');
-    $muxPersistTime = config('constants.ssh.mux_persist_time');
-
-    $scp_command = "timeout $timeout scp ";
-    $muxEnabled = config('constants.ssh.mux_enabled', true) && config('coolify.is_windows_docker_desktop') == false;
-    // ray('SSH Multiplexing Enabled:', $muxEnabled)->blue();
-
-    if ($muxEnabled) {
-        $muxSocket = "/var/www/html/storage/app/ssh/mux/{$server->muxFilename()}";
-        $scp_command .= "-o ControlMaster=auto -o ControlPath=$muxSocket -o ControlPersist={$muxPersistTime} ";
-        ensureMultiplexedConnection($server);
-        // ray('Using SSH Multiplexing')->green();
-    } else {
-        // ray('Not using SSH Multiplexing')->red();
-    }
-
-    if (data_get($server, 'settings.is_cloudflare_tunnel')) {
-        $scp_command .= '-o ProxyCommand="/usr/local/bin/cloudflared access ssh --hostname %h" ';
-    }
-    $scp_command .= "-i {$privateKeyLocation} "
-        .'-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null '
-        .'-o PasswordAuthentication=no '
-        ."-o ConnectTimeout=$connectionTimeout "
-        ."-o ServerAliveInterval=$serverInterval "
-        .'-o RequestTTY=no '
-        .'-o LogLevel=ERROR '
-        ."-P {$port} "
-        ."{$source} "
-        ."{$user}@{$server->ip}:{$dest}";
-
-    return $scp_command;
-}
 function instant_scp(string $source, string $dest, Server $server, $throwError = true)
 {
-    $timeout = config('constants.ssh.command_timeout');
+    $scp_command = SshMultiplexingHelper::generateScpCommand($server, $source, $dest);
-    $scp_command = generateScpCommand($server, $source, $dest);
+    $process = Process::timeout(config('constants.ssh.command_timeout'))->run($scp_command);
-    $process = Process::timeout($timeout)->run($scp_command);
     $output = trim($process->output());
     $exitCode = $process->exitCode();
     if ($exitCode !== 0) {
-        if (! $throwError) {
+        return $throwError ? excludeCertainErrors($process->errorOutput(), $exitCode) : null;
-            return null;
-        }
-
-        return excludeCertainErrors($process->errorOutput(), $exitCode);
-    }
-    if ($output === 'null') {
-        $output = null;
-    }
-
-    return $output;
-}
-function generateSshCommand(Server $server, string $command)
-{
-    if ($server->settings->force_disabled) {
-        throw new \RuntimeException('Server is disabled.');
-    }
-    $user = $server->user;
-    $port = $server->port;
-    $privateKeyLocation = savePrivateKeyToFs($server);
-    $timeout = config('constants.ssh.command_timeout');
-    $connectionTimeout = config('constants.ssh.connection_timeout');
-    $serverInterval = config('constants.ssh.server_interval');
-    $muxPersistTime = config('constants.ssh.mux_persist_time');
-
-    $ssh_command = "timeout $timeout ssh ";
-
-    $muxEnabled = config('constants.ssh.mux_enabled') && config('coolify.is_windows_docker_desktop') == false;
-    // ray('SSH Multiplexing Enabled:', $muxEnabled)->blue();
-    if ($muxEnabled) {
-        // Always use multiplexing when enabled
-        $muxSocket = "/var/www/html/storage/app/ssh/mux/{$server->muxFilename()}";
-        $ssh_command .= "-o ControlMaster=auto -o ControlPath=$muxSocket -o ControlPersist={$muxPersistTime} ";
-        ensureMultiplexedConnection($server);
-        // ray('Using SSH Multiplexing')->green();
-    } else {
-        // ray('Not using SSH Multiplexing')->red();
-    }
-
-    if (data_get($server, 'settings.is_cloudflare_tunnel')) {
-        $ssh_command .= '-o ProxyCommand="/usr/local/bin/cloudflared access ssh --hostname %h" ';
-    }
-    $command = "PATH=\$PATH:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/host/usr/local/sbin:/host/usr/local/bin:/host/usr/sbin:/host/usr/bin:/host/sbin:/host/bin && $command";
-    $delimiter = Hash::make($command);
-    $command = str_replace($delimiter, '', $command);
-    $ssh_command .= "-i {$privateKeyLocation} "
-        .'-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null '
-        .'-o PasswordAuthentication=no '
-        ."-o ConnectTimeout=$connectionTimeout "
-        ."-o ServerAliveInterval=$serverInterval "
-        .'-o RequestTTY=no '
-        .'-o LogLevel=ERROR '
-        ."-p {$port} "
-        ."{$user}@{$server->ip} "
-        ." 'bash -se' << \\$delimiter".PHP_EOL
-        .$command.PHP_EOL
-        .$delimiter;
-
-    return $ssh_command;
-}
-
-function ensureMultiplexedConnection(Server $server)
-{
-    if (! (config('constants.ssh.mux_enabled') && config('coolify.is_windows_docker_desktop') == false)) {
-        return;
-    }
-
-    static $ensuredConnections = [];
-
-    if (isset($ensuredConnections[$server->id])) {
-        if (! shouldResetMultiplexedConnection($server)) {
-            // ray('Using Existing Multiplexed Connection')->green();
-
-            return;
-        }
-    }
-
-    $muxSocket = "/var/www/html/storage/app/ssh/mux/{$server->muxFilename()}";
-    $checkCommand = "ssh -O check -o ControlPath=$muxSocket ";
-    if (data_get($server, 'settings.is_cloudflare_tunnel')) {
-        $checkCommand .= '-o ProxyCommand="/usr/local/bin/cloudflared access ssh --hostname %h" ';
-    }
-    $checkCommand .= " {$server->user}@{$server->ip}";
-
-    $process = Process::run($checkCommand);
-
-    if ($process->exitCode() === 0) {
-        // ray('Existing Multiplexed Connection is Valid')->green();
-        $ensuredConnections[$server->id] = [
-            'timestamp' => now(),
-            'muxSocket' => $muxSocket,
-        ];
-
-        return;
-    }
-
-    // ray('Establishing New Multiplexed Connection')->orange();
-
-    $privateKeyLocation = savePrivateKeyToFs($server);
-    $connectionTimeout = config('constants.ssh.connection_timeout');
-    $serverInterval = config('constants.ssh.server_interval');
-    $muxPersistTime = config('constants.ssh.mux_persist_time');
-
-    $establishCommand = "ssh -fNM -o ControlMaster=auto -o ControlPath=$muxSocket -o ControlPersist={$muxPersistTime} ";
-
-    if (data_get($server, 'settings.is_cloudflare_tunnel')) {
-        $establishCommand .= '-o ProxyCommand="/usr/local/bin/cloudflared access ssh --hostname %h" ';
-    }
-    $establishCommand .= "-i {$privateKeyLocation} "
-        .'-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null '
-        .'-o PasswordAuthentication=no '
-        ."-o ConnectTimeout=$connectionTimeout "
-        ."-o ServerAliveInterval=$serverInterval "
-        .'-o RequestTTY=no '
-        .'-o LogLevel=ERROR '
-        ."-p {$server->port} "
-        ."{$server->user}@{$server->ip}";
-
-    $establishProcess = Process::run($establishCommand);
-
-    if ($establishProcess->exitCode() !== 0) {
-        throw new \RuntimeException('Failed to establish multiplexed connection: '.$establishProcess->errorOutput());
-    }
-
-    $ensuredConnections[$server->id] = [
-        'timestamp' => now(),
-        'muxSocket' => $muxSocket,
-    ];
-
-    // ray('Established New Multiplexed Connection')->green();
-}
-
-function shouldResetMultiplexedConnection(Server $server)
-{
-    if (! (config('constants.ssh.mux_enabled') && config('coolify.is_windows_docker_desktop') == false)) {
-        return false;
-    }
-
-    static $ensuredConnections = [];
-
-    if (! isset($ensuredConnections[$server->id])) {
-        return true;
-    }
-
-    $lastEnsured = $ensuredConnections[$server->id]['timestamp'];
-    $muxPersistTime = config('constants.ssh.mux_persist_time');
-    $resetInterval = strtotime($muxPersistTime) - time();
-
-    return $lastEnsured->addSeconds($resetInterval)->isPast();
-}
-
-function resetMultiplexedConnection(Server $server)
-{
-    if (! (config('constants.ssh.mux_enabled') && config('coolify.is_windows_docker_desktop') == false)) {
-        return;
-    }
-
-    static $ensuredConnections = [];
-
-    if (isset($ensuredConnections[$server->id])) {
-        $muxSocket = $ensuredConnections[$server->id]['muxSocket'];
-        $closeCommand = "ssh -O exit -o ControlPath=$muxSocket {$server->user}@{$server->ip}";
-        Process::run($closeCommand);
-        unset($ensuredConnections[$server->id]);
     }
+    return $output === 'null' ? null : $output;
 }
 
 function instant_remote_process(Collection|array $command, Server $server, bool $throwError = true, bool $no_sudo = false): ?string
 {
-    static $processCount = 0;
+    $command = $command instanceof Collection ? $command->toArray() : $command;
-    $processCount++;
+    if ($server->isNonRoot() && !$no_sudo) {
-
-    $timeout = config('constants.ssh.command_timeout');
-    if ($command instanceof Collection) {
-        $command = $command->toArray();
-    }
-    if ($server->isNonRoot() && ! $no_sudo) {
         $command = parseCommandsByLineForSudo(collect($command), $server);
     }
     $command_string = implode("\n", $command);
 
-    $start_time = microtime(true);
+    // $start_time = microtime(true);
-    $sshCommand = generateSshCommand($server, $command_string);
+    $sshCommand = SshMultiplexingHelper::generateSshCommand($server, $command_string);
-    $process = Process::timeout($timeout)->run($sshCommand);
+    $process = Process::timeout(config('constants.ssh.command_timeout'))->run($sshCommand);
-    $end_time = microtime(true);
+    // $end_time = microtime(true);
 
-    $execution_time = ($end_time - $start_time) * 1000; // Convert to milliseconds
+    // $execution_time = ($end_time - $start_time) * 1000; // Convert to milliseconds
     // ray('SSH command execution time:', $execution_time.' ms')->orange();
 
     $output = trim($process->output());
     $exitCode = $process->exitCode();
 
     if ($exitCode !== 0) {
-        if (! $throwError) {
+        return $throwError ? excludeCertainErrors($process->errorOutput(), $exitCode) : null;
-            return null;
-        }
-
-        return excludeCertainErrors($process->errorOutput(), $exitCode);
     }
-    if ($output === 'null') {
+    return $output === 'null' ? null : $output;
-        $output = null;
-    }
-
-    return $output;
 }
+
 function excludeCertainErrors(string $errorOutput, ?int $exitCode = null)
 {
     $ignoredErrors = collect([
         'Permission denied (publickey',
         'Could not resolve hostname',
     ]);
-    $ignored = false;
+    $ignored = $ignoredErrors->contains(fn($error) => Str::contains($errorOutput, $error));
-    foreach ($ignoredErrors as $ignoredError) {
-        if (Str::contains($errorOutput, $ignoredError)) {
-            $ignored = true;
-            break;
-        }
-    }
     if ($ignored) {
         // TODO: Create new exception and disable in sentry
         throw new \RuntimeException($errorOutput, $exitCode);
     }
     throw new \RuntimeException($errorOutput, $exitCode);
 }
+
 function decode_remote_command_output(?ApplicationDeploymentQueue $application_deployment_queue = null): Collection
 {
-    $application = Application::find(data_get($application_deployment_queue, 'application_id'));
-    $is_debug_enabled = data_get($application, 'settings.is_debug_enabled');
     if (is_null($application_deployment_queue)) {
         return collect([]);
     }
+    $application = Application::find(data_get($application_deployment_queue, 'application_id'));
+    $is_debug_enabled = data_get($application, 'settings.is_debug_enabled');
     try {
         $decoded = json_decode(
             data_get($application_deployment_queue, 'logs'),
@@ -376,20 +127,19 @@ function decode_remote_command_output(?ApplicationDeploymentQueue $application_d
     }
     $seenCommands = collect();
     $formatted = collect($decoded);
-    if (! $is_debug_enabled) {
+    if (!$is_debug_enabled) {
         $formatted = $formatted->filter(fn ($i) => $i['hidden'] === false ?? false);
     }
-    $formatted = $formatted
+    return $formatted
         ->sortBy(fn ($i) => data_get($i, 'order'))
         ->map(function ($i) {
             data_set($i, 'timestamp', Carbon::parse(data_get($i, 'timestamp'))->format('Y-M-d H:i:s.u'));
-
             return $i;
         })
         ->reduce(function ($deploymentLogLines, $logItem) use ($seenCommands) {
             $command = data_get($logItem, 'command');
             $isStderr = data_get($logItem, 'type') === 'stderr';
-            $isNewCommand = ! is_null($command) && ! $seenCommands->first(function ($seenCommand) use ($logItem) {
+            $isNewCommand = !is_null($command) && !$seenCommands->first(function ($seenCommand) use ($logItem) {
                 return data_get($seenCommand, 'command') === data_get($logItem, 'command') && data_get($seenCommand, 'batch') === data_get($logItem, 'batch');
             });
 
@@ -421,36 +171,21 @@ function decode_remote_command_output(?ApplicationDeploymentQueue $application_d
 
             return $deploymentLogLines;
         }, collect());
-
-    return $formatted;
 }
+
 function remove_iip($text)
 {
     $text = preg_replace('/x-access-token:.*?(?=@)/', 'x-access-token:'.REDACTED, $text);
-
     return preg_replace('/\x1b\[[0-9;]*m/', '', $text);
 }
-function remove_mux_and_private_key(Server $server)
-{
-    $muxFilename = $server->muxFilename();
-    $privateKeyLocation = savePrivateKeyToFs($server);
 
-    $closeCommand = "ssh -O exit -o ControlPath=/var/www/html/storage/app/ssh/mux/{$muxFilename} {$server->user}@{$server->ip}";
-    Process::run($closeCommand);
-
-    Storage::disk('ssh-mux')->delete($muxFilename);
-    Storage::disk('ssh-keys')->delete($privateKeyLocation);
-}
 function refresh_server_connection(?PrivateKey $private_key = null)
 {
     if (is_null($private_key)) {
         return;
     }
     foreach ($private_key->servers as $server) {
-        $muxFilename = $server->muxFilename();
+        SshMultiplexingHelper::removeMuxFile($server);
-        $closeCommand = "ssh -O exit -o ControlPath=/var/www/html/storage/app/ssh/mux/{$muxFilename} {$server->user}@{$server->ip}";
-        Process::run($closeCommand);
-        Storage::disk('ssh-mux')->delete($muxFilename);
     }
 }
 
@@ -468,9 +203,8 @@ function checkRequiredCommands(Server $server)
             break;
         }
         $commandFound = instant_remote_process(["docker run --rm --privileged --net=host --pid=host --ipc=host --volume /:/host busybox chroot /host bash -c 'command -v {$command}'"], $server, false);
-        if ($commandFound) {
+        if (!$commandFound) {
-            continue;
+            break;
         }
-        break;
     }
 }

config/constants.php

@@ -6,9 +6,8 @@ return [
         'contact' => 'https://coolify.io/docs/contact',
     ],
     'ssh' => [
-        // Using MUX
+        'mux_enabled' => env('MUX_ENABLED', env('SSH_MUX_ENABLED', true)),
-        'mux_enabled' => env('MUX_ENABLED', env('SSH_MUX_ENABLED', true), true),
+        'mux_persist_time' => env('SSH_MUX_PERSIST_TIME', 3600),
-        'mux_persist_time' => env('SSH_MUX_PERSIST_TIME', '1h'),
         'connection_timeout' => 10,
         'server_interval' => 20,
         'command_timeout' => 7200,

database/migrations/2024_09_16_111428_encrypt_existing_private_keys.php

@@ -0,0 +1,19 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Support\Facades\DB;
+use Illuminate\Support\Facades\Crypt;
+
+class EncryptExistingPrivateKeys extends Migration
+{
+    public function up()
+    {
+        DB::table('private_keys')->chunkById(100, function ($keys) {
+            foreach ($keys as $key) {
+                DB::table('private_keys')
+                    ->where('id', $key->id)
+                    ->update(['private_key' => Crypt::encryptString($key->private_key)]);
+            }
+        });
+    }
+}

database/migrations/2024_09_16_170001_populate_ssh_keys_and_clear_mux_directory.php

@@ -0,0 +1,23 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Support\Facades\Storage;
+use App\Models\PrivateKey;
+
+class PopulateSshKeysAndClearMuxDirectory extends Migration
+{
+    public function up()
+    {
+        Storage::disk('ssh-keys')->deleteDirectory('');
+        Storage::disk('ssh-keys')->makeDirectory('');
+
+        Storage::disk('ssh-mux')->deleteDirectory('');
+        Storage::disk('ssh-mux')->makeDirectory('');
+
+        PrivateKey::chunk(100, function ($keys) {
+            foreach ($keys as $key) {
+                $key->storeInFileSystem();
+            }
+        });
+    }
+}

database/migrations/2024_09_17_111226_add_ssh_key_fingerprint_to_private_keys_table.php

@@ -0,0 +1,31 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+use App\Models\PrivateKey;
+
+class AddSshKeyFingerprintToPrivateKeysTable extends Migration
+{
+    public function up()
+    {
+        Schema::table('private_keys', function (Blueprint $table) {
+            $table->string('fingerprint')->after('private_key')->unique();
+        });
+
+        PrivateKey::whereNull('fingerprint')->each(function ($key) {
+            $fingerprint = PrivateKey::generateFingerprint($key->private_key);
+            if ($fingerprint) {
+                $key->fingerprint = $fingerprint;
+                $key->save();
+            }
+        });
+    }
+
+    public function down()
+    {
+        Schema::table('private_keys', function (Blueprint $table) {
+            $table->dropColumn('fingerprint');
+        });
+    }
+}

database/seeders/DatabaseSeeder.php

@@ -13,6 +13,7 @@ class DatabaseSeeder extends Seeder
             UserSeeder::class,
             TeamSeeder::class,
             PrivateKeySeeder::class,
+            PopulateSshKeysDirectorySeeder::class,
             ServerSeeder::class,
             ServerSettingSeeder::class,
             ProjectSeeder::class,

database/seeders/GithubAppSeeder.php

@@ -31,7 +31,7 @@ class GithubAppSeeder extends Seeder
             'client_id' => 'Iv1.220e564d2b0abd8c',
             'client_secret' => '116d1d80289f378410dd70ab4e4b81dd8d2c52b6',
             'webhook_secret' => '326a47b49054f03288f800d81247ec9414d0abf3',
-            'private_key_id' => 1,
+            'private_key_id' => 2,
             'team_id' => 0,
         ]);
     }

database/seeders/GitlabAppSeeder.php

@@ -20,19 +20,5 @@ class GitlabAppSeeder extends Seeder
             'is_public' => true,
             'team_id' => 0,
         ]);
-        GitlabApp::create([
-            'id' => 2,
-            'name' => 'coolify-laravel-development-private-gitlab',
-            'api_url' => 'https://gitlab.com/api/v4',
-            'html_url' => 'https://gitlab.com',
-            'app_id' => 1234,
-            'app_secret' => '1234',
-            'oauth_id' => 1234,
-            'deploy_key_id' => '1234',
-            'public_key' => 'dfjasiourj',
-            'webhook_token' => '4u3928u4y392',
-            'private_key_id' => 2,
-            'team_id' => 0,
-        ]);
     }
 }

database/seeders/PopulateSshKeysDirectorySeeder.php

@@ -0,0 +1,22 @@
+<?php
+
+namespace Database\Seeders;
+
+use Illuminate\Database\Seeder;
+use Illuminate\Support\Facades\Storage;
+use App\Models\PrivateKey;
+
+class PopulateSshKeysDirectorySeeder extends Seeder
+{
+    public function run()
+    {
+        Storage::disk('ssh-keys')->deleteDirectory('');
+        Storage::disk('ssh-keys')->makeDirectory('');
+
+        PrivateKey::chunk(100, function ($keys) {
+            foreach ($keys as $key) {
+                $key->storeInFileSystem();
+            }
+        });
+    }
+}

database/seeders/PrivateKeySeeder.php

@@ -13,9 +13,8 @@ class PrivateKeySeeder extends Seeder
     public function run(): void
     {
         PrivateKey::create([
-            'id' => 0,
             'team_id' => 0,
-            'name' => 'Testing-host',
+            'name' => 'Testing Host Key',
             'description' => 'This is a test docker container',
             'private_key' => '-----BEGIN OPENSSH PRIVATE KEY-----
 b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW
@@ -25,10 +24,9 @@ AAAECBQw4jg1WRT2IGHMncCiZhURCts2s24HoDS0thHnnRKVuGmoeGq/pojrsyP1pszcNV
 uZx9iFkCELtxrh31QJ68AAAAEXNhaWxANzZmZjY2ZDJlMmRkAQIDBA==
 -----END OPENSSH PRIVATE KEY-----
 ',
-
         ]);
+
         PrivateKey::create([
-            'id' => 1,
             'team_id' => 0,
             'name' => 'development-github-app',
             'description' => 'This is the key for using the development GitHub app',
@@ -61,12 +59,5 @@ a1C8EDKapCw5hAhizEFOUQKOygL8Ipn+tmEUkORYdZ8Q8cWFCv9nIw==
 -----END RSA PRIVATE KEY-----',
             'is_git_related' => true,
         ]);
-        PrivateKey::create([
-            'id' => 2,
-            'team_id' => 0,
-            'name' => 'development-gitlab-app',
-            'description' => 'This is the key for using the development Gitlab app',
-            'private_key' => 'asdf',
-        ]);
     }
 }

database/seeders/ServerSeeder.php

@@ -15,7 +15,7 @@ class ServerSeeder extends Seeder
             'description' => 'This is a test docker container in development mode',
             'ip' => 'coolify-testing-host',
             'team_id' => 0,
-            'private_key_id' => 0,
+            'private_key_id' => 1,
         ]);
     }
 }

resources/views/livewire/security/private-key/create.blade.php

@@ -4,8 +4,8 @@
     <div class="font-bold">You should not use passphrase protected keys.</div>
     </div>
     <div class="flex gap-2 mb-4">
-    <x-forms.button  wire:click="generateNewRSAKey">Generate new RSA SSH Key</x-forms.button>
+        <x-forms.button wire:click="generateNewEDKey">Generate new ED25519 SSH Key (Recommended, fastest and most secure)</x-forms.button>
-    <x-forms.button  wire:click="generateNewEDKey">Generate new ED25519 SSH Key</x-forms.button>
+        <x-forms.button wire:click="generateNewRSAKey">Generate new RSA SSH Key</x-forms.button>
     </div>
     <form class="flex flex-col gap-2" wire:submit='createPrivateKey'>
         <div class="flex gap-2">

resources/views/livewire/server/show-private-key.blade.php

@@ -26,7 +26,8 @@
     <h3 class="pb-4">Choose another Key</h3>
     <div class="grid grid-cols-3 gap-2">
         @forelse ($privateKeys as $private_key)
-            <div class="box group" wire:click='setPrivateKey({{ $private_key->id }})'>
+            <div class="box group cursor-pointer"
+                 wire:click='setPrivateKey({{ $private_key->id }})'>
                 <div class="flex flex-col ">
                     <div class="box-title">{{ $private_key->name }}</div>
                     <div class="box-description">{{ $private_key->description }}</div>

routes/web.php

@@ -271,7 +271,7 @@ Route::middleware(['auth'])->group(function () {
             } else {
                 $server = $execution->scheduledDatabaseBackup->database->destination->server;
             }
-            $privateKeyLocation = savePrivateKeyToFs($server);
+            $privateKeyLocation = $server->privateKey->getKeyLocation();
             $disk = Storage::build([
                 'driver' => 'sftp',
                 'host' => $server->ip,