Merge pull request #1034 from coollabsio/ijpatricio-wip-9

Changes output storage to JSON serialization.

app/Actions/RemoteProcess/DispatchRemoteProcess.php

@@ -20,12 +20,12 @@ class DispatchRemoteProcess
                 ->withProperties($properties)
                 ->performedOn($remoteProcessArgs->model)
                 ->event($remoteProcessArgs->type)
-                ->log("");
+                ->log("[]");
         } else {
             $this->activity = activity()
                 ->withProperties($remoteProcessArgs->toArray())
                 ->event($remoteProcessArgs->type)
-                ->log("");
+                ->log("[]");
         }
     }
 

app/Actions/RemoteProcess/RunRemoteProcess.php

@@ -4,6 +4,7 @@ namespace App\Actions\RemoteProcess;
 
 use App\Enums\ActivityTypes;
 use App\Enums\ProcessStatus;
+use App\Jobs\DeployApplicationJob;
 use Illuminate\Process\ProcessResult;
 use Illuminate\Support\Facades\DB;
 use Illuminate\Support\Facades\Process;
@@ -27,11 +28,6 @@ class RunRemoteProcess
 
     protected int $counter = 1;
 
-    public const MARK_START = "|--";
-    public const MARK_END = "--|";
-    public const SEPARATOR = '|';
-    public const MARK_REGEX = "/(\|--\d+\|\d+\|(?:out|err)--\|)/";
-
     /**
      * Create a new job instance.
      */
@@ -77,6 +73,15 @@ class RunRemoteProcess
         return $processResult;
     }
 
+    protected function getLatestCounter(): int
+    {
+        $description = json_decode($this->activity->description, associative: true, flags: JSON_THROW_ON_ERROR);
+        if ($description === null || count($description) === 0) {
+            return 1;
+        }
+        return end($description)['order'] + 1;
+    }
+
     protected function getCommand(): string
     {
         $user = $this->activity->getExtraProperty('user');
@@ -96,7 +101,7 @@ class RunRemoteProcess
 
         $this->currentTime = $this->elapsedTime();
 
-        $this->activity->description .= $this->encodeOutput($type, $output);
+        $this->activity->description = $this->encodeOutput($type, $output);
 
         if ($this->isAfterLastThrottle()) {
             // Let's write to database.
@@ -109,12 +114,39 @@ class RunRemoteProcess
 
     public function encodeOutput($type, $output)
     {
-        return
+        $outputStack = json_decode($this->activity->description, associative: true, flags: JSON_THROW_ON_ERROR);
-            static::MARK_START . $this->counter++ .
+
-            static::SEPARATOR . $this->elapsedTime() .
+        $outputStack[] = [
-            static::SEPARATOR . $type .
+            'type' => $type,
-            static::MARK_END .
+            'output' => $output,
-            $output;
+            'timestamp' => hrtime(true),
+            'batch' => DeployApplicationJob::$batch_counter,
+            'order' => $this->getLatestCounter(),
+        ];
+
+        return json_encode($outputStack, flags: JSON_THROW_ON_ERROR);
+    }
+
+    public static function decodeOutput(?Activity $activity = null): string
+    {
+        if (is_null($activity)) {
+            return '';
+        }
+
+        try {
+            $decoded = json_decode(
+                data_get($activity, 'description'),
+                associative: true,
+                flags: JSON_THROW_ON_ERROR
+            );
+        } catch (\JsonException $exception) {
+            return '';
+        }
+
+        return collect($decoded)
+            ->sortBy(fn ($i) => $i['order'])
+            ->map(fn ($i) => $i['output'])
+            ->implode("");
     }
 
     /**

app/Actions/RemoteProcess/TidyOutput.php

@@ -1,90 +0,0 @@
-<?php
-
-namespace App\Actions\RemoteProcess;
-
-use Illuminate\Support\Collection;
-use Spatie\Activitylog\Models\Activity;
-
-class TidyOutput
-{
-    protected $output;
-
-    public function __construct(
-        protected Activity $activity
-    )
-    {
-    }
-
-    public function __invoke()
-    {
-        $chunks = preg_split(
-            RunRemoteProcess::MARK_REGEX,
-            $this->activity->description,
-            flags: PREG_SPLIT_DELIM_CAPTURE
-        );
-
-        $tidyRows = $this
-            ->joinMarksWithFollowingItem($chunks)
-            ->reject(fn($i) => $i === '')
-            ->map(function ($i) {
-                if (!preg_match('/\|--(\d+)\|(\d+)\|(out|err)--\|(.*)/', $i, $matches)) {
-                    return $i;
-                }
-                [$wholeLine, $sequence, $elapsedTime, $type, $output] = $matches;
-                return [
-                    'sequence' => $sequence,
-                    'time' => $elapsedTime,
-                    'type' => $type,
-                    'output' => $output,
-                ];
-            });
-
-            return $tidyRows
-                ->sortBy(fn($i) => $i['sequence'])
-                ->map(fn($i) => $i['output'])
-                ->implode("\n");
-    }
-
-    /**
-     * Function to join the defined mark, with the output
-     * that is the following element in the array.
-     *
-     * Turns this:
-     *      [
-     *          "|--1|149|out--|",
-     *          "/root\n",
-     *          "|--2|251|out--|",
-     *          "Welcome 1 times 1\n",
-     *          "|--3|366|out--|",
-     *          "Welcome 2 times 2\n",
-     *          "|--4|466|out--|",
-     *          "Welcome 3 times 3\n",
-     *      ]
-     *
-     *  into this:
-     *
-     *      [
-     *          "|--1|149|out--|/root\n",
-     *          "|--2|251|out--|Welcome 1 times 1\n",
-     *          "|--3|366|out--|Welcome 2 times 2\n",
-     *          "|--4|466|out--|Welcome 3 times 3\n",
-     *      ]
-     */
-    public function joinMarksWithFollowingItem($chunks): Collection
-    {
-        return collect($chunks)->reduce(function ($carry, $item) {
-            $last = $carry->last();
-            if (preg_match(RunRemoteProcess::MARK_REGEX, $last) && !preg_match(RunRemoteProcess::MARK_REGEX, $item)) {
-                // If the last element is a delimiter and the current element is not,
-                // join them together and replace the last element with the joined string
-                $carry->pop();
-                $joined = $last . $item;
-                $carry->push($joined);
-            } else {
-                // Otherwise, just add the current element to the result array
-                $carry->push($item);
-            }
-            return $carry;
-        }, collect());
-    }
-}

app/Http/Controllers/ProjectController.php

@@ -2,10 +2,7 @@
 
 namespace App\Http\Controllers;
 
-use App\Models\Application;
+use Spatie\Activitylog\Models\Activity;
-use App\Models\Environment;
-use Illuminate\Http\Request;
-use Illuminate\Support\Facades\DB;
 
 class ProjectController extends Controller
 {
@@ -64,8 +61,10 @@ class ProjectController extends Controller
         if (!$application) {
             return redirect()->route('home');
         }
+        $activity = Activity::where('properties->deployment_uuid', '=', $deployment_uuid)->first();
 
         return view('project.deployment', [
+            'activity' => $activity,
             'deployment_uuid' => $deployment_uuid,
         ]);
     }

app/Http/Livewire/DeployApplication.php

@@ -2,21 +2,10 @@
 
 namespace App\Http\Livewire;
 
-use App\Jobs\ContainerStatusJob;
 use App\Jobs\DeployApplicationJob;
 use App\Models\Application;
-use App\Models\CoolifyInstanceSettings;
-use DateTimeImmutable;
-use Illuminate\Support\Facades\Http;
-use Illuminate\Support\Facades\Process;
 use Livewire\Component;
-use Symfony\Component\Yaml\Yaml;
 use Visus\Cuid2\Cuid2;
-use Lcobucci\JWT\Encoding\ChainedFormatter;
-use Lcobucci\JWT\Encoding\JoseEncoder;
-use Lcobucci\JWT\Signer\Key\InMemory;
-use Lcobucci\JWT\Signer\Rsa\Sha256;
-use Lcobucci\JWT\Token\Builder;
 
 class DeployApplication extends Component
 {

app/Http/Livewire/PollActivity.php

@@ -24,6 +24,7 @@ class PollActivity extends Component
             $this->isKeepAliveOn = false;
         }
     }
+
     public function render()
     {
         return view('livewire.poll-activity');

app/Jobs/DeployApplicationJob.php

@@ -34,6 +34,7 @@ class DeployApplicationJob implements ShouldQueue
     protected Activity $activity;
     protected string $git_commit;
     protected string $workdir;
+    public static int $batch_counter = 0;
 
     /**
      * Create a new job instance.
@@ -65,7 +66,7 @@ class DeployApplicationJob implements ShouldQueue
             ->performedOn($this->application)
             ->withProperties($remoteProcessArgs->toArray())
             ->event(ActivityTypes::DEPLOYMENT->value)
-            ->log("");
+            ->log("[]");
     }
 
     /**
@@ -94,7 +95,7 @@ class DeployApplicationJob implements ShouldQueue
             "echo -n 'Pulling latest version of the builder image (ghcr.io/coollabsio/coolify-builder)... '",
             "docker run --pull=always -d --name {$this->deployment_uuid} --rm -v /var/run/docker.sock:/var/run/docker.sock ghcr.io/coollabsio/coolify-builder >/dev/null 2>&1",
             "echo 'Done.'",
-        ], 'docker_pull_builder_image');
+        ]);
 
         // Import git repository
         $this->executeNow([
@@ -280,6 +281,7 @@ class DeployApplicationJob implements ShouldQueue
 
     private function executeNow(array|Collection $command, string $propertyName = null, bool $hideFromOutput = false, $setStatus = false)
     {
+        static::$batch_counter++;
         if ($command instanceof Collection) {
             $commandText = $command->implode("\n");
         } else {

database/seeders/ApplicationSeeder.php

@@ -40,31 +40,31 @@ class ApplicationSeeder extends Seeder
             'source_id' => $github_public_source->id,
             'source_type' => GithubApp::class,
         ]);
-        Application::create([
+        // Application::create([
-            'name' => 'Private application (through GitHub App)',
+        //     'name' => 'Private application (through GitHub App)',
-            'git_repository' => 'coollabsio/nodejs-example',
+        //     'git_repository' => 'coollabsio/nodejs-example',
-            'git_branch' => 'main',
+        //     'git_branch' => 'main',
-            'build_pack' => 'nixpacks',
+        //     'build_pack' => 'nixpacks',
-            'ports_exposes' => '3000',
+        //     'ports_exposes' => '3000',
-            'ports_mappings' => '3001:3000',
+        //     'ports_mappings' => '3001:3000',
-            'environment_id' => $environment_1->id,
+        //     'environment_id' => $environment_1->id,
-            'destination_id' => $standalone_docker_1->id,
+        //     'destination_id' => $standalone_docker_1->id,
-            'destination_type' => StandaloneDocker::class,
+        //     'destination_type' => StandaloneDocker::class,
-            'source_id' => $github_private_source->id,
+        //     'source_id' => $github_private_source->id,
-            'source_type' => GithubApp::class,
+        //     'source_type' => GithubApp::class,
-        ]);
+        // ]);
-        Application::create([
+        // Application::create([
-            'name' => 'Public application (from GitHub through Deploy Key)',
+        //     'name' => 'Public application (from GitHub through Deploy Key)',
-            'git_repository' => 'coollabsio/php',
+        //     'git_repository' => 'coollabsio/php',
-            'git_branch' => 'main',
+        //     'git_branch' => 'main',
-            'build_pack' => 'nixpacks',
+        //     'build_pack' => 'nixpacks',
-            'ports_exposes' => '80,3000',
+        //     'ports_exposes' => '80,3000',
-            'ports_mappings' => '3002:80',
+        //     'ports_mappings' => '3002:80',
-            'environment_id' => $environment_1->id,
+        //     'environment_id' => $environment_1->id,
-            'destination_id' => $standalone_docker_1->id,
+        //     'destination_id' => $standalone_docker_1->id,
-            'destination_type' => StandaloneDocker::class,
+        //     'destination_type' => StandaloneDocker::class,
-            'source_id' => $github_private_source_with_deploy_key->id,
+        //     'source_id' => $github_private_source_with_deploy_key->id,
-            'source_type' => GithubApp::class,
+        //     'source_type' => GithubApp::class,
-        ]);
+        // ]);
     }
 }

resources/views/livewire/poll-activity.blade.php

@@ -1,3 +1,3 @@
 <div>
-    <pre style="width: 100%;overflow-y: scroll;" @if ($isKeepAliveOn) wire:poll.750ms="polling" @endif>{{ data_get($activity, 'description') }}</pre>
+    <pre style="width: 100%;overflow-y: scroll;" @if ($isKeepAliveOn) wire:poll.3750ms="polling" @endif>{{ \App\Actions\RemoteProcess\RunRemoteProcess::decodeOutput($activity) }}</pre>
 </div>

resources/views/project/deployment.blade.php

@@ -1,4 +1,4 @@
 <x-layout>
     <h1>Deployment</h1>
-    <livewire:poll-activity :activity="null" :deployment_uuid="$deployment_uuid" />
+    <livewire:poll-activity :activity="$activity" :deployment_uuid="$deployment_uuid" />
 </x-layout>

routes/console.php

@@ -15,5 +15,18 @@ use Illuminate\Support\Facades\Artisan;
 */
 
 Artisan::command('inspire', function () {
-    $this->comment(Inspiring::quote());
+
+    $activity = Spatie\Activitylog\Models\Activity::latest()->first();
+
+    $this->info(
+        collect(
+            json_decode(data_get($activity, 'description'), associative: true, flags: JSON_THROW_ON_ERROR)
+        )
+            ->sortBy('order')
+            ->map(fn($i) => $i['output'])
+            ->implode("\n")
+    );
+
+
+
 })->purpose('Display an inspiring quote');

tests/Feature/DockerCommandsTest.php

@@ -1,5 +1,6 @@
 <?php
 
+use App\Actions\RemoteProcess\RunRemoteProcess;
 use App\Actions\RemoteProcess\TidyOutput;
 use App\Models\User;
 use App\Models\Server;
@@ -33,12 +34,12 @@ it('starts a docker container correctly', function () {
     $host = Server::where('name', 'testing-local-docker-container')->first();
 
     remoteProcess([
-        "docker rm -f $(docker ps --filter='name={$coolifyNamePrefix}*' -aq)"
+        "docker rm -f $(docker ps --filter='name={$coolifyNamePrefix}*' -aq) > /dev/null 2>&1"
     ], $host);
 
     // Assert there's no containers start with coolify_test_*
     $activity = remoteProcess([$areThereCoolifyTestContainers], $host);
-    $tidyOutput = (new TidyOutput($activity))();
+    $tidyOutput = RunRemoteProcess::decodeOutput($activity);
     $containers = formatDockerCmdOutputToJson($tidyOutput);
     expect($containers)->toBeEmpty();
 
@@ -48,7 +49,7 @@ it('starts a docker container correctly', function () {
 
     // docker ps name = $container
     $activity = remoteProcess([$areThereCoolifyTestContainers], $host);
-    $tidyOutput = (new TidyOutput($activity))();
+    $tidyOutput = RunRemoteProcess::decodeOutput($activity);
     $containers = formatDockerCmdOutputToJson($tidyOutput);
     expect($containers->where('Names', $containerName)->count())->toBe(1);
 

tests/Feature/RemoteProcessTest.php

@@ -1,7 +1,6 @@
 <?php
 
 use App\Actions\RemoteProcess\RunRemoteProcess;
-use App\Actions\RemoteProcess\TidyOutput;
 use App\Models\Server;
 use Database\Seeders\DatabaseSeeder;
 use Illuminate\Foundation\Testing\DatabaseMigrations;
@@ -24,13 +23,10 @@ it('outputs correctly', function () {
     ], $host);
 
 
-    preg_match(RunRemoteProcess::MARK_REGEX, $activity->description, $matchesInRawContent);
+    $tidyOutput = RunRemoteProcess::decodeOutput($activity);
-    $out = (new TidyOutput($activity))();
-    preg_match(RunRemoteProcess::MARK_REGEX, $out, $matchesInTidyOutput);
-
-    expect($matchesInRawContent)
-        ->not()->toBeEmpty()
-        ->and($matchesInTidyOutput)
-        ->toBeEmpty();
 
+    expect($tidyOutput)
+        ->toContain('Welcome 1 times')
+        ->toContain('Welcome 3 times')
+        ->not()->toBeJson();
 });