From e6f0059e5e741d951d90848f1aceeb9889661618 Mon Sep 17 00:00:00 2001 From: Joao Patricio Date: Fri, 7 Apr 2023 15:58:45 +0100 Subject: [PATCH 1/6] Changes output to JSON serialization. --- .../RemoteProcess/DispatchRemoteProcess.php | 4 +- .../RemoteProcess/RunRemoteProcess.php | 44 ++++++--- app/Actions/RemoteProcess/TidyOutput.php | 90 ------------------- app/Http/Livewire/DeployApplication.php | 11 --- app/Http/Livewire/PollActivity.php | 1 + app/Jobs/DeployApplicationJob.php | 4 +- .../views/livewire/poll-activity.blade.php | 4 +- routes/console.php | 15 +++- tests/Feature/DockerCommandsTest.php | 5 +- tests/Feature/RemoteProcessTest.php | 14 ++- 10 files changed, 62 insertions(+), 130 deletions(-) delete mode 100644 app/Actions/RemoteProcess/TidyOutput.php diff --git a/app/Actions/RemoteProcess/DispatchRemoteProcess.php b/app/Actions/RemoteProcess/DispatchRemoteProcess.php index c1f0e8036..2d23513ea 100644 --- a/app/Actions/RemoteProcess/DispatchRemoteProcess.php +++ b/app/Actions/RemoteProcess/DispatchRemoteProcess.php @@ -20,12 +20,12 @@ class DispatchRemoteProcess ->withProperties($properties) ->performedOn($remoteProcessArgs->model) ->event($remoteProcessArgs->type) - ->log(""); + ->log("[]"); } else { $this->activity = activity() ->withProperties($remoteProcessArgs->toArray()) ->event($remoteProcessArgs->type) - ->log(""); + ->log("[]"); } } diff --git a/app/Actions/RemoteProcess/RunRemoteProcess.php b/app/Actions/RemoteProcess/RunRemoteProcess.php index 53e0beb10..322fa5bed 100644 --- a/app/Actions/RemoteProcess/RunRemoteProcess.php +++ b/app/Actions/RemoteProcess/RunRemoteProcess.php @@ -27,11 +27,6 @@ class RunRemoteProcess protected int $counter = 1; - public const MARK_START = "|--"; - public const MARK_END = "--|"; - public const SEPARATOR = '|'; - public const MARK_REGEX = "/(\|--\d+\|\d+\|(?:out|err)--\|)/"; - /** * Create a new job instance. */ @@ -93,7 +88,7 @@ class RunRemoteProcess $this->currentTime = $this->elapsedTime(); - $this->activity->description .= $this->encodeOutput($type, $output); + $this->activity->description = $this->encodeOutput($type, $output); if ($this->isAfterLastThrottle()) { // Let's write to database. @@ -106,12 +101,37 @@ class RunRemoteProcess public function encodeOutput($type, $output) { - return - static::MARK_START . $this->counter++ . - static::SEPARATOR . $this->elapsedTime() . - static::SEPARATOR . $type . - static::MARK_END . - $output; + $outputStack = json_decode($this->activity->description, associative: true, flags: JSON_THROW_ON_ERROR); + + $outputStack[] = [ + 'type' => $type, + 'output' => $output, + 'elapsed_tim' => $this->elapsedTime(), + 'order' => $this->counter++, + ]; + + return json_encode($outputStack, flags: JSON_THROW_ON_ERROR); + } + + public static function decodeOutput(?Activity $activity = null): string + { + if(is_null($activity)) { + return ''; + } + + try { + $decoded = json_decode(data_get($activity, 'description'), + associative: true, + flags: JSON_THROW_ON_ERROR + ); + } catch (\JsonException $exception) { + return ''; + } + + return collect($decoded) + ->sortBy(fn($i) => $i['order']) + ->map(fn($i) => $i['output']) + ->implode("\n"); } /** diff --git a/app/Actions/RemoteProcess/TidyOutput.php b/app/Actions/RemoteProcess/TidyOutput.php deleted file mode 100644 index e2bd02f69..000000000 --- a/app/Actions/RemoteProcess/TidyOutput.php +++ /dev/null @@ -1,90 +0,0 @@ -activity->description, - flags: PREG_SPLIT_DELIM_CAPTURE - ); - - $tidyRows = $this - ->joinMarksWithFollowingItem($chunks) - ->reject(fn($i) => $i === '') - ->map(function ($i) { - if (!preg_match('/\|--(\d+)\|(\d+)\|(out|err)--\|(.*)/', $i, $matches)) { - return $i; - } - [$wholeLine, $sequence, $elapsedTime, $type, $output] = $matches; - return [ - 'sequence' => $sequence, - 'time' => $elapsedTime, - 'type' => $type, - 'output' => $output, - ]; - }); - - return $tidyRows - ->sortBy(fn($i) => $i['sequence']) - ->map(fn($i) => $i['output']) - ->implode("\n"); - } - - /** - * Function to join the defined mark, with the output - * that is the following element in the array. - * - * Turns this: - * [ - * "|--1|149|out--|", - * "/root\n", - * "|--2|251|out--|", - * "Welcome 1 times 1\n", - * "|--3|366|out--|", - * "Welcome 2 times 2\n", - * "|--4|466|out--|", - * "Welcome 3 times 3\n", - * ] - * - * into this: - * - * [ - * "|--1|149|out--|/root\n", - * "|--2|251|out--|Welcome 1 times 1\n", - * "|--3|366|out--|Welcome 2 times 2\n", - * "|--4|466|out--|Welcome 3 times 3\n", - * ] - */ - public function joinMarksWithFollowingItem($chunks): Collection - { - return collect($chunks)->reduce(function ($carry, $item) { - $last = $carry->last(); - if (preg_match(RunRemoteProcess::MARK_REGEX, $last) && !preg_match(RunRemoteProcess::MARK_REGEX, $item)) { - // If the last element is a delimiter and the current element is not, - // join them together and replace the last element with the joined string - $carry->pop(); - $joined = $last . $item; - $carry->push($joined); - } else { - // Otherwise, just add the current element to the result array - $carry->push($item); - } - return $carry; - }, collect()); - } -} diff --git a/app/Http/Livewire/DeployApplication.php b/app/Http/Livewire/DeployApplication.php index 5ad4e7f00..dfaa9a725 100644 --- a/app/Http/Livewire/DeployApplication.php +++ b/app/Http/Livewire/DeployApplication.php @@ -2,21 +2,10 @@ namespace App\Http\Livewire; -use App\Jobs\ContainerStatusJob; use App\Jobs\DeployApplicationJob; use App\Models\Application; -use App\Models\CoolifyInstanceSettings; -use DateTimeImmutable; -use Illuminate\Support\Facades\Http; -use Illuminate\Support\Facades\Process; use Livewire\Component; -use Symfony\Component\Yaml\Yaml; use Visus\Cuid2\Cuid2; -use Lcobucci\JWT\Encoding\ChainedFormatter; -use Lcobucci\JWT\Encoding\JoseEncoder; -use Lcobucci\JWT\Signer\Key\InMemory; -use Lcobucci\JWT\Signer\Rsa\Sha256; -use Lcobucci\JWT\Token\Builder; class DeployApplication extends Component { diff --git a/app/Http/Livewire/PollActivity.php b/app/Http/Livewire/PollActivity.php index ae3be1324..da8ed5252 100644 --- a/app/Http/Livewire/PollActivity.php +++ b/app/Http/Livewire/PollActivity.php @@ -24,6 +24,7 @@ class PollActivity extends Component $this->isKeepAliveOn = false; } } + public function render() { return view('livewire.poll-activity'); diff --git a/app/Jobs/DeployApplicationJob.php b/app/Jobs/DeployApplicationJob.php index 2e1ee1aac..b77c5aeed 100644 --- a/app/Jobs/DeployApplicationJob.php +++ b/app/Jobs/DeployApplicationJob.php @@ -64,7 +64,7 @@ class DeployApplicationJob implements ShouldQueue ->performedOn($this->application) ->withProperties($remoteProcessArgs->toArray()) ->event(ActivityTypes::DEPLOYMENT->value) - ->log(""); + ->log("[]"); } /** @@ -139,7 +139,7 @@ class DeployApplicationJob implements ShouldQueue ], setStatus: true); $this->executeNow([ "docker stop -t 0 {$this->deployment_uuid} >/dev/null" - ]); + ], setStatus: true); } private function execute_in_builder(string $command) diff --git a/resources/views/livewire/poll-activity.blade.php b/resources/views/livewire/poll-activity.blade.php index 4a91ba622..46a7f9371 100644 --- a/resources/views/livewire/poll-activity.blade.php +++ b/resources/views/livewire/poll-activity.blade.php @@ -1,3 +1,5 @@
- 
{{ data_get($activity, 'description') }}
+ 
+        {{ \App\Actions\RemoteProcess\RunRemoteProcess::decodeOutput($activity) }}
+
diff --git a/routes/console.php b/routes/console.php index e05f4c9a1..0f43c3b31 100644 --- a/routes/console.php +++ b/routes/console.php @@ -15,5 +15,18 @@ use Illuminate\Support\Facades\Artisan; */ Artisan::command('inspire', function () { - $this->comment(Inspiring::quote()); + + $activity = Spatie\Activitylog\Models\Activity::latest()->first(); + + $this->info( + collect( + json_decode(data_get($activity, 'description'), associative: true, flags: JSON_THROW_ON_ERROR) + ) + ->sortBy('order') + ->map(fn($i) => $i['output']) + ->implode("\n") + ); + + + })->purpose('Display an inspiring quote'); diff --git a/tests/Feature/DockerCommandsTest.php b/tests/Feature/DockerCommandsTest.php index 5e40fe85d..cc6f76444 100644 --- a/tests/Feature/DockerCommandsTest.php +++ b/tests/Feature/DockerCommandsTest.php @@ -1,5 +1,6 @@ toBeEmpty(); @@ -48,7 +49,7 @@ it('starts a docker container correctly', function () { // docker ps name = $container $activity = remoteProcess([$areThereCoolifyTestContainers], $host); - $tidyOutput = (new TidyOutput($activity))(); + $tidyOutput = RunRemoteProcess::decodeOutput($activity); $containers = formatDockerCmdOutputToJson($tidyOutput); expect($containers->where('Names', $containerName)->count())->toBe(1); diff --git a/tests/Feature/RemoteProcessTest.php b/tests/Feature/RemoteProcessTest.php index aeca27edc..1db036ebf 100644 --- a/tests/Feature/RemoteProcessTest.php +++ b/tests/Feature/RemoteProcessTest.php @@ -1,7 +1,6 @@ description, $matchesInRawContent); - $out = (new TidyOutput($activity))(); - preg_match(RunRemoteProcess::MARK_REGEX, $out, $matchesInTidyOutput); - - expect($matchesInRawContent) - ->not()->toBeEmpty() - ->and($matchesInTidyOutput) - ->toBeEmpty(); + $tidyOutput = RunRemoteProcess::decodeOutput($activity); + expect($tidyOutput) + ->toContain('Welcome 1 times') + ->toContain('Welcome 3 times') + ->not()->toBeJson(); }); From b3657dfe2ba1331c3c27c61522c2b2b6257e34fd Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Wed, 12 Apr 2023 09:58:13 +0200 Subject: [PATCH 2/6] fix: small typo --- app/Actions/RemoteProcess/RunRemoteProcess.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/Actions/RemoteProcess/RunRemoteProcess.php b/app/Actions/RemoteProcess/RunRemoteProcess.php index 322fa5bed..31247af89 100644 --- a/app/Actions/RemoteProcess/RunRemoteProcess.php +++ b/app/Actions/RemoteProcess/RunRemoteProcess.php @@ -106,7 +106,7 @@ class RunRemoteProcess $outputStack[] = [ 'type' => $type, 'output' => $output, - 'elapsed_tim' => $this->elapsedTime(), + 'elapsed_time' => $this->elapsedTime(), 'order' => $this->counter++, ]; From 135eb7a11ee4241cace115a9de63dc846204ffe9 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Wed, 12 Apr 2023 10:14:57 +0200 Subject: [PATCH 3/6] fix: show activity on load --- app/Http/Controllers/ProjectController.php | 7 +++---- resources/views/project/deployment.blade.php | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/app/Http/Controllers/ProjectController.php b/app/Http/Controllers/ProjectController.php index 50612aaa4..7a0594e69 100644 --- a/app/Http/Controllers/ProjectController.php +++ b/app/Http/Controllers/ProjectController.php @@ -2,10 +2,7 @@ namespace App\Http\Controllers; -use App\Models\Application; -use App\Models\Environment; -use Illuminate\Http\Request; -use Illuminate\Support\Facades\DB; +use Spatie\Activitylog\Models\Activity; class ProjectController extends Controller { @@ -64,8 +61,10 @@ class ProjectController extends Controller if (!$application) { return redirect()->route('home'); } + $activity = Activity::where('properties->deployment_uuid', '=', $deployment_uuid)->first(); return view('project.deployment', [ + 'activity' => $activity, 'deployment_uuid' => $deployment_uuid, ]); } diff --git a/resources/views/project/deployment.blade.php b/resources/views/project/deployment.blade.php index 750b7d9e9..2f79fdafa 100644 --- a/resources/views/project/deployment.blade.php +++ b/resources/views/project/deployment.blade.php @@ -1,4 +1,4 @@

Deployment

- + From 040e5a51983a8f06b09eb1a79622ec3007bc5a98 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Wed, 12 Apr 2023 11:04:30 +0200 Subject: [PATCH 4/6] should be one line --- resources/views/livewire/poll-activity.blade.php | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/resources/views/livewire/poll-activity.blade.php b/resources/views/livewire/poll-activity.blade.php index 46a7f9371..c02964a56 100644 --- a/resources/views/livewire/poll-activity.blade.php +++ b/resources/views/livewire/poll-activity.blade.php @@ -1,5 +1,3 @@
- 
-        {{ \App\Actions\RemoteProcess\RunRemoteProcess::decodeOutput($activity) }}
-
+ 
{{ \App\Actions\RemoteProcess\RunRemoteProcess::decodeOutput($activity) }}
From 174fc76fb87b6ddad39308e5d8b0cd2c5491843e Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Wed, 12 Apr 2023 13:09:27 +0200 Subject: [PATCH 5/6] fix: order feat: add batch counter --- .../RemoteProcess/RunRemoteProcess.php | 26 ++++++++++++++----- app/Jobs/DeployApplicationJob.php | 6 +++-- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/app/Actions/RemoteProcess/RunRemoteProcess.php b/app/Actions/RemoteProcess/RunRemoteProcess.php index 31247af89..ca433b450 100644 --- a/app/Actions/RemoteProcess/RunRemoteProcess.php +++ b/app/Actions/RemoteProcess/RunRemoteProcess.php @@ -4,6 +4,7 @@ namespace App\Actions\RemoteProcess; use App\Enums\ActivityTypes; use App\Enums\ProcessStatus; +use App\Jobs\DeployApplicationJob; use Illuminate\Process\ProcessResult; use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Process; @@ -69,6 +70,15 @@ class RunRemoteProcess return $processResult; } + protected function getLatestCounter(): int + { + $description = json_decode($this->activity->description, associative: true, flags: JSON_THROW_ON_ERROR); + if ($description === null || count($description) === 0) { + return 1; + } + return end($description)['order'] + 1; + } + protected function getCommand(): string { $user = $this->activity->getExtraProperty('user'); @@ -106,8 +116,9 @@ class RunRemoteProcess $outputStack[] = [ 'type' => $type, 'output' => $output, - 'elapsed_time' => $this->elapsedTime(), - 'order' => $this->counter++, + 'timestamp' => hrtime(true), + 'batch' => DeployApplicationJob::$batch_counter, + 'order' => $this->getLatestCounter(), ]; return json_encode($outputStack, flags: JSON_THROW_ON_ERROR); @@ -115,12 +126,13 @@ class RunRemoteProcess public static function decodeOutput(?Activity $activity = null): string { - if(is_null($activity)) { + if (is_null($activity)) { return ''; } try { - $decoded = json_decode(data_get($activity, 'description'), + $decoded = json_decode( + data_get($activity, 'description'), associative: true, flags: JSON_THROW_ON_ERROR ); @@ -129,9 +141,9 @@ class RunRemoteProcess } return collect($decoded) - ->sortBy(fn($i) => $i['order']) - ->map(fn($i) => $i['output']) - ->implode("\n"); + ->sortBy(fn ($i) => $i['order']) + ->map(fn ($i) => $i['output']) + ->implode(""); } /** diff --git a/app/Jobs/DeployApplicationJob.php b/app/Jobs/DeployApplicationJob.php index b77c5aeed..ac93828a1 100644 --- a/app/Jobs/DeployApplicationJob.php +++ b/app/Jobs/DeployApplicationJob.php @@ -33,6 +33,7 @@ class DeployApplicationJob implements ShouldQueue protected Activity $activity; protected string $git_commit; protected string $workdir; + public static int $batch_counter = 0; /** * Create a new job instance. @@ -93,14 +94,14 @@ class DeployApplicationJob implements ShouldQueue "echo -n 'Pulling latest version of the builder image (ghcr.io/coollabsio/coolify-builder)... '", "docker run --pull=always -d --name {$this->deployment_uuid} --rm -v /var/run/docker.sock:/var/run/docker.sock ghcr.io/coollabsio/coolify-builder >/dev/null 2>&1", "echo 'Done.'", - ], 'docker_pull_builder_image'); + ]); // Import git repository $this->executeNow([ "echo -n 'Importing {$this->application->git_repository}:{$this->application->git_branch} to {$this->workdir}... '", $this->gitImport(), "echo 'Done.'" - ], 'importing_git_repository'); + ]); // Get git commit $this->executeNow([$this->execute_in_builder("cd {$this->workdir} && git rev-parse HEAD")], 'commit_sha', hideFromOutput: true); @@ -256,6 +257,7 @@ class DeployApplicationJob implements ShouldQueue private function executeNow(array $command, string $propertyName = null, bool $hideFromOutput = false, $setStatus = false) { + static::$batch_counter++; $commandText = collect($command)->implode("\n"); $this->activity->properties = $this->activity->properties->merge([ From 02edfaf54b7e6c60f37753c49f82ce5b3f6d2271 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Wed, 12 Apr 2023 13:39:26 +0200 Subject: [PATCH 6/6] fix: tests --- tests/Feature/DockerCommandsTest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Feature/DockerCommandsTest.php b/tests/Feature/DockerCommandsTest.php index cc6f76444..7cfa3cf58 100644 --- a/tests/Feature/DockerCommandsTest.php +++ b/tests/Feature/DockerCommandsTest.php @@ -34,7 +34,7 @@ it('starts a docker container correctly', function () { $host = Server::where('name', 'testing-local-docker-container')->first(); remoteProcess([ - "docker rm -f $(docker ps --filter='name={$coolifyNamePrefix}*' -aq)" + "docker rm -f $(docker ps --filter='name={$coolifyNamePrefix}*' -aq) > /dev/null 2>&1" ], $host); // Assert there's no containers start with coolify_test_*