This commit is contained in:
Andras Bacsai
2023-06-30 15:57:40 +02:00
parent b370826624
commit 55d5b1e8da
13 changed files with 560 additions and 75 deletions

View File

@@ -0,0 +1,163 @@
<?php
namespace App\Actions\CoolifyTask;
use App\Enums\ProcessStatus;
use App\Jobs\ApplicationDeploymentJob;
use App\Models\Application;
use App\Models\ApplicationDeploymentQueue;
use Illuminate\Process\ProcessResult;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Process;
const TIMEOUT = 3600;
const IDLE_TIMEOUT = 3600;
class RunRemoteProcessNew
{
protected Application $application;
protected $time_start;
protected $current_time;
protected $last_write_at = 0;
protected $throttle_interval_ms = 500;
protected int $counter = 1;
public function __construct(
public ApplicationDeploymentQueue $application_deployment_queue,
public bool $hide_from_output = false,
public bool $is_finished = false,
public bool $ignore_errors = false
) {
$this->application = Application::find($application_deployment_queue->application_id)->get();
}
public function __invoke(): ProcessResult
{
$this->time_start = hrtime(true);
$status = ProcessStatus::IN_PROGRESS;
$processResult = Process::timeout(TIMEOUT)->idleTimeout(IDLE_TIMEOUT)->run($this->getCommand(), $this->handleOutput(...));
if ($this->application_deployment_queue->properties->get('status') === ProcessStatus::ERROR->value) {
$status = ProcessStatus::ERROR;
} else {
if (($processResult->exitCode() == 0 && $this->is_finished) || $this->application_deployment_queue->properties->get('status') === ProcessStatus::FINISHED->value) {
$status = ProcessStatus::FINISHED;
}
if ($processResult->exitCode() != 0 && !$this->ignore_errors) {
$status = ProcessStatus::ERROR;
}
}
$this->application_deployment_queue->properties = $this->application_deployment_queue->properties->merge([
'exitCode' => $processResult->exitCode(),
'stdout' => $processResult->output(),
'stderr' => $processResult->errorOutput(),
'status' => $status->value,
]);
$this->application_deployment_queue->save();
if ($processResult->exitCode() != 0 && !$this->ignore_errors) {
throw new \RuntimeException($processResult->errorOutput());
}
return $processResult;
}
protected function getLatestCounter(): int
{
$description = json_decode($this->application_deployment_queue->description, associative: true, flags: JSON_THROW_ON_ERROR);
if ($description === null || count($description) === 0) {
return 1;
}
return end($description)['order'] + 1;
}
protected function getCommand(): string
{
$user = data_get($this->application_deployment_queue, 'properties.user');
$server_ip = data_get($this->application_deployment_queue, 'properties.server_ip');
$private_key_location = data_get($this->application_deployment_queue, 'properties.private_key_location');
$port = data_get($this->application_deployment_queue, 'properties.port');
$command = data_get($this->application_deployment_queue, 'properties.command');
return generate_ssh_command($private_key_location, $server_ip, $user, $port, $command);
}
protected function handleOutput(string $type, string $output)
{
if ($this->hide_from_output) {
return;
}
$this->current_time = $this->elapsedTime();
$this->application_deployment_queue->log = $this->encodeOutput($type, $output);
if ($this->isAfterLastThrottle()) {
// Let's write to database.
DB::transaction(function () {
$this->application_deployment_queue->save();
$this->last_write_at = $this->current_time;
});
}
}
public function encodeOutput($type, $output)
{
$outputStack = json_decode($this->application_deployment_queue->description, associative: true, flags: JSON_THROW_ON_ERROR);
$outputStack[] = [
'type' => $type,
'output' => $output,
'timestamp' => hrtime(true),
'batch' => ApplicationDeploymentJob::$batch_counter,
'order' => $this->getLatestCounter(),
];
return json_encode($outputStack, flags: JSON_THROW_ON_ERROR);
}
public static function decodeOutput(?ApplicationDeploymentQueue $application_deployment_queue = null): string
{
if (is_null($application_deployment_queue)) {
return '';
}
try {
$decoded = json_decode(
data_get($application_deployment_queue, 'description'),
associative: true,
flags: JSON_THROW_ON_ERROR
);
} catch (\JsonException $exception) {
return '';
}
return collect($decoded)
->sortBy(fn ($i) => $i['order'])
->map(fn ($i) => $i['output'])
->implode("");
}
/**
* Determines if it's time to write again to database.
*
* @return bool
*/
protected function isAfterLastThrottle()
{
// If DB was never written, then we immediately decide we have to write.
if ($this->last_write_at === 0) {
return true;
}
return ($this->current_time - $this->throttle_interval_ms) > $this->last_write_at;
}
protected function elapsedTime(): int
{
$timeMs = (hrtime(true) - $this->time_start) / 1_000_000;
return intval($timeMs);
}
}

View File

@@ -0,0 +1,12 @@
<?php
namespace App\Enums;
enum ApplicationDeploymentStatus: string
{
case QUEUED = 'queued';
case IN_PROGRESS = 'in_progress';
case FINISHED = 'finished';
case FAILED = 'failed';
case CANCELLED_BY_USER = 'cancelled-by-user';
}

View File

@@ -61,16 +61,16 @@ class ApplicationController extends Controller
if (!$application) {
return redirect()->route('dashboard');
}
$activity = Activity::where('properties->type_uuid', '=', $deploymentUuid)->first();
if (!$activity) {
return redirect()->route('project.application.deployments', [
'project_uuid' => $project->uuid,
'environment_name' => $environment->name,
'application_uuid' => $application->uuid,
]);
}
$deployment = ApplicationDeploymentQueue::where('deployment_uuid', $deploymentUuid)->first();
if (!$deployment) {
// $activity = Activity::where('properties->type_uuid', '=', $deploymentUuid)->first();
// if (!$activity) {
// return redirect()->route('project.application.deployments', [
// 'project_uuid' => $project->uuid,
// 'environment_name' => $environment->name,
// 'application_uuid' => $application->uuid,
// ]);
// }
$application_deployment_queue = ApplicationDeploymentQueue::where('deployment_uuid', $deploymentUuid)->first();
if (!$application_deployment_queue) {
return redirect()->route('project.application.deployments', [
'project_uuid' => $project->uuid,
'environment_name' => $environment->name,
@@ -79,8 +79,8 @@ class ApplicationController extends Controller
}
return view('project.application.deployment', [
'application' => $application,
'activity' => $activity,
'deployment' => $deployment,
// 'activity' => $activity,
'application_deployment_queue' => $application_deployment_queue,
'deployment_uuid' => $deploymentUuid,
]);
}

View File

@@ -2,31 +2,23 @@
namespace App\Http\Livewire\Project\Application;
use App\Enums\ActivityTypes;
use App\Models\Application;
use Illuminate\Support\Facades\Redis;
use App\Models\ApplicationDeploymentQueue;
use Livewire\Component;
use Spatie\Activitylog\Models\Activity;
class DeploymentLogs extends Component
{
public Application $application;
public $activity;
public ApplicationDeploymentQueue $application_deployment_queue;
public $isKeepAliveOn = true;
public $deployment_uuid;
protected $listeners = ['refreshQueue'];
public function refreshQueue()
{
$this->application_deployment_queue->refresh();
}
public function polling()
{
$this->emit('deploymentFinished');
if (is_null($this->activity) && isset($this->deployment_uuid)) {
$this->activity = Activity::query()
->where('properties->type', '=', ActivityTypes::DEPLOYMENT->value)
->where('properties->type_uuid', '=', $this->deployment_uuid)
->first();
} else {
$this->activity?->refresh();
}
if (data_get($this->activity, 'properties.status') == 'finished' || data_get($this->activity, 'properties.status') == 'failed') {
$this->application_deployment_queue->refresh();
if (data_get($this->application_deployment_queue, 'status') == 'finished' || data_get($this->application_deployment_queue, 'status') == 'failed') {
$this->isKeepAliveOn = false;
}
}

View File

@@ -2,41 +2,46 @@
namespace App\Http\Livewire\Project\Application;
use App\Enums\ProcessStatus;
use App\Enums\ApplicationDeploymentStatus;
use App\Models\Application;
use App\Models\ApplicationDeploymentQueue;
use Illuminate\Support\Facades\Process;
use Livewire\Component;
use Illuminate\Support\Str;
class DeploymentNavbar extends Component
{
public Application $application;
public $activity;
public string $deployment_uuid;
protected $listeners = ['deploymentFinished'];
public ApplicationDeploymentQueue $application_deployment_queue;
public function deploymentFinished()
{
$this->activity->refresh();
$this->application_deployment_queue->refresh();
}
public function show_debug()
{
$application = Application::find($this->application_deployment_queue->application_id);
$application->settings->is_debug_enabled = !$application->settings->is_debug_enabled;
$application->settings->save();
$this->emit('refreshQueue');
}
public function cancel()
{
try {
ray('Cancelling deployment: ' . $this->deployment_uuid . ' of application: ' . $this->application->uuid);
// Update deployment queue
$deployment = ApplicationDeploymentQueue::where('deployment_uuid', $this->deployment_uuid)->first();
$deployment->status = 'cancelled by user';
$deployment->save();
// Update activity
$this->activity->properties = $this->activity->properties->merge([
'exitCode' => 1,
'status' => ProcessStatus::CANCELLED->value,
]);
$this->activity->save();
// Remove builder container
instant_remote_process(["docker rm -f {$this->deployment_uuid}"], $this->application->destination->server, throwError: false, repeat: 25);
queue_next_deployment($this->application);
$application = Application::find($this->application_deployment_queue->application_id);
$server = $application->destination->server;
if ($this->application_deployment_queue->current_process_id) {
$process = Process::run("ps -p {$this->application_deployment_queue->current_process_id} -o command --no-headers");
if (Str::of($process->output())->contains([$server->ip, 'EOF-COOLIFY-SSH'])) {
Process::run("kill -9 {$this->application_deployment_queue->current_process_id}");
}
// TODO: Cancelling text in logs
$this->application_deployment_queue->update([
'current_process_id' => null,
'status' => ApplicationDeploymentStatus::CANCELLED_BY_USER->value,
]);
}
} catch (\Throwable $e) {
return general_error_handler(err: $e, that: $this);
}

View File

@@ -0,0 +1,181 @@
<?php
namespace App\Jobs;
use App\Enums\ApplicationDeploymentStatus;
use App\Models\Application;
use App\Models\ApplicationDeploymentQueue;
use App\Models\ApplicationPreview;
use App\Models\GithubApp;
use App\Models\GitlabApp;
use App\Models\Server;
use App\Models\StandaloneDocker;
use App\Models\SwarmDocker;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Spatie\Url\Url;
use Throwable;
use Visus\Cuid2\Cuid2;
class ApplicationDeploymentJobNew implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public static int $batch_counter = 0;
private int $application_deployment_queue_id;
private ApplicationDeploymentQueue $application_deployment_queue;
private Application $application;
private string $deployment_uuid;
private int $pull_request_id;
private string $commit;
private bool $force_rebuild;
private GithubApp|GitlabApp $source;
private StandaloneDocker|SwarmDocker $destination;
private Server $server;
private string $private_key_location;
private ApplicationPreview|null $preview = null;
private string $container_name;
private string $workdir;
private bool $is_debug_enabled;
public function __construct(int $application_deployment_queue_id)
{
$this->application_deployment_queue = ApplicationDeploymentQueue::find($application_deployment_queue_id);
$this->application = Application::find($this->application_deployment_queue->application_id);
$this->application_deployment_queue_id = $application_deployment_queue_id;
$this->deployment_uuid = $this->application_deployment_queue->deployment_uuid;
$this->pull_request_id = $this->application_deployment_queue->pull_request_id;
$this->commit = $this->application_deployment_queue->commit;
$this->force_rebuild = $this->application_deployment_queue->force_rebuild;
$this->source = $this->application->source->getMorphClass()::where('id', $this->application->source->id)->first();
$this->destination = $this->application->destination->getMorphClass()::where('id', $this->application->destination->id)->first();
$this->server = $this->destination->server;
$this->private_key_location = save_private_key_for_server($this->server);
$this->workdir = "/artifacts/{$this->deployment_uuid}";
$this->is_debug_enabled = $this->application->settings->is_debug_enabled;
$this->container_name = generate_container_name($this->application->uuid);
$this->private_key_location = save_private_key_for_server($this->server);
// Set preview fqdn
if ($this->pull_request_id !== 0) {
$this->preview = ApplicationPreview::findPreviewByApplicationAndPullId($this->application->id, $this->pull_request_id);
if ($this->application->fqdn) {
$preview_fqdn = data_get($this->preview, 'fqdn');
$template = $this->application->preview_url_template;
$url = Url::fromString($this->application->fqdn);
$host = $url->getHost();
$schema = $url->getScheme();
$random = new Cuid2(7);
$preview_fqdn = str_replace('{{random}}', $random, $template);
$preview_fqdn = str_replace('{{domain}}', $host, $preview_fqdn);
$preview_fqdn = str_replace('{{pr_id}}', $this->pull_request_id, $preview_fqdn);
$preview_fqdn = "$schema://$preview_fqdn";
$this->preview->fqdn = $preview_fqdn;
$this->preview->save();
}
}
}
public function handle(): void
{
$this->application_deployment_queue->update([
'status' => ApplicationDeploymentStatus::IN_PROGRESS->value,
]);
try {
if ($this->pull_request_id !== 0) {
// $this->deploy_pull_request();
} else {
$this->deploy();
}
} catch (\Exception $e) {
// $this->execute_now([
// "echo '\nOops something is not okay, are you okay? 😢'",
// "echo '\n\n{$e->getMessage()}'",
// ]);
$this->fail($e->getMessage());
} finally {
// if (isset($this->docker_compose)) {
// Storage::disk('deployments')->put(Str::kebab($this->application->name) . '/docker-compose.yml', $this->docker_compose);
// }
// execute_remote_command(
// commands: [
// "docker rm -f {$this->deployment_uuid} >/dev/null 2>&1"
// ],
// server: $this->server,
// queue: $this->application_deployment_queue,
// hide_from_output: true,
// );
}
}
public function failed(Throwable $exception): void
{
ray($exception);
$this->next(ApplicationDeploymentStatus::FAILED->value);
}
private function execute_in_builder(string $command)
{
return "docker exec {$this->deployment_uuid} bash -c '{$command} |& tee -a /proc/1/fd/1'";
}
private function deploy()
{
execute_remote_command(
commands: [
"echo -n 'Pulling latest version of the builder image (ghcr.io/coollabsio/coolify-builder).'",
],
server: $this->server,
queue: $this->application_deployment_queue,
);
execute_remote_command(
commands: [
"docker run --pull=always -d --name {$this->deployment_uuid} --rm -v /var/run/docker.sock:/var/run/docker.sock ghcr.io/coollabsio/coolify-builder",
],
server: $this->server,
queue: $this->application_deployment_queue,
show_in_output: false,
);
execute_remote_command(
commands: [
"echo 'Done.'",
],
server: $this->server,
queue: $this->application_deployment_queue,
);
execute_remote_command(
commands: [
$this->execute_in_builder("mkdir -p {$this->workdir}")
],
server: $this->server,
queue: $this->application_deployment_queue,
);
execute_remote_command(
commands: [
"echos hello"
],
server: $this->server,
queue: $this->application_deployment_queue,
);
$this->next(ApplicationDeploymentStatus::FINISHED->value);
}
private function next(string $status)
{
// If the deployment is cancelled by the user, don't update the status
if ($this->application_deployment_queue->status !== ApplicationDeploymentStatus::CANCELLED_BY_USER->value) {
$this->application_deployment_queue->update([
'status' => $status,
]);
}
queue_next_deployment($this->application);
}
}

View File

@@ -6,13 +6,5 @@ use Illuminate\Database\Eloquent\Model;
class ApplicationDeploymentQueue extends Model
{
protected $fillable = [
'application_id',
'deployment_uuid',
'pull_request_id',
'force_rebuild',
'commit',
'status',
'is_webhook',
];
protected $guarded = [];
}