From 2320ab0dfc9901ab1366c3ef2bba31e91e4375a3 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Sun, 20 Mar 2022 23:51:50 +0100 Subject: [PATCH 01/12] WIP - Persistent storage --- prisma/schema.prisma | 26 +++++--- src/app.d.ts | 4 +- src/lib/buildPacks/gatsby.ts | 2 +- src/lib/buildPacks/nestjs.ts | 6 +- src/lib/buildPacks/nextjs.ts | 2 +- src/lib/buildPacks/node.ts | 2 +- src/lib/buildPacks/nuxtjs.ts | 2 +- src/lib/buildPacks/react.ts | 2 +- src/lib/buildPacks/rust.ts | 12 ++-- src/lib/buildPacks/static.ts | 4 +- src/lib/buildPacks/svelte.ts | 2 +- src/lib/buildPacks/vuejs.ts | 2 +- src/lib/database/applications.ts | 7 ++- src/lib/docker.ts | 8 +-- src/lib/queues/builder.ts | 24 ++++++-- src/routes/applications/[id]/__layout.svelte | 29 +++++++++ .../applications/[id]/storage/_Storage.svelte | 46 ++++++++++++++ .../applications/[id]/storage/index.json.ts | 58 ++++++++++++++++++ .../applications/[id]/storage/index.svelte | 61 +++++++++++++++++++ 19 files changed, 261 insertions(+), 38 deletions(-) create mode 100644 src/routes/applications/[id]/storage/_Storage.svelte create mode 100644 src/routes/applications/[id]/storage/index.json.ts create mode 100644 src/routes/applications/[id]/storage/index.svelte diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 2a9a43c2b..95310d2aa 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -72,9 +72,9 @@ model TeamInvitation { } model Application { - id String @id @default(cuid()) + id String @id @default(cuid()) name String - fqdn String? @unique + fqdn String? @unique repository String? configHash String? branch String? @@ -86,16 +86,17 @@ model Application { startCommand String? baseDirectory String? publishDirectory String? - createdAt DateTime @default(now()) - updatedAt DateTime @updatedAt + phpModules String? + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt settings ApplicationSettings? teams Team[] destinationDockerId String? - destinationDocker DestinationDocker? @relation(fields: [destinationDockerId], references: [id]) + destinationDocker DestinationDocker? @relation(fields: [destinationDockerId], references: [id]) gitSourceId String? - gitSource GitSource? @relation(fields: [gitSourceId], references: [id]) + gitSource GitSource? @relation(fields: [gitSourceId], references: [id]) secrets Secret[] - phpModules String? + persistentStorage ApplicationPersistentStorage[] } model ApplicationSettings { @@ -110,6 +111,17 @@ model ApplicationSettings { updatedAt DateTime @updatedAt } +model ApplicationPersistentStorage { + id String @id @default(cuid()) + application Application @relation(fields: [applicationId], references: [id]) + applicationId String @unique + path String @unique + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + @@unique([applicationId, path]) +} + model Secret { id String @id @default(cuid()) name String diff --git a/src/app.d.ts b/src/app.d.ts index c3921e0f2..63734a3af 100644 --- a/src/app.d.ts +++ b/src/app.d.ts @@ -7,7 +7,9 @@ declare namespace App { } interface Platform {} interface Session extends SessionData {} - interface Stuff {} + interface Stuff { + application: any; + } } interface SessionData { diff --git a/src/lib/buildPacks/gatsby.ts b/src/lib/buildPacks/gatsby.ts index 3e9c34a2d..a721af128 100644 --- a/src/lib/buildPacks/gatsby.ts +++ b/src/lib/buildPacks/gatsby.ts @@ -8,7 +8,7 @@ const createDockerfile = async (data, imageforBuild): Promise => { Dockerfile.push(`FROM ${imageforBuild}`); Dockerfile.push('WORKDIR /usr/share/nginx/html'); Dockerfile.push(`LABEL coolify.image=true`); - Dockerfile.push(`COPY --from=${applicationId}:${tag}-cache /usr/src/app/${publishDirectory} ./`); + Dockerfile.push(`COPY --from=${applicationId}:${tag}-cache /app/${publishDirectory} ./`); Dockerfile.push(`COPY /nginx.conf /etc/nginx/nginx.conf`); Dockerfile.push(`EXPOSE 80`); Dockerfile.push('CMD ["nginx", "-g", "daemon off;"]'); diff --git a/src/lib/buildPacks/nestjs.ts b/src/lib/buildPacks/nestjs.ts index b30c5ecd9..915bdd3d7 100644 --- a/src/lib/buildPacks/nestjs.ts +++ b/src/lib/buildPacks/nestjs.ts @@ -7,15 +7,13 @@ const createDockerfile = async (data, image): Promise => { const isPnpm = startCommand.includes('pnpm'); Dockerfile.push(`FROM ${image}`); - Dockerfile.push('WORKDIR /usr/src/app'); + Dockerfile.push('WORKDIR /app'); Dockerfile.push(`LABEL coolify.image=true`); if (isPnpm) { Dockerfile.push('RUN curl -f https://get.pnpm.io/v6.16.js | node - add --global pnpm'); Dockerfile.push('RUN pnpm add -g pnpm'); } - Dockerfile.push( - `COPY --from=${applicationId}:${tag}-cache /usr/src/app/${baseDirectory || ''} ./` - ); + Dockerfile.push(`COPY --from=${applicationId}:${tag}-cache /app/${baseDirectory || ''} ./`); Dockerfile.push(`EXPOSE ${port}`); Dockerfile.push(`CMD ${startCommand}`); diff --git a/src/lib/buildPacks/nextjs.ts b/src/lib/buildPacks/nextjs.ts index c8d6a03bc..4f1f3b0d6 100644 --- a/src/lib/buildPacks/nextjs.ts +++ b/src/lib/buildPacks/nextjs.ts @@ -16,7 +16,7 @@ const createDockerfile = async (data, image): Promise => { const Dockerfile: Array = []; const isPnpm = checkPnpm(installCommand, buildCommand, startCommand); Dockerfile.push(`FROM ${image}`); - Dockerfile.push('WORKDIR /usr/src/app'); + Dockerfile.push('WORKDIR /app'); Dockerfile.push(`LABEL coolify.image=true`); if (secrets.length > 0) { secrets.forEach((secret) => { diff --git a/src/lib/buildPacks/node.ts b/src/lib/buildPacks/node.ts index cb99ecec8..54527a728 100644 --- a/src/lib/buildPacks/node.ts +++ b/src/lib/buildPacks/node.ts @@ -17,7 +17,7 @@ const createDockerfile = async (data, image): Promise => { const isPnpm = checkPnpm(installCommand, buildCommand, startCommand); Dockerfile.push(`FROM ${image}`); - Dockerfile.push('WORKDIR /usr/src/app'); + Dockerfile.push('WORKDIR /app'); Dockerfile.push(`LABEL coolify.image=true`); if (secrets.length > 0) { secrets.forEach((secret) => { diff --git a/src/lib/buildPacks/nuxtjs.ts b/src/lib/buildPacks/nuxtjs.ts index ca7c1809c..207afe864 100644 --- a/src/lib/buildPacks/nuxtjs.ts +++ b/src/lib/buildPacks/nuxtjs.ts @@ -16,7 +16,7 @@ const createDockerfile = async (data, image): Promise => { const Dockerfile: Array = []; const isPnpm = checkPnpm(installCommand, buildCommand, startCommand); Dockerfile.push(`FROM ${image}`); - Dockerfile.push('WORKDIR /usr/src/app'); + Dockerfile.push('WORKDIR /app'); Dockerfile.push(`LABEL coolify.image=true`); if (secrets.length > 0) { secrets.forEach((secret) => { diff --git a/src/lib/buildPacks/react.ts b/src/lib/buildPacks/react.ts index 29c462d02..3e372b776 100644 --- a/src/lib/buildPacks/react.ts +++ b/src/lib/buildPacks/react.ts @@ -8,7 +8,7 @@ const createDockerfile = async (data, image): Promise => { Dockerfile.push(`FROM ${image}`); Dockerfile.push(`LABEL coolify.image=true`); Dockerfile.push('WORKDIR /usr/share/nginx/html'); - Dockerfile.push(`COPY --from=${applicationId}:${tag}-cache /usr/src/app/${publishDirectory} ./`); + Dockerfile.push(`COPY --from=${applicationId}:${tag}-cache /app/${publishDirectory} ./`); Dockerfile.push(`COPY /nginx.conf /etc/nginx/nginx.conf`); Dockerfile.push(`EXPOSE 80`); Dockerfile.push('CMD ["nginx", "-g", "daemon off;"]'); diff --git a/src/lib/buildPacks/rust.ts b/src/lib/buildPacks/rust.ts index 49eb59815..586d63140 100644 --- a/src/lib/buildPacks/rust.ts +++ b/src/lib/buildPacks/rust.ts @@ -7,23 +7,21 @@ const createDockerfile = async (data, image, name): Promise => { const { workdir, port, applicationId, tag } = data; const Dockerfile: Array = []; Dockerfile.push(`FROM ${image}`); - Dockerfile.push('WORKDIR /usr/src/app'); + Dockerfile.push('WORKDIR /app'); Dockerfile.push(`LABEL coolify.image=true`); - Dockerfile.push(`COPY --from=${applicationId}:${tag}-cache /usr/src/app/target target`); + Dockerfile.push(`COPY --from=${applicationId}:${tag}-cache /app/target target`); Dockerfile.push(`COPY --from=${applicationId}:${tag}-cache /usr/local/cargo /usr/local/cargo`); Dockerfile.push(`COPY . .`); Dockerfile.push(`RUN cargo build --release --bin ${name}`); Dockerfile.push('FROM debian:buster-slim'); - Dockerfile.push('WORKDIR /usr/src/app'); + Dockerfile.push('WORKDIR /app'); Dockerfile.push( `RUN apt-get update -y && apt-get install -y --no-install-recommends openssl libcurl4 ca-certificates && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/*` ); Dockerfile.push(`RUN update-ca-certificates`); - Dockerfile.push( - `COPY --from=${applicationId}:${tag}-cache /usr/src/app/target/release/${name} ${name}` - ); + Dockerfile.push(`COPY --from=${applicationId}:${tag}-cache /app/target/release/${name} ${name}`); Dockerfile.push(`EXPOSE ${port}`); - Dockerfile.push(`CMD ["/usr/src/app/${name}"]`); + Dockerfile.push(`CMD ["/app/${name}"]`); await fs.writeFile(`${workdir}/Dockerfile`, Dockerfile.join('\n')); }; diff --git a/src/lib/buildPacks/static.ts b/src/lib/buildPacks/static.ts index 075fad0aa..20393ac22 100644 --- a/src/lib/buildPacks/static.ts +++ b/src/lib/buildPacks/static.ts @@ -33,9 +33,7 @@ const createDockerfile = async (data, image): Promise => { }); } if (buildCommand) { - Dockerfile.push( - `COPY --from=${applicationId}:${tag}-cache /usr/src/app/${publishDirectory} ./` - ); + Dockerfile.push(`COPY --from=${applicationId}:${tag}-cache /app/${publishDirectory} ./`); } else { Dockerfile.push(`COPY .${baseDirectory || ''} ./`); } diff --git a/src/lib/buildPacks/svelte.ts b/src/lib/buildPacks/svelte.ts index e6821dafe..181782b89 100644 --- a/src/lib/buildPacks/svelte.ts +++ b/src/lib/buildPacks/svelte.ts @@ -8,7 +8,7 @@ const createDockerfile = async (data, image): Promise => { Dockerfile.push(`FROM ${image}`); Dockerfile.push('WORKDIR /usr/share/nginx/html'); Dockerfile.push(`LABEL coolify.image=true`); - Dockerfile.push(`COPY --from=${applicationId}:${tag}-cache /usr/src/app/${publishDirectory} ./`); + Dockerfile.push(`COPY --from=${applicationId}:${tag}-cache /app/${publishDirectory} ./`); Dockerfile.push(`COPY /nginx.conf /etc/nginx/nginx.conf`); Dockerfile.push(`EXPOSE 80`); Dockerfile.push('CMD ["nginx", "-g", "daemon off;"]'); diff --git a/src/lib/buildPacks/vuejs.ts b/src/lib/buildPacks/vuejs.ts index 4fe90038b..7953977ce 100644 --- a/src/lib/buildPacks/vuejs.ts +++ b/src/lib/buildPacks/vuejs.ts @@ -8,7 +8,7 @@ const createDockerfile = async (data, image): Promise => { Dockerfile.push(`FROM ${image}`); Dockerfile.push('WORKDIR /usr/share/nginx/html'); Dockerfile.push(`LABEL coolify.image=true`); - Dockerfile.push(`COPY --from=${applicationId}:${tag}-cache /usr/src/app/${publishDirectory} ./`); + Dockerfile.push(`COPY --from=${applicationId}:${tag}-cache /app/${publishDirectory} ./`); Dockerfile.push(`COPY /nginx.conf /etc/nginx/nginx.conf`); Dockerfile.push(`EXPOSE 80`); Dockerfile.push('CMD ["nginx", "-g", "daemon off;"]'); diff --git a/src/lib/database/applications.ts b/src/lib/database/applications.ts index d5285e79a..1ee24a567 100644 --- a/src/lib/database/applications.ts +++ b/src/lib/database/applications.ts @@ -134,7 +134,8 @@ export async function getApplication({ id, teamId }) { destinationDocker: true, settings: true, gitSource: { include: { githubApp: true, gitlabApp: true } }, - secrets: true + secrets: true, + persistentStorage: true } }); @@ -268,3 +269,7 @@ export async function createBuild({ } }); } + +export async function getPersistentStorage(id) { + return await prisma.applicationPersistentStorage.findMany({ where: { applicationId: id } }); +} diff --git a/src/lib/docker.ts b/src/lib/docker.ts index ccbc21b2a..717303040 100644 --- a/src/lib/docker.ts +++ b/src/lib/docker.ts @@ -20,7 +20,7 @@ export async function buildCacheImageWithNode(data, imageForBuild) { const isPnpm = checkPnpm(installCommand, buildCommand); const Dockerfile: Array = []; Dockerfile.push(`FROM ${imageForBuild}`); - Dockerfile.push('WORKDIR /usr/src/app'); + Dockerfile.push('WORKDIR /app'); Dockerfile.push(`LABEL coolify.image=true`); if (secrets.length > 0) { secrets.forEach((secret) => { @@ -65,14 +65,14 @@ export async function buildCacheImageWithCargo(data, imageForBuild) { } = data; const Dockerfile: Array = []; Dockerfile.push(`FROM ${imageForBuild} as planner-${applicationId}`); - Dockerfile.push('WORKDIR /usr/src/app'); + Dockerfile.push('WORKDIR /app'); Dockerfile.push('RUN cargo install cargo-chef'); Dockerfile.push('COPY . .'); Dockerfile.push('RUN cargo chef prepare --recipe-path recipe.json'); Dockerfile.push(`FROM ${imageForBuild}`); - Dockerfile.push('WORKDIR /usr/src/app'); + Dockerfile.push('WORKDIR /app'); Dockerfile.push('RUN cargo install cargo-chef'); - Dockerfile.push(`COPY --from=planner-${applicationId} /usr/src/app/recipe.json recipe.json`); + Dockerfile.push(`COPY --from=planner-${applicationId} /app/recipe.json recipe.json`); Dockerfile.push('RUN cargo chef cook --release --recipe-path recipe.json'); await fs.writeFile(`${workdir}/Dockerfile-cache`, Dockerfile.join('\n')); await buildImage({ applicationId, tag, workdir, docker, buildId, isCache: true, debug }); diff --git a/src/lib/queues/builder.ts b/src/lib/queues/builder.ts index f0617d328..cad82bfe0 100644 --- a/src/lib/queues/builder.ts +++ b/src/lib/queues/builder.ts @@ -49,8 +49,10 @@ export default async function (job) { type, pullmergeRequestId = null, sourceBranch = null, - settings + settings, + persistentStorage } = job.data; + console.log(persistentStorage); const { debug } = settings; await asyncSleep(1000); @@ -68,6 +70,10 @@ export default async function (job) { let domain = getDomain(fqdn); const isHttps = fqdn.startsWith('https://'); + let volumes = + persistentStorage?.map((storage) => { + return `${applicationId}-${storage.id}:${storage.path}`; + }) || []; // Previews, we need to get the source branch and set subdomain if (pullmergeRequestId) { branch = sourceBranch; @@ -252,12 +258,22 @@ export default async function (job) { } try { saveBuildLog({ line: 'Deployment started.', buildId, applicationId }); + for await (const volume of volumes) { + const id = volume.split(':')[0]; + try { + await asyncExecShell(`DOCKER_HOST=${host} docker volume inspect ${id}`); + } catch (error) { + await asyncExecShell(`DOCKER_HOST=${host} docker volume create ${id}`); + } + } + volumes = volumes.map((volume) => `-v ${volume} `).join(); + console.log(volumes); const { stderr } = await asyncExecShell( `DOCKER_HOST=${host} docker run ${envFound && `--env-file=${workdir}/.env`} ${labels.join( ' ' - )} --name ${imageId} --network ${ - docker.network - } --restart always -d ${applicationId}:${tag}` + )} --name ${imageId} --network ${docker.network} --restart always ${ + volumes.length > 0 && volumes + } -d ${applicationId}:${tag}` ); if (stderr) console.log(stderr); saveBuildLog({ line: 'Deployment successful!', buildId, applicationId }); diff --git a/src/routes/applications/[id]/__layout.svelte b/src/routes/applications/[id]/__layout.svelte index fb944519b..c41f33282 100644 --- a/src/routes/applications/[id]/__layout.svelte +++ b/src/routes/applications/[id]/__layout.svelte @@ -271,6 +271,35 @@ + + + export let isNew = false; + export let storage = { + id: null, + path: null + }; + import { del, post } from '$lib/api'; + import { page } from '$app/stores'; + import { errorNotification } from '$lib/form'; + const { id } = $page.params; + + async function saveStorage() { + try { + await post(`/applications/${id}/storage.json`, { + path: storage.path + }); + } catch ({ error }) { + return errorNotification(error); + } + } + async function removeStorage() { + try { + await del(`/applications/${id}/storage.json`, { path: storage.path }); + } catch ({ error }) { + return errorNotification(error); + } + } + + + + + + +
+ +
+
+ +
+ diff --git a/src/routes/applications/[id]/storage/index.json.ts b/src/routes/applications/[id]/storage/index.json.ts new file mode 100644 index 000000000..1c0bcab6a --- /dev/null +++ b/src/routes/applications/[id]/storage/index.json.ts @@ -0,0 +1,58 @@ +import { getTeam, getUserDetails } from '$lib/common'; +import * as db from '$lib/database'; +import { ErrorHandler } from '$lib/database'; +import { dockerInstance } from '$lib/docker'; +import type { RequestHandler } from '@sveltejs/kit'; +import jsonwebtoken from 'jsonwebtoken'; + +export const get: RequestHandler = async (event) => { + const { status, body, teamId } = await getUserDetails(event, false); + if (status === 401) return { status, body }; + + const { id } = event.params; + try { + const persistentStorages = await db.getPersistentStorage(id); + return { + body: { + persistentStorages + } + }; + } catch (error) { + return ErrorHandler(error); + } +}; + +export const post: RequestHandler = async (event) => { + const { teamId, status, body } = await getUserDetails(event); + if (status === 401) return { status, body }; + + const { id } = event.params; + const { path } = await event.request.json(); + try { + await db.prisma.applicationPersistentStorage.create({ + data: { path, application: { connect: { id } } } + }); + return { + status: 201 + }; + } catch (error) { + return ErrorHandler(error); + } +}; + +export const del: RequestHandler = async (event) => { + const { teamId, status, body } = await getUserDetails(event); + if (status === 401) return { status, body }; + + const { id } = event.params; + const { path } = await event.request.json(); + + try { + await db.prisma.applicationPersistentStorage.deleteMany({ where: { applicationId: id, path } }); + return { + status: 200 + }; + } catch (error) { + return ErrorHandler(error); + } +}; diff --git a/src/routes/applications/[id]/storage/index.svelte b/src/routes/applications/[id]/storage/index.svelte new file mode 100644 index 000000000..1d64bbf6c --- /dev/null +++ b/src/routes/applications/[id]/storage/index.svelte @@ -0,0 +1,61 @@ + + + + +
+
+ Persistent storage for {getDomain(application.fqdn)} +
+
+ +
+ + + + + + + + {#each persistentStorages as storage} + {#key storage.id} + + + + {/key} + {/each} + + + + +
Path
+
From c013764b61e6d8816bbd387a8f0c0b2c6809f090 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Mon, 21 Mar 2022 16:58:13 +0100 Subject: [PATCH 02/12] WIP Persistent storage --- package.json | 2 +- src/lib/buildPacks/common.ts | 1 + src/lib/buildPacks/php.ts | 16 +++++++++------- src/lib/queues/builder.ts | 2 +- 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/package.json b/package.json index 3e849fa1f..fbd8c3720 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,7 @@ "version": "2.0.31", "license": "AGPL-3.0", "scripts": { - "dev": "docker-compose -f docker-compose-dev.yaml up -d && NODE_ENV=development svelte-kit dev --host 0.0.0.0", + "dev": "docker-compose -f docker-compose-dev.yaml up -d && NODE_ENV=development svelte-kit dev", "dev:stop": "docker-compose -f docker-compose-dev.yaml down", "dev:logs": "docker-compose -f docker-compose-dev.yaml logs -f --tail 10", "studio": "npx prisma studio", diff --git a/src/lib/buildPacks/common.ts b/src/lib/buildPacks/common.ts index 1d2fef25f..b1c02867b 100644 --- a/src/lib/buildPacks/common.ts +++ b/src/lib/buildPacks/common.ts @@ -135,6 +135,7 @@ export async function copyBaseConfigurationFiles(buildPack, workdir, buildId, ap RewriteRule ^(.+)$ index.php [QSA,L] ` ); + await fs.writeFile(`${workdir}/entrypoint.sh`, `chown -R www-data /app`); saveBuildLog({ line: 'Copied default configuration file for PHP.', buildId, applicationId }); } else if (staticDeployments.includes(buildPack)) { await fs.writeFile( diff --git a/src/lib/buildPacks/php.ts b/src/lib/buildPacks/php.ts index 4903213c6..8b9617845 100644 --- a/src/lib/buildPacks/php.ts +++ b/src/lib/buildPacks/php.ts @@ -13,19 +13,21 @@ const createDockerfile = async (data, image): Promise => { Dockerfile.push(`RUN chmod +x /usr/local/bin/install-php-extensions`); Dockerfile.push(`RUN /usr/local/bin/install-php-extensions ${data.phpModules.join(' ')}`); } - Dockerfile.push('RUN a2enmod rewrite'); - Dockerfile.push('WORKDIR /var/www/html'); - Dockerfile.push(`COPY .${baseDirectory || ''} /var/www/html`); - Dockerfile.push(`COPY /.htaccess /var/www/html/.htaccess`); + // Dockerfile.push('RUN a2enmod rewrite'); + // Dockerfile.push(`ENV APACHE_DOCUMENT_ROOT /app`); + // Dockerfile.push(`RUN sed -ri -e 's!/var/www/html!/app!g' /etc/apache2/sites-available/*.conf`); + // Dockerfile.push(`RUN sed -ri -e 's!/var/www/!/app!g' /etc/apache2/apache2.conf /etc/apache2/conf-available/*.conf`) + Dockerfile.push('WORKDIR /app'); + Dockerfile.push(`COPY .${baseDirectory || ''} /app`); + Dockerfile.push(`COPY /.htaccess .`); + Dockerfile.push(`COPY /entrypoint.sh /entrypoint.sh`); Dockerfile.push(`EXPOSE 80`); - Dockerfile.push('CMD ["apache2-foreground"]'); - Dockerfile.push('RUN chown -R www-data /var/www/html'); await fs.writeFile(`${workdir}/Dockerfile`, Dockerfile.join('\n')); }; export default async function (data) { try { - const image = 'php:apache'; + const image = 'webdevops/php-nginx'; await createDockerfile(data, image); await buildImage(data); } catch (error) { diff --git a/src/lib/queues/builder.ts b/src/lib/queues/builder.ts index cad82bfe0..18985a5fb 100644 --- a/src/lib/queues/builder.ts +++ b/src/lib/queues/builder.ts @@ -272,7 +272,7 @@ export default async function (job) { `DOCKER_HOST=${host} docker run ${envFound && `--env-file=${workdir}/.env`} ${labels.join( ' ' )} --name ${imageId} --network ${docker.network} --restart always ${ - volumes.length > 0 && volumes + volumes.length > 0 ? volumes : '' } -d ${applicationId}:${tag}` ); if (stderr) console.log(stderr); From f5e7a84fa6241c66dea85e400bfc59bb707d72ab Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Mon, 21 Mar 2022 21:25:01 +0100 Subject: [PATCH 03/12] Update buildpacks for static sites --- src/lib/buildPacks/common.ts | 28 ++++++++++++++++++---------- src/lib/buildPacks/gatsby.ts | 5 ++--- src/lib/buildPacks/php.ts | 4 ---- src/lib/buildPacks/react.ts | 5 ++--- src/lib/buildPacks/static.ts | 5 ++--- src/lib/buildPacks/svelte.ts | 5 ++--- src/lib/buildPacks/vuejs.ts | 20 ++------------------ src/lib/queues/builder.ts | 5 +---- 8 files changed, 29 insertions(+), 48 deletions(-) diff --git a/src/lib/buildPacks/common.ts b/src/lib/buildPacks/common.ts index b1c02867b..9ded4e0f9 100644 --- a/src/lib/buildPacks/common.ts +++ b/src/lib/buildPacks/common.ts @@ -143,27 +143,35 @@ export async function copyBaseConfigurationFiles(buildPack, workdir, buildId, ap `user nginx; worker_processes auto; - error_log /var/log/nginx/error.log warn; - pid /var/run/nginx.pid; + error_log /docker.stdout; + pid /run/nginx.pid; events { worker_connections 1024; } http { - include /etc/nginx/mime.types; - - access_log off; - sendfile on; - #tcp_nopush on; - keepalive_timeout 65; + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /docker.stdout main; + + sendfile on; + tcp_nopush on; + tcp_nodelay on; + keepalive_timeout 65; + types_hash_max_size 2048; + + include /etc/nginx/mime.types; + default_type application/octet-stream; server { listen 80; server_name localhost; location / { - root /usr/share/nginx/html; + root /app; index index.html; try_files $uri $uri/index.html $uri/ /index.html =404; } @@ -174,7 +182,7 @@ export async function copyBaseConfigurationFiles(buildPack, workdir, buildId, ap # error_page 500 502 503 504 /50x.html; location = /50x.html { - root /usr/share/nginx/html; + root /app; } } diff --git a/src/lib/buildPacks/gatsby.ts b/src/lib/buildPacks/gatsby.ts index a721af128..38989626d 100644 --- a/src/lib/buildPacks/gatsby.ts +++ b/src/lib/buildPacks/gatsby.ts @@ -6,18 +6,17 @@ const createDockerfile = async (data, imageforBuild): Promise => { const Dockerfile: Array = []; Dockerfile.push(`FROM ${imageforBuild}`); - Dockerfile.push('WORKDIR /usr/share/nginx/html'); + Dockerfile.push('WORKDIR /app'); Dockerfile.push(`LABEL coolify.image=true`); Dockerfile.push(`COPY --from=${applicationId}:${tag}-cache /app/${publishDirectory} ./`); Dockerfile.push(`COPY /nginx.conf /etc/nginx/nginx.conf`); Dockerfile.push(`EXPOSE 80`); - Dockerfile.push('CMD ["nginx", "-g", "daemon off;"]'); await fs.writeFile(`${workdir}/Dockerfile`, Dockerfile.join('\n')); }; export default async function (data) { try { - const image = 'nginx:stable-alpine'; + const image = 'webdevops/nginx:alpine'; const imageForBuild = 'node:lts'; await buildCacheImageWithNode(data, imageForBuild); diff --git a/src/lib/buildPacks/php.ts b/src/lib/buildPacks/php.ts index 8b9617845..0db2dd063 100644 --- a/src/lib/buildPacks/php.ts +++ b/src/lib/buildPacks/php.ts @@ -13,10 +13,6 @@ const createDockerfile = async (data, image): Promise => { Dockerfile.push(`RUN chmod +x /usr/local/bin/install-php-extensions`); Dockerfile.push(`RUN /usr/local/bin/install-php-extensions ${data.phpModules.join(' ')}`); } - // Dockerfile.push('RUN a2enmod rewrite'); - // Dockerfile.push(`ENV APACHE_DOCUMENT_ROOT /app`); - // Dockerfile.push(`RUN sed -ri -e 's!/var/www/html!/app!g' /etc/apache2/sites-available/*.conf`); - // Dockerfile.push(`RUN sed -ri -e 's!/var/www/!/app!g' /etc/apache2/apache2.conf /etc/apache2/conf-available/*.conf`) Dockerfile.push('WORKDIR /app'); Dockerfile.push(`COPY .${baseDirectory || ''} /app`); Dockerfile.push(`COPY /.htaccess .`); diff --git a/src/lib/buildPacks/react.ts b/src/lib/buildPacks/react.ts index 3e372b776..719f782eb 100644 --- a/src/lib/buildPacks/react.ts +++ b/src/lib/buildPacks/react.ts @@ -7,17 +7,16 @@ const createDockerfile = async (data, image): Promise => { Dockerfile.push(`FROM ${image}`); Dockerfile.push(`LABEL coolify.image=true`); - Dockerfile.push('WORKDIR /usr/share/nginx/html'); + Dockerfile.push('WORKDIR /app'); Dockerfile.push(`COPY --from=${applicationId}:${tag}-cache /app/${publishDirectory} ./`); Dockerfile.push(`COPY /nginx.conf /etc/nginx/nginx.conf`); Dockerfile.push(`EXPOSE 80`); - Dockerfile.push('CMD ["nginx", "-g", "daemon off;"]'); await fs.writeFile(`${workdir}/Dockerfile`, Dockerfile.join('\n')); }; export default async function (data) { try { - const image = 'nginx:stable-alpine'; + const image = 'webdevops/nginx:alpine'; const imageForBuild = 'node:lts'; await buildCacheImageWithNode(data, imageForBuild); await createDockerfile(data, image); diff --git a/src/lib/buildPacks/static.ts b/src/lib/buildPacks/static.ts index 20393ac22..17900ced3 100644 --- a/src/lib/buildPacks/static.ts +++ b/src/lib/buildPacks/static.ts @@ -15,7 +15,7 @@ const createDockerfile = async (data, image): Promise => { const Dockerfile: Array = []; Dockerfile.push(`FROM ${image}`); - Dockerfile.push('WORKDIR /usr/share/nginx/html'); + Dockerfile.push('WORKDIR /app'); Dockerfile.push(`LABEL coolify.image=true`); if (secrets.length > 0) { secrets.forEach((secret) => { @@ -39,13 +39,12 @@ const createDockerfile = async (data, image): Promise => { } Dockerfile.push(`COPY /nginx.conf /etc/nginx/nginx.conf`); Dockerfile.push(`EXPOSE 80`); - Dockerfile.push('CMD ["nginx", "-g", "daemon off;"]'); await fs.writeFile(`${workdir}/Dockerfile`, Dockerfile.join('\n')); }; export default async function (data) { try { - const image = 'nginx:stable-alpine'; + const image = 'webdevops/nginx:alpine'; const imageForBuild = 'node:lts'; if (data.buildCommand) await buildCacheImageWithNode(data, imageForBuild); await createDockerfile(data, image); diff --git a/src/lib/buildPacks/svelte.ts b/src/lib/buildPacks/svelte.ts index 181782b89..fbcf36abb 100644 --- a/src/lib/buildPacks/svelte.ts +++ b/src/lib/buildPacks/svelte.ts @@ -6,18 +6,17 @@ const createDockerfile = async (data, image): Promise => { const Dockerfile: Array = []; Dockerfile.push(`FROM ${image}`); - Dockerfile.push('WORKDIR /usr/share/nginx/html'); + Dockerfile.push('WORKDIR /app'); Dockerfile.push(`LABEL coolify.image=true`); Dockerfile.push(`COPY --from=${applicationId}:${tag}-cache /app/${publishDirectory} ./`); Dockerfile.push(`COPY /nginx.conf /etc/nginx/nginx.conf`); Dockerfile.push(`EXPOSE 80`); - Dockerfile.push('CMD ["nginx", "-g", "daemon off;"]'); await fs.writeFile(`${workdir}/Dockerfile`, Dockerfile.join('\n')); }; export default async function (data) { try { - const image = 'nginx:stable-alpine'; + const image = 'webdevops/nginx:alpine'; const imageForBuild = 'node:lts'; await buildCacheImageWithNode(data, imageForBuild); diff --git a/src/lib/buildPacks/vuejs.ts b/src/lib/buildPacks/vuejs.ts index 7953977ce..fa80ac435 100644 --- a/src/lib/buildPacks/vuejs.ts +++ b/src/lib/buildPacks/vuejs.ts @@ -6,35 +6,19 @@ const createDockerfile = async (data, image): Promise => { const Dockerfile: Array = []; Dockerfile.push(`FROM ${image}`); - Dockerfile.push('WORKDIR /usr/share/nginx/html'); + Dockerfile.push('WORKDIR /app'); Dockerfile.push(`LABEL coolify.image=true`); Dockerfile.push(`COPY --from=${applicationId}:${tag}-cache /app/${publishDirectory} ./`); Dockerfile.push(`COPY /nginx.conf /etc/nginx/nginx.conf`); Dockerfile.push(`EXPOSE 80`); - Dockerfile.push('CMD ["nginx", "-g", "daemon off;"]'); await fs.writeFile(`${workdir}/Dockerfile`, Dockerfile.join('\n')); }; export default async function (data) { try { - const image = 'nginx:stable-alpine'; + const image = 'webdevops/nginx:alpine'; const imageForBuild = 'node:lts'; await buildCacheImageWithNode(data, imageForBuild); - // await fs.writeFile(`${data.workdir}/default.conf`, `server { - // listen 80; - // server_name localhost; - - // location / { - // root /usr/share/nginx/html; - // try_files $uri $uri/ /index.html; - // } - - // error_page 500 502 503 504 /50x.html; - // location = /50x.html { - // root /usr/share/nginx/html; - // } - // } - // `); await createDockerfile(data, image); await buildImage(data); } catch (error) { diff --git a/src/lib/queues/builder.ts b/src/lib/queues/builder.ts index 18985a5fb..1cbef0f85 100644 --- a/src/lib/queues/builder.ts +++ b/src/lib/queues/builder.ts @@ -52,7 +52,6 @@ export default async function (job) { settings, persistentStorage } = job.data; - console.log(persistentStorage); const { debug } = settings; await asyncSleep(1000); @@ -68,11 +67,10 @@ export default async function (job) { }); let imageId = applicationId; let domain = getDomain(fqdn); - const isHttps = fqdn.startsWith('https://'); let volumes = persistentStorage?.map((storage) => { - return `${applicationId}-${storage.id}:${storage.path}`; + return `${applicationId}-${storage.id}:${type !== 'docker' ? '/app/' : ''}${storage.path}`; }) || []; // Previews, we need to get the source branch and set subdomain if (pullmergeRequestId) { @@ -267,7 +265,6 @@ export default async function (job) { } } volumes = volumes.map((volume) => `-v ${volume} `).join(); - console.log(volumes); const { stderr } = await asyncExecShell( `DOCKER_HOST=${host} docker run ${envFound && `--env-file=${workdir}/.env`} ${labels.join( ' ' From a6d5316090e0f2c4b6aee50b71b4db0269dcd9d2 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Mon, 21 Mar 2022 21:46:49 +0100 Subject: [PATCH 04/12] WIP - Persistent storage --- src/lib/buildPacks/common.ts | 2 +- src/lib/buildPacks/php.ts | 2 +- src/lib/queues/builder.ts | 4 +++- src/routes/applications/[id]/storage/_Storage.svelte | 3 +++ 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/lib/buildPacks/common.ts b/src/lib/buildPacks/common.ts index 9ded4e0f9..3214f7adf 100644 --- a/src/lib/buildPacks/common.ts +++ b/src/lib/buildPacks/common.ts @@ -135,7 +135,7 @@ export async function copyBaseConfigurationFiles(buildPack, workdir, buildId, ap RewriteRule ^(.+)$ index.php [QSA,L] ` ); - await fs.writeFile(`${workdir}/entrypoint.sh`, `chown -R www-data /app`); + await fs.writeFile(`${workdir}/entrypoint.sh`, `chown -R 1000 /app`); saveBuildLog({ line: 'Copied default configuration file for PHP.', buildId, applicationId }); } else if (staticDeployments.includes(buildPack)) { await fs.writeFile( diff --git a/src/lib/buildPacks/php.ts b/src/lib/buildPacks/php.ts index 0db2dd063..3ac7b4670 100644 --- a/src/lib/buildPacks/php.ts +++ b/src/lib/buildPacks/php.ts @@ -16,7 +16,7 @@ const createDockerfile = async (data, image): Promise => { Dockerfile.push('WORKDIR /app'); Dockerfile.push(`COPY .${baseDirectory || ''} /app`); Dockerfile.push(`COPY /.htaccess .`); - Dockerfile.push(`COPY /entrypoint.sh /entrypoint.sh`); + Dockerfile.push(`COPY /entrypoint.sh /opt/docker/provision/entrypoint.d/30-entrypoint.sh`); Dockerfile.push(`EXPOSE 80`); await fs.writeFile(`${workdir}/Dockerfile`, Dockerfile.join('\n')); }; diff --git a/src/lib/queues/builder.ts b/src/lib/queues/builder.ts index 1cbef0f85..4a34718c5 100644 --- a/src/lib/queues/builder.ts +++ b/src/lib/queues/builder.ts @@ -70,7 +70,9 @@ export default async function (job) { let volumes = persistentStorage?.map((storage) => { - return `${applicationId}-${storage.id}:${type !== 'docker' ? '/app/' : ''}${storage.path}`; + return `${applicationId}${storage.path.replace(/\//gi, '-')}:${ + type !== 'docker' ? '/app/' : '' + }${storage.path}`; }) || []; // Previews, we need to get the source branch and set subdomain if (pullmergeRequestId) { diff --git a/src/routes/applications/[id]/storage/_Storage.svelte b/src/routes/applications/[id]/storage/_Storage.svelte index d9bdc9ebe..3fa86ef21 100644 --- a/src/routes/applications/[id]/storage/_Storage.svelte +++ b/src/routes/applications/[id]/storage/_Storage.svelte @@ -11,6 +11,9 @@ async function saveStorage() { try { + storage.path = storage.path.startsWith('/') ? storage.path : `/${storage.path}`; + storage.path = storage.path.endsWith('/') ? storage.path.slice(0, -1) : storage.path; + storage.path.replace(/\/\//g, '/'); await post(`/applications/${id}/storage.json`, { path: storage.path }); From 6617b7811befb55ba7d0654767a9d2104a0d2f69 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Tue, 22 Mar 2022 10:22:20 +0100 Subject: [PATCH 05/12] log ssl errors --- src/lib/queues/ssl.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib/queues/ssl.ts b/src/lib/queues/ssl.ts index d14fc0822..f041153d6 100644 --- a/src/lib/queues/ssl.ts +++ b/src/lib/queues/ssl.ts @@ -4,6 +4,7 @@ export default async function () { try { return await generateSSLCerts(); } catch (error) { + console.log(error); throw error; } } From 6b89857697a44c07c4a368d7156d8e6e00a13841 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Tue, 22 Mar 2022 10:24:52 +0100 Subject: [PATCH 06/12] chore: version++ --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 9a65ee605..4071ffb20 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "coolify", "description": "An open-source & self-hostable Heroku / Netlify alternative.", - "version": "2.0.32", + "version": "2.1.0", "license": "AGPL-3.0", "scripts": { "dev": "docker-compose -f docker-compose-dev.yaml up -d && NODE_ENV=development svelte-kit dev", From 927bf46304958a0c4ad5f60d505bb8eef8a67c94 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Tue, 22 Mar 2022 10:37:33 +0100 Subject: [PATCH 07/12] fix: skip ssl cert in case of error --- src/lib/letsencrypt/index.ts | 86 ++++++++++++++++++++---------------- 1 file changed, 47 insertions(+), 39 deletions(-) diff --git a/src/lib/letsencrypt/index.ts b/src/lib/letsencrypt/index.ts index ff189845f..e85f539de 100644 --- a/src/lib/letsencrypt/index.ts +++ b/src/lib/letsencrypt/index.ts @@ -103,34 +103,38 @@ export async function generateSSLCerts() { orderBy: { createdAt: 'desc' } }); for (const application of applications) { - const { - fqdn, - id, - destinationDocker: { engine, network }, - settings: { previews } - } = application; - const isRunning = await checkContainer(engine, id); - const domain = getDomain(fqdn); - const isHttps = fqdn.startsWith('https://'); - if (isRunning) { - if (isHttps) ssls.push({ domain, id, isCoolify: false }); - } - if (previews) { - const host = getEngine(engine); - const { stdout } = await asyncExecShell( - `DOCKER_HOST=${host} docker container ls --filter="status=running" --filter="network=${network}" --filter="name=${id}-" --format="{{json .Names}}"` - ); - const containers = stdout - .trim() - .split('\n') - .filter((a) => a) - .map((c) => c.replace(/"/g, '')); - if (containers.length > 0) { - for (const container of containers) { - let previewDomain = `${container.split('-')[1]}.${domain}`; - if (isHttps) ssls.push({ domain: previewDomain, id, isCoolify: false }); + try { + const { + fqdn, + id, + destinationDocker: { engine, network }, + settings: { previews } + } = application; + const isRunning = await checkContainer(engine, id); + const domain = getDomain(fqdn); + const isHttps = fqdn.startsWith('https://'); + if (isRunning) { + if (isHttps) ssls.push({ domain, id, isCoolify: false }); + } + if (previews) { + const host = getEngine(engine); + const { stdout } = await asyncExecShell( + `DOCKER_HOST=${host} docker container ls --filter="status=running" --filter="network=${network}" --filter="name=${id}-" --format="{{json .Names}}"` + ); + const containers = stdout + .trim() + .split('\n') + .filter((a) => a) + .map((c) => c.replace(/"/g, '')); + if (containers.length > 0) { + for (const container of containers) { + let previewDomain = `${container.split('-')[1]}.${domain}`; + if (isHttps) ssls.push({ domain: previewDomain, id, isCoolify: false }); + } } } + } catch (error) { + console.log(`Error during generateSSLCerts with ${application.fqdn}: ${error}`); } } const services = await db.prisma.service.findMany({ @@ -145,20 +149,24 @@ export async function generateSSLCerts() { }); for (const service of services) { - const { - fqdn, - id, - type, - destinationDocker: { engine } - } = service; - const found = db.supportedServiceTypesAndVersions.find((a) => a.name === type); - if (found) { - const domain = getDomain(fqdn); - const isHttps = fqdn.startsWith('https://'); - const isRunning = await checkContainer(engine, id); - if (isRunning) { - if (isHttps) ssls.push({ domain, id, isCoolify: false }); + try { + const { + fqdn, + id, + type, + destinationDocker: { engine } + } = service; + const found = db.supportedServiceTypesAndVersions.find((a) => a.name === type); + if (found) { + const domain = getDomain(fqdn); + const isHttps = fqdn.startsWith('https://'); + const isRunning = await checkContainer(engine, id); + if (isRunning) { + if (isHttps) ssls.push({ domain, id, isCoolify: false }); + } } + } catch (error) { + console.log(`Error during generateSSLCerts with ${service.fqdn}: ${error}`); } } const { fqdn } = await db.prisma.setting.findFirst(); From 58a11e37fe18d10d94bd65b255dd0717e0eb87c0 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Tue, 22 Mar 2022 14:58:08 +0100 Subject: [PATCH 08/12] Add schema --- .../migration.sql | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 prisma/migrations/20220322135800_persistent_storage/migration.sql diff --git a/prisma/migrations/20220322135800_persistent_storage/migration.sql b/prisma/migrations/20220322135800_persistent_storage/migration.sql new file mode 100644 index 000000000..d26ae3f8b --- /dev/null +++ b/prisma/migrations/20220322135800_persistent_storage/migration.sql @@ -0,0 +1,18 @@ +-- CreateTable +CREATE TABLE "ApplicationPersistentStorage" ( + "id" TEXT NOT NULL PRIMARY KEY, + "applicationId" TEXT NOT NULL, + "path" TEXT NOT NULL, + "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" DATETIME NOT NULL, + CONSTRAINT "ApplicationPersistentStorage_applicationId_fkey" FOREIGN KEY ("applicationId") REFERENCES "Application" ("id") ON DELETE RESTRICT ON UPDATE CASCADE +); + +-- CreateIndex +CREATE UNIQUE INDEX "ApplicationPersistentStorage_applicationId_key" ON "ApplicationPersistentStorage"("applicationId"); + +-- CreateIndex +CREATE UNIQUE INDEX "ApplicationPersistentStorage_path_key" ON "ApplicationPersistentStorage"("path"); + +-- CreateIndex +CREATE UNIQUE INDEX "ApplicationPersistentStorage_applicationId_path_key" ON "ApplicationPersistentStorage"("applicationId", "path"); From f0ab3750bd474d0a5394a89effc64864d9ded025 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Tue, 22 Mar 2022 15:56:03 +0100 Subject: [PATCH 09/12] Disable PHP modules, as the new image has all activated by default --- src/routes/applications/[id]/index.svelte | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/routes/applications/[id]/index.svelte b/src/routes/applications/[id]/index.svelte index 221aefddc..bcadb5f5b 100644 --- a/src/routes/applications/[id]/index.svelte +++ b/src/routes/applications/[id]/index.svelte @@ -481,7 +481,7 @@ /> {/if} - {#if application.buildPack === 'php'} +
{/if} -