From 4e7e9b2cfc6ba88075347c6165aeece1e84f3ccb Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Thu, 18 Aug 2022 15:29:59 +0200 Subject: [PATCH] feat: public repo deployment --- apps/api/src/jobs/deployApplication.ts | 64 ++++++++-------- apps/api/src/lib/common.ts | 21 ++++++ apps/api/src/lib/importers/github.ts | 74 +++++++++++-------- .../routes/api/v1/applications/handlers.ts | 20 ++--- .../routes/api/v1/destinations/handlers.ts | 1 - .../src/routes/api/v1/services/handlers.ts | 15 +--- .../src/routes/webhooks/traefik/handlers.ts | 1 - apps/ui/src/lib/locales/en.json | 2 +- .../configuration/_PublicRepository.svelte | 11 +-- .../[id]/configuration/buildpack.svelte | 35 +++++---- .../src/routes/applications/[id]/index.svelte | 36 +++++---- .../routes/applications/[id]/storages.svelte | 7 +- 12 files changed, 159 insertions(+), 128 deletions(-) diff --git a/apps/api/src/jobs/deployApplication.ts b/apps/api/src/jobs/deployApplication.ts index 01ecf5374..4db0d5e26 100644 --- a/apps/api/src/jobs/deployApplication.ts +++ b/apps/api/src/jobs/deployApplication.ts @@ -69,6 +69,30 @@ import * as buildpacks from '../lib/buildPacks'; dockerFileLocation, denoMainFile } = message + const currentHash = crypto + .createHash('sha256') + .update( + JSON.stringify({ + pythonWSGI, + pythonModule, + pythonVariable, + deploymentType, + denoOptions, + baseImage, + baseBuildImage, + buildPack, + port, + exposePort, + installCommand, + buildCommand, + startCommand, + secrets, + branch, + repository, + fqdn + }) + ) + .digest('hex'); try { const { debug } = settings; if (concurrency === 1) { @@ -131,7 +155,8 @@ import * as buildpacks from '../lib/buildPacks'; htmlUrl: gitSource.htmlUrl, projectId, deployKeyId: gitSource.gitlabApp?.deployKeyId || null, - privateSshKey: decrypt(gitSource.gitlabApp?.privateSshKey) || null + privateSshKey: decrypt(gitSource.gitlabApp?.privateSshKey) || null, + forPublic: gitSource.forPublic }); if (!commit) { throw new Error('No commit found?'); @@ -146,38 +171,11 @@ import * as buildpacks from '../lib/buildPacks'; } catch (err) { console.log(err); } + if (!pullmergeRequestId) { - const currentHash = crypto - //@ts-ignore - .createHash('sha256') - .update( - JSON.stringify({ - pythonWSGI, - pythonModule, - pythonVariable, - deploymentType, - denoOptions, - baseImage, - baseBuildImage, - buildPack, - port, - exposePort, - installCommand, - buildCommand, - startCommand, - secrets, - branch, - repository, - fqdn - }) - ) - .digest('hex'); + if (configHash !== currentHash) { - await prisma.application.update({ - where: { id: applicationId }, - data: { configHash: currentHash } - }); deployNeeded = true; if (configHash) { await saveBuildLog({ line: 'Configuration changed.', buildId, applicationId }); @@ -201,7 +199,7 @@ import * as buildpacks from '../lib/buildPacks'; } await copyBaseConfigurationFiles(buildPack, workdir, buildId, applicationId, baseImage); if (!imageFound || deployNeeded) { - // if (true) { + // if (true) { if (buildpacks[buildPack]) await buildpacks[buildPack]({ dockerId: destinationDocker.id, @@ -336,6 +334,10 @@ import * as buildpacks from '../lib/buildPacks'; } await saveBuildLog({ line: 'Proxy will be updated shortly.', buildId, applicationId }); await prisma.build.update({ where: { id: message.build_id }, data: { status: 'success' } }); + if (!pullmergeRequestId) await prisma.application.update({ + where: { id: applicationId }, + data: { configHash: currentHash } + }); } } diff --git a/apps/api/src/lib/common.ts b/apps/api/src/lib/common.ts index f3c399eb5..78a23eb2b 100644 --- a/apps/api/src/lib/common.ts +++ b/apps/api/src/lib/common.ts @@ -1176,6 +1176,27 @@ export async function updatePasswordInDb(database, user, newPassword, isRoot) { } } } +export async function checkExposedPort({ id, configuredPort, exposePort, dockerId, remoteIpAddress }: { id: string, configuredPort?: number, exposePort: number, dockerId: string, remoteIpAddress?: string }) { + if (exposePort < 1024 || exposePort > 65535) { + throw { status: 500, message: `Exposed Port needs to be between 1024 and 65535.` } + } + + if (configuredPort) { + if (configuredPort !== exposePort) { + const availablePort = await getFreeExposedPort(id, exposePort, dockerId, remoteIpAddress); + if (availablePort.toString() !== exposePort.toString()) { + throw { status: 500, message: `Port ${exposePort} is already in use.` } + } + } + } else { + + const availablePort = await getFreeExposedPort(id, exposePort, dockerId, remoteIpAddress); +console.log(availablePort, exposePort) + if (availablePort.toString() !== exposePort.toString()) { + throw { status: 500, message: `Port ${exposePort} is already in use.` } + } + } +} export async function getFreeExposedPort(id, exposePort, dockerId, remoteIpAddress) { const { default: getPort } = await import('get-port'); const applicationUsed = await ( diff --git a/apps/api/src/lib/importers/github.ts b/apps/api/src/lib/importers/github.ts index afe1483a3..798931d7a 100644 --- a/apps/api/src/lib/importers/github.ts +++ b/apps/api/src/lib/importers/github.ts @@ -12,7 +12,8 @@ export default async function ({ htmlUrl, branch, buildId, - customPort + customPort, + forPublic }: { applicationId: string; workdir: string; @@ -23,40 +24,55 @@ export default async function ({ branch: string; buildId: string; customPort: number; + forPublic?: boolean; }): Promise { const { default: got } = await import('got') const url = htmlUrl.replace('https://', '').replace('http://', ''); await saveBuildLog({ line: 'GitHub importer started.', buildId, applicationId }); + if (forPublic) { + await saveBuildLog({ + line: `Cloning ${repository}:${branch} branch.`, + buildId, + applicationId + }); + await asyncExecShell( + `git clone -q -b ${branch} https://${url}/${repository}.git ${workdir}/ && cd ${workdir} && git submodule update --init --recursive && git lfs pull && cd .. ` + ); - const body = await prisma.githubApp.findUnique({ where: { id: githubAppId } }); - if (body.privateKey) body.privateKey = decrypt(body.privateKey); - const { privateKey, appId, installationId } = body - const githubPrivateKey = privateKey.replace(/\\n/g, '\n').replace(/"/g, ''); + } else { + const body = await prisma.githubApp.findUnique({ where: { id: githubAppId } }); + if (body.privateKey) body.privateKey = decrypt(body.privateKey); + const { privateKey, appId, installationId } = body + const githubPrivateKey = privateKey.replace(/\\n/g, '\n').replace(/"/g, ''); - const payload = { - iat: Math.round(new Date().getTime() / 1000), - exp: Math.round(new Date().getTime() / 1000 + 60), - iss: appId - }; - const jwtToken = jsonwebtoken.sign(payload, githubPrivateKey, { - algorithm: 'RS256' - }); - const { token } = await got - .post(`${apiUrl}/app/installations/${installationId}/access_tokens`, { - headers: { - Authorization: `Bearer ${jwtToken}`, - Accept: 'application/vnd.github.machine-man-preview+json' - } - }) - .json(); - await saveBuildLog({ - line: `Cloning ${repository}:${branch} branch.`, - buildId, - applicationId - }); - await asyncExecShell( - `git clone -q -b ${branch} https://x-access-token:${token}@${url}/${repository}.git --config core.sshCommand="ssh -p ${customPort}" ${workdir}/ && cd ${workdir} && git submodule update --init --recursive && git lfs pull && cd .. ` - ); + const payload = { + iat: Math.round(new Date().getTime() / 1000), + exp: Math.round(new Date().getTime() / 1000 + 60), + iss: appId + }; + const jwtToken = jsonwebtoken.sign(payload, githubPrivateKey, { + algorithm: 'RS256' + }); + const { token } = await got + .post(`${apiUrl}/app/installations/${installationId}/access_tokens`, { + headers: { + Authorization: `Bearer ${jwtToken}`, + Accept: 'application/vnd.github.machine-man-preview+json' + } + }) + .json(); + await saveBuildLog({ + line: `Cloning ${repository}:${branch} branch.`, + buildId, + applicationId + }); + await asyncExecShell( + `git clone -q -b ${branch} https://x-access-token:${token}@${url}/${repository}.git --config core.sshCommand="ssh -p ${customPort}" ${workdir}/ && cd ${workdir} && git submodule update --init --recursive && git lfs pull && cd .. ` + ); + } const { stdout: commit } = await asyncExecShell(`cd ${workdir}/ && git rev-parse HEAD`); + return commit.replace('\n', ''); + + } diff --git a/apps/api/src/routes/api/v1/applications/handlers.ts b/apps/api/src/routes/api/v1/applications/handlers.ts index 0e2596f79..7ddeb6b1c 100644 --- a/apps/api/src/routes/api/v1/applications/handlers.ts +++ b/apps/api/src/routes/api/v1/applications/handlers.ts @@ -5,7 +5,7 @@ import axios from 'axios'; import { FastifyReply } from 'fastify'; import { day } from '../../../../lib/dayjs'; import { setDefaultBaseImage, setDefaultConfiguration } from '../../../../lib/buildPacks/common'; -import { checkDomainsIsValidInDNS, checkDoubleBranch, decrypt, encrypt, errorHandler, executeDockerCmd, generateSshKeyPair, getContainerUsage, getDomain, getFreeExposedPort, isDev, isDomainConfigured, listSettings, prisma, stopBuild, uniqueName } from '../../../../lib/common'; +import { checkDomainsIsValidInDNS, checkDoubleBranch, checkExposedPort, decrypt, encrypt, errorHandler, executeDockerCmd, generateSshKeyPair, getContainerUsage, getDomain, getFreeExposedPort, isDev, isDomainConfigured, listSettings, prisma, stopBuild, uniqueName } from '../../../../lib/common'; import { checkContainer, formatLabelsOnDocker, isContainerExited, removeContainer } from '../../../../lib/docker'; import { scheduler } from '../../../../lib/scheduler'; @@ -238,6 +238,9 @@ export async function saveApplication(request: FastifyRequest, if (exposePort) { exposePort = Number(exposePort); } + + const { destinationDockerId } = await prisma.application.findUnique({ where: { id } }) + if (exposePort) await checkExposedPort({ id, exposePort, dockerId: destinationDockerId }) if (denoOptions) denoOptions = denoOptions.trim(); const defaultConfiguration = await setDefaultConfiguration({ buildPack, @@ -392,18 +395,7 @@ export async function checkDNS(request: FastifyRequest) { if (found) { throw { status: 500, message: `Domain ${getDomain(fqdn).replace('www.', '')} is already in use!` } } - if (exposePort) { - if (exposePort < 1024 || exposePort > 65535) { - throw { status: 500, message: `Exposed Port needs to be between 1024 and 65535.` } - } - - if (configuredPort !== exposePort) { - const availablePort = await getFreeExposedPort(id, exposePort, dockerId, remoteIpAddress); - if (availablePort.toString() !== exposePort.toString()) { - throw { status: 500, message: `Port ${exposePort} is already in use.` } - } - } - } + await checkExposedPort({ id, configuredPort, exposePort, dockerId, remoteIpAddress }) if (isDNSCheckEnabled && !isDev && !forceSave) { let hostname = request.hostname.split(':')[0]; if (remoteEngine) hostname = remoteIpAddress; @@ -500,7 +492,6 @@ export async function saveApplicationSource(request: FastifyRequest, reply: Fas throw { status: 500, message: `Secret ${name} already exists.` } } else { value = encrypt(value.trim()); - console.log({ value }) await prisma.secret.create({ data: { name, value, isBuildSecret, isPRMRSecret, application: { connect: { id } } } }); diff --git a/apps/api/src/routes/api/v1/destinations/handlers.ts b/apps/api/src/routes/api/v1/destinations/handlers.ts index f310f8a61..2148a1450 100644 --- a/apps/api/src/routes/api/v1/destinations/handlers.ts +++ b/apps/api/src/routes/api/v1/destinations/handlers.ts @@ -79,7 +79,6 @@ export async function newDestination(request: FastifyRequest, re let { name, network, engine, isCoolifyProxyUsed, remoteIpAddress, remoteUser, remotePort } = request.body if (id === 'new') { - console.log(engine) if (engine) { const { stdout } = await asyncExecShell(`DOCKER_HOST=unix:///var/run/docker.sock docker network ls --filter 'name=^${network}$' --format '{{json .}}'`); if (stdout === '') { diff --git a/apps/api/src/routes/api/v1/services/handlers.ts b/apps/api/src/routes/api/v1/services/handlers.ts index 5487054e4..bba1b26bd 100644 --- a/apps/api/src/routes/api/v1/services/handlers.ts +++ b/apps/api/src/routes/api/v1/services/handlers.ts @@ -2,7 +2,7 @@ import type { FastifyReply, FastifyRequest } from 'fastify'; import fs from 'fs/promises'; import yaml from 'js-yaml'; import bcrypt from 'bcryptjs'; -import { prisma, uniqueName, asyncExecShell, getServiceImage, configureServiceType, getServiceFromDB, getContainerUsage, removeService, isDomainConfigured, saveUpdateableFields, fixType, decrypt, encrypt, getServiceMainPort, createDirectories, ComposeFile, makeLabelForServices, getFreePublicPort, getDomain, errorHandler, generatePassword, isDev, stopTcpHttpProxy, supportedServiceTypesAndVersions, executeDockerCmd, listSettings, getFreeExposedPort, checkDomainsIsValidInDNS, persistentVolumes, asyncSleep, isARM, defaultComposeConfiguration } from '../../../../lib/common'; +import { prisma, uniqueName, asyncExecShell, getServiceImage, configureServiceType, getServiceFromDB, getContainerUsage, removeService, isDomainConfigured, saveUpdateableFields, fixType, decrypt, encrypt, getServiceMainPort, createDirectories, ComposeFile, makeLabelForServices, getFreePublicPort, getDomain, errorHandler, generatePassword, isDev, stopTcpHttpProxy, supportedServiceTypesAndVersions, executeDockerCmd, listSettings, getFreeExposedPort, checkDomainsIsValidInDNS, persistentVolumes, asyncSleep, isARM, defaultComposeConfiguration, checkExposedPort } from '../../../../lib/common'; import { day } from '../../../../lib/dayjs'; import { checkContainer, isContainerExited, removeContainer } from '../../../../lib/docker'; import cuid from 'cuid'; @@ -378,18 +378,7 @@ export async function checkService(request: FastifyRequest) { } } } - if (exposePort) { - if (exposePort < 1024 || exposePort > 65535) { - throw { status: 500, message: `Exposed Port needs to be between 1024 and 65535.` } - } - - if (configuredPort !== exposePort) { - const availablePort = await getFreeExposedPort(id, exposePort, dockerId, remoteIpAddress); - if (availablePort.toString() !== exposePort.toString()) { - throw { status: 500, message: `Port ${exposePort} is already in use.` } - } - } - } + await checkExposedPort({ id, configuredPort, exposePort, dockerId, remoteIpAddress }) if (isDNSCheckEnabled && !isDev && !forceSave) { let hostname = request.hostname.split(':')[0]; if (remoteEngine) hostname = remoteIpAddress; diff --git a/apps/api/src/routes/webhooks/traefik/handlers.ts b/apps/api/src/routes/webhooks/traefik/handlers.ts index de805ee71..c4012bc9d 100644 --- a/apps/api/src/routes/webhooks/traefik/handlers.ts +++ b/apps/api/src/routes/webhooks/traefik/handlers.ts @@ -484,7 +484,6 @@ export async function traefikOtherConfiguration(request: FastifyRequestThis is useful for storing data such as a database (SQLite) or a cache." + "persistent_storage_explainer": "You can specify any folder that you want to be persistent across deployments.
/example means it will preserve /app/example in the container as /app is the root directory for your application.

This is useful for storing data such as a database (SQLite) or a cache." }, "deployment_queued": "Deployment queued.", "confirm_to_delete": "Are you sure you would like to delete '{{name}}'?", diff --git a/apps/ui/src/routes/applications/[id]/configuration/_PublicRepository.svelte b/apps/ui/src/routes/applications/[id]/configuration/_PublicRepository.svelte index a2543e381..609a656f8 100644 --- a/apps/ui/src/routes/applications/[id]/configuration/_PublicRepository.svelte +++ b/apps/ui/src/routes/applications/[id]/configuration/_PublicRepository.svelte @@ -25,8 +25,10 @@ const gitUrl = publicRepositoryLink.replace('http://', '').replace('https://', ''); let [host, ...path] = gitUrl.split('/'); const [owner, repository, ...branch] = path; + ownerName = owner; repositoryName = repository; + if (branch[0] === 'tree') { branchName = branch[1]; await saveRepository(); @@ -42,20 +44,20 @@ } const apiUrl = `${protocol}://${host}`; - const repositoryDetails = await get(`${apiUrl}/repos/${owner}/${repository}`); + const repositoryDetails = await get(`${apiUrl}/repos/${ownerName}/${repositoryName}`); projectId = repositoryDetails.id.toString(); let branches: any[] = []; let page = 1; let branchCount = 0; loading.branches = true; - const loadedBranches = await loadBranchesByPage(apiUrl, owner, repository, page); + const loadedBranches = await loadBranchesByPage(apiUrl, ownerName, repositoryName, page); branches = branches.concat(loadedBranches); branchCount = branches.length; if (branchCount === 100) { while (branchCount === 100) { page = page + 1; - const nextBranches = await loadBranchesByPage(apiUrl, owner, repository, page); + const nextBranches = await loadBranchesByPage(apiUrl, ownerName, repositoryName, page); branches = branches.concat(nextBranches); branchCount = nextBranches.length; } @@ -68,7 +70,6 @@ } async function loadBranchesByPage(apiUrl: string, owner: string, repository: string, page = 1) { return await get(`${apiUrl}/repos/${owner}/${repository}/branches?per_page=100&page=${page}`); - // console.log(publicRepositoryLink); } async function saveRepository(event?: any) { try { @@ -81,7 +82,7 @@ type }); await post(`/applications/${id}/configuration/repository`, { - repository: `${repositoryName}/${branchName}`, + repository: `${ownerName}/${repositoryName}`, branch: branchName, projectId, autodeploy: false, diff --git a/apps/ui/src/routes/applications/[id]/configuration/buildpack.svelte b/apps/ui/src/routes/applications/[id]/configuration/buildpack.svelte index 28e32c3d7..9a44b1e03 100644 --- a/apps/ui/src/routes/applications/[id]/configuration/buildpack.svelte +++ b/apps/ui/src/routes/applications/[id]/configuration/buildpack.svelte @@ -48,7 +48,7 @@ export let type: any; export let application: any; export let isPublicRepository: boolean; - console.log(isPublicRepository) + function checkPackageJSONContents({ key, json }: { key: any; json: any }) { return json?.dependencies?.hasOwnProperty(key) || json?.devDependencies?.hasOwnProperty(key); } @@ -237,7 +237,7 @@ if (error.message === 'Bad credentials') { const { token } = await get(`/applications/${id}/configuration/githubToken`); $appSession.tokens.github = token; - return await scanRepository() + return await scanRepository(); } return errorNotification(error); } finally { @@ -246,7 +246,12 @@ } } onMount(async () => { - await scanRepository(); + if (!isPublicRepository) { + await scanRepository(); + } else { + foundConfig = findBuildPack('node', packageManager); + scanning = false; + } }); @@ -263,27 +268,25 @@ {:else} - -
Coolify Buildpacks
- {#each buildPacks.filter(bp => bp.isCoolifyBuildPack === true) as buildPack} -
- -
- {/each} + {#each buildPacks.filter((bp) => bp.isCoolifyBuildPack === true) as buildPack} +
+ +
+ {/each}
Heroku
- {#each buildPacks.filter(bp => bp.isHerokuBuildPack === true) as buildPack} -
- -
- {/each} -
+ {#each buildPacks.filter((bp) => bp.isHerokuBuildPack === true) as buildPack} +
+ +
+ {/each} +
{/if} diff --git a/apps/ui/src/routes/applications/[id]/index.svelte b/apps/ui/src/routes/applications/[id]/index.svelte index 29d495331..0096c044e 100644 --- a/apps/ui/src/routes/applications/[id]/index.svelte +++ b/apps/ui/src/routes/applications/[id]/index.svelte @@ -133,6 +133,7 @@ autodeploy = !autodeploy; } if (name === 'isBot') { + if ($status.application.isRunning) return; isBot = !isBot; application.settings.isBot = isBot; setLocation(application, settings); @@ -345,8 +346,11 @@ - {#if isDisabled} - + {#if isDisabled || application.settings.isPublicRepository} + {:else} {$t('application.git_repository')} - {#if isDisabled} - + {#if isDisabled || application.settings.isPublicRepository} + {:else} changeSettings('isBot')} title="Is your application a bot?" description="You can deploy applications without domains.
They will listen on IP:EXPOSEDPORT instead.

Useful to host Twitch bots." + disabled={$status.application.isRunning} /> {#if !isBot} @@ -770,15 +778,17 @@
{$t('application.features')}
-
- changeSettings('autodeploy')} - title={$t('application.enable_automatic_deployment')} - description={$t('application.enable_auto_deploy_webhooks')} - /> -
+ {#if !application.settings.isPublicRepository} +
+ changeSettings('autodeploy')} + title={$t('application.enable_automatic_deployment')} + description={$t('application.enable_auto_deploy_webhooks')} + /> +
+ {/if} {#if !application.settings.isBot}
-
- -
+ @@ -109,4 +107,7 @@
+
+ +