diff --git a/apps/api/scripts/convert.mjs b/apps/api/scripts/convert.mjs new file mode 100644 index 000000000..f56e2f627 --- /dev/null +++ b/apps/api/scripts/convert.mjs @@ -0,0 +1,95 @@ +import fs from 'fs/promises'; +import yaml from 'js-yaml'; +const templateYml = await fs.readFile('./caprover.yml', 'utf8') +const template = yaml.load(templateYml) + +const newTemplate = { + "templateVersion": "1.0.0", + "serviceDefaultVersion": "latest", + "name": "", + "displayName": "", + "description": "", + "services": { + + }, + "variables": [] +} +const version = template.caproverOneClickApp.variables.find(v => v.id === '$$cap_APP_VERSION').defaultValue || 'latest' + +newTemplate.displayName = template.caproverOneClickApp.displayName +newTemplate.name = template.caproverOneClickApp.displayName.toLowerCase() +newTemplate.documentation = template.caproverOneClickApp.documentation +newTemplate.description = template.caproverOneClickApp.description +newTemplate.serviceDefaultVersion = version + +const varSet = new Set() +const caproverVariables = template.caproverOneClickApp.variables +for (const service of Object.keys(template.services)) { + const serviceTemplate = template.services[service] + const newServiceName = service.replaceAll('cap_appname', 'id') + const newService = { + image: '', + command: '', + environment: [], + volumes: [] + } + const FROM = serviceTemplate.caproverExtra?.dockerfileLines?.find((line) => line.startsWith('FROM')) + if (serviceTemplate.image) { + newService.image = serviceTemplate.image.replaceAll('cap_APP_VERSION', 'core_version') + } else if (FROM) { + newService.image = FROM.split(' ')[1].replaceAll('cap_APP_VERSION', 'core_version') + } + + const CMD = serviceTemplate.caproverExtra?.dockerfileLines?.find((line) => line.startsWith('CMD')) + if (serviceTemplate.command) { + newService.command = serviceTemplate.command + } else if (CMD) { + newService.command = CMD.replace('CMD ', '').replaceAll('"', '').replaceAll('[', '').replaceAll(']', '').replaceAll(',', ' ').replace(/\s+/g, ' ') + } else { + delete newService.command + } + const ENTRYPOINT = serviceTemplate.caproverExtra?.dockerfileLines?.find((line) => line.startsWith('ENTRYPOINT')) + + if (serviceTemplate.entrypoint) { + newService.command = serviceTemplate.entrypoint + + } else if (ENTRYPOINT) { + newService.entrypoint = ENTRYPOINT.replace('ENTRYPOINT ', '').replaceAll('"', '').replaceAll('[', '').replaceAll(']', '').replaceAll(',', ' ').replace(/\s+/g, ' ') + } else { + delete newService.entrypoint + } + + if (serviceTemplate.environment && Object.keys(serviceTemplate.environment).length > 0) { + for (const env of Object.keys(serviceTemplate.environment)) { + if (serviceTemplate.environment[env].startsWith('srv-captain--$$cap_appname')) { + continue; + } + const value = '$$config_' + serviceTemplate.environment[env].replaceAll('srv-captain--$$cap_appname', '$$$id').replace('$$cap', '').replaceAll('captain-overlay-network', `$$$config_${env}`).toLowerCase() + newService.environment.push(`${env}=${value}`) + const foundVariable = varSet.has(env) + if (!foundVariable) { + const foundCaproverVariable = caproverVariables.find((item) => item.id === serviceTemplate.environment[env]) + const defaultValue = foundCaproverVariable?.defaultValue ? foundCaproverVariable?.defaultValue.toString()?.replace('$$cap_gen_random_hex', '$$$generate_hex') : '' + if (defaultValue && defaultValue !== foundCaproverVariable?.defaultValue) { + console.log('changed') + } + newTemplate.variables.push({ + "id": value, + "name": env, + "label": foundCaproverVariable?.label || '', + "defaultValue": defaultValue, + "description": foundCaproverVariable?.description || '', + }) + } + varSet.add(env) + } + } + if (serviceTemplate.volumes && serviceTemplate.volumes.length > 0) { + for (const volume of serviceTemplate.volumes) { + const [source, target] = volume.split(':') + newService.volumes.push(`${source.replaceAll('$$cap_appname-', '$$$id-')}:${target}`) + } + } + newTemplate.services[newServiceName] = newService +} +await fs.writeFile('./caprover_new.yml', yaml.dump([{ ...newTemplate }])) \ No newline at end of file diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index 3322987fe..5f248a5f4 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -11,6 +11,7 @@ import { scheduler } from './lib/scheduler'; import { compareVersions } from 'compare-versions'; import Graceful from '@ladjs/graceful' import axios from 'axios'; +import yaml from 'js-yaml' import fs from 'fs/promises'; import { verifyRemoteDockerEngineFn } from './routes/api/v1/destinations/handlers'; import { checkContainer } from './lib/docker'; @@ -123,7 +124,14 @@ const host = '0.0.0.0'; } }) try { - await migrateServicesToNewTemplate() + const templateYaml = await axios.get('https://gist.githubusercontent.com/andrasbacsai/701c450ef4272a929215cab11d737e3d/raw/4f021329d22934b90c5d67a0e49839a32bd629fd/template.yaml') + const templateJson = yaml.load(templateYaml.data) + if (isDev) { + await fs.writeFile('./template.json', JSON.stringify(templateJson, null, 2)) + } else { + await fs.writeFile('/app/template.json', JSON.stringify(templateJson, null, 2)) + } + await migrateServicesToNewTemplate(templateJson) await fastify.listen({ port, host }) console.log(`Coolify's API is listening on ${host}:${port}`); diff --git a/apps/api/src/lib.ts b/apps/api/src/lib.ts index 2cd1303df..b490845ad 100644 --- a/apps/api/src/lib.ts +++ b/apps/api/src/lib.ts @@ -1,8 +1,7 @@ import { decrypt, encrypt, getDomain, prisma } from "./lib/common"; import { includeServices } from "./lib/services/common"; -import templates from "./lib/templates"; -export async function migrateServicesToNewTemplate() { +export async function migrateServicesToNewTemplate(templates: any) { // This function migrates old hardcoded services to the new template based services try { const services = await prisma.service.findMany({ include: includeServices }) @@ -10,20 +9,25 @@ export async function migrateServicesToNewTemplate() { if (!service.type) { continue; } - if (service.type === 'plausibleanalytics' && service.plausibleAnalytics) await plausibleAnalytics(service) - if (service.type === 'fider' && service.fider) await fider(service) - if (service.type === 'minio' && service.minio) await minio(service) - if (service.type === 'vscodeserver' && service.vscodeserver) await vscodeserver(service) - if (service.type === 'wordpress' && service.wordpress) await wordpress(service) - if (service.type === 'ghost' && service.ghost) await ghost(service) - if (service.type === 'meilisearch' && service.meiliSearch) await meilisearch(service) - if (service.type === 'umami' && service.umami) await umami(service) - if (service.type === 'hasura' && service.hasura) await hasura(service) - if (service.type === 'glitchTip' && service.glitchTip) await glitchtip(service) - if (service.type === 'searxng' && service.searxng) await searxng(service) - if (service.type === 'weblate' && service.weblate) await weblate(service) + let template = templates.find(t => t.name === service.type.toLowerCase()); + if (template) { + template = JSON.parse(JSON.stringify(template).replaceAll('$$id', service.id)) + if (service.type === 'plausibleanalytics' && service.plausibleAnalytics) await plausibleAnalytics(service) + if (service.type === 'fider' && service.fider) await fider(service) + if (service.type === 'minio' && service.minio) await minio(service) + if (service.type === 'vscodeserver' && service.vscodeserver) await vscodeserver(service) + if (service.type === 'wordpress' && service.wordpress) await wordpress(service) + if (service.type === 'ghost' && service.ghost) await ghost(service) + if (service.type === 'meilisearch' && service.meiliSearch) await meilisearch(service) + if (service.type === 'umami' && service.umami) await umami(service) + if (service.type === 'hasura' && service.hasura) await hasura(service) + if (service.type === 'glitchTip' && service.glitchTip) await glitchtip(service) + if (service.type === 'searxng' && service.searxng) await searxng(service) + if (service.type === 'weblate' && service.weblate) await weblate(service) + + await createVolumes(service, template); + } - await createVolumes(service); } } catch (error) { console.log(error) @@ -321,19 +325,15 @@ async function migrateSecrets(secrets: any[], service: any) { await prisma.serviceSecret.findFirst({ where: { name, serviceId: service.id } }) || await prisma.serviceSecret.create({ data: { name, value, service: { connect: { id: service.id } } } }) } } -async function createVolumes(service: any) { +async function createVolumes(service: any, template: any) { const volumes = []; - let template = templates.find(t => t.name === service.type.toLowerCase()); - if (template) { - template = JSON.parse(JSON.stringify(template).replaceAll('$$id', service.id)) - for (const s of Object.keys(template.services)) { - if (template.services[s].volumes && template.services[s].volumes.length > 0) { - for (const volume of template.services[s].volumes) { - const volumeName = volume.split(':')[0] - const volumePath = volume.split(':')[1] - const volumeService = service.id - volumes.push(`${volumeName}@@@${volumePath}@@@${volumeService}`) - } + for (const s of Object.keys(template.services)) { + if (template.services[s].volumes && template.services[s].volumes.length > 0) { + for (const volume of template.services[s].volumes) { + const volumeName = volume.split(':')[0] + const volumePath = volume.split(':')[1] + const volumeService = service.id + volumes.push(`${volumeName}@@@${volumePath}@@@${volumeService}`) } } } diff --git a/apps/api/src/lib/services.ts b/apps/api/src/lib/services.ts index d7f0fd75e..499968a4a 100644 --- a/apps/api/src/lib/services.ts +++ b/apps/api/src/lib/services.ts @@ -1,5 +1,13 @@ -import { createDirectories, getServiceFromDB, getServiceImage, getServiceMainPort, makeLabelForServices } from "./common"; +import { createDirectories, getServiceFromDB, getServiceImage, getServiceMainPort, isDev, makeLabelForServices } from "./common"; +import fs from 'fs/promises'; +export async function getTemplates() { + let templates = []; + if (isDev) { + templates = JSON.parse((await fs.readFile('./template.json')).toString()) + } + return templates +} export async function defaultServiceConfigurations({ id, teamId }) { const service = await getServiceFromDB({ id, teamId }); const { destinationDockerId, destinationDocker, type, serviceSecret } = service; diff --git a/apps/api/src/lib/services/handlers.ts b/apps/api/src/lib/services/handlers.ts index 1170930d3..34f7f08b4 100644 --- a/apps/api/src/lib/services/handlers.ts +++ b/apps/api/src/lib/services/handlers.ts @@ -6,7 +6,7 @@ import { ServiceStartStop } from '../../routes/api/v1/services/types'; import { asyncSleep, ComposeFile, createDirectories, defaultComposeConfiguration, errorHandler, executeDockerCmd, getDomain, getFreePublicPort, getServiceFromDB, getServiceImage, getServiceMainPort, isARM, isDev, makeLabelForServices, persistentVolumes, prisma } from '../common'; import { defaultServiceConfigurations } from '../services'; import { OnlyId } from '../../types'; -import templates from '../templates' + import { parseAndFindServiceTemplates } from '../../routes/api/v1/services/handlers'; import path from 'path'; // export async function startService(request: FastifyRequest) { @@ -711,6 +711,7 @@ export async function startService(request: FastifyRequest) { container_name: service, build: template.services[service].build || undefined, command: template.services[service].command, + entrypoint: template.services[service]?.entrypoint, image: template.services[service].image, expose: template.services[service].ports, // ...(exposePort ? { ports: [`${exposePort}:${port}`] } : {}), diff --git a/apps/api/src/lib/templates.ts b/apps/api/src/lib/templates.ts index 33d67c2f8..c194582d7 100644 --- a/apps/api/src/lib/templates.ts +++ b/apps/api/src/lib/templates.ts @@ -21,7 +21,7 @@ export default [ `WEBLATE_ADMIN_PASSWORD=$$secret_weblate_admin_password`, `POSTGRES_PASSWORD=$$secret_postgres_password`, `POSTGRES_USER=$$config_postgres_user`, - `POSTGRES_DATABASE=$$config_postgres_database`, + `POSTGRES_DATABASE=$$config_postgres_db`, `POSTGRES_HOST=$$id-postgresql`, `POSTGRES_PORT=5432`, `REDIS_HOST=$$id-redis`, @@ -94,13 +94,7 @@ export default [ "defaultValue": "weblate", "description": "", }, - { - "id": "$$config_postgres_database", - "name": "POSTGRES_DATABASE", - "label": "PostgreSQL Database", - "defaultValue": "$$config_postgres_db", - "description": "" - }, + ] }, { @@ -121,7 +115,6 @@ export default [ ], "environment": [ "SEARXNG_BASE_URL=$$config_searxng_base_url", - "SECRET_KEY=$$secret_secret_key", ], "ports": [ "8080" @@ -1462,9 +1455,7 @@ export default [ "label": "Secret Key Base", "defaultValue": "$$generate_passphrase", "description": "", - "extras": { - "length": 64 - } + }, { "id": "$$config_disable_auth", diff --git a/apps/api/src/routes/api/v1/services/handlers.ts b/apps/api/src/routes/api/v1/services/handlers.ts index 53a9a6df5..c17bc25a5 100644 --- a/apps/api/src/routes/api/v1/services/handlers.ts +++ b/apps/api/src/routes/api/v1/services/handlers.ts @@ -2,6 +2,7 @@ import type { FastifyReply, FastifyRequest } from 'fastify'; import fs from 'fs/promises'; import yaml from 'js-yaml'; import bcrypt from 'bcryptjs'; +import crypto from 'crypto'; import { prisma, uniqueName, asyncExecShell, getServiceFromDB, getContainerUsage, isDomainConfigured, saveUpdateableFields, fixType, decrypt, encrypt, ComposeFile, getFreePublicPort, getDomain, errorHandler, generatePassword, isDev, stopTcpHttpProxy, executeDockerCmd, checkDomainsIsValidInDNS, checkExposedPort, listSettings } from '../../../../lib/common'; import { day } from '../../../../lib/dayjs'; import { checkContainer, isContainerExited } from '../../../../lib/docker'; @@ -12,7 +13,7 @@ import type { ActivateWordpressFtp, CheckService, CheckServiceDomain, DeleteServ import { supportedServiceTypesAndVersions } from '../../../../lib/services/supportedVersions'; import { configureServiceType, removeService } from '../../../../lib/services/common'; import { hashPassword } from '../handlers'; -import templates from '../../../../lib/templates'; +import { getTemplates } from '../../../../lib/services'; export async function listServices(request: FastifyRequest) { try { @@ -113,6 +114,7 @@ export async function getServiceStatus(request: FastifyRequest) { } } export async function parseAndFindServiceTemplates(service: any, workdir?: string, isDeploy: boolean = false) { + const templates = await getTemplates() const foundTemplate = templates.find(t => t.name === service.type.toLowerCase()) let parsedTemplate = {} if (foundTemplate) { @@ -162,7 +164,6 @@ export async function parseAndFindServiceTemplates(service: any, workdir?: strin parsedTemplate = JSON.parse(JSON.stringify(parsedTemplate).replaceAll(regex, getDomain(service.fqdn) + "\"")) } else { parsedTemplate = JSON.parse(JSON.stringify(parsedTemplate).replaceAll(regex, value + "\"")) - } } @@ -203,7 +204,7 @@ export async function getService(request: FastifyRequest) { export async function getServiceType(request: FastifyRequest) { try { return { - services: templates + services: await getTemplates() } } catch ({ status, message }) { return errorHandler({ status, message }) @@ -213,6 +214,7 @@ export async function saveServiceType(request: FastifyRequest, try { const { id } = request.params; const { type } = request.body; + const templates = await getTemplates() let foundTemplate = templates.find(t => t.name === type) if (foundTemplate) { let generatedVariables = new Set() @@ -223,22 +225,32 @@ export async function saveServiceType(request: FastifyRequest, if (foundTemplate.variables.length > 0) { foundTemplate.variables = foundTemplate.variables.map(variable => { let { id: variableId } = variable; - console.log(variableId) if (variableId.startsWith('$$secret_')) { - const length = variable?.extras && variable.extras['length'] - if (variable.defaultValue === '$$generate_password') { - variable.value = generatePassword({ length }); - } else if (variable.defaultValue === '$$generate_passphrase') { - variable.value = generatePassword({ length }); + if (variable.defaultValue.startsWith('$$generate_password')) { + const length = variable.defaultValue.replace('$$generate_password(', '').replace('\)', '') || 16 + variable.value = generatePassword({ length: Number(length) }); + } else if (variable.defaultValue.startsWith('$$generate_hex')) { + const length = variable.defaultValue.replace('$$generate_hex(', '').replace('\)', '') || 16 + variable.value = crypto.randomBytes(Number(length)).toString('hex'); } else if (!variable.defaultValue) { variable.defaultValue = undefined } } if (variableId.startsWith('$$config_')) { - if (variable.defaultValue === '$$generate_username') { - variable.value = cuid(); + if (variable.defaultValue.startsWith('$$generate_username')) { + const length = variable.defaultValue.replace('$$generate_username(', '').replace('\)', '') + if (length !== '$$generate_username') { + variable.value = crypto.randomBytes(Number(length)).toString('hex'); + } else { + variable.value = cuid(); + } } else { - variable.value = variable.defaultValue || '' + if (variable.defaultValue.startsWith('$$generate_hex')) { + const length = variable.defaultValue.replace('$$generate_hex(', '').replace('\)', '') || 16 + variable.value = crypto.randomBytes(Number(length)).toString('hex'); + } else { + variable.value = variable.defaultValue || '' + } } } if (variable.value) { @@ -268,17 +280,24 @@ export async function saveServiceType(request: FastifyRequest, if (!variable.value) { continue; } - await prisma.serviceSecret.create({ - data: { name: variable.name, value: encrypt(variable.value), service: { connect: { id } } } - }) + const found = await prisma.serviceSecret.findFirst({ where: { name: variable.name, serviceId: id } }) + if (!found) { + await prisma.serviceSecret.create({ + data: { name: variable.name, value: encrypt(variable.value), service: { connect: { id } } } + }) + } + } if (variable.id.startsWith('$$config_')) { if (!variable.value) { variable.value = ''; } - await prisma.serviceSetting.create({ - data: { name: variable.name, value: variable.value, service: { connect: { id } } } - }) + const found = await prisma.serviceSetting.findFirst({ where: { name: variable.name, serviceId: id } }) + if (!found) { + await prisma.serviceSetting.create({ + data: { name: variable.name, value: variable.value.toString(), service: { connect: { id } } } + }) + } } } } @@ -287,9 +306,12 @@ export async function saveServiceType(request: FastifyRequest, for (const volume of foundTemplate.services[service].volumes) { const [volumeName, path] = volume.split(':') if (!volumeName.startsWith('/')) { - await prisma.servicePersistentStorage.create({ - data: { volumeName, path, containerId: service, predefined: true, service: { connect: { id } } } - }); + const found = await prisma.servicePersistentStorage.findFirst({ where: { volumeName, serviceId: id } }) + if (!found) { + await prisma.servicePersistentStorage.create({ + data: { volumeName, path, containerId: service, predefined: true, service: { connect: { id } } } + }); + } } } } diff --git a/apps/ui/src/routes/services/[id]/index.svelte b/apps/ui/src/routes/services/[id]/index.svelte index f944eab51..b885d7e61 100644 --- a/apps/ui/src/routes/services/[id]/index.svelte +++ b/apps/ui/src/routes/services/[id]/index.svelte @@ -381,7 +381,8 @@ class:border-b={template[oneService].environment.length > 0} class:border-coolgray-500={template[oneService].environment.length > 0} > -
{template[oneService].name}
+
{template[oneService].name || oneService.replace(`${id}-`,'').replace(id,service.type)}
+
diff --git a/apps/ui/src/routes/services/[id]/logs/index.svelte b/apps/ui/src/routes/services/[id]/logs/index.svelte index 39e09ead7..89c226fbf 100644 --- a/apps/ui/src/routes/services/[id]/logs/index.svelte +++ b/apps/ui/src/routes/services/[id]/logs/index.svelte @@ -93,7 +93,7 @@
Service Logs
-
+
{#if template} {#each Object.keys(template) as service}