dave/coolify
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

fixes

Browse Source
This commit is contained in:
Andras Bacsai
2022-10-20 16:06:33 +02:00
parent 9f3732d35b
commit f4019db3d1
9 changed files with 191 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

95
apps/api/scripts/convert.mjs Normal file
View File

@@ -0,0 +1,95 @@
import fs from 'fs/promises';
import yaml from 'js-yaml';
const templateYml = await fs.readFile('./caprover.yml', 'utf8')
const template = yaml.load(templateYml)
const newTemplate = {
"templateVersion": "1.0.0",
"serviceDefaultVersion": "latest",
"name": "",
"displayName": "",
"description": "",
"services": {
},
"variables": []
}
const version = template.caproverOneClickApp.variables.find(v => v.id === '$$cap_APP_VERSION').defaultValue || 'latest'
newTemplate.displayName = template.caproverOneClickApp.displayName
newTemplate.name = template.caproverOneClickApp.displayName.toLowerCase()
newTemplate.documentation = template.caproverOneClickApp.documentation
newTemplate.description = template.caproverOneClickApp.description
newTemplate.serviceDefaultVersion = version
const varSet = new Set()
const caproverVariables = template.caproverOneClickApp.variables
for (const service of Object.keys(template.services)) {
const serviceTemplate = template.services[service]
const newServiceName = service.replaceAll('cap_appname', 'id')
const newService = {
image: '',
command: '',
environment: [],
volumes: []
}
const FROM = serviceTemplate.caproverExtra?.dockerfileLines?.find((line) => line.startsWith('FROM'))
if (serviceTemplate.image) {
newService.image = serviceTemplate.image.replaceAll('cap_APP_VERSION', 'core_version')
} else if (FROM) {
newService.image = FROM.split(' ')[1].replaceAll('cap_APP_VERSION', 'core_version')
}
const CMD = serviceTemplate.caproverExtra?.dockerfileLines?.find((line) => line.startsWith('CMD'))
if (serviceTemplate.command) {
newService.command = serviceTemplate.command
} else if (CMD) {
newService.command = CMD.replace('CMD ', '').replaceAll('"', '').replaceAll('[', '').replaceAll(']', '').replaceAll(',', ' ').replace(/\s+/g, ' ')
} else {
delete newService.command
}
const ENTRYPOINT = serviceTemplate.caproverExtra?.dockerfileLines?.find((line) => line.startsWith('ENTRYPOINT'))
if (serviceTemplate.entrypoint) {
newService.command = serviceTemplate.entrypoint
} else if (ENTRYPOINT) {
newService.entrypoint = ENTRYPOINT.replace('ENTRYPOINT ', '').replaceAll('"', '').replaceAll('[', '').replaceAll(']', '').replaceAll(',', ' ').replace(/\s+/g, ' ')
} else {
delete newService.entrypoint
}
if (serviceTemplate.environment && Object.keys(serviceTemplate.environment).length > 0) {
for (const env of Object.keys(serviceTemplate.environment)) {
if (serviceTemplate.environment[env].startsWith('srv-captain--$$cap_appname')) {
continue;
}
const value = '$$config_' + serviceTemplate.environment[env].replaceAll('srv-captain--$$cap_appname', '$$$id').replace('$$cap', '').replaceAll('captain-overlay-network', `$$$config_${env}`).toLowerCase()
newService.environment.push(`${env}=${value}`)
const foundVariable = varSet.has(env)
if (!foundVariable) {
const foundCaproverVariable = caproverVariables.find((item) => item.id === serviceTemplate.environment[env])
const defaultValue = foundCaproverVariable?.defaultValue ? foundCaproverVariable?.defaultValue.toString()?.replace('$$cap_gen_random_hex', '$$$generate_hex') : ''
if (defaultValue && defaultValue !== foundCaproverVariable?.defaultValue) {
console.log('changed')
}
newTemplate.variables.push({
"id": value,
"name": env,
"label": foundCaproverVariable?.label || '',
"defaultValue": defaultValue,
"description": foundCaproverVariable?.description || '',
})
}
varSet.add(env)
}
}
if (serviceTemplate.volumes && serviceTemplate.volumes.length > 0) {
for (const volume of serviceTemplate.volumes) {
const [source, target] = volume.split(':')
newService.volumes.push(`${source.replaceAll('$$cap_appname-', '$$$id-')}:${target}`)
}
}
newTemplate.services[newServiceName] = newService
}
await fs.writeFile('./caprover_new.yml', yaml.dump([{ ...newTemplate }]))

10
apps/api/src/index.ts
View File

@@ -11,6 +11,7 @@ import { scheduler } from './lib/scheduler';
import { compareVersions } from 'compare-versions'; import { compareVersions } from 'compare-versions';
import Graceful from '@ladjs/graceful' import Graceful from '@ladjs/graceful'
import axios from 'axios'; import axios from 'axios';
import yaml from 'js-yaml'
import fs from 'fs/promises'; import fs from 'fs/promises';
import { verifyRemoteDockerEngineFn } from './routes/api/v1/destinations/handlers'; import { verifyRemoteDockerEngineFn } from './routes/api/v1/destinations/handlers';
import { checkContainer } from './lib/docker'; import { checkContainer } from './lib/docker';
@@ -123,7 +124,14 @@ const host = '0.0.0.0';
} }
}) })
try { try {
await migrateServicesToNewTemplate() const templateYaml = await axios.get('https://gist.githubusercontent.com/andrasbacsai/701c450ef4272a929215cab11d737e3d/raw/4f021329d22934b90c5d67a0e49839a32bd629fd/template.yaml')
const templateJson = yaml.load(templateYaml.data)
if (isDev) {
await fs.writeFile('./template.json', JSON.stringify(templateJson, null, 2))
} else {
await fs.writeFile('/app/template.json', JSON.stringify(templateJson, null, 2))
}
await migrateServicesToNewTemplate(templateJson)
await fastify.listen({ port, host }) await fastify.listen({ port, host })
console.log(`Coolify's API is listening on ${host}:${port}`); console.log(`Coolify's API is listening on ${host}:${port}`);

16
apps/api/src/lib.ts
View File

@@ -1,8 +1,7 @@
import { decrypt, encrypt, getDomain, prisma } from "./lib/common"; import { decrypt, encrypt, getDomain, prisma } from "./lib/common";
import { includeServices } from "./lib/services/common"; import { includeServices } from "./lib/services/common";
import templates from "./lib/templates";
export async function migrateServicesToNewTemplate() { export async function migrateServicesToNewTemplate(templates: any) {
// This function migrates old hardcoded services to the new template based services // This function migrates old hardcoded services to the new template based services
try { try {
const services = await prisma.service.findMany({ include: includeServices }) const services = await prisma.service.findMany({ include: includeServices })
@@ -10,6 +9,9 @@ export async function migrateServicesToNewTemplate() {
if (!service.type) { if (!service.type) {
continue; continue;
} }
let template = templates.find(t => t.name === service.type.toLowerCase());
if (template) {
template = JSON.parse(JSON.stringify(template).replaceAll('$$id', service.id))
if (service.type === 'plausibleanalytics' && service.plausibleAnalytics) await plausibleAnalytics(service) if (service.type === 'plausibleanalytics' && service.plausibleAnalytics) await plausibleAnalytics(service)
if (service.type === 'fider' && service.fider) await fider(service) if (service.type === 'fider' && service.fider) await fider(service)
if (service.type === 'minio' && service.minio) await minio(service) if (service.type === 'minio' && service.minio) await minio(service)
@@ -23,7 +25,9 @@ export async function migrateServicesToNewTemplate() {
if (service.type === 'searxng' && service.searxng) await searxng(service) if (service.type === 'searxng' && service.searxng) await searxng(service)
if (service.type === 'weblate' && service.weblate) await weblate(service) if (service.type === 'weblate' && service.weblate) await weblate(service)
await createVolumes(service); await createVolumes(service, template);
}
} }
} catch (error) { } catch (error) {
console.log(error) console.log(error)
@@ -321,11 +325,8 @@ async function migrateSecrets(secrets: any[], service: any) {
await prisma.serviceSecret.findFirst({ where: { name, serviceId: service.id } }) || await prisma.serviceSecret.create({ data: { name, value, service: { connect: { id: service.id } } } }) await prisma.serviceSecret.findFirst({ where: { name, serviceId: service.id } }) || await prisma.serviceSecret.create({ data: { name, value, service: { connect: { id: service.id } } } })
} }
} }
async function createVolumes(service: any) { async function createVolumes(service: any, template: any) {
const volumes = []; const volumes = [];
let template = templates.find(t => t.name === service.type.toLowerCase());
if (template) {
template = JSON.parse(JSON.stringify(template).replaceAll('$$id', service.id))
for (const s of Object.keys(template.services)) { for (const s of Object.keys(template.services)) {
if (template.services[s].volumes && template.services[s].volumes.length > 0) { if (template.services[s].volumes && template.services[s].volumes.length > 0) {
for (const volume of template.services[s].volumes) { for (const volume of template.services[s].volumes) {
@@ -336,7 +337,6 @@ async function createVolumes(service: any) {
} }
} }
} }
}
for (const volume of volumes) { for (const volume of volumes) {
const [volumeName, path, containerId] = volume.split('@@@') const [volumeName, path, containerId] = volume.split('@@@')
// console.log('Creating volume', volumeName, path, containerId, 'for service', service.id, ', service name:', service.name) // console.log('Creating volume', volumeName, path, containerId, 'for service', service.id, ', service name:', service.name)

10
apps/api/src/lib/services.ts
View File

@@ -1,5 +1,13 @@
import { createDirectories, getServiceFromDB, getServiceImage, getServiceMainPort, makeLabelForServices } from "./common"; import { createDirectories, getServiceFromDB, getServiceImage, getServiceMainPort, isDev, makeLabelForServices } from "./common";
import fs from 'fs/promises';
export async function getTemplates() {
let templates = [];
if (isDev) {
templates = JSON.parse((await fs.readFile('./template.json')).toString())
}
return templates
}
export async function defaultServiceConfigurations({ id, teamId }) { export async function defaultServiceConfigurations({ id, teamId }) {
const service = await getServiceFromDB({ id, teamId }); const service = await getServiceFromDB({ id, teamId });
const { destinationDockerId, destinationDocker, type, serviceSecret } = service; const { destinationDockerId, destinationDocker, type, serviceSecret } = service;

3
apps/api/src/lib/services/handlers.ts
View File

@@ -6,7 +6,7 @@ import { ServiceStartStop } from '../../routes/api/v1/services/types';
import { asyncSleep, ComposeFile, createDirectories, defaultComposeConfiguration, errorHandler, executeDockerCmd, getDomain, getFreePublicPort, getServiceFromDB, getServiceImage, getServiceMainPort, isARM, isDev, makeLabelForServices, persistentVolumes, prisma } from '../common'; import { asyncSleep, ComposeFile, createDirectories, defaultComposeConfiguration, errorHandler, executeDockerCmd, getDomain, getFreePublicPort, getServiceFromDB, getServiceImage, getServiceMainPort, isARM, isDev, makeLabelForServices, persistentVolumes, prisma } from '../common';
import { defaultServiceConfigurations } from '../services'; import { defaultServiceConfigurations } from '../services';
import { OnlyId } from '../../types'; import { OnlyId } from '../../types';
import templates from '../templates'
import { parseAndFindServiceTemplates } from '../../routes/api/v1/services/handlers'; import { parseAndFindServiceTemplates } from '../../routes/api/v1/services/handlers';
import path from 'path'; import path from 'path';
// export async function startService(request: FastifyRequest<ServiceStartStop>) { // export async function startService(request: FastifyRequest<ServiceStartStop>) {
@@ -711,6 +711,7 @@ export async function startService(request: FastifyRequest<ServiceStartStop>) {
container_name: service, container_name: service,
build: template.services[service].build || undefined, build: template.services[service].build || undefined,
command: template.services[service].command, command: template.services[service].command,
entrypoint: template.services[service]?.entrypoint,
image: template.services[service].image, image: template.services[service].image,
expose: template.services[service].ports, expose: template.services[service].ports,
// ...(exposePort ? { ports: [`${exposePort}:${port}`] } : {}), // ...(exposePort ? { ports: [`${exposePort}:${port}`] } : {}),

15
apps/api/src/lib/templates.ts
View File

@@ -21,7 +21,7 @@ export default [
`WEBLATE_ADMIN_PASSWORD=$$secret_weblate_admin_password`, `WEBLATE_ADMIN_PASSWORD=$$secret_weblate_admin_password`,
`POSTGRES_PASSWORD=$$secret_postgres_password`, `POSTGRES_PASSWORD=$$secret_postgres_password`,
`POSTGRES_USER=$$config_postgres_user`, `POSTGRES_USER=$$config_postgres_user`,
`POSTGRES_DATABASE=$$config_postgres_database`, `POSTGRES_DATABASE=$$config_postgres_db`,
`POSTGRES_HOST=$$id-postgresql`, `POSTGRES_HOST=$$id-postgresql`,
`POSTGRES_PORT=5432`, `POSTGRES_PORT=5432`,
`REDIS_HOST=$$id-redis`, `REDIS_HOST=$$id-redis`,
@@ -94,13 +94,7 @@ export default [
"defaultValue": "weblate", "defaultValue": "weblate",
"description": "", "description": "",
}, },
{
"id": "$$config_postgres_database",
"name": "POSTGRES_DATABASE",
"label": "PostgreSQL Database",
"defaultValue": "$$config_postgres_db",
"description": ""
},
] ]
}, },
{ {
@@ -121,7 +115,6 @@ export default [
], ],
"environment": [ "environment": [
"SEARXNG_BASE_URL=$$config_searxng_base_url", "SEARXNG_BASE_URL=$$config_searxng_base_url",
"SECRET_KEY=$$secret_secret_key",
], ],
"ports": [ "ports": [
"8080" "8080"
@@ -1462,9 +1455,7 @@ export default [
"label": "Secret Key Base", "label": "Secret Key Base",
"defaultValue": "$$generate_passphrase", "defaultValue": "$$generate_passphrase",
"description": "", "description": "",
"extras": {
"length": 64
}
}, },
{ {
"id": "$$config_disable_auth", "id": "$$config_disable_auth",

44
apps/api/src/routes/api/v1/services/handlers.ts
View File

@@ -2,6 +2,7 @@ import type { FastifyReply, FastifyRequest } from 'fastify';
import fs from 'fs/promises'; import fs from 'fs/promises';
import yaml from 'js-yaml'; import yaml from 'js-yaml';
import bcrypt from 'bcryptjs'; import bcrypt from 'bcryptjs';
import crypto from 'crypto';
import { prisma, uniqueName, asyncExecShell, getServiceFromDB, getContainerUsage, isDomainConfigured, saveUpdateableFields, fixType, decrypt, encrypt, ComposeFile, getFreePublicPort, getDomain, errorHandler, generatePassword, isDev, stopTcpHttpProxy, executeDockerCmd, checkDomainsIsValidInDNS, checkExposedPort, listSettings } from '../../../../lib/common'; import { prisma, uniqueName, asyncExecShell, getServiceFromDB, getContainerUsage, isDomainConfigured, saveUpdateableFields, fixType, decrypt, encrypt, ComposeFile, getFreePublicPort, getDomain, errorHandler, generatePassword, isDev, stopTcpHttpProxy, executeDockerCmd, checkDomainsIsValidInDNS, checkExposedPort, listSettings } from '../../../../lib/common';
import { day } from '../../../../lib/dayjs'; import { day } from '../../../../lib/dayjs';
import { checkContainer, isContainerExited } from '../../../../lib/docker'; import { checkContainer, isContainerExited } from '../../../../lib/docker';
@@ -12,7 +13,7 @@ import type { ActivateWordpressFtp, CheckService, CheckServiceDomain, DeleteServ
import { supportedServiceTypesAndVersions } from '../../../../lib/services/supportedVersions'; import { supportedServiceTypesAndVersions } from '../../../../lib/services/supportedVersions';
import { configureServiceType, removeService } from '../../../../lib/services/common'; import { configureServiceType, removeService } from '../../../../lib/services/common';
import { hashPassword } from '../handlers'; import { hashPassword } from '../handlers';
import templates from '../../../../lib/templates'; import { getTemplates } from '../../../../lib/services';
export async function listServices(request: FastifyRequest) { export async function listServices(request: FastifyRequest) {
try { try {
@@ -113,6 +114,7 @@ export async function getServiceStatus(request: FastifyRequest<OnlyId>) {
} }
} }
export async function parseAndFindServiceTemplates(service: any, workdir?: string, isDeploy: boolean = false) { export async function parseAndFindServiceTemplates(service: any, workdir?: string, isDeploy: boolean = false) {
const templates = await getTemplates()
const foundTemplate = templates.find(t => t.name === service.type.toLowerCase()) const foundTemplate = templates.find(t => t.name === service.type.toLowerCase())
let parsedTemplate = {} let parsedTemplate = {}
if (foundTemplate) { if (foundTemplate) {
@@ -162,7 +164,6 @@ export async function parseAndFindServiceTemplates(service: any, workdir?: strin
parsedTemplate = JSON.parse(JSON.stringify(parsedTemplate).replaceAll(regex, getDomain(service.fqdn) + "\"")) parsedTemplate = JSON.parse(JSON.stringify(parsedTemplate).replaceAll(regex, getDomain(service.fqdn) + "\""))
} else { } else {
parsedTemplate = JSON.parse(JSON.stringify(parsedTemplate).replaceAll(regex, value + "\"")) parsedTemplate = JSON.parse(JSON.stringify(parsedTemplate).replaceAll(regex, value + "\""))
} }
} }
@@ -203,7 +204,7 @@ export async function getService(request: FastifyRequest<OnlyId>) {
export async function getServiceType(request: FastifyRequest) { export async function getServiceType(request: FastifyRequest) {
try { try {
return { return {
services: templates services: await getTemplates()
} }
} catch ({ status, message }) { } catch ({ status, message }) {
return errorHandler({ status, message }) return errorHandler({ status, message })
@@ -213,6 +214,7 @@ export async function saveServiceType(request: FastifyRequest<SaveServiceType>,
try { try {
const { id } = request.params; const { id } = request.params;
const { type } = request.body; const { type } = request.body;
const templates = await getTemplates()
let foundTemplate = templates.find(t => t.name === type) let foundTemplate = templates.find(t => t.name === type)
if (foundTemplate) { if (foundTemplate) {
let generatedVariables = new Set() let generatedVariables = new Set()
@@ -223,24 +225,34 @@ export async function saveServiceType(request: FastifyRequest<SaveServiceType>,
if (foundTemplate.variables.length > 0) { if (foundTemplate.variables.length > 0) {
foundTemplate.variables = foundTemplate.variables.map(variable => { foundTemplate.variables = foundTemplate.variables.map(variable => {
let { id: variableId } = variable; let { id: variableId } = variable;
console.log(variableId)
if (variableId.startsWith('$$secret_')) { if (variableId.startsWith('$$secret_')) {
const length = variable?.extras && variable.extras['length'] if (variable.defaultValue.startsWith('$$generate_password')) {
if (variable.defaultValue === '$$generate_password') { const length = variable.defaultValue.replace('$$generate_password(', '').replace('\)', '') || 16
variable.value = generatePassword({ length }); variable.value = generatePassword({ length: Number(length) });
} else if (variable.defaultValue === '$$generate_passphrase') { } else if (variable.defaultValue.startsWith('$$generate_hex')) {
variable.value = generatePassword({ length }); const length = variable.defaultValue.replace('$$generate_hex(', '').replace('\)', '') || 16
variable.value = crypto.randomBytes(Number(length)).toString('hex');
} else if (!variable.defaultValue) { } else if (!variable.defaultValue) {
variable.defaultValue = undefined variable.defaultValue = undefined
} }
} }
if (variableId.startsWith('$$config_')) { if (variableId.startsWith('$$config_')) {
if (variable.defaultValue === '$$generate_username') { if (variable.defaultValue.startsWith('$$generate_username')) {
const length = variable.defaultValue.replace('$$generate_username(', '').replace('\)', '')
if (length !== '$$generate_username') {
variable.value = crypto.randomBytes(Number(length)).toString('hex');
} else {
variable.value = cuid(); variable.value = cuid();
}
} else {
if (variable.defaultValue.startsWith('$$generate_hex')) {
const length = variable.defaultValue.replace('$$generate_hex(', '').replace('\)', '') || 16
variable.value = crypto.randomBytes(Number(length)).toString('hex');
} else { } else {
variable.value = variable.defaultValue || '' variable.value = variable.defaultValue || ''
} }
} }
}
if (variable.value) { if (variable.value) {
generatedVariables.add(`${variableId}=${variable.value}`) generatedVariables.add(`${variableId}=${variable.value}`)
} else { } else {
@@ -268,25 +280,34 @@ export async function saveServiceType(request: FastifyRequest<SaveServiceType>,
if (!variable.value) { if (!variable.value) {
continue; continue;
} }
const found = await prisma.serviceSecret.findFirst({ where: { name: variable.name, serviceId: id } })
if (!found) {
await prisma.serviceSecret.create({ await prisma.serviceSecret.create({
data: { name: variable.name, value: encrypt(variable.value), service: { connect: { id } } } data: { name: variable.name, value: encrypt(variable.value), service: { connect: { id } } }
}) })
} }
}
if (variable.id.startsWith('$$config_')) { if (variable.id.startsWith('$$config_')) {
if (!variable.value) { if (!variable.value) {
variable.value = ''; variable.value = '';
} }
const found = await prisma.serviceSetting.findFirst({ where: { name: variable.name, serviceId: id } })
if (!found) {
await prisma.serviceSetting.create({ await prisma.serviceSetting.create({
data: { name: variable.name, value: variable.value, service: { connect: { id } } } data: { name: variable.name, value: variable.value.toString(), service: { connect: { id } } }
}) })
} }
} }
} }
}
for (const service of Object.keys(foundTemplate.services)) { for (const service of Object.keys(foundTemplate.services)) {
if (foundTemplate.services[service].volumes) { if (foundTemplate.services[service].volumes) {
for (const volume of foundTemplate.services[service].volumes) { for (const volume of foundTemplate.services[service].volumes) {
const [volumeName, path] = volume.split(':') const [volumeName, path] = volume.split(':')
if (!volumeName.startsWith('/')) { if (!volumeName.startsWith('/')) {
const found = await prisma.servicePersistentStorage.findFirst({ where: { volumeName, serviceId: id } })
if (!found) {
await prisma.servicePersistentStorage.create({ await prisma.servicePersistentStorage.create({
data: { volumeName, path, containerId: service, predefined: true, service: { connect: { id } } } data: { volumeName, path, containerId: service, predefined: true, service: { connect: { id } } }
}); });
@@ -294,6 +315,7 @@ export async function saveServiceType(request: FastifyRequest<SaveServiceType>,
} }
} }
} }
}
await prisma.service.update({ where: { id }, data: { type, version: foundTemplate.serviceDefaultVersion } }) await prisma.service.update({ where: { id }, data: { type, version: foundTemplate.serviceDefaultVersion } })
return reply.code(201).send() return reply.code(201).send()
} else { } else {

3
apps/ui/src/routes/services/[id]/index.svelte
View File

@@ -381,7 +381,8 @@
class:border-b={template[oneService].environment.length > 0} class:border-b={template[oneService].environment.length > 0}
class:border-coolgray-500={template[oneService].environment.length > 0} class:border-coolgray-500={template[oneService].environment.length > 0}
> >
<div class="title font-bold pb-3">{template[oneService].name}</div> <div class="title font-bold pb-3">{template[oneService].name || oneService.replace(`${id}-`,'').replace(id,service.type)}</div>
<ServiceStatus id={oneService} /> <ServiceStatus id={oneService} />
</div> </div>
<div class="grid grid-flow-row gap-2 px-4"> <div class="grid grid-flow-row gap-2 px-4">

2
apps/ui/src/routes/services/[id]/logs/index.svelte
View File

@@ -93,7 +93,7 @@
<div class="title font-bold pb-3">Service Logs</div> <div class="title font-bold pb-3">Service Logs</div>
</div> </div>
</div> </div>
<div class="flex gap-2 lg:gap-8 pb-4"> <div class="grid grid-cols-4 gap-2 lg:gap-8 pb-4">
{#if template} {#if template}
{#each Object.keys(template) as service} {#each Object.keys(template) as service}
<button <button