dave/coolify
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

saving things

Browse Source
This commit is contained in:
Andras Bacsai
2022-10-21 15:51:32 +02:00
parent 049d5166e8
commit 5d60b5eb8b
12 changed files with 1681 additions and 207 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1315
apps/api/src/lib/services.ts
View File

File diff suppressed because it is too large Load Diff

13
apps/api/src/lib/services/handlers.ts
View File

@@ -3,7 +3,7 @@ import fs from 'fs/promises';
import yaml from 'js-yaml';
import bcrypt from 'bcryptjs';
import { ServiceStartStop } from '../../routes/api/v1/services/types';
import { asyncSleep, ComposeFile, createDirectories, defaultComposeConfiguration, errorHandler, executeDockerCmd, getDomain, getFreePublicPort, getServiceFromDB, getServiceImage, getServiceMainPort, isARM, isDev, makeLabelForServices, persistentVolumes, prisma } from '../common';
import { asyncSleep, ComposeFile, createDirectories, decrypt, defaultComposeConfiguration, errorHandler, executeDockerCmd, getDomain, getFreePublicPort, getServiceFromDB, getServiceImage, getServiceMainPort, isARM, isDev, makeLabelForServices, persistentVolumes, prisma } from '../common';
import { defaultServiceConfigurations } from '../services';
import { OnlyId } from '../../types';
@@ -706,6 +706,16 @@ export async function startService(request: FastifyRequest<ServiceStartStop>) {
if (!value.startsWith('$$secret') && value !== '') {
newEnviroments.push(`${env}=${value}`)
}
}
const secrets = await prisma.serviceSecret.findMany({ where: { serviceId: id } })
for (const secret of secrets) {
const { name, value } = secret
if (value) {
if (template.services[service].environment.find(env => env.startsWith(`${name}=`)) && !newEnviroments.find(env => env.startsWith(`${name}=`))) {
newEnviroments.push(`${name}=${decrypt(value)}`)
}
}
}
config[service] = {
container_name: service,
@@ -757,7 +767,6 @@ export async function startService(request: FastifyRequest<ServiceStartStop>) {
}
const composeFileDestination = `${workdir}/docker-compose.yaml`;
await fs.writeFile(composeFileDestination, yaml.dump(composeFile));
console.log(composeFileDestination)
await startServiceContainers(destinationDocker.id, composeFileDestination)
return {}
} catch ({ status, message }) {

26
apps/api/src/lib/templates.ts
View File

@@ -1,7 +1,7 @@
export default [
{
"templateVersion": "1.0.0",
"serviceDefaultVersion": "latest",
"defaultVersion": "latest",
"name": "weblate",
"displayName": "Weblate",
"description": "",
@@ -99,7 +99,7 @@ export default [
},
{
"templateVersion": "1.0.0",
"serviceDefaultVersion": "2022.10.14-1a5b0965",
"defaultVersion": "2022.10.14-1a5b0965",
"name": "searxng",
"displayName": "SearXNG",
"description": "",
@@ -183,7 +183,7 @@ export default [
},
{
"templateVersion": "1.0.0",
"serviceDefaultVersion": "v2.0.6",
"defaultVersion": "v2.0.6",
"name": "glitchtip",
"displayName": "GlitchTip",
"description": "",
@@ -398,7 +398,7 @@ export default [
},
{
"templateVersion": "1.0.0",
"serviceDefaultVersion": "v2.13.0",
"defaultVersion": "v2.13.0",
"name": "hasura",
"displayName": "Hasura",
"description": "Instant realtime GraphQL APIs on any Postgres application, existing or new.",
@@ -484,7 +484,7 @@ export default [
},
{
"templateVersion": "1.0.0",
"serviceDefaultVersion": "postgresql-v1.38.0",
"defaultVersion": "postgresql-v1.38.0",
"name": "umami",
"displayName": "Umami",
"description": "Umami is a simple, easy to use, self-hosted web analytics solution. The goal is to provide you with a friendly privacy-focused alternative to Google Analytics.",
@@ -713,7 +713,7 @@ export default [
},
{
"templateVersion": "1.0.0",
"serviceDefaultVersion": "v0.29.1",
"defaultVersion": "v0.29.1",
"name": "meilisearch",
"displayName": "MeiliSearch",
"description": "MeiliSearch is a lightning Fast, Ultra Relevant, and Typo-Tolerant Search Engine",
@@ -752,7 +752,7 @@ export default [
},
{
"templateVersion": "1.0.0",
"serviceDefaultVersion": "latest",
"defaultVersion": "latest",
"name": "ghost",
"displayName": "Ghost",
"description": "Ghost is a free and open source blogging platform written in JavaScript and distributed under the MIT License",
@@ -904,7 +904,7 @@ export default [
},
{
"templateVersion": "1.0.0",
"serviceDefaultVersion": "php8.1",
"defaultVersion": "php8.1",
"name": "wordpress",
"displayName": "WordPress",
"description": "WordPress is a content management system based on PHP.",
@@ -1022,7 +1022,7 @@ export default [
},
{
"templateVersion": "1.0.0",
"serviceDefaultVersion": "4.7.1",
"defaultVersion": "4.7.1",
"name": "vscodeserver",
"displayName": "VSCode Server",
"description": "vscode-server by Coder is VS Code running on a remote server, accessible through the browser.",
@@ -1062,7 +1062,7 @@ export default [
},
{
"templateVersion": "1.0.0",
"serviceDefaultVersion": "RELEASE.2022-10-15T19-57-03Z",
"defaultVersion": "RELEASE.2022-10-15T19-57-03Z",
"name": "minio",
"displayName": "MinIO",
"description": " MinIO is a cloud storage server compatible with Amazon S3",
@@ -1132,7 +1132,7 @@ export default [
},
{
"templateVersion": "1.0.0",
"serviceDefaultVersion": "0.21.1",
"defaultVersion": "0.21.1",
"name": "fider",
"displayName": "Fider",
"description": "Fider is a platform to collect and organize customer feedback.",
@@ -1286,7 +1286,7 @@ export default [
},
{
"templateVersion": "1.0.0",
"serviceDefaultVersion": "0.198.1",
"defaultVersion": "0.198.1",
"name": "n8n",
"displayName": "n8n.io",
"description": "n8n is a free and open node based Workflow Automation Tool.",
@@ -1320,7 +1320,7 @@ export default [
},
{
"templateVersion": "1.0.0",
"serviceDefaultVersion": "stable",
"defaultVersion": "stable",
"name": "plausibleanalytics",
"displayName": "PlausibleAnalytics",
"description": "Plausible is a lightweight and open-source website analytics tool.",

171
apps/api/src/routes/api/v1/services/handlers.ts
View File

@@ -115,7 +115,7 @@ export async function getServiceStatus(request: FastifyRequest<OnlyId>) {
}
export async function parseAndFindServiceTemplates(service: any, workdir?: string, isDeploy: boolean = false) {
const templates = await getTemplates()
const foundTemplate = templates.find(t => t.name === service.type.toLowerCase())
const foundTemplate = templates.find(t => t.name.toLowerCase() === service.type.toLowerCase())
let parsedTemplate = {}
if (foundTemplate) {
if (!isDeploy) {
@@ -124,57 +124,87 @@ export async function parseAndFindServiceTemplates(service: any, workdir?: strin
parsedTemplate[realKey] = {
name: value.name,
image: value.image,
environment: []
environment: [],
proxy: {}
}
if (value.environment?.length > 0) {
for (const env of value.environment) {
const [envKey, envValue] = env.split('=')
const label = foundTemplate.variables.find(v => v.name === envKey)?.label
const description = foundTemplate.variables.find(v => v.name === envKey)?.description
const defaultValue = foundTemplate.variables.find(v => v.name === envKey)?.defaultValue
const extras = foundTemplate.variables.find(v => v.name === envKey)?.extras
const variable = foundTemplate.variables.find(v => v.name === envKey) || foundTemplate.variables.find(v => v.id === envValue)
const label = variable?.label
const description = variable?.description
const defaultValue = variable?.defaultValue
const extras = variable?.extras
if (envValue.startsWith('$$config') || extras?.isVisibleOnUI) {
if (envValue.startsWith('$$config_coolify')) {
console.log({envValue,envKey})
}
parsedTemplate[realKey].environment.push(
{ name: envKey, value: envValue, label, description, defaultValue, extras }
)
}
}
}
// TODO: seconday domains are not working - kinda working
if (value?.proxy?.traefik?.configurations) {
for (const proxyValue of value.proxy.traefik.configurations) {
if (proxyValue.domain) {
const variable = foundTemplate.variables.find(v => v.id === proxyValue.domain)
if (variable) {
const { name, label, description, defaultValue, extras } = variable
const found = await prisma.serviceSetting.findFirst({where: {variableName: proxyValue.domain}})
parsedTemplate[realKey].environment.push(
{ name, value: found.value || '', label, description, defaultValue, extras }
)
}
}
}
}
}
} else {
parsedTemplate = foundTemplate
}
let strParsedTemplate = JSON.stringify(parsedTemplate)
// replace $$id and $$workdir
parsedTemplate = JSON.parse(JSON.stringify(parsedTemplate).replaceAll('$$id', service.id).replaceAll('$$core_version', service.version || foundTemplate.serviceDefaultVersion))
strParsedTemplate = strParsedTemplate.replaceAll('$$id', service.id)
strParsedTemplate = strParsedTemplate.replaceAll('$$core_version', service.version || foundTemplate.defaultVersion)
// replace $$fqdn
if (workdir) {
parsedTemplate = JSON.parse(JSON.stringify(parsedTemplate).replaceAll('$$workdir', workdir))
strParsedTemplate = strParsedTemplate.replaceAll('$$workdir', workdir)
}
// replace $$config
if (service.serviceSetting.length > 0) {
for (const setting of service.serviceSetting) {
const { name, value } = setting
const regex = new RegExp(`\\$\\$config_${name}\\"`, 'gi')
const { value, variableName } = setting
if (service.fqdn && value === '$$generate_fqdn') {
parsedTemplate = JSON.parse(JSON.stringify(parsedTemplate).replaceAll(regex, service.fqdn + "\""))
strParsedTemplate = strParsedTemplate.replaceAll(variableName, service.fqdn)
} else if (service.fqdn && value === '$$generate_domain') {
parsedTemplate = JSON.parse(JSON.stringify(parsedTemplate).replaceAll(regex, getDomain(service.fqdn) + "\""))
strParsedTemplate = strParsedTemplate.replaceAll(variableName, getDomain(service.fqdn))
} else if (service.destinationDocker?.network && value === '$$generate_network') {
strParsedTemplate = strParsedTemplate.replaceAll(variableName, service.destinationDocker.network)
} else {
parsedTemplate = JSON.parse(JSON.stringify(parsedTemplate).replaceAll(regex, value + "\""))
strParsedTemplate = strParsedTemplate.replaceAll(variableName, value)
}
}
}
// replace $$secret
if (service.serviceSecret.length > 0) {
for (const secret of service.serviceSecret) {
const { name, value } = secret
parsedTemplate = JSON.parse(JSON.stringify(parsedTemplate).replaceAll(`$$hashed$$secret_${name.toLowerCase()}`, bcrypt.hashSync(value, 10)).replaceAll(`$$secret_${name.toLowerCase()}`, value))
const regexHashed = new RegExp(`\\$\\$hashed\\$\\$secret_${name}\\"`, 'gi')
const regex = new RegExp(`\\$\\$secret_${name}\\"`, 'gi')
if (value) {
strParsedTemplate = strParsedTemplate.replaceAll(regexHashed, bcrypt.hashSync(value, 10) + "\"")
strParsedTemplate = strParsedTemplate.replaceAll(regex, value + "\"")
}
}
}
parsedTemplate = JSON.parse(strParsedTemplate)
}
return parsedTemplate
}
@@ -217,90 +247,43 @@ export async function saveServiceType(request: FastifyRequest<SaveServiceType>,
const templates = await getTemplates()
let foundTemplate = templates.find(t => t.name === type)
if (foundTemplate) {
let generatedVariables = new Set()
let missingVariables = new Set()
foundTemplate = JSON.parse(JSON.stringify(foundTemplate).replaceAll('$$id', id))
if (foundTemplate.variables.length > 0) {
foundTemplate.variables = foundTemplate.variables.map(variable => {
let { id: variableId } = variable;
if (variableId.startsWith('$$secret_')) {
if (variable.defaultValue.startsWith('$$generate_password')) {
const length = variable.defaultValue.replace('$$generate_password(', '').replace('\)', '') || 16
variable.value = generatePassword({ length: Number(length) });
} else if (variable.defaultValue.startsWith('$$generate_hex')) {
const length = variable.defaultValue.replace('$$generate_hex(', '').replace('\)', '') || 16
variable.value = crypto.randomBytes(Number(length)).toString('hex');
} else if (!variable.defaultValue) {
variable.defaultValue = undefined
}
}
if (variableId.startsWith('$$config_')) {
if (variable.defaultValue.startsWith('$$generate_username')) {
const length = variable.defaultValue.replace('$$generate_username(', '').replace('\)', '')
if (length !== '$$generate_username') {
variable.value = crypto.randomBytes(Number(length)).toString('hex');
} else {
variable.value = cuid();
}
} else {
if (variable.defaultValue.startsWith('$$generate_hex')) {
const length = variable.defaultValue.replace('$$generate_hex(', '').replace('\)', '') || 16
variable.value = crypto.randomBytes(Number(length)).toString('hex');
} else {
variable.value = variable.defaultValue || ''
}
}
}
if (variable.value) {
generatedVariables.add(`${variableId}=${variable.value}`)
} else {
missingVariables.add(variableId)
}
return variable
})
if (missingVariables.size > 0) {
foundTemplate.variables = foundTemplate.variables.map(variable => {
if (missingVariables.has(variable.id)) {
variable.value = variable.defaultValue
for (const generatedVariable of generatedVariables) {
let [id, value] = generatedVariable.split('=')
if (variable.value) {
variable.value = variable.value.replaceAll(id, value)
}
}
}
return variable
})
}
for (const variable of foundTemplate.variables) {
if (variable.id.startsWith('$$secret_')) {
if (!variable.value) {
continue;
}
const found = await prisma.serviceSecret.findFirst({ where: { name: variable.name, serviceId: id } })
if (!found) {
await prisma.serviceSecret.create({
data: { name: variable.name, value: encrypt(variable.value), service: { connect: { id } } }
})
}
}
if (variable.id.startsWith('$$config_')) {
if (!variable.value) {
variable.value = '';
}
const found = await prisma.serviceSetting.findFirst({ where: { name: variable.name, serviceId: id } })
if (!found) {
await prisma.serviceSetting.create({
data: { name: variable.name, value: variable.value.toString(), service: { connect: { id } } }
})
}
const { defaultValue } = variable;
const regex = /^\$\$.*\((\d+)\)$/g;
const length = Number(regex.exec(defaultValue)?.[1]) || undefined
if (variable.defaultValue.startsWith('$$generate_password')) {
variable.value = generatePassword({ length });
} else if (variable.defaultValue.startsWith('$$generate_hex')) {
variable.value = generatePassword({ length, isHex: true });
} else if (variable.defaultValue.startsWith('$$generate_username')) {
variable.value = cuid();
} else {
variable.value = variable.defaultValue || '';
}
}
}
for (const variable of foundTemplate.variables) {
if (variable.id.startsWith('$$secret_')) {
const found = await prisma.serviceSecret.findFirst({ where: { name: variable.name, serviceId: id } })
if (!found) {
await prisma.serviceSecret.create({
data: { name: variable.name, value: encrypt(variable.value) || '', service: { connect: { id } } }
})
}
}
if (variable.id.startsWith('$$config_')) {
const found = await prisma.serviceSetting.findFirst({ where: { name: variable.name, serviceId: id } })
if (!found) {
await prisma.serviceSetting.create({
data: { name: variable.name, value: variable.value.toString(), variableName: variable.id, service: { connect: { id } } }
})
}
}
}
for (const service of Object.keys(foundTemplate.services)) {
if (foundTemplate.services[service].volumes) {
for (const volume of foundTemplate.services[service].volumes) {
@@ -316,7 +299,7 @@ export async function saveServiceType(request: FastifyRequest<SaveServiceType>,
}
}
}
await prisma.service.update({ where: { id }, data: { type, version: foundTemplate.serviceDefaultVersion } })
await prisma.service.update({ where: { id }, data: { type, version: foundTemplate.defaultVersion } })
return reply.code(201).send()
} else {
throw { status: 404, message: 'Service type not found.' }

195
apps/api/src/routes/webhooks/traefik/handlers.ts
View File

@@ -4,7 +4,50 @@ import { supportedServiceTypesAndVersions } from "../../../lib/services/supporte
import { includeServices } from "../../../lib/services/common";
import { TraefikOtherConfiguration } from "./types";
import { OnlyId } from "../../../types";
import { getTemplates } from "../../../lib/services";
function generateLoadBalancerService(id, port) {
return {
loadbalancer: {
servers: [
{
url: `http://${id}:${port}`
}
]
}
};
}
function generateHttpRouter(id, nakedDomain, pathPrefix) {
return {
entrypoints: ['web'],
rule: `(Host(\`${nakedDomain}\`) || Host(\`www.${nakedDomain}\`)) && PathPrefix(\`${pathPrefix}\`)`,
service: `${id}`,
middlewares: []
}
}
function generateProtocolRedirectRouter(id, nakedDomain, pathPrefix, fromTo) {
if (fromTo === 'https-to-http') {
return {
entrypoints: ['websecure'],
rule: `(Host(\`${nakedDomain}\`) || Host(\`www.${nakedDomain}\`)) && PathPrefix(\`${pathPrefix}\`)`,
service: `${id}`,
tls: {
domains: {
main: `${nakedDomain}`
}
},
middlewares: ['redirect-to-http']
}
} else if (fromTo === 'http-to-https') {
return {
entrypoints: ['web'],
rule: `(Host(\`${nakedDomain}\`) || Host(\`www.${nakedDomain}\`)) && PathPrefix(\`${pathPrefix}\`)`,
service: `${id}`,
middlewares: ['redirect-to-https']
};
}
}
function configureMiddleware(
{ id, container, port, domain, nakedDomain, isHttps, isWWW, isDualCerts, scriptName, type, isCustomSSL },
traefik
@@ -325,71 +368,127 @@ export async function traefikConfiguration(request, reply) {
});
for (const service of services) {
const {
let {
fqdn,
id,
type,
destinationDockerId,
dualCerts,
serviceSetting
} = service;
if (destinationDockerId) {
const found = supportedServiceTypesAndVersions.find((a) => a.name === type);
const templates = await getTemplates();
let found = templates.find((a) => a.name === type);
type = type.toLowerCase();
if (found) {
const port = found.ports.main;
const publicPort = service[type]?.publicPort;
const isRunning = true;
if (fqdn) {
const domain = getDomain(fqdn);
const nakedDomain = domain.replace(/^www\./, '');
const isHttps = fqdn.startsWith('https://');
const isWWW = fqdn.includes('www.');
if (isRunning) {
// Plausible Analytics custom script
let scriptName = false;
if (type === 'plausibleanalytics') {
const foundScriptName = serviceSetting.find((a) => a.name === 'SCRIPT_NAME')?.value;
if (foundScriptName) {
scriptName = foundScriptName;
found = JSON.parse(JSON.stringify(found).replaceAll('$$id', id));
for (const oneService of Object.keys(found.services)) {
const isProxyConfiguration = found.services[oneService].proxy;
if (isProxyConfiguration) {
const { proxy: { traefik: { configurations } } } = found.services[oneService];
for (const configuration of configurations) {
const publicPort = service[type]?.publicPort;
if (fqdn) {
data.services.push({
id: oneService,
publicPort,
fqdn,
dualCerts,
configuration
});
}
}
let container = id;
let otherDomain = null;
let otherNakedDomain = null;
let otherIsHttps = null;
let otherIsWWW = null;
if (type === 'minio') {
const domain = service.serviceSetting.find((a) => a.name === 'MINIO_SERVER_URL')?.value
otherDomain = getDomain(domain);
otherNakedDomain = otherDomain.replace(/^www\./, '');
otherIsHttps = domain.startsWith('https://');
otherIsWWW = domain.includes('www.');
}
data.services.push({
id,
container,
type,
otherDomain,
otherNakedDomain,
otherIsHttps,
otherIsWWW,
port,
publicPort,
domain,
nakedDomain,
isRunning,
isHttps,
isWWW,
isDualCerts: dualCerts,
scriptName
});
}
}
}
}
}
for (const service of data.services) {
const { id, fqdn, dualCerts, configuration: { port, pathPrefix = '/' }, isCustomSSL = false } = service
const domain = getDomain(fqdn);
const nakedDomain = domain.replace(/^www\./, '');
const isHttps = fqdn.startsWith('https://');
const isWWW = fqdn.includes('www.');
if (isHttps) {
traefik.http.routers[id] = generateHttpRouter(id, nakedDomain, pathPrefix)
traefik.http.routers[`${id}-secure`] = generateProtocolRedirectRouter(id, nakedDomain, pathPrefix, 'http-to-https')
traefik.http.services[id] = generateLoadBalancerService(id, port)
if (dualCerts) {
traefik.http.routers[`${id}-secure`] = {
entrypoints: ['websecure'],
rule: `(Host(\`${nakedDomain}\`) || Host(\`www.${nakedDomain}\`)) && PathPrefix(\`${pathPrefix}\`)`,
service: `${id}`,
tls: isCustomSSL ? true : {
certresolver: 'letsencrypt'
},
middlewares: []
};
} else {
if (isWWW) {
traefik.http.routers[`${id}-secure-www`] = {
entrypoints: ['websecure'],
rule: `Host(\`www.${nakedDomain}\`) && PathPrefix(\`${pathPrefix}\`)`,
service: `${id}`,
tls: isCustomSSL ? true : {
certresolver: 'letsencrypt'
},
middlewares: []
};
traefik.http.routers[`${id}-secure`] = {
entrypoints: ['websecure'],
rule: `Host(\`${nakedDomain}\`) && PathPrefix(\`${pathPrefix}\`)`,
service: `${id}`,
tls: {
domains: {
main: `${domain}`
}
},
middlewares: ['redirect-to-www']
};
traefik.http.routers[`${id}`].middlewares.push('redirect-to-www');
} else {
traefik.http.routers[`${id}-secure-www`] = {
entrypoints: ['websecure'],
rule: `Host(\`www.${nakedDomain}\`) && PathPrefix(\`${pathPrefix}\`)`,
service: `${id}`,
tls: {
domains: {
main: `${domain}`
}
},
middlewares: ['redirect-to-non-www']
};
traefik.http.routers[`${id}-secure`] = {
entrypoints: ['websecure'],
rule: `Host(\`${domain}\`) && PathPrefix(\`${pathPrefix}\`)`,
service: `${id}`,
tls: isCustomSSL ? true : {
certresolver: 'letsencrypt'
},
middlewares: []
};
traefik.http.routers[`${id}`].middlewares.push('redirect-to-non-www');
}
}
} else {
traefik.http.routers[id] = generateHttpRouter(id, nakedDomain, pathPrefix)
traefik.http.routers[`${id}-secure`] = generateProtocolRedirectRouter(id, nakedDomain, pathPrefix, 'https-to-http')
traefik.http.services[id] = generateLoadBalancerService(id, port)
if (!dualCerts) {
if (isWWW) {
traefik.http.routers[`${id}`].middlewares.push('redirect-to-www');
traefik.http.routers[`${id}-secure`].middlewares.push('redirect-to-www');
} else {
traefik.http.routers[`${id}`].middlewares.push('redirect-to-non-www');
traefik.http.routers[`${id}-secure`].middlewares.push('redirect-to-non-www');
}
}
}
}
return {
...traefik
}
const { fqdn, dualCerts } = await prisma.setting.findFirst();
if (fqdn) {
const domain = getDomain(fqdn);