This commit is contained in:
Andras Bacsai
2022-10-21 21:19:30 +02:00
parent 55fc3920fc
commit 9c74a9c1db
5 changed files with 144 additions and 43 deletions

View File

@@ -0,0 +1,21 @@
-- RedefineTables
PRAGMA foreign_keys=OFF;
CREATE TABLE "new_Service" (
"id" TEXT NOT NULL PRIMARY KEY,
"name" TEXT NOT NULL,
"fqdn" TEXT,
"exposePort" INTEGER,
"dualCerts" BOOLEAN NOT NULL DEFAULT false,
"type" TEXT,
"version" TEXT,
"templateVersion" TEXT NOT NULL DEFAULT '0.0.0',
"destinationDockerId" TEXT,
"createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updatedAt" DATETIME NOT NULL,
CONSTRAINT "Service_destinationDockerId_fkey" FOREIGN KEY ("destinationDockerId") REFERENCES "DestinationDocker" ("id") ON DELETE SET NULL ON UPDATE CASCADE
);
INSERT INTO "new_Service" ("createdAt", "destinationDockerId", "dualCerts", "exposePort", "fqdn", "id", "name", "type", "updatedAt", "version") SELECT "createdAt", "destinationDockerId", "dualCerts", "exposePort", "fqdn", "id", "name", "type", "updatedAt", "version" FROM "Service";
DROP TABLE "Service";
ALTER TABLE "new_Service" RENAME TO "Service";
PRAGMA foreign_key_check;
PRAGMA foreign_keys=ON;

View File

@@ -395,6 +395,7 @@ model Service {
dualCerts Boolean @default(false) dualCerts Boolean @default(false)
type String? type String?
version String? version String?
templateVersion String @default("0.0.0")
destinationDockerId String? destinationDockerId String?
createdAt DateTime @default(now()) createdAt DateTime @default(now())
updatedAt DateTime @updatedAt updatedAt DateTime @updatedAt

View File

@@ -16,6 +16,7 @@ import fs from 'fs/promises';
import { verifyRemoteDockerEngineFn } from './routes/api/v1/destinations/handlers'; import { verifyRemoteDockerEngineFn } from './routes/api/v1/destinations/handlers';
import { checkContainer } from './lib/docker'; import { checkContainer } from './lib/docker';
import { migrateServicesToNewTemplate } from './lib'; import { migrateServicesToNewTemplate } from './lib';
import { getTemplates } from './lib/services';
declare module 'fastify' { declare module 'fastify' {
interface FastifyInstance { interface FastifyInstance {
config: { config: {
@@ -124,13 +125,13 @@ const host = '0.0.0.0';
} }
}) })
try { try {
const templateYaml = await axios.get('https://gist.githubusercontent.com/andrasbacsai/701c450ef4272a929215cab11d737e3d/raw/4f021329d22934b90c5d67a0e49839a32bd629fd/template.yaml') const templateJson = await getTemplates()
const templateJson = yaml.load(templateYaml.data)
if (isDev) { if (isDev) {
await fs.writeFile('./template.json', JSON.stringify(templateJson, null, 2)) await fs.writeFile('./template.json', JSON.stringify(templateJson, null, 2))
} else { } else {
await fs.writeFile('/app/template.json', JSON.stringify(templateJson, null, 2)) await fs.writeFile('/app/template.json', JSON.stringify(templateJson, null, 2))
} }
await migrateServicesToNewTemplate(templateJson) await migrateServicesToNewTemplate(templateJson)
await fastify.listen({ port, host }) await fastify.listen({ port, host })

View File

@@ -1,31 +1,87 @@
import { decrypt, encrypt, getDomain, prisma } from "./lib/common"; import cuid from "cuid";
import { decrypt, encrypt, generatePassword, getDomain, prisma } from "./lib/common";
import { includeServices } from "./lib/services/common"; import { includeServices } from "./lib/services/common";
export async function migrateServicesToNewTemplate(templates: any) { export async function migrateServicesToNewTemplate(templates: any) {
// This function migrates old hardcoded services to the new template based services // This function migrates old hardcoded services to the new template based services
try { try {
const services = await prisma.service.findMany({ include: includeServices }) const services: any = await prisma.service.findMany({ include: includeServices })
for (const service of services) { for (const service of services) {
const { id } = service
if (!service.type) { if (!service.type) {
continue; continue;
} }
let template = templates.find(t => t.name === service.type.toLowerCase()); let template = templates.find(t => t.name.toLowerCase() === service.type.toLowerCase());
if (template) { if (template) {
console.log(template.variables.find(v => v.name === "_APP_REDIS_HOST"))
template = JSON.parse(JSON.stringify(template).replaceAll('$$id', service.id)) template = JSON.parse(JSON.stringify(template).replaceAll('$$id', service.id))
if (service.type === 'plausibleanalytics' && service.plausibleAnalytics) await plausibleAnalytics(service) if (service.type === 'plausibleanalytics' && service.plausibleAnalytics) await plausibleAnalytics(service, template)
if (service.type === 'fider' && service.fider) await fider(service) if (service.type === 'fider' && service.fider) await fider(service, template)
if (service.type === 'minio' && service.minio) await minio(service) if (service.type === 'minio' && service.minio) await minio(service, template)
if (service.type === 'vscodeserver' && service.vscodeserver) await vscodeserver(service) if (service.type === 'vscodeserver' && service.vscodeserver) await vscodeserver(service, template)
if (service.type === 'wordpress' && service.wordpress) await wordpress(service) if (service.type === 'wordpress' && service.wordpress) await wordpress(service, template)
if (service.type === 'ghost' && service.ghost) await ghost(service) if (service.type === 'ghost' && service.ghost) await ghost(service, template)
if (service.type === 'meilisearch' && service.meiliSearch) await meilisearch(service) if (service.type === 'meilisearch' && service.meiliSearch) await meilisearch(service, template)
if (service.type === 'umami' && service.umami) await umami(service) if (service.type === 'umami' && service.umami) await umami(service, template)
if (service.type === 'hasura' && service.hasura) await hasura(service) if (service.type === 'hasura' && service.hasura) await hasura(service, template)
if (service.type === 'glitchTip' && service.glitchTip) await glitchtip(service) if (service.type === 'glitchTip' && service.glitchTip) await glitchtip(service, template)
if (service.type === 'searxng' && service.searxng) await searxng(service) if (service.type === 'searxng' && service.searxng) await searxng(service, template)
if (service.type === 'weblate' && service.weblate) await weblate(service) if (service.type === 'weblate' && service.weblate) await weblate(service, template)
if (service.type === 'appwrite' && service.appwrite) await appwrite(service, template)
await createVolumes(service, template); await createVolumes(service, template);
if (template.variables.length > 0) {
for (const variable of template.variables) {
const { defaultValue } = variable;
const regex = /^\$\$.*\((\d+)\)$/g;
const length = Number(regex.exec(defaultValue)?.[1]) || undefined
if (variable.defaultValue.startsWith('$$generate_password')) {
variable.value = generatePassword({ length });
} else if (variable.defaultValue.startsWith('$$generate_hex')) {
variable.value = generatePassword({ length, isHex: true });
} else if (variable.defaultValue.startsWith('$$generate_username')) {
variable.value = cuid();
} else {
variable.value = variable.defaultValue || '';
}
}
}
for (const variable of template.variables) {
if (variable.id.startsWith('$$secret_')) {
const found = await prisma.serviceSecret.findFirst({ where: { name: variable.name, serviceId: id } })
if (!found) {
await prisma.serviceSecret.create({
data: { name: variable.name, value: encrypt(variable.value) || '', service: { connect: { id } } }
})
}
}
if (variable.id.startsWith('$$config_')) {
const found = await prisma.serviceSetting.findFirst({ where: { name: variable.name, serviceId: id } })
if (!found) {
await prisma.serviceSetting.create({
data: { name: variable.name, value: variable.value.toString(), variableName: variable.id, service: { connect: { id } } }
})
}
}
}
for (const service of Object.keys(template.services)) {
if (template.services[service].volumes) {
for (const volume of template.services[service].volumes) {
const [volumeName, path] = volume.split(':')
if (!volumeName.startsWith('/')) {
const found = await prisma.servicePersistentStorage.findFirst({ where: { volumeName, serviceId: id } })
if (!found) {
await prisma.servicePersistentStorage.create({
data: { volumeName, path, containerId: service, predefined: true, service: { connect: { id } } }
});
}
}
}
}
}
await prisma.service.update({ where: { id }, data: { templateVersion: template.templateVersion } })
} }
} }
@@ -34,7 +90,28 @@ export async function migrateServicesToNewTemplate(templates: any) {
} }
} }
async function weblate(service: any) { async function appwrite(service: any, template: any) {
const { opensslKeyV1, executorSecret, redisPassword, mariadbUser, mariadbPassword, mariadbRootUserPassword, mariadbDatabase } = service.appwrite
const secrets = [
`_APP_EXECUTOR_SECRET@@@${executorSecret}`,
`_APP_OPENSSL_KEY_V1@@@${opensslKeyV1}`,
`_APP_REDIS_PASS@@@${redisPassword}`,
`MARIADB_PASSWORD@@@${mariadbPassword}`,
`MARIADB_ROOT_PASSWORD@@@${mariadbRootUserPassword}`,
]
const settings = [
`MARIADB_USER@@@${mariadbUser}`,
`MARIADB_DATABASE@@@${mariadbDatabase}`,
]
await migrateSecrets(secrets, service);
await migrateSettings(settings, service, template);
// Remove old service data
// await prisma.service.update({ where: { id: service.id }, data: { wordpress: { delete: true } } })
}
async function weblate(service: any, template: any) {
const { adminPassword, postgresqlUser, postgresqlPassword, postgresqlDatabase } = service.weblate const { adminPassword, postgresqlUser, postgresqlPassword, postgresqlDatabase } = service.weblate
const secrets = [ const secrets = [
@@ -51,13 +128,13 @@ async function weblate(service: any) {
`POSTGRES_PORT@@@5432`, `POSTGRES_PORT@@@5432`,
`REDIS_HOST@@@$$id-redis`, `REDIS_HOST@@@$$id-redis`,
] ]
await migrateSettings(settings, service, template);
await migrateSecrets(secrets, service); await migrateSecrets(secrets, service);
await migrateSettings(settings, service);
// Remove old service data // Remove old service data
// await prisma.service.update({ where: { id: service.id }, data: { wordpress: { delete: true } } }) // await prisma.service.update({ where: { id: service.id }, data: { wordpress: { delete: true } } })
} }
async function searxng(service: any) { async function searxng(service: any, template: any) {
const { secretKey, redisPassword } = service.searxng const { secretKey, redisPassword } = service.searxng
const secrets = [ const secrets = [
@@ -68,13 +145,13 @@ async function searxng(service: any) {
const settings = [ const settings = [
`SEARXNG_BASE_URL@@@$$generate_fqdn` `SEARXNG_BASE_URL@@@$$generate_fqdn`
] ]
await migrateSettings(settings, service, template);
await migrateSecrets(secrets, service); await migrateSecrets(secrets, service);
await migrateSettings(settings, service);
// Remove old service data // Remove old service data
// await prisma.service.update({ where: { id: service.id }, data: { wordpress: { delete: true } } }) // await prisma.service.update({ where: { id: service.id }, data: { wordpress: { delete: true } } })
} }
async function glitchtip(service: any) { async function glitchtip(service: any, template: any) {
const { postgresqlUser, postgresqlPassword, postgresqlDatabase, secretKeyBase, defaultEmail, defaultUsername, defaultPassword, defaultEmailFrom, emailSmtpHost, emailSmtpPort, emailSmtpUser, emailSmtpPassword, emailSmtpUseTls, emailSmtpUseSsl, emailBackend, mailgunApiKey, sendgridApiKey, enableOpenUserRegistration } = service.glitchTip const { postgresqlUser, postgresqlPassword, postgresqlDatabase, secretKeyBase, defaultEmail, defaultUsername, defaultPassword, defaultEmailFrom, emailSmtpHost, emailSmtpPort, emailSmtpUser, emailSmtpPassword, emailSmtpUseTls, emailSmtpUseSsl, emailBackend, mailgunApiKey, sendgridApiKey, enableOpenUserRegistration } = service.glitchTip
const secrets = [ const secrets = [
@@ -101,13 +178,13 @@ async function glitchtip(service: any) {
`DJANGO_SUPERUSER_EMAIL@@@${defaultEmail}`, `DJANGO_SUPERUSER_EMAIL@@@${defaultEmail}`,
`DJANGO_SUPERUSER_USERNAME@@@${defaultUsername}`, `DJANGO_SUPERUSER_USERNAME@@@${defaultUsername}`,
] ]
await migrateSettings(settings, service, template);
await migrateSecrets(secrets, service); await migrateSecrets(secrets, service);
await migrateSettings(settings, service);
// Remove old service data // Remove old service data
// await prisma.service.update({ where: { id: service.id }, data: { wordpress: { delete: true } } }) // await prisma.service.update({ where: { id: service.id }, data: { wordpress: { delete: true } } })
} }
async function hasura(service: any) { async function hasura(service: any, template: any) {
const { postgresqlUser, postgresqlPassword, postgresqlDatabase, graphQLAdminPassword } = service.hasura const { postgresqlUser, postgresqlPassword, postgresqlDatabase, graphQLAdminPassword } = service.hasura
const secrets = [ const secrets = [
@@ -119,13 +196,13 @@ async function hasura(service: any) {
`POSTGRES_USER@@@${postgresqlUser}`, `POSTGRES_USER@@@${postgresqlUser}`,
`POSTGRES_DB@@@${postgresqlDatabase}`, `POSTGRES_DB@@@${postgresqlDatabase}`,
] ]
await migrateSettings(settings, service, template);
await migrateSecrets(secrets, service); await migrateSecrets(secrets, service);
await migrateSettings(settings, service);
// Remove old service data // Remove old service data
// await prisma.service.update({ where: { id: service.id }, data: { wordpress: { delete: true } } }) // await prisma.service.update({ where: { id: service.id }, data: { wordpress: { delete: true } } })
} }
async function umami(service: any) { async function umami(service: any, template: any) {
const { postgresqlUser, postgresqlPassword, postgresqlDatabase, umamiAdminPassword, hashSalt } = service.umami const { postgresqlUser, postgresqlPassword, postgresqlDatabase, umamiAdminPassword, hashSalt } = service.umami
@@ -139,25 +216,26 @@ async function umami(service: any) {
`POSTGRES_USER@@@${postgresqlUser}`, `POSTGRES_USER@@@${postgresqlUser}`,
`POSTGRES_DB@@@${postgresqlDatabase}`, `POSTGRES_DB@@@${postgresqlDatabase}`,
] ]
await migrateSettings(settings, service, template);
await migrateSecrets(secrets, service); await migrateSecrets(secrets, service);
await migrateSettings(settings, service);
// Remove old service data // Remove old service data
// await prisma.service.update({ where: { id: service.id }, data: { wordpress: { delete: true } } }) // await prisma.service.update({ where: { id: service.id }, data: { wordpress: { delete: true } } })
} }
async function meilisearch(service: any) { async function meilisearch(service: any, template: any) {
const { masterKey } = service.meiliSearch const { masterKey } = service.meiliSearch
const secrets = [ const secrets = [
`MEILI_MASTER_KEY@@@${masterKey}`, `MEILI_MASTER_KEY@@@${masterKey}`,
] ]
// await migrateSettings(settings, service, template);
await migrateSecrets(secrets, service); await migrateSecrets(secrets, service);
// Remove old service data // Remove old service data
// await prisma.service.update({ where: { id: service.id }, data: { wordpress: { delete: true } } }) // await prisma.service.update({ where: { id: service.id }, data: { wordpress: { delete: true } } })
} }
async function ghost(service: any) { async function ghost(service: any, template: any) {
const { defaultEmail, defaultPassword, mariadbUser, mariadbPassword, mariadbRootUser, mariadbRootUserPassword, mariadbDatabase } = service.ghost const { defaultEmail, defaultPassword, mariadbUser, mariadbPassword, mariadbRootUser, mariadbRootUserPassword, mariadbDatabase } = service.ghost
const { fqdn } = service const { fqdn } = service
@@ -182,13 +260,13 @@ async function ghost(service: any) {
`url@@@$$generate_fqdn`, `url@@@$$generate_fqdn`,
`GHOST_ENABLE_HTTPS@@@${isHttps ? 'yes' : 'no'}` `GHOST_ENABLE_HTTPS@@@${isHttps ? 'yes' : 'no'}`
] ]
await migrateSettings(settings, service, template);
await migrateSecrets(secrets, service); await migrateSecrets(secrets, service);
await migrateSettings(settings, service);
// Remove old service data // Remove old service data
// await prisma.service.update({ where: { id: service.id }, data: { wordpress: { delete: true } } }) // await prisma.service.update({ where: { id: service.id }, data: { wordpress: { delete: true } } })
} }
async function wordpress(service: any) { async function wordpress(service: any, template: any) {
const { extraConfig, tablePrefix, ownMysql, mysqlHost, mysqlPort, mysqlUser, mysqlPassword, mysqlRootUser, mysqlRootUserPassword, mysqlDatabase, ftpEnabled, ftpUser, ftpPassword, ftpPublicPort, ftpHostKey, ftpHostKeyPrivate } = service.wordpress const { extraConfig, tablePrefix, ownMysql, mysqlHost, mysqlPort, mysqlUser, mysqlPassword, mysqlRootUser, mysqlRootUserPassword, mysqlDatabase, ftpEnabled, ftpUser, ftpPassword, ftpPublicPort, ftpHostKey, ftpHostKeyPrivate } = service.wordpress
const secrets = [ const secrets = [
@@ -213,13 +291,13 @@ async function wordpress(service: any) {
`COOLIFY_FTP_PUBLIC_PORT@@@${ftpPublicPort}`, `COOLIFY_FTP_PUBLIC_PORT@@@${ftpPublicPort}`,
] ]
await migrateSettings(settings, service, template);
await migrateSecrets(secrets, service); await migrateSecrets(secrets, service);
await migrateSettings(settings, service);
// Remove old service data // Remove old service data
// await prisma.service.update({ where: { id: service.id }, data: { wordpress: { delete: true } } }) // await prisma.service.update({ where: { id: service.id }, data: { wordpress: { delete: true } } })
} }
async function vscodeserver(service: any) { async function vscodeserver(service: any, template: any) {
const { password } = service.vscodeserver const { password } = service.vscodeserver
const secrets = [ const secrets = [
@@ -230,7 +308,7 @@ async function vscodeserver(service: any) {
// Remove old service data // Remove old service data
// await prisma.service.update({ where: { id: service.id }, data: { vscodeserver: { delete: true } } }) // await prisma.service.update({ where: { id: service.id }, data: { vscodeserver: { delete: true } } })
} }
async function minio(service: any) { async function minio(service: any, template: any) {
const { rootUser, rootUserPassword, apiFqdn } = service.minio const { rootUser, rootUserPassword, apiFqdn } = service.minio
const secrets = [ const secrets = [
@@ -242,13 +320,13 @@ async function minio(service: any) {
`MINIO_BROWSER_REDIRECT_URL@@@$$generate_fqdn`, `MINIO_BROWSER_REDIRECT_URL@@@$$generate_fqdn`,
`MINIO_DOMAIN@@@$$generate_domain`, `MINIO_DOMAIN@@@$$generate_domain`,
] ]
await migrateSettings(settings, service); await migrateSettings(settings, service, template);
await migrateSecrets(secrets, service); await migrateSecrets(secrets, service);
// Remove old service data // Remove old service data
// await prisma.service.update({ where: { id: service.id }, data: { minio: { delete: true } } }) // await prisma.service.update({ where: { id: service.id }, data: { minio: { delete: true } } })
} }
async function fider(service: any) { async function fider(service: any, template: any) {
const { postgresqlUser, postgresqlPassword, postgresqlDatabase, jwtSecret, emailNoreply, emailMailgunApiKey, emailMailgunDomain, emailMailgunRegion, emailSmtpHost, emailSmtpPort, emailSmtpUser, emailSmtpPassword, emailSmtpEnableStartTls } = service.fider const { postgresqlUser, postgresqlPassword, postgresqlDatabase, jwtSecret, emailNoreply, emailMailgunApiKey, emailMailgunDomain, emailMailgunRegion, emailSmtpHost, emailSmtpPort, emailSmtpUser, emailSmtpPassword, emailSmtpEnableStartTls } = service.fider
const secrets = [ const secrets = [
@@ -270,14 +348,14 @@ async function fider(service: any) {
`POSTGRES_USER@@@${postgresqlUser}`, `POSTGRES_USER@@@${postgresqlUser}`,
`POSTGRES_DB@@@${postgresqlDatabase}`, `POSTGRES_DB@@@${postgresqlDatabase}`,
] ]
await migrateSettings(settings, service); await migrateSettings(settings, service, template);
await migrateSecrets(secrets, service); await migrateSecrets(secrets, service);
// Remove old service data // Remove old service data
// await prisma.service.update({ where: { id: service.id }, data: { fider: { delete: true } } }) // await prisma.service.update({ where: { id: service.id }, data: { fider: { delete: true } } })
} }
async function plausibleAnalytics(service: any) { async function plausibleAnalytics(service: any, template: any) {
const { email, username, password, postgresqlUser, postgresqlPassword, postgresqlDatabase, secretKeyBase, scriptName } = service.plausibleAnalytics; const { email, username, password, postgresqlUser, postgresqlPassword, postgresqlDatabase, secretKeyBase, scriptName } = service.plausibleAnalytics;
const settings = [ const settings = [
@@ -296,14 +374,14 @@ async function plausibleAnalytics(service: any) {
`POSTGRES_PASSWORD@@@${postgresqlPassword}`, `POSTGRES_PASSWORD@@@${postgresqlPassword}`,
`DATABASE_URL@@@${encrypt(`postgres://${postgresqlUser}:${decrypt(postgresqlPassword)}@$$generate_fqdn:5432/${postgresqlDatabase}`)}`, `DATABASE_URL@@@${encrypt(`postgres://${postgresqlUser}:${decrypt(postgresqlPassword)}@$$generate_fqdn:5432/${postgresqlDatabase}`)}`,
] ]
await migrateSettings(settings, service); await migrateSettings(settings, service, template);
await migrateSecrets(secrets, service); await migrateSecrets(secrets, service);
// Remove old service data // Remove old service data
// await prisma.service.update({ where: { id: service.id }, data: { plausibleAnalytics: { delete: true } } }) // await prisma.service.update({ where: { id: service.id }, data: { plausibleAnalytics: { delete: true } } })
} }
async function migrateSettings(settings: any[], service: any) { async function migrateSettings(settings: any[], service: any, template: any) {
for (const setting of settings) { for (const setting of settings) {
if (!setting) continue; if (!setting) continue;
let [name, value] = setting.split('@@@') let [name, value] = setting.split('@@@')
@@ -311,7 +389,8 @@ async function migrateSettings(settings: any[], service: any) {
continue; continue;
} }
// console.log('Migrating setting', name, value, 'for service', service.id, ', service name:', service.name) // console.log('Migrating setting', name, value, 'for service', service.id, ', service name:', service.name)
await prisma.serviceSetting.findFirst({ where: { name, serviceId: service.id } }) || await prisma.serviceSetting.create({ data: { name, value, service: { connect: { id: service.id } } } }) const variableName = template.variables.find((v: any) => v.name === name)?.id
await prisma.serviceSetting.findFirst({ where: { name, serviceId: service.id } }) || await prisma.serviceSetting.create({ data: { name, value, variableName, service: { connect: { id: service.id } } } })
} }
} }
async function migrateSecrets(secrets: any[], service: any) { async function migrateSecrets(secrets: any[], service: any) {

View File

@@ -282,7 +282,6 @@ export async function saveServiceType(request: FastifyRequest<SaveServiceType>,
}) })
} }
} }
} }
for (const service of Object.keys(foundTemplate.services)) { for (const service of Object.keys(foundTemplate.services)) {
if (foundTemplate.services[service].volumes) { if (foundTemplate.services[service].volumes) {
@@ -299,7 +298,7 @@ export async function saveServiceType(request: FastifyRequest<SaveServiceType>,
} }
} }
} }
await prisma.service.update({ where: { id }, data: { type, version: foundTemplate.defaultVersion } }) await prisma.service.update({ where: { id }, data: { type, version: foundTemplate.defaultVersion, templateVersion: foundTemplate.templateVersion } })
return reply.code(201).send() return reply.code(201).send()
} else { } else {
throw { status: 404, message: 'Service type not found.' } throw { status: 404, message: 'Service type not found.' }