diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index 5ed8110fe..61f1b9d3c 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -6,11 +6,14 @@ import cookie from '@fastify/cookie'; import multipart from '@fastify/multipart'; import path, { join } from 'path'; import autoLoad from '@fastify/autoload'; -import { asyncExecShell, createRemoteEngineConfiguration, getDomain, isDev, listSettings, prisma, version } from './lib/common'; +import { asyncExecShell, cleanupDockerStorage, createRemoteEngineConfiguration, decrypt, executeDockerCmd, executeSSHCmd, generateDatabaseConfiguration, getDomain, isDev, listSettings, prisma, startTraefikProxy, startTraefikTCPProxy, version } from './lib/common'; import { scheduler } from './lib/scheduler'; import { compareVersions } from 'compare-versions'; import Graceful from '@ladjs/graceful' +import axios from 'axios'; +import fs from 'fs/promises'; import { verifyRemoteDockerEngineFn } from './routes/api/v1/destinations/handlers'; +import { checkContainer } from './lib/docker'; declare module 'fastify' { interface FastifyInstance { config: { @@ -130,29 +133,28 @@ const host = '0.0.0.0'; if (!scheduler.workers.has('deployApplication')) { scheduler.run('deployApplication'); } - if (!scheduler.workers.has('infrastructure')) { - scheduler.run('infrastructure'); - } }, 2000) // autoUpdater setInterval(async () => { - scheduler.workers.has('infrastructure') && scheduler.workers.get('infrastructure').postMessage("action:autoUpdater") + await autoUpdater() }, 60000 * 15) // cleanupStorage setInterval(async () => { - scheduler.workers.has('infrastructure') && scheduler.workers.get('infrastructure').postMessage("action:cleanupStorage") + await cleanupStorage() }, 60000 * 10) // checkProxies and checkFluentBit setInterval(async () => { - scheduler.workers.has('infrastructure') && scheduler.workers.get('infrastructure').postMessage("action:checkProxies") - scheduler.workers.has('infrastructure') && scheduler.workers.get('infrastructure').postMessage("action:checkFluentBit") + console.log('checkProxies') + await checkProxies(); + console.log('checkFluentBit') + await checkFluentBit(); }, 10000) setInterval(async () => { - scheduler.workers.has('infrastructure') && scheduler.workers.get('infrastructure').postMessage("action:copySSLCertificates") + await copySSLCertificates(); }, 2000) await Promise.all([ @@ -226,3 +228,233 @@ async function configureRemoteDockers() { console.log(error) } } + +async function autoUpdater() { + try { + const currentVersion = version; + const { data: versions } = await axios + .get( + `https://get.coollabs.io/versions.json` + , { + params: { + appId: process.env['COOLIFY_APP_ID'] || undefined, + version: currentVersion + } + }) + const latestVersion = versions['coolify'].main.version; + const isUpdateAvailable = compareVersions(latestVersion, currentVersion); + if (isUpdateAvailable === 1) { + const activeCount = 0 + if (activeCount === 0) { + if (!isDev) { + const { isAutoUpdateEnabled } = await prisma.setting.findFirst(); + if (isAutoUpdateEnabled) { + await asyncExecShell(`docker pull coollabsio/coolify:${latestVersion}`); + await asyncExecShell(`env | grep COOLIFY > .env`); + await asyncExecShell( + `sed -i '/COOLIFY_AUTO_UPDATE=/cCOOLIFY_AUTO_UPDATE=${isAutoUpdateEnabled}' .env` + ); + await asyncExecShell( + `docker run --rm -tid --env-file .env -v /var/run/docker.sock:/var/run/docker.sock -v coolify-db coollabsio/coolify:${latestVersion} /bin/sh -c "env | grep COOLIFY > .env && echo 'TAG=${latestVersion}' >> .env && docker stop -t 0 coolify coolify-fluentbit && docker rm coolify coolify-fluentbit && docker compose pull && docker compose up -d --force-recreate"` + ); + } + } else { + console.log('Updating (not really in dev mode).'); + } + } + } + } catch (error) { } +} + +async function checkFluentBit() { + if (!isDev) { + const engine = '/var/run/docker.sock'; + const { id } = await prisma.destinationDocker.findFirst({ + where: { engine, network: 'coolify' } + }); + const { found } = await checkContainer({ dockerId: id, container: 'coolify-fluentbit' }); + if (!found) { + await asyncExecShell(`env | grep COOLIFY > .env`); + await asyncExecShell(`docker compose up -d fluent-bit`); + } + } +} +async function checkProxies() { + try { + const { default: isReachable } = await import('is-port-reachable'); + let portReachable; + + const { arch, ipv4, ipv6 } = await listSettings(); + + // Coolify Proxy local + const engine = '/var/run/docker.sock'; + const localDocker = await prisma.destinationDocker.findFirst({ + where: { engine, network: 'coolify', isCoolifyProxyUsed: true } + }); + if (localDocker) { + portReachable = await isReachable(80, { host: ipv4 || ipv6 }) + if (!portReachable) { + await startTraefikProxy(localDocker.id); + } + } + // Coolify Proxy remote + const remoteDocker = await prisma.destinationDocker.findMany({ + where: { remoteEngine: true, remoteVerified: true } + }); + if (remoteDocker.length > 0) { + for (const docker of remoteDocker) { + if (docker.isCoolifyProxyUsed) { + portReachable = await isReachable(80, { host: docker.remoteIpAddress }) + if (!portReachable) { + await startTraefikProxy(docker.id); + } + } + try { + await createRemoteEngineConfiguration(docker.id) + } catch (error) { } + } + } + // TCP Proxies + const databasesWithPublicPort = await prisma.database.findMany({ + where: { publicPort: { not: null } }, + include: { settings: true, destinationDocker: true } + }); + for (const database of databasesWithPublicPort) { + const { destinationDockerId, destinationDocker, publicPort, id } = database; + if (destinationDockerId && destinationDocker.isCoolifyProxyUsed) { + const { privatePort } = generateDatabaseConfiguration(database, arch); + await startTraefikTCPProxy(destinationDocker, id, publicPort, privatePort); + } + } + const wordpressWithFtp = await prisma.wordpress.findMany({ + where: { ftpPublicPort: { not: null } }, + include: { service: { include: { destinationDocker: true } } } + }); + for (const ftp of wordpressWithFtp) { + const { service, ftpPublicPort } = ftp; + const { destinationDockerId, destinationDocker, id } = service; + if (destinationDockerId && destinationDocker.isCoolifyProxyUsed) { + await startTraefikTCPProxy(destinationDocker, id, ftpPublicPort, 22, 'wordpressftp'); + } + } + + // HTTP Proxies + const minioInstances = await prisma.minio.findMany({ + where: { publicPort: { not: null } }, + include: { service: { include: { destinationDocker: true } } } + }); + for (const minio of minioInstances) { + const { service, publicPort } = minio; + const { destinationDockerId, destinationDocker, id } = service; + if (destinationDockerId && destinationDocker.isCoolifyProxyUsed) { + await startTraefikTCPProxy(destinationDocker, id, publicPort, 9000); + } + } + } catch (error) { + + } +} + +async function copySSLCertificates() { + try { + const pAll = await import('p-all'); + const actions = [] + const certificates = await prisma.certificate.findMany({ include: { team: true } }) + const teamIds = certificates.map(c => c.teamId) + const destinations = await prisma.destinationDocker.findMany({ where: { isCoolifyProxyUsed: true, teams: { some: { id: { in: [...teamIds] } } } } }) + for (const certificate of certificates) { + const { id, key, cert } = certificate + const decryptedKey = decrypt(key) + await fs.writeFile(`/tmp/${id}-key.pem`, decryptedKey) + await fs.writeFile(`/tmp/${id}-cert.pem`, cert) + for (const destination of destinations) { + if (destination.remoteEngine) { + if (destination.remoteVerified) { + const { id: dockerId, remoteIpAddress } = destination + actions.push(async () => copyRemoteCertificates(id, dockerId, remoteIpAddress)) + } + } else { + actions.push(async () => copyLocalCertificates(id)) + } + } + } + await pAll.default(actions, { concurrency: 1 }) + } catch (error) { + console.log(error) + } finally { + await asyncExecShell(`find /tmp/ -maxdepth 1 -type f -name '*-*.pem' -delete`) + } +} + +async function copyRemoteCertificates(id: string, dockerId: string, remoteIpAddress: string) { + try { + await asyncExecShell(`scp /tmp/${id}-cert.pem /tmp/${id}-key.pem ${remoteIpAddress}:/tmp/`) + await executeSSHCmd({ dockerId, command: `docker exec coolify-proxy sh -c 'test -d /etc/traefik/acme/custom/ || mkdir -p /etc/traefik/acme/custom/'` }) + await executeSSHCmd({ dockerId, command: `docker cp /tmp/${id}-key.pem coolify-proxy:/etc/traefik/acme/custom/` }) + await executeSSHCmd({ dockerId, command: `docker cp /tmp/${id}-cert.pem coolify-proxy:/etc/traefik/acme/custom/` }) + } catch (error) { + console.log({ error }) + } +} +async function copyLocalCertificates(id: string) { + try { + await asyncExecShell(`docker exec coolify-proxy sh -c 'test -d /etc/traefik/acme/custom/ || mkdir -p /etc/traefik/acme/custom/'`) + await asyncExecShell(`docker cp /tmp/${id}-key.pem coolify-proxy:/etc/traefik/acme/custom/`) + await asyncExecShell(`docker cp /tmp/${id}-cert.pem coolify-proxy:/etc/traefik/acme/custom/`) + } catch (error) { + console.log({ error }) + } +} + +async function cleanupStorage() { + const destinationDockers = await prisma.destinationDocker.findMany(); + let enginesDone = new Set() + for (const destination of destinationDockers) { + if (enginesDone.has(destination.engine) || enginesDone.has(destination.remoteIpAddress)) return + if (destination.engine) enginesDone.add(destination.engine) + if (destination.remoteIpAddress) enginesDone.add(destination.remoteIpAddress) + + let lowDiskSpace = false; + try { + let stdout = null + if (!isDev) { + const output = await executeDockerCmd({ dockerId: destination.id, command: `CONTAINER=$(docker ps -lq | head -1) && docker exec $CONTAINER sh -c 'df -kPT /'` }) + stdout = output.stdout; + } else { + const output = await asyncExecShell( + `df -kPT /` + ); + stdout = output.stdout; + } + let lines = stdout.trim().split('\n'); + let header = lines[0]; + let regex = + /^Filesystem\s+|Type\s+|1024-blocks|\s+Used|\s+Available|\s+Capacity|\s+Mounted on\s*$/g; + const boundaries = []; + let match; + + while ((match = regex.exec(header))) { + boundaries.push(match[0].length); + } + + boundaries[boundaries.length - 1] = -1; + const data = lines.slice(1).map((line) => { + const cl = boundaries.map((boundary) => { + const column = boundary > 0 ? line.slice(0, boundary) : line; + line = line.slice(boundary); + return column.trim(); + }); + return { + capacity: Number.parseInt(cl[5], 10) / 100 + }; + }); + if (data.length > 0) { + const { capacity } = data[0]; + if (capacity > 0.8) { + lowDiskSpace = true; + } + } + } catch (error) { } + await cleanupDockerStorage(destination.id, lowDiskSpace, false) + } +} \ No newline at end of file diff --git a/apps/api/src/lib/scheduler.ts b/apps/api/src/lib/scheduler.ts index 743463757..20add2b06 100644 --- a/apps/api/src/lib/scheduler.ts +++ b/apps/api/src/lib/scheduler.ts @@ -11,15 +11,14 @@ const options: any = { defaultExtension: 'js', logger: new Cabin(), // logger: false, - workerMessageHandler: async ({ name, message }) => { - if (name === 'deployApplication' && message?.deploying) { - if (scheduler.workers.has('autoUpdater') || scheduler.workers.has('cleanupStorage')) { - scheduler.workers.get('deployApplication').postMessage('cancel') - } - } - }, + // workerMessageHandler: async ({ name, message }) => { + // if (name === 'deployApplication' && message?.deploying) { + // if (scheduler.workers.has('autoUpdater') || scheduler.workers.has('cleanupStorage')) { + // scheduler.workers.get('deployApplication').postMessage('cancel') + // } + // } + // }, jobs: [ - { name: 'infrastructure' }, { name: 'deployApplication' }, ], };