dave/coolify
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

feat: docker compose

Browse Source
This commit is contained in:
Andras Bacsai
2022-10-06 15:51:08 +02:00
parent 0c4850b91d
commit 9bb125cebd
14 changed files with 227 additions and 473 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
apps/api/src/jobs/deployApplication.ts
View File

@@ -85,6 +85,7 @@ import * as buildpacks from '../lib/buildPacks';
baseDirectory,
publishDirectory,
dockerFileLocation,
dockerComposeConfiguration,
denoMainFile
} = application
const currentHash = crypto
@@ -112,17 +113,6 @@ import * as buildpacks from '../lib/buildPacks';
)
.digest('hex');
const { debug } = settings;
// if (concurrency === 1) {
// await prisma.build.updateMany({
// where: {
// status: { in: ['queued', 'running'] },
// id: { not: buildId },
// applicationId,
// createdAt: { lt: new Date(new Date().getTime() - 10 * 1000) }
// },
// data: { status: 'failed' }
// });
// }
let imageId = applicationId;
let domain = getDomain(fqdn);
const volumes =
@@ -138,6 +128,10 @@ import * as buildpacks from '../lib/buildPacks';
repository = sourceRepository || repository;
}
try {
dockerComposeConfiguration = JSON.parse(dockerComposeConfiguration)
} catch (error) { }
let deployNeeded = true;
let destinationType;
@@ -264,6 +258,7 @@ import * as buildpacks from '../lib/buildPacks';
pythonModule,
pythonVariable,
dockerFileLocation,
dockerComposeConfiguration,
denoMainFile,
denoOptions,
baseImage,

296
apps/api/src/jobs/infrastructure.ts
View File

@@ -1,296 +0,0 @@
import { parentPort } from 'node:worker_threads';
import axios from 'axios';
import { compareVersions } from 'compare-versions';
import { asyncExecShell, cleanupDockerStorage, executeDockerCmd, isDev, prisma, startTraefikTCPProxy, generateDatabaseConfiguration, startTraefikProxy, listSettings, version, createRemoteEngineConfiguration, decrypt, executeSSHCmd } from '../lib/common';
import { checkContainer } from '../lib/docker';
import fs from 'fs/promises'
async function autoUpdater() {
try {
const currentVersion = version;
const { data: versions } = await axios
.get(
`https://get.coollabs.io/versions.json`
, {
params: {
appId: process.env['COOLIFY_APP_ID'] || undefined,
version: currentVersion
}
})
const latestVersion = versions['coolify'].main.version;
const isUpdateAvailable = compareVersions(latestVersion, currentVersion);
if (isUpdateAvailable === 1) {
const activeCount = 0
if (activeCount === 0) {
if (!isDev) {
const { isAutoUpdateEnabled } = await prisma.setting.findFirst();
if (isAutoUpdateEnabled) {
await asyncExecShell(`docker pull coollabsio/coolify:${latestVersion}`);
await asyncExecShell(`env | grep COOLIFY > .env`);
await asyncExecShell(
`sed -i '/COOLIFY_AUTO_UPDATE=/cCOOLIFY_AUTO_UPDATE=${isAutoUpdateEnabled}' .env`
);
await asyncExecShell(
`docker run --rm -tid --env-file .env -v /var/run/docker.sock:/var/run/docker.sock -v coolify-db coollabsio/coolify:${latestVersion} /bin/sh -c "env | grep COOLIFY > .env && echo 'TAG=${latestVersion}' >> .env && docker stop -t 0 coolify coolify-fluentbit && docker rm coolify coolify-fluentbit && docker compose pull && docker compose up -d --force-recreate"`
);
}
} else {
console.log('Updating (not really in dev mode).');
}
}
}
} catch (error) { }
}
async function checkFluentBit() {
if (!isDev) {
const engine = '/var/run/docker.sock';
const { id } = await prisma.destinationDocker.findFirst({
where: { engine, network: 'coolify' }
});
const { found } = await checkContainer({ dockerId: id, container: 'coolify-fluentbit' });
if (!found) {
await asyncExecShell(`env | grep COOLIFY > .env`);
await asyncExecShell(`docker compose up -d fluent-bit`);
}
}
}
async function copyRemoteCertificates(id: string, dockerId: string, remoteIpAddress: string) {
try {
await asyncExecShell(`scp /tmp/${id}-cert.pem /tmp/${id}-key.pem ${remoteIpAddress}:/tmp/`)
await executeSSHCmd({ dockerId, command: `docker exec coolify-proxy sh -c 'test -d /etc/traefik/acme/custom/ || mkdir -p /etc/traefik/acme/custom/'` })
await executeSSHCmd({ dockerId, command: `docker cp /tmp/${id}-key.pem coolify-proxy:/etc/traefik/acme/custom/` })
await executeSSHCmd({ dockerId, command: `docker cp /tmp/${id}-cert.pem coolify-proxy:/etc/traefik/acme/custom/` })
} catch (error) {
console.log({ error })
}
}
async function copyLocalCertificates(id: string) {
try {
await asyncExecShell(`docker exec coolify-proxy sh -c 'test -d /etc/traefik/acme/custom/ || mkdir -p /etc/traefik/acme/custom/'`)
await asyncExecShell(`docker cp /tmp/${id}-key.pem coolify-proxy:/etc/traefik/acme/custom/`)
await asyncExecShell(`docker cp /tmp/${id}-cert.pem coolify-proxy:/etc/traefik/acme/custom/`)
} catch (error) {
console.log({ error })
}
}
async function copySSLCertificates() {
try {
const pAll = await import('p-all');
const actions = []
const certificates = await prisma.certificate.findMany({ include: { team: true } })
const teamIds = certificates.map(c => c.teamId)
const destinations = await prisma.destinationDocker.findMany({ where: { isCoolifyProxyUsed: true, teams: { some: { id: { in: [...teamIds] } } } } })
for (const certificate of certificates) {
const { id, key, cert } = certificate
const decryptedKey = decrypt(key)
await fs.writeFile(`/tmp/${id}-key.pem`, decryptedKey)
await fs.writeFile(`/tmp/${id}-cert.pem`, cert)
for (const destination of destinations) {
if (destination.remoteEngine) {
if (destination.remoteVerified) {
const { id: dockerId, remoteIpAddress } = destination
actions.push(async () => copyRemoteCertificates(id, dockerId, remoteIpAddress))
}
} else {
actions.push(async () => copyLocalCertificates(id))
}
}
}
await pAll.default(actions, { concurrency: 1 })
} catch (error) {
console.log(error)
} finally {
await asyncExecShell(`find /tmp/ -maxdepth 1 -type f -name '*-*.pem' -delete`)
}
}
async function checkProxies() {
try {
const { default: isReachable } = await import('is-port-reachable');
let portReachable;
const { arch, ipv4, ipv6 } = await listSettings();
// Coolify Proxy local
const engine = '/var/run/docker.sock';
const localDocker = await prisma.destinationDocker.findFirst({
where: { engine, network: 'coolify', isCoolifyProxyUsed: true }
});
if (localDocker) {
portReachable = await isReachable(80, { host: ipv4 || ipv6 })
if (!portReachable) {
await startTraefikProxy(localDocker.id);
}
}
// Coolify Proxy remote
const remoteDocker = await prisma.destinationDocker.findMany({
where: { remoteEngine: true, remoteVerified: true }
});
if (remoteDocker.length > 0) {
for (const docker of remoteDocker) {
if (docker.isCoolifyProxyUsed) {
portReachable = await isReachable(80, { host: docker.remoteIpAddress })
if (!portReachable) {
await startTraefikProxy(docker.id);
}
}
try {
await createRemoteEngineConfiguration(docker.id)
} catch (error) { }
}
}
// TCP Proxies
const databasesWithPublicPort = await prisma.database.findMany({
where: { publicPort: { not: null } },
include: { settings: true, destinationDocker: true }
});
for (const database of databasesWithPublicPort) {
const { destinationDockerId, destinationDocker, publicPort, id } = database;
if (destinationDockerId && destinationDocker.isCoolifyProxyUsed) {
const { privatePort } = generateDatabaseConfiguration(database, arch);
await startTraefikTCPProxy(destinationDocker, id, publicPort, privatePort);
}
}
const wordpressWithFtp = await prisma.wordpress.findMany({
where: { ftpPublicPort: { not: null } },
include: { service: { include: { destinationDocker: true } } }
});
for (const ftp of wordpressWithFtp) {
const { service, ftpPublicPort } = ftp;
const { destinationDockerId, destinationDocker, id } = service;
if (destinationDockerId && destinationDocker.isCoolifyProxyUsed) {
await startTraefikTCPProxy(destinationDocker, id, ftpPublicPort, 22, 'wordpressftp');
}
}
// HTTP Proxies
const minioInstances = await prisma.minio.findMany({
where: { publicPort: { not: null } },
include: { service: { include: { destinationDocker: true } } }
});
for (const minio of minioInstances) {
const { service, publicPort } = minio;
const { destinationDockerId, destinationDocker, id } = service;
if (destinationDockerId && destinationDocker.isCoolifyProxyUsed) {
await startTraefikTCPProxy(destinationDocker, id, publicPort, 9000);
}
}
} catch (error) {
}
}
async function cleanupPrismaEngines() {
if (!isDev) {
try {
const { stdout } = await asyncExecShell(`ps -ef | grep /app/prisma-engines/query-engine | grep -v grep | wc -l | xargs`)
if (stdout.trim() != null && stdout.trim() != '' && Number(stdout.trim()) > 1) {
await asyncExecShell(`killall -q -e /app/prisma-engines/query-engine -o 1m`)
}
} catch (error) { }
}
}
async function cleanupStorage() {
const destinationDockers = await prisma.destinationDocker.findMany();
let enginesDone = new Set()
for (const destination of destinationDockers) {
if (enginesDone.has(destination.engine) || enginesDone.has(destination.remoteIpAddress)) return
if (destination.engine) enginesDone.add(destination.engine)
if (destination.remoteIpAddress) enginesDone.add(destination.remoteIpAddress)
let lowDiskSpace = false;
try {
let stdout = null
if (!isDev) {
const output = await executeDockerCmd({ dockerId: destination.id, command: `CONTAINER=$(docker ps -lq | head -1) && docker exec $CONTAINER sh -c 'df -kPT /'` })
stdout = output.stdout;
} else {
const output = await asyncExecShell(
`df -kPT /`
);
stdout = output.stdout;
}
let lines = stdout.trim().split('\n');
let header = lines[0];
let regex =
/^Filesystem\s+|Type\s+|1024-blocks|\s+Used|\s+Available|\s+Capacity|\s+Mounted on\s*$/g;
const boundaries = [];
let match;
while ((match = regex.exec(header))) {
boundaries.push(match[0].length);
}
boundaries[boundaries.length - 1] = -1;
const data = lines.slice(1).map((line) => {
const cl = boundaries.map((boundary) => {
const column = boundary > 0 ? line.slice(0, boundary) : line;
line = line.slice(boundary);
return column.trim();
});
return {
capacity: Number.parseInt(cl[5], 10) / 100
};
});
if (data.length > 0) {
const { capacity } = data[0];
if (capacity > 0.8) {
lowDiskSpace = true;
}
}
} catch (error) { }
await cleanupDockerStorage(destination.id, lowDiskSpace, false)
}
}
(async () => {
let status = {
cleanupStorage: false,
autoUpdater: false,
copySSLCertificates: false,
}
if (parentPort) {
parentPort.on('message', async (message) => {
if (parentPort) {
if (message === 'error') throw new Error('oops');
if (message === 'cancel') {
parentPort.postMessage('cancelled');
process.exit(1);
}
if (message === 'action:cleanupStorage') {
if (!status.autoUpdater) {
status.cleanupStorage = true
await cleanupStorage();
status.cleanupStorage = false
}
return;
}
if (message === 'action:cleanupPrismaEngines') {
await cleanupPrismaEngines();
return;
}
if (message === 'action:checkProxies') {
await checkProxies();
return;
}
if (message === 'action:checkFluentBit') {
await checkFluentBit();
return;
}
if (message === 'action:copySSLCertificates') {
if (!status.copySSLCertificates) {
status.copySSLCertificates = true
await copySSLCertificates();
status.copySSLCertificates = false
}
return;
}
if (message === 'action:autoUpdater') {
if (!status.cleanupStorage) {
status.autoUpdater = true
await autoUpdater();
status.autoUpdater = false
}
return;
}
}
});
} else process.exit(0);
})();

6
apps/api/src/lib/buildPacks/compose.ts
View File

@@ -16,7 +16,8 @@ export default async function (data) {
baseDirectory,
secrets,
pullmergeRequestId,
port
port,
dockerComposeConfiguration
} = data
const fileYml = `${workdir}${baseDirectory}/docker-compose.yml`;
const fileYaml = `${workdir}${baseDirectory}/docker-compose.yaml`;
@@ -76,6 +77,9 @@ export default async function (data) {
value['env_file'] = envFound ? [`${workdir}/.env`] : []
value['labels'] = labels
value['volumes'] = volumes
if (dockerComposeConfiguration[key].port) {
value['expose'] = [dockerComposeConfiguration[key].port]
}
if (value['networks']?.length > 0) {
value['networks'].forEach((network) => {
networks[network] = {

6
apps/api/src/routes/api/v1/applications/handlers.ts
View File

@@ -1223,7 +1223,7 @@ export async function getPreviews(request: FastifyRequest<OnlyId>) {
export async function getApplicationLogs(request: FastifyRequest<GetApplicationLogs>) {
try {
const { id } = request.params;
const { id, containerId } = request.params;
let { since = 0 } = request.query
if (since !== 0) {
since = day(since).unix();
@@ -1234,10 +1234,8 @@ export async function getApplicationLogs(request: FastifyRequest<GetApplicationL
});
if (destinationDockerId) {
try {
// const found = await checkContainer({ dockerId, container: id })
// if (found) {
const { default: ansi } = await import('strip-ansi')
const { stdout, stderr } = await executeDockerCmd({ dockerId, command: `docker logs --since ${since} --tail 5000 --timestamps ${id}` })
const { stdout, stderr } = await executeDockerCmd({ dockerId, command: `docker logs --since ${since} --tail 5000 --timestamps ${containerId}` })
const stripLogsStdout = stdout.toString().split('\n').map((l) => ansi(l)).filter((a) => a);
const stripLogsStderr = stderr.toString().split('\n').map((l) => ansi(l)).filter((a) => a);
const logs = stripLogsStderr.concat(stripLogsStdout)

3
apps/api/src/routes/api/v1/applications/index.ts
View File

@@ -45,7 +45,8 @@ const root: FastifyPluginAsync = async (fastify): Promise<void> => {
fastify.get<RestartPreviewApplication>('/:id/previews/:pullmergeRequestId/status', async (request) => await getPreviewStatus(request));
fastify.post<RestartPreviewApplication>('/:id/previews/:pullmergeRequestId/restart', async (request, reply) => await restartPreview(request, reply));
fastify.get<GetApplicationLogs>('/:id/logs', async (request) => await getApplicationLogs(request));
// fastify.get<GetApplicationLogs>('/:id/logs', async (request) => await getApplicationLogs(request));
fastify.get<GetApplicationLogs>('/:id/logs/:containerId', async (request) => await getApplicationLogs(request));
fastify.get<GetBuilds>('/:id/logs/build', async (request) => await getBuilds(request));
fastify.get<GetBuildIdLogs>('/:id/logs/build/:buildId', async (request) => await getBuildIdLogs(request));

6
apps/api/src/routes/api/v1/applications/types.ts
View File

@@ -87,7 +87,11 @@ export interface DeleteStorage extends OnlyId {
path: string,
}
}
export interface GetApplicationLogs extends OnlyId {
export interface GetApplicationLogs {
Params: {
id: string,
containerId: string
}
Querystring: {
since: number,
}