Merge branch 'main' into ui

This commit is contained in:
Andras Bacsai
2022-09-19 08:57:48 +02:00
53 changed files with 1312 additions and 927 deletions

View File

@@ -38,8 +38,16 @@ import * as buildpacks from '../lib/buildPacks';
for (const queueBuild of queuedBuilds) {
actions.push(async () => {
let application = await prisma.application.findUnique({ where: { id: queueBuild.applicationId }, include: { destinationDocker: true, gitSource: { include: { githubApp: true, gitlabApp: true } }, persistentStorage: true, secrets: true, settings: true, teams: true } })
let { id: buildId, type, sourceBranch = null, pullmergeRequestId = null, forceRebuild } = queueBuild
let { id: buildId, type, sourceBranch = null, pullmergeRequestId = null, previewApplicationId = null, forceRebuild } = queueBuild
application = decryptApplication(application)
const originalApplicationId = application.id
if (pullmergeRequestId) {
const previewApplications = await prisma.previewApplication.findMany({ where: { applicationId: originalApplicationId, pullmergeRequestId } })
if (previewApplications.length > 0) {
previewApplicationId = previewApplications[0].id
}
}
const usableApplicationId = previewApplicationId || originalApplicationId
try {
if (queueBuild.status === 'running') {
await saveBuildLog({ line: 'Building halted, restarting...', buildId, applicationId: application.id });
@@ -104,17 +112,17 @@ import * as buildpacks from '../lib/buildPacks';
)
.digest('hex');
const { debug } = settings;
if (concurrency === 1) {
await prisma.build.updateMany({
where: {
status: { in: ['queued', 'running'] },
id: { not: buildId },
applicationId,
createdAt: { lt: new Date(new Date().getTime() - 10 * 1000) }
},
data: { status: 'failed' }
});
}
// if (concurrency === 1) {
// await prisma.build.updateMany({
// where: {
// status: { in: ['queued', 'running'] },
// id: { not: buildId },
// applicationId,
// createdAt: { lt: new Date(new Date().getTime() - 10 * 1000) }
// },
// data: { status: 'failed' }
// });
// }
let imageId = applicationId;
let domain = getDomain(fqdn);
const volumes =
@@ -338,10 +346,15 @@ import * as buildpacks from '../lib/buildPacks';
await saveBuildLog({ line: 'Deployment successful!', buildId, applicationId });
} catch (error) {
await saveBuildLog({ line: error, buildId, applicationId });
await prisma.build.updateMany({
where: { id: buildId, status: { in: ['queued', 'running'] } },
data: { status: 'failed' }
});
const foundBuild = await prisma.build.findUnique({ where: { id: buildId } })
if (foundBuild) {
await prisma.build.update({
where: { id: buildId },
data: {
status: 'failed'
}
});
}
throw new Error(error);
}
await saveBuildLog({ line: 'Proxy will be updated shortly.', buildId, applicationId });
@@ -353,10 +366,15 @@ import * as buildpacks from '../lib/buildPacks';
}
}
catch (error) {
await prisma.build.updateMany({
where: { id: buildId, status: { in: ['queued', 'running'] } },
data: { status: 'failed' }
});
const foundBuild = await prisma.build.findUnique({ where: { id: buildId } })
if (foundBuild) {
await prisma.build.update({
where: { id: buildId },
data: {
status: 'failed'
}
});
}
if (error !== 1) {
await saveBuildLog({ line: error, buildId, applicationId: application.id });
}

View File

@@ -29,7 +29,7 @@ async function autoUpdater() {
`sed -i '/COOLIFY_AUTO_UPDATE=/cCOOLIFY_AUTO_UPDATE=${isAutoUpdateEnabled}' .env`
);
await asyncExecShell(
`docker run --rm -tid --env-file .env -v /var/run/docker.sock:/var/run/docker.sock -v coolify-db coollabsio/coolify:${latestVersion} /bin/sh -c "env | grep COOLIFY > .env && echo 'TAG=${latestVersion}' >> .env && docker stop -t 0 coolify && docker rm coolify && docker compose up -d --force-recreate"`
`docker run --rm -tid --env-file .env -v /var/run/docker.sock:/var/run/docker.sock -v coolify-db coollabsio/coolify:${latestVersion} /bin/sh -c "env | grep COOLIFY > .env && echo 'TAG=${latestVersion}' >> .env && docker stop -t 0 coolify coolify-fluentbit && docker rm coolify coolify-fluentbit && docker compose pull && docker compose up -d --force-recreate"`
);
}
} else {

View File

@@ -1,4 +1,4 @@
import { base64Encode, executeDockerCmd, generateTimestamp, getDomain, isDev, prisma, version } from "../common";
import { base64Encode, encrypt, executeDockerCmd, generateTimestamp, getDomain, isDev, prisma, version } from "../common";
import { promises as fs } from 'fs';
import { day } from "../dayjs";
@@ -461,17 +461,32 @@ export const saveBuildLog = async ({
buildId: string;
applicationId: string;
}): Promise<any> => {
const { default: got } = await import('got')
if (line && typeof line === 'string' && line.includes('ghs_')) {
const regex = /ghs_.*@/g;
line = line.replace(regex, '<SENSITIVE_DATA_DELETED>@');
}
const addTimestamp = `[${generateTimestamp()}] ${line}`;
if (isDev) console.debug(`[${applicationId}] ${addTimestamp}`);
return await prisma.buildLog.create({
data: {
line: addTimestamp, buildId, time: Number(day().valueOf()), applicationId
}
});
const fluentBitUrl = isDev ? 'http://localhost:24224' : 'http://coolify-fluentbit:24224';
if (isDev) {
console.debug(`[${applicationId}] ${addTimestamp}`);
}
try {
return await got.post(`${fluentBitUrl}/${applicationId}_buildlog_${buildId}.csv`, {
json: {
line: encrypt(line)
}
})
} catch(error) {
return await prisma.buildLog.create({
data: {
line: addTimestamp, buildId, time: Number(day().valueOf()), applicationId
}
});
}
};
export async function copyBaseConfigurationFiles(
@@ -707,7 +722,6 @@ export async function buildCacheImageWithNode(data, imageForBuild) {
Dockerfile.push(`RUN ${installCommand}`);
}
Dockerfile.push(`RUN ${buildCommand}`);
console.log(Dockerfile.join('\n'))
await fs.writeFile(`${workdir}/Dockerfile-cache`, Dockerfile.join('\n'));
await buildImage({ ...data, isCache: true });
}

View File

@@ -21,7 +21,7 @@ import { scheduler } from './scheduler';
import { supportedServiceTypesAndVersions } from './services/supportedVersions';
import { includeServices } from './services/common';
export const version = '3.10.3';
export const version = '3.10.4';
export const isDev = process.env.NODE_ENV === 'development';
const algorithm = 'aes-256-ctr';
@@ -45,9 +45,7 @@ export function getAPIUrl() {
if (process.env.CODESANDBOX_HOST) {
return `https://${process.env.CODESANDBOX_HOST.replace(/\$PORT/, '3001')}`;
}
return isDev
? 'http://localhost:3001'
: 'http://localhost:3000';
return isDev ? 'http://host.docker.internal:3001' : 'http://localhost:3000';
}
export function getUIUrl() {

View File

@@ -1374,10 +1374,6 @@ async function startAppWriteService(request: FastifyRequest<ServiceStartStop>) {
const teamId = request.user.teamId;
const { version, fqdn, destinationDocker, secrets, exposePort, network, port, workdir, image, appwrite } = await defaultServiceConfigurations({ id, teamId })
let isStatsEnabled = false
if (secrets.find(s => s === '_APP_USAGE_STATS=enabled')) {
isStatsEnabled = true
}
const {
opensslKeyV1,
executorSecret,
@@ -1755,50 +1751,48 @@ async function startAppWriteService(request: FastifyRequest<ServiceStartStop>) {
},
};
if (isStatsEnabled) {
dockerCompose[id].depends_on.push(`${id}-influxdb`);
dockerCompose[`${id}-usage`] = {
image: `${image}:${version}`,
container_name: `${id}-usage`,
labels: makeLabelForServices('appwrite'),
entrypoint: "usage",
depends_on: [
`${id}-mariadb`,
`${id}-influxdb`,
],
environment: [
"_APP_ENV=production",
`_APP_OPENSSL_KEY_V1=${opensslKeyV1}`,
`_APP_DB_HOST=${mariadbHost}`,
`_APP_DB_PORT=${mariadbPort}`,
`_APP_DB_SCHEMA=${mariadbDatabase}`,
`_APP_DB_USER=${mariadbUser}`,
`_APP_DB_PASS=${mariadbPassword}`,
`_APP_INFLUXDB_HOST=${id}-influxdb`,
"_APP_INFLUXDB_PORT=8086",
`_APP_REDIS_HOST=${id}-redis`,
"_APP_REDIS_PORT=6379",
...secrets
],
...defaultComposeConfiguration(network),
}
dockerCompose[`${id}-influxdb`] = {
image: "appwrite/influxdb:1.5.0",
container_name: `${id}-influxdb`,
volumes: [
`${id}-influxdb:/var/lib/influxdb:rw`
],
...defaultComposeConfiguration(network),
}
dockerCompose[`${id}-telegraf`] = {
image: "appwrite/telegraf:1.4.0",
container_name: `${id}-telegraf`,
environment: [
`_APP_INFLUXDB_HOST=${id}-influxdb`,
"_APP_INFLUXDB_PORT=8086",
],
...defaultComposeConfiguration(network),
}
dockerCompose[id].depends_on.push(`${id}-influxdb`);
dockerCompose[`${id}-usage`] = {
image: `${image}:${version}`,
container_name: `${id}-usage`,
labels: makeLabelForServices('appwrite'),
entrypoint: "usage",
depends_on: [
`${id}-mariadb`,
`${id}-influxdb`,
],
environment: [
"_APP_ENV=production",
`_APP_OPENSSL_KEY_V1=${opensslKeyV1}`,
`_APP_DB_HOST=${mariadbHost}`,
`_APP_DB_PORT=${mariadbPort}`,
`_APP_DB_SCHEMA=${mariadbDatabase}`,
`_APP_DB_USER=${mariadbUser}`,
`_APP_DB_PASS=${mariadbPassword}`,
`_APP_INFLUXDB_HOST=${id}-influxdb`,
"_APP_INFLUXDB_PORT=8086",
`_APP_REDIS_HOST=${id}-redis`,
"_APP_REDIS_PORT=6379",
...secrets
],
...defaultComposeConfiguration(network),
}
dockerCompose[`${id}-influxdb`] = {
image: "appwrite/influxdb:1.5.0",
container_name: `${id}-influxdb`,
volumes: [
`${id}-influxdb:/var/lib/influxdb:rw`
],
...defaultComposeConfiguration(network),
}
dockerCompose[`${id}-telegraf`] = {
image: "appwrite/telegraf:1.4.0",
container_name: `${id}-telegraf`,
environment: [
`_APP_INFLUXDB_HOST=${id}-influxdb`,
"_APP_INFLUXDB_PORT=8086",
],
...defaultComposeConfiguration(network),
}
const composeFile: any = {

View File

@@ -1,3 +1,24 @@
/*
Example of a supported version:
{
// Name used to identify the service internally
name: 'umami',
// Fancier name to show to the user
fancyName: 'Umami',
// Docker base image for the service
baseImage: 'ghcr.io/mikecao/umami',
// Optional: If there is any dependent image, you should list it here
images: [],
// Usable tags
versions: ['postgresql-latest'],
// Which tag is the recommended
recommendedVersion: 'postgresql-latest',
// Application's default port, Umami listens on 3000
ports: {
main: 3000
}
}
*/
export const supportedServiceTypesAndVersions = [
{
name: 'plausibleanalytics',
@@ -151,7 +172,7 @@ export const supportedServiceTypesAndVersions = [
fancyName: 'Appwrite',
baseImage: 'appwrite/appwrite',
images: ['mariadb:10.7', 'redis:6.2-alpine', 'appwrite/telegraf:1.4.0'],
versions: ['latest', '0.15.3'],
versions: ['latest', '1.0','0.15.3'],
recommendedVersion: '0.15.3',
ports: {
main: 80

View File

@@ -5,6 +5,7 @@ import axios from 'axios';
import { FastifyReply } from 'fastify';
import fs from 'fs/promises';
import yaml from 'js-yaml';
import csv from 'csvtojson';
import { day } from '../../../../lib/dayjs';
import { makeLabelForStandaloneApplication, setDefaultBaseImage, setDefaultConfiguration } from '../../../../lib/buildPacks/common';
@@ -12,8 +13,9 @@ import { checkDomainsIsValidInDNS, checkDoubleBranch, checkExposedPort, createDi
import { checkContainer, formatLabelsOnDocker, isContainerExited, removeContainer } from '../../../../lib/docker';
import type { FastifyRequest } from 'fastify';
import type { GetImages, CancelDeployment, CheckDNS, CheckRepository, DeleteApplication, DeleteSecret, DeleteStorage, GetApplicationLogs, GetBuildIdLogs, GetBuildLogs, SaveApplication, SaveApplicationSettings, SaveApplicationSource, SaveDeployKey, SaveDestination, SaveSecret, SaveStorage, DeployApplication, CheckDomain, StopPreviewApplication } from './types';
import type { GetImages, CancelDeployment, CheckDNS, CheckRepository, DeleteApplication, DeleteSecret, DeleteStorage, GetApplicationLogs, GetBuildIdLogs, SaveApplication, SaveApplicationSettings, SaveApplicationSource, SaveDeployKey, SaveDestination, SaveSecret, SaveStorage, DeployApplication, CheckDomain, StopPreviewApplication, RestartPreviewApplication, GetBuilds } from './types';
import { OnlyId } from '../../../../types';
import path from 'node:path';
function filterObject(obj, callback) {
return Object.fromEntries(Object.entries(obj).
@@ -83,8 +85,6 @@ export async function getApplicationStatus(request: FastifyRequest<OnlyId>) {
isExited = status.status.isExited;
isRestarting = status.status.isRestarting
}
// isExited = await isContainerExited(application.destinationDocker.id, id);
}
return {
isRunning,
@@ -164,7 +164,8 @@ export async function getApplicationFromDB(id: string, teamId: string) {
gitSource: { include: { githubApp: true, gitlabApp: true } },
secrets: true,
persistentStorage: true,
connectedDatabase: true
connectedDatabase: true,
previewApplication: true
}
});
if (!application) {
@@ -350,6 +351,7 @@ export async function stopPreviewApplication(request: FastifyRequest<StopPreview
if (found) {
await removeContainer({ id: container, dockerId: application.destinationDocker.id });
}
await prisma.previewApplication.deleteMany({ where: { applicationId: application.id, pullmergeRequestId } })
}
return reply.code(201).send();
} catch ({ status, message }) {
@@ -617,7 +619,7 @@ export async function deployApplication(request: FastifyRequest<DeployApplicatio
githubAppId: application.gitSource?.githubApp?.id,
gitlabAppId: application.gitSource?.gitlabApp?.id,
status: 'queued',
type: 'manual'
type: pullmergeRequestId ? application.gitSource?.githubApp?.id ? 'manual_pr' : 'manual_mr' : 'manual'
}
});
return {
@@ -808,7 +810,6 @@ export async function saveSecret(request: FastifyRequest<SaveSecret>, reply: Fas
try {
const { id } = request.params
let { name, value, isBuildSecret, isPRMRSecret, isNew } = request.body
if (isNew) {
const found = await prisma.secret.findFirst({ where: { name, applicationId: id, isPRMRSecret } });
if (found) {
@@ -820,14 +821,24 @@ export async function saveSecret(request: FastifyRequest<SaveSecret>, reply: Fas
});
}
} else {
value = encrypt(value.trim());
if (value) {
value = encrypt(value.trim());
}
const found = await prisma.secret.findFirst({ where: { applicationId: id, name, isPRMRSecret } });
if (found) {
await prisma.secret.updateMany({
where: { applicationId: id, name, isPRMRSecret },
data: { value, isBuildSecret, isPRMRSecret }
});
if (!value && isPRMRSecret) {
await prisma.secret.deleteMany({
where: { applicationId: id, name, isPRMRSecret }
});
} else {
await prisma.secret.updateMany({
where: { applicationId: id, name, isPRMRSecret },
data: { value, isBuildSecret, isPRMRSecret }
});
}
} else {
await prisma.secret.create({
data: { name, value, isBuildSecret, isPRMRSecret, application: { connect: { id } } }
@@ -894,6 +905,181 @@ export async function deleteStorage(request: FastifyRequest<DeleteStorage>) {
}
}
export async function restartPreview(request: FastifyRequest<RestartPreviewApplication>, reply: FastifyReply) {
try {
const { id, pullmergeRequestId } = request.params
const { teamId } = request.user
let application: any = await getApplicationFromDB(id, teamId);
if (application?.destinationDockerId) {
const buildId = cuid();
const { id: dockerId, network } = application.destinationDocker;
const { secrets, port, repository, persistentStorage, id: applicationId, buildPack, exposePort } = application;
const envs = [
`PORT=${port}`
];
if (secrets.length > 0) {
secrets.forEach((secret) => {
if (pullmergeRequestId) {
const isSecretFound = secrets.filter(s => s.name === secret.name && s.isPRMRSecret)
if (isSecretFound.length > 0) {
envs.push(`${secret.name}=${isSecretFound[0].value}`);
} else {
envs.push(`${secret.name}=${secret.value}`);
}
} else {
if (!secret.isPRMRSecret) {
envs.push(`${secret.name}=${secret.value}`);
}
}
});
}
const { workdir } = await createDirectories({ repository, buildId });
const labels = []
let image = null
const { stdout: container } = await executeDockerCmd({ dockerId, command: `docker container ls --filter 'label=com.docker.compose.service=${id}-${pullmergeRequestId}' --format '{{json .}}'` })
const containersArray = container.trim().split('\n');
for (const container of containersArray) {
const containerObj = formatLabelsOnDocker(container);
image = containerObj[0].Image
Object.keys(containerObj[0].Labels).forEach(function (key) {
if (key.startsWith('coolify')) {
labels.push(`${key}=${containerObj[0].Labels[key]}`)
}
})
}
let imageFound = false;
try {
await executeDockerCmd({
dockerId,
command: `docker image inspect ${image}`
})
imageFound = true;
} catch (error) {
//
}
if (!imageFound) {
throw { status: 500, message: 'Image not found, cannot restart application.' }
}
await fs.writeFile(`${workdir}/.env`, envs.join('\n'));
let envFound = false;
try {
envFound = !!(await fs.stat(`${workdir}/.env`));
} catch (error) {
//
}
const volumes =
persistentStorage?.map((storage) => {
return `${applicationId}${storage.path.replace(/\//gi, '-')}:${buildPack !== 'docker' ? '/app' : ''
}${storage.path}`;
}) || [];
const composeVolumes = volumes.map((volume) => {
return {
[`${volume.split(':')[0]}`]: {
name: volume.split(':')[0]
}
};
});
const composeFile = {
version: '3.8',
services: {
[`${applicationId}-${pullmergeRequestId}`]: {
image,
container_name: `${applicationId}-${pullmergeRequestId}`,
volumes,
env_file: envFound ? [`${workdir}/.env`] : [],
labels,
depends_on: [],
expose: [port],
...(exposePort ? { ports: [`${exposePort}:${port}`] } : {}),
...defaultComposeConfiguration(network),
}
},
networks: {
[network]: {
external: true
}
},
volumes: Object.assign({}, ...composeVolumes)
};
await fs.writeFile(`${workdir}/docker-compose.yml`, yaml.dump(composeFile));
await executeDockerCmd({ dockerId, command: `docker stop -t 0 ${id}-${pullmergeRequestId}` })
await executeDockerCmd({ dockerId, command: `docker rm ${id}-${pullmergeRequestId}` })
await executeDockerCmd({ dockerId, command: `docker compose --project-directory ${workdir} up -d` })
return reply.code(201).send();
}
throw { status: 500, message: 'Application cannot be restarted.' }
} catch ({ status, message }) {
return errorHandler({ status, message })
}
}
export async function getPreviewStatus(request: FastifyRequest<RestartPreviewApplication>) {
try {
const { id, pullmergeRequestId } = request.params
const { teamId } = request.user
let isRunning = false;
let isExited = false;
let isRestarting = false;
let isBuilding = false
const application: any = await getApplicationFromDB(id, teamId);
if (application?.destinationDockerId) {
const status = await checkContainer({ dockerId: application.destinationDocker.id, container: `${id}-${pullmergeRequestId}` });
if (status?.found) {
isRunning = status.status.isRunning;
isExited = status.status.isExited;
isRestarting = status.status.isRestarting
}
const building = await prisma.build.findMany({ where: { applicationId: id, pullmergeRequestId, status: { in: ['queued', 'running'] } } })
isBuilding = building.length > 0
}
return {
isBuilding,
isRunning,
isRestarting,
isExited,
};
} catch ({ status, message }) {
return errorHandler({ status, message })
}
}
export async function loadPreviews(request: FastifyRequest<OnlyId>) {
try {
const { id } = request.params
const application = await prisma.application.findUnique({ where: { id }, include: { destinationDocker: true } });
const { stdout } = await executeDockerCmd({ dockerId: application.destinationDocker.id, command: `docker container ls --filter 'name=${id}-' --format "{{json .}}"` })
if (stdout === '') {
throw { status: 500, message: 'No previews found.' }
}
const containers = formatLabelsOnDocker(stdout).filter(container => container.Labels['coolify.configuration'] && container.Labels['coolify.type'] === 'standalone-application')
const jsonContainers = containers
.map((container) =>
JSON.parse(Buffer.from(container.Labels['coolify.configuration'], 'base64').toString())
)
.filter((container) => {
return container.pullmergeRequestId && container.applicationId === id;
});
for (const container of jsonContainers) {
const found = await prisma.previewApplication.findMany({ where: { applicationId: container.applicationId, pullmergeRequestId: container.pullmergeRequestId } })
if (found.length === 0) {
await prisma.previewApplication.create({
data: {
pullmergeRequestId: container.pullmergeRequestId,
sourceBranch: container.branch,
customDomain: container.fqdn,
application: { connect: { id: container.applicationId } }
}
})
}
}
return {
previews: await prisma.previewApplication.findMany({ where: { applicationId: id } })
}
} catch ({ status, message }) {
return errorHandler({ status, message })
}
}
export async function getPreviews(request: FastifyRequest<OnlyId>) {
try {
const { id } = request.params
@@ -909,26 +1095,7 @@ export async function getPreviews(request: FastifyRequest<OnlyId>) {
const applicationSecrets = secrets.filter((secret) => !secret.isPRMRSecret);
const PRMRSecrets = secrets.filter((secret) => secret.isPRMRSecret);
const application = await prisma.application.findUnique({ where: { id }, include: { destinationDocker: true } });
const { stdout } = await executeDockerCmd({ dockerId: application.destinationDocker.id, command: `docker container ls --filter 'name=${id}-' --format "{{json .}}"` })
if (stdout === '') {
return {
containers: [],
applicationSecrets: [],
PRMRSecrets: []
}
}
const containers = formatLabelsOnDocker(stdout).filter(container => container.Labels['coolify.configuration'] && container.Labels['coolify.type'] === 'standalone-application')
const jsonContainers = containers
.map((container) =>
JSON.parse(Buffer.from(container.Labels['coolify.configuration'], 'base64').toString())
)
.filter((container) => {
return container.pullmergeRequestId && container.applicationId === id;
});
return {
containers: jsonContainers,
applicationSecrets: applicationSecrets.sort((a, b) => {
return ('' + a.name).localeCompare(b.name);
}),
@@ -980,7 +1147,7 @@ export async function getApplicationLogs(request: FastifyRequest<GetApplicationL
return errorHandler({ status, message })
}
}
export async function getBuildLogs(request: FastifyRequest<GetBuildLogs>) {
export async function getBuilds(request: FastifyRequest<GetBuilds>) {
try {
const { id } = request.params
let { buildId, skip = 0 } = request.query
@@ -997,17 +1164,15 @@ export async function getBuildLogs(request: FastifyRequest<GetBuildLogs>) {
builds = await prisma.build.findMany({
where: { applicationId: id },
orderBy: { createdAt: 'desc' },
take: 5,
skip
take: 5 + skip
});
}
builds = builds.map((build) => {
const updatedAt = day(build.updatedAt).utc();
build.took = updatedAt.diff(day(build.createdAt)) / 1000;
build.since = updatedAt.fromNow();
return build;
});
if (build.status === 'running') {
build.elapsed = (day().utc().diff(day(build.createdAt)) / 1000).toFixed(0);
}
return build
})
return {
builds,
buildCount
@@ -1019,22 +1184,49 @@ export async function getBuildLogs(request: FastifyRequest<GetBuildLogs>) {
export async function getBuildIdLogs(request: FastifyRequest<GetBuildIdLogs>) {
try {
const { buildId } = request.params
// TODO: Fluentbit could still hold the logs, so we need to check if the logs are done
const { buildId, id } = request.params
let { sequence = 0 } = request.query
if (typeof sequence !== 'number') {
sequence = Number(sequence)
}
let logs = await prisma.buildLog.findMany({
where: { buildId, time: { gt: sequence } },
orderBy: { time: 'asc' }
});
let file = `/app/logs/${id}_buildlog_${buildId}.csv`
if (isDev) {
file = `${process.cwd()}/../../logs/${id}_buildlog_${buildId}.csv`
}
const data = await prisma.build.findFirst({ where: { id: buildId } });
const createdAt = day(data.createdAt).utc();
try {
await fs.stat(file)
} catch (error) {
let logs = await prisma.buildLog.findMany({
where: { buildId, time: { gt: sequence } },
orderBy: { time: 'asc' }
});
const data = await prisma.build.findFirst({ where: { id: buildId } });
const createdAt = day(data.createdAt).utc();
return {
logs: logs.map(log => {
log.time = Number(log.time)
return log
}),
fromDb: true,
took: day().diff(createdAt) / 1000,
status: data?.status || 'queued'
}
}
let fileLogs = (await fs.readFile(file)).toString()
let decryptedLogs = await csv({ noheader: true }).fromString(fileLogs)
let logs = decryptedLogs.map(log => {
const parsed = {
time: log['field1'],
line: decrypt(log['field2'] + '","' + log['field3'])
}
return parsed
}).filter(log => log.time > sequence)
return {
logs: logs.map(log => {
log.time = Number(log.time)
return log
}),
logs,
fromDb: false,
took: day().diff(createdAt) / 1000,
status: data?.status || 'queued'
}

View File

@@ -1,8 +1,8 @@
import { FastifyPluginAsync } from 'fastify';
import { OnlyId } from '../../../../types';
import { cancelDeployment, checkDNS, checkDomain, checkRepository, deleteApplication, deleteSecret, deleteStorage, deployApplication, getApplication, getApplicationLogs, getApplicationStatus, getBuildIdLogs, getBuildLogs, getBuildPack, getGitHubToken, getGitLabSSHKey, getImages, getPreviews, getSecrets, getStorages, getUsage, listApplications, newApplication, restartApplication, saveApplication, saveApplicationSettings, saveApplicationSource, saveBuildPack, saveConnectedDatabase, saveDeployKey, saveDestination, saveGitLabSSHKey, saveRepository, saveSecret, saveStorage, stopApplication, stopPreviewApplication } from './handlers';
import { cancelDeployment, checkDNS, checkDomain, checkRepository, deleteApplication, deleteSecret, deleteStorage, deployApplication, getApplication, getApplicationLogs, getApplicationStatus, getBuildIdLogs, getBuildPack, getBuilds, getGitHubToken, getGitLabSSHKey, getImages, getPreviews, getPreviewStatus, getSecrets, getStorages, getUsage, listApplications, loadPreviews, newApplication, restartApplication, restartPreview, saveApplication, saveApplicationSettings, saveApplicationSource, saveBuildPack, saveConnectedDatabase, saveDeployKey, saveDestination, saveGitLabSSHKey, saveRepository, saveSecret, saveStorage, stopApplication, stopPreviewApplication } from './handlers';
import type { CancelDeployment, CheckDNS, CheckDomain, CheckRepository, DeleteApplication, DeleteSecret, DeleteStorage, DeployApplication, GetApplicationLogs, GetBuildIdLogs, GetBuildLogs, GetImages, SaveApplication, SaveApplicationSettings, SaveApplicationSource, SaveDeployKey, SaveDestination, SaveSecret, SaveStorage, StopPreviewApplication } from './types';
import type { CancelDeployment, CheckDNS, CheckDomain, CheckRepository, DeleteApplication, DeleteSecret, DeleteStorage, DeployApplication, GetApplicationLogs, GetBuildIdLogs, GetBuilds, GetImages, RestartPreviewApplication, SaveApplication, SaveApplicationSettings, SaveApplicationSource, SaveDeployKey, SaveDestination, SaveSecret, SaveStorage, StopPreviewApplication } from './types';
const root: FastifyPluginAsync = async (fastify): Promise<void> => {
fastify.addHook('onRequest', async (request) => {
@@ -37,9 +37,12 @@ const root: FastifyPluginAsync = async (fastify): Promise<void> => {
fastify.delete<DeleteStorage>('/:id/storages', async (request) => await deleteStorage(request));
fastify.get<OnlyId>('/:id/previews', async (request) => await getPreviews(request));
fastify.post<OnlyId>('/:id/previews/load', async (request) => await loadPreviews(request));
fastify.get<RestartPreviewApplication>('/:id/previews/:pullmergeRequestId/status', async (request) => await getPreviewStatus(request));
fastify.post<RestartPreviewApplication>('/:id/previews/:pullmergeRequestId/restart', async (request, reply) => await restartPreview(request, reply));
fastify.get<GetApplicationLogs>('/:id/logs', async (request) => await getApplicationLogs(request));
fastify.get<GetBuildLogs>('/:id/logs/build', async (request) => await getBuildLogs(request));
fastify.get<GetBuilds>('/:id/logs/build', async (request) => await getBuilds(request));
fastify.get<GetBuildIdLogs>('/:id/logs/build/:buildId', async (request) => await getBuildIdLogs(request));
fastify.get('/:id/usage', async (request) => await getUsage(request))

View File

@@ -89,7 +89,7 @@ export interface GetApplicationLogs extends OnlyId {
since: number,
}
}
export interface GetBuildLogs extends OnlyId {
export interface GetBuilds extends OnlyId {
Querystring: {
buildId: string
skip: number,
@@ -97,6 +97,7 @@ export interface GetBuildLogs extends OnlyId {
}
export interface GetBuildIdLogs {
Params: {
id: string,
buildId: string
},
Querystring: {
@@ -126,4 +127,10 @@ export interface StopPreviewApplication extends OnlyId {
Body: {
pullmergeRequestId: string | null,
}
}
export interface RestartPreviewApplication {
Params: {
id: string,
pullmergeRequestId: string | null,
}
}

View File

@@ -73,7 +73,7 @@ export async function update(request: FastifyRequest<Update>) {
`sed -i '/COOLIFY_AUTO_UPDATE=/cCOOLIFY_AUTO_UPDATE=${isAutoUpdateEnabled}' .env`
);
await asyncExecShell(
`docker run --rm -tid --env-file .env -v /var/run/docker.sock:/var/run/docker.sock -v coolify-db coollabsio/coolify:${latestVersion} /bin/sh -c "env | grep COOLIFY > .env && echo 'TAG=${latestVersion}' >> .env && docker stop -t 0 coolify && docker rm coolify && docker compose up -d --force-recreate"`
`docker run --rm -tid --env-file .env -v /var/run/docker.sock:/var/run/docker.sock -v coolify-db coollabsio/coolify:${latestVersion} /bin/sh -c "env | grep COOLIFY > .env && echo 'TAG=${latestVersion}' >> .env && docker stop -t 0 coolify coolify-fluentbit && docker rm coolify coolify-fluentbit && docker compose pull && docker compose up -d --force-recreate"`
);
return {};
} else {

View File

@@ -8,9 +8,7 @@ export async function listServers(request: FastifyRequest) {
try {
const userId = request.user.userId;
const teamId = request.user.teamId;
const servers = await prisma.destinationDocker.findMany({ where: { teams: { some: { id: teamId === '0' ? undefined : teamId } }, remoteEngine: false }, distinct: ['engine'] })
// const remoteServers = await prisma.destinationDocker.findMany({ where: { teams: { some: { id: teamId === '0' ? undefined : teamId } } }, distinct: ['remoteIpAddress', 'engine'] })
const servers = await prisma.destinationDocker.findMany({ where: { teams: { some: { id: teamId === '0' ? undefined : teamId } }}, distinct: ['remoteIpAddress', 'engine'] })
return {
servers
}
@@ -67,8 +65,7 @@ export async function showUsage(request: FastifyRequest) {
const { stdout: stats } = await executeSSHCmd({ dockerId: id, command: `vmstat -s` })
const { stdout: disks } = await executeSSHCmd({ dockerId: id, command: `df -m / --output=size,used,pcent|grep -v 'Used'| xargs` })
const { stdout: cpus } = await executeSSHCmd({ dockerId: id, command: `nproc --all` })
// const { stdout: cpuUsage } = await executeSSHCmd({ dockerId: id, command: `echo $[100-$(vmstat 1 2|tail -1|awk '{print $15}')]` })
// console.log(cpuUsage)
const { stdout: cpuUsage } = await executeSSHCmd({ dockerId: id, command: `echo $[100-$(vmstat 1 2|tail -1|awk '{print $15}')]` })
const parsed: any = parseFromText(stats)
return {
usage: {
@@ -81,8 +78,8 @@ export async function showUsage(request: FastifyRequest) {
freeMemPercentage: (parsed.totalMemoryKB - parsed.usedMemoryKB) / parsed.totalMemoryKB * 100
},
cpu: {
load: 0,
usage: 0,
load: [0,0,0],
usage: cpuUsage,
count: cpus
},
disk: {

View File

@@ -456,7 +456,7 @@ export async function activatePlausibleUsers(request: FastifyRequest<OnlyId>, re
if (destinationDockerId) {
await executeDockerCmd({
dockerId: destinationDocker.id,
command: `docker exec ${id} 'psql -H postgresql://${postgresqlUser}:${postgresqlPassword}@localhost:5432/${postgresqlDatabase} -c "UPDATE users SET email_verified = true;"'`
command: `docker exec ${id}-postgresql psql -H postgresql://${postgresqlUser}:${postgresqlPassword}@localhost:5432/${postgresqlDatabase} -c "UPDATE users SET email_verified = true;"`
})
return await reply.code(201).send()
}
@@ -476,7 +476,7 @@ export async function cleanupPlausibleLogs(request: FastifyRequest<OnlyId>, repl
if (destinationDockerId) {
await executeDockerCmd({
dockerId: destinationDocker.id,
command: `docker exec ${id}-clickhouse sh -c "/usr/bin/clickhouse-client -q \\"SELECT name FROM system.tables WHERE name LIKE '%log%';\\"| xargs -I{} /usr/bin/clickhouse-client -q \"TRUNCATE TABLE system.{};\""`
command: `docker exec ${id}-clickhouse /usr/bin/clickhouse-client -q \\"SELECT name FROM system.tables WHERE name LIKE '%log%';\\"| xargs -I{} /usr/bin/clickhouse-client -q \"TRUNCATE TABLE system.{};\"`
})
return await reply.code(201).send()
}

View File

@@ -1,7 +1,7 @@
import axios from "axios";
import cuid from "cuid";
import crypto from "crypto";
import { encrypt, errorHandler, getUIUrl, isDev, prisma } from "../../../lib/common";
import { encrypt, errorHandler, getDomain, getUIUrl, isDev, prisma } from "../../../lib/common";
import { checkContainer, removeContainer } from "../../../lib/docker";
import { createdBranchDatabase, getApplicationFromDBWebhook, removeBranchDatabase } from "../../api/v1/applications/handlers";
@@ -169,10 +169,29 @@ export async function gitHubEvents(request: FastifyRequest<GitHubEvents>): Promi
pullmergeRequestAction === 'reopened' ||
pullmergeRequestAction === 'synchronize'
) {
await prisma.application.update({
where: { id: application.id },
data: { updatedAt: new Date() }
});
let previewApplicationId = undefined
if (pullmergeRequestId) {
const foundPreviewApplications = await prisma.previewApplication.findMany({ where: { applicationId: application.id, pullmergeRequestId } })
if (foundPreviewApplications.length > 0) {
previewApplicationId = foundPreviewApplications[0].id
} else {
const protocol = application.fqdn.includes('https://') ? 'https://' : 'http://'
const previewApplication = await prisma.previewApplication.create({
data: {
pullmergeRequestId,
sourceBranch,
customDomain: `${protocol}${pullmergeRequestId}.${getDomain(application.fqdn)}`,
application: { connect: { id: application.id } }
}
})
previewApplicationId = previewApplication.id
}
}
// if (application.connectedDatabase && pullmergeRequestAction === 'opened' || pullmergeRequestAction === 'reopened') {
// // Coolify hosted database
// if (application.connectedDatabase.databaseId) {
@@ -187,6 +206,7 @@ export async function gitHubEvents(request: FastifyRequest<GitHubEvents>): Promi
data: {
id: buildId,
pullmergeRequestId,
previewApplicationId,
sourceBranch,
applicationId: application.id,
destinationDockerId: application.destinationDocker.id,
@@ -198,7 +218,9 @@ export async function gitHubEvents(request: FastifyRequest<GitHubEvents>): Promi
}
});
return {
message: 'Queued. Thank you!'
};
} else if (pullmergeRequestAction === 'closed') {
if (application.destinationDockerId) {
const id = `${application.id}-${pullmergeRequestId}`;
@@ -206,13 +228,22 @@ export async function gitHubEvents(request: FastifyRequest<GitHubEvents>): Promi
await removeContainer({ id, dockerId: application.destinationDocker.id });
} catch (error) { }
}
if (application.connectedDatabase.databaseId) {
const databaseId = application.connectedDatabase.databaseId;
const database = await prisma.database.findUnique({ where: { id: databaseId } });
if (database) {
await removeBranchDatabase(database, pullmergeRequestId);
const foundPreviewApplications = await prisma.previewApplication.findMany({ where: { applicationId: application.id, pullmergeRequestId } })
if (foundPreviewApplications.length > 0) {
for (const preview of foundPreviewApplications) {
await prisma.previewApplication.delete({ where: { id: preview.id } })
}
}
return {
message: 'PR closed. Thank you!'
};
// if (application?.connectedDatabase?.databaseId) {
// const databaseId = application.connectedDatabase.databaseId;
// const database = await prisma.database.findUnique({ where: { id: databaseId } });
// if (database) {
// await removeBranchDatabase(database, pullmergeRequestId);
// }
// }
}
}
}

View File

@@ -2,7 +2,7 @@ import axios from "axios";
import cuid from "cuid";
import crypto from "crypto";
import type { FastifyReply, FastifyRequest } from "fastify";
import { errorHandler, getAPIUrl, getUIUrl, isDev, listSettings, prisma } from "../../../lib/common";
import { errorHandler, getAPIUrl, getDomain, getUIUrl, isDev, listSettings, prisma } from "../../../lib/common";
import { checkContainer, removeContainer } from "../../../lib/docker";
import { getApplicationFromDB, getApplicationFromDBWebhook } from "../../api/v1/applications/handlers";
@@ -91,8 +91,8 @@ export async function gitLabEvents(request: FastifyRequest<GitLabEvents>) {
}
}
} else if (objectKind === 'merge_request') {
const { object_attributes: { work_in_progress: isDraft, action, source_branch: sourceBranch, target_branch: targetBranch, iid: pullmergeRequestId }, project: { id } } = request.body
const { object_attributes: { work_in_progress: isDraft, action, source_branch: sourceBranch, target_branch: targetBranch }, project: { id } } = request.body
const pullmergeRequestId = request.body.object_attributes.iid.toString();
const projectId = Number(id);
if (!allowedActions.includes(action)) {
throw { status: 500, message: 'Action not allowed.' }
@@ -130,10 +130,29 @@ export async function gitLabEvents(request: FastifyRequest<GitLabEvents>) {
where: { id: application.id },
data: { updatedAt: new Date() }
});
let previewApplicationId = undefined
if (pullmergeRequestId) {
const foundPreviewApplications = await prisma.previewApplication.findMany({ where: { applicationId: application.id, pullmergeRequestId } })
if (foundPreviewApplications.length > 0) {
previewApplicationId = foundPreviewApplications[0].id
} else {
const protocol = application.fqdn.includes('https://') ? 'https://' : 'http://'
const previewApplication = await prisma.previewApplication.create({
data: {
pullmergeRequestId,
sourceBranch,
customDomain: `${protocol}${pullmergeRequestId}.${getDomain(application.fqdn)}`,
application: { connect: { id: application.id } }
}
})
previewApplicationId = previewApplication.id
}
}
await prisma.build.create({
data: {
id: buildId,
pullmergeRequestId: pullmergeRequestId.toString(),
pullmergeRequestId,
previewApplicationId,
sourceBranch,
applicationId: application.id,
destinationDockerId: application.destinationDocker.id,
@@ -150,8 +169,19 @@ export async function gitLabEvents(request: FastifyRequest<GitLabEvents>) {
} else if (action === 'close') {
if (application.destinationDockerId) {
const id = `${application.id}-${pullmergeRequestId}`;
await removeContainer({ id, dockerId: application.destinationDocker.id });
try {
await removeContainer({ id, dockerId: application.destinationDocker.id });
} catch (error) { }
}
const foundPreviewApplications = await prisma.previewApplication.findMany({ where: { applicationId: application.id, pullmergeRequestId } })
if (foundPreviewApplications.length > 0) {
for (const preview of foundPreviewApplications) {
await prisma.previewApplication.delete({ where: { id: preview.id } })
}
}
return {
message: 'MR closed. Thank you!'
};
}
}

View File

@@ -12,7 +12,7 @@ function configureMiddleware(
if (isHttps) {
traefik.http.routers[id] = {
entrypoints: ['web'],
rule: `Host(\`${nakedDomain}\`) || Host(\`www.${nakedDomain}\`)`,
rule: `(Host(\`${nakedDomain}\`) || Host(\`www.${nakedDomain}\`)) && PathPrefix(\`/\`)`,
service: `${id}`,
middlewares: ['redirect-to-https']
};
@@ -53,7 +53,7 @@ function configureMiddleware(
if (isDualCerts) {
traefik.http.routers[`${id}-secure`] = {
entrypoints: ['websecure'],
rule: `Host(\`${nakedDomain}\`) || Host(\`www.${nakedDomain}\`)`,
rule: `(Host(\`${nakedDomain}\`) || Host(\`www.${nakedDomain}\`)) && PathPrefix(\`/\`)`,
service: `${id}`,
tls: {
certresolver: 'letsencrypt'
@@ -64,7 +64,7 @@ function configureMiddleware(
if (isWWW) {
traefik.http.routers[`${id}-secure-www`] = {
entrypoints: ['websecure'],
rule: `Host(\`www.${nakedDomain}\`)`,
rule: `Host(\`www.${nakedDomain}\`) && PathPrefix(\`/\`)`,
service: `${id}`,
tls: {
certresolver: 'letsencrypt'
@@ -73,7 +73,7 @@ function configureMiddleware(
};
traefik.http.routers[`${id}-secure`] = {
entrypoints: ['websecure'],
rule: `Host(\`${nakedDomain}\`)`,
rule: `Host(\`${nakedDomain}\`) && PathPrefix(\`/\`)`,
service: `${id}`,
tls: {
domains: {
@@ -86,7 +86,7 @@ function configureMiddleware(
} else {
traefik.http.routers[`${id}-secure-www`] = {
entrypoints: ['websecure'],
rule: `Host(\`www.${nakedDomain}\`)`,
rule: `Host(\`www.${nakedDomain}\`) && PathPrefix(\`/\`)`,
service: `${id}`,
tls: {
domains: {
@@ -97,7 +97,7 @@ function configureMiddleware(
};
traefik.http.routers[`${id}-secure`] = {
entrypoints: ['websecure'],
rule: `Host(\`${domain}\`)`,
rule: `Host(\`${domain}\`) && PathPrefix(\`/\`)`,
service: `${id}`,
tls: {
certresolver: 'letsencrypt'
@@ -110,14 +110,14 @@ function configureMiddleware(
} else {
traefik.http.routers[id] = {
entrypoints: ['web'],
rule: `Host(\`${nakedDomain}\`) || Host(\`www.${nakedDomain}\`)`,
rule: `(Host(\`${nakedDomain}\`) || Host(\`www.${nakedDomain}\`)) && PathPrefix(\`/\`)`,
service: `${id}`,
middlewares: []
};
traefik.http.routers[`${id}-secure`] = {
entrypoints: ['websecure'],
rule: `Host(\`${nakedDomain}\`) || Host(\`www.${nakedDomain}\`)`,
rule: `(Host(\`${nakedDomain}\`) || Host(\`www.${nakedDomain}\`)) && PathPrefix(\`/\`)`,
service: `${id}`,
tls: {
domains: {