wip: trpc

This commit is contained in:
Andras Bacsai
2022-12-13 12:47:14 +01:00
parent 1639d1725a
commit 1180d3fdde
28 changed files with 1341 additions and 175 deletions

View File

@@ -35,6 +35,7 @@
"fastify-plugin": "4.4.0",
"got": "^12.5.3",
"is-port-reachable": "4.0.0",
"js-yaml": "4.1.0",
"jsonwebtoken": "8.5.1",
"node-fetch": "3.3.0",
"prisma": "4.6.1",
@@ -47,11 +48,12 @@
"zod": "3.19.1"
},
"devDependencies": {
"@types/bcryptjs": "^2.4.2",
"@types/js-yaml": "^4.0.5",
"@types/jsonwebtoken": "^8.5.9",
"@types/node": "18.11.9",
"@types/node-fetch": "2.6.2",
"@types/shell-quote": "^1.7.1",
"@types/bcryptjs": "^2.4.2",
"@types/ws": "8.5.3",
"npm-run-all": "4.1.5",
"rimraf": "3.0.2",

View File

@@ -7,6 +7,7 @@ import { uniqueNamesGenerator, adjectives, colors, animals } from 'unique-names-
import type { Config } from 'unique-names-generator';
import { env } from '../env';
import { day } from './dayjs';
import { executeCommand } from './executeCommand';
const customConfig: Config = {
dictionaries: [adjectives, colors, animals],
@@ -132,3 +133,51 @@ export async function removeService({ id }: { id: string }): Promise<void> {
await prisma.service.delete({ where: { id } });
}
export const createDirectories = async ({
repository,
buildId
}: {
repository: string;
buildId: string;
}): Promise<{ workdir: string; repodir: string }> => {
if (repository) repository = repository.replaceAll(' ', '');
const repodir = `/tmp/build-sources/${repository}/`;
const workdir = `/tmp/build-sources/${repository}/${buildId}`;
let workdirFound = false;
try {
workdirFound = !!(await fs.stat(workdir));
} catch (error) {}
if (workdirFound) {
await executeCommand({ command: `rm -fr ${workdir}` });
}
await executeCommand({ command: `mkdir -p ${workdir}` });
return {
workdir,
repodir
};
};
export async function saveDockerRegistryCredentials({ url, username, password, workdir }) {
if (!username || !password) {
return null;
}
let decryptedPassword = decrypt(password);
const location = `${workdir}/.docker`;
try {
await fs.mkdir(`${workdir}/.docker`);
} catch (error) {
console.log(error);
}
const payload = JSON.stringify({
auths: {
[url]: {
auth: Buffer.from(`${username}:${decryptedPassword}`).toString('base64')
}
}
});
await fs.writeFile(`${location}/config.json`, payload);
return location;
}

View File

@@ -122,3 +122,36 @@ export async function stopTcpHttpProxy(
return error;
}
}
export function formatLabelsOnDocker(data: any) {
return data
.trim()
.split('\n')
.map((a) => JSON.parse(a))
.map((container) => {
const labels = container.Labels.split(',');
let jsonLabels = {};
labels.forEach((l) => {
const name = l.split('=')[0];
const value = l.split('=')[1];
jsonLabels = { ...jsonLabels, ...{ [name]: value } };
});
container.Labels = jsonLabels;
return container;
});
}
export function defaultComposeConfiguration(network: string): any {
return {
networks: [network],
restart: 'on-failure',
deploy: {
restart_policy: {
condition: 'on-failure',
delay: '5s',
max_attempts: 10,
window: '120s'
}
}
};
}

View File

@@ -23,7 +23,7 @@ export async function executeCommand({
sshCommand?: boolean;
shell?: boolean;
stream?: boolean;
dockerId?: string;
dockerId?: string | null;
buildId?: string;
applicationId?: string;
debug?: boolean;
@@ -31,8 +31,8 @@ export async function executeCommand({
const { execa, execaCommand } = await import('execa');
const { parse } = await import('shell-quote');
const parsedCommand = parse(command);
const dockerCommand = parsedCommand[0]?.toString();
const dockerArgs = parsedCommand.slice(1).toString();
const dockerCommand = parsedCommand[0];
const dockerArgs = parsedCommand.slice(1);
if (dockerId && dockerCommand && dockerArgs) {
const destinationDocker = await prisma.destinationDocker.findUnique({
@@ -41,14 +41,12 @@ export async function executeCommand({
if (!destinationDocker) {
throw new Error('Destination docker not found');
}
let {
remoteEngine,
remoteIpAddress,
engine = 'unix:///var/run/docker.sock'
} = destinationDocker;
let { remoteEngine, remoteIpAddress, engine } = destinationDocker;
if (remoteEngine) {
await createRemoteEngineConfiguration(dockerId);
engine = `ssh://${remoteIpAddress}-remote`;
} else {
engine = 'unix:///var/run/docker.sock';
}
if (env.CODESANDBOX_HOST) {
@@ -60,16 +58,19 @@ export async function executeCommand({
if (shell) {
return execaCommand(`ssh ${remoteIpAddress}-remote ${command}`);
}
//@ts-ignore
return await execa('ssh', [`${remoteIpAddress}-remote`, dockerCommand, ...dockerArgs]);
}
if (stream) {
return await new Promise(async (resolve, reject) => {
let subprocess = null;
if (shell) {
//@ts-ignore
subprocess = execaCommand(command, {
env: { DOCKER_BUILDKIT: '1', DOCKER_HOST: engine }
});
} else {
//@ts-ignore
subprocess = execa(dockerCommand, dockerArgs, {
env: { DOCKER_BUILDKIT: '1', DOCKER_HOST: engine }
});
@@ -112,7 +113,8 @@ export async function executeCommand({
});
subprocess.on('exit', async (code: number) => {
if (code === 0) {
resolve('success');
//@ts-ignore
resolve(code);
} else {
if (!debug) {
for (const log of logs) {
@@ -127,9 +129,11 @@ export async function executeCommand({
} else {
if (shell) {
return await execaCommand(command, {
//@ts-ignore
env: { DOCKER_BUILDKIT: '1', DOCKER_HOST: engine }
});
} else {
//@ts-ignore
return await execa(dockerCommand, dockerArgs, {
env: { DOCKER_BUILDKIT: '1', DOCKER_HOST: engine }
});
@@ -139,6 +143,7 @@ export async function executeCommand({
if (shell) {
return execaCommand(command, { shell: true });
}
//@ts-ignore
return await execa(dockerCommand, dockerArgs);
}
}

View File

@@ -12,7 +12,7 @@ const prismaGlobal = global as typeof global & {
export const prisma: PrismaClient =
prismaGlobal.prisma ||
new PrismaClient({
log: env.NODE_ENV === 'development' ? ['query', 'error', 'warn'] : ['error']
log: env.NODE_ENV !== 'development' ? ['query', 'error', 'warn'] : ['error']
});
if (env.NODE_ENV !== 'production') {

View File

@@ -0,0 +1,391 @@
import { z } from 'zod';
import fs from 'fs/promises';
import yaml from 'js-yaml';
import { privateProcedure, router } from '../../trpc';
import { prisma } from '../../../prisma';
import { executeCommand } from '../../../lib/executeCommand';
import {
checkContainer,
defaultComposeConfiguration,
formatLabelsOnDocker,
removeContainer
} from '../../../lib/docker';
import { deployApplication, generateConfigHash, getApplicationFromDB } from './lib';
import cuid from 'cuid';
import { createDirectories, saveDockerRegistryCredentials } from '../../../lib/common';
export const applicationsRouter = router({
getApplicationById: privateProcedure
.input(z.object({ id: z.string() }))
.query(async ({ ctx, input }) => {
const id: string = input.id;
const teamId = ctx.user?.teamId;
if (!teamId) {
throw { status: 400, message: 'Team not found.' };
}
const application = await getApplicationFromDB(id, teamId);
return {
success: true,
data: { ...application }
};
}),
save: privateProcedure
.input(
z.object({
id: z.string()
})
)
.mutation(async ({ ctx, input }) => {
const { id } = input;
const teamId = ctx.user?.teamId;
// const buildId = await deployApplication(id, teamId);
return {
// buildId
};
}),
status: privateProcedure.input(z.object({ id: z.string() })).query(async ({ ctx, input }) => {
const id: string = input.id;
const teamId = ctx.user?.teamId;
if (!teamId) {
throw { status: 400, message: 'Team not found.' };
}
let payload = [];
const application: any = await getApplicationFromDB(id, teamId);
if (application?.destinationDockerId) {
if (application.buildPack === 'compose') {
const { stdout: containers } = await executeCommand({
dockerId: application.destinationDocker.id,
command: `docker ps -a --filter "label=coolify.applicationId=${id}" --format '{{json .}}'`
});
const containersArray = containers.trim().split('\n');
if (containersArray.length > 0 && containersArray[0] !== '') {
for (const container of containersArray) {
let isRunning = false;
let isExited = false;
let isRestarting = false;
const containerObj = JSON.parse(container);
const status = containerObj.State;
if (status === 'running') {
isRunning = true;
}
if (status === 'exited') {
isExited = true;
}
if (status === 'restarting') {
isRestarting = true;
}
payload.push({
name: containerObj.Names,
status: {
isRunning,
isExited,
isRestarting
}
});
}
}
} else {
let isRunning = false;
let isExited = false;
let isRestarting = false;
const status = await checkContainer({
dockerId: application.destinationDocker.id,
container: id
});
if (status?.found) {
isRunning = status.status.isRunning;
isExited = status.status.isExited;
isRestarting = status.status.isRestarting;
payload.push({
name: id,
status: {
isRunning,
isExited,
isRestarting
}
});
}
}
}
return payload;
}),
cleanup: privateProcedure.query(async ({ ctx }) => {
const teamId = ctx.user?.teamId;
let applications = await prisma.application.findMany({
where: { teams: { some: { id: teamId === '0' ? undefined : teamId } } },
include: { settings: true, destinationDocker: true, teams: true }
});
for (const application of applications) {
if (
!application.buildPack ||
!application.destinationDockerId ||
!application.branch ||
(!application.settings?.isBot && !application?.fqdn)
) {
if (application?.destinationDockerId && application.destinationDocker?.network) {
const { stdout: containers } = await executeCommand({
dockerId: application.destinationDocker.id,
command: `docker ps -a --filter network=${application.destinationDocker.network} --filter name=${application.id} --format '{{json .}}'`
});
if (containers) {
const containersArray = containers.trim().split('\n');
for (const container of containersArray) {
const containerObj = JSON.parse(container);
const id = containerObj.ID;
await removeContainer({ id, dockerId: application.destinationDocker.id });
}
}
}
await prisma.applicationSettings.deleteMany({ where: { applicationId: application.id } });
await prisma.buildLog.deleteMany({ where: { applicationId: application.id } });
await prisma.build.deleteMany({ where: { applicationId: application.id } });
await prisma.secret.deleteMany({ where: { applicationId: application.id } });
await prisma.applicationPersistentStorage.deleteMany({
where: { applicationId: application.id }
});
await prisma.applicationConnectedDatabase.deleteMany({
where: { applicationId: application.id }
});
await prisma.application.deleteMany({ where: { id: application.id } });
}
}
return {};
}),
stop: privateProcedure.input(z.object({ id: z.string() })).mutation(async ({ ctx, input }) => {
const { id } = input;
const teamId = ctx.user?.teamId;
const application: any = await getApplicationFromDB(id, teamId);
if (application?.destinationDockerId) {
const { id: dockerId } = application.destinationDocker;
if (application.buildPack === 'compose') {
const { stdout: containers } = await executeCommand({
dockerId: application.destinationDocker.id,
command: `docker ps -a --filter "label=coolify.applicationId=${id}" --format '{{json .}}'`
});
const containersArray = containers.trim().split('\n');
if (containersArray.length > 0 && containersArray[0] !== '') {
for (const container of containersArray) {
const containerObj = JSON.parse(container);
await removeContainer({
id: containerObj.ID,
dockerId: application.destinationDocker.id
});
}
}
return;
}
const { found } = await checkContainer({ dockerId, container: id });
if (found) {
await removeContainer({ id, dockerId: application.destinationDocker.id });
}
}
return {};
}),
restart: privateProcedure.input(z.object({ id: z.string() })).mutation(async ({ ctx, input }) => {
const { id } = input;
const teamId = ctx.user?.teamId;
let application = await getApplicationFromDB(id, teamId);
if (application?.destinationDockerId) {
const buildId = cuid();
const { id: dockerId, network } = application.destinationDocker;
const {
dockerRegistry,
secrets,
pullmergeRequestId,
port,
repository,
persistentStorage,
id: applicationId,
buildPack,
exposePort
} = application;
let location = null;
const labels = [];
let image = null;
const envs = [`PORT=${port}`];
if (secrets.length > 0) {
secrets.forEach((secret) => {
if (pullmergeRequestId) {
const isSecretFound = secrets.filter((s) => s.name === secret.name && s.isPRMRSecret);
if (isSecretFound.length > 0) {
envs.push(`${secret.name}=${isSecretFound[0].value}`);
} else {
envs.push(`${secret.name}=${secret.value}`);
}
} else {
if (!secret.isPRMRSecret) {
envs.push(`${secret.name}=${secret.value}`);
}
}
});
}
const { workdir } = await createDirectories({ repository, buildId });
const { stdout: container } = await executeCommand({
dockerId,
command: `docker container ls --filter 'label=com.docker.compose.service=${id}' --format '{{json .}}'`
});
const containersArray = container.trim().split('\n');
for (const container of containersArray) {
const containerObj = formatLabelsOnDocker(container);
image = containerObj[0].Image;
Object.keys(containerObj[0].Labels).forEach(function (key) {
if (key.startsWith('coolify')) {
labels.push(`${key}=${containerObj[0].Labels[key]}`);
}
});
}
if (dockerRegistry) {
const { url, username, password } = dockerRegistry;
location = await saveDockerRegistryCredentials({ url, username, password, workdir });
}
let imageFoundLocally = false;
try {
await executeCommand({
dockerId,
command: `docker image inspect ${image}`
});
imageFoundLocally = true;
} catch (error) {
//
}
let imageFoundRemotely = false;
try {
await executeCommand({
dockerId,
command: `docker ${location ? `--config ${location}` : ''} pull ${image}`
});
imageFoundRemotely = true;
} catch (error) {
//
}
if (!imageFoundLocally && !imageFoundRemotely) {
throw { status: 500, message: 'Image not found, cannot restart application.' };
}
await fs.writeFile(`${workdir}/.env`, envs.join('\n'));
let envFound = false;
try {
envFound = !!(await fs.stat(`${workdir}/.env`));
} catch (error) {
//
}
const volumes =
persistentStorage?.map((storage) => {
return `${applicationId}${storage.path.replace(/\//gi, '-')}:${
buildPack !== 'docker' ? '/app' : ''
}${storage.path}`;
}) || [];
const composeVolumes = volumes.map((volume) => {
return {
[`${volume.split(':')[0]}`]: {
name: volume.split(':')[0]
}
};
});
const composeFile = {
version: '3.8',
services: {
[applicationId]: {
image,
container_name: applicationId,
volumes,
env_file: envFound ? [`${workdir}/.env`] : [],
labels,
depends_on: [],
expose: [port],
...(exposePort ? { ports: [`${exposePort}:${port}`] } : {}),
...defaultComposeConfiguration(network)
}
},
networks: {
[network]: {
external: true
}
},
volumes: Object.assign({}, ...composeVolumes)
};
await fs.writeFile(`${workdir}/docker-compose.yml`, yaml.dump(composeFile));
try {
await executeCommand({ dockerId, command: `docker stop -t 0 ${id}` });
await executeCommand({ dockerId, command: `docker rm ${id}` });
} catch (error) {
//
}
await executeCommand({
dockerId,
command: `docker compose --project-directory ${workdir} up -d`
});
}
return {};
}),
deploy: privateProcedure
.input(
z.object({
id: z.string()
})
)
.mutation(async ({ ctx, input }) => {
const { id } = input;
const teamId = ctx.user?.teamId;
const buildId = await deployApplication(id, teamId);
return {
buildId
};
}),
forceRedeploy: privateProcedure
.input(
z.object({
id: z.string()
})
)
.mutation(async ({ ctx, input }) => {
const { id } = input;
const teamId = ctx.user?.teamId;
const buildId = await deployApplication(id, teamId, true);
return {
buildId
};
}),
delete: privateProcedure
.input(z.object({ force: z.boolean(), id: z.string() }))
.mutation(async ({ ctx, input }) => {
const { id, force } = input;
const teamId = ctx.user?.teamId;
const application = await prisma.application.findUnique({
where: { id },
include: { destinationDocker: true }
});
if (!force && application?.destinationDockerId && application.destinationDocker?.network) {
const { stdout: containers } = await executeCommand({
dockerId: application.destinationDocker.id,
command: `docker ps -a --filter network=${application.destinationDocker.network} --filter name=${id} --format '{{json .}}'`
});
if (containers) {
const containersArray = containers.trim().split('\n');
for (const container of containersArray) {
const containerObj = JSON.parse(container);
const id = containerObj.ID;
await removeContainer({ id, dockerId: application.destinationDocker.id });
}
}
}
await prisma.applicationSettings.deleteMany({ where: { application: { id } } });
await prisma.buildLog.deleteMany({ where: { applicationId: id } });
await prisma.build.deleteMany({ where: { applicationId: id } });
await prisma.secret.deleteMany({ where: { applicationId: id } });
await prisma.applicationPersistentStorage.deleteMany({ where: { applicationId: id } });
await prisma.applicationConnectedDatabase.deleteMany({ where: { applicationId: id } });
if (teamId === '0') {
await prisma.application.deleteMany({ where: { id } });
} else {
await prisma.application.deleteMany({ where: { id, teams: { some: { id: teamId } } } });
}
return {};
})
});

View File

@@ -1,157 +1,85 @@
import { z } from 'zod';
import { privateProcedure, router } from '../trpc';
import { decrypt, isARM } from '../../lib/common';
import { prisma } from '../../prisma';
import { executeCommand } from '../../lib/executeCommand';
import { checkContainer, removeContainer } from '../../lib/docker';
import cuid from 'cuid';
import crypto from 'node:crypto';
import { decrypt, isARM } from '../../../lib/common';
export const applicationsRouter = router({
status: privateProcedure.input(z.object({ id: z.string() })).query(async ({ ctx, input }) => {
const id: string = input.id;
const teamId = ctx.user?.teamId;
if (!teamId) {
throw { status: 400, message: 'Team not found.' };
import { prisma } from '../../../prisma';
export async function deployApplication(
id: string,
teamId: string,
forceRebuild: boolean = false
): Promise<string | Error> {
const buildId = cuid();
const application = await getApplicationFromDB(id, teamId);
if (application) {
if (!application?.configHash) {
await generateConfigHash(
id,
application.buildPack,
application.port,
application.exposePort,
application.installCommand,
application.buildCommand,
application.startCommand
);
}
let payload = [];
const application: any = await getApplicationFromDB(id, teamId);
if (application?.destinationDockerId) {
if (application.buildPack === 'compose') {
const { stdout: containers } = await executeCommand({
dockerId: application.destinationDocker.id,
command: `docker ps -a --filter "label=coolify.applicationId=${id}" --format '{{json .}}'`
});
const containersArray = containers.trim().split('\n');
if (containersArray.length > 0 && containersArray[0] !== '') {
for (const container of containersArray) {
let isRunning = false;
let isExited = false;
let isRestarting = false;
const containerObj = JSON.parse(container);
const status = containerObj.State;
if (status === 'running') {
isRunning = true;
}
if (status === 'exited') {
isExited = true;
}
if (status === 'restarting') {
isRestarting = true;
}
payload.push({
name: containerObj.Names,
status: {
isRunning,
isExited,
isRestarting
}
});
}
await prisma.application.update({ where: { id }, data: { updatedAt: new Date() } });
if (application.gitSourceId) {
await prisma.build.create({
data: {
id: buildId,
applicationId: id,
branch: application.branch,
forceRebuild,
destinationDockerId: application.destinationDocker?.id,
gitSourceId: application.gitSource?.id,
githubAppId: application.gitSource?.githubApp?.id,
gitlabAppId: application.gitSource?.gitlabApp?.id,
status: 'queued',
type: 'manual'
}
} else {
let isRunning = false;
let isExited = false;
let isRestarting = false;
const status = await checkContainer({
dockerId: application.destinationDocker.id,
container: id
});
if (status?.found) {
isRunning = status.status.isRunning;
isExited = status.status.isExited;
isRestarting = status.status.isRestarting;
payload.push({
name: id,
status: {
isRunning,
isExited,
isRestarting
}
});
}
}
}
return payload;
}),
cleanup: privateProcedure.query(async ({ ctx }) => {
const teamId = ctx.user?.teamId;
let applications = await prisma.application.findMany({
where: { teams: { some: { id: teamId === '0' ? undefined : teamId } } },
include: { settings: true, destinationDocker: true, teams: true }
});
for (const application of applications) {
if (
!application.buildPack ||
!application.destinationDockerId ||
!application.branch ||
(!application.settings?.isBot && !application?.fqdn)
) {
if (application?.destinationDockerId && application.destinationDocker?.network) {
const { stdout: containers } = await executeCommand({
dockerId: application.destinationDocker.id,
command: `docker ps -a --filter network=${application.destinationDocker.network} --filter name=${application.id} --format '{{json .}}'`
});
if (containers) {
const containersArray = containers.trim().split('\n');
for (const container of containersArray) {
const containerObj = JSON.parse(container);
const id = containerObj.ID;
await removeContainer({ id, dockerId: application.destinationDocker.id });
}
}
}
await prisma.applicationSettings.deleteMany({ where: { applicationId: application.id } });
await prisma.buildLog.deleteMany({ where: { applicationId: application.id } });
await prisma.build.deleteMany({ where: { applicationId: application.id } });
await prisma.secret.deleteMany({ where: { applicationId: application.id } });
await prisma.applicationPersistentStorage.deleteMany({
where: { applicationId: application.id }
});
await prisma.applicationConnectedDatabase.deleteMany({
where: { applicationId: application.id }
});
await prisma.application.deleteMany({ where: { id: application.id } });
}
}
return {};
}),
delete: privateProcedure
.input(z.object({ force: z.boolean(), id: z.string() }))
.mutation(async ({ ctx, input }) => {
const { id, force } = input;
const teamId = ctx.user?.teamId;
const application = await prisma.application.findUnique({
where: { id },
include: { destinationDocker: true }
});
if (!force && application?.destinationDockerId && application.destinationDocker?.network) {
const { stdout: containers } = await executeCommand({
dockerId: application.destinationDocker.id,
command: `docker ps -a --filter network=${application.destinationDocker.network} --filter name=${id} --format '{{json .}}'`
});
if (containers) {
const containersArray = containers.trim().split('\n');
for (const container of containersArray) {
const containerObj = JSON.parse(container);
const id = containerObj.ID;
await removeContainer({ id, dockerId: application.destinationDocker.id });
}
} else {
await prisma.build.create({
data: {
id: buildId,
applicationId: id,
branch: 'latest',
forceRebuild,
destinationDockerId: application.destinationDocker?.id,
status: 'queued',
type: 'manual'
}
}
await prisma.applicationSettings.deleteMany({ where: { application: { id } } });
await prisma.buildLog.deleteMany({ where: { applicationId: id } });
await prisma.build.deleteMany({ where: { applicationId: id } });
await prisma.secret.deleteMany({ where: { applicationId: id } });
await prisma.applicationPersistentStorage.deleteMany({ where: { applicationId: id } });
await prisma.applicationConnectedDatabase.deleteMany({ where: { applicationId: id } });
if (teamId === '0') {
await prisma.application.deleteMany({ where: { id } });
} else {
await prisma.application.deleteMany({ where: { id, teams: { some: { id: teamId } } } });
}
return {};
})
});
});
}
return buildId;
}
throw { status: 500, message: 'Application cannot be deployed.' };
}
export async function generateConfigHash(
id: string,
buildPack: string,
port: number,
exposePort: number,
installCommand: string,
buildCommand: string,
startCommand: string
): Promise<any> {
const configHash = crypto
.createHash('sha256')
.update(
JSON.stringify({
buildPack,
port,
exposePort,
installCommand,
buildCommand,
startCommand
})
)
.digest('hex');
return await prisma.application.update({ where: { id }, data: { configHash } });
}
export async function getApplicationFromDB(id: string, teamId: string) {
let application = await prisma.application.findFirst({
where: { id, teams: { some: { id: teamId === '0' ? undefined : teamId } } },

View File

@@ -29,5 +29,5 @@ const isAdmin = t.middleware(async ({ ctx, next }) => {
});
});
export const router = t.router;
export const privateProcedure = t.procedure.use(isAdmin).use(logger);
export const publicProcedure = t.procedure.use(logger);
export const privateProcedure = t.procedure.use(isAdmin);
export const publicProcedure = t.procedure;