dave/coolify
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Merge pull request #487 from coollabsio/next

Browse Source 
v3.1.1
This commit is contained in:
Andras Bacsai
2022-07-13 15:43:46 +02:00
committed by GitHub
parent 002637ec5f d5f2d22663
commit d5f9d57be9
14 changed files with 229 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files

116
apps/api/src/jobs/cleanupStorage.ts
View File

@@ -1,5 +1,5 @@
import { parentPort } from 'node:worker_threads';
import { asyncExecShell, isDev, prisma, version } from '../lib/common';
import { asyncExecShell, cleanupDockerStorage, isDev, prisma, version } from '../lib/common';
import { getEngine } from '../lib/docker';
(async () => {
@@ -9,82 +9,52 @@ import { getEngine } from '../lib/docker';
for (const engine of engines) {
let lowDiskSpace = false;
const host = getEngine(engine);
// try {
// let stdout = null
// if (!isDev) {
// const output = await asyncExecShell(
// `DOCKER_HOST=${host} docker exec coolify sh -c 'df -kPT /'`
// );
// stdout = output.stdout;
// } else {
// const output = await asyncExecShell(
// `df -kPT /`
// );
// stdout = output.stdout;
// }
// let lines = stdout.trim().split('\n');
// let header = lines[0];
// let regex =
// /^Filesystem\s+|Type\s+|1024-blocks|\s+Used|\s+Available|\s+Capacity|\s+Mounted on\s*$/g;
// const boundaries = [];
// let match;
// while ((match = regex.exec(header))) {
// boundaries.push(match[0].length);
// }
// boundaries[boundaries.length - 1] = -1;
// const data = lines.slice(1).map((line) => {
// const cl = boundaries.map((boundary) => {
// const column = boundary > 0 ? line.slice(0, boundary) : line;
// line = line.slice(boundary);
// return column.trim();
// });
// return {
// capacity: Number.parseInt(cl[5], 10) / 100
// };
// });
// if (data.length > 0) {
// const { capacity } = data[0];
// if (capacity > 0.6) {
// lowDiskSpace = true;
// }
// }
// } catch (error) {
// console.log(error);
// }
if (!isDev) {
// Cleanup old coolify images
try {
let { stdout: images } = await asyncExecShell(
`DOCKER_HOST=${host} docker images coollabsio/coolify --filter before="coollabsio/coolify:${version}" -q | xargs `
try {
let stdout = null
if (!isDev) {
const output = await asyncExecShell(
`DOCKER_HOST=${host} docker exec coolify sh -c 'df -kPT /'`
);
images = images.trim();
if (images) {
await asyncExecShell(`DOCKER_HOST=${host} docker rmi -f ${images}`);
stdout = output.stdout;
} else {
const output = await asyncExecShell(
`df -kPT /`
);
stdout = output.stdout;
}
let lines = stdout.trim().split('\n');
let header = lines[0];
let regex =
/^Filesystem\s+|Type\s+|1024-blocks|\s+Used|\s+Available|\s+Capacity|\s+Mounted on\s*$/g;
const boundaries = [];
let match;
while ((match = regex.exec(header))) {
boundaries.push(match[0].length);
}
boundaries[boundaries.length - 1] = -1;
const data = lines.slice(1).map((line) => {
const cl = boundaries.map((boundary) => {
const column = boundary > 0 ? line.slice(0, boundary) : line;
line = line.slice(boundary);
return column.trim();
});
return {
capacity: Number.parseInt(cl[5], 10) / 100
};
});
if (data.length > 0) {
const { capacity } = data[0];
if (capacity > 0.8) {
lowDiskSpace = true;
}
} catch (error) {
//console.log(error);
}
try {
await asyncExecShell(`DOCKER_HOST=${host} docker container prune -f`);
} catch (error) {
//console.log(error);
}
try {
await asyncExecShell(`DOCKER_HOST=${host} docker image prune -f`);
} catch (error) {
//console.log(error);
}
try {
await asyncExecShell(`DOCKER_HOST=${host} docker image prune -a -f`);
} catch (error) {
//console.log(error);
}
} else {
console.log(`[DEV MODE] Low disk space: ${lowDiskSpace}`);
} catch (error) {
console.log(error);
}
await cleanupDockerStorage(host, lowDiskSpace, false)
}
await prisma.$disconnect();
await prisma.$disconnect();
} else process.exit(0);
})();

68
apps/api/src/lib/common.ts
View File

@@ -14,8 +14,9 @@ import cuid from 'cuid';
import { checkContainer, getEngine, removeContainer } from './docker';
import { day } from './dayjs';
import * as serviceFields from './serviceFields'
import axios from 'axios';
export const version = '3.1.0';
export const version = '3.1.1';
export const isDev = process.env.NODE_ENV === 'development';
const algorithm = 'aes-256-ctr';
@@ -30,13 +31,29 @@ export const defaultProxyImage = `coolify-haproxy-alpine:latest`;
export const defaultProxyImageTcp = `coolify-haproxy-tcp-alpine:latest`;
export const defaultProxyImageHttp = `coolify-haproxy-http-alpine:latest`;
export const defaultTraefikImage = `traefik:v2.6`;
export function getAPIUrl() {
if (process.env.GITPOD_WORKSPACE_URL) {
const { href } = new URL(process.env.GITPOD_WORKSPACE_URL)
const newURL = href.replace('https://', 'https://3001-').replace(/\/$/, '')
return newURL
}
return isDev ? 'http://localhost:3001' : 'http://localhost:3000';
}
export function getUIUrl() {
if (process.env.GITPOD_WORKSPACE_URL) {
const { href } = new URL(process.env.GITPOD_WORKSPACE_URL)
const newURL = href.replace('https://', 'https://3000-').replace(/\/$/, '')
return newURL
}
return 'http://localhost:3000';
}
const mainTraefikEndpoint = isDev
? 'http://host.docker.internal:3001/webhooks/traefik/main.json'
? `${getAPIUrl()}/webhooks/traefik/main.json`
: 'http://coolify:3000/webhooks/traefik/main.json';
const otherTraefikEndpoint = isDev
? 'http://host.docker.internal:3001/webhooks/traefik/other.json'
? `${getAPIUrl()}/webhooks/traefik/other.json`
: 'http://coolify:3000/webhooks/traefik/other.json';
@@ -1478,4 +1495,49 @@ async function cleanupDB(buildId: string) {
if (data?.status === 'queued' || data?.status === 'running') {
await prisma.build.update({ where: { id: buildId }, data: { status: 'failed' } });
}
}
export function convertTolOldVolumeNames(type) {
if (type === 'nocodb') {
return 'nc'
}
}
export async function getAvailableServices(): Promise<any> {
const { data } = await axios.get(`https://gist.githubusercontent.com/andrasbacsai/4aac36d8d6214dbfc34fa78110554a50/raw/291a957ee6ac01d480465623e183a30230ad921f/availableServices.json`)
return data
}
export async function cleanupDockerStorage(host, lowDiskSpace, force) {
// Cleanup old coolify images
try {
let { stdout: images } = await asyncExecShell(
`DOCKER_HOST=${host} docker images coollabsio/coolify --filter before="coollabsio/coolify:${version}" -q | xargs `
);
images = images.trim();
if (images) {
await asyncExecShell(`DOCKER_HOST=${host} docker rmi -f ${images}`);
}
} catch (error) {
//console.log(error);
}
if (lowDiskSpace || force) {
if (isDev) {
if (!force) console.log(`[DEV MODE] Low disk space: ${lowDiskSpace}`);
return
}
try {
await asyncExecShell(`DOCKER_HOST=${host} docker container prune -f`);
} catch (error) {
//console.log(error);
}
try {
await asyncExecShell(`DOCKER_HOST=${host} docker image prune -f`);
} catch (error) {
//console.log(error);
}
try {
await asyncExecShell(`DOCKER_HOST=${host} docker image prune -a -f`);
} catch (error) {
//console.log(error);
}
}
}

11
apps/api/src/routes/api/v1/handlers.ts
View File

@@ -4,7 +4,7 @@ import axios from 'axios';
import compare from 'compare-versions';
import cuid from 'cuid';
import bcrypt from 'bcryptjs';
import { asyncExecShell, asyncSleep, errorHandler, isDev, prisma, uniqueName, version } from '../../../lib/common';
import { asyncExecShell, asyncSleep, cleanupDockerStorage, errorHandler, isDev, prisma, uniqueName, version } from '../../../lib/common';
import type { FastifyReply, FastifyRequest } from 'fastify';
import type { Login, Update } from '.';
@@ -15,7 +15,14 @@ export async function hashPassword(password: string): Promise<string> {
return bcrypt.hash(password, saltRounds);
}
export async function cleanupManually() {
try {
await cleanupDockerStorage('unix:///var/run/docker.sock', true, true)
return {}
} catch ({ status, message }) {
return errorHandler({ status, message })
}
}
export async function checkUpdate(request: FastifyRequest) {
try {
const currentVersion = version;

2
apps/api/src/routes/api/v1/iam/handlers.ts
View File

@@ -445,7 +445,7 @@ export async function setPermission(request: FastifyRequest, reply: FastifyReply
export async function changePassword(request: FastifyRequest, reply: FastifyReply) {
try {
const { id } = request.body;
await prisma.user.update({ where: { id: undefined }, data: { password: 'RESETME' } });
await prisma.user.update({ where: { id }, data: { password: 'RESETME' } });
return reply.code(201).send()
} catch ({ status, message }) {
return errorHandler({ status, message })

6
apps/api/src/routes/api/v1/index.ts
View File

@@ -1,6 +1,6 @@
import { FastifyPluginAsync } from 'fastify';
import { scheduler } from '../../../lib/scheduler';
import { checkUpdate, login, showDashboard, update, showUsage, getCurrentUser } from './handlers';
import { checkUpdate, login, showDashboard, update, showUsage, getCurrentUser, cleanupManually } from './handlers';
export interface Update {
Body: { latestVersion: string }
@@ -46,6 +46,10 @@ const root: FastifyPluginAsync = async (fastify, opts): Promise<void> => {
fastify.get('/usage', {
onRequest: [fastify.authenticate]
}, async () => await showUsage());
fastify.post('/internal/cleanup', {
onRequest: [fastify.authenticate]
}, async () => await cleanupManually());
};
export default root;

66
apps/api/src/routes/api/v1/services/handlers.ts
View File

@@ -2,11 +2,75 @@ import type { FastifyReply, FastifyRequest } from 'fastify';
import fs from 'fs/promises';
import yaml from 'js-yaml';
import bcrypt from 'bcryptjs';
import { prisma, uniqueName, asyncExecShell, getServiceImage, getServiceImages, configureServiceType, getServiceFromDB, getContainerUsage, removeService, isDomainConfigured, saveUpdateableFields, fixType, decrypt, encrypt, getServiceMainPort, createDirectories, ComposeFile, makeLabelForServices, getFreePort, getDomain, errorHandler, supportedServiceTypesAndVersions, generatePassword, isDev, stopTcpHttpProxy } from '../../../../lib/common';
import { prisma, uniqueName, asyncExecShell, getServiceImage, getServiceImages, configureServiceType, getServiceFromDB, getContainerUsage, removeService, isDomainConfigured, saveUpdateableFields, fixType, decrypt, encrypt, getServiceMainPort, createDirectories, ComposeFile, makeLabelForServices, getFreePort, getDomain, errorHandler, supportedServiceTypesAndVersions, generatePassword, isDev, stopTcpHttpProxy, getAvailableServices } from '../../../../lib/common';
import { day } from '../../../../lib/dayjs';
import { checkContainer, dockerInstance, getEngine, removeContainer } from '../../../../lib/docker';
import cuid from 'cuid';
async function startServiceNew(request: FastifyRequest) {
try {
const { id } = request.params;
const teamId = request.user.teamId;
const service = await getServiceFromDB({ id, teamId });
const { type, version, destinationDockerId, destinationDocker, serviceSecret, exposePort } =
service;
const network = destinationDockerId && destinationDocker.network;
const host = getEngine(destinationDocker.engine);
const port = getServiceMainPort(type);
const { workdir } = await createDirectories({ repository: type, buildId: id });
const image = getServiceImage(type);
const config = (await getAvailableServices()).find((name) => name.name === type).compose
const environmentVariables = {}
if (serviceSecret.length > 0) {
serviceSecret.forEach((secret) => {
environmentVariables[secret.name] = secret.value;
});
}
config.services[id] = JSON.parse(JSON.stringify(config.services[type]))
config.services[id].container_name = id
config.services[id].image = `${image}:${version}`
config.services[id].ports = (exposePort ? [`${exposePort}:${port}`] : []),
config.services[id].restart = "always"
config.services[id].networks = [network]
config.services[id].labels = makeLabelForServices(type)
config.services[id].deploy = {
restart_policy: {
condition: 'on-failure',
delay: '5s',
max_attempts: 3,
window: '120s'
}
}
config.networks = {
[network]: {
external: true
}
}
config.volumes = {}
config.services[id].volumes.forEach((volume, index) => {
let oldVolumeName = volume.split(':')[0]
const path = volume.split(':')[1]
oldVolumeName = convertTolOldVolumeNames(type)
const volumeName = `${id}-${oldVolumeName}`
config.volumes[volumeName] = {
name: volumeName
}
config.services[id].volumes[index] = `${volumeName}:${path}`
})
delete config.services[type]
config.services[id].environment = environmentVariables
const composeFileDestination = `${workdir}/docker-compose.yaml`;
await fs.writeFile(composeFileDestination, yaml.dump(config));
await asyncExecShell(`DOCKER_HOST=${host} docker compose -f ${composeFileDestination} pull`);
await asyncExecShell(`DOCKER_HOST=${host} docker compose -f ${composeFileDestination} up -d`);
return {}
} catch ({ status, message }) {
return errorHandler({ status, message })
}
}
export async function listServices(request: FastifyRequest) {
try {
const userId = request.user.userId;

6
apps/api/src/routes/webhooks/github/handlers.ts
View File

@@ -2,7 +2,7 @@ import axios from "axios";
import cuid from "cuid";
import crypto from "crypto";
import type { FastifyReply, FastifyRequest } from "fastify";
import { encrypt, errorHandler, isDev, prisma } from "../../../lib/common";
import { encrypt, errorHandler, getAPIUrl, getUIUrl, isDev, prisma } from "../../../lib/common";
import { checkContainer, removeContainer } from "../../../lib/docker";
import { scheduler } from "../../../lib/scheduler";
import { getApplicationFromDB, getApplicationFromDBWebhook } from "../../api/v1/applications/handlers";
@@ -19,7 +19,7 @@ export async function installGithub(request: FastifyRequest, reply: FastifyReply
data: { installationId: Number(installation_id) }
});
if (isDev) {
return reply.redirect(`http://localhost:3000/sources/${gitSourceId}`)
return reply.redirect(`${getUIUrl()}/sources/${gitSourceId}`)
} else {
return reply.redirect(`/sources/${gitSourceId}`)
}
@@ -55,7 +55,7 @@ export async function configureGitHubApp(request, reply) {
}
});
if (isDev) {
return reply.redirect(`http://localhost:3000/sources/${state}`)
return reply.redirect(`${getUIUrl()}/sources/${state}`)
} else {
return reply.redirect(`/sources/${state}`)
}

6
apps/api/src/routes/webhooks/gitlab/handlers.ts
View File

@@ -2,7 +2,7 @@ import axios from "axios";
import cuid from "cuid";
import crypto from "crypto";
import type { FastifyReply, FastifyRequest } from "fastify";
import { encrypt, errorHandler, isDev, listSettings, prisma } from "../../../lib/common";
import { encrypt, errorHandler, getAPIUrl, isDev, listSettings, prisma } from "../../../lib/common";
import { checkContainer, removeContainer } from "../../../lib/docker";
import { scheduler } from "../../../lib/scheduler";
import { getApplicationFromDB, getApplicationFromDBWebhook } from "../../api/v1/applications/handlers";
@@ -16,7 +16,7 @@ export async function configureGitLabApp(request: FastifyRequest, reply: Fastify
let domain = `http://${request.hostname}`;
if (fqdn) domain = fqdn;
if (isDev) {
domain = `http://localhost:3001`;
domain = getAPIUrl();
}
const params = new URLSearchParams({
client_id: appId,
@@ -28,7 +28,7 @@ export async function configureGitLabApp(request: FastifyRequest, reply: Fastify
});
const { data } = await axios.post(`${htmlUrl}/oauth/token`, params)
if (isDev) {
return reply.redirect(`http://localhost:3000/webhooks/success?token=${data.access_token}`)
return reply.redirect(`${getAPIUrl()}/webhooks/success?token=${data.access_token}`)
}
return reply.redirect(`/webhooks/success?token=${data.access_token}`)
} catch ({ status, message, ...other }) {

20
apps/ui/src/lib/api.ts
View File

@@ -1,14 +1,28 @@
import { browser, dev } from '$app/env';
import { dev } from '$app/env';
import Cookies from 'js-cookie';
export function getAPIUrl() {
if (GITPOD_WORKSPACE_URL) {
const {href} = new URL(GITPOD_WORKSPACE_URL)
const newURL = href.replace('https://','https://3001-').replace(/\/$/,'')
const { href } = new URL(GITPOD_WORKSPACE_URL)
const newURL = href.replace('https://', 'https://3001-').replace(/\/$/, '')
return newURL
}
return dev ? 'http://localhost:3001' : 'http://localhost:3000';
}
export function getWebhookUrl(type: string) {
console.log(GITPOD_WORKSPACE_URL)
if (GITPOD_WORKSPACE_URL) {
const { href } = new URL(GITPOD_WORKSPACE_URL)
const newURL = href.replace('https://', 'https://3001-').replace(/\/$/, '')
if (type === 'github') {
return `${newURL}/webhooks/github/events`
}
if (type === 'gitlab') {
return `${newURL}/webhooks/gitlab/events`
}
}
return `https://webhook.site/0e5beb2c-4e9b-40e2-a89e-32295e570c21/events`;
}
async function send({
method,
path,

9
apps/ui/src/lib/components/Usage.svelte
View File

@@ -23,7 +23,7 @@
};
import { appSession } from '$lib/store';
import { onDestroy, onMount } from 'svelte';
import { get } from '$lib/api';
import { get, post } from '$lib/api';
import { errorNotification } from '$lib/common';
import Trend from './Trend.svelte';
async function getStatus() {
@@ -59,6 +59,9 @@
cpu: 'stable',
disk: 'stable'
};
async function manuallyCleanupStorage() {
return await post('/internal/cleanup', {});
}
</script>
{#if $appSession.teamId === '0'}
@@ -129,6 +132,9 @@
<dd class="mt-1 text-3xl font-semibold text-white">
{usage?.disk.usedGb}<span class="text-sm">GB</span>
</dd>
<button on:click={manuallyCleanupStorage} class="bg-coollabs hover:bg-coollabs-100"
>Cleanup Storage</button
>
</div>
<div
class="overflow-hidden rounded px-4 py-5 text-center sm:p-6 sm:text-left"
@@ -143,5 +149,6 @@
</dd>
</div>
</dl>
<div class="px-6 pt-20 text-2xl font-bold">Resources</div>
{/if}

6
apps/ui/src/routes/applications/[id]/configuration/_GitlabRepositories.svelte
View File

@@ -8,7 +8,7 @@
import { onMount } from 'svelte';
import { dev } from '$app/env';
import { goto } from '$app/navigation';
import { del, get, post } from '$lib/api';
import { del, get, getAPIUrl, getWebhookUrl, post } from '$lib/api';
import { t } from '$lib/translations';
import { errorNotification } from '$lib/common';
import { appSession } from '$lib/store';
@@ -18,7 +18,7 @@
const from = $page.url.searchParams.get('from');
let url = settings?.fqdn ? settings.fqdn : window.location.origin;
if (dev) url = `http://localhost:3001`;
if (dev) url = getAPIUrl();
const updateDeployKeyIdUrl = `/applications/${id}/configuration/deploykey`;
@@ -228,7 +228,7 @@
}
async function setWebhook(url: any, webhookToken: any) {
const host = dev
? 'https://webhook.site/0e5beb2c-4e9b-40e2-a89e-32295e570c21'
? getWebhookUrl('gitlab')
: `${window.location.origin}/webhooks/gitlab/events`;
try {
await post(

6
apps/ui/src/routes/sources/[id]/_Github.svelte
View File

@@ -2,7 +2,7 @@
export let source: any;
export let settings: any;
import { page } from '$app/stores';
import { post } from '$lib/api';
import { getAPIUrl, getWebhookUrl, post } from '$lib/api';
import Explainer from '$lib/components/Explainer.svelte';
import { toast } from '@zerodevx/svelte-toast';
import { t } from '$lib/translations';
@@ -67,7 +67,7 @@
const { organization, htmlUrl } = source;
const { fqdn } = settings;
const host = dev
? 'http://localhost:3001'
? getAPIUrl()
: fqdn
? fqdn
: `http://${window.location.host}` || '';
@@ -81,7 +81,7 @@
url: host,
hook_attributes: {
url: dev
? 'https://webhook.site/0e5beb2c-4e9b-40e2-a89e-32295e570c21/events'
? getWebhookUrl('github')
: `${host}/webhooks/github/events`
},
redirect_url: `${host}/webhooks/github`,

4
apps/ui/src/routes/sources/[id]/_Gitlab.svelte
View File

@@ -4,7 +4,7 @@
import Explainer from '$lib/components/Explainer.svelte';
import { page } from '$app/stores';
import { onMount } from 'svelte';
import { post } from '$lib/api';
import { getAPIUrl, post } from '$lib/api';
import { dev } from '$app/env';
import CopyPasswordField from '$lib/components/CopyPasswordField.svelte';
import { toast } from '@zerodevx/svelte-toast';
@@ -17,7 +17,7 @@
let url = settings.fqdn ? settings.fqdn : window.location.origin;
if (dev) {
url = `http://localhost:3001`;
url = getAPIUrl();
}
let loading = false;

3
package.json
View File

@@ -1,7 +1,7 @@
{
"name": "coolify",
"description": "An open-source & self-hostable Heroku / Netlify alternative.",
"version": "3.1.0",
"version": "3.1.1",
"license": "AGPL-3.0",
"scripts": {
"db:studio": "pnpm run --filter coolify-api db:studio",
@@ -18,6 +18,7 @@
"build": "NODE_ENV=production run-p -n build:*",
"build:api": "NODE_ENV=production pnpm run --filter coolify-api build",
"build:ui": "NODE_ENV=production pnpm run --filter coolify-ui build",
"dockerlogin":"echo $DOCKER_PASS | docker login --username=$DOCKER_USER --password-stdin",
"release:staging:amd": "cross-var docker buildx build --platform linux/amd64 -t coollabsio/coolify:$npm_package_version --push .",
"release:local":"rm -fr ./local-serve && mkdir ./local-serve && pnpm build && cp -Rp apps/api/build/* ./local-serve && cp -Rp apps/ui/build/ ./local-serve/public && cp -Rp apps/api/prisma/ ./local-serve/prisma && cp -Rp apps/api/package.json ./local-serve && cp .env ./local-serve && cd ./local-serve && pnpm install . && pnpm start"
},