From 3fd50ebb1241488563e277d22e95628ef7d81f37 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Sat, 30 Apr 2022 13:54:19 +0200 Subject: [PATCH] fix: checking low disk space --- package.json | 2 +- pnpm-lock.yaml | 14 +++---- src/lib/queues/cleanup.ts | 83 ++++++++++++++++++++++++++++----------- 3 files changed, 69 insertions(+), 30 deletions(-) diff --git a/package.json b/package.json index 934d8e6d1..1cffb1afb 100644 --- a/package.json +++ b/package.json @@ -55,8 +55,8 @@ "svelte-check": "2.7.0", "svelte-preprocess": "4.10.6", "svelte-select": "4.4.7", - "tailwindcss": "3.0.24", "sveltekit-i18n": "2.1.2", + "tailwindcss": "3.0.24", "ts-node": "10.7.0", "tslib": "2.3.1", "typescript": "4.6.3" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9c608add7..8af565eeb 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -48,8 +48,8 @@ specifiers: svelte-kit-cookie-session: 2.1.3 svelte-preprocess: 4.10.6 svelte-select: 4.4.7 - tailwindcss: 3.0.24 sveltekit-i18n: 2.1.2 + tailwindcss: 3.0.24 tailwindcss-scrollbar: 0.1.0 ts-node: 10.7.0 tslib: 2.3.1 @@ -108,9 +108,9 @@ devDependencies: svelte-check: 2.7.0_postcss@8.4.12+svelte@3.47.0 svelte-preprocess: 4.10.6_41810887ae6c6d59323116f47e33fa38 svelte-select: 4.4.7 + sveltekit-i18n: 2.1.2_svelte@3.47.0 tailwindcss: 3.0.24_ts-node@10.7.0 ts-node: 10.7.0_de7c86b0cde507c63a0402da5b982bd3 - sveltekit-i18n: 2.1.2_svelte@3.46.4 tslib: 2.3.1 typescript: 4.6.3 @@ -424,7 +424,7 @@ packages: - supports-color dev: true - /@sveltekit-i18n/base/1.1.1_svelte@3.46.4: + /@sveltekit-i18n/base/1.1.1_svelte@3.47.0: resolution: { integrity: sha512-J/sMU0OwS3dCLOuilHMBqu8vZHuuXiNV9vFJx8Nb4/b5BlR/KCZ4bCXI8wZR02GHeCOYKZxWus07CM1scxa/jw== @@ -432,7 +432,7 @@ packages: peerDependencies: svelte: ^3.x dependencies: - svelte: 3.46.4 + svelte: 3.47.0 optionalDependencies: '@sveltekit-i18n/parser-default': 1.0.3 dev: true @@ -4977,7 +4977,7 @@ packages: engines: { node: '>= 8' } dev: true - /sveltekit-i18n/2.1.2_svelte@3.46.4: + /sveltekit-i18n/2.1.2_svelte@3.47.0: resolution: { integrity: sha512-s5YxcbNd2EWNZaZR1A4Drt8s53E4fpUkN4XIWd3VRpw1pihZVWssqmBW1qkjQ6AB0kiu1Qwule+vt1HkbQOjrg== @@ -4985,9 +4985,9 @@ packages: peerDependencies: svelte: ^3.x dependencies: - '@sveltekit-i18n/base': 1.1.1_svelte@3.46.4 + '@sveltekit-i18n/base': 1.1.1_svelte@3.47.0 '@sveltekit-i18n/parser-default': 1.0.3 - svelte: 3.46.4 + svelte: 3.47.0 dev: true /table/6.7.2: diff --git a/src/lib/queues/cleanup.ts b/src/lib/queues/cleanup.ts index d5dc49381..e5a65e0a5 100644 --- a/src/lib/queues/cleanup.ts +++ b/src/lib/queues/cleanup.ts @@ -4,34 +4,73 @@ export default async function (): Promise { const destinationDockers = await prisma.destinationDocker.findMany(); const engines = [...new Set(destinationDockers.map(({ engine }) => engine))]; for (const engine of engines) { + let lowDiskSpace = false; const host = getEngine(engine); - // Cleanup old coolify images try { - let { stdout: images } = await asyncExecShell( - `DOCKER_HOST=${host} docker images coollabsio/coolify --filter before="coollabsio/coolify:${version}" -q | xargs ` + const { stdout } = await asyncExecShell( + `DOCKER_HOST=${host} docker exec coolify sh -c 'df -kPT /'` ); - images = images.trim(); - if (images) { - await asyncExecShell(`DOCKER_HOST=${host} docker rmi -f ${images}`); + let lines = stdout.trim().split('\n'); + let header = lines[0]; + let regex = + /^Filesystem\s+|Type\s+|1024-blocks|\s+Used|\s+Available|\s+Capacity|\s+Mounted on\s*$/g; + const boundaries = []; + let match; + + while ((match = regex.exec(header))) { + boundaries.push(match[0].length); + } + + boundaries[boundaries.length - 1] = -1; + const data = lines.slice(1).map((line) => { + const cl = boundaries.map((boundary) => { + const column = boundary > 0 ? line.slice(0, boundary) : line; + line = line.slice(boundary); + return column.trim(); + }); + return { + capacity: Number.parseInt(cl[5], 10) / 100 + }; + }); + if (data.length > 0) { + const { capacity } = data[0]; + if (capacity > 0.8) { + lowDiskSpace = true; + } } } catch (error) { - //console.log(error); + console.log(error); } - try { - await asyncExecShell(`DOCKER_HOST=${host} docker container prune -f`); - } catch (error) { - //console.log(error); - } - try { - await asyncExecShell(`DOCKER_HOST=${host} docker image prune -f --filter "until=2h"`); - } catch (error) { - //console.log(error); - } - // Cleanup old images older than a day - try { - await asyncExecShell(`DOCKER_HOST=${host} docker image prune --filter "until=72h" -a -f`); - } catch (error) { - //console.log(error); + console.log(`Is LowDiskSpace detected? ${lowDiskSpace}`); + if (lowDiskSpace) { + // Cleanup old coolify images + try { + let { stdout: images } = await asyncExecShell( + `DOCKER_HOST=${host} docker images coollabsio/coolify --filter before="coollabsio/coolify:${version}" -q | xargs ` + ); + images = images.trim(); + if (images) { + await asyncExecShell(`DOCKER_HOST=${host} docker rmi -f ${images}`); + } + } catch (error) { + //console.log(error); + } + try { + await asyncExecShell(`DOCKER_HOST=${host} docker container prune -f`); + } catch (error) { + //console.log(error); + } + try { + await asyncExecShell(`DOCKER_HOST=${host} docker image prune -f --filter "until=2h"`); + } catch (error) { + //console.log(error); + } + // Cleanup old images older than a day + try { + await asyncExecShell(`DOCKER_HOST=${host} docker image prune --filter "until=72h" -a -f`); + } catch (error) { + //console.log(error); + } } } }