diff --git a/.gitignore b/.gitignore index 500cd6776..eddb26abd 100644 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,5 @@ client apps/api/db/*.db local-serve apps/api/db/migration.db-journal -apps/api/core* \ No newline at end of file +apps/api/core* +logs \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 675fe5b5b..b7bd56212 100644 --- a/Dockerfile +++ b/Dockerfile @@ -33,6 +33,7 @@ RUN chmod +x ~/.docker/cli-plugins/docker-compose /usr/bin/docker RUN (curl -sSL "https://github.com/buildpacks/pack/releases/download/v0.27.0/pack-v0.27.0-linux.tgz" | tar -C /usr/local/bin/ --no-same-owner -xzv pack) COPY --from=build /app/apps/api/build/ . +COPY --from=build /app/others/fluentbit/ ./fluentbit COPY --from=build /app/apps/ui/build/ ./public COPY --from=build /app/apps/api/prisma/ ./prisma COPY --from=build /app/apps/api/package.json . diff --git a/apps/api/package.json b/apps/api/package.json index f436482c8..f3635517b 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -29,6 +29,8 @@ "bree": "9.1.2", "cabin": "9.1.2", "compare-versions": "5.0.1", + "csv-parse": "^5.3.0", + "csvtojson": "^2.0.10", "cuid": "2.1.8", "dayjs": "1.11.5", "dockerode": "3.3.4", diff --git a/apps/api/src/lib/buildPacks/common.ts b/apps/api/src/lib/buildPacks/common.ts index eed68511a..a357b64b6 100644 --- a/apps/api/src/lib/buildPacks/common.ts +++ b/apps/api/src/lib/buildPacks/common.ts @@ -1,4 +1,4 @@ -import { base64Encode, executeDockerCmd, generateTimestamp, getDomain, isDev, prisma, version } from "../common"; +import { base64Encode, encrypt, executeDockerCmd, generateTimestamp, getDomain, isDev, prisma, version } from "../common"; import { promises as fs } from 'fs'; import { day } from "../dayjs"; @@ -461,17 +461,30 @@ export const saveBuildLog = async ({ buildId: string; applicationId: string; }): Promise => { + const { default: got } = await import('got') + if (line && typeof line === 'string' && line.includes('ghs_')) { const regex = /ghs_.*@/g; line = line.replace(regex, '@'); } const addTimestamp = `[${generateTimestamp()}] ${line}`; - if (isDev) console.debug(`[${applicationId}] ${addTimestamp}`); - return await prisma.buildLog.create({ - data: { - line: addTimestamp, buildId, time: Number(day().valueOf()), applicationId + const fluentBitUrl = isDev ? 'http://localhost:24224' : 'http://coolify-fluentbit:24224'; + + if (isDev) { + console.debug(`[${applicationId}] ${addTimestamp}`); + } + // return await prisma.buildLog.create({ + // data: { + // line: addTimestamp, buildId, time: Number(day().valueOf()), applicationId + // } + // }); + + return await got.post(`${fluentBitUrl}/${applicationId}_buildlog_${buildId}.csv`, { + json: { + line: encrypt(line) } - }); + }) + }; export async function copyBaseConfigurationFiles( diff --git a/apps/api/src/routes/api/v1/applications/handlers.ts b/apps/api/src/routes/api/v1/applications/handlers.ts index 69b30cedf..a8881850b 100644 --- a/apps/api/src/routes/api/v1/applications/handlers.ts +++ b/apps/api/src/routes/api/v1/applications/handlers.ts @@ -5,6 +5,7 @@ import axios from 'axios'; import { FastifyReply } from 'fastify'; import fs from 'fs/promises'; import yaml from 'js-yaml'; +import csv from 'csvtojson'; import { day } from '../../../../lib/dayjs'; import { makeLabelForStandaloneApplication, setDefaultBaseImage, setDefaultConfiguration } from '../../../../lib/buildPacks/common'; @@ -14,6 +15,7 @@ import { checkContainer, formatLabelsOnDocker, isContainerExited, removeContaine import type { FastifyRequest } from 'fastify'; import type { GetImages, CancelDeployment, CheckDNS, CheckRepository, DeleteApplication, DeleteSecret, DeleteStorage, GetApplicationLogs, GetBuildIdLogs, GetBuildLogs, SaveApplication, SaveApplicationSettings, SaveApplicationSource, SaveDeployKey, SaveDestination, SaveSecret, SaveStorage, DeployApplication, CheckDomain, StopPreviewApplication, RestartPreviewApplication } from './types'; import { OnlyId } from '../../../../types'; +import path from 'node:path'; function filterObject(obj, callback) { return Object.fromEntries(Object.entries(obj). @@ -1178,22 +1180,38 @@ export async function getBuildLogs(request: FastifyRequest) { export async function getBuildIdLogs(request: FastifyRequest) { try { - const { buildId } = request.params + // TODO: Fluentbit could still hold the logs, so we need to check if the logs are done + const { buildId, id } = request.params let { sequence = 0 } = request.query if (typeof sequence !== 'number') { sequence = Number(sequence) } - let logs = await prisma.buildLog.findMany({ - where: { buildId, time: { gt: sequence } }, - orderBy: { time: 'asc' } - }); + let file = `/app/logs/${id}_buildlog_${buildId}.csv` + if (isDev) { + file = `${process.cwd()}/../../logs/${id}_buildlog_${buildId}.csv` + } const data = await prisma.build.findFirst({ where: { id: buildId } }); const createdAt = day(data.createdAt).utc(); + try { + await fs.stat(file) + } catch (error) { + return { + logs: [], + took: day().diff(createdAt) / 1000, + status: data?.status || 'queued' + } + } + let fileLogs = (await fs.readFile(file)).toString() + let decryptedLogs = await csv({ noheader: true }).fromString(fileLogs) + let logs = decryptedLogs.map(log => { + const parsed = { + time: log['field1'], + line: decrypt(log['field2'] + '","' + log['field3']) + } + return parsed + }).filter(log => log.time > sequence) return { - logs: logs.map(log => { - log.time = Number(log.time) - return log - }), + logs, took: day().diff(createdAt) / 1000, status: data?.status || 'queued' } diff --git a/apps/api/src/routes/api/v1/applications/types.ts b/apps/api/src/routes/api/v1/applications/types.ts index de8165c5b..3db320596 100644 --- a/apps/api/src/routes/api/v1/applications/types.ts +++ b/apps/api/src/routes/api/v1/applications/types.ts @@ -97,6 +97,7 @@ export interface GetBuildLogs extends OnlyId { } export interface GetBuildIdLogs { Params: { + id: string, buildId: string }, Querystring: { diff --git a/apps/ui/src/routes/applications/[id]/logs/_BuildLog.svelte b/apps/ui/src/routes/applications/[id]/logs/_BuildLog.svelte index 6d1947e83..fbf67c618 100644 --- a/apps/ui/src/routes/applications/[id]/logs/_BuildLog.svelte +++ b/apps/ui/src/routes/applications/[id]/logs/_BuildLog.svelte @@ -11,7 +11,7 @@ import LoadingLogs from '$lib/components/LoadingLogs.svelte'; import { errorNotification } from '$lib/common'; import Tooltip from '$lib/components/Tooltip.svelte'; - + import { day } from '$lib/dayjs'; let logs: any = []; let currentStatus: any; let streamInterval: any; @@ -159,7 +159,7 @@ bind:this={logsEl} > {#each logs as log} -
{log.line + '\n'}
+
[{day.unix(log.time).format('HH:mm:ss.SSS')}] {log.line + '\n'}
{/each} {:else} diff --git a/docker-compose-dev.yaml b/docker-compose-dev.yaml index 4769b6128..4f0a4959a 100644 --- a/docker-compose-dev.yaml +++ b/docker-compose-dev.yaml @@ -1,38 +1,18 @@ version: '3.8' services: - postgres: - image: postgres:14.5-alpine - restart: always - container_name: coolify-pg + fluent-bit: + image: fluent/fluent-bit:1.9.8 + container_name: coolify-fluentbit + volumes: + - ./logs:/logs + - ./others/fluentbit/fluent-bit.conf:/fluent-bit/etc/fluent-bit.conf + - ./others/fluentbit/parser.conf:/fluent-bit/etc/parser.conf + ports: + - "24224:24224" networks: - coolify-infra - command: - - "postgres" - - "-c" - - "log_connections=yes" - - "-c" - - "max_connections=1337" - environment: - POSTGRES_DB: coolify - POSTGRES_USER: coolify - POSTGRES_PASSWORD: coolify - POSTGRES_HOST_AUTH_METHOD: "trust" - volumes: - - 'coolify-pgdb:/var/lib/postgresql/data' networks: coolify-infra: attachable: true name: coolify-infra - -volumes: - coolify-db: - name: coolify-db - coolify-pgdb: - name: coolify-pgdb - coolify-ssl-certs: - name: coolify-ssl-certs - coolify-letsencrypt: - name: coolify-letsencrypt - coolify-traefik-letsencrypt: - name: coolify-traefik-letsencrypt diff --git a/docker-compose.yaml b/docker-compose.yaml index 567222545..34d0fad83 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -20,28 +20,22 @@ services: - '.env' networks: - coolify-infra - # postgres: - # image: postgres:14.5-alpine - # restart: always - # container_name: coolify-pg - # command: - # - "postgres" - # - "-c" - # - "log_connections=yes" - # - "-c" - # - "max_connections=1337" - # volumes: - # - 'coolify-pgdb:/var/lib/postgresql/data' - # ports: - # - target: 5432 - # published: 5432 - # protocol: tcp + fluent-bit: + image: fluent/fluent-bit:1.9.8 + container_name: coolify-fluentbit + volumes: + - 'coolify-logs:/app/logs' + - /app/fluentbit/:/fluent-bit/etc/ + networks: + - coolify-infra networks: coolify-infra: attachable: true name: coolify-infra volumes: + coolify-logs: + name: coolify-logs coolify-db: name: coolify-db coolify-pgdb: diff --git a/others/docker-compose-dev.yaml b/others/docker-compose-dev.yaml deleted file mode 100644 index 6c89e87d5..000000000 --- a/others/docker-compose-dev.yaml +++ /dev/null @@ -1,35 +0,0 @@ -version: '3.8' - -services: - redis: - image: redis:6.2-alpine - container_name: coolify-redis - networks: - - coolify-infra - ports: - - target: 6379 - published: 6379 - protocol: tcp - mode: host - # fluentbit: - # container_name: coolify-fluentbit - # build: - # context: ./data/fluentd - # dockerfile: Dockerfile-dev - # ports: - # - target: 24224 - # published: 24224 - # protocol: tcp - # mode: host - # - target: 24224 - # published: 24224 - # protocol: udp - # mode: host - # networks: - # - coolify-infra - # extra_hosts: - # - 'host.docker.internal:host-gateway' -networks: - coolify-infra: - attachable: true - name: coolify-infra diff --git a/others/docker-compose-traefik.yaml b/others/docker-compose-traefik.yaml deleted file mode 100644 index 09fd32525..000000000 --- a/others/docker-compose-traefik.yaml +++ /dev/null @@ -1,29 +0,0 @@ -version: '3.8' - -services: - proxy: - image: traefik:v2.6 - command: - - --api.insecure=true - - --entrypoints.web.address=:80 - - --entrypoints.websecure.address=:443 - - --providers.docker=false - - --providers.docker.exposedbydefault=false - - --providers.http.endpoint=http://host.docker.internal:3000/traefik.json - - --providers.http.pollTimeout=5s - - --log.level=error - ports: - - '80:80' - - '443:443' - - '8080:8080' - volumes: - - /var/run/docker.sock:/var/run/docker.sock - extra_hosts: - - 'host.docker.internal:host-gateway' - networks: - - coolify-infra - -networks: - coolify-infra: - attachable: true - name: coolify-infra diff --git a/others/fluentbit/fluent-bit.conf b/others/fluentbit/fluent-bit.conf new file mode 100644 index 000000000..67de89be5 --- /dev/null +++ b/others/fluentbit/fluent-bit.conf @@ -0,0 +1,30 @@ +[SERVICE] + Parsers_file /fluent-bit/etc/parser.conf + Flush 1 + Grace 30 +[INPUT] + Name http + Host 0.0.0.0 + Port 24224 +[FILTER] + Name parser + Match * + Key_Name log + Parser docker + Reserve_Data True +[OUTPUT] + Name file + Match * + Path /logs + Mkdir true + Format csv +# [OUTPUT] +# Name influxdb +# match * +# Host coolify-influxdb +# Port 8086 +# Database coolify +# Bucket coolify +# Org coolify +# HTTP_Token 12345678 +# Sequence_Tag _seq \ No newline at end of file diff --git a/others/fluentbit/parser.conf b/others/fluentbit/parser.conf new file mode 100644 index 000000000..02f6fd02c --- /dev/null +++ b/others/fluentbit/parser.conf @@ -0,0 +1,6 @@ +[PARSER] + Name docker + Format json + Time_Key time + Time_Format %Y-%m-%dT%H:%M:%S.%L + Time_Keep On \ No newline at end of file diff --git a/others/fluentd/Dockerfile-dev b/others/fluentd/Dockerfile-dev deleted file mode 100644 index bb5524df9..000000000 --- a/others/fluentd/Dockerfile-dev +++ /dev/null @@ -1,6 +0,0 @@ -FROM fluent/fluent-bit:1.9.0 -COPY fluentbit-dev.conf /tmp/fluentbit.conf -ENTRYPOINT ["/fluent-bit/bin/fluent-bit", "-c", "/tmp/fluentbit.conf"] -# USER root -# RUN ["gem", "install", "fluent-plugin-mongo"] -# USER fluent \ No newline at end of file diff --git a/others/fluentd/fluentbit-dev.conf b/others/fluentd/fluentbit-dev.conf deleted file mode 100644 index 8d06fd0af..000000000 --- a/others/fluentd/fluentbit-dev.conf +++ /dev/null @@ -1,24 +0,0 @@ -[INPUT] - Name forward - Listen 0.0.0.0 - Port 24224 - Buffer_Chunk_Size 32KB - Buffer_Max_Size 64KB - -[OUTPUT] - Name influxdb - Match * - Host coolify-influxdb - Port 8086 - Bucket containerlogs - Org organization - HTTP_Token supertoken - Sequence_Tag _seq - Tag_Keys container_name -[OUTPUT] - Name http - Match * - Host host.docker.internal - Port 3000 - URI /logs.json - Format json \ No newline at end of file diff --git a/others/fluentd/fluentd-dev.conf b/others/fluentd/fluentd-dev.conf deleted file mode 100644 index 9ae4a06c0..000000000 --- a/others/fluentd/fluentd-dev.conf +++ /dev/null @@ -1,28 +0,0 @@ - - @type forward - port 24224 - bind 0.0.0.0 - - - - @type http - endpoint http://host.docker.internal:3000/logs.json - - flush_at_shutdown true - flush_mode immediate - flush_thread_count 8 - flush_thread_interval 1 - flush_thread_burst_interval 1 - retry_forever true - retry_type exponential_backoff - - - - - @type parser - key_name log - reserve_data true - - @type json - - \ No newline at end of file diff --git a/others/traefik/docker-compose-tcp.yaml b/others/traefik/docker-compose-tcp.yaml deleted file mode 100644 index 110630d2e..000000000 --- a/others/traefik/docker-compose-tcp.yaml +++ /dev/null @@ -1,23 +0,0 @@ -version: '3.5' - -services: - ${ID}: - container_name: proxy-for-${PORT} - image: traefik:v2.6 - command: - - --api.insecure=true - - --entrypoints.web.address=:${PORT} - - --providers.docker=false - - --providers.docker.exposedbydefault=false - - --providers.http.endpoint=http://host.docker.internal:3000/traefik.json?id=${ID} - - --providers.http.pollTimeout=5s - - --log.level=error - ports: - - '${PORT}:${PORT}' - networks: - - ${NETWORK} - -networks: - net: - external: false - name: ${NETWORK} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ad1e1c1a5..98d3a1183 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -33,6 +33,8 @@ importers: bree: 9.1.2 cabin: 9.1.2 compare-versions: 5.0.1 + csv-parse: ^5.3.0 + csvtojson: ^2.0.10 cuid: 2.1.8 dayjs: 1.11.5 dockerode: 3.3.4 @@ -65,7 +67,7 @@ importers: typescript: 4.8.2 unique-names-generator: 4.7.1 dependencies: - '@breejs/ts-worker': 2.0.0_d3un4r7p64mpe4ydkpns6lvpxy + '@breejs/ts-worker': 2.0.0_zx7xfusupi724hd5vcuaoj6jni '@fastify/autoload': 5.3.1 '@fastify/cookie': 8.1.0 '@fastify/cors': 8.1.0 @@ -80,6 +82,8 @@ importers: bree: 9.1.2 cabin: 9.1.2 compare-versions: 5.0.1 + csv-parse: 5.3.0 + csvtojson: 2.0.10 cuid: 2.1.8 dayjs: 1.11.5 dockerode: 3.3.4 @@ -241,11 +245,12 @@ packages: engines: {node: '>= 10'} dev: false - /@breejs/ts-worker/2.0.0_d3un4r7p64mpe4ydkpns6lvpxy: + /@breejs/ts-worker/2.0.0_zx7xfusupi724hd5vcuaoj6jni: resolution: {integrity: sha512-6anHRcmgYlF7mrm/YVRn6rx2cegLuiY3VBxkkimOTWC/dVQeH336imVSuIKEGKTwiuNTPr2hswVdDSneNuXg3A==} engines: {node: '>= 12.11'} peerDependencies: bree: '>=9.0.0' + tsconfig-paths: '>= 4' dependencies: bree: 9.1.2 ts-node: 10.8.2_r4hqq7vrw4pxsipnb7ha25ylfe @@ -1870,6 +1875,10 @@ packages: readable-stream: 3.6.0 dev: false + /bluebird/3.7.2: + resolution: {integrity: sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==} + dev: false + /bn.js/4.12.0: resolution: {integrity: sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==} dev: false @@ -2292,6 +2301,20 @@ packages: engines: {node: '>=4'} hasBin: true + /csv-parse/5.3.0: + resolution: {integrity: sha512-UXJCGwvJ2fep39purtAn27OUYmxB1JQto+zhZ4QlJpzsirtSFbzLvip1aIgziqNdZp/TptvsKEV5BZSxe10/DQ==} + dev: false + + /csvtojson/2.0.10: + resolution: {integrity: sha512-lUWFxGKyhraKCW8Qghz6Z0f2l/PqB1W3AO0HKJzGIQ5JRSlR651ekJDiGJbBT4sRNNv5ddnSGVEnsxP9XRCVpQ==} + engines: {node: '>=4.0.0'} + hasBin: true + dependencies: + bluebird: 3.7.2 + lodash: 4.17.21 + strip-bom: 2.0.0 + dev: false + /cuid/2.1.8: resolution: {integrity: sha512-xiEMER6E7TlTPnDxrM4eRiC6TRgjNX9xzEZ5U/Se2YJKr7Mq4pJn/2XEHjl3STcSh96GmkHPcBXLES8M29wyyg==} dev: false @@ -3899,6 +3922,10 @@ packages: has-symbols: 1.0.3 dev: true + /is-utf8/0.2.1: + resolution: {integrity: sha512-rMYPYvCzsXywIsldgLaSoPlw5PfoB/ssr7hY4pLfcodrA5M/eArza1a9VmTiNIBNMjOGr1Ow9mTyU2o69U6U9Q==} + dev: false + /is-uuid/1.0.2: resolution: {integrity: sha512-tCByphFcJgf2qmiMo5hMCgNAquNSagOetVetDvBXswGkNfoyEMvGH1yDlF8cbZbKnbVBr4Y5/rlpMz9umxyBkQ==} dev: false @@ -5595,6 +5622,13 @@ packages: ansi-regex: 6.0.1 dev: false + /strip-bom/2.0.0: + resolution: {integrity: sha512-kwrX1y7czp1E69n2ajbG65mIo9dqvJ+8aBQXOGVxqwvNbsXdFM6Lq37dLAY3mknUwru8CfcCbfOLL/gMo+fi3g==} + engines: {node: '>=0.10.0'} + dependencies: + is-utf8: 0.2.1 + dev: false + /strip-bom/3.0.0: resolution: {integrity: sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==} engines: {node: '>=4'}