v1.0.11 (#41)

Features: 
- Build packs for popular frontend frameworks. It will help to understand which build packs should be chosen.

Fixes:
- Github queries optimized.
- Save repositories to store (faster navigation).
- Remove unnecessary data on dashboard requests.
- Speed up static site builds with a lot.

UI:
- Redesign of the application deployment page.
- Redesign of database deployments page.

api/buildPacks/gatsby/index.js

@@ -0,0 +1,25 @@
+const fs = require('fs').promises
+const { buildImage } = require('../helpers')
+const { streamEvents, docker } = require('../../libs/docker')
+
+//    'HEALTHCHECK --timeout=10s --start-period=10s --interval=5s CMD curl -I -s -f http://localhost/ || exit 1',
+const publishStaticDocker = (configuration) => {
+  return [
+    'FROM nginx:stable-alpine',
+    'COPY nginx.conf /etc/nginx/nginx.conf',
+    'WORKDIR /usr/share/nginx/html',
+    `COPY --from=${configuration.build.container.name}:${configuration.build.container.tag}-cache /usr/src/app/${configuration.publish.directory} ./`,
+    'EXPOSE 80',
+    'CMD ["nginx", "-g", "daemon off;"]'
+  ].join('\n')
+}
+
+module.exports = async function (configuration) {
+  await buildImage(configuration, true)
+  await fs.writeFile(`${configuration.general.workdir}/Dockerfile`, publishStaticDocker(configuration))
+  const stream = await docker.engine.buildImage(
+    { src: ['.'], context: configuration.general.workdir },
+    { t: `${configuration.build.container.name}:${configuration.build.container.tag}` }
+  )
+  await streamEvents(stream, configuration)
+}

api/buildPacks/helpers.js

@@ -10,11 +10,11 @@ const buildImageNodeDocker = (configuration) => {
     `RUN ${configuration.build.command.build}`
   ].join('\n')
 }
-async function buildImage (configuration) {
+async function buildImage (configuration, cacheBuild) {
   await fs.writeFile(`${configuration.general.workdir}/Dockerfile`, buildImageNodeDocker(configuration))
   const stream = await docker.engine.buildImage(
     { src: ['.'], context: configuration.general.workdir },
-    { t: `${configuration.build.container.name}:${configuration.build.container.tag}` }
+    { t: `${configuration.build.container.name}:${cacheBuild ? `${configuration.build.container.tag}-cache` : configuration.build.container.tag}` }
   )
   await streamEvents(stream, configuration)
 }

api/buildPacks/index.js

@@ -1,7 +1,13 @@
-const static = require('./static')
+const Static = require('./static')
+const react = require('./react')
+const nextjs = require('./nextjs')
+const nuxtjs = require('./nuxtjs')
+const gatsby = require('./gatsby')
+const vuejs = require('./vuejs')
+const svelte = require('./svelte')
 const nodejs = require('./nodejs')
 const php = require('./php')
-const custom = require('./custom')
+const docker = require('./docker')
 const rust = require('./rust')
 
-module.exports = { static, nodejs, php, custom, rust }
+module.exports = { static: Static, nodejs, php, docker, rust, react, vuejs, nextjs, nuxtjs, svelte, gatsby }

api/buildPacks/nextjs/index.js

@@ -0,0 +1,28 @@
+const fs = require('fs').promises
+const { buildImage } = require('../helpers')
+const { streamEvents, docker } = require('../../libs/docker')
+//  `HEALTHCHECK --timeout=10s --start-period=10s --interval=5s CMD curl -I -s -f http://localhost:${configuration.publish.port}${configuration.publish.path} || exit 1`,
+const publishNodejsDocker = (configuration) => {
+  return [
+    'FROM node:lts',
+    'WORKDIR /usr/src/app',
+    configuration.build.command.build
+      ? `COPY --from=${configuration.build.container.name}:${configuration.build.container.tag} /usr/src/app/${configuration.publish.directory} ./`
+      : `
+      COPY ${configuration.build.directory}/package*.json ./
+      RUN ${configuration.build.command.installation}
+      COPY ./${configuration.build.directory} ./`,
+    `EXPOSE ${configuration.publish.port}`,
+    'CMD [ "yarn", "start" ]'
+  ].join('\n')
+}
+
+module.exports = async function (configuration) {
+  await buildImage(configuration)
+  await fs.writeFile(`${configuration.general.workdir}/Dockerfile`, publishNodejsDocker(configuration))
+  const stream = await docker.engine.buildImage(
+    { src: ['.'], context: configuration.general.workdir },
+    { t: `${configuration.build.container.name}:${configuration.build.container.tag}` }
+  )
+  await streamEvents(stream, configuration)
+}

api/buildPacks/nuxtjs/index.js

@@ -0,0 +1,28 @@
+const fs = require('fs').promises
+const { buildImage } = require('../helpers')
+const { streamEvents, docker } = require('../../libs/docker')
+//  `HEALTHCHECK --timeout=10s --start-period=10s --interval=5s CMD curl -I -s -f http://localhost:${configuration.publish.port}${configuration.publish.path} || exit 1`,
+const publishNodejsDocker = (configuration) => {
+  return [
+    'FROM node:lts',
+    'WORKDIR /usr/src/app',
+    configuration.build.command.build
+      ? `COPY --from=${configuration.build.container.name}:${configuration.build.container.tag} /usr/src/app/${configuration.publish.directory} ./`
+      : `
+      COPY ${configuration.build.directory}/package*.json ./
+      RUN ${configuration.build.command.installation}
+      COPY ./${configuration.build.directory} ./`,
+    `EXPOSE ${configuration.publish.port}`,
+    'CMD [ "yarn", "start" ]'
+  ].join('\n')
+}
+
+module.exports = async function (configuration) {
+  await buildImage(configuration)
+  await fs.writeFile(`${configuration.general.workdir}/Dockerfile`, publishNodejsDocker(configuration))
+  const stream = await docker.engine.buildImage(
+    { src: ['.'], context: configuration.general.workdir },
+    { t: `${configuration.build.container.name}:${configuration.build.container.tag}` }
+  )
+  await streamEvents(stream, configuration)
+}

api/buildPacks/react/index.js

@@ -0,0 +1,25 @@
+const fs = require('fs').promises
+const { buildImage } = require('../helpers')
+const { streamEvents, docker } = require('../../libs/docker')
+
+//    'HEALTHCHECK --timeout=10s --start-period=10s --interval=5s CMD curl -I -s -f http://localhost/ || exit 1',
+const publishStaticDocker = (configuration) => {
+  return [
+    'FROM nginx:stable-alpine',
+    'COPY nginx.conf /etc/nginx/nginx.conf',
+    'WORKDIR /usr/share/nginx/html',
+    `COPY --from=${configuration.build.container.name}:${configuration.build.container.tag}-cache /usr/src/app/${configuration.publish.directory} ./`,
+    'EXPOSE 80',
+    'CMD ["nginx", "-g", "daemon off;"]'
+  ].join('\n')
+}
+
+module.exports = async function (configuration) {
+  await buildImage(configuration, true)
+  await fs.writeFile(`${configuration.general.workdir}/Dockerfile`, publishStaticDocker(configuration))
+  const stream = await docker.engine.buildImage(
+    { src: ['.'], context: configuration.general.workdir },
+    { t: `${configuration.build.container.name}:${configuration.build.container.tag}` }
+  )
+  await streamEvents(stream, configuration)
+}

api/buildPacks/static/index.js

@@ -9,7 +9,7 @@ const publishStaticDocker = (configuration) => {
     'COPY nginx.conf /etc/nginx/nginx.conf',
     'WORKDIR /usr/share/nginx/html',
     configuration.build.command.build
-      ? `COPY --from=${configuration.build.container.name}:${configuration.build.container.tag} /usr/src/app/${configuration.publish.directory} ./`
+      ? `COPY --from=${configuration.build.container.name}:${configuration.build.container.tag}-cache /usr/src/app/${configuration.publish.directory} ./`
       : `COPY ./${configuration.build.directory} ./`,
     'EXPOSE 80',
     'CMD ["nginx", "-g", "daemon off;"]'
@@ -17,9 +17,8 @@ const publishStaticDocker = (configuration) => {
 }
 
 module.exports = async function (configuration) {
-  if (configuration.build.command.build) await buildImage(configuration)
+  if (configuration.build.command.build) await buildImage(configuration, true)
   await fs.writeFile(`${configuration.general.workdir}/Dockerfile`, publishStaticDocker(configuration))
-
   const stream = await docker.engine.buildImage(
     { src: ['.'], context: configuration.general.workdir },
     { t: `${configuration.build.container.name}:${configuration.build.container.tag}` }

api/buildPacks/svelte/index.js

@@ -0,0 +1,25 @@
+const fs = require('fs').promises
+const { buildImage } = require('../helpers')
+const { streamEvents, docker } = require('../../libs/docker')
+
+//    'HEALTHCHECK --timeout=10s --start-period=10s --interval=5s CMD curl -I -s -f http://localhost/ || exit 1',
+const publishStaticDocker = (configuration) => {
+  return [
+    'FROM nginx:stable-alpine',
+    'COPY nginx.conf /etc/nginx/nginx.conf',
+    'WORKDIR /usr/share/nginx/html',
+    `COPY --from=${configuration.build.container.name}:${configuration.build.container.tag}-cache /usr/src/app/${configuration.publish.directory} ./`,
+    'EXPOSE 80',
+    'CMD ["nginx", "-g", "daemon off;"]'
+  ].join('\n')
+}
+
+module.exports = async function (configuration) {
+  await buildImage(configuration, true)
+  await fs.writeFile(`${configuration.general.workdir}/Dockerfile`, publishStaticDocker(configuration))
+  const stream = await docker.engine.buildImage(
+    { src: ['.'], context: configuration.general.workdir },
+    { t: `${configuration.build.container.name}:${configuration.build.container.tag}` }
+  )
+  await streamEvents(stream, configuration)
+}

api/buildPacks/vuejs/index.js

@@ -0,0 +1,25 @@
+const fs = require('fs').promises
+const { buildImage } = require('../helpers')
+const { streamEvents, docker } = require('../../libs/docker')
+
+//    'HEALTHCHECK --timeout=10s --start-period=10s --interval=5s CMD curl -I -s -f http://localhost/ || exit 1',
+const publishStaticDocker = (configuration) => {
+  return [
+    'FROM nginx:stable-alpine',
+    'COPY nginx.conf /etc/nginx/nginx.conf',
+    'WORKDIR /usr/share/nginx/html',
+    `COPY --from=${configuration.build.container.name}:${configuration.build.container.tag}-cache /usr/src/app/${configuration.publish.directory} ./`,
+    'EXPOSE 80',
+    'CMD ["nginx", "-g", "daemon off;"]'
+  ].join('\n')
+}
+
+module.exports = async function (configuration) {
+  await buildImage(configuration, true)
+  await fs.writeFile(`${configuration.general.workdir}/Dockerfile`, publishStaticDocker(configuration))
+  const stream = await docker.engine.buildImage(
+    { src: ['.'], context: configuration.general.workdir },
+    { t: `${configuration.build.container.name}:${configuration.build.container.tag}` }
+  )
+  await streamEvents(stream, configuration)
+}

api/libs/applications/cleanup/index.js

@@ -2,11 +2,16 @@ const { docker } = require('../../docker')
 const { execShellAsync } = require('../../common')
 const Deployment = require('../../../models/Deployment')
 
-async function purgeImagesContainers (configuration) {
+async function purgeImagesContainers (configuration, deleteAll = false) {
   const { name, tag } = configuration.build.container
   await execShellAsync('docker container prune -f')
-  const IDsToDelete = (await execShellAsync(`docker images ls --filter=reference='${name}' --filter=before='${name}:${tag}' --format '{{json .ID }}'`)).trim().replace(/"/g, '').split('\n')
-  if (IDsToDelete.length !== 0) for (const id of IDsToDelete) await execShellAsync(`docker rmi -f ${id}`)
+  if (deleteAll) {
+    const IDsToDelete = (await execShellAsync(`docker images ls --filter=reference='${name}' --format '{{json .ID }}'`)).trim().replace(/"/g, '').split('\n')
+    if (IDsToDelete.length > 0) await execShellAsync(`docker rmi -f ${IDsToDelete.toString().replace(',', ' ')}`)
+  } else {
+    const IDsToDelete = (await execShellAsync(`docker images ls --filter=reference='${name}' --filter=before='${name}:${tag}' --format '{{json .ID }}'`)).trim().replace(/"/g, '').split('\n')
+    if (IDsToDelete.length > 1) await execShellAsync(`docker rmi -f ${IDsToDelete.toString().replace(',', ' ')}`)
+  }
   await execShellAsync('docker image prune -f')
 }
 

api/libs/applications/configuration.js

@@ -23,14 +23,10 @@ function setDefaultConfiguration (configuration) {
 
   if (!configuration.publish.path) configuration.publish.path = '/'
   if (!configuration.publish.port) {
-    if (configuration.build.pack === 'php') {
-      configuration.publish.port = 80
-    } else if (configuration.build.pack === 'static') {
-      configuration.publish.port = 80
-    } else if (configuration.build.pack === 'nodejs') {
-      configuration.publish.port = 3000
-    } else if (configuration.build.pack === 'rust') {
+    if (configuration.build.pack === 'nodejs' && configuration.build.pack === 'vuejs' && configuration.build.pack === 'nuxtjs' && configuration.build.pack === 'rust' && configuration.build.pack === 'nextjs') {
       configuration.publish.port = 3000
+    } else {
+      configuration.publish.port = 80
     }
   }
 

api/libs/applications/deploy/copyFiles.js

@@ -1,5 +1,6 @@
 const fs = require('fs').promises
 module.exports = async function (configuration) {
+  const staticDeployments = ['react', 'vuejs', 'static', 'svelte', 'gatsby']
   try {
     // TODO: Write full .dockerignore for all deployments!!
     if (configuration.build.pack === 'php') {
@@ -12,7 +13,7 @@ module.exports = async function (configuration) {
       `)
     }
     // await fs.writeFile(`${configuration.general.workdir}/.dockerignore`, 'node_modules')
-    if (configuration.build.pack === 'static') {
+    if (staticDeployments.includes(configuration.build.pack)) {
       await fs.writeFile(
         `${configuration.general.workdir}/nginx.conf`,
         `user  nginx;

api/libs/applications/github/cloneRepository.js

@@ -1,38 +1,44 @@
 const jwt = require('jsonwebtoken')
 const axios = require('axios')
-const { execShellAsync, cleanupTmp } = require('../../common')
+const { execShellAsync } = require('../../common')
 
 module.exports = async function (configuration) {
-  const { workdir } = configuration.general
-  const { organization, name, branch } = configuration.repository
-  const github = configuration.github
-
-  const githubPrivateKey = process.env.GITHUB_APP_PRIVATE_KEY.replace(/\\n/g, '\n').replace(/"/g, '')
-
-  const payload = {
-    iat: Math.round(new Date().getTime() / 1000),
-    exp: Math.round(new Date().getTime() / 1000 + 60),
-    iss: parseInt(github.app.id)
-  }
-
-  const jwtToken = jwt.sign(payload, githubPrivateKey, {
-    algorithm: 'RS256'
-  })
-  const accessToken = await axios({
-    method: 'POST',
-    url: `https://api.github.com/app/installations/${github.installation.id}/access_tokens`,
-    data: {},
-    headers: {
-      Authorization: 'Bearer ' + jwtToken,
-      Accept: 'application/vnd.github.machine-man-preview+json'
+  try {
+    const { workdir } = configuration.general
+    const { organization, name, branch } = configuration.repository
+    const github = configuration.github
+    if (!github.installation.id || !github.app.id) {
+      throw new Error('Github installation ID is invalid.')
     }
-  })
-  await execShellAsync(
-      `mkdir -p ${workdir} && git clone -q -b ${branch} https://x-access-token:${accessToken.data.token}@github.com/${organization}/${name}.git ${workdir}/`
-  )
-  configuration.build.container.tag = (
-    await execShellAsync(`cd ${configuration.general.workdir}/ && git rev-parse HEAD`)
-  )
-    .replace('\n', '')
-    .slice(0, 7)
+    const githubPrivateKey = process.env.GITHUB_APP_PRIVATE_KEY.replace(/\\n/g, '\n').replace(/"/g, '')
+
+    const payload = {
+      iat: Math.round(new Date().getTime() / 1000),
+      exp: Math.round(new Date().getTime() / 1000 + 60),
+      iss: parseInt(github.app.id)
+    }
+
+    const jwtToken = jwt.sign(payload, githubPrivateKey, {
+      algorithm: 'RS256'
+    })
+    const accessToken = await axios({
+      method: 'POST',
+      url: `https://api.github.com/app/installations/${github.installation.id}/access_tokens`,
+      data: {},
+      headers: {
+        Authorization: 'Bearer ' + jwtToken,
+        Accept: 'application/vnd.github.machine-man-preview+json'
+      }
+    })
+    await execShellAsync(
+        `mkdir -p ${workdir} && git clone -q -b ${branch} https://x-access-token:${accessToken.data.token}@github.com/${organization}/${name}.git ${workdir}/`
+    )
+    configuration.build.container.tag = (
+      await execShellAsync(`cd ${configuration.general.workdir}/ && git rev-parse HEAD`)
+    )
+      .replace('\n', '')
+      .slice(0, 7)
+  } catch (error) {
+    throw new Error(error)
+  }
 }

api/routes/v1/application/remove.js

@@ -1,7 +1,8 @@
 const { docker } = require('../../../libs/docker')
-const { execShellAsync } = require('../../../libs/common')
+const { execShellAsync, delay } = require('../../../libs/common')
 const ApplicationLog = require('../../../models/Logs/Application')
 const Deployment = require('../../../models/Deployment')
+const { purgeImagesContainers } = require('../../../libs/applications/cleanup')
 
 module.exports = async function (fastify) {
   fastify.post('/', async (request, reply) => {
@@ -25,6 +26,8 @@ module.exports = async function (fastify) {
         }
         await execShellAsync(`docker stack rm ${found.build.container.name}`)
         reply.code(200).send({ organization, name, branch })
+        await delay(10000)
+        await purgeImagesContainers(found, true)
       } else {
         reply.code(500).send({ message: 'Nothing to do.' })
       }

api/routes/v1/dashboard/index.js

@@ -1,27 +1,10 @@
 const { docker } = require('../../../libs/docker')
-const Deployment = require('../../../models/Deployment')
 const ServerLog = require('../../../models/Logs/Server')
 const { saveServerLog } = require('../../../libs/logging')
 
 module.exports = async function (fastify) {
   fastify.get('/', async (request, reply) => {
     try {
-      const latestDeployments = await Deployment.aggregate([
-        {
-          $sort: { createdAt: -1 }
-        },
-        {
-          $group:
-          {
-            _id: {
-              repoId: '$repoId',
-              branch: '$branch'
-            },
-            createdAt: { $last: '$createdAt' },
-            progress: { $first: '$progress' }
-          }
-        }
-      ])
       const serverLogs = await ServerLog.find()
       const dockerServices = await docker.engine.listServices()
       let applications = dockerServices.filter(r => r.Spec.Labels.managedBy === 'coolify' && r.Spec.Labels.type === 'application' && r.Spec.Labels.configuration)
@@ -29,25 +12,31 @@ module.exports = async function (fastify) {
       let services = dockerServices.filter(r => r.Spec.Labels.managedBy === 'coolify' && r.Spec.Labels.type === 'service' && r.Spec.Labels.configuration)
       applications = applications.map(r => {
         if (JSON.parse(r.Spec.Labels.configuration)) {
-          const configuration = JSON.parse(r.Spec.Labels.configuration)
-          const status = latestDeployments.find(l => configuration.repository.id === l._id.repoId && configuration.repository.branch === l._id.branch)
-          if (status && status.progress) r.progress = status.progress
-          r.Spec.Labels.configuration = configuration
-          return r
+          return {
+            configuration: JSON.parse(r.Spec.Labels.configuration),
+            UpdatedAt: r.UpdatedAt
+          }
         }
         return {}
       })
       databases = databases.map(r => {
-        const configuration = r.Spec.Labels.configuration ? JSON.parse(r.Spec.Labels.configuration) : null
-        r.Spec.Labels.configuration = configuration
-        return r
+        if (JSON.parse(r.Spec.Labels.configuration)) {
+          return {
+            configuration: JSON.parse(r.Spec.Labels.configuration)
+          }
+        }
+        return {}
       })
       services = services.map(r => {
-        const configuration = r.Spec.Labels.configuration ? JSON.parse(r.Spec.Labels.configuration) : null
-        r.Spec.Labels.configuration = configuration
-        return r
+        if (JSON.parse(r.Spec.Labels.configuration)) {
+          return {
+            serviceName: r.Spec.Labels.serviceName,
+            configuration: JSON.parse(r.Spec.Labels.configuration)
+          }
+        }
+        return {}
       })
-      applications = [...new Map(applications.map(item => [item.Spec.Labels.configuration.publish.domain + item.Spec.Labels.configuration.publish.path, item])).values()]
+      applications = [...new Map(applications.map(item => [item.configuration.publish.domain + item.configuration.publish.path, item])).values()]
       return {
         serverLogs,
         applications: {

api/routes/v1/databases/index.js

@@ -66,7 +66,6 @@ module.exports = async function (fastify) {
       if (!defaultDatabaseName) defaultDatabaseName = nickname
 
       reply.code(201).send({ message: 'Deploying.' })
-      // TODO: Persistent volume, custom inputs
       const deployId = cuid()
       const configuration = {
         general: {

api/routes/v1/webhooks/deploy.js

@@ -12,7 +12,6 @@ const cloneRepository = require('../../../libs/applications/github/cloneReposito
 const { purgeImagesContainers } = require('../../../libs/applications/cleanup')
 
 module.exports = async function (fastify) {
-  // TODO: Add this to fastify plugin
   const postSchema = {
     body: {
       type: 'object',

api/server.js

@@ -1,12 +1,11 @@
 require('dotenv').config()
 const fs = require('fs')
 const util = require('util')
-const axios = require('axios')
 const mongoose = require('mongoose')
 const path = require('path')
 const { saveServerLog } = require('./libs/logging')
 const { execShellAsync } = require('./libs/common')
-const { purgeImagesContainers, cleanupStuckedDeploymentsInDB } = require('./libs/applications/cleanup')
+const { cleanupStuckedDeploymentsInDB } = require('./libs/applications/cleanup')
 const fastify = require('fastify')({
   trustProxy: true,
   logger: {