v1.0.9 (#37)

Features: - Integrated the first service: [Plausible Analytics](https://plausible.io)! Fixes: - UI/UX fixes and new designs
2021-04-22 23:48:29 +02:00
parent f742c2a3e2
commit 3744c64459
32 changed files with 997 additions and 109 deletions
--- a/api/app.js
+++ b/api/app.js
@@ -12,6 +12,8 @@ module.exports = async function (fastify, opts) {
    server.register(require('./routes/v1/application/deploy'), { prefix: '/application/deploy' })
    server.register(require('./routes/v1/application/deploy/logs'), { prefix: '/application/deploy/logs' })
    server.register(require('./routes/v1/databases'), { prefix: '/databases' })
+    server.register(require('./routes/v1/services'), { prefix: '/services' })
+    server.register(require('./routes/v1/services/deploy'), { prefix: '/services/deploy' })
    server.register(require('./routes/v1/server'), { prefix: '/server' })
  })
  // Public routes
--- a/api/libs/applications/configuration.js
+++ b/api/libs/applications/configuration.js
@@ -2,7 +2,7 @@ const { uniqueNamesGenerator, adjectives, colors, animals } = require('unique-na
 const cuid = require('cuid')
 const crypto = require('crypto')
 const { docker } = require('../docker')
-const { execShellAsync } = require('../common')
+const { execShellAsync, baseServiceConfiguration } = require('../common')

 function getUniq () {
  return uniqueNamesGenerator({ dictionaries: [adjectives, animals, colors], length: 2 })
@@ -15,25 +15,6 @@ function setDefaultConfiguration (configuration) {
  const shaBase = JSON.stringify({ repository: configuration.repository })
  const sha256 = crypto.createHash('sha256').update(shaBase).digest('hex')

-  const baseServiceConfiguration = {
-    replicas: 1,
-    restart_policy: {
-      condition: 'any',
-      max_attempts: 3
-    },
-    update_config: {
-      parallelism: 1,
-      delay: '10s',
-      order: 'start-first'
-    },
-    rollback_config: {
-      parallelism: 1,
-      delay: '10s',
-      order: 'start-first',
-      failure_action: 'rollback'
-    }
-  }
-
  configuration.build.container.name = sha256.slice(0, 15)

  configuration.general.nickname = nickname
@@ -133,4 +114,4 @@ async function precheckDeployment ({ services, configuration }) {
    forceUpdate
  }
 }
-module.exports = { setDefaultConfiguration, updateServiceLabels, precheckDeployment }
+module.exports = { setDefaultConfiguration, updateServiceLabels, precheckDeployment, baseServiceConfiguration }
--- a/api/libs/common.js
+++ b/api/libs/common.js
@@ -6,6 +6,24 @@ const User = require('../models/User')
 const algorithm = 'aes-256-cbc'
 const key = process.env.SECRETS_ENCRYPTION_KEY

+const baseServiceConfiguration = {
+  replicas: 1,
+  restart_policy: {
+    condition: 'any',
+    max_attempts: 3
+  },
+  update_config: {
+    parallelism: 1,
+    delay: '10s',
+    order: 'start-first'
+  },
+  rollback_config: {
+    parallelism: 1,
+    delay: '10s',
+    order: 'start-first',
+    failure_action: 'rollback'
+  }
+}
 function delay (t) {
  return new Promise(function (resolve) {
    setTimeout(function () {
@@ -94,5 +112,6 @@ module.exports = {
  checkImageAvailable,
  encryptData,
  decryptData,
-  verifyUserId
+  verifyUserId,
+  baseServiceConfiguration
 }
--- a/api/libs/services/plausible/index.js
+++ b/api/libs/services/plausible/index.js
@@ -0,0 +1,185 @@
+const { execShellAsync, cleanupTmp, baseServiceConfiguration } = require('../../common')
+const yaml = require('js-yaml')
+const fs = require('fs').promises
+const generator = require('generate-password')
+const { docker } = require('../../docker')
+
+async function plausible ({ email, userName, userPassword, baseURL, traefikURL }) {
+  const deployId = 'plausible'
+  const workdir = '/tmp/plausible'
+  const secretKey = generator.generate({ length: 64, numbers: true, strict: true })
+  const generateEnvsPostgres = {
+    POSTGRESQL_PASSWORD: generator.generate({ length: 24, numbers: true, strict: true }),
+    POSTGRESQL_USERNAME: generator.generate({ length: 10, numbers: true, strict: true }),
+    POSTGRESQL_DATABASE: 'plausible'
+  }
+
+  const secrets = [
+    { name: 'ADMIN_USER_EMAIL', value: email },
+    { name: 'ADMIN_USER_NAME', value: userName },
+    { name: 'ADMIN_USER_PWD', value: userPassword },
+    { name: 'BASE_URL', value: baseURL },
+    { name: 'SECRET_KEY_BASE', value: secretKey },
+    { name: 'DISABLE_AUTH', value: 'false' },
+    { name: 'DISABLE_REGISTRATION', value: 'true' },
+    { name: 'DATABASE_URL', value: `postgresql://${generateEnvsPostgres.POSTGRESQL_USERNAME}:${generateEnvsPostgres.POSTGRESQL_PASSWORD}@plausible_db:5432/${generateEnvsPostgres.POSTGRESQL_DATABASE}` },
+    { name: 'CLICKHOUSE_DATABASE_URL', value: 'http://plausible_events_db:8123/plausible' }
+  ]
+
+  const generateEnvsClickhouse = {}
+  for (const secret of secrets) generateEnvsClickhouse[secret.name] = secret.value
+
+  const clickhouseConfigXml = `
+    <yandex>
+      <logger>
+          <level>warning</level>
+          <console>true</console>
+      </logger>
+
+      <!-- Stop all the unnecessary logging -->
+      <query_thread_log remove="remove"/>
+      <query_log remove="remove"/>
+      <text_log remove="remove"/>
+      <trace_log remove="remove"/>
+      <metric_log remove="remove"/>
+      <asynchronous_metric_log remove="remove"/>
+  </yandex>`
+  const clickhouseUserConfigXml = `
+    <yandex>
+      <profiles>
+          <default>
+              <log_queries>0</log_queries>
+              <log_query_threads>0</log_query_threads>
+          </default>
+      </profiles>
+  </yandex>`
+
+  const clickhouseConfigs = [
+    { source: 'plausible-clickhouse-user-config.xml', target: '/etc/clickhouse-server/users.d/logging.xml' },
+    { source: 'plausible-clickhouse-config.xml', target: '/etc/clickhouse-server/config.d/logging.xml' },
+    { source: 'plausible-init.query', target: '/docker-entrypoint-initdb.d/init.query' },
+    { source: 'plausible-init-db.sh', target: '/docker-entrypoint-initdb.d/init-db.sh' }
+  ]
+
+  const initQuery = 'CREATE DATABASE IF NOT EXISTS plausible;'
+  const initScript = 'clickhouse client --queries-file /docker-entrypoint-initdb.d/init.query'
+  await execShellAsync(`mkdir -p ${workdir}`)
+  await fs.writeFile(`${workdir}/clickhouse-config.xml`, clickhouseConfigXml)
+  await fs.writeFile(`${workdir}/clickhouse-user-config.xml`, clickhouseUserConfigXml)
+  await fs.writeFile(`${workdir}/init.query`, initQuery)
+  await fs.writeFile(`${workdir}/init-db.sh`, initScript)
+  const stack = {
+    version: '3.8',
+    services: {
+      [deployId]: {
+        image: 'plausible/analytics:latest',
+        command: 'sh -c "sleep 10 && /entrypoint.sh db createdb && /entrypoint.sh db migrate && /entrypoint.sh db init-admin && /entrypoint.sh run"',
+        networks: [`${docker.network}`],
+        volumes: [`${deployId}-postgres-data:/var/lib/postgresql/data`],
+        environment: generateEnvsClickhouse,
+        deploy: {
+          ...baseServiceConfiguration,
+          labels: [
+            'managedBy=coolify',
+            'type=service',
+            'serviceName=plausible',
+            'configuration=' + JSON.stringify({ email, userName, userPassword, baseURL, secretKey, generateEnvsPostgres, generateEnvsClickhouse }),
+            'traefik.enable=true',
+            'traefik.http.services.' +
+            deployId +
+            '.loadbalancer.server.port=8000',
+            'traefik.http.routers.' +
+            deployId +
+            '.entrypoints=websecure',
+            'traefik.http.routers.' +
+            deployId +
+            '.rule=Host(`' +
+            traefikURL +
+            '`) && PathPrefix(`/`)',
+            'traefik.http.routers.' +
+            deployId +
+            '.tls.certresolver=letsencrypt',
+            'traefik.http.routers.' +
+            deployId +
+            '.middlewares=global-compress'
+          ]
+        }
+      },
+      plausible_db: {
+        image: 'bitnami/postgresql:13.2.0',
+        networks: [`${docker.network}`],
+        environment: generateEnvsPostgres,
+        deploy: {
+          ...baseServiceConfiguration,
+          labels: [
+            'managedBy=coolify',
+            'type=service',
+            'serviceName=plausible'
+          ]
+        }
+      },
+      plausible_events_db: {
+        image: 'yandex/clickhouse-server:21.3.2.5',
+        networks: [`${docker.network}`],
+        volumes: [`${deployId}-clickhouse-data:/var/lib/clickhouse`],
+        ulimits: {
+          nofile: {
+            soft: 262144,
+            hard: 262144
+          }
+        },
+        configs: [...clickhouseConfigs],
+        deploy: {
+          ...baseServiceConfiguration,
+          labels: [
+            'managedBy=coolify',
+            'type=service',
+            'serviceName=plausible'
+          ]
+        }
+      }
+    },
+    networks: {
+      [`${docker.network}`]: {
+        external: true
+      }
+    },
+    volumes: {
+      [`${deployId}-clickhouse-data`]: {
+        external: true
+      },
+      [`${deployId}-postgres-data`]: {
+        external: true
+      }
+    },
+    configs: {
+      'plausible-clickhouse-user-config.xml': {
+        file: `${workdir}/clickhouse-user-config.xml`
+      },
+      'plausible-clickhouse-config.xml': {
+        file: `${workdir}/clickhouse-config.xml`
+      },
+      'plausible-init.query': {
+        file: `${workdir}/init.query`
+      },
+      'plausible-init-db.sh': {
+        file: `${workdir}/init-db.sh`
+      }
+    }
+  }
+  await fs.writeFile(`${workdir}/stack.yml`, yaml.dump(stack))
+  await execShellAsync('docker stack rm plausible')
+  await execShellAsync(
+    `cat ${workdir}/stack.yml | docker stack deploy --prune -c - ${deployId}`
+  )
+  cleanupTmp(workdir)
+}
+
+async function activateAdminUser () {
+  const { POSTGRESQL_USERNAME, POSTGRESQL_PASSWORD, POSTGRESQL_DATABASE } = JSON.parse(JSON.parse((await execShellAsync('docker service inspect plausible_plausible --format=\'{{json .Spec.Labels.configuration}}\'')))).generateEnvsPostgres
+  const containers = (await execShellAsync('docker ps -a --format=\'{{json .Names}}\'')).replace(/"/g, '').trim().split('\n')
+  const postgresDB = containers.find(container => container.startsWith('plausible_plausible_db'))
+  await execShellAsync(`docker exec ${postgresDB} psql -H postgresql://${POSTGRESQL_USERNAME}:${POSTGRESQL_PASSWORD}@localhost:5432/${POSTGRESQL_DATABASE} -c "UPDATE users SET email_verified = true;"`)
+}
+
+module.exports = { plausible, activateAdminUser }
--- a/api/routes/v1/dashboard/index.js
+++ b/api/routes/v1/dashboard/index.js
@@ -23,9 +23,10 @@ module.exports = async function (fastify) {
        }
      ])
      const serverLogs = await ServerLog.find()
-      const services = await docker.engine.listServices()
-      let applications = services.filter(r => r.Spec.Labels.managedBy === 'coolify' && r.Spec.Labels.type === 'application' && r.Spec.Labels.configuration)
-      let databases = services.filter(r => r.Spec.Labels.managedBy === 'coolify' && r.Spec.Labels.type === 'database' && r.Spec.Labels.configuration)
+      const dockerServices = await docker.engine.listServices()
+      let applications = dockerServices.filter(r => r.Spec.Labels.managedBy === 'coolify' && r.Spec.Labels.type === 'application' && r.Spec.Labels.configuration)
+      let databases = dockerServices.filter(r => r.Spec.Labels.managedBy === 'coolify' && r.Spec.Labels.type === 'database' && r.Spec.Labels.configuration)
+      let services = dockerServices.filter(r => r.Spec.Labels.managedBy === 'coolify' && r.Spec.Labels.type === 'service' && r.Spec.Labels.configuration)
      applications = applications.map(r => {
        if (JSON.parse(r.Spec.Labels.configuration)) {
          const configuration = JSON.parse(r.Spec.Labels.configuration)
@@ -41,6 +42,11 @@ module.exports = async function (fastify) {
        r.Spec.Labels.configuration = configuration
        return r
      })
+      services = services.map(r => {
+        const configuration = r.Spec.Labels.configuration ? JSON.parse(r.Spec.Labels.configuration) : null
+        r.Spec.Labels.configuration = configuration
+        return r
+      })
      applications = [...new Map(applications.map(item => [item.Spec.Labels.configuration.publish.domain + item.Spec.Labels.configuration.publish.path, item])).values()]
      return {
        serverLogs,
@@ -49,6 +55,9 @@ module.exports = async function (fastify) {
        },
        databases: {
          deployed: databases
+        },
+        services: {
+          deployed: services
        }
      }
    } catch (error) {
--- a/api/routes/v1/databases/index.js
+++ b/api/routes/v1/databases/index.js
@@ -18,13 +18,15 @@ module.exports = async function (fastify) {
      const database = (await docker.engine.listServices()).find(r => r.Spec.Labels.managedBy === 'coolify' && r.Spec.Labels.type === 'database' && JSON.parse(r.Spec.Labels.configuration).general.deployId === deployId)
      if (database) {
        const jsonEnvs = {}
-        for (const d of database.Spec.TaskTemplate.ContainerSpec.Env) {
-          const s = d.split('=')
-          jsonEnvs[s[0]] = s[1]
+        if (database.Spec.TaskTemplate.ContainerSpec.Env) {
+          for (const d of database.Spec.TaskTemplate.ContainerSpec.Env) {
+            const s = d.split('=')
+            jsonEnvs[s[0]] = s[1]
+          }
        }
        const payload = {
          config: JSON.parse(database.Spec.Labels.configuration),
-          envs: jsonEnvs
+          envs: jsonEnvs || null
        }
        reply.code(200).send(payload)
      } else {
@@ -39,7 +41,7 @@ module.exports = async function (fastify) {
    body: {
      type: 'object',
      properties: {
-        type: { type: 'string', enum: ['mongodb', 'postgresql', 'mysql', 'couchdb'] }
+        type: { type: 'string', enum: ['mongodb', 'postgresql', 'mysql', 'couchdb', 'clickhouse'] }
      },
      required: ['type']
    }
@@ -82,9 +84,11 @@ module.exports = async function (fastify) {
          name: nickname
        }
      }
+      await execShellAsync(`mkdir -p ${configuration.general.workdir}`)
      let generateEnvs = {}
      let image = null
      let volume = null
+      let ulimits = {}
      if (type === 'mongodb') {
        generateEnvs = {
          MONGODB_ROOT_PASSWORD: passwords[0],
@@ -119,6 +123,15 @@ module.exports = async function (fastify) {
        }
        image = 'bitnami/mysql:8.0'
        volume = `${configuration.general.deployId}-${type}-data:/bitnami/mysql/data`
+      } else if (type === 'clickhouse') {
+        image = 'yandex/clickhouse-server'
+        volume = `${configuration.general.deployId}-${type}-data:/var/lib/clickhouse`
+        ulimits = {
+          nofile: {
+            soft: 262144,
+            hard: 262144
+          }
+        }
      }

      const stack = {
@@ -129,6 +142,7 @@ module.exports = async function (fastify) {
            networks: [`${docker.network}`],
            environment: generateEnvs,
            volumes: [volume],
+            ulimits,
            deploy: {
              replicas: 1,
              update_config: {
@@ -160,12 +174,12 @@ module.exports = async function (fastify) {
          }
        }
      }
-      await execShellAsync(`mkdir -p ${configuration.general.workdir}`)
      await fs.writeFile(`${configuration.general.workdir}/stack.yml`, yaml.dump(stack))
      await execShellAsync(
-              `cat ${configuration.general.workdir}/stack.yml | docker stack deploy -c - ${configuration.general.deployId}`
+        `cat ${configuration.general.workdir}/stack.yml | docker stack deploy -c - ${configuration.general.deployId}`
      )
    } catch (error) {
+      console.log(error)
      await saveServerLog(error)
      throw new Error(error)
    }
--- a/api/routes/v1/services/deploy.js
+++ b/api/routes/v1/services/deploy.js
@@ -0,0 +1,15 @@
+const { plausible, activateAdminUser } = require('../../../libs/services/plausible')
+
+module.exports = async function (fastify) {
+  fastify.post('/plausible', async (request, reply) => {
+    let { email, userName, userPassword, baseURL } = request.body
+    const traefikURL = baseURL
+    baseURL = `https://${baseURL}`
+    await plausible({ email, userName, userPassword, baseURL, traefikURL })
+    return {}
+  })
+  fastify.patch('/plausible/activate', async (request, reply) => {
+    await activateAdminUser()
+    return 'OK'
+  })
+}
--- a/api/routes/v1/services/index.js
+++ b/api/routes/v1/services/index.js
@@ -0,0 +1,27 @@
+const { execShellAsync } = require('../../../libs/common')
+const { docker } = require('../../../libs/docker')
+
+module.exports = async function (fastify) {
+  fastify.get('/:serviceName', async (request, reply) => {
+    const { serviceName } = request.params
+    try {
+      const service = (await docker.engine.listServices()).find(r => r.Spec.Labels.managedBy === 'coolify' && r.Spec.Labels.type === 'service' && r.Spec.Labels.serviceName === serviceName && r.Spec.Name === `${serviceName}_${serviceName}`)
+      if (service) {
+        const payload = {
+          config: JSON.parse(service.Spec.Labels.configuration)
+        }
+        reply.code(200).send(payload)
+      } else {
+        throw new Error()
+      }
+    } catch (error) {
+      console.log(error)
+      throw new Error('No service found?')
+    }
+  })
+  fastify.delete('/:serviceName', async (request, reply) => {
+    const { serviceName } = request.params
+    await execShellAsync(`docker stack rm ${serviceName}`)
+    reply.code(200).send({})
+  })
+}