dave/coolify
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

fix: application persistent storage things

Browse Source
This commit is contained in:
Andras Bacsai
2022-11-14 10:40:28 +01:00
parent bec6b961f3
commit b4f9d29129
10 changed files with 203 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
apps/api/prisma/migrations/20221114093217_application_storage_path_migration/migration.sql Normal file
View File

@@ -0,0 +1,45 @@
-- RedefineTables
PRAGMA foreign_keys=OFF;
CREATE TABLE "new_Setting" (
"id" TEXT NOT NULL PRIMARY KEY,
"fqdn" TEXT,
"isAPIDebuggingEnabled" BOOLEAN DEFAULT false,
"isRegistrationEnabled" BOOLEAN NOT NULL DEFAULT false,
"dualCerts" BOOLEAN NOT NULL DEFAULT false,
"minPort" INTEGER NOT NULL DEFAULT 9000,
"maxPort" INTEGER NOT NULL DEFAULT 9100,
"proxyPassword" TEXT NOT NULL,
"proxyUser" TEXT NOT NULL,
"proxyHash" TEXT,
"proxyDefaultRedirect" TEXT,
"isAutoUpdateEnabled" BOOLEAN NOT NULL DEFAULT false,
"isDNSCheckEnabled" BOOLEAN NOT NULL DEFAULT true,
"DNSServers" TEXT,
"isTraefikUsed" BOOLEAN NOT NULL DEFAULT true,
"createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updatedAt" DATETIME NOT NULL,
"ipv4" TEXT,
"ipv6" TEXT,
"arch" TEXT,
"concurrentBuilds" INTEGER NOT NULL DEFAULT 1,
"applicationStoragePathMigrationFinished" BOOLEAN NOT NULL DEFAULT false
);
INSERT INTO "new_Setting" ("DNSServers", "arch", "concurrentBuilds", "createdAt", "dualCerts", "fqdn", "id", "ipv4", "ipv6", "isAPIDebuggingEnabled", "isAutoUpdateEnabled", "isDNSCheckEnabled", "isRegistrationEnabled", "isTraefikUsed", "maxPort", "minPort", "proxyDefaultRedirect", "proxyHash", "proxyPassword", "proxyUser", "updatedAt") SELECT "DNSServers", "arch", "concurrentBuilds", "createdAt", "dualCerts", "fqdn", "id", "ipv4", "ipv6", "isAPIDebuggingEnabled", "isAutoUpdateEnabled", "isDNSCheckEnabled", "isRegistrationEnabled", "isTraefikUsed", "maxPort", "minPort", "proxyDefaultRedirect", "proxyHash", "proxyPassword", "proxyUser", "updatedAt" FROM "Setting";
DROP TABLE "Setting";
ALTER TABLE "new_Setting" RENAME TO "Setting";
CREATE UNIQUE INDEX "Setting_fqdn_key" ON "Setting"("fqdn");
CREATE TABLE "new_ApplicationPersistentStorage" (
"id" TEXT NOT NULL PRIMARY KEY,
"applicationId" TEXT NOT NULL,
"path" TEXT NOT NULL,
"oldPath" BOOLEAN NOT NULL DEFAULT false,
"createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updatedAt" DATETIME NOT NULL,
CONSTRAINT "ApplicationPersistentStorage_applicationId_fkey" FOREIGN KEY ("applicationId") REFERENCES "Application" ("id") ON DELETE RESTRICT ON UPDATE CASCADE
);
INSERT INTO "new_ApplicationPersistentStorage" ("applicationId", "createdAt", "id", "path", "updatedAt") SELECT "applicationId", "createdAt", "id", "path", "updatedAt" FROM "ApplicationPersistentStorage";
DROP TABLE "ApplicationPersistentStorage";
ALTER TABLE "new_ApplicationPersistentStorage" RENAME TO "ApplicationPersistentStorage";
CREATE UNIQUE INDEX "ApplicationPersistentStorage_applicationId_path_key" ON "ApplicationPersistentStorage"("applicationId", "path");
PRAGMA foreign_key_check;
PRAGMA foreign_keys=ON;

44
apps/api/prisma/schema.prisma
View File

@@ -19,27 +19,28 @@ model Certificate {
}
model Setting {
id String @id @default(cuid())
fqdn String? @unique
isAPIDebuggingEnabled Boolean? @default(false)
isRegistrationEnabled Boolean @default(false)
dualCerts Boolean @default(false)
minPort Int @default(9000)
maxPort Int @default(9100)
proxyPassword String
proxyUser String
proxyHash String?
proxyDefaultRedirect String?
isAutoUpdateEnabled Boolean @default(false)
isDNSCheckEnabled Boolean @default(true)
DNSServers String?
isTraefikUsed Boolean @default(true)
createdAt DateTime @default(now())
updatedAt DateTime @updatedAt
ipv4 String?
ipv6 String?
arch String?
concurrentBuilds Int @default(1)
id String @id @default(cuid())
fqdn String? @unique
isAPIDebuggingEnabled Boolean? @default(false)
isRegistrationEnabled Boolean @default(false)
dualCerts Boolean @default(false)
minPort Int @default(9000)
maxPort Int @default(9100)
proxyPassword String
proxyUser String
proxyHash String?
proxyDefaultRedirect String?
isAutoUpdateEnabled Boolean @default(false)
isDNSCheckEnabled Boolean @default(true)
DNSServers String?
isTraefikUsed Boolean @default(true)
createdAt DateTime @default(now())
updatedAt DateTime @updatedAt
ipv4 String?
ipv6 String?
arch String?
concurrentBuilds Int @default(1)
applicationStoragePathMigrationFinished Boolean @default(false)
}
model User {
@@ -186,6 +187,7 @@ model ApplicationPersistentStorage {
id String @id @default(cuid())
applicationId String
path String
oldPath Boolean @default(false)
createdAt DateTime @default(now())
updatedAt DateTime @updatedAt
application Application @relation(fields: [applicationId], references: [id])

5
apps/api/src/index.ts
View File

@@ -17,7 +17,7 @@ import yaml from 'js-yaml'
import fs from 'fs/promises';
import { verifyRemoteDockerEngineFn } from './routes/api/v1/destinations/handlers';
import { checkContainer } from './lib/docker';
import { migrateServicesToNewTemplate } from './lib';
import { migrateApplicationPersistentStorage, migrateServicesToNewTemplate } from './lib';
import { refreshTags, refreshTemplates } from './routes/api/v1/handlers';
declare module 'fastify' {
@@ -142,7 +142,8 @@ const host = '0.0.0.0';
await socketIOServer(fastify)
console.log(`Coolify's API is listening on ${host}:${port}`);
migrateServicesToNewTemplate()
migrateServicesToNewTemplate();
await migrateApplicationPersistentStorage();
await initServer();
const graceful = new Graceful({ brees: [scheduler] });

6
apps/api/src/jobs/deployApplication.ts
View File

@@ -117,8 +117,10 @@ import * as buildpacks from '../lib/buildPacks';
let domain = getDomain(fqdn);
const volumes =
persistentStorage?.map((storage) => {
return `${applicationId}${storage.path.replace(/\//gi, '-')}:${buildPack !== 'docker' ? '/app' : ''
}${storage.path}`;
if (storage.oldPath) {
return `${applicationId}${storage.path.replace(/\//gi, '-').replace('-app','')}:${storage.path}`;
}
return `${applicationId}${storage.path.replace(/\//gi, '-')}:${storage.path}`;
}) || [];
// Previews, we need to get the source branch and set subdomain
if (pullmergeRequestId) {

32
apps/api/src/lib.ts
View File

@@ -2,6 +2,32 @@ import cuid from "cuid";
import { decrypt, encrypt, fixType, generatePassword, prisma } from "./lib/common";
import { getTemplates } from "./lib/services";
export async function migrateApplicationPersistentStorage() {
const settings = await prisma.setting.findFirst()
if (settings) {
const { id: settingsId, applicationStoragePathMigrationFinished } = settings
try {
if (!applicationStoragePathMigrationFinished) {
const applications = await prisma.application.findMany({ include: { persistentStorage: true } });
for (const application of applications) {
if (application.persistentStorage && application.persistentStorage.length > 0 && application?.buildPack !== 'docker') {
for (const storage of application.persistentStorage) {
let { id, path } = storage
if (!path.startsWith('/app')) {
path = `/app${path}`
await prisma.applicationPersistentStorage.update({ where: { id }, data: { path, oldPath: true } })
}
}
}
}
}
} catch (error) {
console.log(error)
} finally {
await prisma.setting.update({ where: { id: settingsId }, data: { applicationStoragePathMigrationFinished: true } })
}
}
}
export async function migrateServicesToNewTemplate() {
// This function migrates old hardcoded services to the new template based services
try {
@@ -456,9 +482,9 @@ async function migrateSettings(settings: any[], service: any, template: any) {
variableName = `$$config_${name.toLowerCase()}`
}
// console.log('Migrating setting', name, value, 'for service', service.id, ', service name:', service.name, 'variableName: ', variableName)
await prisma.serviceSetting.findFirst({ where: { name: minio, serviceId: service.id } }) || await prisma.serviceSetting.create({ data: { name: minio, value, variableName, service: { connect: { id: service.id } } } })
} catch(error) {
} catch (error) {
console.log(error)
}
}
@@ -473,7 +499,7 @@ async function migrateSecrets(secrets: any[], service: any) {
}
// console.log('Migrating secret', name, value, 'for service', service.id, ', service name:', service.name)
await prisma.serviceSecret.findFirst({ where: { name, serviceId: service.id } }) || await prisma.serviceSecret.create({ data: { name, value, service: { connect: { id: service.id } } } })
} catch(error) {
} catch (error) {
console.log(error)
}
}

26
apps/api/src/lib/buildPacks/compose.ts
View File

@@ -64,13 +64,14 @@ export default async function (data) {
} catch (error) {
//
}
const composeVolumes = volumes.map((volume) => {
return {
[`${volume.split(':')[0]}`]: {
name: volume.split(':')[0]
}
};
});
const composeVolumes = [];
for (const volume of volumes) {
let [v, _] = volume.split(':');
composeVolumes[v] = {
name: v,
}
}
let networks = {}
for (let [key, value] of Object.entries(dockerComposeYaml.services)) {
value['container_name'] = `${applicationId}-${key}`
@@ -79,16 +80,19 @@ export default async function (data) {
// TODO: If we support separated volume for each service, we need to add it here
if (value['volumes'].length > 0) {
value['volumes'] = value['volumes'].map((volume) => {
let [v, path] = volume.split(':');
let [v, path, permission] = volume.split(':');
v = `${applicationId}-${v}`
composeVolumes[v] = {
name: v
}
return `${v}:${path}`
return `${v}:${path}${permission ? ':' + permission : ''}`
})
}
if (volumes.length > 0) {
value['volumes'] = [...value['volumes'] || '', volumes]
for (const volume of volumes) {
value['volumes'].push(volume)
}
}
if (dockerComposeConfiguration[key].port) {
value['expose'] = [dockerComposeConfiguration[key].port]
@@ -104,7 +108,7 @@ export default async function (data) {
dockerComposeYaml.services[key] = { ...dockerComposeYaml.services[key], restart: defaultComposeConfiguration(network).restart, deploy: defaultComposeConfiguration(network).deploy }
}
if (Object.keys(composeVolumes).length > 0) {
dockerComposeYaml['volumes'] = {...composeVolumes}
dockerComposeYaml['volumes'] = { ...composeVolumes }
}
dockerComposeYaml['networks'] = Object.assign({ ...networks }, { [network]: { external: true } })
await fs.writeFile(`${workdir}/docker-compose.${isYml ? 'yml' : 'yaml'}`, yaml.dump(dockerComposeYaml));