feat: use registry for building

apps/api/src/index.ts

@@ -202,14 +202,14 @@ async function getIPAddress() {
 	try {
 		const settings = await listSettings();
 		if (!settings.ipv4) {
-			console.log(`Getting public IPv4 address...`);
 			const ipv4 = await publicIpv4({ timeout: 2000 })
+			console.log(`Getting public IPv4 address...`);
 			await prisma.setting.update({ where: { id: settings.id }, data: { ipv4 } })
 		}
 
 		if (!settings.ipv6) {
-			console.log(`Getting public IPv6 address...`);
 			const ipv6 = await publicIpv6({ timeout: 2000 })
+			console.log(`Getting public IPv6 address...`);
 			await prisma.setting.update({ where: { id: settings.id }, data: { ipv6 } })
 		}
 
@@ -223,13 +223,13 @@ async function getTagsTemplates() {
 			const tags = await fs.readFile('./devTags.json', 'utf8')
 			await fs.writeFile('./templates.json', JSON.stringify(yaml.load(templates)))
 			await fs.writeFile('./tags.json', tags)
-			console.log('Tags and templates loaded in dev mode...')
+			console.log('[004] Tags and templates loaded in dev mode...')
 		} else {
 			const tags = await got.get('https://get.coollabs.io/coolify/service-tags.json').text()
 			const response = await got.get('https://get.coollabs.io/coolify/service-templates.yaml').text()
 			await fs.writeFile('/app/templates.json', JSON.stringify(yaml.load(response)))
 			await fs.writeFile('/app/tags.json', tags)
-			console.log('Tags and templates loaded...')
+			console.log('[004] Tags and templates loaded...')
 		}
 
 	} catch (error) {
@@ -239,15 +239,22 @@ async function getTagsTemplates() {
 }
 async function initServer() {
 	try {
-		console.log(`Initializing server...`);
+		console.log(`[001] Initializing server...`);
 		await asyncExecShell(`docker network create --attachable coolify`);
 	} catch (error) { }
 	try {
+		console.log(`[002] Set stuck builds to failed...`);
 		const isOlder = compareVersions('3.8.1', version);
 		if (isOlder === 1) {
 			await prisma.build.updateMany({ where: { status: { in: ['running', 'queued'] } }, data: { status: 'failed' } });
 		}
 	} catch (error) { }
+	try {
+		console.log('[003] Cleaning up old build sources under /tmp/build-sources/...');
+		await fs.rm('/tmp/build-sources', { recursive: true, force: true })
+	} catch (error) {
+		console.log(error)
+	}
 }
 
 async function getArch() {

apps/api/src/jobs/deployApplication.ts

@@ -38,56 +38,70 @@ import * as buildpacks from '../lib/buildPacks';
 					for (const queueBuild of queuedBuilds) {
 						actions.push(async () => {
 							let application = await prisma.application.findUnique({ where: { id: queueBuild.applicationId }, include: { destinationDocker: true, gitSource: { include: { githubApp: true, gitlabApp: true } }, persistentStorage: true, secrets: true, settings: true, teams: true } })
+
 							let { id: buildId, type, sourceBranch = null, pullmergeRequestId = null, previewApplicationId = null, forceRebuild, sourceRepository = null } = queueBuild
+
 							application = decryptApplication(application)
+
 							const originalApplicationId = application.id
+							const {
+								id: applicationId,
+								name,
+								destinationDocker,
+								destinationDockerId,
+								gitSource,
+								configHash,
+								fqdn,
+								projectId,
+								secrets,
+								phpModules,
+								settings,
+								persistentStorage,
+								pythonWSGI,
+								pythonModule,
+								pythonVariable,
+								denoOptions,
+								exposePort,
+								baseImage,
+								baseBuildImage,
+								deploymentType,
+							} = application
+
+							let {
+								branch,
+								repository,
+								buildPack,
+								port,
+								installCommand,
+								buildCommand,
+								startCommand,
+								baseDirectory,
+								publishDirectory,
+								dockerFileLocation,
+								dockerComposeConfiguration,
+								denoMainFile
+							} = application
+
+							let imageId = applicationId;
+							let domain = getDomain(fqdn);
+
 							if (pullmergeRequestId) {
 								const previewApplications = await prisma.previewApplication.findMany({ where: { applicationId: originalApplicationId, pullmergeRequestId } })
 								if (previewApplications.length > 0) {
 									previewApplicationId = previewApplications[0].id
 								}
+								// Previews, we need to get the source branch and set subdomain
+								branch = sourceBranch;
+								domain = `${pullmergeRequestId}.${domain}`;
+								imageId = `${applicationId}-${pullmergeRequestId}`;
+								repository = sourceRepository || repository;
 							}
-							const usableApplicationId = previewApplicationId || originalApplicationId
+							const { workdir, repodir } = await createDirectories({ repository, buildId });
 							try {
 								if (queueBuild.status === 'running') {
 									await saveBuildLog({ line: 'Building halted, restarting...', buildId, applicationId: application.id });
 								}
-								const {
+
-									id: applicationId,
-									name,
-									destinationDocker,
-									destinationDockerId,
-									gitSource,
-									configHash,
-									fqdn,
-									projectId,
-									secrets,
-									phpModules,
-									settings,
-									persistentStorage,
-									pythonWSGI,
-									pythonModule,
-									pythonVariable,
-									denoOptions,
-									exposePort,
-									baseImage,
-									baseBuildImage,
-									deploymentType,
-								} = application
-								let {
-									branch,
-									repository,
-									buildPack,
-									port,
-									installCommand,
-									buildCommand,
-									startCommand,
-									baseDirectory,
-									publishDirectory,
-									dockerFileLocation,
-									dockerComposeConfiguration,
-									denoMainFile
-								} = application
 								const currentHash = crypto
 									.createHash('sha256')
 									.update(
@@ -113,22 +127,14 @@ import * as buildpacks from '../lib/buildPacks';
 									)
 									.digest('hex');
 								const { debug } = settings;
-								let imageId = applicationId;
-								let domain = getDomain(fqdn);
 								const volumes =
 									persistentStorage?.map((storage) => {
 										if (storage.oldPath) {
-											return `${applicationId}${storage.path.replace(/\//gi, '-').replace('-app','')}:${storage.path}`;
+											return `${applicationId}${storage.path.replace(/\//gi, '-').replace('-app', '')}:${storage.path}`;
 										}
 										return `${applicationId}${storage.path.replace(/\//gi, '-')}:${storage.path}`;
 									}) || [];
-								// Previews, we need to get the source branch and set subdomain
+
-								if (pullmergeRequestId) {
-									branch = sourceBranch;
-									domain = `${pullmergeRequestId}.${domain}`;
-									imageId = `${applicationId}-${pullmergeRequestId}`;
-									repository = sourceRepository || repository;
-								}
 
 								try {
 									dockerComposeConfiguration = JSON.parse(dockerComposeConfiguration)
@@ -141,7 +147,7 @@ import * as buildpacks from '../lib/buildPacks';
 								}
 								if (destinationType === 'docker') {
 									await prisma.build.update({ where: { id: buildId }, data: { status: 'running' } });
-									const { workdir, repodir } = await createDirectories({ repository, buildId });
+
 									const configuration = await setDefaultConfiguration(application);
 
 									buildPack = configuration.buildPack;
@@ -419,6 +425,8 @@ import * as buildpacks from '../lib/buildPacks';
 								if (error !== 1) {
 									await saveBuildLog({ line: error, buildId, applicationId: application.id });
 								}
+							} finally {
+								await fs.rm(workdir, { recursive: true, force: true });
 							}
 						});
 					}

apps/api/src/lib/buildPacks/common.ts

@@ -1,4 +1,4 @@
-import { base64Encode, encrypt, executeDockerCmd, generateTimestamp, getDomain, isDev, prisma, version } from "../common";
+import { base64Encode, decrypt, encrypt, executeDockerCmd, generateTimestamp, getDomain, isDev, prisma, version } from "../common";
 import { promises as fs } from 'fs';
 import { day } from "../dayjs";
 
@@ -577,6 +577,28 @@ export function checkPnpm(installCommand = null, buildCommand = null, startComma
 	);
 }
 
+export async function saveDockerRegistryCredentials({ url, username, password, workdir }) {
+	let decryptedPassword = decrypt(password);
+	const location = `${workdir}/.docker`;
+
+	if (!username || !password) {
+		return null
+	}
+
+	try {
+		await fs.mkdir(`${workdir}/.docker`);
+	} catch (error) {
+		console.log(error);
+	}
+	await fs.writeFile(`${location}/config.json`, JSON.stringify({
+		"auths": {
+			[url]: {
+				"auth": Buffer.from(`${username}:${decryptedPassword}`).toString('base64')
+			}
+		}
+	}))
+	return location
+}
 export async function buildImage({
 	applicationId,
 	tag,
@@ -602,8 +624,9 @@ export async function buildImage({
 	}
 	const dockerFile = isCache ? `${dockerFileLocation}-cache` : `${dockerFileLocation}`
 	const cache = `${applicationId}:${tag}${isCache ? '-cache' : ''}`
-
+	const { dockerRegistry: { url, username, password } } = await prisma.application.findUnique({ where: { id: applicationId }, select: { dockerRegistry: true } })
-	await executeDockerCmd({ debug, buildId, applicationId, dockerId, command: `docker build --progress plain -f ${workdir}/${dockerFile} -t ${cache} --build-arg SOURCE_COMMIT=${commit} ${workdir}` })
+	const location = await saveDockerRegistryCredentials({ url, username, password, workdir })
+	await executeDockerCmd({ debug, buildId, applicationId, dockerId, command: `docker ${location ? `--config ${location}` : ''} build --progress plain -f ${workdir}/${dockerFile} -t ${cache} --build-arg SOURCE_COMMIT=${commit} ${workdir}` })
 
 	const { status } = await prisma.build.findUnique({ where: { id: buildId } })
 	if (status === 'canceled') {