feat: use registry for building

apps/api/src/index.ts

@@ -202,14 +202,14 @@ async function getIPAddress() {
 	try {
 		const settings = await listSettings();
 		if (!settings.ipv4) {
-			console.log(`Getting public IPv4 address...`);
 			const ipv4 = await publicIpv4({ timeout: 2000 })
+			console.log(`Getting public IPv4 address...`);
 			await prisma.setting.update({ where: { id: settings.id }, data: { ipv4 } })
 		}
 
 		if (!settings.ipv6) {
-			console.log(`Getting public IPv6 address...`);
 			const ipv6 = await publicIpv6({ timeout: 2000 })
+			console.log(`Getting public IPv6 address...`);
 			await prisma.setting.update({ where: { id: settings.id }, data: { ipv6 } })
 		}
 
@@ -223,13 +223,13 @@ async function getTagsTemplates() {
 			const tags = await fs.readFile('./devTags.json', 'utf8')
 			await fs.writeFile('./templates.json', JSON.stringify(yaml.load(templates)))
 			await fs.writeFile('./tags.json', tags)
-			console.log('Tags and templates loaded in dev mode...')
+			console.log('[004] Tags and templates loaded in dev mode...')
 		} else {
 			const tags = await got.get('https://get.coollabs.io/coolify/service-tags.json').text()
 			const response = await got.get('https://get.coollabs.io/coolify/service-templates.yaml').text()
 			await fs.writeFile('/app/templates.json', JSON.stringify(yaml.load(response)))
 			await fs.writeFile('/app/tags.json', tags)
-			console.log('Tags and templates loaded...')
+			console.log('[004] Tags and templates loaded...')
 		}
 
 	} catch (error) {
@@ -239,15 +239,22 @@ async function getTagsTemplates() {
 }
 async function initServer() {
 	try {
-		console.log(`Initializing server...`);
+		console.log(`[001] Initializing server...`);
 		await asyncExecShell(`docker network create --attachable coolify`);
 	} catch (error) { }
 	try {
+		console.log(`[002] Set stuck builds to failed...`);
 		const isOlder = compareVersions('3.8.1', version);
 		if (isOlder === 1) {
 			await prisma.build.updateMany({ where: { status: { in: ['running', 'queued'] } }, data: { status: 'failed' } });
 		}
 	} catch (error) { }
+	try {
+		console.log('[003] Cleaning up old build sources under /tmp/build-sources/...');
+		await fs.rm('/tmp/build-sources', { recursive: true, force: true })
+	} catch (error) {
+		console.log(error)
+	}
 }
 
 async function getArch() {

apps/api/src/jobs/deployApplication.ts

@@ -38,20 +38,12 @@ import * as buildpacks from '../lib/buildPacks';
 					for (const queueBuild of queuedBuilds) {
 						actions.push(async () => {
 							let application = await prisma.application.findUnique({ where: { id: queueBuild.applicationId }, include: { destinationDocker: true, gitSource: { include: { githubApp: true, gitlabApp: true } }, persistentStorage: true, secrets: true, settings: true, teams: true } })
+
 							let { id: buildId, type, sourceBranch = null, pullmergeRequestId = null, previewApplicationId = null, forceRebuild, sourceRepository = null } = queueBuild
+
 							application = decryptApplication(application)
+
 							const originalApplicationId = application.id
-							if (pullmergeRequestId) {
-								const previewApplications = await prisma.previewApplication.findMany({ where: { applicationId: originalApplicationId, pullmergeRequestId } })
-								if (previewApplications.length > 0) {
-									previewApplicationId = previewApplications[0].id
-								}
-							}
-							const usableApplicationId = previewApplicationId || originalApplicationId
-							try {
-								if (queueBuild.status === 'running') {
-									await saveBuildLog({ line: 'Building halted, restarting...', buildId, applicationId: application.id });
-								}
 							const {
 								id: applicationId,
 								name,
@@ -74,6 +66,7 @@ import * as buildpacks from '../lib/buildPacks';
 								baseBuildImage,
 								deploymentType,
 							} = application
+
 							let {
 								branch,
 								repository,
@@ -88,6 +81,27 @@ import * as buildpacks from '../lib/buildPacks';
 								dockerComposeConfiguration,
 								denoMainFile
 							} = application
+
+							let imageId = applicationId;
+							let domain = getDomain(fqdn);
+
+							if (pullmergeRequestId) {
+								const previewApplications = await prisma.previewApplication.findMany({ where: { applicationId: originalApplicationId, pullmergeRequestId } })
+								if (previewApplications.length > 0) {
+									previewApplicationId = previewApplications[0].id
+								}
+								// Previews, we need to get the source branch and set subdomain
+								branch = sourceBranch;
+								domain = `${pullmergeRequestId}.${domain}`;
+								imageId = `${applicationId}-${pullmergeRequestId}`;
+								repository = sourceRepository || repository;
+							}
+							const { workdir, repodir } = await createDirectories({ repository, buildId });
+							try {
+								if (queueBuild.status === 'running') {
+									await saveBuildLog({ line: 'Building halted, restarting...', buildId, applicationId: application.id });
+								}
+
 								const currentHash = crypto
 									.createHash('sha256')
 									.update(
@@ -113,8 +127,6 @@ import * as buildpacks from '../lib/buildPacks';
 									)
 									.digest('hex');
 								const { debug } = settings;
-								let imageId = applicationId;
-								let domain = getDomain(fqdn);
 								const volumes =
 									persistentStorage?.map((storage) => {
 										if (storage.oldPath) {
@@ -122,13 +134,7 @@ import * as buildpacks from '../lib/buildPacks';
 										}
 										return `${applicationId}${storage.path.replace(/\//gi, '-')}:${storage.path}`;
 									}) || [];
-								// Previews, we need to get the source branch and set subdomain
+
-								if (pullmergeRequestId) {
-									branch = sourceBranch;
-									domain = `${pullmergeRequestId}.${domain}`;
-									imageId = `${applicationId}-${pullmergeRequestId}`;
-									repository = sourceRepository || repository;
-								}
 
 								try {
 									dockerComposeConfiguration = JSON.parse(dockerComposeConfiguration)
@@ -141,7 +147,7 @@ import * as buildpacks from '../lib/buildPacks';
 								}
 								if (destinationType === 'docker') {
 									await prisma.build.update({ where: { id: buildId }, data: { status: 'running' } });
-									const { workdir, repodir } = await createDirectories({ repository, buildId });
+
 									const configuration = await setDefaultConfiguration(application);
 
 									buildPack = configuration.buildPack;
@@ -419,6 +425,8 @@ import * as buildpacks from '../lib/buildPacks';
 								if (error !== 1) {
 									await saveBuildLog({ line: error, buildId, applicationId: application.id });
 								}
+							} finally {
+								await fs.rm(workdir, { recursive: true, force: true });
 							}
 						});
 					}

apps/api/src/lib/buildPacks/common.ts

@@ -1,4 +1,4 @@
-import { base64Encode, encrypt, executeDockerCmd, generateTimestamp, getDomain, isDev, prisma, version } from "../common";
+import { base64Encode, decrypt, encrypt, executeDockerCmd, generateTimestamp, getDomain, isDev, prisma, version } from "../common";
 import { promises as fs } from 'fs';
 import { day } from "../dayjs";
 
@@ -577,6 +577,28 @@ export function checkPnpm(installCommand = null, buildCommand = null, startComma
 	);
 }
 
+export async function saveDockerRegistryCredentials({ url, username, password, workdir }) {
+	let decryptedPassword = decrypt(password);
+	const location = `${workdir}/.docker`;
+
+	if (!username || !password) {
+		return null
+	}
+
+	try {
+		await fs.mkdir(`${workdir}/.docker`);
+	} catch (error) {
+		console.log(error);
+	}
+	await fs.writeFile(`${location}/config.json`, JSON.stringify({
+		"auths": {
+			[url]: {
+				"auth": Buffer.from(`${username}:${decryptedPassword}`).toString('base64')
+			}
+		}
+	}))
+	return location
+}
 export async function buildImage({
 	applicationId,
 	tag,
@@ -602,8 +624,9 @@ export async function buildImage({
 	}
 	const dockerFile = isCache ? `${dockerFileLocation}-cache` : `${dockerFileLocation}`
 	const cache = `${applicationId}:${tag}${isCache ? '-cache' : ''}`
-
+	const { dockerRegistry: { url, username, password } } = await prisma.application.findUnique({ where: { id: applicationId }, select: { dockerRegistry: true } })
-	await executeDockerCmd({ debug, buildId, applicationId, dockerId, command: `docker build --progress plain -f ${workdir}/${dockerFile} -t ${cache} --build-arg SOURCE_COMMIT=${commit} ${workdir}` })
+	const location = await saveDockerRegistryCredentials({ url, username, password, workdir })
+	await executeDockerCmd({ debug, buildId, applicationId, dockerId, command: `docker ${location ? `--config ${location}` : ''} build --progress plain -f ${workdir}/${dockerFile} -t ${cache} --build-arg SOURCE_COMMIT=${commit} ${workdir}` })
 
 	const { status } = await prisma.build.findUnique({ where: { id: buildId } })
 	if (status === 'canceled') {