ci: Docker move build stage outside container (no-changelog) (#16009)

scripts/build-n8n.mjs

@@ -0,0 +1,219 @@
+#!/usr/bin/env node
+/**
+ * This script is used to build the n8n application for production.
+ * It will:
+ * 1. Clean the previous build output
+ * 2. Run pnpm install and build
+ * 3. Prepare for deployment - clean package.json files
+ * 4. Create a pruned production deployment in 'compiled'
+ */
+
+import { $, echo, fs, chalk } from 'zx';
+import path from 'path';
+
+// Check if running in a CI environment
+const isCI = process.env.CI === 'true';
+
+// Disable verbose output and force color only if not in CI
+$.verbose = !isCI;
+process.env.FORCE_COLOR = isCI ? '0' : '1';
+
+const scriptDir = path.dirname(new URL(import.meta.url).pathname);
+const isInScriptsDir = path.basename(scriptDir) === 'scripts';
+const rootDir = isInScriptsDir ? path.join(scriptDir, '..') : scriptDir;
+
+// --- Configuration ---
+const config = {
+	compiledAppDir: process.env.BUILD_OUTPUT_DIR || path.join(rootDir, 'compiled'),
+	rootDir: rootDir,
+};
+
+// Define backend patches to keep during deployment
+const PATCHES_TO_KEEP = ['pdfjs-dist', 'pkce-challenge', 'bull'];
+
+// --- Helper Functions ---
+const timers = new Map();
+
+function startTimer(name) {
+	timers.set(name, Date.now());
+}
+
+function getElapsedTime(name) {
+	const start = timers.get(name);
+	if (!start) return 0;
+	return Math.floor((Date.now() - start) / 1000);
+}
+
+function formatDuration(seconds) {
+	const hours = Math.floor(seconds / 3600);
+	const minutes = Math.floor((seconds % 3600) / 60);
+	const secs = seconds % 60;
+
+	if (hours > 0) return `${hours}h ${minutes}m ${secs}s`;
+	if (minutes > 0) return `${minutes}m ${secs}s`;
+	return `${secs}s`;
+}
+
+function printHeader(title) {
+	echo('');
+	echo(chalk.blue.bold(`===== ${title} =====`));
+}
+
+function printDivider() {
+	echo(chalk.gray('-----------------------------------------------'));
+}
+
+// --- Main Build Process ---
+printHeader('n8n Build & Production Preparation');
+echo(`INFO: Output Directory: ${config.compiledAppDir}`);
+printDivider();
+
+startTimer('total_build');
+
+// 0. Clean Previous Build Output
+echo(chalk.yellow(`INFO: Cleaning previous output directory: ${config.compiledAppDir}...`));
+await fs.remove(config.compiledAppDir);
+printDivider();
+
+// 1. Local Application Pre-build
+echo(chalk.yellow('INFO: Starting local application pre-build...'));
+startTimer('package_build');
+
+echo(chalk.yellow('INFO: Running pnpm install and build...'));
+await $`cd ${config.rootDir} && pnpm install --frozen-lockfile`;
+await $`cd ${config.rootDir} && pnpm build`;
+echo(chalk.green('✅ pnpm install and build completed'));
+
+const packageBuildTime = getElapsedTime('package_build');
+echo(chalk.green(`✅ Package build completed in ${formatDuration(packageBuildTime)}`));
+printDivider();
+
+// 2. Prepare for deployment - clean package.json files
+echo(chalk.yellow('INFO: Performing pre-deploy cleanup on package.json files...'));
+
+// Find and backup package.json files
+const packageJsonFiles = await $`cd ${config.rootDir} && find . -name "package.json" \
+-not -path "./node_modules/*" \
+-not -path "*/node_modules/*" \
+-not -path "./compiled/*" \
+-type f`.lines();
+
+// Backup all package.json files
+// This is only needed locally, not in CI
+if (process.env.CI !== 'true') {
+	for (const file of packageJsonFiles) {
+		if (file) {
+			const fullPath = path.join(config.rootDir, file);
+			await fs.copy(fullPath, `${fullPath}.bak`);
+		}
+	}
+}
+// Run FE trim script
+await $`cd ${config.rootDir} && node .github/scripts/trim-fe-packageJson.js`;
+echo(chalk.yellow('INFO: Performing selective patch cleanup...'));
+
+const packageJsonPath = path.join(config.rootDir, 'package.json');
+
+if (await fs.pathExists(packageJsonPath)) {
+	try {
+		// 1. Read the package.json file
+		const packageJsonContent = await fs.readFile(packageJsonPath, 'utf8');
+		let packageJson = JSON.parse(packageJsonContent);
+
+		// 2. Modify the patchedDependencies directly in JavaScript
+		if (packageJson.pnpm && packageJson.pnpm.patchedDependencies) {
+			const filteredPatches = {};
+			for (const [key, value] of Object.entries(packageJson.pnpm.patchedDependencies)) {
+				// Check if the key (patch name) starts with any of the allowed patches
+				const shouldKeep = PATCHES_TO_KEEP.some((patchPrefix) => key.startsWith(patchPrefix));
+				if (shouldKeep) {
+					filteredPatches[key] = value;
+				}
+			}
+			packageJson.pnpm.patchedDependencies = filteredPatches;
+		}
+
+		// 3. Write the modified package.json back
+		await fs.writeFile(packageJsonPath, JSON.stringify(packageJson, null, 2), 'utf8');
+
+		echo(chalk.green('✅ Kept backend patches: ' + PATCHES_TO_KEEP.join(', ')));
+		echo(
+			chalk.gray(
+				`Removed FE/dev patches that are not in the list of backend patches to keep: ${PATCHES_TO_KEEP.join(', ')}`,
+			),
+		);
+	} catch (error) {
+		echo(chalk.red(`ERROR: Failed to cleanup patches in package.json: ${error.message}`));
+		process.exit(1);
+	}
+}
+
+echo(chalk.yellow(`INFO: Creating pruned production deployment in '${config.compiledAppDir}'...`));
+startTimer('package_deploy');
+
+await fs.ensureDir(config.compiledAppDir);
+
+await $`cd ${config.rootDir} && NODE_ENV=production DOCKER_BUILD=true pnpm --filter=n8n --prod --legacy deploy --no-optional ./compiled`;
+
+const packageDeployTime = getElapsedTime('package_deploy');
+
+// Restore package.json files
+// This is only needed locally, not in CI
+if (process.env.CI !== 'true') {
+	for (const file of packageJsonFiles) {
+		if (file) {
+			const fullPath = path.join(config.rootDir, file);
+			const backupPath = `${fullPath}.bak`;
+			if (await fs.pathExists(backupPath)) {
+				await fs.move(backupPath, fullPath, { overwrite: true });
+			}
+		}
+	}
+}
+
+// Calculate output size
+const compiledAppOutputSize = (await $`du -sh ${config.compiledAppDir} | cut -f1`).stdout.trim();
+
+// Generate build manifest
+const buildManifest = {
+	buildTime: new Date().toISOString(),
+	artifactSize: compiledAppOutputSize,
+	buildDuration: {
+		packageBuild: packageBuildTime,
+		packageDeploy: packageDeployTime,
+		total: getElapsedTime('total_build'),
+	},
+};
+
+await fs.writeJson(path.join(config.compiledAppDir, 'build-manifest.json'), buildManifest, {
+	spaces: 2,
+});
+
+echo(chalk.green(`✅ Package deployment completed in ${formatDuration(packageDeployTime)}`));
+echo(`INFO: Size of ${config.compiledAppDir}: ${compiledAppOutputSize}`);
+printDivider();
+
+// Calculate total time
+const totalBuildTime = getElapsedTime('total_build');
+
+// --- Final Output ---
+echo('');
+echo(chalk.green.bold('================ BUILD SUMMARY ================'));
+echo(chalk.green(`✅ n8n built successfully!`));
+echo('');
+echo(chalk.blue('📦 Build Output:'));
+echo(`   Directory:      ${path.resolve(config.compiledAppDir)}`);
+echo(`   Size:           ${compiledAppOutputSize}`);
+echo('');
+echo(chalk.blue('⏱️  Build Times:'));
+echo(`   Package Build:  ${formatDuration(packageBuildTime)}`);
+echo(`   Package Deploy: ${formatDuration(packageDeployTime)}`);
+echo(chalk.gray('   -----------------------------'));
+echo(chalk.bold(`   Total Time:     ${formatDuration(totalBuildTime)}`));
+echo('');
+echo(chalk.blue('📋 Build Manifest:'));
+echo(`   ${path.resolve(config.compiledAppDir)}/build-manifest.json`);
+echo(chalk.green.bold('=============================================='));
+
+// Exit with success
+process.exit(0);

scripts/dockerize-n8n.mjs

@@ -0,0 +1,88 @@
+#!/usr/bin/env node
+/**
+ * This script is used to build the n8n docker image locally.
+ * It simulates how we build for CI and should allow for local testing.
+ * By default it outputs the tag 'dev' and the image name 'n8n-local:dev'.
+ * It can be overridden by setting the IMAGE_BASE_NAME and IMAGE_TAG environment variables.
+ */
+
+import { $, echo, fs, chalk } from 'zx';
+import path from 'path';
+
+// Disable verbose mode for cleaner output
+$.verbose = false;
+process.env.FORCE_COLOR = '1';
+
+// --- Determine script location ---
+const scriptDir = path.dirname(new URL(import.meta.url).pathname);
+const isInScriptsDir = path.basename(scriptDir) === 'scripts';
+const rootDir = isInScriptsDir ? path.join(scriptDir, '..') : scriptDir;
+
+// --- Configuration ---
+const config = {
+	dockerfilePath: path.join(rootDir, 'docker/images/n8n/Dockerfile'),
+	imageBaseName: process.env.IMAGE_BASE_NAME || 'n8n-local',
+	imageTag: process.env.IMAGE_TAG || 'dev',
+	buildContext: rootDir,
+	compiledAppDir: path.join(rootDir, 'compiled'),
+};
+
+config.fullImageName = `${config.imageBaseName}:${config.imageTag}`;
+
+// --- Check Prerequisites ---
+echo(chalk.blue.bold('===== Docker Build for n8n ====='));
+echo(`INFO: Image: ${config.fullImageName}`);
+echo(chalk.gray('-----------------------------------------------'));
+
+// Check if compiled directory exists
+if (!(await fs.pathExists(config.compiledAppDir))) {
+	echo(chalk.red(`Error: Compiled app directory not found at ${config.compiledAppDir}`));
+	echo(chalk.yellow('Please run build-n8n.mjs first!'));
+	process.exit(1);
+}
+
+// Check Docker
+try {
+	await $`command -v docker`;
+} catch {
+	echo(chalk.red('Error: Docker is not installed or not in PATH'));
+	process.exit(1);
+}
+
+// --- Build Docker Image ---
+const startTime = Date.now();
+echo(chalk.yellow('INFO: Building Docker image...'));
+
+try {
+	const buildOutput = await $`docker build \
+		-t ${config.fullImageName} \
+		-f ${config.dockerfilePath} \
+		${config.buildContext}`;
+
+	echo(buildOutput.stdout);
+} catch (error) {
+	echo(chalk.red(`ERROR: Docker build failed: ${error.stderr || error.message}`));
+	process.exit(1);
+}
+
+const buildTime = Math.floor((Date.now() - startTime) / 1000);
+
+// Get image size
+let imageSize = 'Unknown';
+try {
+	const sizeOutput = await $`docker images ${config.fullImageName} --format "{{.Size}}"`;
+	imageSize = sizeOutput.stdout.trim();
+} catch (error) {
+	echo(chalk.yellow('Warning: Could not get image size'));
+}
+
+// --- Summary ---
+echo('');
+echo(chalk.green.bold('================ DOCKER BUILD COMPLETE ================'));
+echo(chalk.green(`✅ Image built: ${config.fullImageName}`));
+echo(`   Size: ${imageSize}`);
+echo(`   Build time: ${buildTime}s`);
+echo(chalk.green.bold('===================================================='));
+
+// Exit with success
+process.exit(0);

scripts/scan-n8n-image.mjs

@@ -0,0 +1,152 @@
+#!/usr/bin/env node
+/**
+ * This script is used to scan the n8n docker image for vulnerabilities.
+ * It uses Trivy to scan the image.
+ */
+
+import { $, echo, fs, chalk } from 'zx';
+import path from 'path';
+
+$.verbose = false;
+process.env.FORCE_COLOR = '1';
+
+const scriptDir = path.dirname(new URL(import.meta.url).pathname);
+const isInScriptsDir = path.basename(scriptDir) === 'scripts';
+const rootDir = isInScriptsDir ? path.join(scriptDir, '..') : scriptDir;
+
+// --- Configuration ---
+const config = {
+	imageBaseName: process.env.IMAGE_BASE_NAME || 'n8n-local',
+	imageTag: process.env.IMAGE_TAG || 'dev',
+	trivyImage: process.env.TRIVY_IMAGE || 'aquasec/trivy:latest',
+	severity: process.env.TRIVY_SEVERITY || 'CRITICAL,HIGH,MEDIUM,LOW',
+	outputFormat: process.env.TRIVY_FORMAT || 'table',
+	outputFile: process.env.TRIVY_OUTPUT || null,
+	scanTimeout: process.env.TRIVY_TIMEOUT || '10m',
+	ignoreUnfixed: process.env.TRIVY_IGNORE_UNFIXED === 'true',
+	scanners: process.env.TRIVY_SCANNERS || 'vuln',
+	quiet: process.env.TRIVY_QUIET === 'true',
+	rootDir: rootDir,
+};
+
+config.fullImageName = `${config.imageBaseName}:${config.imageTag}`;
+
+const printHeader = (title) =>
+	!config.quiet && echo(`\n${chalk.blue.bold(`===== ${title} =====`)}`);
+
+const printSummary = (status, time, message) => {
+	if (config.quiet) return;
+
+	echo('\n' + chalk.blue.bold('===== Scan Summary ====='));
+	echo(status === 'success' ? chalk.green.bold(message) : chalk.yellow.bold(message));
+	echo(chalk[status === 'success' ? 'green' : 'yellow'](`   Scan time: ${time}s`));
+
+	if (config.outputFile) {
+		const resolvedPath = path.isAbsolute(config.outputFile)
+			? config.outputFile
+			: path.join(config.rootDir, config.outputFile);
+		echo(chalk[status === 'success' ? 'green' : 'yellow'](`   Report saved to: ${resolvedPath}`));
+	}
+
+	echo('\n' + chalk.gray('Scan Configuration:'));
+	echo(chalk.gray(`  • Target Image: ${config.fullImageName}`));
+	echo(chalk.gray(`  • Severity Levels: ${config.severity}`));
+	echo(chalk.gray(`  • Scanners: ${config.scanners}`));
+	if (config.ignoreUnfixed) echo(chalk.gray(`  • Ignored unfixed: yes`));
+	echo(chalk.blue.bold('========================'));
+};
+
+// --- Main Process ---
+(async () => {
+	printHeader('Trivy Security Scan for n8n Image');
+
+	try {
+		await $`command -v docker`;
+	} catch {
+		echo(chalk.red('Error: Docker is not installed or not in PATH'));
+		process.exit(1);
+	}
+
+	try {
+		await $`docker image inspect ${config.fullImageName} > /dev/null 2>&1`;
+	} catch {
+		echo(chalk.red(`Error: Docker image '${config.fullImageName}' not found`));
+		echo(chalk.yellow('Please run dockerize-n8n.mjs first!'));
+		process.exit(1);
+	}
+
+	// Pull latest Trivy image silently
+	try {
+		await $`docker pull ${config.trivyImage} > /dev/null 2>&1`;
+	} catch {
+		// Silent fallback to cached version
+	}
+
+	// Build Trivy command
+	const trivyArgs = [
+		'run',
+		'--rm',
+		'-v',
+		'/var/run/docker.sock:/var/run/docker.sock',
+		config.trivyImage,
+		'image',
+		'--severity',
+		config.severity,
+		'--format',
+		config.outputFormat,
+		'--timeout',
+		config.scanTimeout,
+		'--scanners',
+		config.scanners,
+		'--no-progress',
+	];
+
+	if (config.ignoreUnfixed) trivyArgs.push('--ignore-unfixed');
+	if (config.quiet && config.outputFormat === 'table') trivyArgs.push('--quiet');
+
+	// Handle output file - resolve relative to root directory
+	if (config.outputFile) {
+		const outputPath = path.isAbsolute(config.outputFile)
+			? config.outputFile
+			: path.join(config.rootDir, config.outputFile);
+		await fs.ensureDir(path.dirname(outputPath));
+		trivyArgs.push('--output', '/tmp/trivy-output', '-v', `${outputPath}:/tmp/trivy-output`);
+	}
+
+	trivyArgs.push(config.fullImageName);
+
+	// Run the scan
+	const startTime = Date.now();
+
+	try {
+		const result = await $`docker ${trivyArgs}`;
+
+		// Print Trivy output first
+		if (!config.outputFile && result.stdout) {
+			echo(result.stdout);
+		}
+
+		// Then print our summary
+		const scanTime = Math.floor((Date.now() - startTime) / 1000);
+		printSummary('success', scanTime, '✅ Security scan completed successfully');
+
+		process.exit(0);
+	} catch (error) {
+		const scanTime = Math.floor((Date.now() - startTime) / 1000);
+
+		// Trivy returns exit code 1 when vulnerabilities are found
+		if (error.exitCode === 1) {
+			// Print Trivy output first
+			if (!config.outputFile && error.stdout) {
+				echo(error.stdout);
+			}
+
+			// Then print our summary
+			printSummary('warning', scanTime, '⚠️  Vulnerabilities found!');
+			process.exit(1);
+		} else {
+			echo(chalk.red(`❌ Scan failed: ${error.message}`));
+			process.exit(error.exitCode || 1);
+		}
+	}
+})();