feat(benchmark): New options for n8n benchmark (#10741)

packages/@n8n/benchmark/scripts/run.mjs

@@ -36,6 +36,8 @@ async function main() {
 			n8nLicenseCert: config.n8nLicenseCert,
 			n8nTag: config.n8nTag,
 			n8nSetupsToUse,
+			vus: config.vus,
+			duration: config.duration,
 		});
 	} else {
 		await runLocally({
@@ -46,6 +48,8 @@ async function main() {
 			n8nTag: config.n8nTag,
 			runDir: config.runDir,
 			n8nSetupsToUse,
+			vus: config.vus,
+			duration: config.duration,
 		});
 	}
 }
@@ -66,6 +70,8 @@ function readAvailableN8nSetups() {
  * @property {string} [k6ApiToken]
  * @property {string} [n8nLicenseCert]
  * @property {string} [runDir]
+ * @property {string} [vus]
+ * @property {string} [duration]
  *
  * @returns {Promise<Config>}
  */
@@ -87,6 +93,8 @@ async function parseAndValidateConfig() {
 	const n8nLicenseCert = args.n8nLicenseCert || process.env.N8N_LICENSE_CERT || undefined;
 	const runDir = args.runDir || undefined;
 	const env = args.env || 'local';
+	const vus = args.vus;
+	const duration = args.duration;
 
 	if (!env) {
 		printUsage();
@@ -102,6 +110,8 @@ async function parseAndValidateConfig() {
 		k6ApiToken,
 		n8nLicenseCert,
 		runDir,
+		vus,
+		duration,
 	};
 }
 
@@ -141,6 +151,8 @@ function printUsage() {
 	console.log('  --debug              Enable verbose output');
 	console.log('  --n8nTag             Docker tag for n8n image. Default is latest');
 	console.log('  --benchmarkTag       Docker tag for benchmark cli image. Default is latest');
+	console.log('  --vus                How many concurrent requests to make');
+	console.log('  --duration           Test duration, e.g. 1m or 30s');
 	console.log(
 		'  --k6ApiToken         API token for k6 cloud. Default is read from K6_API_TOKEN env var. If omitted, k6 cloud will not be used',
 	);

packages/@n8n/benchmark/scripts/runForN8nSetup.mjs

@@ -6,6 +6,7 @@
 import path from 'path';
 import { $, argv, fs } from 'zx';
 import { DockerComposeClient } from './clients/dockerComposeClient.mjs';
+import { flagsObjectToCliArgs } from './utils/flags.mjs';
 
 const paths = {
 	n8nSetupsDir: path.join(__dirname, 'n8nSetups'),
@@ -27,6 +28,8 @@ async function main() {
 	const n8nLicenseCert = argv.n8nLicenseCert || process.env.N8N_LICENSE_CERT || undefined;
 	const n8nLicenseActivationKey = process.env.N8N_LICENSE_ACTIVATION_KEY || '';
 	const n8nLicenseTenantId = argv.n8nLicenseTenantId || process.env.N8N_LICENSE_TENANT_ID || '';
+	const vus = argv.vus;
+	const duration = argv.duration;
 
 	if (!fs.existsSync(baseRunDir)) {
 		console.error(
@@ -66,7 +69,21 @@ async function main() {
 	try {
 		await dockerComposeClient.$('up', '-d', '--remove-orphans', 'n8n');
 
-		await dockerComposeClient.$('run', 'benchmark', 'run', `--scenarioNamePrefix=${n8nSetupToUse}`);
+		const tags = Object.entries({
+			N8nVersion: n8nTag,
+			N8nSetup: n8nSetupToUse,
+		})
+			.map(([key, value]) => `${key}=${value}`)
+			.join(',');
+
+		const cliArgs = flagsObjectToCliArgs({
+			scenarioNamePrefix: n8nSetupToUse,
+			vus,
+			duration,
+			tags,
+		});
+
+		await dockerComposeClient.$('run', 'benchmark', 'run', ...cliArgs);
 	} catch (error) {
 		console.error('An error occurred while running the benchmarks:');
 		console.error(error.message);

packages/@n8n/benchmark/scripts/runInCloud.mjs

@@ -13,6 +13,7 @@ import { sleep, which, $, tmpdir } from 'zx';
 import path from 'path';
 import { SshClient } from './clients/sshClient.mjs';
 import { TerraformClient } from './clients/terraformClient.mjs';
+import { flagsObjectToCliArgs } from './utils/flags.mjs';
 
 /**
  * @typedef {Object} BenchmarkEnv
@@ -30,6 +31,8 @@ import { TerraformClient } from './clients/terraformClient.mjs';
  * @property {string} benchmarkTag
  * @property {string} [k6ApiToken]
  * @property {string} [n8nLicenseCert]
+ * @property {string} [vus]
+ * @property {string} [duration]
  *
  * @param {Config} config
  */
@@ -93,17 +96,16 @@ async function runBenchmarkForN8nSetup({ config, sshClient, scriptsDir, n8nSetup
 	console.log(`Running benchmarks for ${n8nSetup}...`);
 	const runScriptPath = path.join(scriptsDir, 'runForN8nSetup.mjs');
 
-	const flags = {
+	const cliArgs = flagsObjectToCliArgs({
 		n8nDockerTag: config.n8nTag,
 		benchmarkDockerTag: config.benchmarkTag,
 		k6ApiToken: config.k6ApiToken,
 		n8nLicenseCert: config.n8nLicenseCert,
-	};
+		vus: config.vus,
+		duration: config.duration,
+	});
 
-	const flagsString = Object.entries(flags)
-		.filter(([, value]) => value !== undefined)
-		.map(([key, value]) => `--${key}=${value}`)
-		.join(' ');
+	const flagsString = cliArgs.join(' ');
 
 	await sshClient.ssh(`npx zx ${runScriptPath} ${flagsString} ${n8nSetup}`, {
 		// Test run should always log its output

packages/@n8n/benchmark/scripts/runLocally.mjs

@@ -11,6 +11,7 @@
 // @ts-check
 import { $ } from 'zx';
 import path from 'path';
+import { flagsObjectToCliArgs } from './utils/flags.mjs';
 
 /**
  * @typedef {Object} BenchmarkEnv
@@ -30,19 +31,21 @@ const paths = {
  * @property {string} [runDir]
  * @property {string} [k6ApiToken]
  * @property {string} [n8nLicenseCert]
+ * @property {string} [vus]
+ * @property {string} [duration]
  *
  * @param {Config} config
  */
 export async function runLocally(config) {
 	const runScriptPath = path.join(paths.scriptsDir, 'runForN8nSetup.mjs');
 
-	const flags = Object.entries({
+	const cliArgs = flagsObjectToCliArgs({
 		n8nDockerTag: config.n8nTag,
 		benchmarkDockerTag: config.benchmarkTag,
 		runDir: config.runDir,
-	})
-		.filter(([, value]) => value !== undefined)
-		.map(([key, value]) => `--${key}=${value}`);
+		vus: config.vus,
+		duration: config.duration,
+	});
 
 	try {
 		for (const n8nSetup of config.n8nSetupsToUse) {
@@ -54,7 +57,7 @@ export async function runLocally(config) {
 					K6_API_TOKEN: config.k6ApiToken,
 					N8N_LICENSE_CERT: config.n8nLicenseCert,
 				},
-			})`npx ${runScriptPath} ${flags} ${n8nSetup}`;
+			})`npx ${runScriptPath} ${cliArgs} ${n8nSetup}`;
 		}
 	} catch (error) {
 		console.error('An error occurred while running the benchmarks:');

packages/@n8n/benchmark/scripts/utils/flags.mjs

@@ -0,0 +1,14 @@
+// @ts-check
+
+/**
+ * Converts an object of flags to an array of CLI arguments.
+ *
+ * @param {Record<string, string | undefined>} flags
+ *
+ * @returns {string[]}
+ */
+export function flagsObjectToCliArgs(flags) {
+	return Object.entries(flags)
+		.filter(([, value]) => value !== undefined)
+		.map(([key, value]) => `--${key}=${value}`);
+}