diff --git a/.github/workflows/benchmark-nightly.yml b/.github/workflows/benchmark-nightly.yml index d557a0c9ee..b95bf33f56 100644 --- a/.github/workflows/benchmark-nightly.yml +++ b/.github/workflows/benchmark-nightly.yml @@ -70,7 +70,13 @@ jobs: working-directory: packages/@n8n/benchmark - name: Run the benchmark - run: pnpm benchmark-in-cloud --n8nTag ${{ env.N8N_TAG }} --benchmarkTag ${{ env.N8N_BENCHMARK_TAG }} ${{ env.DEBUG }} + run: | + pnpm benchmark-in-cloud \ + --vus 5 \ + --duration 1m \ + --n8nTag ${{ env.N8N_TAG }} \ + --benchmarkTag ${{ env.N8N_BENCHMARK_TAG }} \ + ${{ env.DEBUG }} working-directory: packages/@n8n/benchmark # We need to login again because the access token expires diff --git a/packages/@n8n/benchmark/package.json b/packages/@n8n/benchmark/package.json index a9ecd000b0..8a87384df7 100644 --- a/packages/@n8n/benchmark/package.json +++ b/packages/@n8n/benchmark/package.json @@ -33,7 +33,6 @@ "dependencies": { "@oclif/core": "4.0.7", "axios": "catalog:", - "convict": "6.2.4", "dotenv": "8.6.0", "zx": "^8.1.4" }, diff --git a/packages/@n8n/benchmark/scripts/run.mjs b/packages/@n8n/benchmark/scripts/run.mjs index f6f5da1e37..0ea074ad73 100755 --- a/packages/@n8n/benchmark/scripts/run.mjs +++ b/packages/@n8n/benchmark/scripts/run.mjs @@ -36,6 +36,8 @@ async function main() { n8nLicenseCert: config.n8nLicenseCert, n8nTag: config.n8nTag, n8nSetupsToUse, + vus: config.vus, + duration: config.duration, }); } else { await runLocally({ @@ -46,6 +48,8 @@ async function main() { n8nTag: config.n8nTag, runDir: config.runDir, n8nSetupsToUse, + vus: config.vus, + duration: config.duration, }); } } @@ -66,6 +70,8 @@ function readAvailableN8nSetups() { * @property {string} [k6ApiToken] * @property {string} [n8nLicenseCert] * @property {string} [runDir] + * @property {string} [vus] + * @property {string} [duration] * * @returns {Promise} */ @@ -87,6 +93,8 @@ async function parseAndValidateConfig() { const n8nLicenseCert = args.n8nLicenseCert || process.env.N8N_LICENSE_CERT || undefined; const runDir = args.runDir || undefined; const env = args.env || 'local'; + const vus = args.vus; + const duration = args.duration; if (!env) { printUsage(); @@ -102,6 +110,8 @@ async function parseAndValidateConfig() { k6ApiToken, n8nLicenseCert, runDir, + vus, + duration, }; } @@ -141,6 +151,8 @@ function printUsage() { console.log(' --debug Enable verbose output'); console.log(' --n8nTag Docker tag for n8n image. Default is latest'); console.log(' --benchmarkTag Docker tag for benchmark cli image. Default is latest'); + console.log(' --vus How many concurrent requests to make'); + console.log(' --duration Test duration, e.g. 1m or 30s'); console.log( ' --k6ApiToken API token for k6 cloud. Default is read from K6_API_TOKEN env var. If omitted, k6 cloud will not be used', ); diff --git a/packages/@n8n/benchmark/scripts/runForN8nSetup.mjs b/packages/@n8n/benchmark/scripts/runForN8nSetup.mjs index b27968daa1..819b7dc5f8 100755 --- a/packages/@n8n/benchmark/scripts/runForN8nSetup.mjs +++ b/packages/@n8n/benchmark/scripts/runForN8nSetup.mjs @@ -6,6 +6,7 @@ import path from 'path'; import { $, argv, fs } from 'zx'; import { DockerComposeClient } from './clients/dockerComposeClient.mjs'; +import { flagsObjectToCliArgs } from './utils/flags.mjs'; const paths = { n8nSetupsDir: path.join(__dirname, 'n8nSetups'), @@ -27,6 +28,8 @@ async function main() { const n8nLicenseCert = argv.n8nLicenseCert || process.env.N8N_LICENSE_CERT || undefined; const n8nLicenseActivationKey = process.env.N8N_LICENSE_ACTIVATION_KEY || ''; const n8nLicenseTenantId = argv.n8nLicenseTenantId || process.env.N8N_LICENSE_TENANT_ID || ''; + const vus = argv.vus; + const duration = argv.duration; if (!fs.existsSync(baseRunDir)) { console.error( @@ -66,7 +69,21 @@ async function main() { try { await dockerComposeClient.$('up', '-d', '--remove-orphans', 'n8n'); - await dockerComposeClient.$('run', 'benchmark', 'run', `--scenarioNamePrefix=${n8nSetupToUse}`); + const tags = Object.entries({ + N8nVersion: n8nTag, + N8nSetup: n8nSetupToUse, + }) + .map(([key, value]) => `${key}=${value}`) + .join(','); + + const cliArgs = flagsObjectToCliArgs({ + scenarioNamePrefix: n8nSetupToUse, + vus, + duration, + tags, + }); + + await dockerComposeClient.$('run', 'benchmark', 'run', ...cliArgs); } catch (error) { console.error('An error occurred while running the benchmarks:'); console.error(error.message); diff --git a/packages/@n8n/benchmark/scripts/runInCloud.mjs b/packages/@n8n/benchmark/scripts/runInCloud.mjs index ba1c39241c..fc5b59b93b 100755 --- a/packages/@n8n/benchmark/scripts/runInCloud.mjs +++ b/packages/@n8n/benchmark/scripts/runInCloud.mjs @@ -13,6 +13,7 @@ import { sleep, which, $, tmpdir } from 'zx'; import path from 'path'; import { SshClient } from './clients/sshClient.mjs'; import { TerraformClient } from './clients/terraformClient.mjs'; +import { flagsObjectToCliArgs } from './utils/flags.mjs'; /** * @typedef {Object} BenchmarkEnv @@ -30,6 +31,8 @@ import { TerraformClient } from './clients/terraformClient.mjs'; * @property {string} benchmarkTag * @property {string} [k6ApiToken] * @property {string} [n8nLicenseCert] + * @property {string} [vus] + * @property {string} [duration] * * @param {Config} config */ @@ -93,17 +96,16 @@ async function runBenchmarkForN8nSetup({ config, sshClient, scriptsDir, n8nSetup console.log(`Running benchmarks for ${n8nSetup}...`); const runScriptPath = path.join(scriptsDir, 'runForN8nSetup.mjs'); - const flags = { + const cliArgs = flagsObjectToCliArgs({ n8nDockerTag: config.n8nTag, benchmarkDockerTag: config.benchmarkTag, k6ApiToken: config.k6ApiToken, n8nLicenseCert: config.n8nLicenseCert, - }; + vus: config.vus, + duration: config.duration, + }); - const flagsString = Object.entries(flags) - .filter(([, value]) => value !== undefined) - .map(([key, value]) => `--${key}=${value}`) - .join(' '); + const flagsString = cliArgs.join(' '); await sshClient.ssh(`npx zx ${runScriptPath} ${flagsString} ${n8nSetup}`, { // Test run should always log its output diff --git a/packages/@n8n/benchmark/scripts/runLocally.mjs b/packages/@n8n/benchmark/scripts/runLocally.mjs index be3b832410..760d87defc 100755 --- a/packages/@n8n/benchmark/scripts/runLocally.mjs +++ b/packages/@n8n/benchmark/scripts/runLocally.mjs @@ -11,6 +11,7 @@ // @ts-check import { $ } from 'zx'; import path from 'path'; +import { flagsObjectToCliArgs } from './utils/flags.mjs'; /** * @typedef {Object} BenchmarkEnv @@ -30,19 +31,21 @@ const paths = { * @property {string} [runDir] * @property {string} [k6ApiToken] * @property {string} [n8nLicenseCert] + * @property {string} [vus] + * @property {string} [duration] * * @param {Config} config */ export async function runLocally(config) { const runScriptPath = path.join(paths.scriptsDir, 'runForN8nSetup.mjs'); - const flags = Object.entries({ + const cliArgs = flagsObjectToCliArgs({ n8nDockerTag: config.n8nTag, benchmarkDockerTag: config.benchmarkTag, runDir: config.runDir, - }) - .filter(([, value]) => value !== undefined) - .map(([key, value]) => `--${key}=${value}`); + vus: config.vus, + duration: config.duration, + }); try { for (const n8nSetup of config.n8nSetupsToUse) { @@ -54,7 +57,7 @@ export async function runLocally(config) { K6_API_TOKEN: config.k6ApiToken, N8N_LICENSE_CERT: config.n8nLicenseCert, }, - })`npx ${runScriptPath} ${flags} ${n8nSetup}`; + })`npx ${runScriptPath} ${cliArgs} ${n8nSetup}`; } } catch (error) { console.error('An error occurred while running the benchmarks:'); diff --git a/packages/@n8n/benchmark/scripts/utils/flags.mjs b/packages/@n8n/benchmark/scripts/utils/flags.mjs new file mode 100644 index 0000000000..23ab57c29c --- /dev/null +++ b/packages/@n8n/benchmark/scripts/utils/flags.mjs @@ -0,0 +1,14 @@ +// @ts-check + +/** + * Converts an object of flags to an array of CLI arguments. + * + * @param {Record} flags + * + * @returns {string[]} + */ +export function flagsObjectToCliArgs(flags) { + return Object.entries(flags) + .filter(([, value]) => value !== undefined) + .map(([key, value]) => `--${key}=${value}`); +} diff --git a/packages/@n8n/benchmark/src/commands/list.ts b/packages/@n8n/benchmark/src/commands/list.ts index fcc60b1b81..485493e43d 100644 --- a/packages/@n8n/benchmark/src/commands/list.ts +++ b/packages/@n8n/benchmark/src/commands/list.ts @@ -1,15 +1,19 @@ import { Command } from '@oclif/core'; import { ScenarioLoader } from '@/scenario/scenarioLoader'; -import { loadConfig } from '@/config/config'; +import { testScenariosPath } from '@/config/commonFlags'; export default class ListCommand extends Command { static description = 'List all available scenarios'; + static flags = { + testScenariosPath, + }; + async run() { - const config = loadConfig(); + const { flags } = await this.parse(ListCommand); const scenarioLoader = new ScenarioLoader(); - const allScenarios = scenarioLoader.loadAll(config.get('testScenariosPath')); + const allScenarios = scenarioLoader.loadAll(flags.testScenariosPath); console.log('Available test scenarios:'); console.log(''); diff --git a/packages/@n8n/benchmark/src/commands/run.ts b/packages/@n8n/benchmark/src/commands/run.ts index 5e2c1d3dc6..f5aa9bb089 100644 --- a/packages/@n8n/benchmark/src/commands/run.ts +++ b/packages/@n8n/benchmark/src/commands/run.ts @@ -1,59 +1,97 @@ import { Command, Flags } from '@oclif/core'; -import { loadConfig } from '@/config/config'; import { ScenarioLoader } from '@/scenario/scenarioLoader'; import { ScenarioRunner } from '@/testExecution/scenarioRunner'; import { N8nApiClient } from '@/n8nApiClient/n8nApiClient'; import { ScenarioDataFileLoader } from '@/scenario/scenarioDataLoader'; +import type { K6Tag } from '@/testExecution/k6Executor'; import { K6Executor } from '@/testExecution/k6Executor'; +import { testScenariosPath } from '@/config/commonFlags'; export default class RunCommand extends Command { static description = 'Run all (default) or specified test scenarios'; - // TODO: Add support for filtering scenarios static flags = { - scenarios: Flags.string({ - char: 't', - description: 'Comma-separated list of test scenarios to run', - required: false, - }), + testScenariosPath, scenarioNamePrefix: Flags.string({ - description: 'Prefix for the scenario name. Defaults to Unnamed', - required: false, + description: 'Prefix for the scenario name', + default: 'Unnamed', + }), + n8nBaseUrl: Flags.string({ + description: 'The base URL for the n8n instance', + default: 'http://localhost:5678', + env: 'N8N_BASE_URL', + }), + n8nUserEmail: Flags.string({ + description: 'The email address of the n8n user', + default: 'benchmark-user@n8n.io', + env: 'N8N_USER_EMAIL', + }), + k6ExecutablePath: Flags.string({ + doc: 'The path to the k6 binary', + default: 'k6', + env: 'K6_PATH', + }), + k6ApiToken: Flags.string({ + doc: 'The API token for k6 cloud', + default: undefined, + env: 'K6_API_TOKEN', + }), + n8nUserPassword: Flags.string({ + description: 'The password of the n8n user', + default: 'VerySecret!123', + env: 'N8N_USER_PASSWORD', + }), + tags: Flags.string({ + char: 't', + description: 'Tags to attach to the run. Comma separated list of key=value pairs', + }), + vus: Flags.integer({ + description: 'Number of concurrent requests to make', + default: 5, + }), + duration: Flags.string({ + description: 'Duration of the test with a unit, e.g. 1m', + default: '1m', }), }; async run() { - const config = await this.loadConfigAndMergeWithFlags(); + const { flags } = await this.parse(RunCommand); + const tags = await this.parseTags(); const scenarioLoader = new ScenarioLoader(); const scenarioRunner = new ScenarioRunner( - new N8nApiClient(config.get('n8n.baseUrl')), + new N8nApiClient(flags.n8nBaseUrl), new ScenarioDataFileLoader(), new K6Executor({ - k6ExecutablePath: config.get('k6.executablePath'), - k6ApiToken: config.get('k6.apiToken'), - n8nApiBaseUrl: config.get('n8n.baseUrl'), + duration: flags.duration, + vus: flags.vus, + k6ExecutablePath: flags.k6ExecutablePath, + k6ApiToken: flags.k6ApiToken, + n8nApiBaseUrl: flags.n8nBaseUrl, + tags, }), { - email: config.get('n8n.user.email'), - password: config.get('n8n.user.password'), + email: flags.n8nUserEmail, + password: flags.n8nUserPassword, }, - config.get('scenarioNamePrefix'), + flags.scenarioNamePrefix, ); - const allScenarios = scenarioLoader.loadAll(config.get('testScenariosPath')); + const allScenarios = scenarioLoader.loadAll(flags.testScenariosPath); await scenarioRunner.runManyScenarios(allScenarios); } - private async loadConfigAndMergeWithFlags() { - const config = loadConfig(); + private async parseTags(): Promise { const { flags } = await this.parse(RunCommand); - - if (flags.scenarioNamePrefix) { - config.set('scenarioNamePrefix', flags.scenarioNamePrefix); + if (!flags.tags) { + return []; } - return config; + return flags.tags.split(',').map((tag) => { + const [name, value] = tag.split('='); + return { name, value }; + }); } } diff --git a/packages/@n8n/benchmark/src/config/commonFlags.ts b/packages/@n8n/benchmark/src/config/commonFlags.ts new file mode 100644 index 0000000000..fb53682324 --- /dev/null +++ b/packages/@n8n/benchmark/src/config/commonFlags.ts @@ -0,0 +1,6 @@ +import { Flags } from '@oclif/core'; + +export const testScenariosPath = Flags.string({ + description: 'The path to the scenarios', + default: 'scenarios', +}); diff --git a/packages/@n8n/benchmark/src/config/config.ts b/packages/@n8n/benchmark/src/config/config.ts deleted file mode 100644 index ad8be8b4ea..0000000000 --- a/packages/@n8n/benchmark/src/config/config.ts +++ /dev/null @@ -1,64 +0,0 @@ -import convict from 'convict'; -import dotenv from 'dotenv'; - -dotenv.config(); - -const configSchema = { - testScenariosPath: { - doc: 'The path to the scenarios', - format: String, - default: 'scenarios', - }, - n8n: { - baseUrl: { - doc: 'The base URL for the n8n instance', - format: String, - default: 'http://localhost:5678', - env: 'N8N_BASE_URL', - }, - user: { - email: { - doc: 'The email address of the n8n user', - format: String, - default: 'benchmark-user@n8n.io', - env: 'N8N_USER_EMAIL', - }, - password: { - doc: 'The password of the n8n user', - format: String, - default: 'VerySecret!123', - env: 'N8N_USER_PASSWORD', - }, - }, - }, - scenarioNamePrefix: { - doc: 'Prefix for the scenario name', - format: String, - default: 'Unnamed', - env: 'N8N_BENCHMARK_SCENARIO_NAME_PREFIX', - }, - k6: { - executablePath: { - doc: 'The path to the k6 binary', - format: String, - default: 'k6', - env: 'K6_PATH', - }, - apiToken: { - doc: 'The API token for k6 cloud', - format: String, - default: undefined, - env: 'K6_API_TOKEN', - }, - }, -}; - -export type Config = ReturnType; - -export function loadConfig() { - const config = convict(configSchema); - - config.validate({ allowed: 'strict' }); - - return config; -} diff --git a/packages/@n8n/benchmark/src/testExecution/k6Executor.ts b/packages/@n8n/benchmark/src/testExecution/k6Executor.ts index da272ce69f..27bf2b3c8d 100644 --- a/packages/@n8n/benchmark/src/testExecution/k6Executor.ts +++ b/packages/@n8n/benchmark/src/testExecution/k6Executor.ts @@ -3,10 +3,20 @@ import path from 'path'; import { $, which, tmpfile } from 'zx'; import type { Scenario } from '@/types/scenario'; +export type K6Tag = { + name: string; + value: string; +}; + export type K6ExecutorOpts = { k6ExecutablePath: string; + /** How many concurrent requests to make */ + vus: number; + /** Test duration, e.g. 1m or 30s */ + duration: string; k6ApiToken?: string; n8nApiBaseUrl: string; + tags?: K6Tag[]; }; export type K6RunOpts = { @@ -19,7 +29,7 @@ export type K6RunOpts = { * @example ['--duration', '1m'] * @example ['--quiet'] */ -type K6CliFlag = [string] | [string, string]; +type K6CliFlag = [string | number] | [string, string | number]; /** * Executes test scenarios using k6 @@ -45,7 +55,11 @@ export function handleSummary(data) { const augmentedTestScriptPath = this.augmentSummaryScript(scenario, scenarioRunName); const runDirPath = path.dirname(augmentedTestScriptPath); - const flags: K6CliFlag[] = [['--quiet'], ['--duration', '3m'], ['--vus', '5']]; + const flags: K6CliFlag[] = [ + ['--quiet'], + ['--duration', this.opts.duration], + ['--vus', this.opts.vus], + ]; if (this.opts.k6ApiToken) { flags.push(['--out', 'cloud']); diff --git a/packages/@n8n/benchmark/src/testExecution/k6Summary.ts b/packages/@n8n/benchmark/src/testExecution/k6Summary.ts index c40b0cae97..7277945d3f 100644 --- a/packages/@n8n/benchmark/src/testExecution/k6Summary.ts +++ b/packages/@n8n/benchmark/src/testExecution/k6Summary.ts @@ -243,7 +243,7 @@ interface RootGroup { name: string; path: string; id: string; - groups: any[]; + groups: unknown[]; checks: Check[]; } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5aa9c26537..cdb5b09169 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -226,9 +226,6 @@ importers: axios: specifier: 'catalog:' version: 1.7.4(debug@4.3.6) - convict: - specifier: 6.2.4 - version: 6.2.4 dotenv: specifier: 8.6.0 version: 8.6.0 @@ -21628,7 +21625,7 @@ snapshots: eslint-import-resolver-node@0.3.9: dependencies: - debug: 3.2.7(supports-color@5.5.0) + debug: 3.2.7(supports-color@8.1.1) is-core-module: 2.13.1 resolve: 1.22.8 transitivePeerDependencies: @@ -21653,7 +21650,7 @@ snapshots: eslint-module-utils@2.8.0(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.5.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.5.2))(eslint-plugin-import@2.29.1)(eslint@8.57.0))(eslint@8.57.0): dependencies: - debug: 3.2.7(supports-color@5.5.0) + debug: 3.2.7(supports-color@8.1.1) optionalDependencies: '@typescript-eslint/parser': 7.2.0(eslint@8.57.0)(typescript@5.5.2) eslint: 8.57.0 @@ -21673,7 +21670,7 @@ snapshots: array.prototype.findlastindex: 1.2.3 array.prototype.flat: 1.3.2 array.prototype.flatmap: 1.3.2 - debug: 3.2.7(supports-color@5.5.0) + debug: 3.2.7(supports-color@8.1.1) doctrine: 2.1.0 eslint: 8.57.0 eslint-import-resolver-node: 0.3.9 @@ -22543,7 +22540,7 @@ snapshots: array-parallel: 0.1.3 array-series: 0.1.5 cross-spawn: 4.0.2 - debug: 3.2.7(supports-color@5.5.0) + debug: 3.2.7(supports-color@8.1.1) transitivePeerDependencies: - supports-color @@ -25564,7 +25561,7 @@ snapshots: pdf-parse@1.1.1: dependencies: - debug: 3.2.7(supports-color@5.5.0) + debug: 3.2.7(supports-color@8.1.1) node-ensure: 0.0.0 transitivePeerDependencies: - supports-color @@ -26450,7 +26447,7 @@ snapshots: rhea@1.0.24: dependencies: - debug: 3.2.7(supports-color@5.5.0) + debug: 3.2.7(supports-color@8.1.1) transitivePeerDependencies: - supports-color