chore(core): Add credential support for benchmark scenarios (#18229)

packages/@n8n/benchmark/scenarios/credential-http-node/credential-bearer.json

@@ -0,0 +1,8 @@
+{
+	"type": "httpBearerAuth",
+	"name": "Dummy HTTP credential",
+	"data": {
+		"token": "dummy token"
+	},
+	"id": "0fqzOReozl2aQvtl"
+}

packages/@n8n/benchmark/scenarios/credential-http-node/credential-http-node.json

@@ -0,0 +1,241 @@
+{
+	"name": "Credential HTTP Request",
+	"nodes": [
+		{
+			"parameters": {
+				"httpMethod": "POST",
+				"path": "benchmark-credential-http-node",
+				"responseMode": "responseNode",
+				"options": {}
+			},
+			"type": "n8n-nodes-base.webhook",
+			"typeVersion": 2,
+			"position": [-64, 32],
+			"id": "7dd66dcc-03c7-4898-ab3d-d2765730e3f3",
+			"name": "Webhook",
+			"webhookId": "92b141cd-6e59-4425-9c0a-e2ee0f4faad2"
+		},
+		{
+			"parameters": {
+				"respondWith": "allIncomingItems",
+				"options": {}
+			},
+			"type": "n8n-nodes-base.respondToWebhook",
+			"typeVersion": 1.1,
+			"position": [1072, 32],
+			"id": "e074e6a7-2417-4fde-8b6b-bc68069e833b",
+			"name": "Respond to Webhook"
+		},
+		{
+			"parameters": {
+				"url": "http://mockapi:8080/users/clair.bahringer/received_events/public",
+				"authentication": "genericCredentialType",
+				"genericAuthType": "httpBearerAuth",
+				"options": {
+					"response": {
+						"response": {
+							"fullResponse": true
+						}
+					}
+				}
+			},
+			"type": "n8n-nodes-base.httpRequest",
+			"typeVersion": 4.2,
+			"position": [304, -176],
+			"id": "28ddbbf9-56a5-431e-afe9-3c44b21aa676",
+			"name": "Mock public received events",
+			"credentials": {
+				"httpBearerAuth": {
+					"id": "0fqzOReozl2aQvtl",
+					"name": "Dummy HTTP credential"
+				}
+			}
+		},
+		{
+			"parameters": {
+				"url": "http://mockapi:8080/repos/udke6pujoywnagxkcvab2riw23khzn2tibo2vincws32qexb50ey7h97d42vnzyol0rxypgsg4pomsf7sgnmdaihstljw8edcijrwmy7mfi76yif19c4/47i31dh737el215j62ts2f2782nw3ss26rul3s8jw13u3vu0xm349a5hyay5asmwnlnf7nx8p9h4g62so6s1cis7xv9puj5j98t4m980sbe2455fn1obccjl/events",
+				"authentication": "genericCredentialType",
+				"genericAuthType": "httpBearerAuth",
+				"options": {
+					"response": {
+						"response": {
+							"fullResponse": true
+						}
+					}
+				}
+			},
+			"type": "n8n-nodes-base.httpRequest",
+			"typeVersion": 4.2,
+			"position": [304, 32],
+			"id": "3ce8827c-6226-467e-a4da-9891c1acd863",
+			"name": "Mock repository events",
+			"credentials": {
+				"httpBearerAuth": {
+					"id": "0fqzOReozl2aQvtl",
+					"name": "Dummy HTTP credential"
+				}
+			}
+		},
+		{
+			"parameters": {
+				"url": "http://mockapi:8080/orgs/g02pp066qoyithcjevhd6m1wfii3c4x51k39n9apybljhx69/events",
+				"authentication": "genericCredentialType",
+				"genericAuthType": "httpBearerAuth",
+				"options": {
+					"response": {
+						"response": {
+							"fullResponse": true
+						}
+					}
+				}
+			},
+			"type": "n8n-nodes-base.httpRequest",
+			"typeVersion": 4.2,
+			"position": [304, 224],
+			"id": "a8e416ab-50cc-4e50-8d9a-9fcf5d4bbdc8",
+			"name": "Mock organization events",
+			"credentials": {
+				"httpBearerAuth": {
+					"id": "0fqzOReozl2aQvtl",
+					"name": "Dummy HTTP credential"
+				}
+			}
+		},
+		{
+			"parameters": {
+				"numberInputs": 3
+			},
+			"type": "n8n-nodes-base.merge",
+			"typeVersion": 3,
+			"position": [608, 32],
+			"id": "542a27d4-3a03-4c22-a79a-7266050c519e",
+			"name": "Merge"
+		},
+		{
+			"parameters": {
+				"assignments": {
+					"assignments": [
+						{
+							"id": "89608adb-f487-416f-a7d8-3ebb1f7b50e5",
+							"name": "statusCode",
+							"value": "={{ $json.statusCode }}",
+							"type": "number"
+						}
+					]
+				},
+				"options": {}
+			},
+			"id": "35d4bfbb-d4be-49f4-a5dd-bd5c67a48200",
+			"name": "Select statusCode",
+			"type": "n8n-nodes-base.set",
+			"typeVersion": 3.4,
+			"position": [832, 32]
+		}
+	],
+	"pinData": {
+		"Webhook": [
+			{
+				"json": {
+					"headers": {
+						"host": "localhost:5678",
+						"user-agent": "curl/8.6.0",
+						"accept": "*/*"
+					},
+					"params": {},
+					"query": {},
+					"body": {},
+					"webhookUrl": "http://localhost:5678/webhook-test/benchmark-credential-http-node",
+					"executionMode": "test"
+				}
+			}
+		]
+	},
+	"connections": {
+		"Webhook": {
+			"main": [
+				[
+					{
+						"node": "Mock public received events",
+						"type": "main",
+						"index": 0
+					},
+					{
+						"node": "Mock repository events",
+						"type": "main",
+						"index": 0
+					},
+					{
+						"node": "Mock organization events",
+						"type": "main",
+						"index": 0
+					}
+				]
+			]
+		},
+		"Mock public received events": {
+			"main": [
+				[
+					{
+						"node": "Merge",
+						"type": "main",
+						"index": 0
+					}
+				]
+			]
+		},
+		"Mock repository events": {
+			"main": [
+				[
+					{
+						"node": "Merge",
+						"type": "main",
+						"index": 1
+					}
+				]
+			]
+		},
+		"Mock organization events": {
+			"main": [
+				[
+					{
+						"node": "Merge",
+						"type": "main",
+						"index": 2
+					}
+				]
+			]
+		},
+		"Merge": {
+			"main": [
+				[
+					{
+						"node": "Select statusCode",
+						"type": "main",
+						"index": 0
+					}
+				]
+			]
+		},
+		"Select statusCode": {
+			"main": [
+				[
+					{
+						"node": "Respond to Webhook",
+						"type": "main",
+						"index": 0
+					}
+				]
+			]
+		}
+	},
+	"active": true,
+	"settings": {
+		"executionOrder": "v1"
+	},
+	"versionId": "96bfc5ef-9421-47f5-9fdd-432dbd4bc4ca",
+	"meta": {
+		"instanceId": "4141065f11bd5ed73fef4f9b1d91842ded0ec4058e6640a98aa14384e269204b"
+	},
+	"id": "6V8rTiIDqZOniAs1",
+	"tags": []
+}

packages/@n8n/benchmark/scenarios/credential-http-node/credential-http-node.manifest.json

@@ -0,0 +1,10 @@
+{
+	"$schema": "../scenario.schema.json",
+	"name": "CredentialHttpNode",
+	"description": "Webhook -> 3x HTTP request to a mock API -> Merge -> Respond to Webhook. Requires a mock API running at http://mockapi:8080",
+	"scenarioData": {
+		"workflowFiles": ["credential-http-node.json"],
+		"credentialFiles": ["credential-bearer.json"]
+	},
+	"scriptPath": "credential-http-node.script.js"
+}

packages/@n8n/benchmark/scenarios/credential-http-node/credential-http-node.script.js

@@ -0,0 +1,30 @@
+import http from 'k6/http';
+import { check } from 'k6';
+
+const apiBaseUrl = __ENV.API_BASE_URL;
+
+export default function () {
+	const res = http.post(`${apiBaseUrl}/webhook/benchmark-http-node`);
+
+	if (res.status !== 200) {
+		console.error(
+			`Invalid response. Received status ${res.status}. Body: ${JSON.stringify(res.body)}`,
+		);
+	}
+
+	check(res, {
+		'is status 200': (r) => r.status === 200,
+		'http requests were OK': (r) => {
+			if (r.status !== 200) return false;
+
+			try {
+				// Response body is an array of the request status codes made with HttpNodes
+				const body = JSON.parse(r.body);
+				return Array.isArray(body) ? body.every((request) => request.statusCode === 200) : false;
+			} catch (error) {
+				console.error('Error parsing response body: ', error);
+				return false;
+			}
+		},
+	});
+}

packages/@n8n/benchmark/scenarios/scenario.schema.json

@@ -8,6 +8,12 @@
 					"items": {
 						"type": "string"
 					}
+				},
+				"credentialFiles": {
+					"type": "array",
+					"items": {
+						"type": "string"
+					}
 				}
 			},
 			"required": [],

packages/@n8n/benchmark/src/n8n-api-client/credentials-api-client.ts

@@ -0,0 +1,28 @@
+import type { Credential } from '@/n8n-api-client/n8n-api-client.types';
+
+import type { AuthenticatedN8nApiClient } from './authenticated-n8n-api-client';
+
+export class CredentialApiClient {
+	constructor(private readonly apiClient: AuthenticatedN8nApiClient) {}
+
+	async getAllCredentials(): Promise<Credential[]> {
+		const response = await this.apiClient.get<{ count: number; data: Credential[] }>(
+			'/credentials',
+		);
+
+		return response.data.data;
+	}
+
+	async createCredential(credential: Credential): Promise<Credential> {
+		const response = await this.apiClient.post<{ data: Credential }>('/credentials', {
+			...credential,
+			id: undefined,
+		});
+
+		return response.data.data;
+	}
+
+	async deleteCredential(credentialId: Credential['id']): Promise<void> {
+		await this.apiClient.delete(`/credentials/${credentialId}`);
+	}
+}

packages/@n8n/benchmark/src/n8n-api-client/n8n-api-client.types.ts

@@ -6,3 +6,9 @@ export type Workflow = {
 	name: string;
 	tags?: string[];
 };
+
+export type Credential = {
+	id: string;
+	name: string;
+	type: string;
+};

packages/@n8n/benchmark/src/scenario/scenario-data-loader.ts

@@ -1,27 +1,48 @@
 import * as fs from 'node:fs';
 import * as path from 'node:path';
 
-import type { Workflow } from '@/n8n-api-client/n8n-api-client.types';
+import type { Workflow, Credential } from '@/n8n-api-client/n8n-api-client.types';
 import type { Scenario } from '@/types/scenario';
 
+export type LoadableScenarioData = {
+	workflows: Workflow[];
+	credentials: Credential[];
+};
+
 /**
  * Loads scenario data files from FS
  */
 export class ScenarioDataFileLoader {
-	async loadDataForScenario(scenario: Scenario): Promise<{
+	async loadDataForScenario(scenario: Scenario): Promise<LoadableScenarioData> {
-		workflows: Workflow[];
-	}> {
 		const workflows = await Promise.all(
 			scenario.scenarioData.workflowFiles?.map((workflowFilePath) =>
 				this.loadSingleWorkflowFromFile(path.join(scenario.scenarioDirPath, workflowFilePath)),
 			) ?? [],
 		);
 
+		const credentials = await Promise.all(
+			scenario.scenarioData.credentialFiles?.map((credentialFilePath) =>
+				this.loadSingleCredentialFromFile(path.join(scenario.scenarioDirPath, credentialFilePath)),
+			) ?? [],
+		);
+
 		return {
 			workflows,
+			credentials,
 		};
 	}
 
+	private loadSingleCredentialFromFile(credentialFilePath: string): Credential {
+		const fileContent = fs.readFileSync(credentialFilePath, 'utf8');
+
+		try {
+			return JSON.parse(fileContent) as Credential;
+		} catch (error) {
+			const e = error as Error;
+			throw new Error(`Failed to parse credential file ${credentialFilePath}: ${e.message}`);
+		}
+	}
+
 	private loadSingleWorkflowFromFile(workflowFilePath: string): Workflow {
 		const fileContent = fs.readFileSync(workflowFilePath, 'utf8');
 

packages/@n8n/benchmark/src/test-execution/scenario-data-importer.ts

@@ -1,25 +1,79 @@
 import type { AuthenticatedN8nApiClient } from '@/n8n-api-client/authenticated-n8n-api-client';
-import type { Workflow } from '@/n8n-api-client/n8n-api-client.types';
+import { CredentialApiClient } from '@/n8n-api-client/credentials-api-client';
+import type { Workflow, Credential } from '@/n8n-api-client/n8n-api-client.types';
 import { WorkflowApiClient } from '@/n8n-api-client/workflows-api-client';
+import type { LoadableScenarioData } from '@/scenario/scenario-data-loader';
 
 /**
  * Imports scenario data into an n8n instance
  */
 export class ScenarioDataImporter {
 	private readonly workflowApiClient: WorkflowApiClient;
+	private readonly credentialApiClient: CredentialApiClient;
 
 	constructor(n8nApiClient: AuthenticatedN8nApiClient) {
 		this.workflowApiClient = new WorkflowApiClient(n8nApiClient);
+		this.credentialApiClient = new CredentialApiClient(n8nApiClient);
 	}
 
-	async importTestScenarioData(workflows: Workflow[]) {
+	private replaceValuesInObject(obj: unknown, searchText: string, targetText: string) {
-		const existingWorkflows = await this.workflowApiClient.getAllWorkflows();
+		if (Array.isArray(obj)) {
+			obj.map((item) => this.replaceValuesInObject(item, searchText, targetText));
+		} else if (typeof obj === 'object' && obj !== null) {
+			for (const [key, value] of Object.entries(obj)) {
+				if (typeof value === 'string' && value === searchText) {
+					(obj as Record<string, unknown>)[key] = targetText;
+				} else {
+					this.replaceValuesInObject(value, searchText, targetText);
+				}
+			}
+		}
+	}
 
-		for (const workflow of workflows) {
+	async importTestScenarioData(data: LoadableScenarioData) {
+		const existingWorkflows = await this.workflowApiClient.getAllWorkflows();
+		const existingCredentials = await this.credentialApiClient.getAllCredentials();
+
+		for (const credential of data.credentials) {
+			const createdCredential = await this.importCredentials({ existingCredentials, credential });
+
+			// We need to update the id and name of the credential in the workflows
+			for (const workflow of data.workflows) {
+				this.replaceValuesInObject(workflow, credential.id, createdCredential.id);
+				this.replaceValuesInObject(workflow, credential.name, createdCredential.name);
+			}
+		}
+
+		for (const workflow of data.workflows) {
 			await this.importWorkflow({ existingWorkflows, workflow });
 		}
 	}
 
+	/**
+	 * Imports a single credential into n8n removing any existing credentials with the same name
+	 * @param opts
+	 * @returns
+	 */
+	private async importCredentials(opts: {
+		existingCredentials: Credential[];
+		credential: Credential;
+	}) {
+		const existingCredentials = this.findExistingCredentials(
+			opts.existingCredentials,
+			opts.credential,
+		);
+		if (existingCredentials.length > 0) {
+			for (const toDelete of existingCredentials) {
+				await this.credentialApiClient.deleteCredential(toDelete.id);
+			}
+		}
+
+		return await this.credentialApiClient.createCredential({
+			...opts.credential,
+			name: this.getBenchmarkCredentialName(opts.credential),
+		});
+	}
+
 	/**
 	 * Imports a single workflow into n8n removing any existing workflows with the same name
 	 */
@@ -40,6 +94,17 @@ export class ScenarioDataImporter {
 		return await this.workflowApiClient.activateWorkflow(createdWorkflow);
 	}
 
+	private findExistingCredentials(
+		existingCredentials: Credential[],
+		credentialToImport: Credential,
+	): Credential[] {
+		const benchmarkCredentialName = this.getBenchmarkCredentialName(credentialToImport);
+
+		return existingCredentials.filter(
+			(existingCredential) => existingCredential.name === benchmarkCredentialName,
+		);
+	}
+
 	private findExistingWorkflows(
 		existingWorkflows: Workflow[],
 		workflowToImport: Workflow,
@@ -51,6 +116,10 @@ export class ScenarioDataImporter {
 		);
 	}
 
+	private getBenchmarkCredentialName(credential: Credential) {
+		return `[BENCHMARK] ${credential.name}`;
+	}
+
 	private getBenchmarkWorkflowName(workflow: Workflow) {
 		return `[BENCHMARK] ${workflow.name}`;
 	}

packages/@n8n/benchmark/src/test-execution/scenario-runner.ts

@@ -47,7 +47,7 @@ export class ScenarioRunner {
 
 		console.log('Loading and importing data');
 		const testData = await this.dataLoader.loadDataForScenario(scenario);
-		await testDataImporter.importTestScenarioData(testData.workflows);
+		await testDataImporter.importTestScenarioData(testData);
 
 		// Wait for 1s before executing the scenario to ensure that the workflows are activated.
 		// In multi-main mode it can take some time before the workflow becomes active.

packages/@n8n/benchmark/src/types/scenario.ts

@@ -1,6 +1,8 @@
 export type ScenarioData = {
 	/** Relative paths to the workflow files */
 	workflowFiles?: string[];
+	/** Relative paths to the credential files */
+	credentialFiles?: string[];
 };
 
 /**