perf(core): Load node types on demand on runners (no-changelog) (#11559)

packages/@n8n/task-runner/src/__tests__/node-types.test.ts

@@ -63,4 +63,35 @@ describe('TaskRunnerNodeTypes', () => {
 			});
 		});
 	});
+
+	describe('addNodeTypeDescriptions', () => {
+		it('should add new node types', () => {
+			const nodeTypes = new TaskRunnerNodeTypes(TYPES);
+
+			const nodeTypeDescriptions = [
+				{ name: 'new-type', version: 1 },
+				{ name: 'new-type', version: 2 },
+			] as INodeTypeDescription[];
+
+			nodeTypes.addNodeTypeDescriptions(nodeTypeDescriptions);
+
+			expect(nodeTypes.getByNameAndVersion('new-type', 1)).toEqual({
+				description: { name: 'new-type', version: 1 },
+			});
+			expect(nodeTypes.getByNameAndVersion('new-type', 2)).toEqual({
+				description: { name: 'new-type', version: 2 },
+			});
+		});
+	});
+
+	describe('onlyUnknown', () => {
+		it('should return only unknown node types', () => {
+			const nodeTypes = new TaskRunnerNodeTypes(TYPES);
+
+			const candidate = { name: 'unknown', version: 1 };
+
+			expect(nodeTypes.onlyUnknown([candidate])).toEqual([candidate]);
+			expect(nodeTypes.onlyUnknown([SINGLE_VERSIONED])).toEqual([]);
+		});
+	});
 });

packages/@n8n/task-runner/src/js-task-runner/js-task-runner.ts

@@ -17,6 +17,7 @@ import type {
 	IRunExecutionData,
 	WorkflowExecuteMode,
 	EnvProviderState,
+	INodeTypeDescription,
 } from 'n8n-workflow';
 import * as a from 'node:assert';
 import { runInNewContext, type Context } from 'node:vm';
@@ -119,6 +120,29 @@ export class JsTaskRunner extends TaskRunner {
 			neededBuiltIns.toDataRequestParams(),
 		);
 
+		/**
+		 * We request node types only when we know a task needs all nodes, because
+		 * needing all nodes means that the task relies on paired item functionality,
+		 * which is the same requirement for needing node types.
+		 */
+		if (neededBuiltIns.needsAllNodes) {
+			const uniqueNodeTypes = new Map(
+				data.workflow.nodes.map((node) => [
+					`${node.type}|${node.typeVersion}`,
+					{ name: node.type, version: node.typeVersion },
+				]),
+			);
+
+			const unknownNodeTypes = this.nodeTypes.onlyUnknown([...uniqueNodeTypes.values()]);
+
+			const nodeTypes = await this.requestNodeTypes<INodeTypeDescription[]>(
+				task.taskId,
+				unknownNodeTypes,
+			);
+
+			this.nodeTypes.addNodeTypeDescriptions(nodeTypes);
+		}
+
 		const workflowParams = data.workflow;
 		const workflow = new Workflow({
 			...workflowParams,

packages/@n8n/task-runner/src/message-types.ts

@@ -1,6 +1,11 @@
 import type { INodeTypeBaseDescription } from 'n8n-workflow';
 
-import type { RPC_ALLOW_LIST, TaskDataRequestParams, TaskResultData } from './runner-types';
+import type {
+	NeededNodeType,
+	RPC_ALLOW_LIST,
+	TaskDataRequestParams,
+	TaskResultData,
+} from './runner-types';
 
 export namespace BrokerMessage {
 	export namespace ToRunner {
@@ -47,6 +52,8 @@ export namespace BrokerMessage {
 
 		export interface NodeTypes {
 			type: 'broker:nodetypes';
+			taskId: string;
+			requestId: string;
 			nodeTypes: INodeTypeBaseDescription[];
 		}
 
@@ -87,6 +94,13 @@ export namespace BrokerMessage {
 			requestParams: TaskDataRequestParams;
 		}
 
+		export interface NodeTypesRequest {
+			type: 'broker:nodetypesrequest';
+			taskId: string;
+			requestId: string;
+			requestParams: NeededNodeType[];
+		}
+
 		export interface RPC {
 			type: 'broker:rpc';
 			callId: string;
@@ -95,7 +109,7 @@ export namespace BrokerMessage {
 			params: unknown[];
 		}
 
-		export type All = TaskReady | TaskDone | TaskError | TaskDataRequest | RPC;
+		export type All = TaskReady | TaskDone | TaskError | TaskDataRequest | NodeTypesRequest | RPC;
 	}
 }
 
@@ -120,6 +134,13 @@ export namespace RequesterMessage {
 			data: unknown;
 		}
 
+		export interface NodeTypesResponse {
+			type: 'requester:nodetypesresponse';
+			taskId: string;
+			requestId: string;
+			nodeTypes: INodeTypeBaseDescription[];
+		}
+
 		export interface RPCResponse {
 			type: 'requester:rpcresponse';
 			taskId: string;
@@ -134,7 +155,13 @@ export namespace RequesterMessage {
 			taskType: string;
 		}
 
-		export type All = TaskSettings | TaskCancel | RPCResponse | TaskDataResponse | TaskRequest;
+		export type All =
+			| TaskSettings
+			| TaskCancel
+			| RPCResponse
+			| TaskDataResponse
+			| NodeTypesResponse
+			| TaskRequest;
 	}
 }
 
@@ -183,6 +210,25 @@ export namespace RunnerMessage {
 			requestParams: TaskDataRequestParams;
 		}
 
+		export interface NodeTypesRequest {
+			type: 'runner:nodetypesrequest';
+			taskId: string;
+			requestId: string;
+
+			/**
+			 * Which node types should be included in the runner's node types request.
+			 *
+			 * Node types are needed only when the script relies on paired item functionality.
+			 * If so, we need only the node types not already cached in the runner.
+			 *
+			 * TODO: In future we can trim this down to only node types in the paired item chain,
+			 * rather than assuming we need all node types in the workflow.
+			 *
+			 * @example [{ name: 'n8n-nodes-base.httpRequest', version: 1 }]
+			 */
+			requestParams: NeededNodeType[];
+		}
+
 		export interface RPC {
 			type: 'runner:rpc';
 			callId: string;
@@ -199,6 +245,7 @@ export namespace RunnerMessage {
 			| TaskRejected
 			| TaskOffer
 			| RPC
-			| TaskDataRequest;
+			| TaskDataRequest
+			| NodeTypesRequest;
 	}
 }

packages/@n8n/task-runner/src/node-types.ts

@@ -7,6 +7,8 @@ import {
 	type IVersionedNodeType,
 } from 'n8n-workflow';
 
+import type { NeededNodeType } from './runner-types';
+
 type VersionedTypes = Map<number, INodeTypeDescription>;
 
 export const DEFAULT_NODETYPE_VERSION = 1;
@@ -61,4 +63,30 @@ export class TaskRunnerNodeTypes implements INodeTypes {
 	getKnownTypes(): IDataObject {
 		throw new ApplicationError('Unimplemented `getKnownTypes`', { level: 'error' });
 	}
+
+	addNodeTypeDescriptions(nodeTypeDescriptions: INodeTypeDescription[]) {
+		const newNodeTypes = this.parseNodeTypes(nodeTypeDescriptions);
+
+		for (const [name, newVersions] of newNodeTypes.entries()) {
+			if (!this.nodeTypesByVersion.has(name)) {
+				this.nodeTypesByVersion.set(name, newVersions);
+			} else {
+				const existingVersions = this.nodeTypesByVersion.get(name)!;
+				for (const [version, nodeType] of newVersions.entries()) {
+					existingVersions.set(version, nodeType);
+				}
+			}
+		}
+	}
+
+	/** Filter out node type versions that are already registered. */
+	onlyUnknown(nodeTypes: NeededNodeType[]) {
+		return nodeTypes.filter(({ name, version }) => {
+			const existingVersions = this.nodeTypesByVersion.get(name);
+
+			if (!existingVersions) return true;
+
+			return !existingVersions.has(version);
+		});
+	}
 }

packages/@n8n/task-runner/src/runner-types.ts

@@ -112,3 +112,6 @@ export const RPC_ALLOW_LIST = [
 	'helpers.httpRequest',
 	'logNodeOutput',
 ] as const;
+
+/** Node types needed for the runner to execute a task. */
+export type NeededNodeType = { name: string; version: number };

packages/@n8n/task-runner/src/task-runner.ts

@@ -1,4 +1,4 @@
-import { ApplicationError, type INodeTypeDescription } from 'n8n-workflow';
+import { ApplicationError } from 'n8n-workflow';
 import { nanoid } from 'nanoid';
 import { type MessageEvent, WebSocket } from 'ws';
 
@@ -25,6 +25,12 @@ interface DataRequest {
 	reject: (error: unknown) => void;
 }
 
+interface NodeTypesRequest {
+	requestId: string;
+	resolve: (data: unknown) => void;
+	reject: (error: unknown) => void;
+}
+
 interface RPCCall {
 	callId: string;
 	resolve: (data: unknown) => void;
@@ -58,6 +64,8 @@ export abstract class TaskRunner {
 
 	dataRequests: Map<DataRequest['requestId'], DataRequest> = new Map();
 
+	nodeTypesRequests: Map<NodeTypesRequest['requestId'], NodeTypesRequest> = new Map();
+
 	rpcCalls: Map<RPCCall['callId'], RPCCall> = new Map();
 
 	nodeTypes: TaskRunnerNodeTypes = new TaskRunnerNodeTypes([]);
@@ -168,15 +176,11 @@ export abstract class TaskRunner {
 				this.handleRpcResponse(message.callId, message.status, message.data);
 				break;
 			case 'broker:nodetypes':
-				this.setNodeTypes(message.nodeTypes as unknown as INodeTypeDescription[]);
+				this.processNodeTypesResponse(message.requestId, message.nodeTypes);
 				break;
 		}
 	}
 
-	setNodeTypes(nodeTypes: INodeTypeDescription[]) {
-		this.nodeTypes = new TaskRunnerNodeTypes(nodeTypes);
-	}
-
 	processDataResponse(requestId: string, data: unknown) {
 		const request = this.dataRequests.get(requestId);
 		if (!request) {
@@ -187,6 +191,16 @@ export abstract class TaskRunner {
 		request.resolve(data);
 	}
 
+	processNodeTypesResponse(requestId: string, nodeTypes: unknown) {
+		const request = this.nodeTypesRequests.get(requestId);
+
+		if (!request) return;
+
+		// Deleting of the request is handled in `requestNodeTypes`, using a
+		// `finally` wrapped around the return
+		request.resolve(nodeTypes);
+	}
+
 	hasOpenTasks() {
 		return Object.values(this.runningTasks).length < this.maxConcurrency;
 	}
@@ -282,6 +296,34 @@ export abstract class TaskRunner {
 		throw new ApplicationError('Unimplemented');
 	}
 
+	async requestNodeTypes<T = unknown>(
+		taskId: Task['taskId'],
+		requestParams: RunnerMessage.ToBroker.NodeTypesRequest['requestParams'],
+	) {
+		const requestId = nanoid();
+
+		const nodeTypesPromise = new Promise<T>((resolve, reject) => {
+			this.nodeTypesRequests.set(requestId, {
+				requestId,
+				resolve: resolve as (data: unknown) => void,
+				reject,
+			});
+		});
+
+		this.send({
+			type: 'runner:nodetypesrequest',
+			taskId,
+			requestId,
+			requestParams,
+		});
+
+		try {
+			return await nodeTypesPromise;
+		} finally {
+			this.nodeTypesRequests.delete(requestId);
+		}
+	}
+
 	async requestData<T = unknown>(
 		taskId: Task['taskId'],
 		requestParams: RunnerMessage.ToBroker.TaskDataRequest['requestParams'],