feat: Only send needed data to task runner (no-changelog) (#11487)

packages/@n8n/task-runner/src/js-task-runner/built-ins-parser/__tests__/built-ins-parser-state.test.ts

@@ -0,0 +1,117 @@
+import { BuiltInsParserState } from '../built-ins-parser-state';
+
+describe('BuiltInsParserState', () => {
+	describe('toDataRequestSpecification', () => {
+		it('should return empty array when no properties are marked as needed', () => {
+			const state = new BuiltInsParserState();
+
+			expect(state.toDataRequestParams()).toEqual({
+				dataOfNodes: [],
+				env: false,
+				input: false,
+				prevNode: false,
+			});
+		});
+
+		it('should return all nodes and input when markNeedsAllNodes is called', () => {
+			const state = new BuiltInsParserState();
+			state.markNeedsAllNodes();
+
+			expect(state.toDataRequestParams()).toEqual({
+				dataOfNodes: 'all',
+				env: false,
+				input: true,
+				prevNode: false,
+			});
+		});
+
+		it('should return specific node names when nodes are marked as needed individually', () => {
+			const state = new BuiltInsParserState();
+			state.markNodeAsNeeded('Node1');
+			state.markNodeAsNeeded('Node2');
+
+			expect(state.toDataRequestParams()).toEqual({
+				dataOfNodes: ['Node1', 'Node2'],
+				env: false,
+				input: false,
+				prevNode: false,
+			});
+		});
+
+		it('should ignore individual nodes when needsAllNodes is marked as true', () => {
+			const state = new BuiltInsParserState();
+			state.markNodeAsNeeded('Node1');
+			state.markNeedsAllNodes();
+			state.markNodeAsNeeded('Node2'); // should be ignored since all nodes are needed
+
+			expect(state.toDataRequestParams()).toEqual({
+				dataOfNodes: 'all',
+				env: false,
+				input: true,
+				prevNode: false,
+			});
+		});
+
+		it('should mark env as needed when markEnvAsNeeded is called', () => {
+			const state = new BuiltInsParserState();
+			state.markEnvAsNeeded();
+
+			expect(state.toDataRequestParams()).toEqual({
+				dataOfNodes: [],
+				env: true,
+				input: false,
+				prevNode: false,
+			});
+		});
+
+		it('should mark input as needed when markInputAsNeeded is called', () => {
+			const state = new BuiltInsParserState();
+			state.markInputAsNeeded();
+
+			expect(state.toDataRequestParams()).toEqual({
+				dataOfNodes: [],
+				env: false,
+				input: true,
+				prevNode: false,
+			});
+		});
+
+		it('should mark prevNode as needed when markPrevNodeAsNeeded is called', () => {
+			const state = new BuiltInsParserState();
+			state.markPrevNodeAsNeeded();
+
+			expect(state.toDataRequestParams()).toEqual({
+				dataOfNodes: [],
+				env: false,
+				input: false,
+				prevNode: true,
+			});
+		});
+
+		it('should return correct specification when multiple properties are marked as needed', () => {
+			const state = new BuiltInsParserState();
+			state.markNeedsAllNodes();
+			state.markEnvAsNeeded();
+			state.markInputAsNeeded();
+			state.markPrevNodeAsNeeded();
+
+			expect(state.toDataRequestParams()).toEqual({
+				dataOfNodes: 'all',
+				env: true,
+				input: true,
+				prevNode: true,
+			});
+		});
+
+		it('should return correct specification when all properties are marked as needed', () => {
+			const state = BuiltInsParserState.newNeedsAllDataState();
+
+			expect(state.toDataRequestParams()).toEqual({
+				dataOfNodes: 'all',
+				env: true,
+				input: true,
+				prevNode: true,
+			});
+		});
+	});
+});

packages/@n8n/task-runner/src/js-task-runner/built-ins-parser/__tests__/built-ins-parser.test.ts

@@ -0,0 +1,251 @@
+import { getAdditionalKeys } from 'n8n-core';
+import type { IDataObject, INodeType, IWorkflowExecuteAdditionalData } from 'n8n-workflow';
+import { Workflow, WorkflowDataProxy } from 'n8n-workflow';
+
+import { newCodeTaskData } from '../../__tests__/test-data';
+import { BuiltInsParser } from '../built-ins-parser';
+import { BuiltInsParserState } from '../built-ins-parser-state';
+
+describe('BuiltInsParser', () => {
+	const parser = new BuiltInsParser();
+
+	const parseAndExpectOk = (code: string) => {
+		const result = parser.parseUsedBuiltIns(code);
+		if (!result.ok) {
+			fail(result.error);
+		}
+
+		return result.result;
+	};
+
+	describe('Env, input, execution and prevNode', () => {
+		const cases: Array<[string, BuiltInsParserState]> = [
+			['$env', new BuiltInsParserState({ needs$env: true })],
+			['$execution', new BuiltInsParserState({ needs$execution: true })],
+			['$prevNode', new BuiltInsParserState({ needs$prevNode: true })],
+		];
+
+		test.each(cases)("should identify built-ins in '%s'", (code, expected) => {
+			const state = parseAndExpectOk(code);
+			expect(state).toEqual(expected);
+		});
+	});
+
+	describe('Input', () => {
+		it('should mark input as needed when $input is used', () => {
+			const state = parseAndExpectOk(`
+				$input.item.json.age = 10 + Math.floor(Math.random() * 30);
+				$input.item.json.password = $input.item.json.password.split('').map(() => '*').join("")
+				delete $input.item.json.lastname
+				const emailParts = $input.item.json.email.split("@")
+				$input.item.json.emailData = {
+					user: emailParts[0],
+					domain: emailParts[1]
+				}
+
+				return $input.item;
+			`);
+
+			expect(state).toEqual(new BuiltInsParserState({ needs$input: true }));
+		});
+
+		it('should mark input as needed when $json is used', () => {
+			const state = parseAndExpectOk(`
+				$json.age = 10 + Math.floor(Math.random() * 30);
+				return $json;
+				`);
+
+			expect(state).toEqual(new BuiltInsParserState({ needs$input: true }));
+		});
+	});
+
+	describe('$(...)', () => {
+		const cases: Array<[string, BuiltInsParserState]> = [
+			[
+				'$("nodeName").first()',
+				new BuiltInsParserState({ neededNodeNames: new Set(['nodeName']) }),
+			],
+			[
+				'$("nodeName").all(); $("secondNode").matchingItem()',
+				new BuiltInsParserState({ neededNodeNames: new Set(['nodeName', 'secondNode']) }),
+			],
+		];
+
+		test.each(cases)("should identify nodes in '%s'", (code, expected) => {
+			const state = parseAndExpectOk(code);
+			expect(state).toEqual(expected);
+		});
+
+		it('should need all nodes when $() is called with a variable', () => {
+			const state = parseAndExpectOk('var n = "name"; $(n)');
+			expect(state).toEqual(new BuiltInsParserState({ needsAllNodes: true, needs$input: true }));
+		});
+
+		it('should require all nodes when there are multiple usages of $() and one is with a variable', () => {
+			const state = parseAndExpectOk(`
+				$("nodeName");
+				$("secondNode");
+				var n = "name";
+				$(n)
+			`);
+			expect(state).toEqual(new BuiltInsParserState({ needsAllNodes: true, needs$input: true }));
+		});
+
+		test.each([
+			['without parameters', '$()'],
+			['number literal', '$(123)'],
+		])('should ignore when $ is called %s', (_, code) => {
+			const state = parseAndExpectOk(code);
+			expect(state).toEqual(new BuiltInsParserState());
+		});
+
+		test.each([
+			'$("node").item',
+			'$("node")["item"]',
+			'$("node").pairedItem()',
+			'$("node")["pairedItem"]()',
+			'$("node").itemMatching(0)',
+			'$("node")["itemMatching"](0)',
+			'$("node")[variable]',
+			'var a = $("node")',
+			'let a = $("node")',
+			'const a = $("node")',
+			'a = $("node")',
+		])('should require all nodes if %s is used', (code) => {
+			const state = parseAndExpectOk(code);
+			expect(state).toEqual(new BuiltInsParserState({ needsAllNodes: true, needs$input: true }));
+		});
+
+		test.each(['$("node").first()', '$("node").last()', '$("node").all()', '$("node").params'])(
+			'should require only accessed node if %s is used',
+			(code) => {
+				const state = parseAndExpectOk(code);
+				expect(state).toEqual(
+					new BuiltInsParserState({
+						needsAllNodes: false,
+						neededNodeNames: new Set(['node']),
+					}),
+				);
+			},
+		);
+	});
+
+	describe('ECMAScript syntax', () => {
+		describe('ES2020', () => {
+			it('should parse optional chaining', () => {
+				parseAndExpectOk(`
+					const a = { b: { c: 1 } };
+					return a.b?.c;
+				`);
+			});
+
+			it('should parse nullish coalescing', () => {
+				parseAndExpectOk(`
+					const a = null;
+					return a ?? 1;
+				`);
+			});
+		});
+
+		describe('ES2021', () => {
+			it('should parse numeric separators', () => {
+				parseAndExpectOk(`
+					const a = 1_000_000;
+					return a;
+				`);
+			});
+		});
+	});
+
+	describe('WorkflowDataProxy built-ins', () => {
+		it('should have a known list of built-ins', () => {
+			const data = newCodeTaskData([]);
+			const dataProxy = new WorkflowDataProxy(
+				new Workflow({
+					...data.workflow,
+					nodeTypes: {
+						getByName() {
+							return undefined as unknown as INodeType;
+						},
+						getByNameAndVersion() {
+							return undefined as unknown as INodeType;
+						},
+						getKnownTypes() {
+							return undefined as unknown as IDataObject;
+						},
+					},
+				}),
+				data.runExecutionData,
+				data.runIndex,
+				0,
+				data.activeNodeName,
+				data.connectionInputData,
+				data.siblingParameters,
+				data.mode,
+				getAdditionalKeys(
+					data.additionalData as IWorkflowExecuteAdditionalData,
+					data.mode,
+					data.runExecutionData,
+				),
+				data.executeData,
+				data.defaultReturnRunIndex,
+				data.selfData,
+				data.contextNodeName,
+				// Make sure that even if we don't receive the envProviderState for
+				// whatever reason, we don't expose the task runner's env to the code
+				data.envProviderState ?? {
+					env: {},
+					isEnvAccessBlocked: false,
+					isProcessAvailable: true,
+				},
+			).getDataProxy({ throwOnMissingExecutionData: false });
+
+			/**
+			 * NOTE! If you are adding new built-ins to the WorkflowDataProxy class
+			 * make sure the built-ins parser and Task Runner handle them properly.
+			 */
+			expect(Object.keys(dataProxy)).toStrictEqual([
+				'$',
+				'$input',
+				'$binary',
+				'$data',
+				'$env',
+				'$evaluateExpression',
+				'$item',
+				'$fromAI',
+				'$fromai',
+				'$fromAi',
+				'$items',
+				'$json',
+				'$node',
+				'$self',
+				'$parameter',
+				'$prevNode',
+				'$runIndex',
+				'$mode',
+				'$workflow',
+				'$itemIndex',
+				'$now',
+				'$today',
+				'$jmesPath',
+				'DateTime',
+				'Interval',
+				'Duration',
+				'$execution',
+				'$vars',
+				'$secrets',
+				'$executionId',
+				'$resumeWebhookUrl',
+				'$getPairedItem',
+				'$jmespath',
+				'$position',
+				'$thisItem',
+				'$thisItemIndex',
+				'$thisRunIndex',
+				'$nodeVersion',
+				'$nodeId',
+				'$webhookId',
+			]);
+		});
+	});
+});

packages/@n8n/task-runner/src/js-task-runner/built-ins-parser/acorn-helpers.ts

@@ -0,0 +1,28 @@
+import type {
+	AssignmentExpression,
+	Identifier,
+	Literal,
+	MemberExpression,
+	Node,
+	VariableDeclarator,
+} from 'acorn';
+
+export function isLiteral(node?: Node): node is Literal {
+	return node?.type === 'Literal';
+}
+
+export function isIdentifier(node?: Node): node is Identifier {
+	return node?.type === 'Identifier';
+}
+
+export function isMemberExpression(node?: Node): node is MemberExpression {
+	return node?.type === 'MemberExpression';
+}
+
+export function isVariableDeclarator(node?: Node): node is VariableDeclarator {
+	return node?.type === 'VariableDeclarator';
+}
+
+export function isAssignmentExpression(node?: Node): node is AssignmentExpression {
+	return node?.type === 'AssignmentExpression';
+}

packages/@n8n/task-runner/src/js-task-runner/built-ins-parser/built-ins-parser-state.ts

@@ -0,0 +1,74 @@
+import type { N8nMessage } from '../../runner-types';
+
+/**
+ * Class to keep track of which built-in variables are accessed in the code
+ */
+export class BuiltInsParserState {
+	neededNodeNames: Set<string> = new Set();
+
+	needsAllNodes = false;
+
+	needs$env = false;
+
+	needs$input = false;
+
+	needs$execution = false;
+
+	needs$prevNode = false;
+
+	constructor(opts: Partial<BuiltInsParserState> = {}) {
+		Object.assign(this, opts);
+	}
+
+	/**
+	 * Marks that all nodes are needed, including input data
+	 */
+	markNeedsAllNodes() {
+		this.needsAllNodes = true;
+		this.needs$input = true;
+		this.neededNodeNames = new Set();
+	}
+
+	markNodeAsNeeded(nodeName: string) {
+		if (this.needsAllNodes) {
+			return;
+		}
+
+		this.neededNodeNames.add(nodeName);
+	}
+
+	markEnvAsNeeded() {
+		this.needs$env = true;
+	}
+
+	markInputAsNeeded() {
+		this.needs$input = true;
+	}
+
+	markExecutionAsNeeded() {
+		this.needs$execution = true;
+	}
+
+	markPrevNodeAsNeeded() {
+		this.needs$prevNode = true;
+	}
+
+	toDataRequestParams(): N8nMessage.ToRequester.TaskDataRequest['requestParams'] {
+		return {
+			dataOfNodes: this.needsAllNodes ? 'all' : Array.from(this.neededNodeNames),
+			env: this.needs$env,
+			input: this.needs$input,
+			prevNode: this.needs$prevNode,
+		};
+	}
+
+	static newNeedsAllDataState() {
+		const obj = new BuiltInsParserState();
+		obj.markNeedsAllNodes();
+		obj.markEnvAsNeeded();
+		obj.markInputAsNeeded();
+		obj.markExecutionAsNeeded();
+		obj.markPrevNodeAsNeeded();
+		return obj;
+	}
+}

packages/@n8n/task-runner/src/js-task-runner/built-ins-parser/built-ins-parser.ts

@@ -0,0 +1,142 @@
+import type { CallExpression, Identifier, Node, Program } from 'acorn';
+import { parse } from 'acorn';
+import { ancestor } from 'acorn-walk';
+import type { Result } from 'n8n-workflow';
+import { toResult } from 'n8n-workflow';
+
+import {
+	isAssignmentExpression,
+	isIdentifier,
+	isLiteral,
+	isMemberExpression,
+	isVariableDeclarator,
+} from './acorn-helpers';
+import { BuiltInsParserState } from './built-ins-parser-state';
+
+/**
+ * Class for parsing Code Node code to identify which built-in variables
+ * are accessed
+ */
+export class BuiltInsParser {
+	/**
+	 * Parses which built-in variables are accessed in the given code
+	 */
+	public parseUsedBuiltIns(code: string): Result<BuiltInsParserState, Error> {
+		return toResult(() => {
+			const wrappedCode = `async function VmCodeWrapper() { ${code} }`;
+			const ast = parse(wrappedCode, { ecmaVersion: 2025, sourceType: 'module' });
+
+			return this.identifyBuiltInsByWalkingAst(ast);
+		});
+	}
+
+	/** Traverse the AST of the script and mark any data needed for it to run. */
+	private identifyBuiltInsByWalkingAst(ast: Program) {
+		const accessedBuiltIns = new BuiltInsParserState();
+
+		ancestor(
+			ast,
+			{
+				CallExpression: this.visitCallExpression,
+				Identifier: this.visitIdentifier,
+			},
+			undefined,
+			accessedBuiltIns,
+		);
+
+		return accessedBuiltIns;
+	}
+
+	private visitCallExpression = (
+		node: CallExpression,
+		state: BuiltInsParserState,
+		ancestors: Node[],
+	) => {
+		// $(...)
+		const isDollar = node.callee.type === 'Identifier' && node.callee.name === '$';
+		if (!isDollar) return;
+
+		// $(): This is not valid, ignore
+		if (node.arguments.length === 0) {
+			return;
+		}
+
+		const firstArg = node.arguments[0];
+		if (!isLiteral(firstArg)) {
+			// $(variable): Can't easily determine statically, mark all nodes as needed
+			state.markNeedsAllNodes();
+			return;
+		}
+
+		if (typeof firstArg.value !== 'string') {
+			// $(123): Static value, but not a string --> invalid code --> ignore
+			return;
+		}
+
+		// $("node"): Static value, mark 'nodeName' as needed
+		state.markNodeAsNeeded(firstArg.value);
+
+		// Determine how $("node") is used
+		this.handlePrevNodeCall(node, state, ancestors);
+	};
+
+	private handlePrevNodeCall(_node: CallExpression, state: BuiltInsParserState, ancestors: Node[]) {
+		// $("node").item, .pairedItem or .itemMatching: In a case like this, the execution
+		// engine will traverse back from current node (i.e. the Code Node) to
+		// the "node" node and use `pairedItem`s to find which item is linked
+		// to the current item. So, we need to mark all nodes as needed.
+		// TODO: We could also mark all the nodes between the current node and
+		// the "node" node as needed, but that would require more complex logic.
+		const directParent = ancestors[ancestors.length - 2];
+		if (isMemberExpression(directParent)) {
+			const accessedProperty = directParent.property;
+
+			if (directParent.computed) {
+				// $("node")["item"], ["pairedItem"] or ["itemMatching"]
+				if (isLiteral(accessedProperty)) {
+					if (this.isPairedItemProperty(accessedProperty.value)) {
+						state.markNeedsAllNodes();
+					}
+					// Else: $("node")[123]: Static value, but not any of the ones above --> ignore
+				}
+				// $("node")[variable]
+				else if (isIdentifier(accessedProperty)) {
+					state.markNeedsAllNodes();
+				}
+			}
+			// $("node").item, .pairedItem or .itemMatching
+			else if (isIdentifier(accessedProperty) && this.isPairedItemProperty(accessedProperty.name)) {
+				state.markNeedsAllNodes();
+			}
+		} else if (isVariableDeclarator(directParent) || isAssignmentExpression(directParent)) {
+			// const variable = $("node") or variable = $("node"):
+			// In this case we would need to track down all the possible use sites
+			// of 'variable' and determine if `.item` is accessed on it. This is
+			// more complex and skipped for now.
+			// TODO: Optimize for this case
+			state.markNeedsAllNodes();
+		} else {
+			// Something else than the cases above. Mark all nodes as needed as it
+			// could be a dynamic access.
+			state.markNeedsAllNodes();
+		}
+	}
+
+	private visitIdentifier = (node: Identifier, state: BuiltInsParserState) => {
+		if (node.name === '$env') {
+			state.markEnvAsNeeded();
+		} else if (node.name === '$input' || node.name === '$json') {
+			state.markInputAsNeeded();
+		} else if (node.name === '$execution') {
+			state.markExecutionAsNeeded();
+		} else if (node.name === '$prevNode') {
+			state.markPrevNodeAsNeeded();
+		}
+	};
+
+	private isPairedItemProperty(
+		property?: string | boolean | null | number | RegExp | bigint,
+	): boolean {
+		return property === 'item' || property === 'pairedItem' || property === 'itemMatching';
+	}
+}