fix(editor): Use pinned data to resolve expressions in unexecuted nodes (#9693)

Co-authored-by: Milorad Filipovic <milorad@n8n.io>
Co-authored-by: Mutasem Aldmour <mutasem@n8n.io>

packages/workflow/src/Workflow.ts

@@ -415,7 +415,7 @@ export class Workflow {
 	 *
 	 * @param {string} nodeName Name of the node to return the pinData of
 	 */
-	getPinDataOfNode(nodeName: string): IDataObject[] | undefined {
+	getPinDataOfNode(nodeName: string): INodeExecutionData[] | undefined {
 		return this.pinData ? this.pinData[nodeName] : undefined;
 	}
 

packages/workflow/src/WorkflowDataProxy.ts

@@ -29,6 +29,7 @@ import { deepCopy } from './utils';
 import { getGlobalState } from './GlobalState';
 import { ApplicationError } from './errors/application.error';
 import { SCRIPTING_NODE_TYPES } from './Constants';
+import { getPinDataIfManualExecution } from './WorkflowDataProxyHelpers';
 
 export function isResourceLocatorValue(value: unknown): value is INodeParameterResourceLocator {
 	return Boolean(
@@ -241,6 +242,29 @@ export class WorkflowDataProxy {
 		});
 	}
 
+	private getNodeExecutionOrPinnedData({
+		nodeName,
+		branchIndex,
+		runIndex,
+		shortSyntax = false,
+	}: {
+		nodeName: string;
+		branchIndex?: number;
+		runIndex?: number;
+		shortSyntax?: boolean;
+	}) {
+		try {
+			return this.getNodeExecutionData(nodeName, shortSyntax, branchIndex, runIndex);
+		} catch (e) {
+			const pinData = getPinDataIfManualExecution(this.workflow, nodeName, this.mode);
+			if (pinData) {
+				return pinData;
+			}
+
+			throw e;
+		}
+	}
+
 	/**
 	 * Returns the node ExecutionData
 	 *
@@ -283,7 +307,7 @@ export class WorkflowDataProxy {
 
 			if (
 				!that.runExecutionData.resultData.runData.hasOwnProperty(nodeName) &&
-				!that.workflow.getPinDataOfNode(nodeName)
+				!getPinDataIfManualExecution(that.workflow, nodeName, that.mode)
 			) {
 				throw new ExpressionError('Referenced node is unexecuted', {
 					runIndex: that.runIndex,
@@ -383,7 +407,10 @@ export class WorkflowDataProxy {
 					}
 
 					if (['binary', 'data', 'json'].includes(name)) {
-						const executionData = that.getNodeExecutionData(nodeName, shortSyntax, undefined);
+						const executionData = that.getNodeExecutionOrPinnedData({
+							nodeName,
+							shortSyntax,
+						});
 
 						if (executionData.length === 0) {
 							if (that.workflow.getParentNodes(nodeName).length === 0) {
@@ -619,11 +646,6 @@ export class WorkflowDataProxy {
 	getDataProxy(): IWorkflowDataProxyData {
 		const that = this;
 
-		const getNodeOutput = (nodeName: string, branchIndex: number, runIndex?: number) => {
-			runIndex = runIndex === undefined ? -1 : runIndex;
-			return that.getNodeExecutionData(nodeName, false, branchIndex, runIndex);
-		};
-
 		// replacing proxies with the actual data.
 		const jmespathWrapper = (data: IDataObject | IDataObject[], query: string) => {
 			if (typeof data !== 'object' || typeof query !== 'string') {
@@ -662,7 +684,7 @@ export class WorkflowDataProxy {
 
 			if (context?.nodeCause) {
 				const nodeName = context.nodeCause;
-				const pinData = this.workflow.getPinDataOfNode(nodeName);
+				const pinData = getPinDataIfManualExecution(that.workflow, nodeName, that.mode);
 
 				if (pinData) {
 					if (!context) {
@@ -776,7 +798,8 @@ export class WorkflowDataProxy {
 
 				const previousNodeOutputData =
 					taskData?.data?.main?.[previousNodeOutput] ??
-					(that.workflow.getPinDataOfNode(sourceData.previousNode) as INodeExecutionData[]);
+					getPinDataIfManualExecution(that.workflow, sourceData.previousNode, that.mode) ??
+					[];
 				const source = taskData?.source ?? [];
 
 				if (pairedItem.item >= previousNodeOutputData.length) {
@@ -897,10 +920,22 @@ export class WorkflowDataProxy {
 			}
 
 			taskData =
-				that.runExecutionData!.resultData.runData[sourceData.previousNode][
+				that.runExecutionData!.resultData.runData[sourceData.previousNode]?.[
 					sourceData?.previousNodeRun || 0
 				];
 
+			if (!taskData) {
+				const pinData = getPinDataIfManualExecution(
+					that.workflow,
+					sourceData.previousNode,
+					that.mode,
+				);
+
+				if (pinData) {
+					taskData = { data: { main: [pinData] }, startTime: 0, executionTime: 0, source: [] };
+				}
+			}
+
 			const previousNodeOutput = sourceData.previousNodeOutput || 0;
 			if (previousNodeOutput >= taskData.data!.main.length) {
 				throw createExpressionError('Can’t get data for expression', {
@@ -944,7 +979,7 @@ export class WorkflowDataProxy {
 				const ensureNodeExecutionData = () => {
 					if (
 						!that?.runExecutionData?.resultData?.runData.hasOwnProperty(nodeName) &&
-						!that.workflow.getPinDataOfNode(nodeName)
+						!getPinDataIfManualExecution(that.workflow, nodeName, that.mode)
 					) {
 						throw createExpressionError('Referenced node is unexecuted', {
 							runIndex: that.runIndex,
@@ -1009,8 +1044,20 @@ export class WorkflowDataProxy {
 										itemIndex = that.itemIndex;
 									}
 
+									if (!that.connectionInputData.length) {
+										const pinnedData = getPinDataIfManualExecution(
+											that.workflow,
+											nodeName,
+											that.mode,
+										);
+
+										if (pinnedData) {
+											return pinnedData[itemIndex];
+										}
+									}
+
 									const executionData = that.connectionInputData;
-									const input = executionData[itemIndex];
+									const input = executionData?.[itemIndex];
 									if (!input) {
 										throw createExpressionError('Can’t get data for expression', {
 											messageTemplate: 'Can’t get data for expression under ‘%%PARAMETER%%’ field',
@@ -1061,6 +1108,7 @@ export class WorkflowDataProxy {
 								}
 								return pairedItemMethod;
 							}
+
 							if (property === 'first') {
 								ensureNodeExecutionData();
 								return (branchIndex?: number, runIndex?: number) => {
@@ -1070,7 +1118,11 @@ export class WorkflowDataProxy {
 										that.workflow.getNodeConnectionIndexes(that.activeNodeName, nodeName)
 											?.sourceIndex ??
 										0;
-									const executionData = getNodeOutput(nodeName, branchIndex, runIndex);
+									const executionData = that.getNodeExecutionOrPinnedData({
+										nodeName,
+										branchIndex,
+										runIndex,
+									});
 									if (executionData[0]) return executionData[0];
 									return undefined;
 								};
@@ -1084,7 +1136,11 @@ export class WorkflowDataProxy {
 										that.workflow.getNodeConnectionIndexes(that.activeNodeName, nodeName)
 											?.sourceIndex ??
 										0;
-									const executionData = getNodeOutput(nodeName, branchIndex, runIndex);
+									const executionData = that.getNodeExecutionOrPinnedData({
+										nodeName,
+										branchIndex,
+										runIndex,
+									});
 									if (!executionData.length) return undefined;
 									if (executionData[executionData.length - 1]) {
 										return executionData[executionData.length - 1];
@@ -1101,7 +1157,7 @@ export class WorkflowDataProxy {
 										that.workflow.getNodeConnectionIndexes(that.activeNodeName, nodeName)
 											?.sourceIndex ??
 										0;
-									return getNodeOutput(nodeName, branchIndex, runIndex);
+									return that.getNodeExecutionOrPinnedData({ nodeName, branchIndex, runIndex });
 								};
 							}
 							if (property === 'context') {

packages/workflow/src/WorkflowDataProxyHelpers.ts

@@ -0,0 +1,12 @@
+import type { INodeExecutionData, Workflow, WorkflowExecuteMode } from '.';
+
+export function getPinDataIfManualExecution(
+	workflow: Workflow,
+	nodeName: string,
+	mode: WorkflowExecuteMode,
+): INodeExecutionData[] | undefined {
+	if (mode !== 'manual') {
+		return undefined;
+	}
+	return workflow.getPinDataOfNode(nodeName);
+}

packages/workflow/test/NodeTypes.ts

@@ -571,6 +571,37 @@ const setNode: LoadedClass<INodeType> = {
 	},
 };
 
+const manualTriggerNode: LoadedClass<INodeType> = {
+	sourcePath: '',
+	type: {
+		description: {
+			displayName: 'Manual Trigger',
+			name: 'n8n-nodes-base.manualTrigger',
+			icon: 'fa:mouse-pointer',
+			group: ['trigger'],
+			version: 1,
+			description: 'Runs the flow on clicking a button in n8n',
+			eventTriggerDescription: '',
+			maxNodes: 1,
+			defaults: {
+				name: 'When clicking ‘Test workflow’',
+				color: '#909298',
+			},
+			inputs: [],
+			outputs: ['main'],
+			properties: [
+				{
+					displayName:
+						'This node is where the workflow execution starts (when you click the ‘test’ button on the canvas).<br><br> <a data-action="showNodeCreator">Explore other ways to trigger your workflow</a> (e.g on a schedule, or a webhook)',
+					name: 'notice',
+					type: 'notice',
+					default: '',
+				},
+			],
+		},
+	},
+};
+
 export class NodeTypes implements INodeTypes {
 	nodeTypes: INodeTypeData = {
 		'n8n-nodes-base.stickyNote': stickyNode,
@@ -628,6 +659,7 @@ export class NodeTypes implements INodeTypes {
 				},
 			},
 		},
+		'n8n-nodes-base.manualTrigger': manualTriggerNode,
 	};
 
 	getByName(nodeType: string): INodeType | IVersionedNodeType {

packages/workflow/test/WorkflowDataProxy.test.ts

@@ -1,4 +1,11 @@
-import type { IExecuteData, INode, IRun, IWorkflowBase } from '@/Interfaces';
+import type {
+	IExecuteData,
+	INode,
+	IPinData,
+	IRun,
+	IWorkflowBase,
+	WorkflowExecuteMode,
+} from '@/Interfaces';
 import { Workflow } from '@/Workflow';
 import { WorkflowDataProxy } from '@/WorkflowDataProxy';
 import { ExpressionError } from '@/errors/expression.error';
@@ -13,7 +20,12 @@ const loadFixture = (fixture: string) => {
 	return { workflow, run };
 };
 
-const getProxyFromFixture = (workflow: IWorkflowBase, run: IRun | null, activeNode: string) => {
+const getProxyFromFixture = (
+	workflow: IWorkflowBase,
+	run: IRun | null,
+	activeNode: string,
+	mode?: WorkflowExecuteMode,
+) => {
 	const taskData = run?.data.resultData.runData[activeNode]?.[0];
 	const lastNodeConnectionInputData = taskData?.data?.main[0];
 
@@ -29,6 +41,16 @@ const getProxyFromFixture = (workflow: IWorkflowBase, run: IRun | null, activeNo
 		};
 	}
 
+	let pinData: IPinData = {};
+	if (workflow.pinData) {
+		// json key is stored as part of workflow
+		// but dropped when copy/pasting
+		// so adding here to keep updating tests simple
+		for (let nodeName in workflow.pinData) {
+			pinData[nodeName] = workflow.pinData[nodeName].map((item) => ({ json: item }));
+		}
+	}
+
 	const dataProxy = new WorkflowDataProxy(
 		new Workflow({
 			id: '123',
@@ -37,6 +59,7 @@ const getProxyFromFixture = (workflow: IWorkflowBase, run: IRun | null, activeNo
 			connections: workflow.connections,
 			active: false,
 			nodeTypes: Helpers.NodeTypes(),
+			pinData,
 		}),
 		run?.data ?? null,
 		0,
@@ -44,7 +67,7 @@ const getProxyFromFixture = (workflow: IWorkflowBase, run: IRun | null, activeNo
 		activeNode,
 		lastNodeConnectionInputData ?? [],
 		{},
-		'manual',
+		mode ?? 'integrated',
 		{},
 		executeData,
 	);
@@ -323,4 +346,61 @@ describe('WorkflowDataProxy', () => {
 			}
 		});
 	});
+
+	describe('Pinned data with manual execution', () => {
+		const fixture = loadFixture('pindata');
+		const proxy = getProxyFromFixture(fixture.workflow, null, 'NotPinnedSet1', 'manual');
+
+		test('$(PinnedSet).item.json', () => {
+			expect(proxy.$('PinnedSet').item.json).toEqual({ firstName: 'Joe', lastName: 'Smith' });
+		});
+
+		test('$(PinnedSet).item.json.firstName', () => {
+			expect(proxy.$('PinnedSet').item.json.firstName).toBe('Joe');
+		});
+
+		test('$(PinnedSet).pairedItem().json.firstName', () => {
+			expect(proxy.$('PinnedSet').pairedItem().json.firstName).toBe('Joe');
+		});
+
+		test('$(PinnedSet).first().json.firstName', () => {
+			expect(proxy.$('PinnedSet').first().json.firstName).toBe('Joe');
+		});
+
+		test('$(PinnedSet).first().json.firstName', () => {
+			expect(proxy.$('PinnedSet').first().json.firstName).toBe('Joe');
+		});
+
+		test('$(PinnedSet).last().json.firstName', () => {
+			expect(proxy.$('PinnedSet').last().json.firstName).toBe('Joan');
+		});
+
+		test('$(PinnedSet).all()[0].json.firstName', () => {
+			expect(proxy.$('PinnedSet').all()[0].json.firstName).toBe('Joe');
+		});
+
+		test('$(PinnedSet).all()[1].json.firstName', () => {
+			expect(proxy.$('PinnedSet').all()[1].json.firstName).toBe('Joan');
+		});
+
+		test('$(PinnedSet).all()[2]', () => {
+			expect(proxy.$('PinnedSet').all()[2]).toBeUndefined();
+		});
+
+		test('$(PinnedSet).itemMatching(0).json.firstName', () => {
+			expect(proxy.$('PinnedSet').itemMatching(0).json.firstName).toBe('Joe');
+		});
+
+		test('$(PinnedSet).itemMatching(1).json.firstName', () => {
+			expect(proxy.$('PinnedSet').itemMatching(1).json.firstName).toBe('Joan');
+		});
+
+		test('$(PinnedSet).itemMatching(2)', () => {
+			expect(proxy.$('PinnedSet').itemMatching(2)).toBeUndefined();
+		});
+
+		test('$node[PinnedSet].json.firstName', () => {
+			expect(proxy.$node.PinnedSet.json.firstName).toBe('Joe');
+		});
+	});
 });

packages/workflow/test/fixtures/WorkflowDataProxy/pindata_run.json

@@ -0,0 +1,7 @@
+{
+	"data": {
+		"resultData": {
+			"runData": {}
+		}
+	}
+}

packages/workflow/test/fixtures/WorkflowDataProxy/pindata_workflow.json

@@ -0,0 +1,106 @@
+{
+  "meta": {
+    "instanceId": "a786b722078489c1fa382391a9f3476c2784761624deb2dfb4634827256d51a0"
+  },
+  "nodes": [
+    {
+      "parameters": {
+        "assignments": {
+          "assignments": [
+            {
+              "id": "3058c300-b377-41b7-9c90-a01372f9b581",
+              "name": "firstName",
+              "value": "Joe",
+              "type": "string"
+            },
+            {
+              "id": "bb871662-c23c-4234-ac0c-b78c279bbf34",
+              "name": "lastName",
+              "value": "Smith",
+              "type": "string"
+            }
+          ]
+        },
+        "options": {}
+      },
+      "id": "baee2bf4-5083-4cbe-8e51-4eddcf859ef5",
+      "name": "PinnedSet",
+      "type": "n8n-nodes-base.set",
+      "typeVersion": 3.3,
+      "position": [
+        1120,
+        380
+      ]
+    },
+    {
+      "parameters": {
+        "assignments": {
+          "assignments": [
+            {
+              "id": "a482f1fd-4815-4da4-a733-7beafb43c500",
+              "name": "test",
+              "value": "={{ $('PinnedSet').all().json }}\n{{ $('PinnedSet').item.json.firstName }}\n{{ $('PinnedSet').first().json.firstName }}\n{{ $('PinnedSet').itemMatching(0).json.firstName }}\n{{ $('PinnedSet').itemMatching(1).json.firstName }}\n{{ $('PinnedSet').last().json.firstName }}\n{{ $('PinnedSet').all()[0].json.firstName }}\n{{ $('PinnedSet').all()[1].json.firstName }}\n\n{{ $input.first().json.firstName }}\n{{ $input.last().json.firstName }}\n{{ $input.item.json.firstName }}\n\n{{ $json.firstName }}\n{{ $data.firstName }}\n\n{{ $items()[0].json.firstName }}",
+              "type": "string"
+            }
+          ]
+        },
+        "options": {}
+      },
+      "id": "2a543169-e2c1-4764-ac63-09534310b2b9",
+      "name": "NotPinnedSet1",
+      "type": "n8n-nodes-base.set",
+      "typeVersion": 3.3,
+      "position": [
+        1360,
+        380
+      ]
+    },
+    {
+      "parameters": {},
+      "id": "f36672e5-8c87-480e-a5b8-de9da6b63192",
+      "name": "Start",
+      "type": "n8n-nodes-base.manualTrigger",
+      "position": [
+        920,
+        380
+      ],
+      "typeVersion": 1
+    }
+  ],
+  "connections": {
+    "PinnedSet": {
+      "main": [
+        [
+          {
+            "node": "NotPinnedSet1",
+            "type": "main",
+            "index": 0
+          }
+        ]
+      ]
+    },
+    "Start": {
+      "main": [
+        [
+          {
+            "node": "PinnedSet",
+            "type": "main",
+            "index": 0
+          }
+        ]
+      ]
+    }
+  },
+  "pinData": {
+    "PinnedSet": [
+      {
+        "firstName": "Joe",
+        "lastName": "Smith"
+      },
+      {
+        "firstName": "Joan",
+        "lastName": "Summers"
+      }
+    ]
+  }
+}