feat(core): Add support for building LLM applications (#7235)

This extracts all core and editor changes from #7246 and #7137, so that
we can get these changes merged first.

ADO-1120

[DB Tests](https://github.com/n8n-io/n8n/actions/runs/6379749011)
[E2E Tests](https://github.com/n8n-io/n8n/actions/runs/6379751480)
[Workflow Tests](https://github.com/n8n-io/n8n/actions/runs/6379752828)

---------

Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
Co-authored-by: Oleg Ivaniv <me@olegivaniv.com>
Co-authored-by: Alex Grozav <alex@grozav.com>
Co-authored-by: कारतोफ्फेलस्क्रिप्ट™ <aditya@netroy.in>

packages/cli/src/ActiveWorkflowRunner.ts

@@ -515,6 +515,7 @@ export class ActiveWorkflowRunner implements IWebhookManager {
 			},
 			executionData: {
 				contextData: {},
+				metadata: {},
 				nodeExecutionStack,
 				waitingExecution: {},
 				waitingExecutionSource: {},

packages/cli/src/GenericHelpers.ts

@@ -141,6 +141,7 @@ export async function createErrorExecution(
 		},
 		executionData: {
 			contextData: {},
+			metadata: {},
 			nodeExecutionStack: [
 				{
 					node,

packages/cli/src/Server.ts

@@ -249,6 +249,7 @@ export class Server extends AbstractServer {
 			urlBaseWebhook,
 			urlBaseEditor: instanceBaseUrl,
 			versionCli: '',
+			isBetaRelease: config.getEnv('generic.isBetaRelease'),
 			oauthCallbackUrls: {
 				oauth1: `${instanceBaseUrl}/${this.restEndpoint}/oauth1-credential/callback`,
 				oauth2: `${instanceBaseUrl}/${this.restEndpoint}/oauth2-credential/callback`,

packages/cli/src/WorkflowExecuteAdditionalData.ts

@@ -32,6 +32,7 @@ import type {
 import {
 	ErrorReporterProxy as ErrorReporter,
 	LoggerProxy as Logger,
+	NodeOperationError,
 	Workflow,
 	WorkflowHooks,
 } from 'n8n-workflow';
@@ -46,6 +47,7 @@ import type {
 	IWorkflowExecuteProcess,
 	IWorkflowExecutionDataProcess,
 	IWorkflowErrorData,
+	IPushDataType,
 	ExecutionPayload,
 } from '@/Interfaces';
 import { NodeTypes } from '@/NodeTypes';
@@ -69,6 +71,41 @@ import { restoreBinaryDataId } from './executionLifecycleHooks/restoreBinaryData
 
 const ERROR_TRIGGER_TYPE = config.getEnv('nodes.errorTriggerType');
 
+export function objectToError(errorObject: unknown, workflow: Workflow): Error {
+	// TODO: Expand with other error types
+	if (errorObject instanceof Error) {
+		// If it's already an Error instance, return it as is.
+		return errorObject;
+	} else if (errorObject && typeof errorObject === 'object' && 'message' in errorObject) {
+		// If it's an object with a 'message' property, create a new Error instance.
+		let error: Error | undefined;
+		if ('node' in errorObject) {
+			const node = workflow.getNode((errorObject.node as { name: string }).name);
+			if (node) {
+				error = new NodeOperationError(
+					node,
+					errorObject as unknown as Error,
+					errorObject as object,
+				);
+			}
+		}
+
+		if (error === undefined) {
+			error = new Error(errorObject.message as string);
+		}
+
+		if ('stack' in errorObject) {
+			// If there's a 'stack' property, set it on the new Error instance.
+			error.stack = errorObject.stack as string;
+		}
+
+		return error;
+	} else {
+		// If it's neither an Error nor an object with a 'message' property, create a generic Error.
+		return new Error('An error occurred');
+	}
+}
+
 /**
  * Checks if there was an error and if errorWorkflow or a trigger is defined. If so it collects
  * all the data and executes it
@@ -369,6 +406,7 @@ export function hookFunctionsPreExecute(parentProcessMode?: string): IWorkflowEx
 							},
 							executionData: {
 								contextData: {},
+								metadata: {},
 								nodeExecutionStack: [],
 								waitingExecution: {},
 								waitingExecutionSource: {},
@@ -709,6 +747,7 @@ export async function getRunData(
 		},
 		executionData: {
 			contextData: {},
+			metadata: {},
 			nodeExecutionStack,
 			waitingExecution: {},
 			waitingExecutionSource: {},
@@ -743,7 +782,7 @@ export async function getWorkflowData(
 
 		workflowData = await WorkflowsService.get({ id: workflowInfo.id }, { relations });
 
-		if (workflowData === undefined) {
+		if (workflowData === undefined || workflowData === null) {
 			throw new Error(`The workflow with the id "${workflowInfo.id}" does not exist.`);
 		}
 	} else {
@@ -910,11 +949,14 @@ async function executeWorkflow(
 			executionId,
 			fullExecutionData,
 		);
-		throw {
-			...error,
-			stack: error.stack,
-			message: error.message,
-		};
+		throw objectToError(
+			{
+				...error,
+				stack: error.stack,
+				message: error.message,
+			},
+			workflow,
+		);
 	}
 
 	await externalHooks.run('workflow.postExecute', [data, workflowData, executionId]);
@@ -932,10 +974,13 @@ async function executeWorkflow(
 	// Workflow did fail
 	const { error } = data.data.resultData;
 	// eslint-disable-next-line @typescript-eslint/no-throw-literal
-	throw {
-		...error,
-		stack: error!.stack,
-	};
+	throw objectToError(
+		{
+			...error,
+			stack: error!.stack,
+		},
+		workflow,
+	);
 }
 
 export function setExecutionStatus(status: ExecutionStatus) {
@@ -951,8 +996,7 @@ export function setExecutionStatus(status: ExecutionStatus) {
 		});
 }
 
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
-export function sendMessageToUI(source: string, messages: any[]) {
+export function sendDataToUI(type: string, data: IDataObject | IDataObject[]) {
 	const { sessionId } = this;
 	if (sessionId === undefined) {
 		return;
@@ -961,14 +1005,7 @@ export function sendMessageToUI(source: string, messages: any[]) {
 	// Push data to session which started workflow
 	try {
 		const pushInstance = Container.get(Push);
-		pushInstance.send(
-			'sendConsoleMessage',
-			{
-				source: `[Node: "${source}"]`,
-				messages,
-			},
-			sessionId,
-		);
+		pushInstance.send(type as IPushDataType, data, sessionId);
 	} catch (error) {
 		Logger.warn(`There was a problem sending message to UI: ${error.message}`);
 	}

packages/cli/src/WorkflowHelpers.ts

@@ -74,6 +74,7 @@ export function generateFailedExecutionFromError(
 			},
 			executionData: {
 				contextData: {},
+				metadata: {},
 				nodeExecutionStack: [
 					{
 						node,
@@ -252,6 +253,7 @@ export async function executeErrorWorkflow(
 			},
 			executionData: {
 				contextData: {},
+				metadata: {},
 				nodeExecutionStack,
 				waitingExecution: {},
 				waitingExecutionSource: {},

packages/cli/src/WorkflowRunner.ts

@@ -327,7 +327,7 @@ export class WorkflowRunner {
 				executionId,
 			});
 
-			additionalData.sendMessageToUI = WorkflowExecuteAdditionalData.sendMessageToUI.bind({
+			additionalData.sendDataToUI = WorkflowExecuteAdditionalData.sendDataToUI.bind({
 				sessionId: data.sessionId,
 			});
 
@@ -344,8 +344,7 @@ export class WorkflowRunner {
 			} else if (
 				data.runData === undefined ||
 				data.startNodes === undefined ||
-				data.startNodes.length === 0 ||
-				data.destinationNode === undefined
+				data.startNodes.length === 0
 			) {
 				Logger.debug(`Execution ID ${executionId} will run executing all nodes.`, { executionId });
 				// Execute all nodes
@@ -736,11 +735,11 @@ export class WorkflowRunner {
 				if (responsePromise) {
 					responsePromise.resolve(WebhookHelpers.decodeWebhookResponse(message.data.response));
 				}
-			} else if (message.type === 'sendMessageToUI') {
+			} else if (message.type === 'sendDataToUI') {
 				// eslint-disable-next-line @typescript-eslint/no-unsafe-call
-				WorkflowExecuteAdditionalData.sendMessageToUI.bind({ sessionId: data.sessionId })(
-					message.data.source,
-					message.data.message,
+				WorkflowExecuteAdditionalData.sendDataToUI.bind({ sessionId: data.sessionId })(
+					message.data.type,
+					message.data.data,
 				);
 			} else if (message.type === 'processError') {
 				clearTimeout(executionTimeout);

packages/cli/src/WorkflowRunnerProcess.ts

@@ -189,13 +189,13 @@ class WorkflowRunnerProcess {
 			executionId: inputData.executionId,
 		});
 		// eslint-disable-next-line @typescript-eslint/no-explicit-any
-		additionalData.sendMessageToUI = async (source: string, message: any) => {
+		additionalData.sendDataToUI = async (type: string, data: IDataObject | IDataObject[]) => {
 			if (workflowRunner.data!.executionMode !== 'manual') {
 				return;
 			}
 
 			try {
-				await sendToParentProcess('sendMessageToUI', { source, message });
+				await sendToParentProcess('sendDataToUI', { type, data });
 			} catch (error) {
 				ErrorReporter.error(error);
 				this.logger.error(
@@ -291,8 +291,7 @@ class WorkflowRunnerProcess {
 		if (
 			this.data.runData === undefined ||
 			this.data.startNodes === undefined ||
-			this.data.startNodes.length === 0 ||
-			this.data.destinationNode === undefined
+			this.data.startNodes.length === 0
 		) {
 			// Execute all nodes
 

packages/cli/src/config/schema.ts

@@ -431,6 +431,13 @@ export const schema = {
 			format: ['main', 'webhook', 'worker'] as const,
 			default: 'main',
 		},
+
+		isBetaRelease: {
+			doc: 'If it is a beta release',
+			format: 'Boolean',
+			default: false,
+			env: 'IS_BETA_RELEASE',
+		},
 	},
 
 	// How n8n can be reached (Editor & REST-API)