feat(core): Add support for building LLM applications (#7235)

This extracts all core and editor changes from #7246 and #7137, so that
we can get these changes merged first.

ADO-1120

[DB Tests](https://github.com/n8n-io/n8n/actions/runs/6379749011)
[E2E Tests](https://github.com/n8n-io/n8n/actions/runs/6379751480)
[Workflow Tests](https://github.com/n8n-io/n8n/actions/runs/6379752828)

---------

Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
Co-authored-by: Oleg Ivaniv <me@olegivaniv.com>
Co-authored-by: Alex Grozav <alex@grozav.com>
Co-authored-by: कारतोफ्फेलस्क्रिप्ट™ <aditya@netroy.in>
This commit is contained in:
कारतोफ्फेलस्क्रिप्ट™
2023-10-02 17:33:43 +02:00
committed by GitHub
parent 04dfcd73be
commit 00a4b8b0c6
93 changed files with 6209 additions and 728 deletions

View File

@@ -105,6 +105,7 @@ export class Expression {
* @param {(IRunExecutionData | null)} runExecutionData
* @param {boolean} [returnObjectAsString=false]
*/
// TODO: Clean that up at some point and move all the options into an options object
resolveSimpleParameterValue(
parameterValue: NodeParameterValue,
siblingParameters: INodeParameters,
@@ -119,6 +120,7 @@ export class Expression {
executeData?: IExecuteData,
returnObjectAsString = false,
selfData = {},
contextNodeName?: string,
): NodeParameterValue | INodeParameters | NodeParameterValue[] | INodeParameters[] {
// Check if it is an expression
if (typeof parameterValue !== 'string' || parameterValue.charAt(0) !== '=') {
@@ -147,6 +149,7 @@ export class Expression {
executeData,
-1,
selfData,
contextNodeName,
);
const data = dataProxy.getDataProxy();
@@ -476,6 +479,7 @@ export class Expression {
* @param {(IRunExecutionData | null)} runExecutionData
* @param {boolean} [returnObjectAsString=false]
*/
// TODO: Clean that up at some point and move all the options into an options object
getParameterValue(
parameterValue: NodeParameterValueType | INodeParameterResourceLocator,
runExecutionData: IRunExecutionData | null,
@@ -489,6 +493,7 @@ export class Expression {
executeData?: IExecuteData,
returnObjectAsString = false,
selfData = {},
contextNodeName?: string,
): NodeParameterValueType {
// Helper function which returns true when the parameter is a complex one or array
const isComplexParameter = (value: NodeParameterValueType) => {
@@ -514,6 +519,7 @@ export class Expression {
executeData,
returnObjectAsString,
selfData,
contextNodeName,
);
}
@@ -531,6 +537,7 @@ export class Expression {
executeData,
returnObjectAsString,
selfData,
contextNodeName,
);
};
@@ -550,6 +557,7 @@ export class Expression {
executeData,
returnObjectAsString,
selfData,
contextNodeName,
);
}

View File

@@ -565,6 +565,7 @@ export interface IN8nRequestOperationPaginationOffset extends IN8nRequestOperati
}
export interface IGetNodeParameterOptions {
contextNode?: INode;
// extract value from regex, works only when parameter type is resourceLocator
extractValue?: boolean;
// get raw value of parameter with unresolved expressions
@@ -760,17 +761,37 @@ type BaseExecutionFunctions = FunctionsBaseWithRequiredKeys<'getMode'> & {
getInputSourceData(inputIndex?: number, inputName?: string): ISourceData;
};
// TODO: Create later own type only for Config-Nodes
export type IExecuteFunctions = ExecuteFunctions.GetNodeParameterFn &
BaseExecutionFunctions & {
executeWorkflow(
workflowInfo: IExecuteWorkflowInfo,
inputData?: INodeExecutionData[],
): Promise<any>;
getInputConnectionData(
inputName: ConnectionTypes,
itemIndex: number,
inputIndex?: number,
nodeNameOverride?: string,
): Promise<unknown>;
getInputData(inputIndex?: number, inputName?: string): INodeExecutionData[];
getNodeOutputs(): INodeOutputConfiguration[];
putExecutionToWait(waitTill: Date): Promise<void>;
sendMessageToUI(message: any): void;
sendResponse(response: IExecuteResponsePromiseData): void;
// TODO: Make this one then only available in the new config one
addInputData(
connectionType: ConnectionTypes,
data: INodeExecutionData[][] | ExecutionError,
runIndex?: number,
): { index: number };
addOutputData(
connectionType: ConnectionTypes,
currentNodeRunIndex: number,
data: INodeExecutionData[][] | ExecutionError,
): void;
nodeHelpers: NodeHelperFunctions;
helpers: RequestHelperFunctions &
BaseHelperFunctions &
@@ -1009,6 +1030,7 @@ export interface INodeParameters {
export type NodePropertyTypes =
| 'boolean'
| 'button'
| 'collection'
| 'color'
| 'dateTime'
@@ -1049,6 +1071,7 @@ export interface ILoadOptions {
}
export interface INodePropertyTypeOptions {
action?: string; // Supported by: button
alwaysOpenEditWindow?: boolean; // Supported by: json
codeAutocomplete?: CodeAutocompleteTypes; // Supported by: string
editor?: EditorType; // Supported by: string
@@ -1256,8 +1279,14 @@ export namespace MultiPartFormData {
>;
}
export interface SupplyData {
metadata?: IDataObject;
response: unknown;
}
export interface INodeType {
description: INodeTypeDescription;
supplyData?(this: IExecuteFunctions): Promise<SupplyData>;
execute?(
this: IExecuteFunctions,
): Promise<INodeExecutionData[][] | NodeExecutionWithMetadata[][] | null>;
@@ -1324,7 +1353,7 @@ export interface INodeCredentialDescription {
testedBy?: ICredentialTestRequest | string; // Name of a function inside `loadOptions.credentialTest`
}
export type INodeIssueTypes = 'credentials' | 'execution' | 'parameters' | 'typeUnknown';
export type INodeIssueTypes = 'credentials' | 'execution' | 'input' | 'parameters' | 'typeUnknown';
export interface INodeIssueObjectProperty {
[key: string]: string[];
@@ -1339,6 +1368,7 @@ export interface INodeIssueData {
export interface INodeIssues {
execution?: boolean;
credentials?: INodeIssueObjectProperty;
input?: INodeIssueObjectProperty;
parameters?: INodeIssueObjectProperty;
typeUnknown?: boolean;
[key: string]: undefined | boolean | INodeIssueObjectProperty;
@@ -1466,15 +1496,77 @@ export interface IPostReceiveSort extends IPostReceiveBase {
};
}
export type ConnectionTypes =
| 'ai_chain'
| 'ai_document'
| 'ai_embedding'
| 'ai_languageModel'
| 'ai_memory'
| 'ai_outputParser'
| 'ai_retriever'
| 'ai_textSplitter'
| 'ai_tool'
| 'ai_vectorRetriever'
| 'ai_vectorStore'
| 'main';
export const enum NodeConnectionType {
// eslint-disable-next-line @typescript-eslint/naming-convention
AiChain = 'ai_chain',
// eslint-disable-next-line @typescript-eslint/naming-convention
AiDocument = 'ai_document',
// eslint-disable-next-line @typescript-eslint/naming-convention
AiEmbedding = 'ai_embedding',
// eslint-disable-next-line @typescript-eslint/naming-convention
AiLanguageModel = 'ai_languageModel',
// eslint-disable-next-line @typescript-eslint/naming-convention
AiMemory = 'ai_memory',
// eslint-disable-next-line @typescript-eslint/naming-convention
AiOutputParser = 'ai_outputParser',
// eslint-disable-next-line @typescript-eslint/naming-convention
AiRetriever = 'ai_retriever',
// eslint-disable-next-line @typescript-eslint/naming-convention
AiTextSplitter = 'ai_textSplitter',
// eslint-disable-next-line @typescript-eslint/naming-convention
AiTool = 'ai_tool',
// eslint-disable-next-line @typescript-eslint/naming-convention
AiVectorRetriever = 'ai_vectorRetriever',
// eslint-disable-next-line @typescript-eslint/naming-convention
AiVectorStore = 'ai_vectorStore',
// eslint-disable-next-line @typescript-eslint/naming-convention
Main = 'main',
}
export interface INodeInputFilter {
// TODO: Later add more filter options like categories, subcatogries,
// regex, allow to exclude certain nodes, ... ?
// Potentially change totally after alpha/beta. Is not a breaking change after all.
nodes: string[]; // Allowed nodes
}
export interface INodeInputConfiguration {
displayName?: string;
maxConnections?: number;
required?: boolean;
filter?: INodeInputFilter;
type: ConnectionTypes;
}
export interface INodeOutputConfiguration {
displayName?: string;
required?: boolean;
type: ConnectionTypes;
}
export interface INodeTypeDescription extends INodeTypeBaseDescription {
version: number | number[];
defaults: INodeParameters;
eventTriggerDescription?: string;
activationMessage?: string;
inputs: string[];
inputs: Array<ConnectionTypes | INodeInputConfiguration> | string;
requiredInputs?: string | number[] | number; // Ony available with executionOrder => "v1"
inputNames?: string[];
outputs: string[];
outputs: Array<ConnectionTypes | INodeInputConfiguration> | string;
outputNames?: string[];
properties: INodeProperties[];
credentials?: INodeCredentialDescription[];
@@ -1655,6 +1747,10 @@ export interface IRunExecutionData {
executionData?: {
contextData: IExecuteContextData;
nodeExecutionStack: IExecuteData[];
metadata: {
// node-name: metadata by runIndex
[key: string]: ITaskMetadata[];
};
waitingExecution: IWaitingForExecution;
waitingExecutionSource: IWaitingForExecutionSource | null;
};
@@ -1666,14 +1762,25 @@ export interface IRunData {
[key: string]: ITaskData[];
}
export interface ITaskSubRunMetadata {
node: string;
runIndex: number;
}
export interface ITaskMetadata {
subRun?: ITaskSubRunMetadata[];
}
// The data that gets returned when a node runs
export interface ITaskData {
startTime: number;
executionTime: number;
executionStatus?: ExecutionStatus;
data?: ITaskDataConnections;
inputOverride?: ITaskDataConnections;
error?: ExecutionError;
source: Array<ISourceData | null>; // Is an array as nodes have multiple inputs
metadata?: ITaskMetadata;
}
export interface ISourceData {
@@ -1769,7 +1876,7 @@ export interface IWorkflowExecuteAdditionalData {
restApiUrl: string;
instanceBaseUrl: string;
setExecutionStatus?: (status: ExecutionStatus) => void;
sendMessageToUI?: (source: string, message: any) => void;
sendDataToUI?: (type: string, data: IDataObject | IDataObject[]) => void;
timezone: string;
webhookBaseUrl: string;
webhookWaitingBaseUrl: string;
@@ -2138,6 +2245,7 @@ export interface IN8nUISettings {
urlBaseWebhook: string;
urlBaseEditor: string;
versionCli: string;
isBetaRelease: boolean;
n8nMetadata?: {
userId?: string;
[key: string]: string | number | undefined;

View File

@@ -36,6 +36,10 @@ import type {
INodePropertyOptions,
ResourceMapperValue,
ValidationResult,
ConnectionTypes,
INodeTypeDescription,
INodeOutputConfiguration,
INodeInputConfiguration,
GenericValue,
} from './Interfaces';
import { isResourceMapperValue, isValidResourceLocatorParameterValue } from './type-guards';
@@ -1005,6 +1009,65 @@ export function getNodeWebhookUrl(
return `${baseUrl}/${getNodeWebhookPath(workflowId, node, path, isFullPath)}`;
}
export function getConnectionTypes(
connections: Array<ConnectionTypes | INodeInputConfiguration | INodeOutputConfiguration>,
): ConnectionTypes[] {
return connections
.map((connection) => {
if (typeof connection === 'string') {
return connection;
}
return connection.type;
})
.filter((connection) => connection !== undefined);
}
export function getNodeInputs(
workflow: Workflow,
node: INode,
nodeTypeData: INodeTypeDescription,
): Array<ConnectionTypes | INodeInputConfiguration> {
if (Array.isArray(nodeTypeData.inputs)) {
return nodeTypeData.inputs;
}
// Calculate the outputs dynamically
try {
return (workflow.expression.getSimpleParameterValue(
node,
nodeTypeData.inputs,
'internal',
'',
{},
) || []) as ConnectionTypes[];
} catch (e) {
throw new Error(`Could not calculate inputs dynamically for node "${node.name}"`);
}
}
export function getNodeOutputs(
workflow: Workflow,
node: INode,
nodeTypeData: INodeTypeDescription,
): Array<ConnectionTypes | INodeOutputConfiguration> {
if (Array.isArray(nodeTypeData.outputs)) {
return nodeTypeData.outputs;
}
// Calculate the outputs dynamically
try {
return (workflow.expression.getSimpleParameterValue(
node,
nodeTypeData.outputs,
'internal',
'',
{},
) || []) as ConnectionTypes[];
} catch (e) {
throw new Error(`Could not calculate outputs dynamically for node "${node.name}"`);
}
}
/**
* Returns all the parameter-issues of the node
*
@@ -1049,7 +1112,7 @@ export function nodeIssuesToString(issues: INodeIssues, node?: INode): string[]
nodeIssues.push('Execution Error.');
}
const objectProperties = ['parameters', 'credentials'];
const objectProperties = ['parameters', 'credentials', 'input'];
let issueText: string;
let parameterName: string;

View File

@@ -41,6 +41,7 @@ import type {
IRun,
IRunNodeResponse,
NodeParameterValueType,
ConnectionTypes,
} from './Interfaces';
import { Node } from './Interfaces';
import type { IDeferredPromise } from './DeferredPromise';
@@ -557,11 +558,11 @@ export class Workflow {
/**
* Finds the highest parent nodes of the node with the given name
*
* @param {string} [type='main']
* @param {ConnectionTypes} [type='main']
*/
getHighestNode(
nodeName: string,
type = 'main',
type: ConnectionTypes = 'main',
nodeConnectionIndex?: number,
checkedNodes?: string[],
): string[] {
@@ -639,17 +640,25 @@ export class Workflow {
* @param {string} [type='main']
* @param {*} [depth=-1]
*/
getChildNodes(nodeName: string, type = 'main', depth = -1): string[] {
getChildNodes(
nodeName: string,
type: ConnectionTypes | 'ALL' | 'ALL_NON_MAIN' = 'main',
depth = -1,
): string[] {
return this.getConnectedNodes(this.connectionsBySourceNode, nodeName, type, depth);
}
/**
* Returns all the nodes before the given one
*
* @param {string} [type='main']
* @param {ConnectionTypes} [type='main']
* @param {*} [depth=-1]
*/
getParentNodes(nodeName: string, type = 'main', depth = -1): string[] {
getParentNodes(
nodeName: string,
type: ConnectionTypes | 'ALL' | 'ALL_NON_MAIN' = 'main',
depth = -1,
): string[] {
return this.getConnectedNodes(this.connectionsByDestinationNode, nodeName, type, depth);
}
@@ -657,15 +666,15 @@ export class Workflow {
* Gets all the nodes which are connected nodes starting from
* the given one
*
* @param {string} [type='main']
* @param {ConnectionTypes} [type='main']
* @param {*} [depth=-1]
*/
getConnectedNodes(
connections: IConnections,
nodeName: string,
type = 'main',
connectionType: ConnectionTypes | 'ALL' | 'ALL_NON_MAIN' = 'main',
depth = -1,
checkedNodes?: string[],
checkedNodesIncoming?: string[],
): string[] {
depth = depth === -1 ? -1 : depth;
const newDepth = depth === -1 ? depth : depth - 1;
@@ -679,57 +688,71 @@ export class Workflow {
return [];
}
if (!connections[nodeName].hasOwnProperty(type)) {
// Node does not have incoming connections of given type
return [];
let types: ConnectionTypes[];
if (connectionType === 'ALL') {
types = Object.keys(connections[nodeName]) as ConnectionTypes[];
} else if (connectionType === 'ALL_NON_MAIN') {
types = Object.keys(connections[nodeName]).filter(
(type) => type !== 'main',
) as ConnectionTypes[];
} else {
types = [connectionType];
}
checkedNodes = checkedNodes || [];
if (checkedNodes.includes(nodeName)) {
// Node got checked already before
return [];
}
checkedNodes.push(nodeName);
const returnNodes: string[] = [];
let addNodes: string[];
let nodeIndex: number;
let i: number;
let parentNodeName: string;
connections[nodeName][type].forEach((connectionsByIndex) => {
connectionsByIndex.forEach((connection) => {
if (checkedNodes!.includes(connection.node)) {
// Node got checked already before
return;
}
const returnNodes: string[] = [];
returnNodes.unshift(connection.node);
types.forEach((type) => {
if (!connections[nodeName].hasOwnProperty(type)) {
// Node does not have incoming connections of given type
return;
}
addNodes = this.getConnectedNodes(
connections,
connection.node,
type,
newDepth,
checkedNodes,
);
const checkedNodes = checkedNodesIncoming ? [...checkedNodesIncoming] : [];
for (i = addNodes.length; i--; i > 0) {
// Because nodes can have multiple parents it is possible that
// parts of the tree is parent of both and to not add nodes
// twice check first if they already got added before.
parentNodeName = addNodes[i];
nodeIndex = returnNodes.indexOf(parentNodeName);
if (checkedNodes.includes(nodeName)) {
// Node got checked already before
return;
}
if (nodeIndex !== -1) {
// Node got found before so remove it from current location
// that node-order stays correct
returnNodes.splice(nodeIndex, 1);
checkedNodes.push(nodeName);
connections[nodeName][type].forEach((connectionsByIndex) => {
connectionsByIndex.forEach((connection) => {
if (checkedNodes.includes(connection.node)) {
// Node got checked already before
return;
}
returnNodes.unshift(parentNodeName);
}
returnNodes.unshift(connection.node);
addNodes = this.getConnectedNodes(
connections,
connection.node,
connectionType,
newDepth,
checkedNodes,
);
for (i = addNodes.length; i--; i > 0) {
// Because nodes can have multiple parents it is possible that
// parts of the tree is parent of both and to not add nodes
// twice check first if they already got added before.
parentNodeName = addNodes[i];
nodeIndex = returnNodes.indexOf(parentNodeName);
if (nodeIndex !== -1) {
// Node got found before so remove it from current location
// that node-order stays correct
returnNodes.splice(nodeIndex, 1);
}
returnNodes.unshift(parentNodeName);
}
});
});
});
@@ -755,7 +778,7 @@ export class Workflow {
searchNodesBFS(connections: IConnections, sourceNode: string, maxDepth = -1): IConnectedNode[] {
const returnConns: IConnectedNode[] = [];
const type = 'main';
const type: ConnectionTypes = 'main';
let queue: IConnectedNode[] = [];
queue.push({
name: sourceNode,
@@ -821,7 +844,7 @@ export class Workflow {
getNodeConnectionIndexes(
nodeName: string,
parentNodeName: string,
type = 'main',
type: ConnectionTypes = 'main',
depth = -1,
checkedNodes?: string[],
): INodeConnection | undefined {

View File

@@ -60,6 +60,8 @@ export class WorkflowDataProxy {
private activeNodeName: string;
private contextNodeName: string;
private connectionInputData: INodeExecutionData[];
private siblingParameters: INodeParameters;
@@ -76,6 +78,7 @@ export class WorkflowDataProxy {
private timezone: string;
// TODO: Clean that up at some point and move all the options into an options object
constructor(
workflow: Workflow,
runExecutionData: IRunExecutionData | null,
@@ -90,17 +93,19 @@ export class WorkflowDataProxy {
executeData?: IExecuteData,
defaultReturnRunIndex = -1,
selfData = {},
contextNodeName?: string,
) {
this.activeNodeName = activeNodeName;
this.contextNodeName = contextNodeName || activeNodeName;
this.workflow = workflow;
this.runExecutionData = isScriptingNode(activeNodeName, workflow)
this.runExecutionData = isScriptingNode(this.contextNodeName, workflow)
? runExecutionData !== null
? augmentObject(runExecutionData)
: null
: runExecutionData;
this.connectionInputData = isScriptingNode(activeNodeName, workflow)
this.connectionInputData = isScriptingNode(this.contextNodeName, workflow)
? augmentArray(connectionInputData)
: connectionInputData;
@@ -264,6 +269,9 @@ export class WorkflowDataProxy {
that.timezone,
that.additionalKeys,
that.executeData,
false,
{},
that.contextNodeName,
);
}
@@ -342,13 +350,13 @@ export class WorkflowDataProxy {
// (example "IF" node. If node is connected to "true" or to "false" output)
if (outputIndex === undefined) {
const nodeConnection = that.workflow.getNodeConnectionIndexes(
that.activeNodeName,
that.contextNodeName,
nodeName,
'main',
);
if (nodeConnection === undefined) {
throw new ExpressionError(`connect "${that.activeNodeName}" to "${nodeName}"`, {
throw new ExpressionError(`connect "${that.contextNodeName}" to "${nodeName}"`, {
runIndex: that.runIndex,
itemIndex: that.itemIndex,
});
@@ -890,7 +898,7 @@ export class WorkflowDataProxy {
message: 'Cant get data',
},
nodeCause: nodeBeforeLast,
description: 'Could not resolve, proably no pairedItem exists',
description: 'Could not resolve, probably no pairedItem exists',
type: 'no pairing info',
moreInfoLink: true,
});
@@ -1022,7 +1030,7 @@ export class WorkflowDataProxy {
// Before resolving the pairedItem make sure that the requested node comes in the
// graph before the current one
const parentNodes = that.workflow.getParentNodes(that.activeNodeName);
const parentNodes = that.workflow.getParentNodes(that.contextNodeName);
if (!parentNodes.includes(nodeName)) {
throw createExpressionError('Invalid expression', {
messageTemplate: 'Invalid expression under %%PARAMETER%%',
@@ -1180,6 +1188,9 @@ export class WorkflowDataProxy {
that.timezone,
that.additionalKeys,
that.executeData,
false,
{},
that.contextNodeName,
);
},
$item: (itemIndex: number, runIndex?: number) => {
@@ -1197,6 +1208,7 @@ export class WorkflowDataProxy {
that.additionalKeys,
that.executeData,
defaultReturnRunIndex,
that.contextNodeName,
);
return dataProxy.getDataProxy();
},
@@ -1253,10 +1265,10 @@ export class WorkflowDataProxy {
if (name === 'isProxy') return true;
if (['$data', '$json'].includes(name as string)) {
return that.nodeDataGetter(that.activeNodeName, true)?.json;
return that.nodeDataGetter(that.contextNodeName, true)?.json;
}
if (name === '$binary') {
return that.nodeDataGetter(that.activeNodeName, true)?.binary;
return that.nodeDataGetter(that.contextNodeName, true)?.binary;
}
return Reflect.get(target, name, receiver);