✨ Nodes as JSON and authentication redesign (#2401)

* ✨ change FE to handle new object type

* 🚸 improve UX of handling invalid credentials

* 🚧 WIP

* 🎨 fix typescript issues

* 🐘 add migrations for all supported dbs

* ✏️ add description to migrations

* ⚡ add credential update on import

* ⚡ resolve after merge issues

* 👕 fix lint issues

* ⚡ check credentials on workflow create/update

* update interface

* 👕 fix ts issues

* ⚡ adaption to new credentials UI

* 🐛 intialize cache on BE for credentials check

* 🐛 fix undefined oldCredentials

* 🐛 fix deleting credential

* 🐛 fix check for undefined keys

* 🐛 fix disabling edit in execution

* 🎨 just show credential name on execution view

* ✏️  remove TODO

* ⚡ implement review suggestions

* ⚡ add cache to getCredentialsByType

* ⏪ use getter instead of cache

* ✏️ fix variable name typo

* 🐘 include waiting nodes to migrations

* 🐛 fix reverting migrations command

* ⚡ update typeorm command

* ✨ create db:revert command

* 👕 fix lint error

* ✨ Add optional authenticate method to credentials

* ⚡ Simplify code and add authentication support to MattermostApi

* 👕 Fix lint issue

* ⚡ Add support to own-mode

* 👕 Fix lint issue

* ✨ Add support for predefined auth types bearer and headerAuth

* ⚡ Make sure that DateTime Node always returns strings

* ⚡ Add support for moment types to If Node

* ⚡ Make it possible for HTTP Request Node to use all credential types

* ✨ Add basicAuth support

* Add a new dropcontact node

* ✨ First basic implementation of mainly JSON based nodes

* ✨ Add fixedCollection support, added value parameter and
expression support for value and property

* Improvements to #2389

* ⚡ Add credentials verification

* ⚡ Small improvement

* ⚡ set default time to 45 seconds

* ✨ Add support for preSend and postReceive methods

* ➕ Add lodash merge and set depedency to workflow

* 👕 Fix lint issue

* ⚡ Improvements

* ⚡ Improvements

* ⚡ Improvements

* ⚡ Improvements

* ⚡ Improvements

* 🐛 Set siren and language correctly

* ⚡ Add support for requestDefaults

* ⚡ Add support for baseURL to httpRequest

* ⚡ Move baseURL to correct location

* ✨ Add support for options loading

* 🐛 Fix error with fullAccess nodes

* ✨ Add credential test functionality

* 🐛 Fix issue with OAuth autentication and lint issue

* ⚡ Fix build issue

* 🐛 Fix issue that url got always overwritten to empty

* ✨ Add pagination support

* ⚡ Code fix required after merge

* ⚡ Remove not needed imports

* ⚡ Fix credential test

* ✨ Add expression support for request properties and $self
support on properties

* ⚡ Rename $self to $value

* 👕 Fix lint issue

* ⚡ Add example how to send data in path

* ✨ Make it possible to not sent in dot notation

* ✨ Add support for postReceive:rootProperty

* ⚡ Fix typo

* ✨ Add support for postReceive:set

* ⚡ Some fixes

* ⚡ Small improvement

* ;zap: Separate RoutingNode code

* ⚡ Simplify code and fix bug

* ⚡ Remove unused code

* ✨ Make it possible to define "request" and "requestProperty" on
options

* 👕 Fix lint issue

* ⚡ Change $credentials variables name

* ✨ Enable expressions and access to credentials in requestDefaults

* ⚡ Make parameter option loading use RoutingNode.makeRoutingRequest

* ✨ Allow requestOperations overwrite on LoadOptions

* ✨ Make it possible to access current node parameters in loadOptions

* ⚡ Rename parameters variable to make future proof

* ⚡ Make it possible to use offset-pagination with body

* ✨ Add support for queryAuth

* ⚡ Never return more items than requested

* ✨ Make it possible to overwrite requestOperations on parameter
and option level

* 👕 Fix lint issue

* ✨ Allow simplified auth also with regular nodes

* ✨ Add support for receiving binary data

* 🐛 Fix example node

* ⚡ Rename property "name" to "displayName" in loadOptions

* ⚡ Send data by default as "query" if nothing is set

* ⚡ Rename $self to $parent

* ⚡ Change to work with INodeExecutionData instead of IDataObject

* ⚡ Improve binaryData handling

* ⚡ Property design improvements

* ⚡ Fix property name

* 🚨 Add some tests

* ⚡ Add also test for request

* ⚡ Improve test and fix issues

* ⚡ Improvements to loadOptions

* ⚡ Normalize loadOptions with rest of code

* ⚡ Add info text

* ✨ Add support for $value in postReceive

* 🚨 Add tests for RoutingNode.runNode

* ⚡ Remove TODOs and make url property optional

* ⚡ Fix bug and lint issue

* 🐛 Fix bug that not the correct property got used

* 🚨 Add tests for CredentialsHelper.authenticate

* ⚡ Improve code and resolve expressions also everywhere for
loadOptions and credential test requests

* ✨ Make it possible to define multiple preSend and postReceive
actions

* ✨ Allow to define tests on credentials

* ⚡ Remove test data

* ⬆️ Update package-lock.json file

* ⚡ Remove old not longer used code

Co-authored-by: Ben Hesseldieck <b.hesseldieck@gmail.com>
Co-authored-by: Mutasem <mutdmour@gmail.com>
Co-authored-by: PaulineDropcontact <pauline@dropcontact.io>
Co-authored-by: ricardo <ricardoespinoza105@gmail.com>

packages/workflow/test/Helpers.ts

@@ -1,14 +1,526 @@
+import get from 'lodash.get';
 import {
+	CredentialInformation,
+	IAdditionalCredentialOptions,
+	IAllExecuteFunctions,
+	IContextObject,
+	ICredentialDataDecryptedObject,
+	ICredentials,
+	ICredentialsEncrypted,
+	ICredentialsHelper,
+	IDataObject,
+	IExecuteFunctions,
+	IExecuteResponsePromiseData,
+	IExecuteSingleFunctions,
+	IExecuteWorkflowInfo,
+	IHttpRequestOptions,
+	IN8nHttpFullResponse,
+	IN8nHttpResponse,
+	INode,
+	INodeCredentialsDetails,
+	INodeExecutionData,
+	INodeParameters,
 	INodeType,
 	INodeTypeData,
 	INodeTypes,
+	IRunExecutionData,
+	ITaskDataConnections,
+	IWorkflowBase,
+	IWorkflowDataProxyAdditionalKeys,
+	IWorkflowDataProxyData,
+	IWorkflowExecuteAdditionalData,
 	NodeHelpers,
+	NodeParameterValue,
+	Workflow,
+	WorkflowDataProxy,
+	WorkflowExecuteMode,
+	WorkflowHooks,
 } from '../src';
 
 export interface INodeTypesObject {
 	[key: string]: INodeType;
 }
 
+export class Credentials extends ICredentials {
+	hasNodeAccess(nodeType: string): boolean {
+		return true;
+	}
+
+	setData(data: ICredentialDataDecryptedObject, encryptionKey: string): void {
+		this.data = JSON.stringify(data);
+	}
+
+	setDataKey(key: string, data: CredentialInformation, encryptionKey: string): void {
+		let fullData;
+		try {
+			fullData = this.getData(encryptionKey);
+		} catch (e) {
+			fullData = {};
+		}
+
+		fullData[key] = data;
+
+		return this.setData(fullData, encryptionKey);
+	}
+
+	getData(encryptionKey: string, nodeType?: string): ICredentialDataDecryptedObject {
+		if (this.data === undefined) {
+			throw new Error('No data is set so nothing can be returned.');
+		}
+		return JSON.parse(this.data);
+	}
+
+	getDataKey(key: string, encryptionKey: string, nodeType?: string): CredentialInformation {
+		const fullData = this.getData(encryptionKey, nodeType);
+
+		if (fullData === null) {
+			throw new Error(`No data was set.`);
+		}
+
+		// eslint-disable-next-line no-prototype-builtins
+		if (!fullData.hasOwnProperty(key)) {
+			throw new Error(`No data for key "${key}" exists.`);
+		}
+
+		return fullData[key];
+	}
+
+	getDataToSave(): ICredentialsEncrypted {
+		if (this.data === undefined) {
+			throw new Error(`No credentials were set to save.`);
+		}
+
+		return {
+			id: this.id,
+			name: this.name,
+			type: this.type,
+			data: this.data,
+			nodesAccess: this.nodesAccess,
+		};
+	}
+}
+
+export class CredentialsHelper extends ICredentialsHelper {
+	async authenticate(
+		credentials: ICredentialDataDecryptedObject,
+		typeName: string,
+		requestParams: IHttpRequestOptions,
+	): Promise<IHttpRequestOptions> {
+		return requestParams;
+	}
+
+	getParentTypes(name: string): string[] {
+		return [];
+	}
+
+	async getDecrypted(
+		nodeCredentials: INodeCredentialsDetails,
+		type: string,
+	): Promise<ICredentialDataDecryptedObject> {
+		return {};
+	}
+
+	async getCredentials(
+		nodeCredentials: INodeCredentialsDetails,
+		type: string,
+	): Promise<ICredentials> {
+		return new Credentials({ id: null, name: '' }, '', [], '');
+	}
+
+	async updateCredentials(
+		nodeCredentials: INodeCredentialsDetails,
+		type: string,
+		data: ICredentialDataDecryptedObject,
+	): Promise<void> {}
+}
+
+export function getNodeParameter(
+	workflow: Workflow,
+	runExecutionData: IRunExecutionData | null,
+	runIndex: number,
+	connectionInputData: INodeExecutionData[],
+	node: INode,
+	parameterName: string,
+	itemIndex: number,
+	mode: WorkflowExecuteMode,
+	additionalKeys: IWorkflowDataProxyAdditionalKeys,
+	fallbackValue?: any,
+): NodeParameterValue | INodeParameters | NodeParameterValue[] | INodeParameters[] | object {
+	const nodeType = workflow.nodeTypes.getByNameAndVersion(node.type, node.typeVersion);
+	if (nodeType === undefined) {
+		throw new Error(`Node type "${node.type}" is not known so can not return paramter value!`);
+	}
+
+	const value = get(node.parameters, parameterName, fallbackValue);
+
+	if (value === undefined) {
+		throw new Error(`Could not get parameter "${parameterName}"!`);
+	}
+
+	let returnData;
+	try {
+		returnData = workflow.expression.getParameterValue(
+			value,
+			runExecutionData,
+			runIndex,
+			itemIndex,
+			node.name,
+			connectionInputData,
+			mode,
+			additionalKeys,
+		);
+	} catch (e) {
+		e.message += ` [Error in parameter: "${parameterName}"]`;
+		throw e;
+	}
+
+	return returnData;
+}
+
+export function getExecuteFunctions(
+	workflow: Workflow,
+	runExecutionData: IRunExecutionData,
+	runIndex: number,
+	connectionInputData: INodeExecutionData[],
+	inputData: ITaskDataConnections,
+	node: INode,
+	itemIndex: number,
+	additionalData: IWorkflowExecuteAdditionalData,
+	mode: WorkflowExecuteMode,
+): IExecuteFunctions {
+	return ((workflow, runExecutionData, connectionInputData, inputData, node) => {
+		return {
+			continueOnFail: () => {
+				return false;
+			},
+			evaluateExpression: (expression: string, itemIndex: number) => {
+				return expression;
+			},
+			async executeWorkflow(
+				workflowInfo: IExecuteWorkflowInfo,
+				inputData?: INodeExecutionData[],
+			): Promise<any> {
+				return additionalData.executeWorkflow(workflowInfo, additionalData, inputData);
+			},
+			getContext(type: string): IContextObject {
+				return NodeHelpers.getContext(runExecutionData, type, node);
+			},
+			async getCredentials(
+				type: string,
+				itemIndex?: number,
+			): Promise<ICredentialDataDecryptedObject | undefined> {
+				return {
+					apiKey: '12345',
+				};
+			},
+			getExecutionId: (): string => {
+				return additionalData.executionId!;
+			},
+			getInputData: (inputIndex = 0, inputName = 'main') => {
+				if (!inputData.hasOwnProperty(inputName)) {
+					// Return empty array because else it would throw error when nothing is connected to input
+					return [];
+				}
+
+				if (inputData[inputName].length < inputIndex) {
+					throw new Error(`Could not get input index "${inputIndex}" of input "${inputName}"!`);
+				}
+
+				if (inputData[inputName][inputIndex] === null) {
+					// return [];
+					throw new Error(`Value "${inputIndex}" of input "${inputName}" did not get set!`);
+				}
+
+				return inputData[inputName][inputIndex] as INodeExecutionData[];
+			},
+			getNodeParameter: (
+				parameterName: string,
+				itemIndex: number,
+				fallbackValue?: any,
+			):
+				| NodeParameterValue
+				| INodeParameters
+				| NodeParameterValue[]
+				| INodeParameters[]
+				| object => {
+				return getNodeParameter(
+					workflow,
+					runExecutionData,
+					runIndex,
+					connectionInputData,
+					node,
+					parameterName,
+					itemIndex,
+					mode,
+					{},
+					fallbackValue,
+				);
+			},
+			getMode: (): WorkflowExecuteMode => {
+				return mode;
+			},
+			getNode: () => {
+				return JSON.parse(JSON.stringify(node));
+			},
+			getRestApiUrl: (): string => {
+				return additionalData.restApiUrl;
+			},
+			getTimezone: (): string => {
+				return additionalData.timezone;
+			},
+			getWorkflow: () => {
+				return {
+					id: workflow.id,
+					name: workflow.name,
+					active: workflow.active,
+				};
+			},
+			getWorkflowDataProxy: (itemIndex: number): IWorkflowDataProxyData => {
+				const dataProxy = new WorkflowDataProxy(
+					workflow,
+					runExecutionData,
+					runIndex,
+					itemIndex,
+					node.name,
+					connectionInputData,
+					{},
+					mode,
+					{},
+				);
+				return dataProxy.getDataProxy();
+			},
+			getWorkflowStaticData(type: string): IDataObject {
+				return workflow.getStaticData(type, node);
+			},
+			prepareOutputData: NodeHelpers.prepareOutputData,
+			async putExecutionToWait(waitTill: Date): Promise<void> {
+				runExecutionData.waitTill = waitTill;
+			},
+			sendMessageToUI(...args: any[]): void {
+				if (mode !== 'manual') {
+					return;
+				}
+				try {
+					if (additionalData.sendMessageToUI) {
+						additionalData.sendMessageToUI(node.name, args);
+					}
+				} catch (error) {
+					// eslint-disable-next-line @typescript-eslint/restrict-template-expressions
+					console.error(`There was a problem sending messsage to UI: ${error.message}`);
+				}
+			},
+			async sendResponse(response: IExecuteResponsePromiseData): Promise<void> {
+				await additionalData.hooks?.executeHookFunctions('sendResponse', [response]);
+			},
+			helpers: {
+				async httpRequest(
+					requestOptions: IHttpRequestOptions,
+				): Promise<IN8nHttpFullResponse | IN8nHttpResponse> {
+					return {
+						body: {
+							headers: {},
+							statusCode: 200,
+							requestOptions,
+						},
+					};
+				},
+				async requestWithAuthentication(
+					this: IAllExecuteFunctions,
+					credentialsType: string,
+					requestOptions: IHttpRequestOptions,
+					additionalCredentialOptions?: IAdditionalCredentialOptions,
+				): Promise<any> {
+					return {
+						body: {
+							headers: {},
+							statusCode: 200,
+							credentialsType,
+							requestOptions,
+							additionalCredentialOptions,
+						},
+					};
+				},
+				async httpRequestWithAuthentication(
+					this: IAllExecuteFunctions,
+					credentialsType: string,
+					requestOptions: IHttpRequestOptions,
+					additionalCredentialOptions?: IAdditionalCredentialOptions,
+				): Promise<any> {
+					return {
+						body: {
+							headers: {},
+							statusCode: 200,
+							credentialsType,
+							requestOptions,
+							additionalCredentialOptions,
+						},
+					};
+				},
+			},
+		};
+	})(workflow, runExecutionData, connectionInputData, inputData, node);
+}
+
+export function getExecuteSingleFunctions(
+	workflow: Workflow,
+	runExecutionData: IRunExecutionData,
+	runIndex: number,
+	connectionInputData: INodeExecutionData[],
+	inputData: ITaskDataConnections,
+	node: INode,
+	itemIndex: number,
+	additionalData: IWorkflowExecuteAdditionalData,
+	mode: WorkflowExecuteMode,
+): IExecuteSingleFunctions {
+	return ((workflow, runExecutionData, connectionInputData, inputData, node, itemIndex) => {
+		return {
+			continueOnFail: () => {
+				return false;
+			},
+			evaluateExpression: (expression: string, evaluateItemIndex: number | undefined) => {
+				return expression;
+			},
+			getContext(type: string): IContextObject {
+				return NodeHelpers.getContext(runExecutionData, type, node);
+			},
+			async getCredentials(type: string): Promise<ICredentialDataDecryptedObject | undefined> {
+				return {
+					apiKey: '12345',
+				};
+			},
+			getInputData: (inputIndex = 0, inputName = 'main') => {
+				if (!inputData.hasOwnProperty(inputName)) {
+					// Return empty array because else it would throw error when nothing is connected to input
+					return { json: {} };
+				}
+
+				if (inputData[inputName].length < inputIndex) {
+					throw new Error(`Could not get input index "${inputIndex}" of input "${inputName}"!`);
+				}
+
+				const allItems = inputData[inputName][inputIndex];
+
+				if (allItems === null) {
+					// return [];
+					throw new Error(`Value "${inputIndex}" of input "${inputName}" did not get set!`);
+				}
+
+				if (allItems[itemIndex] === null) {
+					// return [];
+					throw new Error(
+						`Value "${inputIndex}" of input "${inputName}" with itemIndex "${itemIndex}" did not get set!`,
+					);
+				}
+
+				return allItems[itemIndex];
+			},
+			getMode: (): WorkflowExecuteMode => {
+				return mode;
+			},
+			getNode: () => {
+				return JSON.parse(JSON.stringify(node));
+			},
+			getRestApiUrl: (): string => {
+				return additionalData.restApiUrl;
+			},
+			getTimezone: (): string => {
+				return additionalData.timezone;
+			},
+			getNodeParameter: (
+				parameterName: string,
+				fallbackValue?: any,
+			):
+				| NodeParameterValue
+				| INodeParameters
+				| NodeParameterValue[]
+				| INodeParameters[]
+				| object => {
+				return getNodeParameter(
+					workflow,
+					runExecutionData,
+					runIndex,
+					connectionInputData,
+					node,
+					parameterName,
+					itemIndex,
+					mode,
+					{},
+					fallbackValue,
+				);
+			},
+			getWorkflow: () => {
+				return {
+					id: workflow.id,
+					name: workflow.name,
+					active: workflow.active,
+				};
+			},
+			getWorkflowDataProxy: (): IWorkflowDataProxyData => {
+				const dataProxy = new WorkflowDataProxy(
+					workflow,
+					runExecutionData,
+					runIndex,
+					itemIndex,
+					node.name,
+					connectionInputData,
+					{},
+					mode,
+					{},
+				);
+				return dataProxy.getDataProxy();
+			},
+			getWorkflowStaticData(type: string): IDataObject {
+				return workflow.getStaticData(type, node);
+			},
+			helpers: {
+				async httpRequest(
+					requestOptions: IHttpRequestOptions,
+				): Promise<IN8nHttpFullResponse | IN8nHttpResponse> {
+					return {
+						body: {
+							headers: {},
+							statusCode: 200,
+							requestOptions,
+						},
+					};
+				},
+				async requestWithAuthentication(
+					this: IAllExecuteFunctions,
+					credentialsType: string,
+					requestOptions: IHttpRequestOptions,
+					additionalCredentialOptions?: IAdditionalCredentialOptions,
+				): Promise<any> {
+					return {
+						body: {
+							headers: {},
+							statusCode: 200,
+							credentialsType,
+							requestOptions,
+							additionalCredentialOptions,
+						},
+					};
+				},
+				async httpRequestWithAuthentication(
+					this: IAllExecuteFunctions,
+					credentialsType: string,
+					requestOptions: IHttpRequestOptions,
+					additionalCredentialOptions?: IAdditionalCredentialOptions,
+				): Promise<any> {
+					return {
+						body: {
+							headers: {},
+							statusCode: 200,
+							credentialsType,
+							requestOptions,
+							additionalCredentialOptions,
+						},
+					};
+				},
+			},
+		};
+	})(workflow, runExecutionData, connectionInputData, inputData, node, itemIndex);
+}
+
 class NodeTypesClass implements INodeTypes {
 	nodeTypes: INodeTypeData = {
 		'test.set': {
@@ -120,3 +632,27 @@ export function NodeTypes(): NodeTypesClass {
 
 	return nodeTypesInstance;
 }
+
+export function WorkflowExecuteAdditionalData(): IWorkflowExecuteAdditionalData {
+	const workflowData: IWorkflowBase = {
+		name: '',
+		createdAt: new Date(),
+		updatedAt: new Date(),
+		active: true,
+		nodes: [],
+		connections: {},
+	};
+
+	return {
+		credentialsHelper: new CredentialsHelper(''),
+		hooks: new WorkflowHooks({}, 'trigger', '1', workflowData),
+		executeWorkflow: async (workflowInfo: IExecuteWorkflowInfo): Promise<any> => {},
+		sendMessageToUI: (message: string) => {},
+		restApiUrl: '',
+		encryptionKey: 'test',
+		timezone: 'America/New_York',
+		webhookBaseUrl: 'webhook',
+		webhookWaitingBaseUrl: 'webhook-waiting',
+		webhookTestBaseUrl: 'webhook-test',
+	};
+}