fix(core & function nodes): Update function nodes to work with binary-data-mode 'filesystem'. (#3845)

* Initial Fix

* Self-Review #1

* Lint

* Added support for FunctionItem. Minor updates.

* Self-review

* review comments. Added testing.

* Self Review

* Fixed memory handling on data manager use.

* Fixes for unnecessary memory leaks.

packages/core/src/BinaryDataManager/index.ts

@@ -50,15 +50,23 @@ export class BinaryDataManager {
 	): Promise<IBinaryData> {
 		const retBinaryData = binaryData;
 
+		// If a manager handles this binary, return the binary data with it's reference id.
 		if (this.managers[this.binaryDataMode]) {
 			return this.managers[this.binaryDataMode]
 				.storeBinaryData(binaryBuffer, executionId)
 				.then((filename) => {
+					// Add data manager reference id.
 					retBinaryData.id = this.generateBinaryId(filename);
+
+					// Prevent preserving data in memory if handled by a data manager.
+					retBinaryData.data = this.binaryDataMode;
+
+					// Short-circuit return to prevent further actions.
 					return retBinaryData;
 				});
 		}
 
+		// Else fallback to storing this data in memory.
 		retBinaryData.data = binaryBuffer.toString(BINARY_ENCODING);
 		return binaryData;
 	}

packages/core/src/Interfaces.ts

@@ -45,6 +45,7 @@ export interface IExecuteFunctions extends IExecuteFunctionsBase {
 			mimeType?: string,
 		): Promise<IBinaryData>;
 		getBinaryDataBuffer(itemIndex: number, propertyName: string): Promise<Buffer>;
+		setBinaryDataBuffer(data: IBinaryData, binaryData: Buffer): Promise<IBinaryData>;
 		request: (uriOrObject: string | IDataObject | any, options?: IDataObject) => Promise<any>; // tslint:disable-line:no-any
 		requestWithAuthentication(
 			this: IAllExecuteFunctions,
@@ -80,6 +81,7 @@ export interface IExecuteFunctions extends IExecuteFunctionsBase {
 export interface IExecuteSingleFunctions extends IExecuteSingleFunctionsBase {
 	helpers: {
 		getBinaryDataBuffer(propertyName: string, inputIndex?: number): Promise<Buffer>;
+		setBinaryDataBuffer(data: IBinaryData, binaryData: Buffer): Promise<IBinaryData>;
 		httpRequest(requestOptions: IHttpRequestOptions): Promise<any>; // tslint:disable-line:no-any
 		prepareBinaryData(
 			binaryData: Buffer,

packages/core/src/NodeExecuteFunctions.ts

@@ -813,6 +813,22 @@ export async function getBinaryDataBuffer(
 	return BinaryDataManager.getInstance().retrieveBinaryData(binaryData);
 }
 
+/**
+ * Store an incoming IBinaryData & related buffer using the configured binary data manager.
+ *
+ * @export
+ * @param {IBinaryData} data
+ * @param {Buffer} binaryData
+ * @returns {Promise<IBinaryData>}
+ */
+export async function setBinaryDataBuffer(
+	data: IBinaryData,
+	binaryData: Buffer,
+	executionId: string,
+): Promise<IBinaryData> {
+	return BinaryDataManager.getInstance().storeBinaryData(data, binaryData, executionId);
+}
+
 /**
  * Takes a buffer and converts it into the format n8n uses. It encodes the binary data as
  * base64 and adds metadata.
@@ -882,7 +898,7 @@ export async function prepareBinaryData(
 		}
 	}
 
-	return BinaryDataManager.getInstance().storeBinaryData(returnData, binaryData, executionId);
+	return setBinaryDataBuffer(returnData, binaryData, executionId);
 }
 
 /**
@@ -1950,6 +1966,9 @@ export function getExecutePollFunctions(
 			},
 			helpers: {
 				httpRequest,
+				async setBinaryDataBuffer(data: IBinaryData, binaryData: Buffer): Promise<IBinaryData> {
+					return setBinaryDataBuffer.call(this, data, binaryData, additionalData.executionId!);
+				},
 				async prepareBinaryData(
 					binaryData: Buffer,
 					filePath?: string,
@@ -2121,6 +2140,9 @@ export function getExecuteTriggerFunctions(
 						additionalCredentialOptions,
 					);
 				},
+				async setBinaryDataBuffer(data: IBinaryData, binaryData: Buffer): Promise<IBinaryData> {
+					return setBinaryDataBuffer.call(this, data, binaryData, additionalData.executionId!);
+				},
 				async prepareBinaryData(
 					binaryData: Buffer,
 					filePath?: string,
@@ -2381,6 +2403,9 @@ export function getExecuteFunctions(
 						additionalCredentialOptions,
 					);
 				},
+				async setBinaryDataBuffer(data: IBinaryData, binaryData: Buffer): Promise<IBinaryData> {
+					return setBinaryDataBuffer.call(this, data, binaryData, additionalData.executionId!);
+				},
 				async prepareBinaryData(
 					binaryData: Buffer,
 					filePath?: string,
@@ -2624,6 +2649,9 @@ export function getExecuteSingleFunctions(
 						additionalCredentialOptions,
 					);
 				},
+				async setBinaryDataBuffer(data: IBinaryData, binaryData: Buffer): Promise<IBinaryData> {
+					return setBinaryDataBuffer.call(this, data, binaryData, additionalData.executionId!);
+				},
 				async prepareBinaryData(
 					binaryData: Buffer,
 					filePath?: string,
@@ -3121,6 +3149,9 @@ export function getExecuteWebhookFunctions(
 						additionalCredentialOptions,
 					);
 				},
+				async setBinaryDataBuffer(data: IBinaryData, binaryData: Buffer): Promise<IBinaryData> {
+					return setBinaryDataBuffer.call(this, data, binaryData, additionalData.executionId!);
+				},
 				async prepareBinaryData(
 					binaryData: Buffer,
 					filePath?: string,