refactor(core): Reduce memory usage in the Webhook node (#4640)

use file streaming to pass webhook binaries around

packages/core/src/NodeExecuteFunctions.ts

@@ -64,6 +64,7 @@ import {
 	NodeExecutionWithMetadata,
 	IPairedItemData,
 	deepCopy,
+	BinaryFileType,
 } from 'n8n-workflow';
 
 import { Agent } from 'https';
@@ -77,8 +78,8 @@ import FormData from 'form-data';
 import path from 'path';
 import { OptionsWithUri, OptionsWithUrl, RequestCallback, RequiredUriUrl } from 'request';
 import requestPromise, { RequestPromiseOptions } from 'request-promise-native';
-import { fromBuffer } from 'file-type';
-import { lookup } from 'mime-types';
+import FileType from 'file-type';
+import { lookup, extension } from 'mime-types';
 import { IncomingHttpHeaders } from 'http';
 import axios, {
 	AxiosError,
@@ -830,6 +831,13 @@ export async function getBinaryDataBuffer(
 	return BinaryDataManager.getInstance().retrieveBinaryData(binaryData);
 }
 
+function fileTypeFromMimeType(mimeType: string): BinaryFileType | undefined {
+	if (mimeType.startsWith('image/')) return 'image';
+	if (mimeType.startsWith('video/')) return 'video';
+	if (mimeType.startsWith('text/') || mimeType.startsWith('application/json')) return 'text';
+	return;
+}
+
 /**
  * Store an incoming IBinaryData & related buffer using the configured binary data manager.
  *
@@ -846,10 +854,60 @@ export async function setBinaryDataBuffer(
 	return BinaryDataManager.getInstance().storeBinaryData(data, binaryData, executionId);
 }
 
+export async function copyBinaryFile(
+	executionId: string,
+	filePath: string,
+	fileName: string,
+	mimeType?: string,
+): Promise<IBinaryData> {
+	let fileExtension: string | undefined;
+	if (!mimeType) {
+		// If no mime type is given figure it out
+
+		if (filePath) {
+			// Use file path to guess mime type
+			const mimeTypeLookup = lookup(filePath);
+			if (mimeTypeLookup) {
+				mimeType = mimeTypeLookup;
+			}
+		}
+
+		if (!mimeType) {
+			// read the first bytes of the file to guess mime type
+			const fileTypeData = await FileType.fromFile(filePath);
+			if (fileTypeData) {
+				mimeType = fileTypeData.mime;
+				fileExtension = fileTypeData.ext;
+			}
+		}
+
+		if (!mimeType) {
+			// Fall back to text
+			mimeType = 'text/plain';
+		}
+	} else if (!fileExtension) {
+		fileExtension = extension(mimeType) || undefined;
+	}
+
+	const returnData: IBinaryData = {
+		mimeType,
+		fileType: fileTypeFromMimeType(mimeType),
+		fileExtension,
+		data: '',
+	};
+
+	if (fileName) {
+		returnData.fileName = fileName;
+	} else if (filePath) {
+		returnData.fileName = path.parse(filePath).base;
+	}
+
+	return BinaryDataManager.getInstance().copyBinaryFile(returnData, filePath, executionId);
+}
+
 /**
  * Takes a buffer and converts it into the format n8n uses. It encodes the binary data as
  * base64 and adds metadata.
- *
  */
 export async function prepareBinaryData(
 	binaryData: Buffer,
@@ -871,7 +929,7 @@ export async function prepareBinaryData(
 
 		if (!mimeType) {
 			// Use buffer to guess mime type
-			const fileTypeData = await fromBuffer(binaryData);
+			const fileTypeData = await FileType.fromBuffer(binaryData);
 			if (fileTypeData) {
 				mimeType = fileTypeData.mime;
 				fileExtension = fileTypeData.ext;
@@ -882,10 +940,13 @@ export async function prepareBinaryData(
 			// Fall back to text
 			mimeType = 'text/plain';
 		}
+	} else if (!fileExtension) {
+		fileExtension = extension(mimeType) || undefined;
 	}
 
 	const returnData: IBinaryData = {
 		mimeType,
+		fileType: fileTypeFromMimeType(mimeType),
 		fileExtension,
 		data: '',
 	};
@@ -3076,6 +3137,19 @@ export function getExecuteWebhookFunctions(
 				async setBinaryDataBuffer(data: IBinaryData, binaryData: Buffer): Promise<IBinaryData> {
 					return setBinaryDataBuffer.call(this, data, binaryData, additionalData.executionId!);
 				},
+				async copyBinaryFile(
+					filePath: string,
+					fileName: string,
+					mimeType?: string,
+				): Promise<IBinaryData> {
+					return copyBinaryFile.call(
+						this,
+						additionalData.executionId!,
+						filePath,
+						fileName,
+						mimeType,
+					);
+				},
 				async prepareBinaryData(
 					binaryData: Buffer,
 					filePath?: string,