refactor(Read Binary File Node): Use node streams for to reduce memory usage (#5069)

packages/core/src/BinaryDataManager/FileSystem.ts

@@ -2,6 +2,7 @@ import fs from 'fs/promises';
 import { jsonParse } from 'n8n-workflow';
 import path from 'path';
 import { v4 as uuid } from 'uuid';
+import type { Readable } from 'stream';
 
 import { BinaryMetadata, IBinaryDataConfig, IBinaryDataManager } from '../Interfaces';
 
@@ -66,10 +67,10 @@ export class BinaryDataFileSystem implements IBinaryDataManager {
 		return jsonParse(await fs.readFile(this.getMetadataPath(identifier), { encoding: 'utf-8' }));
 	}
 
-	async storeBinaryData(binaryBuffer: Buffer, executionId: string): Promise<string> {
+	async storeBinaryData(binaryData: Buffer | Readable, executionId: string): Promise<string> {
 		const binaryDataId = this.generateFileName(executionId);
 		await this.addBinaryIdToPersistMeta(executionId, binaryDataId);
-		await this.saveToLocalStorage(binaryBuffer, binaryDataId);
+		await this.saveToLocalStorage(binaryData, binaryDataId);
 		return binaryDataId;
 	}
 
@@ -234,8 +235,8 @@ export class BinaryDataFileSystem implements IBinaryDataManager {
 		await fs.cp(source, this.getBinaryPath(identifier));
 	}
 
-	private async saveToLocalStorage(data: Buffer, identifier: string) {
-		await fs.writeFile(this.getBinaryPath(identifier), data);
+	private async saveToLocalStorage(binaryData: Buffer | Readable, identifier: string) {
+		await fs.writeFile(this.getBinaryPath(identifier), binaryData);
 	}
 
 	private async retrieveFromLocalStorage(identifier: string): Promise<Buffer> {

packages/core/src/BinaryDataManager/index.ts

@@ -1,9 +1,11 @@
-import prettyBytes from 'pretty-bytes';
+import concatStream from 'concat-stream';
+import { readFile, stat } from 'fs/promises';
 import type { IBinaryData, INodeExecutionData } from 'n8n-workflow';
+import prettyBytes from 'pretty-bytes';
+import type { Readable } from 'stream';
 import { BINARY_ENCODING } from '../Constants';
 import type { BinaryMetadata, IBinaryDataConfig, IBinaryDataManager } from '../Interfaces';
 import { BinaryDataFileSystem } from './FileSystem';
-import { readFile, stat } from 'fs/promises';
 
 export class BinaryDataManager {
 	static instance: BinaryDataManager | undefined;
@@ -79,29 +81,35 @@ export class BinaryDataManager {
 
 	async storeBinaryData(
 		binaryData: IBinaryData,
-		binaryBuffer: Buffer,
+		input: Buffer | Readable,
 		executionId: string,
 	): Promise<IBinaryData> {
-		binaryData.fileSize = prettyBytes(binaryBuffer.length);
-
 		// If a manager handles this binary, return the binary data with its reference id.
 		const manager = this.managers[this.binaryDataMode];
 		if (manager) {
-			const identifier = await manager.storeBinaryData(binaryBuffer, executionId);
+			const identifier = await manager.storeBinaryData(input, executionId);
 			// Add data manager reference id.
 			binaryData.id = this.generateBinaryId(identifier);
 
 			// Prevent preserving data in memory if handled by a data manager.
 			binaryData.data = this.binaryDataMode;
 
+			const fileSize = await manager.getFileSize(identifier);
+			binaryData.fileSize = prettyBytes(fileSize);
+
 			await manager.storeBinaryMetadata(identifier, {
 				fileName: binaryData.fileName,
 				mimeType: binaryData.mimeType,
-				fileSize: binaryBuffer.length,
+				fileSize,
 			});
 		} else {
 			// Else fallback to storing this data in memory.
-			binaryData.data = binaryBuffer.toString(BINARY_ENCODING);
+			const buffer = await new Promise<Buffer>((resolve) => {
+				if (Buffer.isBuffer(input)) resolve(input);
+				else input.pipe(concatStream(resolve));
+			});
+			binaryData.data = buffer.toString(BINARY_ENCODING);
+			binaryData.fileSize = prettyBytes(buffer.length);
 		}
 
 		return binaryData;

packages/core/src/Interfaces.ts

@@ -1,4 +1,5 @@
-import {
+import type { Readable } from 'stream';
+import type {
 	IPollResponse,
 	ITriggerResponse,
 	IWorkflowSettings as IWorkflowSettingsWorkflow,
@@ -67,7 +68,7 @@ export interface IBinaryDataManager {
 	copyBinaryFile(filePath: string, executionId: string): Promise<string>;
 	storeBinaryMetadata(identifier: string, metadata: BinaryMetadata): Promise<void>;
 	getBinaryMetadata(identifier: string): Promise<BinaryMetadata>;
-	storeBinaryData(binaryBuffer: Buffer, executionId: string): Promise<string>;
+	storeBinaryData(binaryData: Buffer | Readable, executionId: string): Promise<string>;
 	retrieveBinaryDataByIdentifier(identifier: string): Promise<Buffer>;
 	getBinaryPath(identifier: string): string;
 	markDataForDeletionByExecutionId(executionId: string): Promise<void>;

packages/core/src/NodeExecuteFunctions.ts

@@ -91,6 +91,8 @@ import axios, {
 	Method,
 } from 'axios';
 import url, { URL, URLSearchParams } from 'url';
+import type { Readable } from 'stream';
+
 import { BinaryDataManager } from './BinaryDataManager';
 import type { IResponseError, IWorkflowSettings } from './Interfaces';
 import { extractValue } from './ExtractValue';
@@ -840,12 +842,12 @@ export async function getBinaryDataBuffer(
  *
  * @export
  * @param {IBinaryData} data
- * @param {Buffer} binaryData
+ * @param {Buffer | Readable} binaryData
  * @returns {Promise<IBinaryData>}
  */
 export async function setBinaryDataBuffer(
 	data: IBinaryData,
-	binaryData: Buffer,
+	binaryData: Buffer | Readable,
 	executionId: string,
 ): Promise<IBinaryData> {
 	return BinaryDataManager.getInstance().storeBinaryData(data, binaryData, executionId);
@@ -907,7 +909,7 @@ export async function copyBinaryFile(
  * base64 and adds metadata.
  */
 async function prepareBinaryData(
-	binaryData: Buffer,
+	binaryData: Buffer | Readable,
 	executionId: string,
 	filePath?: string,
 	mimeType?: string,
@@ -924,7 +926,8 @@ async function prepareBinaryData(
 			}
 		}
 
-		if (!mimeType) {
+		// TODO: detect filetype from streams
+		if (!mimeType && Buffer.isBuffer(binaryData)) {
 			// Use buffer to guess mime type
 			const fileTypeData = await FileType.fromBuffer(binaryData);
 			if (fileTypeData) {