✨ Introduce binary data management (#2059)

* introduce binary data management * merge fixes * fixes * init binary data manager for other modes * improve binary manager * improve binary manager * delete binary data on executions delete * lazy delete non-saved executions binary data * merge fixes + error handing * improve structure * leftovers and cleanups * formatting * fix config description * fixes * fix races * duplicate binary data for execute workflow node * clean up and cr * update mode name, add binary mode to diagnostics * update mode name, add prefix to filename * update filename * allow multiple modes, backward compatibility * improve file and id naming * use execution id for binary data storage * delete binary data by execution id * add meta for persisted binary data * delete marked persisted files * mark deletion by executionid * add env var for persisted binary data ttl * improvements * lint fix * fix env var description * cleanup * cleanup * ⚡ Minor improvements Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
2025-12-21 03:42:16 +00:00 · 2021-12-23 22:29:04 +01:00
parent 416e15cdb6
commit 1e42effc3a
22 changed files with 743 additions and 40 deletions
--- a/packages/core/src/BinaryDataManager/index.ts
+++ b/packages/core/src/BinaryDataManager/index.ts
@@ -0,0 +1,187 @@
+import { IBinaryData, INodeExecutionData } from 'n8n-workflow';
+import { BINARY_ENCODING } from '../Constants';
+import { IBinaryDataConfig, IBinaryDataManager } from '../Interfaces';
+import { BinaryDataFileSystem } from './FileSystem';
+
+export class BinaryDataManager {
+	private static instance: BinaryDataManager;
+
+	private managers: {
+		[key: string]: IBinaryDataManager;
+	};
+
+	private binaryDataMode: string;
+
+	private availableModes: string[];
+
+	constructor(config: IBinaryDataConfig) {
+		this.binaryDataMode = config.mode;
+		this.availableModes = config.availableModes.split(',');
+		this.managers = {};
+	}
+
+	static async init(config: IBinaryDataConfig, mainManager = false): Promise<void> {
+		if (BinaryDataManager.instance) {
+			throw new Error('Binary Data Manager already initialized');
+		}
+
+		BinaryDataManager.instance = new BinaryDataManager(config);
+
+		if (BinaryDataManager.instance.availableModes.includes('filesystem')) {
+			BinaryDataManager.instance.managers.filesystem = new BinaryDataFileSystem(config);
+			await BinaryDataManager.instance.managers.filesystem.init(mainManager);
+		}
+
+		return undefined;
+	}
+
+	static getInstance(): BinaryDataManager {
+		if (!BinaryDataManager.instance) {
+			throw new Error('Binary Data Manager not initialized');
+		}
+
+		return BinaryDataManager.instance;
+	}
+
+	async storeBinaryData(
+		binaryData: IBinaryData,
+		binaryBuffer: Buffer,
+		executionId: string,
+	): Promise<IBinaryData> {
+		const retBinaryData = binaryData;
+
+		if (this.managers[this.binaryDataMode]) {
+			return this.managers[this.binaryDataMode]
+				.storeBinaryData(binaryBuffer, executionId)
+				.then((filename) => {
+					retBinaryData.id = this.generateBinaryId(filename);
+					return retBinaryData;
+				});
+		}
+
+		retBinaryData.data = binaryBuffer.toString(BINARY_ENCODING);
+		return binaryData;
+	}
+
+	async retrieveBinaryData(binaryData: IBinaryData): Promise<Buffer> {
+		if (binaryData.id) {
+			return this.retrieveBinaryDataByIdentifier(binaryData.id);
+		}
+
+		return Buffer.from(binaryData.data, BINARY_ENCODING);
+	}
+
+	async retrieveBinaryDataByIdentifier(identifier: string): Promise<Buffer> {
+		const { mode, id } = this.splitBinaryModeFileId(identifier);
+		if (this.managers[mode]) {
+			return this.managers[mode].retrieveBinaryDataByIdentifier(id);
+		}
+
+		throw new Error('Storage mode used to store binary data not available');
+	}
+
+	async markDataForDeletionByExecutionId(executionId: string): Promise<void> {
+		if (this.managers[this.binaryDataMode]) {
+			return this.managers[this.binaryDataMode].markDataForDeletionByExecutionId(executionId);
+		}
+
+		return Promise.resolve();
+	}
+
+	async persistBinaryDataForExecutionId(executionId: string): Promise<void> {
+		if (this.managers[this.binaryDataMode]) {
+			return this.managers[this.binaryDataMode].persistBinaryDataForExecutionId(executionId);
+		}
+
+		return Promise.resolve();
+	}
+
+	async deleteBinaryDataByExecutionId(executionId: string): Promise<void> {
+		if (this.managers[this.binaryDataMode]) {
+			return this.managers[this.binaryDataMode].deleteBinaryDataByExecutionId(executionId);
+		}
+
+		return Promise.resolve();
+	}
+
+	async duplicateBinaryData(
+		inputData: Array<INodeExecutionData[] | null> | unknown,
+		executionId: string,
+	): Promise<INodeExecutionData[][]> {
+		if (inputData && this.managers[this.binaryDataMode]) {
+			const returnInputData = (inputData as INodeExecutionData[][]).map(
+				async (executionDataArray) => {
+					if (executionDataArray) {
+						return Promise.all(
+							executionDataArray.map((executionData) => {
+								if (executionData.binary) {
+									return this.duplicateBinaryDataInExecData(executionData, executionId);
+								}
+
+								return executionData;
+							}),
+						);
+					}
+
+					return executionDataArray;
+				},
+			);
+
+			return Promise.all(returnInputData);
+		}
+
+		return Promise.resolve(inputData as INodeExecutionData[][]);
+	}
+
+	private generateBinaryId(filename: string) {
+		return `${this.binaryDataMode}:${filename}`;
+	}
+
+	private splitBinaryModeFileId(fileId: string): { mode: string; id: string } {
+		const [mode, id] = fileId.split(':');
+		return { mode, id };
+	}
+
+	private async duplicateBinaryDataInExecData(
+		executionData: INodeExecutionData,
+		executionId: string,
+	): Promise<INodeExecutionData> {
+		const binaryManager = this.managers[this.binaryDataMode];
+
+		if (executionData.binary) {
+			const binaryDataKeys = Object.keys(executionData.binary);
+			const bdPromises = binaryDataKeys.map(async (key: string) => {
+				if (!executionData.binary) {
+					return { key, newId: undefined };
+				}
+
+				const binaryDataId = executionData.binary[key].id;
+				if (!binaryDataId) {
+					return { key, newId: undefined };
+				}
+
+				return binaryManager
+					?.duplicateBinaryDataByIdentifier(
+						this.splitBinaryModeFileId(binaryDataId).id,
+						executionId,
+					)
+					.then((filename) => ({
+						newId: this.generateBinaryId(filename),
+						key,
+					}));
+			});
+
+			return Promise.all(bdPromises).then((b) => {
+				return b.reduce((acc, curr) => {
+					if (acc.binary && curr) {
+						acc.binary[curr.key].id = curr.newId;
+					}
+
+					return acc;
+				}, executionData);
+			});
+		}
+
+		return executionData;
+	}
+}