admin/n8n-enterprise-unlocked
1
0
Fork 0
mirror of https://github.com/Abdulazizzn/n8n-enterprise-unlocked.git synced 2025-12-17 01:56:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(core): Remove storeMetadata and getSize from binary data manager interface (no-changelog) (#7195)

Browse Source 
Depends on: #7164 | Story:
[PAY-838](https://linear.app/n8n/issue/PAY-838/introduce-object-store-service-for-binary-data)

This PR removes `storeMetadata` and `getSize` from the binary data
manager interface, as these are specific to filesystem mode. Also this
disambiguates identifiers:

```
binaryDataId
filesystem:289b4aac51e-dac6-4167-b793-6d5c415e2b47 {mode}:{fileId}

fileId - FS
289b4aac51e-dac6-4167-b793-6d5c415e2b47 {executionId}{uuid}

fileId - S3
/workflows/{workflowId}/executions/{executionId}/binary_data/b4aac51e-dac6-4167-b793-6d5c415e2b47
```

Note: The object store changes originally in this PR were extracted out
into the final PR.

---------

Co-authored-by: कारतोफ्फेलस्क्रिप्ट™ <aditya@netroy.in>
This commit is contained in:
Iván Ovejero
2023-09-25 10:07:06 +02:00
committed by GitHub
parent 6d6e2488c6
commit dcc9cc13ed
7 changed files with 143 additions and 132 deletions
Download Patch File Download Diff File Expand all files Collapse all files

98
packages/core/src/BinaryData/BinaryData.service.ts
View File

@@ -5,12 +5,13 @@ import { Service } from 'typedi';
import { BINARY_ENCODING, LoggerProxy as Logger, IBinaryData } from 'n8n-workflow';
import { FileSystemManager } from './FileSystem.manager';
import { InvalidBinaryDataManagerError, InvalidBinaryDataModeError, areValidModes } from './utils';
import { UnknownBinaryDataManager, InvalidBinaryDataMode } from './errors';
import { LogCatch } from '../decorators/LogCatch.decorator';
import { areValidModes } from './utils';
import type { Readable } from 'stream';
import type { BinaryData } from './types';
import type { INodeExecutionData } from 'n8n-workflow';
import { LogCatch } from '../decorators/LogCatch.decorator';
@Service()
export class BinaryDataService {
@@ -21,7 +22,7 @@ export class BinaryDataService {
private managers: Record<string, BinaryData.Manager> = {};
async init(config: BinaryData.Config) {
if (!areValidModes(config.availableModes)) throw new InvalidBinaryDataModeError();
if (!areValidModes(config.availableModes)) throw new InvalidBinaryDataMode();
this.availableModes = config.availableModes;
this.mode = config.mode;
@@ -45,47 +46,39 @@ export class BinaryDataService {
return binaryData;
}
const identifier = await manager.copyByPath(path, executionId);
binaryData.id = this.createIdentifier(identifier);
binaryData.data = this.mode; // clear binary data from memory
const fileSize = await manager.getSize(identifier);
binaryData.fileSize = prettyBytes(fileSize);
await manager.storeMetadata(identifier, {
const { fileId, fileSize } = await manager.copyByFilePath(path, executionId, {
fileName: binaryData.fileName,
mimeType: binaryData.mimeType,
fileSize,
});
binaryData.id = this.createBinaryDataId(fileId);
binaryData.fileSize = prettyBytes(fileSize);
binaryData.data = this.mode; // clear binary data from memory
return binaryData;
}
@LogCatch((error) => Logger.error('Failed to write binary data file', { error }))
async store(binaryData: IBinaryData, input: Buffer | Readable, executionId: string) {
async store(binaryData: IBinaryData, bufferOrStream: Buffer | Readable, executionId: string) {
const manager = this.managers[this.mode];
if (!manager) {
const buffer = await this.binaryToBuffer(input);
const buffer = await this.binaryToBuffer(bufferOrStream);
binaryData.data = buffer.toString(BINARY_ENCODING);
binaryData.fileSize = prettyBytes(buffer.length);
return binaryData;
}
const identifier = await manager.store(input, executionId);
binaryData.id = this.createIdentifier(identifier);
binaryData.data = this.mode; // clear binary data from memory
const fileSize = await manager.getSize(identifier);
binaryData.fileSize = prettyBytes(fileSize);
await manager.storeMetadata(identifier, {
const { fileId, fileSize } = await manager.store(bufferOrStream, executionId, {
fileName: binaryData.fileName,
mimeType: binaryData.mimeType,
fileSize,
});
binaryData.id = this.createBinaryDataId(fileId);
binaryData.fileSize = prettyBytes(fileSize);
binaryData.data = this.mode; // clear binary data from memory
return binaryData;
}
@@ -96,34 +89,32 @@ export class BinaryDataService {
});
}
getAsStream(identifier: string, chunkSize?: number) {
const { mode, id } = this.splitBinaryModeFileId(identifier);
getAsStream(binaryDataId: string, chunkSize?: number) {
const [mode, fileId] = binaryDataId.split(':');
return this.getManager(mode).getStream(id, chunkSize);
return this.getManager(mode).getAsStream(fileId, chunkSize);
}
async getBinaryDataBuffer(binaryData: IBinaryData) {
if (binaryData.id) return this.retrieveBinaryDataByIdentifier(binaryData.id);
async getAsBuffer(binaryData: IBinaryData) {
if (binaryData.id) {
const [mode, fileId] = binaryData.id.split(':');
return this.getManager(mode).getAsBuffer(fileId);
}
return Buffer.from(binaryData.data, BINARY_ENCODING);
}
async retrieveBinaryDataByIdentifier(identifier: string) {
const { mode, id } = this.splitBinaryModeFileId(identifier);
getPath(binaryDataId: string) {
const [mode, fileId] = binaryDataId.split(':');
return this.getManager(mode).getBuffer(id);
return this.getManager(mode).getPath(fileId);
}
getPath(identifier: string) {
const { mode, id } = this.splitBinaryModeFileId(identifier);
async getMetadata(binaryDataId: string) {
const [mode, fileId] = binaryDataId.split(':');
return this.getManager(mode).getPath(id);
}
async getMetadata(identifier: string) {
const { mode, id } = this.splitBinaryModeFileId(identifier);
return this.getManager(mode).getMetadata(id);
return this.getManager(mode).getMetadata(fileId);
}
async deleteManyByExecutionIds(executionIds: string[]) {
@@ -167,14 +158,11 @@ export class BinaryDataService {
// private methods
// ----------------------------------
private createIdentifier(filename: string) {
return `${this.mode}:${filename}`;
}
private splitBinaryModeFileId(fileId: string) {
const [mode, id] = fileId.split(':');
return { mode, id };
/**
* Create an identifier `${mode}:{fileId}` for `IBinaryData['id']`.
*/
private createBinaryDataId(fileId: string) {
return `${this.mode}:${fileId}`;
}
private async duplicateBinaryDataInExecData(
@@ -195,12 +183,12 @@ export class BinaryDataService {
return { key, newId: undefined };
}
return manager
?.copyByIdentifier(this.splitBinaryModeFileId(binaryDataId).id, executionId)
.then((filename) => ({
newId: this.createIdentifier(filename),
key,
}));
const [_mode, fileId] = binaryDataId.split(':');
return manager?.copyByFileId(fileId, executionId).then((newFileId) => ({
newId: this.createBinaryDataId(newFileId),
key,
}));
});
return Promise.all(bdPromises).then((b) => {
@@ -222,6 +210,6 @@ export class BinaryDataService {
if (manager) return manager;
throw new InvalidBinaryDataManagerError(mode);
throw new UnknownBinaryDataManager(mode);
}
}