admin/n8n-enterprise-unlocked
1
0
Fork 0
mirror of https://github.com/Abdulazizzn/n8n-enterprise-unlocked.git synced 2025-12-17 10:02:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(core): Introduce object store service (#7225)

Browse Source 
Depends on https://github.com/n8n-io/n8n/pull/7220 | Story:
[PAY-840](https://linear.app/n8n/issue/PAY-840/introduce-object-store-service-and-manager-for-binary-data)

This PR introduces an object store service for Enterprise edition. Note
that the service is tested but currently unused - it will be integrated
soon as a binary data manager, and later for execution data.
`amazonaws.com` in the host is temporarily hardcoded until we integrate
the service and test against AWS, Cloudflare and Backblaze, in the next
PR.

This is ready for review - the PR it depends on is approved and waiting
for CI.

---------

Co-authored-by: कारतोफ्फेलस्क्रिप्ट™ <aditya@netroy.in>
This commit is contained in:
Iván Ovejero
2023-09-27 09:42:35 +02:00
committed by GitHub
parent 2c4e25c06b
commit fa845453bb
15 changed files with 12628 additions and 4086 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
packages/core/src/BinaryData/BinaryData.service.ts
View File

@@ -1,13 +1,13 @@
/* eslint-disable @typescript-eslint/naming-convention */
import { readFile, stat } from 'fs/promises';
import concatStream from 'concat-stream';
import prettyBytes from 'pretty-bytes';
import { Service } from 'typedi';
import { BINARY_ENCODING, LoggerProxy as Logger, IBinaryData } from 'n8n-workflow';
import { FileSystemManager } from './FileSystem.manager';
import { UnknownBinaryDataManager, InvalidBinaryDataMode } from './errors';
import { LogCatch } from '../decorators/LogCatch.decorator';
import { areValidModes } from './utils';
import { areValidModes, toBuffer } from './utils';
import type { Readable } from 'stream';
import type { BinaryData } from './types';
@@ -15,8 +15,6 @@ import type { INodeExecutionData } from 'n8n-workflow';
@Service()
export class BinaryDataService {
private availableModes: BinaryData.Mode[] = [];
private mode: BinaryData.Mode = 'default';
private managers: Record<string, BinaryData.Manager> = {};
@@ -24,10 +22,10 @@ export class BinaryDataService {
async init(config: BinaryData.Config) {
if (!areValidModes(config.availableModes)) throw new InvalidBinaryDataMode();
this.availableModes = config.availableModes;
this.mode = config.mode;
if (this.availableModes.includes('filesystem')) {
if (config.availableModes.includes('filesystem')) {
const { FileSystemManager } = await import('./FileSystem.manager');
this.managers.filesystem = new FileSystemManager(config.localStoragePath);
await this.managers.filesystem.init();
@@ -80,7 +78,7 @@ export class BinaryDataService {
const manager = this.managers[this.mode];
if (!manager) {
const buffer = await this.binaryToBuffer(bufferOrStream);
const buffer = await this.toBuffer(bufferOrStream);
binaryData.data = buffer.toString(BINARY_ENCODING);
binaryData.fileSize = prettyBytes(buffer.length);
@@ -106,11 +104,8 @@ export class BinaryDataService {
return binaryData;
}
async binaryToBuffer(body: Buffer | Readable) {
return new Promise<Buffer>((resolve) => {
if (Buffer.isBuffer(body)) resolve(body);
else body.pipe(concatStream(resolve));
});
async toBuffer(bufferOrStream: Buffer | Readable) {
return toBuffer(bufferOrStream);
}
async getAsStream(binaryDataId: string, chunkSize?: number) {
@@ -141,12 +136,12 @@ export class BinaryDataService {
return this.getManager(mode).getMetadata(fileId);
}
async deleteManyByExecutionIds(executionIds: string[]) {
async deleteMany(ids: BinaryData.IdsForDeletion) {
const manager = this.managers[this.mode];
if (!manager) return;
await manager.deleteManyByExecutionIds(executionIds);
await manager.deleteMany(ids);
}
@LogCatch((error) =>