refactor(core): Generalize binary data manager interface (no-changelog) (#7164)

Depends on: #7092 | Story:
[PAY-768](https://linear.app/n8n/issue/PAY-768)

This PR: 
- Generalizes the `IBinaryDataManager` interface.
- Adjusts `Filesystem.ts` to satisfy the interface.
- Sets up an S3 client stub to be filled in in the next PR.
- Turns `BinaryDataManager` into an injectable service.
- Adjusts the config schema and adds new validators.

Note that the PR looks large but all the main changes are in
`packages/core/src/binaryData`.

Out of scope:
- `BinaryDataManager` (now `BinaryDataService`) and `Filesystem.ts` (now
`fs.client.ts`) were slightly refactored for maintainability, but fully
overhauling them is **not** the focus of this PR, which is meant to
clear the way for the S3 implementation. Future improvements for these
two should include setting up a backwards-compatible dir structure that
makes it easier to locate binary data files to delete, removing
duplication, simplifying cloning methods, using integers for binary data
size instead of `prettyBytes()`, writing tests for existing binary data
logic, etc.

---------

Co-authored-by: कारतोफ्फेलस्क्रिप्ट™ <aditya@netroy.in>
This commit is contained in:
Iván Ovejero
2023-09-22 17:22:12 +02:00
committed by GitHub
parent 4614e1e1c9
commit 6d6e2488c6
46 changed files with 602 additions and 552 deletions

View File

@@ -1,4 +1,3 @@
import { BinaryDataManager } from '@/BinaryDataManager';
import {
getBinaryDataBuffer,
parseIncomingMessage,
@@ -16,25 +15,24 @@ import type {
Workflow,
WorkflowHooks,
} from 'n8n-workflow';
import { BinaryDataService } from '@/BinaryData/BinaryData.service';
import nock from 'nock';
import { tmpdir } from 'os';
import { join } from 'path';
import { initLogger } from './helpers/utils';
import Container from 'typedi';
const temporaryDir = mkdtempSync(join(tmpdir(), 'n8n'));
describe('NodeExecuteFunctions', () => {
describe('test binary data helper methods', () => {
// Reset BinaryDataManager for each run. This is a dirty operation, as individual managers are not cleaned.
beforeEach(() => {
BinaryDataManager.instance = undefined;
});
test("test getBinaryDataBuffer(...) & setBinaryDataBuffer(...) methods in 'default' mode", async () => {
// Setup a 'default' binary data manager instance
await BinaryDataManager.init({
Container.set(BinaryDataService, new BinaryDataService());
await Container.get(BinaryDataService).init({
mode: 'default',
availableModes: 'default',
availableModes: ['default'],
localStoragePath: temporaryDir,
});
@@ -80,10 +78,12 @@ describe('NodeExecuteFunctions', () => {
});
test("test getBinaryDataBuffer(...) & setBinaryDataBuffer(...) methods in 'filesystem' mode", async () => {
Container.set(BinaryDataService, new BinaryDataService());
// Setup a 'filesystem' binary data manager instance
await BinaryDataManager.init({
await Container.get(BinaryDataService).init({
mode: 'filesystem',
availableModes: 'filesystem',
availableModes: ['filesystem'],
localStoragePath: temporaryDir,
});