refactor(core): Generalize binary data manager interface (no-changelog) (#7164)

Depends on: #7092 | Story:
[PAY-768](https://linear.app/n8n/issue/PAY-768)

This PR: 
- Generalizes the `IBinaryDataManager` interface.
- Adjusts `Filesystem.ts` to satisfy the interface.
- Sets up an S3 client stub to be filled in in the next PR.
- Turns `BinaryDataManager` into an injectable service.
- Adjusts the config schema and adds new validators.

Note that the PR looks large but all the main changes are in
`packages/core/src/binaryData`.

Out of scope:
- `BinaryDataManager` (now `BinaryDataService`) and `Filesystem.ts` (now
`fs.client.ts`) were slightly refactored for maintainability, but fully
overhauling them is **not** the focus of this PR, which is meant to
clear the way for the S3 implementation. Future improvements for these
two should include setting up a backwards-compatible dir structure that
makes it easier to locate binary data files to delete, removing
duplication, simplifying cloning methods, using integers for binary data
size instead of `prettyBytes()`, writing tests for existing binary data
logic, etc.

---------

Co-authored-by: कारतोफ्फेलस्क्रिप्ट™ <aditya@netroy.in>
This commit is contained in:
Iván Ovejero
2023-09-22 17:22:12 +02:00
committed by GitHub
parent 4614e1e1c9
commit 6d6e2488c6
46 changed files with 602 additions and 552 deletions

View File

@@ -2,28 +2,21 @@ import path from 'path';
import convict from 'convict';
import { UserSettings } from 'n8n-core';
import { jsonParse } from 'n8n-workflow';
import { ensureStringArray } from './utils';
convict.addFormat({
name: 'nodes-list',
// @ts-ignore
validate(values: string[], { env }: { env: string }): void {
try {
if (!Array.isArray(values)) {
throw new Error();
}
name: 'json-string-array',
coerce: (rawStr: string) =>
jsonParse<string[]>(rawStr, {
errorMessage: `Expected this value "${rawStr}" to be valid JSON`,
}),
validate: ensureStringArray,
});
for (const value of values) {
if (typeof value !== 'string') {
throw new Error();
}
}
} catch (error) {
throw new TypeError(`${env} is not a valid Array of strings.`);
}
},
coerce(rawValue: string): string[] {
return jsonParse(rawValue, { errorMessage: 'nodes-list needs to be valid JSON' });
},
convict.addFormat({
name: 'comma-separated-list',
coerce: (rawStr: string) => rawStr.split(','),
validate: ensureStringArray,
});
export const schema = {
@@ -788,13 +781,13 @@ export const schema = {
nodes: {
include: {
doc: 'Nodes to load',
format: 'nodes-list',
format: 'json-string-array',
default: undefined,
env: 'NODES_INCLUDE',
},
exclude: {
doc: 'Nodes not to load',
format: 'nodes-list',
format: 'json-string-array',
default: undefined,
env: 'NODES_EXCLUDE',
},
@@ -902,7 +895,7 @@ export const schema = {
binaryDataManager: {
availableModes: {
format: String,
format: 'comma-separated-list',
default: 'filesystem',
env: 'N8N_AVAILABLE_BINARY_DATA_MODES',
doc: 'Available modes of binary data storage, as comma separated strings',