admin/n8n-enterprise-unlocked
1
0
Fork 0
mirror of https://github.com/Abdulazizzn/n8n-enterprise-unlocked.git synced 2025-12-17 10:02:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

✨ Introduce binary data management (#2059)

Browse Source 
* introduce binary data management

* merge fixes

* fixes

* init binary data manager for other modes

* improve binary manager

* improve binary manager

* delete binary data on executions delete

* lazy delete non-saved executions binary data

* merge fixes + error handing

* improve structure

* leftovers and cleanups

* formatting

* fix config description

* fixes

* fix races

* duplicate binary data for execute workflow node

* clean up and cr

* update mode name, add binary mode to diagnostics

* update mode name, add prefix to filename

* update filename

* allow multiple modes, backward compatibility

* improve file and id naming

* use execution id for binary data storage

* delete binary data by execution id

* add meta for persisted binary data

* delete marked persisted files

* mark deletion by executionid

* add env var for persisted binary data ttl

* improvements

* lint fix

* fix env var description

* cleanup

* cleanup

* ⚡ Minor improvements

Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
This commit is contained in:
Ahsan Virani
2021-12-23 22:29:04 +01:00
committed by GitHub
parent 416e15cdb6
commit 1e42effc3a
22 changed files with 743 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

92
packages/core/src/NodeExecuteFunctions.ts
View File

@@ -73,9 +73,9 @@ import { lookup } from 'mime-types';
import axios, { AxiosProxyConfig, AxiosRequestConfig, Method } from 'axios';
import { URL, URLSearchParams } from 'url';
import { BinaryDataManager } from './BinaryDataManager';
// eslint-disable-next-line import/no-cycle
import {
BINARY_ENCODING,
ICredentialTestFunctions,
IHookFunctions,
ILoadOptionsFunctions,
@@ -682,7 +682,7 @@ export async function getBinaryDataBuffer(
inputIndex: number,
): Promise<Buffer> {
const binaryData = inputData.main![inputIndex]![itemIndex]!.binary![propertyName]!;
return Buffer.from(binaryData.data, BINARY_ENCODING);
return BinaryDataManager.getInstance().retrieveBinaryData(binaryData);
}
/**
@@ -697,6 +697,7 @@ export async function getBinaryDataBuffer(
*/
export async function prepareBinaryData(
binaryData: Buffer,
executionId: string,
filePath?: string,
mimeType?: string,
): Promise<IBinaryData> {
@@ -727,10 +728,7 @@ export async function prepareBinaryData(
const returnData: IBinaryData = {
mimeType,
// TODO: Should program it in a way that it does not have to converted to base64
// It should only convert to and from base64 when saved in database because
// of for example an error or when there is a wait node.
data: binaryData.toString(BINARY_ENCODING),
data: '',
};
if (filePath) {
@@ -753,7 +751,7 @@ export async function prepareBinaryData(
}
}
return returnData;
return BinaryDataManager.getInstance().storeBinaryData(returnData, binaryData, executionId);
}
/**
@@ -1370,7 +1368,19 @@ export function getExecutePollFunctions(
},
helpers: {
httpRequest,
prepareBinaryData,
async prepareBinaryData(
binaryData: Buffer,
filePath?: string,
mimeType?: string,
): Promise<IBinaryData> {
return prepareBinaryData.call(
this,
binaryData,
additionalData.executionId!,
filePath,
mimeType,
);
},
request: proxyRequestToAxios,
async requestOAuth2(
this: IAllExecuteFunctions,
@@ -1476,8 +1486,19 @@ export function getExecuteTriggerFunctions(
},
helpers: {
httpRequest,
prepareBinaryData,
async prepareBinaryData(
binaryData: Buffer,
filePath?: string,
mimeType?: string,
): Promise<IBinaryData> {
return prepareBinaryData.call(
this,
binaryData,
additionalData.executionId!,
filePath,
mimeType,
);
},
request: proxyRequestToAxios,
async requestOAuth2(
this: IAllExecuteFunctions,
@@ -1553,7 +1574,14 @@ export function getExecuteFunctions(
workflowInfo: IExecuteWorkflowInfo,
inputData?: INodeExecutionData[],
): Promise<any> {
return additionalData.executeWorkflow(workflowInfo, additionalData, inputData);
return additionalData
.executeWorkflow(workflowInfo, additionalData, inputData)
.then(async (result) =>
BinaryDataManager.getInstance().duplicateBinaryData(
result,
additionalData.executionId!,
),
);
},
getContext(type: string): IContextObject {
return NodeHelpers.getContext(runExecutionData, type, node);
@@ -1672,7 +1700,19 @@ export function getExecuteFunctions(
},
helpers: {
httpRequest,
prepareBinaryData,
async prepareBinaryData(
binaryData: Buffer,
filePath?: string,
mimeType?: string,
): Promise<IBinaryData> {
return prepareBinaryData.call(
this,
binaryData,
additionalData.executionId!,
filePath,
mimeType,
);
},
async getBinaryDataBuffer(
itemIndex: number,
propertyName: string,
@@ -1853,7 +1893,19 @@ export function getExecuteSingleFunctions(
},
helpers: {
httpRequest,
prepareBinaryData,
async prepareBinaryData(
binaryData: Buffer,
filePath?: string,
mimeType?: string,
): Promise<IBinaryData> {
return prepareBinaryData.call(
this,
binaryData,
additionalData.executionId!,
filePath,
mimeType,
);
},
request: proxyRequestToAxios,
async requestOAuth2(
this: IAllExecuteFunctions,
@@ -2234,7 +2286,19 @@ export function getExecuteWebhookFunctions(
prepareOutputData: NodeHelpers.prepareOutputData,
helpers: {
httpRequest,
prepareBinaryData,
async prepareBinaryData(
binaryData: Buffer,
filePath?: string,
mimeType?: string,
): Promise<IBinaryData> {
return prepareBinaryData.call(
this,
binaryData,
additionalData.executionId!,
filePath,
mimeType,
);
},
request: proxyRequestToAxios,
async requestOAuth2(
this: IAllExecuteFunctions,