Introduce binary data management (#2059)

* introduce binary data management

* merge fixes

* fixes

* init binary data manager for other modes

* improve binary manager

* improve binary manager

* delete binary data on executions delete

* lazy delete non-saved executions binary data

* merge fixes + error handing

* improve structure

* leftovers and cleanups

* formatting

* fix config description

* fixes

* fix races

* duplicate binary data for execute workflow node

* clean up and cr

* update mode name, add binary mode to diagnostics

* update mode name, add prefix to filename

* update filename

* allow multiple modes, backward compatibility

* improve file and id naming

* use execution id for binary data storage

* delete binary data by execution id

* add meta for persisted binary data

* delete marked persisted files

* mark deletion by executionid

* add env var for persisted binary data ttl

* improvements

* lint fix

* fix env var description

* cleanup

* cleanup

*  Minor improvements

Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
This commit is contained in:
Ahsan Virani
2021-12-23 22:29:04 +01:00
committed by GitHub
parent 416e15cdb6
commit 1e42effc3a
22 changed files with 743 additions and 40 deletions

View File

@@ -16,7 +16,7 @@ import * as express from 'express';
// eslint-disable-next-line import/no-extraneous-dependencies
import { get } from 'lodash';
import { BINARY_ENCODING, NodeExecuteFunctions } from 'n8n-core';
import { BINARY_ENCODING, BinaryDataManager, NodeExecuteFunctions } from 'n8n-core';
import {
createDeferredPromise,
@@ -37,6 +37,7 @@ import {
Workflow,
WorkflowExecuteMode,
} from 'n8n-workflow';
// eslint-disable-next-line import/no-cycle
import {
GenericHelpers,
@@ -447,7 +448,7 @@ export async function executeWebhook(
IExecutionDb | undefined
>;
executePromise
.then((data) => {
.then(async (data) => {
if (data === undefined) {
if (!didSendResponse) {
responseCallback(null, {
@@ -611,7 +612,10 @@ export async function executeWebhook(
if (!didSendResponse) {
// Send the webhook response manually
res.setHeader('Content-Type', binaryData.mimeType);
res.end(Buffer.from(binaryData.data, BINARY_ENCODING));
const binaryDataBuffer = await BinaryDataManager.getInstance().retrieveBinaryData(
binaryData,
);
res.end(binaryDataBuffer);
responseCallback(null, {
noWebhookResponse: true,