admin/n8n-enterprise-unlocked
1
0
Fork 0
mirror of https://github.com/Abdulazizzn/n8n-enterprise-unlocked.git synced 2025-12-18 02:21:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(core)!: Make getBinaryStream async (#7247)

Browse Source 
Story: [PAY-846](https://linear.app/n8n/issue/PAY-846) | Related:
https://github.com/n8n-io/n8n/pull/7225

For the S3 backend for external storage of binary data and execution
data, the `getAsStream` method in the binary data manager interface used
by FS and S3 will need to become async. This is a breaking change for
nodes-base.
This commit is contained in:
Iván Ovejero
2023-09-25 16:59:45 +02:00
committed by GitHub
parent 484035eb51
commit 75541e91f2
23 changed files with 74 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
packages/nodes-base/nodes/Aws/S3/V2/AwsS3V2.node.ts
View File

@@ -870,7 +870,10 @@ export class AwsS3V2 implements INodeType {
let uploadData: Buffer | Readable;
multipartHeaders['Content-Type'] = binaryPropertyData.mimeType;
if (binaryPropertyData.id) {
uploadData = this.helpers.getBinaryStream(binaryPropertyData.id, UPLOAD_CHUNK_SIZE);
uploadData = await this.helpers.getBinaryStream(
binaryPropertyData.id,
UPLOAD_CHUNK_SIZE,
);
const createMultiPartUpload = await awsApiRequestREST.call(
this,
servicePath,

2
packages/nodes-base/nodes/Crypto/Crypto.node.ts
View File

@@ -486,7 +486,7 @@ export class Crypto implements INodeType {
const binaryPropertyName = this.getNodeParameter('binaryPropertyName', i);
const binaryData = this.helpers.assertBinaryData(i, binaryPropertyName);
if (binaryData.id) {
const binaryStream = this.helpers.getBinaryStream(binaryData.id);
const binaryStream = await this.helpers.getBinaryStream(binaryData.id);
hashOrHmac.setEncoding(encoding);
await pipeline(binaryStream, hashOrHmac);
newValue = hashOrHmac.read();

4
packages/nodes-base/nodes/Ftp/Ftp.node.ts
View File

@@ -649,7 +649,7 @@ export class Ftp implements INodeType {
let uploadData: Buffer | Readable;
if (binaryData.id) {
uploadData = this.helpers.getBinaryStream(binaryData.id);
uploadData = await this.helpers.getBinaryStream(binaryData.id);
} else {
uploadData = Buffer.from(binaryData.data, BINARY_ENCODING);
}
@@ -759,7 +759,7 @@ export class Ftp implements INodeType {
let uploadData: Buffer | Readable;
if (binaryData.id) {
uploadData = this.helpers.getBinaryStream(binaryData.id);
uploadData = await this.helpers.getBinaryStream(binaryData.id);
} else {
uploadData = Buffer.from(binaryData.data, BINARY_ENCODING);
}

2
packages/nodes-base/nodes/Google/CloudStorage/ObjectDescription.ts
View File

@@ -160,7 +160,7 @@ export const objectOperations: INodeProperties[] = [
const binaryData = this.helpers.assertBinaryData(binaryPropertyName);
if (binaryData.id) {
content = this.helpers.getBinaryStream(binaryData.id);
content = await this.helpers.getBinaryStream(binaryData.id);
const binaryMetadata = await this.helpers.getBinaryMetadata(binaryData.id);
contentType = binaryMetadata.mimeType ?? 'application/octet-stream';
contentLength = binaryMetadata.fileSize;

2
packages/nodes-base/nodes/Google/Drive/v1/GoogleDriveV1.node.ts
View File

@@ -2442,7 +2442,7 @@ export class GoogleDriveV1 implements INodeType {
const binaryData = this.helpers.assertBinaryData(i, binaryPropertyName);
if (binaryData.id) {
// Stream data in 256KB chunks, and upload the via the resumable upload api
fileContent = this.helpers.getBinaryStream(binaryData.id, UPLOAD_CHUNK_SIZE);
fileContent = await this.helpers.getBinaryStream(binaryData.id, UPLOAD_CHUNK_SIZE);
const metadata = await this.helpers.getBinaryMetadata(binaryData.id);
contentLength = metadata.fileSize;
originalFilename = metadata.fileName;

2
packages/nodes-base/nodes/Google/Drive/v2/helpers/utils.ts
View File

@@ -40,7 +40,7 @@ export async function getItemBinaryData(
if (binaryData.id) {
// Stream data in 256KB chunks, and upload the via the resumable upload api
fileContent = this.helpers.getBinaryStream(binaryData.id, chunkSize);
fileContent = await this.helpers.getBinaryStream(binaryData.id, chunkSize);
const metadata = await this.helpers.getBinaryMetadata(binaryData.id);
contentLength = metadata.fileSize;
originalFilename = metadata.fileName;

2
packages/nodes-base/nodes/Google/YouTube/YouTube.node.ts
View File

@@ -838,7 +838,7 @@ export class YouTube implements INodeType {
if (binaryData.id) {
// Stream data in 256KB chunks, and upload the via the resumable upload api
fileContent = this.helpers.getBinaryStream(binaryData.id, UPLOAD_CHUNK_SIZE);
fileContent = await this.helpers.getBinaryStream(binaryData.id, UPLOAD_CHUNK_SIZE);
const metadata = await this.helpers.getBinaryMetadata(binaryData.id);
contentLength = metadata.fileSize;
mimeType = metadata.mimeType ?? binaryData.mimeType;

26
packages/nodes-base/nodes/HttpRequest/GenericFunctions.ts
View File

@@ -138,21 +138,31 @@ export const binaryContentTypes = [
export type BodyParametersReducer = (
acc: IDataObject,
cur: { name: string; value: string },
) => IDataObject;
) => Promise<IDataObject>;
export const prepareRequestBody = (
export async function reduceAsync<T, R>(
arr: T[],
reducer: (acc: Awaited<Promise<R>>, cur: T) => Promise<R>,
init: Promise<R> = Promise.resolve({} as R),
): Promise<R> {
return arr.reduce(async (promiseAcc, item) => {
return reducer(await promiseAcc, item);
}, init);
}
export const prepareRequestBody = async (
parameters: BodyParameter[],
bodyType: string,
version: number,
defaultReducer: BodyParametersReducer,
) => {
if (bodyType === 'json' && version >= 4) {
return parameters.reduce((acc, entry) => {
const value = entry.value;
set(acc, entry.name, value);
return acc;
}, {} as IDataObject);
return parameters.reduce(async (acc, entry) => {
const result = await acc;
set(result, entry.name, entry.value);
return result;
}, Promise.resolve({}));
} else {
return parameters.reduce(defaultReducer, {});
return reduceAsync(parameters, defaultReducer);
}
};

13
packages/nodes-base/nodes/HttpRequest/V3/HttpRequestV3.node.ts
View File

@@ -31,6 +31,7 @@ import {
binaryContentTypes,
getOAuth2AdditionalParameters,
prepareRequestBody,
reduceAsync,
replaceNullValues,
sanitizeUiMessage,
} from '../GenericFunctions';
@@ -1161,7 +1162,7 @@ export class HttpRequestV3 implements INodeType {
});
}
const parametersToKeyValue = (
const parametersToKeyValue = async (
accumulator: { [key: string]: any },
cur: { name: string; value: string; parameterType?: string; inputDataFieldName?: string },
) => {
@@ -1171,7 +1172,7 @@ export class HttpRequestV3 implements INodeType {
let uploadData: Buffer | Readable;
const itemBinaryData = items[itemIndex].binary![cur.inputDataFieldName];
if (itemBinaryData.id) {
uploadData = this.helpers.getBinaryStream(itemBinaryData.id);
uploadData = await this.helpers.getBinaryStream(itemBinaryData.id);
} else {
uploadData = Buffer.from(itemBinaryData.data, BINARY_ENCODING);
}
@@ -1192,7 +1193,7 @@ export class HttpRequestV3 implements INodeType {
// Get parameters defined in the UI
if (sendBody && bodyParameters) {
if (specifyBody === 'keypair' || bodyContentType === 'multipart-form-data') {
requestOptions.body = prepareRequestBody(
requestOptions.body = await prepareRequestBody(
bodyParameters,
bodyContentType,
nodeVersion,
@@ -1243,7 +1244,7 @@ export class HttpRequestV3 implements INodeType {
const itemBinaryData = this.helpers.assertBinaryData(itemIndex, inputDataFieldName);
if (itemBinaryData.id) {
uploadData = this.helpers.getBinaryStream(itemBinaryData.id);
uploadData = await this.helpers.getBinaryStream(itemBinaryData.id);
const metadata = await this.helpers.getBinaryMetadata(itemBinaryData.id);
contentLength = metadata.fileSize;
} else {
@@ -1264,7 +1265,7 @@ export class HttpRequestV3 implements INodeType {
// Get parameters defined in the UI
if (sendQuery && queryParameters) {
if (specifyQuery === 'keypair') {
requestOptions.qs = queryParameters.reduce(parametersToKeyValue, {});
requestOptions.qs = await reduceAsync(queryParameters, parametersToKeyValue);
} else if (specifyQuery === 'json') {
// query is specified using JSON
try {
@@ -1287,7 +1288,7 @@ export class HttpRequestV3 implements INodeType {
if (sendHeaders && headerParameters) {
let additionalHeaders: IDataObject = {};
if (specifyHeaders === 'keypair') {
additionalHeaders = headerParameters.reduce(parametersToKeyValue, {});
additionalHeaders = await reduceAsync(headerParameters, parametersToKeyValue);
} else if (specifyHeaders === 'json') {
// body is specified using JSON
try {

12
packages/nodes-base/nodes/HttpRequest/test/utils/utils.test.ts
View File

@@ -2,7 +2,7 @@ import { prepareRequestBody } from '../../GenericFunctions';
import type { BodyParameter, BodyParametersReducer } from '../../GenericFunctions';
describe('HTTP Node Utils, prepareRequestBody', () => {
it('should call default reducer', () => {
it('should call default reducer', async () => {
const bodyParameters: BodyParameter[] = [
{
name: 'foo.bar',
@@ -11,15 +11,13 @@ describe('HTTP Node Utils, prepareRequestBody', () => {
];
const defaultReducer: BodyParametersReducer = jest.fn();
prepareRequestBody(bodyParameters, 'json', 3, defaultReducer);
await prepareRequestBody(bodyParameters, 'json', 3, defaultReducer);
expect(defaultReducer).toBeCalledTimes(1);
expect(defaultReducer).toBeCalledWith({}, { name: 'foo.bar', value: 'baz' }, 0, [
{ name: 'foo.bar', value: 'baz' },
]);
expect(defaultReducer).toBeCalledWith({}, { name: 'foo.bar', value: 'baz' });
});
it('should call process dot notations', () => {
it('should call process dot notations', async () => {
const bodyParameters: BodyParameter[] = [
{
name: 'foo.bar.spam',
@@ -28,7 +26,7 @@ describe('HTTP Node Utils, prepareRequestBody', () => {
];
const defaultReducer: BodyParametersReducer = jest.fn();
const result = prepareRequestBody(bodyParameters, 'json', 4, defaultReducer);
const result = await prepareRequestBody(bodyParameters, 'json', 4, defaultReducer);
expect(defaultReducer).toBeCalledTimes(0);
expect(result).toBeDefined();

2
packages/nodes-base/nodes/Jira/Jira.node.ts
View File

@@ -1058,7 +1058,7 @@ export class Jira implements INodeType {
let uploadData: Buffer | Readable;
if (binaryData.id) {
uploadData = this.helpers.getBinaryStream(binaryData.id);
uploadData = await this.helpers.getBinaryStream(binaryData.id);
} else {
uploadData = Buffer.from(binaryData.data, BINARY_ENCODING);
}

2
packages/nodes-base/nodes/Slack/V2/SlackV2.node.ts
View File

@@ -1059,7 +1059,7 @@ export class SlackV2 implements INodeType {
let uploadData: Buffer | Readable;
if (binaryData.id) {
uploadData = this.helpers.getBinaryStream(binaryData.id);
uploadData = await this.helpers.getBinaryStream(binaryData.id);
} else {
uploadData = Buffer.from(binaryData.data, BINARY_ENCODING);
}

2
packages/nodes-base/nodes/SpreadsheetFile/v2/SpreadsheetFileV2.node.ts
View File

@@ -92,7 +92,7 @@ export class SpreadsheetFileV2 implements INodeType {
},
});
if (binaryData.id) {
const stream = this.helpers.getBinaryStream(binaryData.id);
const stream = await this.helpers.getBinaryStream(binaryData.id);
await pipeline(stream, parser);
} else {
parser.write(binaryData.data, BINARY_ENCODING);

2
packages/nodes-base/nodes/Ssh/Ssh.node.ts
View File

@@ -441,7 +441,7 @@ export class Ssh implements INodeType {
let uploadData: Buffer | Readable;
if (binaryData.id) {
uploadData = this.helpers.getBinaryStream(binaryData.id);
uploadData = await this.helpers.getBinaryStream(binaryData.id);
} else {
uploadData = Buffer.from(binaryData.data, BINARY_ENCODING);
}

2
packages/nodes-base/nodes/Telegram/Telegram.node.ts
View File

@@ -2005,7 +2005,7 @@ export class Telegram implements INodeType {
let uploadData: Buffer | Readable;
if (itemBinaryData.id) {
uploadData = this.helpers.getBinaryStream(itemBinaryData.id);
uploadData = await this.helpers.getBinaryStream(itemBinaryData.id);
} else {
uploadData = Buffer.from(itemBinaryData.data, BINARY_ENCODING);
}

2
packages/nodes-base/nodes/WriteBinaryFile/WriteBinaryFile.node.ts
View File

@@ -91,7 +91,7 @@ export class WriteBinaryFile implements INodeType {
let fileContent: Buffer | Readable;
if (binaryData.id) {
fileContent = this.helpers.getBinaryStream(binaryData.id);
fileContent = await this.helpers.getBinaryStream(binaryData.id);
} else {
fileContent = Buffer.from(binaryData.data, BINARY_ENCODING);
}