feat(AwsS3 Node): Small overhaul of the node with multipart uploading (#6017)

* Create new version for S3 * Update S3 to new aws s3 methods * Switch from SAOP to Rest api * Add multipart request * Seperate stream into chunks and send the multipart * Fix chunk into buffer * Fix wrong sha256 mismatch * Add abort multipart on error * Complete multipart and list parts * Change format to xml and add a minmum size of 5MB for each part * Fix returned data for uploading a file * Remove console.logs * Seperate needed headers and multipart headers * Throw error on aborting, remove console.logs * Remove soap request from generic function * Keep buffer * Add unit test for V2 * fix upload file content body * removed unused import * Fix bug where the object was too smal and used only one part * Fix naming for bucket name * Fix issue with file name not returning data * Add parent name * Remove console.logs * Add content type * fix headears for other upload mode --------- Co-authored-by: Marcus <marcus@n8n.io>
2025-12-18 02:21:13 +00:00 · 2023-06-15 13:19:22 +02:00
parent da105f468b
commit 109442f38f
17 changed files with 3682 additions and 911 deletions
--- a/packages/nodes-base/nodes/Aws/S3/V2/GenericFunctions.ts
+++ b/packages/nodes-base/nodes/Aws/S3/V2/GenericFunctions.ts
@@ -0,0 +1,132 @@
+import get from 'lodash.get';
+
+import { parseString } from 'xml2js';
+
+import type {
+	IDataObject,
+	IExecuteFunctions,
+	IHookFunctions,
+	ILoadOptionsFunctions,
+	IWebhookFunctions,
+	IHttpRequestOptions,
+} from 'n8n-workflow';
+
+export async function awsApiRequest(
+	this: IHookFunctions | IExecuteFunctions | ILoadOptionsFunctions | IWebhookFunctions,
+	service: string,
+	method: string,
+	path: string,
+	body?: string | Buffer | any,
+	query: IDataObject = {},
+	headers?: object,
+	option: IDataObject = {},
+	_region?: string,
+): Promise<any> {
+	const requestOptions = {
+		qs: {
+			...query,
+			service,
+			path,
+			query,
+		},
+		method,
+		body,
+		url: '',
+		headers,
+	} as IHttpRequestOptions;
+	if (Object.keys(option).length !== 0) {
+		Object.assign(requestOptions, option);
+	}
+	return this.helpers.requestWithAuthentication.call(this, 'aws', requestOptions);
+}
+
+export async function awsApiRequestREST(
+	this: IHookFunctions | IExecuteFunctions | ILoadOptionsFunctions,
+	service: string,
+	method: string,
+	path: string,
+	body?: string | Buffer | any,
+	query: IDataObject = {},
+	headers?: object,
+	options: IDataObject = {},
+	region?: string,
+): Promise<any> {
+	const response = await awsApiRequest.call(
+		this,
+		service,
+		method,
+		path,
+		body,
+		query,
+		headers,
+		options,
+		region,
+	);
+	try {
+		if (response.includes('<?xml version="1.0" encoding="UTF-8"?>')) {
+			return await new Promise((resolve, reject) => {
+				parseString(response as string, { explicitArray: false }, (err, data) => {
+					if (err) {
+						return reject(err);
+					}
+					resolve(data);
+				});
+			});
+		}
+		return JSON.parse(response as string);
+	} catch (error) {
+		return response;
+	}
+}
+
+export async function awsApiRequestRESTAllItems(
+	this: IHookFunctions | IExecuteFunctions | ILoadOptionsFunctions,
+	propertyName: string,
+	service: string,
+	method: string,
+	path: string,
+	body?: string,
+	query: IDataObject = {},
+	headers?: object,
+	option: IDataObject = {},
+	region?: string,
+): Promise<any> {
+	const returnData: IDataObject[] = [];
+
+	let responseData;
+	do {
+		responseData = await awsApiRequestREST.call(
+			this,
+			service,
+			method,
+			path,
+			body,
+			query,
+			headers,
+			option,
+			region,
+		);
+		//https://forums.aws.amazon.com/thread.jspa?threadID=55746
+		if (get(responseData, `${propertyName.split('.')[0]}.NextContinuationToken`)) {
+			query['continuation-token'] = get(
+				responseData,
+				`${propertyName.split('.')[0]}.NextContinuationToken`,
+			);
+		}
+		if (get(responseData, propertyName)) {
+			if (Array.isArray(get(responseData, propertyName))) {
+				returnData.push.apply(returnData, get(responseData, propertyName) as IDataObject[]);
+			} else {
+				returnData.push(get(responseData, propertyName) as IDataObject);
+			}
+		}
+		const limit = query.limit as number | undefined;
+		if (limit && limit <= returnData.length) {
+			return returnData;
+		}
+	} while (
+		get(responseData, `${propertyName.split('.')[0]}.IsTruncated`) !== undefined &&
+		get(responseData, `${propertyName.split('.')[0]}.IsTruncated`) !== 'false'
+	);
+	return returnData;
+}