refactor(Google Drive Node): Use node streams for uploading and downloading files (#5017)

* use streams to upload files to google drive

* use streams to download files from google drive

* use resumable uploads api for google drive

* avoid dangling promises, and reduce memory usage in error logging

packages/nodes-base/nodes/Google/Drive/GoogleDrive.node.ts

@@ -1,4 +1,4 @@
-import { IExecuteFunctions } from 'n8n-core';
+import { BINARY_ENCODING, IExecuteFunctions } from 'n8n-core';
 
 import {
 	IDataObject,
@@ -13,6 +13,9 @@ import {
 import { googleApiRequest, googleApiRequestAllItems } from './GenericFunctions';
 
 import { v4 as uuid } from 'uuid';
+import type { Readable } from 'stream';
+
+const UPLOAD_CHUNK_SIZE = 256 * 1024;
 
 interface GoogleDriveFilesItem {
 	id: string;
@@ -2306,6 +2309,7 @@ export class GoogleDrive implements INodeType {
 						const downloadOptions = this.getNodeParameter('options', i);
 
 						const requestOptions = {
+							useStream: true,
 							resolveWithFullResponse: true,
 							encoding: null,
 							json: false,
@@ -2316,7 +2320,7 @@ export class GoogleDrive implements INodeType {
 							'GET',
 							`/drive/v3/files/${fileId}`,
 							{},
-							{ fields: 'mimeType', supportsTeamDrives: true },
+							{ fields: 'mimeType,name', supportsTeamDrives: true },
 						);
 						let response;
 
@@ -2370,15 +2374,8 @@ export class GoogleDrive implements INodeType {
 							);
 						}
 
-						let mimeType: string | undefined;
-						let fileName: string | undefined = undefined;
-						if (response.headers['content-type']) {
-							mimeType = response.headers['content-type'];
-						}
-
-						if (downloadOptions.fileName) {
-							fileName = downloadOptions.fileName as string;
-						}
+						const mimeType = file.mimeType ?? response.headers['content-type'] ?? undefined;
+						const fileName = downloadOptions.fileName ?? file.name ?? undefined;
 
 						const newItem: INodeExecutionData = {
 							json: items[i].json,
@@ -2400,10 +2397,8 @@ export class GoogleDrive implements INodeType {
 							i,
 						) as string;
 
-						const data = Buffer.from(response.body as string);
-
 						items[i].binary![dataPropertyNameDownload] = await this.helpers.prepareBinaryData(
-							data as unknown as Buffer,
+							response.body as unknown as Readable,
 							fileName,
 							mimeType,
 						);
@@ -2511,9 +2506,11 @@ export class GoogleDrive implements INodeType {
 						// ----------------------------------
 						const resolveData = this.getNodeParameter('resolveData', 0);
 
-						let mimeType = 'text/plain';
-						let body;
+						let contentLength: number;
+						let fileContent: Buffer | Readable;
 						let originalFilename: string | undefined;
+						let mimeType = 'text/plain';
+
 						if (this.getNodeParameter('binaryData', i)) {
 							// Is binary file to upload
 							const item = items[i];
@@ -2526,7 +2523,8 @@ export class GoogleDrive implements INodeType {
 
 							const propertyNameUpload = this.getNodeParameter('binaryPropertyName', i) as string;
 
-							if (item.binary[propertyNameUpload] === undefined) {
+							const binary = item.binary[propertyNameUpload];
+							if (binary === undefined) {
 								throw new NodeOperationError(
 									this.getNode(),
 									`No binary data property "${propertyNameUpload}" does not exists on item!`,
@@ -2534,48 +2532,86 @@ export class GoogleDrive implements INodeType {
 								);
 							}
 
-							if (item.binary[propertyNameUpload].mimeType) {
-								mimeType = item.binary[propertyNameUpload].mimeType;
+							if (binary.id) {
+								// Stream data in 256KB chunks, and upload the via the resumable upload api
+								fileContent = this.helpers.getBinaryStream(binary.id, UPLOAD_CHUNK_SIZE);
+								const metadata = await this.helpers.getBinaryMetadata(binary.id);
+								contentLength = metadata.fileSize;
+								originalFilename = metadata.fileName;
+								if (metadata.mimeType) mimeType = binary.mimeType;
+							} else {
+								fileContent = Buffer.from(binary.data, BINARY_ENCODING);
+								contentLength = fileContent.length;
+								originalFilename = binary.fileName;
+								mimeType = binary.mimeType;
 							}
-
-							if (item.binary[propertyNameUpload].fileName) {
-								originalFilename = item.binary[propertyNameUpload].fileName;
-							}
-
-							body = await this.helpers.getBinaryDataBuffer(i, propertyNameUpload);
 						} else {
 							// Is text file
-							body = Buffer.from(this.getNodeParameter('fileContent', i) as string, 'utf8');
+							fileContent = Buffer.from(this.getNodeParameter('fileContent', i) as string, 'utf8');
+							contentLength = fileContent.byteLength;
 						}
 
 						const name = this.getNodeParameter('name', i) as string;
 						const parents = this.getNodeParameter('parents', i) as string[];
 
-						let qs: IDataObject = {
-							fields: queryFields,
-							uploadType: 'media',
-						};
+						let uploadId;
+						if (Buffer.isBuffer(fileContent)) {
+							const response = await googleApiRequest.call(
+								this,
+								'POST',
+								'/upload/drive/v3/files',
+								fileContent,
+								{
+									fields: queryFields,
+									uploadType: 'media',
+								},
+								undefined,
+								{
+									headers: {
+										'Content-Type': mimeType,
+										'Content-Length': contentLength,
+									},
+									encoding: null,
+									json: false,
+								},
+							);
+							uploadId = JSON.parse(response).id;
+						} else {
+							const resumableUpload = await googleApiRequest.call(
+								this,
+								'POST',
+								'/upload/drive/v3/files',
+								undefined,
+								{ uploadType: 'resumable' },
+								undefined,
+								{
+									resolveWithFullResponse: true,
+								},
+							);
+							const uploadUrl = resumableUpload.headers.location;
 
-						const requestOptions = {
-							headers: {
-								'Content-Type': mimeType,
-								'Content-Length': body.byteLength,
-							},
-							encoding: null,
-							json: false,
-						};
+							let offset = 0;
+							for await (const chunk of fileContent) {
+								const nextOffset = offset + chunk.length;
+								try {
+									const response = await this.helpers.httpRequest({
+										method: 'PUT',
+										url: uploadUrl,
+										headers: {
+											'Content-Length': chunk.length,
+											'Content-Range': `bytes ${offset}-${nextOffset - 1}/${contentLength}`,
+										},
+										body: chunk,
+									});
+									uploadId = response.id;
+								} catch (error) {
+									if (error.response?.status !== 308) throw error;
+								}
+								offset = nextOffset;
+							}
+						}
 
-						let response = await googleApiRequest.call(
-							this,
-							'POST',
-							'/upload/drive/v3/files',
-							body,
-							qs,
-							undefined,
-							requestOptions,
-						);
-
-						body = {
+						const requestBody = {
 							mimeType,
 							name,
 							originalFilename,
@@ -2588,7 +2624,7 @@ export class GoogleDrive implements INodeType {
 						) as IDataObject[];
 
 						if (properties.length) {
-							Object.assign(body, {
+							Object.assign(requestBody, {
 								properties: properties.reduce(
 									(obj, value) => Object.assign(obj, { [`${value.key}`]: value.value }),
 									{},
@@ -2603,7 +2639,7 @@ export class GoogleDrive implements INodeType {
 						) as IDataObject[];
 
 						if (properties.length) {
-							Object.assign(body, {
+							Object.assign(requestBody, {
 								appProperties: appProperties.reduce(
 									(obj, value) => Object.assign(obj, { [`${value.key}`]: value.value }),
 									{},
@@ -2611,18 +2647,16 @@ export class GoogleDrive implements INodeType {
 							});
 						}
 
-						qs = {
-							addParents: parents.join(','),
-							// When set to true shared drives can be used.
-							supportsAllDrives: true,
-						};
-
-						response = await googleApiRequest.call(
+						let response = await googleApiRequest.call(
 							this,
 							'PATCH',
-							`/drive/v3/files/${JSON.parse(response).id}`,
-							body,
-							qs,
+							`/drive/v3/files/${uploadId}`,
+							requestBody,
+							{
+								addParents: parents.join(','),
+								// When set to true shared drives can be used.
+								supportsAllDrives: true,
+							},
 						);
 
 						if (resolveData) {