refactor(Google Drive Node): Use node streams for uploading and downloading files (#5017)

* use streams to upload files to google drive

* use streams to download files from google drive

* use resumable uploads api for google drive

* avoid dangling promises, and reduce memory usage in error logging
This commit is contained in:
कारतोफ्फेलस्क्रिप्ट™
2023-01-04 12:29:56 +01:00
committed by GitHub
parent 8b19fdd5f0
commit 54126b2c87
6 changed files with 229 additions and 163 deletions

View File

@@ -1,10 +1,10 @@
import concatStream from 'concat-stream';
import { readFile, stat } from 'fs/promises';
import type { IBinaryData, INodeExecutionData } from 'n8n-workflow';
import type { BinaryMetadata, IBinaryData, INodeExecutionData } from 'n8n-workflow';
import prettyBytes from 'pretty-bytes';
import type { Readable } from 'stream';
import { BINARY_ENCODING } from '../Constants';
import type { BinaryMetadata, IBinaryDataConfig, IBinaryDataManager } from '../Interfaces';
import type { IBinaryDataConfig, IBinaryDataManager } from '../Interfaces';
import { BinaryDataFileSystem } from './FileSystem';
export class BinaryDataManager {
@@ -88,6 +88,7 @@ export class BinaryDataManager {
const manager = this.managers[this.binaryDataMode];
if (manager) {
const identifier = await manager.storeBinaryData(input, executionId);
// Add data manager reference id.
binaryData.id = this.generateBinaryId(identifier);
@@ -115,6 +116,15 @@ export class BinaryDataManager {
return binaryData;
}
getBinaryStream(identifier: string, chunkSize?: number): Readable {
const { mode, id } = this.splitBinaryModeFileId(identifier);
if (this.managers[mode]) {
return this.managers[mode].getBinaryStream(id, chunkSize);
}
throw new Error('Storage mode used to store binary data not available');
}
async retrieveBinaryData(binaryData: IBinaryData): Promise<Buffer> {
if (binaryData.id) {
return this.retrieveBinaryDataByIdentifier(binaryData.id);