fix(Google Gemini Node): Use streams when uploading file (#19537)

packages/@n8n/nodes-langchain/nodes/vendors/GoogleGemini/GoogleGemini.node.test.ts

@@ -17,6 +17,7 @@ describe('GoogleGemini Node', () => {
 	const getConnectedToolsMock = jest.spyOn(helpers, 'getConnectedTools');
 	const downloadFileMock = jest.spyOn(utils, 'downloadFile');
 	const uploadFileMock = jest.spyOn(utils, 'uploadFile');
+	const transferFileMock = jest.spyOn(utils, 'transferFile');
 
 	beforeEach(() => {
 		jest.clearAllMocks();
@@ -629,11 +630,7 @@ describe('GoogleGemini Node', () => {
 						return undefined;
 				}
 			});
-			downloadFileMock.mockResolvedValue({
-				fileContent: Buffer.from('test'),
-				mimeType: 'application/pdf',
-			});
-			uploadFileMock.mockResolvedValue({
+			transferFileMock.mockResolvedValue({
 				fileUri: 'https://generativelanguage.googleapis.com/v1/files/abc123',
 				mimeType: 'application/pdf',
 			});
@@ -648,11 +645,11 @@ describe('GoogleGemini Node', () => {
 					pairedItem: { item: 0 },
 				},
 			]);
-			expect(downloadFileMock).toHaveBeenCalledWith(
+			expect(transferFileMock).toHaveBeenCalledWith(
+				0,
 				'https://example.com/file.pdf',
 				'application/octet-stream',
 			);
-			expect(uploadFileMock).toHaveBeenCalledWith(Buffer.from('test'), 'application/pdf');
 		});
 
 		it('should upload file from binary data', async () => {
@@ -666,16 +663,7 @@ describe('GoogleGemini Node', () => {
 						return undefined;
 				}
 			});
-			const mockBinaryData: IBinaryData = {
-				mimeType: 'application/pdf',
-				fileName: 'test.pdf',
-				fileSize: '1024',
-				fileExtension: 'pdf',
-				data: 'test',
-			};
-			executeFunctionsMock.helpers.assertBinaryData.mockReturnValue(mockBinaryData);
-			executeFunctionsMock.helpers.getBinaryDataBuffer.mockResolvedValue(Buffer.from('test'));
-			uploadFileMock.mockResolvedValue({
+			transferFileMock.mockResolvedValue({
 				fileUri: 'https://generativelanguage.googleapis.com/v1/files/abc123',
 				mimeType: 'application/pdf',
 			});
@@ -691,7 +679,7 @@ describe('GoogleGemini Node', () => {
 					pairedItem: { item: 0 },
 				},
 			]);
-			expect(uploadFileMock).toHaveBeenCalledWith(Buffer.from('test'), 'application/pdf');
+			expect(transferFileMock).toHaveBeenCalledWith(0, undefined, 'application/octet-stream');
 		});
 	});
 

packages/@n8n/nodes-langchain/nodes/vendors/GoogleGemini/actions/file/upload.operation.ts

@@ -1,7 +1,7 @@
 import type { IExecuteFunctions, INodeExecutionData, INodeProperties } from 'n8n-workflow';
 import { updateDisplayOptions } from 'n8n-workflow';
 
-import { downloadFile, uploadFile } from '../../helpers/utils';
+import { transferFile } from '../../helpers/utils';
 
 export const properties: INodeProperties[] = [
 	{
@@ -60,34 +60,19 @@ export const description = updateDisplayOptions(displayOptions, properties);
 
 export async function execute(this: IExecuteFunctions, i: number): Promise<INodeExecutionData[]> {
 	const inputType = this.getNodeParameter('inputType', i, 'url') as string;
+
+	let fileUrl: string | undefined;
 	if (inputType === 'url') {
-		const fileUrl = this.getNodeParameter('fileUrl', i, '') as string;
-		const { fileContent, mimeType } = await downloadFile.call(
-			this,
-			fileUrl,
-			'application/octet-stream',
-		);
-		const response = await uploadFile.call(this, fileContent, mimeType);
-		return [
-			{
-				json: response,
-				pairedItem: {
-					item: i,
-				},
-			},
-		];
-	} else {
-		const binaryPropertyName = this.getNodeParameter('binaryPropertyName', i, 'data');
-		const binaryData = this.helpers.assertBinaryData(i, binaryPropertyName);
-		const buffer = await this.helpers.getBinaryDataBuffer(i, binaryPropertyName);
-		const response = await uploadFile.call(this, buffer, binaryData.mimeType);
-		return [
-			{
-				json: response,
-				pairedItem: {
-					item: i,
-				},
-			},
-		];
+		fileUrl = this.getNodeParameter('fileUrl', i, '') as string;
 	}
+
+	const response = await transferFile.call(this, i, fileUrl, 'application/octet-stream');
+	return [
+		{
+			json: response,
+			pairedItem: {
+				item: i,
+			},
+		},
+	];
 }

packages/@n8n/nodes-langchain/nodes/vendors/GoogleGemini/helpers/utils.test.ts

@@ -1,9 +1,14 @@
+import axios from 'axios';
 import { mockDeep } from 'jest-mock-extended';
-import type { IExecuteFunctions } from 'n8n-workflow';
+import type { IBinaryData, IExecuteFunctions } from 'n8n-workflow';
+import { NodeOperationError } from 'n8n-workflow';
 
-import { downloadFile, uploadFile } from './utils';
+import { downloadFile, uploadFile, transferFile } from './utils';
 import * as transport from '../transport';
 
+jest.mock('axios');
+const mockedAxios = axios as jest.Mocked<typeof axios>;
+
 describe('GoogleGemini -> utils', () => {
 	const mockExecuteFunctions = mockDeep<IExecuteFunctions>();
 	const apiRequestMock = jest.spyOn(transport, 'apiRequest');
@@ -176,5 +181,355 @@ describe('GoogleGemini -> utils', () => {
 			});
 			expect(apiRequestMock).toHaveBeenCalledWith('GET', '/v1beta/files/test123');
 		});
+
+		it('should poll until file is active', async () => {
+			const fileContent = Buffer.from('test file content');
+			const mimeType = 'application/pdf';
+
+			apiRequestMock.mockResolvedValueOnce({
+				headers: {
+					'x-goog-upload-url': 'https://upload.googleapis.com/upload/123',
+				},
+			});
+
+			mockExecuteFunctions.helpers.httpRequest.mockResolvedValueOnce({
+				file: {
+					name: 'files/abc123',
+					uri: 'https://generativelanguage.googleapis.com/v1/files/abc123',
+					mimeType: 'application/pdf',
+					state: 'PROCESSING',
+				},
+			});
+
+			apiRequestMock
+				.mockResolvedValueOnce({
+					name: 'files/abc123',
+					uri: 'https://generativelanguage.googleapis.com/v1/files/abc123',
+					mimeType: 'application/pdf',
+					state: 'PROCESSING',
+				})
+				.mockResolvedValueOnce({
+					name: 'files/abc123',
+					uri: 'https://generativelanguage.googleapis.com/v1/files/abc123',
+					mimeType: 'application/pdf',
+					state: 'ACTIVE',
+				});
+
+			jest.spyOn(global, 'setTimeout').mockImplementation((callback: any) => {
+				callback();
+				return {} as any;
+			});
+
+			const result = await uploadFile.call(mockExecuteFunctions, fileContent, mimeType);
+
+			expect(result).toEqual({
+				fileUri: 'https://generativelanguage.googleapis.com/v1/files/abc123',
+				mimeType: 'application/pdf',
+			});
+
+			expect(apiRequestMock).toHaveBeenCalledWith('GET', '/v1beta/files/abc123');
+		});
+
+		it('should throw error when upload fails', async () => {
+			const fileContent = Buffer.from('test file content');
+			const mimeType = 'application/pdf';
+
+			apiRequestMock.mockResolvedValueOnce({
+				headers: {
+					'x-goog-upload-url': 'https://upload.googleapis.com/upload/123',
+				},
+			});
+
+			mockExecuteFunctions.helpers.httpRequest.mockResolvedValueOnce({
+				file: {
+					name: 'files/abc123',
+					uri: 'https://generativelanguage.googleapis.com/v1/files/abc123',
+					mimeType: 'application/pdf',
+					state: 'FAILED',
+					error: { message: 'Upload failed' },
+				},
+			});
+
+			mockExecuteFunctions.getNode.mockReturnValue({ name: 'Google Gemini' } as any);
+
+			await expect(uploadFile.call(mockExecuteFunctions, fileContent, mimeType)).rejects.toThrow(
+				new NodeOperationError(mockExecuteFunctions.getNode(), 'Upload failed', {
+					description: 'Error uploading file',
+				}),
+			);
+		});
+	});
+
+	describe('transferFile', () => {
+		it('should transfer file from URL using axios', async () => {
+			const mockStream = {
+				pipe: jest.fn(),
+				on: jest.fn(),
+			} as any;
+
+			mockedAxios.get.mockResolvedValue({
+				data: mockStream,
+				headers: {
+					'content-type': 'application/pdf; charset=utf-8',
+				},
+			});
+
+			apiRequestMock.mockResolvedValueOnce({
+				headers: {
+					'x-goog-upload-url': 'https://upload.googleapis.com/upload/123',
+				},
+			});
+
+			mockExecuteFunctions.helpers.httpRequest.mockResolvedValueOnce({
+				body: {
+					file: {
+						name: 'files/abc123',
+						uri: 'https://generativelanguage.googleapis.com/v1/files/abc123',
+						mimeType: 'application/pdf',
+						state: 'ACTIVE',
+					},
+				},
+			});
+
+			const result = await transferFile.call(
+				mockExecuteFunctions,
+				0,
+				'https://example.com/file.pdf',
+				'application/octet-stream',
+			);
+
+			expect(result).toEqual({
+				fileUri: 'https://generativelanguage.googleapis.com/v1/files/abc123',
+				mimeType: 'application/pdf',
+			});
+
+			expect(mockedAxios.get).toHaveBeenCalledWith('https://example.com/file.pdf', {
+				params: undefined,
+				responseType: 'stream',
+			});
+
+			expect(apiRequestMock).toHaveBeenCalledWith('POST', '/upload/v1beta/files', {
+				headers: {
+					'X-Goog-Upload-Protocol': 'resumable',
+					'X-Goog-Upload-Command': 'start',
+					'X-Goog-Upload-Header-Content-Type': 'application/pdf',
+					'Content-Type': 'application/json',
+				},
+				option: { returnFullResponse: true },
+			});
+
+			expect(mockExecuteFunctions.helpers.httpRequest).toHaveBeenCalledWith({
+				method: 'POST',
+				url: 'https://upload.googleapis.com/upload/123',
+				headers: {
+					'X-Goog-Upload-Offset': '0',
+					'X-Goog-Upload-Command': 'upload, finalize',
+					'Content-Type': 'application/pdf',
+				},
+				body: mockStream,
+				returnFullResponse: true,
+			});
+		});
+
+		it('should transfer file from binary data without id', async () => {
+			const mockBinaryData: IBinaryData = {
+				mimeType: 'application/pdf',
+				fileName: 'test.pdf',
+				fileSize: '1024',
+				fileExtension: 'pdf',
+				data: 'test',
+			};
+
+			mockExecuteFunctions.getNodeParameter.mockReturnValue('data');
+			mockExecuteFunctions.helpers.assertBinaryData.mockReturnValue(mockBinaryData);
+			mockExecuteFunctions.helpers.getBinaryDataBuffer.mockResolvedValue(Buffer.from('test'));
+
+			apiRequestMock.mockResolvedValueOnce({
+				headers: {
+					'x-goog-upload-url': 'https://upload.googleapis.com/upload/123',
+				},
+			});
+
+			mockExecuteFunctions.helpers.httpRequest.mockResolvedValueOnce({
+				file: {
+					name: 'files/abc123',
+					uri: 'https://generativelanguage.googleapis.com/v1/files/abc123',
+					mimeType: 'application/pdf',
+					state: 'ACTIVE',
+				},
+			});
+
+			const result = await transferFile.call(
+				mockExecuteFunctions,
+				0,
+				undefined,
+				'application/octet-stream',
+			);
+
+			expect(result).toEqual({
+				fileUri: 'https://generativelanguage.googleapis.com/v1/files/abc123',
+				mimeType: 'application/pdf',
+			});
+
+			expect(mockExecuteFunctions.helpers.assertBinaryData).toHaveBeenCalledWith(0, 'data');
+			expect(mockExecuteFunctions.helpers.getBinaryDataBuffer).toHaveBeenCalledWith(0, 'data');
+		});
+
+		it('should transfer file from binary data with id using stream', async () => {
+			const mockBinaryData: IBinaryData = {
+				id: 'binary-123',
+				mimeType: 'application/pdf',
+				fileName: 'test.pdf',
+				fileSize: '1024',
+				fileExtension: 'pdf',
+				data: 'test',
+			};
+
+			const mockStream = {
+				pipe: jest.fn(),
+				on: jest.fn(),
+			} as any;
+
+			mockExecuteFunctions.getNodeParameter.mockReturnValue('data');
+			mockExecuteFunctions.helpers.assertBinaryData.mockReturnValue(mockBinaryData);
+			mockExecuteFunctions.helpers.getBinaryStream.mockResolvedValue(mockStream);
+
+			apiRequestMock.mockResolvedValueOnce({
+				headers: {
+					'x-goog-upload-url': 'https://upload.googleapis.com/upload/123',
+				},
+			});
+
+			mockExecuteFunctions.helpers.httpRequest.mockResolvedValueOnce({
+				body: {
+					file: {
+						name: 'files/abc123',
+						uri: 'https://generativelanguage.googleapis.com/v1/files/abc123',
+						mimeType: 'application/pdf',
+						state: 'ACTIVE',
+					},
+				},
+			});
+
+			const result = await transferFile.call(
+				mockExecuteFunctions,
+				0,
+				undefined,
+				'application/octet-stream',
+			);
+
+			expect(result).toEqual({
+				fileUri: 'https://generativelanguage.googleapis.com/v1/files/abc123',
+				mimeType: 'application/pdf',
+			});
+
+			expect(mockExecuteFunctions.helpers.getBinaryStream).toHaveBeenCalledWith(
+				'binary-123',
+				262144,
+			);
+		});
+
+		it('should throw error when binary property name is missing', async () => {
+			mockExecuteFunctions.getNodeParameter.mockReturnValue('');
+			mockExecuteFunctions.getNode.mockReturnValue({ name: 'Google Gemini' } as any);
+
+			await expect(
+				transferFile.call(mockExecuteFunctions, 0, undefined, 'application/octet-stream'),
+			).rejects.toThrow(
+				new NodeOperationError(mockExecuteFunctions.getNode(), 'Binary property name is required', {
+					description: 'Error uploading file',
+				}),
+			);
+		});
+
+		it('should throw error when upload URL is not received', async () => {
+			const mockStream = {
+				pipe: jest.fn(),
+				on: jest.fn(),
+			} as any;
+
+			mockedAxios.get.mockResolvedValue({
+				data: mockStream,
+				headers: {
+					'content-type': 'application/pdf',
+				},
+			});
+
+			apiRequestMock.mockResolvedValueOnce({
+				headers: {},
+			});
+
+			mockExecuteFunctions.getNode.mockReturnValue({ name: 'Google Gemini' } as any);
+
+			await expect(
+				transferFile.call(
+					mockExecuteFunctions,
+					0,
+					'https://example.com/file.pdf',
+					'application/octet-stream',
+				),
+			).rejects.toThrow(
+				new NodeOperationError(mockExecuteFunctions.getNode(), 'Failed to get upload URL'),
+			);
+		});
+
+		it('should poll until file is active and throw error on failure', async () => {
+			const mockStream = {
+				pipe: jest.fn(),
+				on: jest.fn(),
+			} as any;
+
+			mockedAxios.get.mockResolvedValue({
+				data: mockStream,
+				headers: {
+					'content-type': 'application/pdf',
+				},
+			});
+
+			apiRequestMock.mockResolvedValueOnce({
+				headers: {
+					'x-goog-upload-url': 'https://upload.googleapis.com/upload/123',
+				},
+			});
+
+			mockExecuteFunctions.helpers.httpRequest.mockResolvedValueOnce({
+				body: {
+					file: {
+						name: 'files/abc123',
+						uri: 'https://generativelanguage.googleapis.com/v1/files/abc123',
+						mimeType: 'application/pdf',
+						state: 'PROCESSING',
+					},
+				},
+			});
+
+			apiRequestMock.mockResolvedValueOnce({
+				name: 'files/abc123',
+				uri: 'https://generativelanguage.googleapis.com/v1/files/abc123',
+				mimeType: 'application/pdf',
+				state: 'FAILED',
+				error: { message: 'Processing failed' },
+			});
+
+			jest.spyOn(global, 'setTimeout').mockImplementation((callback: any) => {
+				callback();
+				return {} as any;
+			});
+
+			mockExecuteFunctions.getNode.mockReturnValue({ name: 'Google Gemini' } as any);
+
+			await expect(
+				transferFile.call(
+					mockExecuteFunctions,
+					0,
+					'https://example.com/file.pdf',
+					'application/octet-stream',
+				),
+			).rejects.toThrow(
+				new NodeOperationError(mockExecuteFunctions.getNode(), 'Processing failed', {
+					description: 'Error uploading file',
+				}),
+			);
+		});
 	});
 });

packages/@n8n/nodes-langchain/nodes/vendors/GoogleGemini/helpers/utils.ts

@@ -3,6 +3,9 @@ import { NodeOperationError } from 'n8n-workflow';
 
 import { apiRequest } from '../transport';
 
+import axios from 'axios';
+import type Stream from 'node:stream';
+
 interface File {
 	name: string;
 	uri: string;
@@ -11,6 +14,8 @@ interface File {
 	error?: { message: string };
 }
 
+const CHUNK_SIZE = 256 * 1024;
+
 export async function downloadFile(
 	this: IExecuteFunctions,
 	url: string,
@@ -82,3 +87,81 @@ export async function uploadFile(this: IExecuteFunctions, fileContent: Buffer, m
 
 	return { fileUri: uploadResponse.file.uri, mimeType: uploadResponse.file.mimeType };
 }
+
+export async function transferFile(
+	this: IExecuteFunctions,
+	i: number,
+	downloadUrl?: string,
+	fallbackMimeType?: string,
+	qs?: IDataObject,
+) {
+	let stream: Stream;
+	let mimeType: string;
+
+	if (downloadUrl) {
+		const downloadResponse = await axios.get(downloadUrl, {
+			params: qs,
+			responseType: 'stream',
+		});
+
+		mimeType = downloadResponse.headers['content-type']?.split(';')?.[0] ?? fallbackMimeType;
+		stream = downloadResponse.data;
+	} else {
+		const binaryPropertyName = this.getNodeParameter('binaryPropertyName', i, 'data');
+		if (!binaryPropertyName) {
+			throw new NodeOperationError(this.getNode(), 'Binary property name is required', {
+				description: 'Error uploading file',
+			});
+		}
+		const binaryData = this.helpers.assertBinaryData(i, binaryPropertyName);
+		if (!binaryData.id) {
+			const buffer = await this.helpers.getBinaryDataBuffer(i, binaryPropertyName);
+			return await uploadFile.call(this, buffer, binaryData.mimeType);
+		} else {
+			stream = await this.helpers.getBinaryStream(binaryData.id, CHUNK_SIZE);
+			mimeType = binaryData.mimeType;
+		}
+	}
+
+	const uploadInitResponse = (await apiRequest.call(this, 'POST', '/upload/v1beta/files', {
+		headers: {
+			'X-Goog-Upload-Protocol': 'resumable',
+			'X-Goog-Upload-Command': 'start',
+			'X-Goog-Upload-Header-Content-Type': mimeType,
+			'Content-Type': 'application/json',
+		},
+		option: { returnFullResponse: true },
+	})) as { headers: IDataObject };
+
+	const uploadUrl = uploadInitResponse.headers['x-goog-upload-url'] as string;
+	if (!uploadUrl) {
+		throw new NodeOperationError(this.getNode(), 'Failed to get upload URL');
+	}
+
+	const uploadResponse = (await this.helpers.httpRequest({
+		method: 'POST',
+		url: uploadUrl,
+		headers: {
+			'X-Goog-Upload-Offset': '0',
+			'X-Goog-Upload-Command': 'upload, finalize',
+			'Content-Type': mimeType,
+		},
+		body: stream,
+		returnFullResponse: true,
+	})) as { body: { file: File } };
+
+	let file = uploadResponse.body.file;
+
+	while (file.state !== 'ACTIVE' && file.state !== 'FAILED') {
+		await new Promise((resolve) => setTimeout(resolve, 1000));
+		file = (await apiRequest.call(this, 'GET', `/v1beta/${file.name}`)) as File;
+	}
+
+	if (file.state === 'FAILED') {
+		throw new NodeOperationError(this.getNode(), file.error?.message ?? 'Unknown error', {
+			description: 'Error uploading file',
+		});
+	}
+
+	return { fileUri: file.uri, mimeType: file.mimeType };
+}