From 82e707cdd7b05b9a719cd5fe86ab71aeb39d4b30 Mon Sep 17 00:00:00 2001 From: Michael Kret <88898367+michael-radency@users.noreply.github.com> Date: Tue, 16 Sep 2025 10:32:01 +0300 Subject: [PATCH] fix(Google Gemini Node): Use streams when uploading file (#19537) --- .../GoogleGemini/GoogleGemini.node.test.ts | 24 +- .../actions/file/upload.operation.ts | 43 +-- .../GoogleGemini/helpers/utils.test.ts | 359 +++++++++++++++++- .../vendors/GoogleGemini/helpers/utils.ts | 83 ++++ 4 files changed, 460 insertions(+), 49 deletions(-) diff --git a/packages/@n8n/nodes-langchain/nodes/vendors/GoogleGemini/GoogleGemini.node.test.ts b/packages/@n8n/nodes-langchain/nodes/vendors/GoogleGemini/GoogleGemini.node.test.ts index 613617309a..c92d39a1b2 100644 --- a/packages/@n8n/nodes-langchain/nodes/vendors/GoogleGemini/GoogleGemini.node.test.ts +++ b/packages/@n8n/nodes-langchain/nodes/vendors/GoogleGemini/GoogleGemini.node.test.ts @@ -17,6 +17,7 @@ describe('GoogleGemini Node', () => { const getConnectedToolsMock = jest.spyOn(helpers, 'getConnectedTools'); const downloadFileMock = jest.spyOn(utils, 'downloadFile'); const uploadFileMock = jest.spyOn(utils, 'uploadFile'); + const transferFileMock = jest.spyOn(utils, 'transferFile'); beforeEach(() => { jest.clearAllMocks(); @@ -629,11 +630,7 @@ describe('GoogleGemini Node', () => { return undefined; } }); - downloadFileMock.mockResolvedValue({ - fileContent: Buffer.from('test'), - mimeType: 'application/pdf', - }); - uploadFileMock.mockResolvedValue({ + transferFileMock.mockResolvedValue({ fileUri: 'https://generativelanguage.googleapis.com/v1/files/abc123', mimeType: 'application/pdf', }); @@ -648,11 +645,11 @@ describe('GoogleGemini Node', () => { pairedItem: { item: 0 }, }, ]); - expect(downloadFileMock).toHaveBeenCalledWith( + expect(transferFileMock).toHaveBeenCalledWith( + 0, 'https://example.com/file.pdf', 'application/octet-stream', ); - expect(uploadFileMock).toHaveBeenCalledWith(Buffer.from('test'), 'application/pdf'); }); it('should upload file from binary data', async () => { @@ -666,16 +663,7 @@ describe('GoogleGemini Node', () => { return undefined; } }); - const mockBinaryData: IBinaryData = { - mimeType: 'application/pdf', - fileName: 'test.pdf', - fileSize: '1024', - fileExtension: 'pdf', - data: 'test', - }; - executeFunctionsMock.helpers.assertBinaryData.mockReturnValue(mockBinaryData); - executeFunctionsMock.helpers.getBinaryDataBuffer.mockResolvedValue(Buffer.from('test')); - uploadFileMock.mockResolvedValue({ + transferFileMock.mockResolvedValue({ fileUri: 'https://generativelanguage.googleapis.com/v1/files/abc123', mimeType: 'application/pdf', }); @@ -691,7 +679,7 @@ describe('GoogleGemini Node', () => { pairedItem: { item: 0 }, }, ]); - expect(uploadFileMock).toHaveBeenCalledWith(Buffer.from('test'), 'application/pdf'); + expect(transferFileMock).toHaveBeenCalledWith(0, undefined, 'application/octet-stream'); }); }); diff --git a/packages/@n8n/nodes-langchain/nodes/vendors/GoogleGemini/actions/file/upload.operation.ts b/packages/@n8n/nodes-langchain/nodes/vendors/GoogleGemini/actions/file/upload.operation.ts index 947edc9065..aa1aabc928 100644 --- a/packages/@n8n/nodes-langchain/nodes/vendors/GoogleGemini/actions/file/upload.operation.ts +++ b/packages/@n8n/nodes-langchain/nodes/vendors/GoogleGemini/actions/file/upload.operation.ts @@ -1,7 +1,7 @@ import type { IExecuteFunctions, INodeExecutionData, INodeProperties } from 'n8n-workflow'; import { updateDisplayOptions } from 'n8n-workflow'; -import { downloadFile, uploadFile } from '../../helpers/utils'; +import { transferFile } from '../../helpers/utils'; export const properties: INodeProperties[] = [ { @@ -60,34 +60,19 @@ export const description = updateDisplayOptions(displayOptions, properties); export async function execute(this: IExecuteFunctions, i: number): Promise { const inputType = this.getNodeParameter('inputType', i, 'url') as string; + + let fileUrl: string | undefined; if (inputType === 'url') { - const fileUrl = this.getNodeParameter('fileUrl', i, '') as string; - const { fileContent, mimeType } = await downloadFile.call( - this, - fileUrl, - 'application/octet-stream', - ); - const response = await uploadFile.call(this, fileContent, mimeType); - return [ - { - json: response, - pairedItem: { - item: i, - }, - }, - ]; - } else { - const binaryPropertyName = this.getNodeParameter('binaryPropertyName', i, 'data'); - const binaryData = this.helpers.assertBinaryData(i, binaryPropertyName); - const buffer = await this.helpers.getBinaryDataBuffer(i, binaryPropertyName); - const response = await uploadFile.call(this, buffer, binaryData.mimeType); - return [ - { - json: response, - pairedItem: { - item: i, - }, - }, - ]; + fileUrl = this.getNodeParameter('fileUrl', i, '') as string; } + + const response = await transferFile.call(this, i, fileUrl, 'application/octet-stream'); + return [ + { + json: response, + pairedItem: { + item: i, + }, + }, + ]; } diff --git a/packages/@n8n/nodes-langchain/nodes/vendors/GoogleGemini/helpers/utils.test.ts b/packages/@n8n/nodes-langchain/nodes/vendors/GoogleGemini/helpers/utils.test.ts index a023e94c22..35b1ed2eb0 100644 --- a/packages/@n8n/nodes-langchain/nodes/vendors/GoogleGemini/helpers/utils.test.ts +++ b/packages/@n8n/nodes-langchain/nodes/vendors/GoogleGemini/helpers/utils.test.ts @@ -1,9 +1,14 @@ +import axios from 'axios'; import { mockDeep } from 'jest-mock-extended'; -import type { IExecuteFunctions } from 'n8n-workflow'; +import type { IBinaryData, IExecuteFunctions } from 'n8n-workflow'; +import { NodeOperationError } from 'n8n-workflow'; -import { downloadFile, uploadFile } from './utils'; +import { downloadFile, uploadFile, transferFile } from './utils'; import * as transport from '../transport'; +jest.mock('axios'); +const mockedAxios = axios as jest.Mocked; + describe('GoogleGemini -> utils', () => { const mockExecuteFunctions = mockDeep(); const apiRequestMock = jest.spyOn(transport, 'apiRequest'); @@ -176,5 +181,355 @@ describe('GoogleGemini -> utils', () => { }); expect(apiRequestMock).toHaveBeenCalledWith('GET', '/v1beta/files/test123'); }); + + it('should poll until file is active', async () => { + const fileContent = Buffer.from('test file content'); + const mimeType = 'application/pdf'; + + apiRequestMock.mockResolvedValueOnce({ + headers: { + 'x-goog-upload-url': 'https://upload.googleapis.com/upload/123', + }, + }); + + mockExecuteFunctions.helpers.httpRequest.mockResolvedValueOnce({ + file: { + name: 'files/abc123', + uri: 'https://generativelanguage.googleapis.com/v1/files/abc123', + mimeType: 'application/pdf', + state: 'PROCESSING', + }, + }); + + apiRequestMock + .mockResolvedValueOnce({ + name: 'files/abc123', + uri: 'https://generativelanguage.googleapis.com/v1/files/abc123', + mimeType: 'application/pdf', + state: 'PROCESSING', + }) + .mockResolvedValueOnce({ + name: 'files/abc123', + uri: 'https://generativelanguage.googleapis.com/v1/files/abc123', + mimeType: 'application/pdf', + state: 'ACTIVE', + }); + + jest.spyOn(global, 'setTimeout').mockImplementation((callback: any) => { + callback(); + return {} as any; + }); + + const result = await uploadFile.call(mockExecuteFunctions, fileContent, mimeType); + + expect(result).toEqual({ + fileUri: 'https://generativelanguage.googleapis.com/v1/files/abc123', + mimeType: 'application/pdf', + }); + + expect(apiRequestMock).toHaveBeenCalledWith('GET', '/v1beta/files/abc123'); + }); + + it('should throw error when upload fails', async () => { + const fileContent = Buffer.from('test file content'); + const mimeType = 'application/pdf'; + + apiRequestMock.mockResolvedValueOnce({ + headers: { + 'x-goog-upload-url': 'https://upload.googleapis.com/upload/123', + }, + }); + + mockExecuteFunctions.helpers.httpRequest.mockResolvedValueOnce({ + file: { + name: 'files/abc123', + uri: 'https://generativelanguage.googleapis.com/v1/files/abc123', + mimeType: 'application/pdf', + state: 'FAILED', + error: { message: 'Upload failed' }, + }, + }); + + mockExecuteFunctions.getNode.mockReturnValue({ name: 'Google Gemini' } as any); + + await expect(uploadFile.call(mockExecuteFunctions, fileContent, mimeType)).rejects.toThrow( + new NodeOperationError(mockExecuteFunctions.getNode(), 'Upload failed', { + description: 'Error uploading file', + }), + ); + }); + }); + + describe('transferFile', () => { + it('should transfer file from URL using axios', async () => { + const mockStream = { + pipe: jest.fn(), + on: jest.fn(), + } as any; + + mockedAxios.get.mockResolvedValue({ + data: mockStream, + headers: { + 'content-type': 'application/pdf; charset=utf-8', + }, + }); + + apiRequestMock.mockResolvedValueOnce({ + headers: { + 'x-goog-upload-url': 'https://upload.googleapis.com/upload/123', + }, + }); + + mockExecuteFunctions.helpers.httpRequest.mockResolvedValueOnce({ + body: { + file: { + name: 'files/abc123', + uri: 'https://generativelanguage.googleapis.com/v1/files/abc123', + mimeType: 'application/pdf', + state: 'ACTIVE', + }, + }, + }); + + const result = await transferFile.call( + mockExecuteFunctions, + 0, + 'https://example.com/file.pdf', + 'application/octet-stream', + ); + + expect(result).toEqual({ + fileUri: 'https://generativelanguage.googleapis.com/v1/files/abc123', + mimeType: 'application/pdf', + }); + + expect(mockedAxios.get).toHaveBeenCalledWith('https://example.com/file.pdf', { + params: undefined, + responseType: 'stream', + }); + + expect(apiRequestMock).toHaveBeenCalledWith('POST', '/upload/v1beta/files', { + headers: { + 'X-Goog-Upload-Protocol': 'resumable', + 'X-Goog-Upload-Command': 'start', + 'X-Goog-Upload-Header-Content-Type': 'application/pdf', + 'Content-Type': 'application/json', + }, + option: { returnFullResponse: true }, + }); + + expect(mockExecuteFunctions.helpers.httpRequest).toHaveBeenCalledWith({ + method: 'POST', + url: 'https://upload.googleapis.com/upload/123', + headers: { + 'X-Goog-Upload-Offset': '0', + 'X-Goog-Upload-Command': 'upload, finalize', + 'Content-Type': 'application/pdf', + }, + body: mockStream, + returnFullResponse: true, + }); + }); + + it('should transfer file from binary data without id', async () => { + const mockBinaryData: IBinaryData = { + mimeType: 'application/pdf', + fileName: 'test.pdf', + fileSize: '1024', + fileExtension: 'pdf', + data: 'test', + }; + + mockExecuteFunctions.getNodeParameter.mockReturnValue('data'); + mockExecuteFunctions.helpers.assertBinaryData.mockReturnValue(mockBinaryData); + mockExecuteFunctions.helpers.getBinaryDataBuffer.mockResolvedValue(Buffer.from('test')); + + apiRequestMock.mockResolvedValueOnce({ + headers: { + 'x-goog-upload-url': 'https://upload.googleapis.com/upload/123', + }, + }); + + mockExecuteFunctions.helpers.httpRequest.mockResolvedValueOnce({ + file: { + name: 'files/abc123', + uri: 'https://generativelanguage.googleapis.com/v1/files/abc123', + mimeType: 'application/pdf', + state: 'ACTIVE', + }, + }); + + const result = await transferFile.call( + mockExecuteFunctions, + 0, + undefined, + 'application/octet-stream', + ); + + expect(result).toEqual({ + fileUri: 'https://generativelanguage.googleapis.com/v1/files/abc123', + mimeType: 'application/pdf', + }); + + expect(mockExecuteFunctions.helpers.assertBinaryData).toHaveBeenCalledWith(0, 'data'); + expect(mockExecuteFunctions.helpers.getBinaryDataBuffer).toHaveBeenCalledWith(0, 'data'); + }); + + it('should transfer file from binary data with id using stream', async () => { + const mockBinaryData: IBinaryData = { + id: 'binary-123', + mimeType: 'application/pdf', + fileName: 'test.pdf', + fileSize: '1024', + fileExtension: 'pdf', + data: 'test', + }; + + const mockStream = { + pipe: jest.fn(), + on: jest.fn(), + } as any; + + mockExecuteFunctions.getNodeParameter.mockReturnValue('data'); + mockExecuteFunctions.helpers.assertBinaryData.mockReturnValue(mockBinaryData); + mockExecuteFunctions.helpers.getBinaryStream.mockResolvedValue(mockStream); + + apiRequestMock.mockResolvedValueOnce({ + headers: { + 'x-goog-upload-url': 'https://upload.googleapis.com/upload/123', + }, + }); + + mockExecuteFunctions.helpers.httpRequest.mockResolvedValueOnce({ + body: { + file: { + name: 'files/abc123', + uri: 'https://generativelanguage.googleapis.com/v1/files/abc123', + mimeType: 'application/pdf', + state: 'ACTIVE', + }, + }, + }); + + const result = await transferFile.call( + mockExecuteFunctions, + 0, + undefined, + 'application/octet-stream', + ); + + expect(result).toEqual({ + fileUri: 'https://generativelanguage.googleapis.com/v1/files/abc123', + mimeType: 'application/pdf', + }); + + expect(mockExecuteFunctions.helpers.getBinaryStream).toHaveBeenCalledWith( + 'binary-123', + 262144, + ); + }); + + it('should throw error when binary property name is missing', async () => { + mockExecuteFunctions.getNodeParameter.mockReturnValue(''); + mockExecuteFunctions.getNode.mockReturnValue({ name: 'Google Gemini' } as any); + + await expect( + transferFile.call(mockExecuteFunctions, 0, undefined, 'application/octet-stream'), + ).rejects.toThrow( + new NodeOperationError(mockExecuteFunctions.getNode(), 'Binary property name is required', { + description: 'Error uploading file', + }), + ); + }); + + it('should throw error when upload URL is not received', async () => { + const mockStream = { + pipe: jest.fn(), + on: jest.fn(), + } as any; + + mockedAxios.get.mockResolvedValue({ + data: mockStream, + headers: { + 'content-type': 'application/pdf', + }, + }); + + apiRequestMock.mockResolvedValueOnce({ + headers: {}, + }); + + mockExecuteFunctions.getNode.mockReturnValue({ name: 'Google Gemini' } as any); + + await expect( + transferFile.call( + mockExecuteFunctions, + 0, + 'https://example.com/file.pdf', + 'application/octet-stream', + ), + ).rejects.toThrow( + new NodeOperationError(mockExecuteFunctions.getNode(), 'Failed to get upload URL'), + ); + }); + + it('should poll until file is active and throw error on failure', async () => { + const mockStream = { + pipe: jest.fn(), + on: jest.fn(), + } as any; + + mockedAxios.get.mockResolvedValue({ + data: mockStream, + headers: { + 'content-type': 'application/pdf', + }, + }); + + apiRequestMock.mockResolvedValueOnce({ + headers: { + 'x-goog-upload-url': 'https://upload.googleapis.com/upload/123', + }, + }); + + mockExecuteFunctions.helpers.httpRequest.mockResolvedValueOnce({ + body: { + file: { + name: 'files/abc123', + uri: 'https://generativelanguage.googleapis.com/v1/files/abc123', + mimeType: 'application/pdf', + state: 'PROCESSING', + }, + }, + }); + + apiRequestMock.mockResolvedValueOnce({ + name: 'files/abc123', + uri: 'https://generativelanguage.googleapis.com/v1/files/abc123', + mimeType: 'application/pdf', + state: 'FAILED', + error: { message: 'Processing failed' }, + }); + + jest.spyOn(global, 'setTimeout').mockImplementation((callback: any) => { + callback(); + return {} as any; + }); + + mockExecuteFunctions.getNode.mockReturnValue({ name: 'Google Gemini' } as any); + + await expect( + transferFile.call( + mockExecuteFunctions, + 0, + 'https://example.com/file.pdf', + 'application/octet-stream', + ), + ).rejects.toThrow( + new NodeOperationError(mockExecuteFunctions.getNode(), 'Processing failed', { + description: 'Error uploading file', + }), + ); + }); }); }); diff --git a/packages/@n8n/nodes-langchain/nodes/vendors/GoogleGemini/helpers/utils.ts b/packages/@n8n/nodes-langchain/nodes/vendors/GoogleGemini/helpers/utils.ts index 8b337fc27b..b4d3e49dd9 100644 --- a/packages/@n8n/nodes-langchain/nodes/vendors/GoogleGemini/helpers/utils.ts +++ b/packages/@n8n/nodes-langchain/nodes/vendors/GoogleGemini/helpers/utils.ts @@ -3,6 +3,9 @@ import { NodeOperationError } from 'n8n-workflow'; import { apiRequest } from '../transport'; +import axios from 'axios'; +import type Stream from 'node:stream'; + interface File { name: string; uri: string; @@ -11,6 +14,8 @@ interface File { error?: { message: string }; } +const CHUNK_SIZE = 256 * 1024; + export async function downloadFile( this: IExecuteFunctions, url: string, @@ -82,3 +87,81 @@ export async function uploadFile(this: IExecuteFunctions, fileContent: Buffer, m return { fileUri: uploadResponse.file.uri, mimeType: uploadResponse.file.mimeType }; } + +export async function transferFile( + this: IExecuteFunctions, + i: number, + downloadUrl?: string, + fallbackMimeType?: string, + qs?: IDataObject, +) { + let stream: Stream; + let mimeType: string; + + if (downloadUrl) { + const downloadResponse = await axios.get(downloadUrl, { + params: qs, + responseType: 'stream', + }); + + mimeType = downloadResponse.headers['content-type']?.split(';')?.[0] ?? fallbackMimeType; + stream = downloadResponse.data; + } else { + const binaryPropertyName = this.getNodeParameter('binaryPropertyName', i, 'data'); + if (!binaryPropertyName) { + throw new NodeOperationError(this.getNode(), 'Binary property name is required', { + description: 'Error uploading file', + }); + } + const binaryData = this.helpers.assertBinaryData(i, binaryPropertyName); + if (!binaryData.id) { + const buffer = await this.helpers.getBinaryDataBuffer(i, binaryPropertyName); + return await uploadFile.call(this, buffer, binaryData.mimeType); + } else { + stream = await this.helpers.getBinaryStream(binaryData.id, CHUNK_SIZE); + mimeType = binaryData.mimeType; + } + } + + const uploadInitResponse = (await apiRequest.call(this, 'POST', '/upload/v1beta/files', { + headers: { + 'X-Goog-Upload-Protocol': 'resumable', + 'X-Goog-Upload-Command': 'start', + 'X-Goog-Upload-Header-Content-Type': mimeType, + 'Content-Type': 'application/json', + }, + option: { returnFullResponse: true }, + })) as { headers: IDataObject }; + + const uploadUrl = uploadInitResponse.headers['x-goog-upload-url'] as string; + if (!uploadUrl) { + throw new NodeOperationError(this.getNode(), 'Failed to get upload URL'); + } + + const uploadResponse = (await this.helpers.httpRequest({ + method: 'POST', + url: uploadUrl, + headers: { + 'X-Goog-Upload-Offset': '0', + 'X-Goog-Upload-Command': 'upload, finalize', + 'Content-Type': mimeType, + }, + body: stream, + returnFullResponse: true, + })) as { body: { file: File } }; + + let file = uploadResponse.body.file; + + while (file.state !== 'ACTIVE' && file.state !== 'FAILED') { + await new Promise((resolve) => setTimeout(resolve, 1000)); + file = (await apiRequest.call(this, 'GET', `/v1beta/${file.name}`)) as File; + } + + if (file.state === 'FAILED') { + throw new NodeOperationError(this.getNode(), file.error?.message ?? 'Unknown error', { + description: 'Error uploading file', + }); + } + + return { fileUri: file.uri, mimeType: file.mimeType }; +}