fix(HTTP Request Node): Respect the original encoding of the incoming response (#9869)

packages/core/src/BinaryData/BinaryData.service.ts

@@ -3,7 +3,7 @@ import prettyBytes from 'pretty-bytes';
 import Container, { Service } from 'typedi';
 import { BINARY_ENCODING } from 'n8n-workflow';
 import { InvalidModeError } from '../errors/invalid-mode.error';
-import { areConfigModes, toBuffer } from './utils';
+import { areConfigModes, binaryToBuffer } from './utils';
 
 import type { Readable } from 'stream';
 import type { BinaryData } from './types';
@@ -84,7 +84,7 @@ export class BinaryDataService {
 		const manager = this.managers[this.mode];
 
 		if (!manager) {
-			const buffer = await this.toBuffer(bufferOrStream);
+			const buffer = await binaryToBuffer(bufferOrStream);
 			binaryData.data = buffer.toString(BINARY_ENCODING);
 			binaryData.fileSize = prettyBytes(buffer.length);
 
@@ -110,10 +110,6 @@ export class BinaryDataService {
 		return binaryData;
 	}
 
-	async toBuffer(bufferOrStream: Buffer | Readable) {
-		return await toBuffer(bufferOrStream);
-	}
-
 	async getAsStream(binaryDataId: string, chunkSize?: number) {
 		const [mode, fileId] = binaryDataId.split(':');
 

packages/core/src/BinaryData/ObjectStore.manager.ts

@@ -1,7 +1,7 @@
 import fs from 'node:fs/promises';
 import { Service } from 'typedi';
 import { v4 as uuid } from 'uuid';
-import { toBuffer } from './utils';
+import { binaryToBuffer } from './utils';
 import { ObjectStoreService } from '../ObjectStore/ObjectStore.service.ee';
 
 import type { Readable } from 'node:stream';
@@ -22,7 +22,7 @@ export class ObjectStoreManager implements BinaryData.Manager {
 		metadata: BinaryData.PreWriteMetadata,
 	) {
 		const fileId = this.toFileId(workflowId, executionId);
-		const buffer = await this.toBuffer(bufferOrStream);
+		const buffer = await binaryToBuffer(bufferOrStream);
 
 		await this.objectStoreService.put(fileId, buffer, metadata);
 
@@ -100,8 +100,4 @@ export class ObjectStoreManager implements BinaryData.Manager {
 
 		return `workflows/${workflowId}/executions/${executionId}/binary_data/${uuid()}`;
 	}
-
-	private async toBuffer(bufferOrStream: Buffer | Readable) {
-		return await toBuffer(bufferOrStream);
-	}
 }

packages/core/src/BinaryData/utils.ts

@@ -32,7 +32,8 @@ export async function doesNotExist(dir: string) {
 	}
 }
 
-export async function toBuffer(body: Buffer | Readable) {
+/** Converts a buffer or a readable stream to a buffer */
+export async function binaryToBuffer(body: Buffer | Readable) {
 	if (Buffer.isBuffer(body)) return body;
 	return await new Promise<Buffer>((resolve, reject) => {
 		body

packages/core/src/NodeExecuteFunctions.ts

@@ -158,6 +158,7 @@ import type { BinaryData } from './BinaryData/types';
 import merge from 'lodash/merge';
 import { InstanceSettings } from './InstanceSettings';
 import { SSHClientsManager } from './SSHClientsManager';
+import { binaryToBuffer } from './BinaryData/utils';
 
 axios.defaults.timeout = 300000;
 // Prevent axios from adding x-form-www-urlencoded headers by default
@@ -764,6 +765,15 @@ export function parseIncomingMessage(message: IncomingMessage) {
 	}
 }
 
+async function binaryToString(body: Buffer | Readable, encoding?: BufferEncoding) {
+	const buffer = await binaryToBuffer(body);
+	if (!encoding && body instanceof IncomingMessage) {
+		parseIncomingMessage(body);
+		encoding = body.encoding;
+	}
+	return buffer.toString(encoding);
+}
+
 export async function proxyRequestToAxios(
 	workflow: Workflow | undefined,
 	additionalData: IWorkflowExecuteAdditionalData | undefined,
@@ -837,9 +847,7 @@ export async function proxyRequestToAxios(
 				let responseData = response.data;
 
 				if (Buffer.isBuffer(responseData) || responseData instanceof Readable) {
-					responseData = await Container.get(BinaryDataService)
-						.toBuffer(responseData)
-						.then((buffer) => buffer.toString('utf-8'));
+					responseData = await binaryToString(responseData);
 				}
 
 				if (configObject.simple === false) {
@@ -3091,17 +3099,14 @@ const getRequestHelperFunctions = (
 				let contentBody: Exclude<IN8nHttpResponse, Buffer>;
 
 				if (newResponse.body instanceof Readable && paginationOptions.binaryResult !== true) {
-					const data = await this.helpers
-						.binaryToBuffer(newResponse.body as Buffer | Readable)
-						.then((body) => body.toString());
 					// Keep the original string version that we can use it to hash if needed
-					contentBody = data;
+					contentBody = await binaryToString(newResponse.body as Buffer | Readable);
 
 					const responseContentType = newResponse.headers['content-type']?.toString() ?? '';
 					if (responseContentType.includes('application/json')) {
-						newResponse.body = jsonParse(data, { fallbackValue: {} });
+						newResponse.body = jsonParse(contentBody, { fallbackValue: {} });
 					} else {
-						newResponse.body = data;
+						newResponse.body = contentBody;
 					}
 					tempResponseData.__bodyResolved = true;
 					tempResponseData.body = newResponse.body;
@@ -3187,9 +3192,7 @@ const getRequestHelperFunctions = (
 						// now an error manually if the response code is not a success one.
 						let data = tempResponseData.body;
 						if (data instanceof Readable && paginationOptions.binaryResult !== true) {
-							data = await this.helpers
-								.binaryToBuffer(tempResponseData.body as Buffer | Readable)
-								.then((body) => body.toString());
+							data = await binaryToString(data as Buffer | Readable);
 						} else if (typeof data === 'object') {
 							data = JSON.stringify(data);
 						}
@@ -3400,8 +3403,8 @@ const getBinaryHelperFunctions = (
 	getBinaryPath,
 	getBinaryStream,
 	getBinaryMetadata,
-	binaryToBuffer: async (body: Buffer | Readable) =>
-		await Container.get(BinaryDataService).toBuffer(body),
+	binaryToBuffer,
+	binaryToString,
 	prepareBinaryData: async (binaryData, filePath, mimeType) =>
 		await prepareBinaryData(binaryData, executionId!, workflowId, filePath, mimeType),
 	setBinaryDataBuffer: async (data, binaryData) =>
@@ -3743,8 +3746,6 @@ export function getExecuteFunctions(
 				);
 				return dataProxy.getDataProxy();
 			},
-			binaryToBuffer: async (body: Buffer | Readable) =>
-				await Container.get(BinaryDataService).toBuffer(body),
 			async putExecutionToWait(waitTill: Date): Promise<void> {
 				runExecutionData.waitTill = waitTill;
 				if (additionalData.setExecutionStatus) {