From 461b39c5df5dd446cb8ceef469b204c7c5111229 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E0=A4=95=E0=A4=BE=E0=A4=B0=E0=A4=A4=E0=A5=8B=E0=A4=AB?= =?UTF-8?q?=E0=A5=8D=E0=A4=AB=E0=A5=87=E0=A4=B2=E0=A4=B8=E0=A5=8D=E0=A4=95?= =?UTF-8?q?=E0=A5=8D=E0=A4=B0=E0=A4=BF=E0=A4=AA=E0=A5=8D=E0=A4=9F=E2=84=A2?= Date: Thu, 28 Nov 2024 14:31:54 +0100 Subject: [PATCH] fix(HTTP Request Node): Use iconv-lite to decode http responses, to support more encoding types (#11930) --- packages/@n8n/imap/package.json | 2 +- packages/core/package.json | 1 + packages/core/src/NodeExecuteFunctions.ts | 7 +- .../core/test/NodeExecuteFunctions.test.ts | 99 ++++++++++++++++++- packages/nodes-base/package.json | 2 +- pnpm-lock.yaml | 54 +++++----- pnpm-workspace.yaml | 1 + 7 files changed, 132 insertions(+), 34 deletions(-) diff --git a/packages/@n8n/imap/package.json b/packages/@n8n/imap/package.json index 81643869a0..5a1d6b54f1 100644 --- a/packages/@n8n/imap/package.json +++ b/packages/@n8n/imap/package.json @@ -20,7 +20,7 @@ "dist/**/*" ], "dependencies": { - "iconv-lite": "0.6.3", + "iconv-lite": "catalog:", "imap": "0.8.19", "quoted-printable": "1.0.1", "utf8": "3.0.0", diff --git a/packages/core/package.json b/packages/core/package.json index 855ecd15e4..cedac2f2f5 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -47,6 +47,7 @@ "fast-glob": "catalog:", "file-type": "16.5.4", "form-data": "catalog:", + "iconv-lite": "catalog:", "lodash": "catalog:", "luxon": "catalog:", "mime-types": "2.1.35", diff --git a/packages/core/src/NodeExecuteFunctions.ts b/packages/core/src/NodeExecuteFunctions.ts index 750ff80dd9..b8635f2e1d 100644 --- a/packages/core/src/NodeExecuteFunctions.ts +++ b/packages/core/src/NodeExecuteFunctions.ts @@ -28,6 +28,7 @@ import { createReadStream } from 'fs'; import { access as fsAccess, writeFile as fsWriteFile } from 'fs/promises'; import { IncomingMessage } from 'http'; import { Agent, type AgentOptions } from 'https'; +import iconv from 'iconv-lite'; import get from 'lodash/get'; import isEmpty from 'lodash/isEmpty'; import merge from 'lodash/merge'; @@ -745,13 +746,13 @@ export function parseIncomingMessage(message: IncomingMessage) { } } -export async function binaryToString(body: Buffer | Readable, encoding?: BufferEncoding) { - const buffer = await binaryToBuffer(body); +export async function binaryToString(body: Buffer | Readable, encoding?: string) { if (!encoding && body instanceof IncomingMessage) { parseIncomingMessage(body); encoding = body.encoding; } - return buffer.toString(encoding); + const buffer = await binaryToBuffer(body); + return iconv.decode(buffer, encoding ?? 'utf-8'); } export async function proxyRequestToAxios( diff --git a/packages/core/test/NodeExecuteFunctions.test.ts b/packages/core/test/NodeExecuteFunctions.test.ts index a796314b57..bf985fe729 100644 --- a/packages/core/test/NodeExecuteFunctions.test.ts +++ b/packages/core/test/NodeExecuteFunctions.test.ts @@ -1,5 +1,5 @@ import { mkdtempSync, readFileSync } from 'fs'; -import type { IncomingMessage } from 'http'; +import { IncomingMessage } from 'http'; import type { Agent } from 'https'; import { mock } from 'jest-mock-extended'; import type { @@ -16,12 +16,14 @@ import type { import nock from 'nock'; import { tmpdir } from 'os'; import { join } from 'path'; +import { Readable } from 'stream'; import type { SecureContextOptions } from 'tls'; import Container from 'typedi'; import { BinaryDataService } from '@/BinaryData/BinaryData.service'; import { InstanceSettings } from '@/InstanceSettings'; import { + binaryToString, copyInputItems, getBinaryDataBuffer, isFilePathBlocked, @@ -549,6 +551,101 @@ describe('NodeExecuteFunctions', () => { }, ); }); + + describe('binaryToString', () => { + const ENCODING_SAMPLES = { + utf8: { + text: 'Hello, 世界! τεστ мир ⚡️ é à ü ñ', + buffer: Buffer.from([ + 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x2c, 0x20, 0xe4, 0xb8, 0x96, 0xe7, 0x95, 0x8c, 0x21, 0x20, + 0xcf, 0x84, 0xce, 0xb5, 0xcf, 0x83, 0xcf, 0x84, 0x20, 0xd0, 0xbc, 0xd0, 0xb8, 0xd1, 0x80, + 0x20, 0xe2, 0x9a, 0xa1, 0xef, 0xb8, 0x8f, 0x20, 0xc3, 0xa9, 0x20, 0xc3, 0xa0, 0x20, 0xc3, + 0xbc, 0x20, 0xc3, 0xb1, + ]), + }, + + 'iso-8859-15': { + text: 'Café € personnalité', + buffer: Buffer.from([ + 0x43, 0x61, 0x66, 0xe9, 0x20, 0xa4, 0x20, 0x70, 0x65, 0x72, 0x73, 0x6f, 0x6e, 0x6e, 0x61, + 0x6c, 0x69, 0x74, 0xe9, + ]), + }, + + latin1: { + text: 'señor année déjà', + buffer: Buffer.from([ + 0x73, 0x65, 0xf1, 0x6f, 0x72, 0x20, 0x61, 0x6e, 0x6e, 0xe9, 0x65, 0x20, 0x64, 0xe9, 0x6a, + 0xe0, + ]), + }, + + ascii: { + text: 'Hello, World! 123', + buffer: Buffer.from([ + 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x2c, 0x20, 0x57, 0x6f, 0x72, 0x6c, 0x64, 0x21, 0x20, 0x31, + 0x32, 0x33, + ]), + }, + + 'windows-1252': { + text: '€ Smart "quotes" • bullet', + buffer: Buffer.from([ + 0x80, 0x20, 0x53, 0x6d, 0x61, 0x72, 0x74, 0x20, 0x22, 0x71, 0x75, 0x6f, 0x74, 0x65, 0x73, + 0x22, 0x20, 0x95, 0x20, 0x62, 0x75, 0x6c, 0x6c, 0x65, 0x74, + ]), + }, + + 'shift-jis': { + text: 'こんにちは世界', + buffer: Buffer.from([ + 0x82, 0xb1, 0x82, 0xf1, 0x82, 0xc9, 0x82, 0xbf, 0x82, 0xcd, 0x90, 0xa2, 0x8a, 0x45, + ]), + }, + + big5: { + text: '哈囉世界', + buffer: Buffer.from([0xab, 0xa2, 0xc5, 0x6f, 0xa5, 0x40, 0xac, 0xc9]), + }, + + 'koi8-r': { + text: 'Привет мир', + buffer: Buffer.from([0xf0, 0xd2, 0xc9, 0xd7, 0xc5, 0xd4, 0x20, 0xcd, 0xc9, 0xd2]), + }, + }; + + describe('should handle Buffer', () => { + for (const [encoding, { text, buffer }] of Object.entries(ENCODING_SAMPLES)) { + test(`with ${encoding}`, async () => { + const data = await binaryToString(buffer, encoding); + expect(data).toBe(text); + }); + } + }); + + describe('should handle streams', () => { + for (const [encoding, { text, buffer }] of Object.entries(ENCODING_SAMPLES)) { + test(`with ${encoding}`, async () => { + const stream = Readable.from(buffer); + const data = await binaryToString(stream, encoding); + expect(data).toBe(text); + }); + } + }); + + describe('should handle IncomingMessage', () => { + for (const [encoding, { text, buffer }] of Object.entries(ENCODING_SAMPLES)) { + test(`with ${encoding}`, async () => { + const response = Readable.from(buffer) as IncomingMessage; + response.headers = { 'content-type': `application/json;charset=${encoding}` }; + // @ts-expect-error need this hack to fake `instanceof IncomingMessage` checks + response.__proto__ = IncomingMessage.prototype; + const data = await binaryToString(response); + expect(data).toBe(text); + }); + } + }); + }); }); describe('isFilePathBlocked', () => { diff --git a/packages/nodes-base/package.json b/packages/nodes-base/package.json index a77db99ccc..d195b512d0 100644 --- a/packages/nodes-base/package.json +++ b/packages/nodes-base/package.json @@ -868,7 +868,7 @@ "get-system-fonts": "2.0.2", "gm": "1.25.0", "html-to-text": "9.0.5", - "iconv-lite": "0.6.3", + "iconv-lite": "catalog:", "ics": "2.40.0", "isbot": "3.6.13", "iso-639-1": "2.1.15", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3eabd32411..adb0314865 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -45,6 +45,9 @@ catalogs: form-data: specifier: 4.0.0 version: 4.0.0 + iconv-lite: + specifier: 0.6.3 + version: 0.6.3 lodash: specifier: 4.17.21 version: 4.17.21 @@ -280,7 +283,7 @@ importers: version: 4.0.7 axios: specifier: 'catalog:' - version: 1.7.4 + version: 1.7.4(debug@4.3.7) dotenv: specifier: 8.6.0 version: 8.6.0 @@ -348,7 +351,7 @@ importers: dependencies: axios: specifier: 'catalog:' - version: 1.7.4 + version: 1.7.4(debug@4.3.7) packages/@n8n/codemirror-lang: dependencies: @@ -378,7 +381,7 @@ importers: packages/@n8n/imap: dependencies: iconv-lite: - specifier: 0.6.3 + specifier: 'catalog:' version: 0.6.3 imap: specifier: 0.8.19 @@ -422,7 +425,7 @@ importers: version: 3.666.0(@aws-sdk/client-sts@3.666.0) '@getzep/zep-cloud': specifier: 1.0.12 - version: 1.0.12(@langchain/core@0.3.15(openai@4.69.0(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13)(langchain@0.3.5(7umjwzmwnymi4lyinuvazmp6ki)) + version: 1.0.12(@langchain/core@0.3.15(openai@4.69.0(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13)(langchain@0.3.5(4ubssgvn2k3t3hxnzmxuoc2aja)) '@getzep/zep-js': specifier: 0.9.0 version: 0.9.0 @@ -449,7 +452,7 @@ importers: version: 0.3.1(@aws-sdk/client-sso-oidc@3.666.0(@aws-sdk/client-sts@3.666.0))(@langchain/core@0.3.15(openai@4.69.0(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13) '@langchain/community': specifier: 0.3.11 - version: 0.3.11(ndajhtzj4xqxxpqz4t56suvqri) + version: 0.3.11(qw65fvnztmfuymkskopqa7kjky) '@langchain/core': specifier: 'catalog:' version: 0.3.15(openai@4.69.0(encoding@0.1.13)(zod@3.23.8)) @@ -536,7 +539,7 @@ importers: version: 23.0.1 langchain: specifier: 0.3.5 - version: 0.3.5(7umjwzmwnymi4lyinuvazmp6ki) + version: 0.3.5(4ubssgvn2k3t3hxnzmxuoc2aja) lodash: specifier: 'catalog:' version: 4.17.21 @@ -795,7 +798,7 @@ importers: version: 1.11.0 axios: specifier: 'catalog:' - version: 1.7.4 + version: 1.7.4(debug@4.3.7) bcryptjs: specifier: 2.4.3 version: 2.4.3 @@ -1126,7 +1129,7 @@ importers: version: 1.11.0 axios: specifier: 'catalog:' - version: 1.7.4 + version: 1.7.4(debug@4.3.7) concat-stream: specifier: 2.0.0 version: 2.0.0 @@ -1142,6 +1145,9 @@ importers: form-data: specifier: 'catalog:' version: 4.0.0 + iconv-lite: + specifier: 'catalog:' + version: 0.6.3 lodash: specifier: 'catalog:' version: 4.17.21 @@ -1416,7 +1422,7 @@ importers: version: 10.11.0(vue@3.5.11(typescript@5.7.2)) axios: specifier: 'catalog:' - version: 1.7.4 + version: 1.7.4(debug@4.3.7) bowser: specifier: 2.11.0 version: 2.11.0 @@ -1678,7 +1684,7 @@ importers: specifier: 9.0.5 version: 9.0.5 iconv-lite: - specifier: 0.6.3 + specifier: 'catalog:' version: 0.6.3 ics: specifier: 2.40.0 @@ -1899,7 +1905,7 @@ importers: version: 0.15.2 axios: specifier: 'catalog:' - version: 1.7.4 + version: 1.7.4(debug@4.3.7) callsites: specifier: 3.1.0 version: 3.1.0 @@ -14121,7 +14127,7 @@ snapshots: '@gar/promisify@1.1.3': optional: true - '@getzep/zep-cloud@1.0.12(@langchain/core@0.3.15(openai@4.69.0(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13)(langchain@0.3.5(7umjwzmwnymi4lyinuvazmp6ki))': + '@getzep/zep-cloud@1.0.12(@langchain/core@0.3.15(openai@4.69.0(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13)(langchain@0.3.5(4ubssgvn2k3t3hxnzmxuoc2aja))': dependencies: form-data: 4.0.0 node-fetch: 2.7.0(encoding@0.1.13) @@ -14130,7 +14136,7 @@ snapshots: zod: 3.23.8 optionalDependencies: '@langchain/core': 0.3.15(openai@4.69.0(encoding@0.1.13)(zod@3.23.8)) - langchain: 0.3.5(7umjwzmwnymi4lyinuvazmp6ki) + langchain: 0.3.5(4ubssgvn2k3t3hxnzmxuoc2aja) transitivePeerDependencies: - encoding @@ -14597,7 +14603,7 @@ snapshots: - aws-crt - encoding - '@langchain/community@0.3.11(ndajhtzj4xqxxpqz4t56suvqri)': + '@langchain/community@0.3.11(qw65fvnztmfuymkskopqa7kjky)': dependencies: '@ibm-cloud/watsonx-ai': 1.1.2 '@langchain/core': 0.3.15(openai@4.69.0(encoding@0.1.13)(zod@3.23.8)) @@ -14607,7 +14613,7 @@ snapshots: flat: 5.0.2 ibm-cloud-sdk-core: 5.1.0 js-yaml: 4.1.0 - langchain: 0.3.5(7umjwzmwnymi4lyinuvazmp6ki) + langchain: 0.3.5(4ubssgvn2k3t3hxnzmxuoc2aja) langsmith: 0.2.3(openai@4.69.0(encoding@0.1.13)(zod@3.23.8)) uuid: 10.0.0 zod: 3.23.8 @@ -14620,7 +14626,7 @@ snapshots: '@aws-sdk/client-s3': 3.666.0 '@aws-sdk/credential-provider-node': 3.666.0(@aws-sdk/client-sso-oidc@3.666.0(@aws-sdk/client-sts@3.666.0))(@aws-sdk/client-sts@3.666.0) '@azure/storage-blob': 12.18.0(encoding@0.1.13) - '@getzep/zep-cloud': 1.0.12(@langchain/core@0.3.15(openai@4.69.0(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13)(langchain@0.3.5(7umjwzmwnymi4lyinuvazmp6ki)) + '@getzep/zep-cloud': 1.0.12(@langchain/core@0.3.15(openai@4.69.0(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13)(langchain@0.3.5(4ubssgvn2k3t3hxnzmxuoc2aja)) '@getzep/zep-js': 0.9.0 '@google-ai/generativelanguage': 2.6.0(encoding@0.1.13) '@google-cloud/storage': 7.12.1(encoding@0.1.13) @@ -15301,7 +15307,7 @@ snapshots: '@rudderstack/rudder-sdk-node@2.0.9(tslib@2.6.2)': dependencies: - axios: 1.7.4 + axios: 1.7.4(debug@4.3.7) axios-retry: 3.7.0 component-type: 1.2.1 join-component: 1.1.0 @@ -17557,17 +17563,9 @@ snapshots: '@babel/runtime': 7.24.7 is-retry-allowed: 2.2.0 - axios@1.7.4: - dependencies: - follow-redirects: 1.15.6(debug@4.3.6) - form-data: 4.0.0 - proxy-from-env: 1.1.0 - transitivePeerDependencies: - - debug - axios@1.7.4(debug@4.3.7): dependencies: - follow-redirects: 1.15.6(debug@4.3.7) + follow-redirects: 1.15.6(debug@4.3.6) form-data: 4.0.0 proxy-from-env: 1.1.0 transitivePeerDependencies: @@ -21409,7 +21407,7 @@ snapshots: kuler@2.0.0: {} - langchain@0.3.5(7umjwzmwnymi4lyinuvazmp6ki): + langchain@0.3.5(4ubssgvn2k3t3hxnzmxuoc2aja): dependencies: '@langchain/core': 0.3.15(openai@4.69.0(encoding@0.1.13)(zod@3.23.8)) '@langchain/openai': 0.3.11(@langchain/core@0.3.15(openai@4.69.0(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13) @@ -21433,7 +21431,7 @@ snapshots: '@langchain/groq': 0.1.2(@langchain/core@0.3.15(openai@4.69.0(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13) '@langchain/mistralai': 0.1.1(@langchain/core@0.3.15(openai@4.69.0(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13) '@langchain/ollama': 0.1.1(@langchain/core@0.3.15(openai@4.69.0(encoding@0.1.13)(zod@3.23.8))) - axios: 1.7.4 + axios: 1.7.4(debug@4.3.7) cheerio: 1.0.0 handlebars: 4.7.8 transitivePeerDependencies: diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 8b692007af..306d776861 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -18,6 +18,7 @@ catalog: fast-glob: 3.2.12 flatted: 3.2.7 form-data: 4.0.0 + iconv-lite: 0.6.3 lodash: 4.17.21 luxon: 3.4.4 nanoid: 3.3.6