fix(HTTP Request Node): Handle response errors correctly when continueOnFail is set (#18207)

Co-authored-by: Michael Kret <michael.k@radency.com>

packages/nodes-base/nodes/HttpRequest/V3/HttpRequestV3.node.ts

@@ -748,276 +748,289 @@ export class HttpRequestV3 implements INodeType {
 
 		let responseData: any;
 		for (let itemIndex = 0; itemIndex < items.length; itemIndex++) {
-			responseData = promisesResponses.shift();
+			try {
+				responseData = promisesResponses.shift();
 
-			if (errorItems[itemIndex]) {
+				if (errorItems[itemIndex]) {
-				returnItems.push({
-					json: { error: errorItems[itemIndex] },
-					pairedItem: { item: itemIndex },
-				});
-
-				continue;
-			}
-
-			if (responseData!.status !== 'fulfilled') {
-				if (responseData.reason.statusCode === 429) {
-					responseData.reason.message =
-						"Try spacing your requests out using the batching settings under 'Options'";
-				}
-				if (!this.continueOnFail()) {
-					if (autoDetectResponseFormat && responseData.reason.error instanceof Buffer) {
-						responseData.reason.error = Buffer.from(responseData.reason.error as Buffer).toString();
-					}
-
-					let error;
-					if (responseData?.reason instanceof NodeApiError) {
-						error = responseData.reason;
-						set(error, 'context.itemIndex', itemIndex);
-					} else {
-						const errorData = (
-							responseData.reason ? responseData.reason : responseData
-						) as JsonObject;
-						error = new NodeApiError(this.getNode(), errorData, { itemIndex });
-					}
-
-					set(error, 'context.request', sanitizedRequests[itemIndex]);
-
-					throw error;
-				} else {
-					removeCircularRefs(responseData.reason as JsonObject);
-					// Return the actual reason as error
 					returnItems.push({
-						json: {
+						json: { error: errorItems[itemIndex] },
-							error: responseData.reason,
+						pairedItem: { item: itemIndex },
-						},
-						pairedItem: {
-							item: itemIndex,
-						},
 					});
+
 					continue;
 				}
-			}
 
-			let responses: any[];
+				if (responseData!.status !== 'fulfilled') {
-			if (Array.isArray(responseData.value)) {
+					if (responseData.reason.statusCode === 429) {
-				responses = responseData.value;
+						responseData.reason.message =
-			} else {
+							"Try spacing your requests out using the batching settings under 'Options'";
-				responses = [responseData.value];
-			}
-
-			let responseFormat = this.getNodeParameter(
-				'options.response.response.responseFormat',
-				0,
-				'autodetect',
-			) as string;
-
-			fullResponse = this.getNodeParameter(
-				'options.response.response.fullResponse',
-				0,
-				false,
-			) as boolean;
-
-			// eslint-disable-next-line prefer-const
-			for (let [index, response] of Object.entries(responses)) {
-				if (response?.request?.constructor.name === 'ClientRequest') delete response.request;
-
-				if (this.getMode() === 'manual' && index === '0') {
-					// For manual executions save the first response in the context
-					// so that we can use it in the frontend and so make it easier for
-					// the users to create the required pagination expressions
-					const nodeContext = this.getContext('node');
-					if (pagination && pagination.paginationMode !== 'off') {
-						nodeContext.response = responseData.value[0];
-					} else {
-						nodeContext.response = responseData.value;
 					}
-				}
+					if (!this.continueOnFail()) {
-
+						if (autoDetectResponseFormat && responseData.reason.error instanceof Buffer) {
-				const responseContentType = response.headers['content-type'] ?? '';
+							responseData.reason.error = Buffer.from(
-				if (autoDetectResponseFormat) {
+								responseData.reason.error as Buffer,
-					if (responseContentType.includes('application/json')) {
+							).toString();
-						responseFormat = 'json';
-						if (!response.__bodyResolved) {
-							const neverError = this.getNodeParameter(
-								'options.response.response.neverError',
-								0,
-								false,
-							) as boolean;
-
-							const data = await this.helpers.binaryToString(response.body as Buffer | Readable);
-							response.body = jsonParse(data, {
-								...(neverError
-									? { fallbackValue: {} }
-									: { errorMessage: 'Invalid JSON in response body' }),
-							});
-						}
-					} else if (binaryContentTypes.some((e) => responseContentType.includes(e))) {
-						responseFormat = 'file';
-					} else {
-						responseFormat = 'text';
-						if (!response.__bodyResolved) {
-							const data = await this.helpers.binaryToString(response.body as Buffer | Readable);
-							response.body = !data ? undefined : data;
-						}
-					}
-				}
-				// This is a no-op outside of tool usage
-				const optimizeResponse = configureResponseOptimizer(this, itemIndex);
-
-				if (autoDetectResponseFormat && !fullResponse) {
-					delete response.headers;
-					delete response.statusCode;
-					delete response.statusMessage;
-				}
-				if (!fullResponse) {
-					response = optimizeResponse(response.body);
-				} else {
-					response.body = optimizeResponse(response.body);
-				}
-				if (responseFormat === 'file') {
-					const outputPropertyName = this.getNodeParameter(
-						'options.response.response.outputPropertyName',
-						0,
-						'data',
-					) as string;
-
-					const newItem: INodeExecutionData = {
-						json: {},
-						binary: {},
-						pairedItem: {
-							item: itemIndex,
-						},
-					};
-
-					if (items[itemIndex].binary !== undefined) {
-						// Create a shallow copy of the binary data so that the old
-						// data references which do not get changed still stay behind
-						// but the incoming data does not get changed.
-						Object.assign(newItem.binary as IBinaryKeyData, items[itemIndex].binary);
-					}
-
-					let binaryData: Buffer | Readable;
-					if (fullResponse) {
-						const returnItem: IDataObject = {};
-						for (const property of fullResponseProperties) {
-							if (property === 'body') {
-								continue;
-							}
-							returnItem[property] = response[property];
 						}
 
-						newItem.json = returnItem;
+						let error;
-						binaryData = response?.body;
+						if (responseData?.reason instanceof NodeApiError) {
-					} else {
+							error = responseData.reason;
-						newItem.json = items[itemIndex].json;
+							set(error, 'context.itemIndex', itemIndex);
-						binaryData = response;
+						} else {
-					}
+							const errorData = (
-					const preparedBinaryData = await this.helpers.prepareBinaryData(
+								responseData.reason ? responseData.reason : responseData
-						binaryData,
+							) as JsonObject;
-						undefined,
+							error = new NodeApiError(this.getNode(), errorData, { itemIndex });
-						mimeTypeFromResponse(responseContentType),
-					);
-
-					preparedBinaryData.fileName = setFilename(
-						preparedBinaryData,
-						requestOptions,
-						responseFileName,
-					);
-
-					newItem.binary![outputPropertyName] = preparedBinaryData;
-
-					returnItems.push(newItem);
-				} else if (responseFormat === 'text') {
-					const outputPropertyName = this.getNodeParameter(
-						'options.response.response.outputPropertyName',
-						0,
-						'data',
-					) as string;
-					if (fullResponse) {
-						const returnItem: IDataObject = {};
-						for (const property of fullResponseProperties) {
-							if (property === 'body') {
-								returnItem[outputPropertyName] = toText(response[property]);
-								continue;
-							}
-							returnItem[property] = response[property];
 						}
-						returnItems.push({
+
-							json: returnItem,
+						set(error, 'context.request', sanitizedRequests[itemIndex]);
-							pairedItem: {
+
-								item: itemIndex,
+						throw error;
-							},
-						});
 					} else {
+						removeCircularRefs(responseData.reason as JsonObject);
+						// Return the actual reason as error
 						returnItems.push({
 							json: {
-								[outputPropertyName]: toText(response),
+								error: responseData.reason,
 							},
 							pairedItem: {
 								item: itemIndex,
 							},
 						});
+						continue;
 					}
-				} else {
+				}
-					// responseFormat: 'json'
-					if (fullResponse) {
-						const returnItem: IDataObject = {};
-						for (const property of fullResponseProperties) {
-							returnItem[property] = response[property];
-						}
 
-						if (responseFormat === 'json' && typeof returnItem.body === 'string') {
+				let responses: any[];
-							try {
+				if (Array.isArray(responseData.value)) {
-								returnItem.body = JSON.parse(returnItem.body);
+					responses = responseData.value;
-							} catch (error) {
+				} else {
-								throw new NodeOperationError(
+					responses = [responseData.value];
-									this.getNode(),
+				}
-									'Response body is not valid JSON. Change "Response Format" to "Text"',
+
-									{ itemIndex },
+				let responseFormat = this.getNodeParameter(
-								);
+					'options.response.response.responseFormat',
+					0,
+					'autodetect',
+				) as string;
+
+				fullResponse = this.getNodeParameter(
+					'options.response.response.fullResponse',
+					0,
+					false,
+				) as boolean;
+
+				// eslint-disable-next-line prefer-const
+				for (let [index, response] of Object.entries(responses)) {
+					if (response?.request?.constructor.name === 'ClientRequest') delete response.request;
+
+					if (this.getMode() === 'manual' && index === '0') {
+						// For manual executions save the first response in the context
+						// so that we can use it in the frontend and so make it easier for
+						// the users to create the required pagination expressions
+						const nodeContext = this.getContext('node');
+						if (pagination && pagination.paginationMode !== 'off') {
+							nodeContext.response = responseData.value[0];
+						} else {
+							nodeContext.response = responseData.value;
+						}
+					}
+
+					const responseContentType = response.headers['content-type'] ?? '';
+					if (autoDetectResponseFormat) {
+						if (responseContentType.includes('application/json')) {
+							responseFormat = 'json';
+							if (!response.__bodyResolved) {
+								const neverError = this.getNodeParameter(
+									'options.response.response.neverError',
+									0,
+									false,
+								) as boolean;
+
+								const data = await this.helpers.binaryToString(response.body as Buffer | Readable);
+								response.body = jsonParse(data, {
+									...(neverError
+										? { fallbackValue: {} }
+										: { errorMessage: 'Invalid JSON in response body' }),
+								});
+							}
+						} else if (binaryContentTypes.some((e) => responseContentType.includes(e))) {
+							responseFormat = 'file';
+						} else {
+							responseFormat = 'text';
+							if (!response.__bodyResolved) {
+								const data = await this.helpers.binaryToString(response.body as Buffer | Readable);
+								response.body = !data ? undefined : data;
 							}
 						}
+					}
+					// This is a no-op outside of tool usage
+					const optimizeResponse = configureResponseOptimizer(this, itemIndex);
 
-						returnItems.push({
+					if (autoDetectResponseFormat && !fullResponse) {
-							json: returnItem,
+						delete response.headers;
+						delete response.statusCode;
+						delete response.statusMessage;
+					}
+					if (!fullResponse) {
+						response = optimizeResponse(response.body);
+					} else {
+						response.body = optimizeResponse(response.body);
+					}
+					if (responseFormat === 'file') {
+						const outputPropertyName = this.getNodeParameter(
+							'options.response.response.outputPropertyName',
+							0,
+							'data',
+						) as string;
+
+						const newItem: INodeExecutionData = {
+							json: {},
+							binary: {},
 							pairedItem: {
 								item: itemIndex,
 							},
-						});
+						};
-					} else {
+
-						if (responseFormat === 'json' && typeof response === 'string') {
+						if (items[itemIndex].binary !== undefined) {
-							try {
+							// Create a shallow copy of the binary data so that the old
-								if (typeof response !== 'object') {
+							// data references which do not get changed still stay behind
-									response = JSON.parse(response);
+							// but the incoming data does not get changed.
-								}
+							Object.assign(newItem.binary as IBinaryKeyData, items[itemIndex].binary);
-							} catch (error) {
-								throw new NodeOperationError(
-									this.getNode(),
-									'Response body is not valid JSON. Change "Response Format" to "Text"',
-									{ itemIndex },
-								);
-							}
 						}
 
-						if (Array.isArray(response)) {
+						let binaryData: Buffer | Readable;
-							response.forEach((item) =>
+						if (fullResponse) {
-								returnItems.push({
+							const returnItem: IDataObject = {};
-									json: item,
+							for (const property of fullResponseProperties) {
-									pairedItem: {
+								if (property === 'body') {
-										item: itemIndex,
+									continue;
-									},
+								}
-								}),
+								returnItem[property] = response[property];
-							);
+							}
+
+							newItem.json = returnItem;
+							binaryData = response?.body;
+						} else {
+							newItem.json = items[itemIndex].json;
+							binaryData = response;
+						}
+						const preparedBinaryData = await this.helpers.prepareBinaryData(
+							binaryData,
+							undefined,
+							mimeTypeFromResponse(responseContentType),
+						);
+
+						preparedBinaryData.fileName = setFilename(
+							preparedBinaryData,
+							requestOptions,
+							responseFileName,
+						);
+
+						newItem.binary![outputPropertyName] = preparedBinaryData;
+
+						returnItems.push(newItem);
+					} else if (responseFormat === 'text') {
+						const outputPropertyName = this.getNodeParameter(
+							'options.response.response.outputPropertyName',
+							0,
+							'data',
+						) as string;
+						if (fullResponse) {
+							const returnItem: IDataObject = {};
+							for (const property of fullResponseProperties) {
+								if (property === 'body') {
+									returnItem[outputPropertyName] = toText(response[property]);
+									continue;
+								}
+								returnItem[property] = response[property];
+							}
+							returnItems.push({
+								json: returnItem,
+								pairedItem: {
+									item: itemIndex,
+								},
+							});
 						} else {
 							returnItems.push({
-								json: response,
+								json: {
+									[outputPropertyName]: toText(response),
+								},
 								pairedItem: {
 									item: itemIndex,
 								},
 							});
 						}
+					} else {
+						// responseFormat: 'json'
+						if (fullResponse) {
+							const returnItem: IDataObject = {};
+							for (const property of fullResponseProperties) {
+								returnItem[property] = response[property];
+							}
+
+							if (responseFormat === 'json' && typeof returnItem.body === 'string') {
+								try {
+									returnItem.body = JSON.parse(returnItem.body);
+								} catch (error) {
+									throw new NodeOperationError(
+										this.getNode(),
+										'Response body is not valid JSON. Change "Response Format" to "Text"',
+										{ itemIndex },
+									);
+								}
+							}
+
+							returnItems.push({
+								json: returnItem,
+								pairedItem: {
+									item: itemIndex,
+								},
+							});
+						} else {
+							if (responseFormat === 'json' && typeof response === 'string') {
+								try {
+									if (typeof response !== 'object') {
+										response = JSON.parse(response);
+									}
+								} catch (error) {
+									throw new NodeOperationError(
+										this.getNode(),
+										'Response body is not valid JSON. Change "Response Format" to "Text"',
+										{ itemIndex },
+									);
+								}
+							}
+
+							if (Array.isArray(response)) {
+								response.forEach((item) =>
+									returnItems.push({
+										json: item,
+										pairedItem: {
+											item: itemIndex,
+										},
+									}),
+								);
+							} else {
+								returnItems.push({
+									json: response,
+									pairedItem: {
+										item: itemIndex,
+									},
+								});
+							}
+						}
 					}
 				}
+			} catch (error) {
+				if (!this.continueOnFail()) throw error;
+
+				returnItems.push({
+					json: { error: error.message },
+					pairedItem: { item: itemIndex },
+				});
+
+				continue;
 			}
 		}
 

packages/nodes-base/nodes/HttpRequest/test/node/HttpRequest.test.ts

@@ -106,6 +106,7 @@ describe('Test HTTP Request Node', () => {
 				completed: false,
 				userId: 15,
 			});
+		nock(baseUrl).get('/html').reply(200, '<html><body><h1>Test</h1></body></html>');
 
 		//PUT
 		nock(baseUrl).put('/todos/10', { userId: '42' }).reply(200, {

packages/nodes-base/nodes/HttpRequest/test/node/workflow.use_error_output.json

@@ -3,53 +3,95 @@
 	"nodes": [
 		{
 			"parameters": {},
-			"id": "6e15f2de-79fe-41f3-b76e-53ebfa2e4437",
+			"id": "6707decf-7ae3-46f1-8603-b0fe4844f240",
 			"name": "When clicking ‘Execute workflow’",
 			"type": "n8n-nodes-base.manualTrigger",
 			"typeVersion": 1,
-			"position": [-60, 580]
+			"position": [16, 560]
 		},
 		{
 			"parameters": {},
-			"id": "af1bb989-3c6d-4323-8e88-649e45d64c00",
+			"id": "09b63a84-789d-4a31-b546-849ba47ed689",
 			"name": "Success path",
 			"type": "n8n-nodes-base.noOp",
 			"typeVersion": 1,
-			"position": [460, 460]
+			"position": [464, 272]
-		},
-		{
-			"parameters": {},
-			"id": "43c4ee19-6a9d-4b1d-aefa-2c24abe45189",
-			"name": "Error path",
-			"type": "n8n-nodes-base.noOp",
-			"typeVersion": 1,
-			"position": [460, 700]
 		},
 		{
 			"parameters": {
 				"method": "POST",
-				"url": "https://webhook.site/e18fe8f9-ec77-4574-a40d-8ae054191e1e",
+				"url": "https://dummyjson.com/todos/1",
 				"sendBody": true,
 				"specifyBody": "json",
 				"jsonBody": "{\n  \"q\": \"abc\",\n}",
 				"options": {}
 			},
-			"id": "31641920-7f43-473f-ad96-b121122802bb",
+			"id": "c6841eb1-7913-4c8d-8c9d-b88a908125ed",
-			"name": "HTTP Request",
+			"name": "Invalid JSON Body",
 			"type": "n8n-nodes-base.httpRequest",
 			"typeVersion": 4.2,
-			"position": [180, 580],
+			"position": [240, 368],
 			"alwaysOutputData": false,
 			"onError": "continueErrorOutput"
+		},
+		{
+			"parameters": {},
+			"id": "21750b7e-c18a-43a5-a068-f8aa85d1cadf",
+			"name": "Success path1",
+			"type": "n8n-nodes-base.noOp",
+			"typeVersion": 1,
+			"position": [464, 656]
+		},
+		{
+			"parameters": {
+				"url": "https://dummyjson.com/html",
+				"options": {
+					"response": {
+						"response": {
+							"responseFormat": "json"
+						}
+					}
+				}
+			},
+			"id": "ae2891f0-1968-4f57-a1a0-87d6f6dab57d",
+			"name": "Invalid JSON Response",
+			"type": "n8n-nodes-base.httpRequest",
+			"typeVersion": 4.2,
+			"position": [240, 752],
+			"alwaysOutputData": false,
+			"onError": "continueErrorOutput"
+		},
+		{
+			"parameters": {},
+			"id": "baa75f00-e0dd-40b2-b718-1e8311549a05",
+			"name": "Request body error",
+			"type": "n8n-nodes-base.noOp",
+			"typeVersion": 1,
+			"position": [464, 464]
+		},
+		{
+			"parameters": {},
+			"id": "4733c47f-38c8-44a8-9d77-d9e733777cbf",
+			"name": "Response body error",
+			"type": "n8n-nodes-base.noOp",
+			"typeVersion": 1,
+			"position": [464, 848]
 		}
 	],
 	"pinData": {
-		"Error path": [
+		"Request body error": [
 			{
 				"json": {
 					"error": "JSON parameter needs to be valid JSON"
 				}
 			}
+		],
+		"Response body error": [
+			{
+				"json": {
+					"error": "Response body is not valid JSON. Change \"Response Format\" to \"Text\""
+				}
+			}
 		]
 	},
 	"connections": {
@@ -57,14 +99,19 @@
 			"main": [
 				[
 					{
-						"node": "HTTP Request",
+						"node": "Invalid JSON Body",
+						"type": "main",
+						"index": 0
+					},
+					{
+						"node": "Invalid JSON Response",
 						"type": "main",
 						"index": 0
 					}
 				]
 			]
 		},
-		"HTTP Request": {
+		"Invalid JSON Body": {
 			"main": [
 				[
 					{
@@ -75,7 +122,25 @@
 				],
 				[
 					{
-						"node": "Error path",
+						"node": "Request body error",
+						"type": "main",
+						"index": 0
+					}
+				]
+			]
+		},
+		"Invalid JSON Response": {
+			"main": [
+				[
+					{
+						"node": "Success path1",
+						"type": "main",
+						"index": 0
+					}
+				],
+				[
+					{
+						"node": "Response body error",
 						"type": "main",
 						"index": 0
 					}
@@ -87,11 +152,10 @@
 	"settings": {
 		"executionOrder": "v1"
 	},
-	"versionId": "f445a6e9-818f-4b70-8b4f-e2a68fc5d6e4",
+	"versionId": "c7026310-8c4f-4889-a45b-befebacc7dde",
 	"meta": {
-		"templateCredsSetupCompleted": true,
+		"instanceId": "27cc9b56542ad45b38725555722c50a1c3fee1670bbb67980558314ee08517c4"
-		"instanceId": "be251a83c052a9862eeac953816fbb1464f89dfbf79d7ac490a8e336a8cc8bfd"
 	},
-	"id": "f3rDILaMkFqisP3P",
+	"id": "2Zd3If2j9PglrHri",
 	"tags": []
 }