refactor(core): Parse Webhook request bodies on-demand (#6394)

Also,
1. Consistent CORS support ~on all three webhook types~ waiting webhooks never supported CORS. I'll fix that in another PR
2. [Fixes binary-data handling when request body is text, json, or xml](https://linear.app/n8n/issue/NODE-505/webhook-binary-data-handling-fails-for-textplain-files).
3. Reduced number of middleware that each request has to go through.
4. Removed the need to maintain webhook endpoints in the auth-exception list.
5. Skip all middlewares (apart from `compression`) on Webhook routes. 
6. move `multipart/form-data` support out of individual nodes
7. upgrade `formidable`
8. fix the filenames on binary-data in webhooks nodes
9. add unit tests and integration tests for webhook request handling, and increase test coverage

packages/cli/src/WebhookHelpers.ts

@@ -1,21 +1,20 @@
 /* eslint-disable @typescript-eslint/no-unsafe-argument */
-
 /* eslint-disable @typescript-eslint/prefer-optional-chain */
 /* eslint-disable @typescript-eslint/no-shadow */
 /* eslint-disable @typescript-eslint/no-unsafe-assignment */
 /* eslint-disable id-denylist */
 /* eslint-disable prefer-spread */
-
 /* eslint-disable @typescript-eslint/prefer-nullish-coalescing */
 /* eslint-disable @typescript-eslint/no-unsafe-member-access */
-
 /* eslint-disable @typescript-eslint/restrict-template-expressions */
-
 import type express from 'express';
+import { Container } from 'typedi';
 import get from 'lodash/get';
 import stream from 'stream';
 import { promisify } from 'util';
-import { Container } from 'typedi';
+import { parse as parseQueryString } from 'querystring';
+import { Parser as XmlParser } from 'xml2js';
+import formidable from 'formidable';
 
 import { BinaryDataManager, NodeExecuteFunctions } from 'n8n-core';
 
@@ -26,6 +25,7 @@ import type {
 	IDeferredPromise,
 	IExecuteData,
 	IExecuteResponsePromiseData,
+	IHttpRequestMethods,
 	IN8nHttpFullResponse,
 	INode,
 	IRunExecutionData,
@@ -40,6 +40,7 @@ import {
 	BINARY_ENCODING,
 	createDeferredPromise,
 	ErrorReporterProxy as ErrorReporter,
+	jsonParse,
 	LoggerProxy as Logger,
 	NodeHelpers,
 } from 'n8n-workflow';
@@ -47,8 +48,11 @@ import {
 import type {
 	IExecutionDb,
 	IResponseCallbackData,
+	IWebhookManager,
 	IWorkflowDb,
 	IWorkflowExecutionDataProcess,
+	WebhookCORSRequest,
+	WebhookRequest,
 } from '@/Interfaces';
 import * as GenericHelpers from '@/GenericHelpers';
 import * as ResponseHelper from '@/ResponseHelper';
@@ -63,11 +67,67 @@ import { OwnershipService } from './services/ownership.service';
 
 const pipeline = promisify(stream.pipeline);
 
-export const WEBHOOK_METHODS = ['DELETE', 'GET', 'HEAD', 'PATCH', 'POST', 'PUT'];
+export const WEBHOOK_METHODS: IHttpRequestMethods[] = [
+	'DELETE',
+	'GET',
+	'HEAD',
+	'PATCH',
+	'POST',
+	'PUT',
+];
+
+const xmlParser = new XmlParser({
+	async: true,
+	normalize: true, // Trim whitespace inside text nodes
+	normalizeTags: true, // Transform tags to lowercase
+	explicitArray: false, // Only put properties in array if length > 1
+});
+
+export const webhookRequestHandler =
+	(webhookManager: IWebhookManager) =>
+	async (req: WebhookRequest | WebhookCORSRequest, res: express.Response) => {
+		const { path } = req.params;
+		const method = req.method;
+
+		if (method !== 'OPTIONS' && !WEBHOOK_METHODS.includes(method)) {
+			return ResponseHelper.sendErrorResponse(
+				res,
+				new Error(`The method ${method} is not supported.`),
+			);
+		}
+
+		// Setup CORS headers only if the incoming request has an `origin` header
+		if ('origin' in req.headers) {
+			if (webhookManager.getWebhookMethods) {
+				try {
+					const allowedMethods = await webhookManager.getWebhookMethods(path);
+					res.header('Access-Control-Allow-Methods', ['OPTIONS', ...allowedMethods].join(', '));
+				} catch (error) {
+					return ResponseHelper.sendErrorResponse(res, error as Error);
+				}
+			}
+			res.header('Access-Control-Allow-Origin', req.headers.origin);
+		}
+
+		if (method === 'OPTIONS') {
+			return ResponseHelper.sendSuccessResponse(res, {}, true, 204);
+		}
+
+		let response;
+		try {
+			response = await webhookManager.executeWebhook(req, res);
+		} catch (error) {
+			return ResponseHelper.sendErrorResponse(res, error as Error);
+		}
+
+		// Don't respond, if already responded
+		if (response.noWebhookResponse !== true) {
+			ResponseHelper.sendSuccessResponse(res, response.data, true, response.responseCode);
+		}
+	};
 
 /**
  * Returns all the webhooks which should be created for the given workflow
- *
  */
 export function getWorkflowWebhooks(
 	workflow: Workflow,
@@ -134,9 +194,6 @@ export function encodeWebhookResponse(
 
 /**
  * Executes a webhook
- *
- * @param {(string | undefined)} sessionId
- * @param {((error: Error | null, data: IResponseCallbackData) => void)} responseCallback
  */
 export async function executeWebhook(
 	workflow: Workflow,
@@ -147,7 +204,7 @@ export async function executeWebhook(
 	sessionId: string | undefined,
 	runExecutionData: IRunExecutionData | undefined,
 	executionId: string | undefined,
-	req: express.Request,
+	req: WebhookRequest,
 	res: express.Response,
 	responseCallback: (error: Error | null, data: IResponseCallbackData) => void,
 	destinationNode?: string,
@@ -227,6 +284,16 @@ export async function executeWebhook(
 	additionalData.httpRequest = req;
 	additionalData.httpResponse = res;
 
+	const binaryData = workflow.expression.getSimpleParameterValue(
+		workflowStartNode,
+		'={{$parameter["options"]["binaryData"]}}',
+		executionMode,
+		additionalData.timezone,
+		additionalKeys,
+		undefined,
+		false,
+	);
+
 	let didSendResponse = false;
 	let runExecutionDataMerge = {};
 	try {
@@ -234,6 +301,46 @@ export async function executeWebhook(
 		// the workflow should be executed or not
 		let webhookResultData: IWebhookResponseData;
 
+		// if `Webhook` or `Wait` node, and binaryData is enabled, skip pre-parse the request-body
+		if (!binaryData) {
+			const { contentType, encoding } = req;
+			if (contentType === 'multipart/form-data') {
+				const form = formidable({
+					multiples: true,
+					encoding: encoding as formidable.BufferEncoding,
+					// TODO: pass a custom `fileWriteStreamHandler` to create binary data files directly
+				});
+				req.body = await new Promise((resolve) => {
+					form.parse(req, async (err, data, files) => {
+						resolve({ data, files });
+					});
+				});
+			} else {
+				await req.readRawBody();
+				const { rawBody } = req;
+				if (rawBody?.length) {
+					try {
+						if (contentType === 'application/json') {
+							req.body = jsonParse(rawBody.toString(encoding));
+						} else if (contentType?.endsWith('/xml') || contentType?.endsWith('+xml')) {
+							req.body = await xmlParser.parseStringPromise(rawBody.toString(encoding));
+						} else if (contentType === 'application/x-www-form-urlencoded') {
+							req.body = parseQueryString(rawBody.toString(encoding), undefined, undefined, {
+								maxKeys: 1000,
+							});
+						} else if (contentType === 'text/plain') {
+							req.body = rawBody.toString(encoding);
+						}
+					} catch (error) {
+						throw new ResponseHelper.UnprocessableRequestError(
+							'Failed to parse request body',
+							error.message,
+						);
+					}
+				}
+			}
+		}
+
 		try {
 			webhookResultData = await workflow.runWebhook(
 				webhookData,
@@ -685,11 +792,13 @@ export async function executeWebhook(
 
 		return executionId;
 	} catch (e) {
-		if (!didSendResponse) {
-			responseCallback(new Error('There was a problem executing the workflow'), {});
-		}
-
-		throw new ResponseHelper.InternalServerError(e.message);
+		const error =
+			e instanceof ResponseHelper.UnprocessableRequestError
+				? e
+				: new Error('There was a problem executing the workflow', { cause: e });
+		if (didSendResponse) throw error;
+		responseCallback(error, {});
+		return;
 	}
 }