mirror of
https://github.com/Abdulazizzn/n8n-enterprise-unlocked.git
synced 2025-12-18 10:31:15 +00:00
refactor(core): Parse Webhook request bodies on-demand (#6394)
Also, 1. Consistent CORS support ~on all three webhook types~ waiting webhooks never supported CORS. I'll fix that in another PR 2. [Fixes binary-data handling when request body is text, json, or xml](https://linear.app/n8n/issue/NODE-505/webhook-binary-data-handling-fails-for-textplain-files). 3. Reduced number of middleware that each request has to go through. 4. Removed the need to maintain webhook endpoints in the auth-exception list. 5. Skip all middlewares (apart from `compression`) on Webhook routes. 6. move `multipart/form-data` support out of individual nodes 7. upgrade `formidable` 8. fix the filenames on binary-data in webhooks nodes 9. add unit tests and integration tests for webhook request handling, and increase test coverage
This commit is contained in:
कारतोफ्फेलस्क्रिप्ट™
committed by
GitHub
GitHub
parent
369a2e9796
commit
31d8f478ee
@@ -1,40 +1,28 @@
|
||||
import { Container } from 'typedi';
|
||||
import { readFile } from 'fs/promises';
|
||||
import type { Server } from 'http';
|
||||
import type { Url } from 'url';
|
||||
import express from 'express';
|
||||
import bodyParser from 'body-parser';
|
||||
import bodyParserXml from 'body-parser-xml';
|
||||
import compression from 'compression';
|
||||
import parseUrl from 'parseurl';
|
||||
import type { RedisOptions } from 'ioredis';
|
||||
|
||||
import type { WebhookHttpMethod } from 'n8n-workflow';
|
||||
import { LoggerProxy } from 'n8n-workflow';
|
||||
import config from '@/config';
|
||||
import { N8N_VERSION, inDevelopment } from '@/constants';
|
||||
import { N8N_VERSION, inDevelopment, inTest } from '@/constants';
|
||||
import { ActiveWorkflowRunner } from '@/ActiveWorkflowRunner';
|
||||
import * as Db from '@/Db';
|
||||
import type { IExternalHooksClass } from '@/Interfaces';
|
||||
import { ExternalHooks } from '@/ExternalHooks';
|
||||
import {
|
||||
send,
|
||||
sendErrorResponse,
|
||||
sendSuccessResponse,
|
||||
ServiceUnavailableError,
|
||||
} from '@/ResponseHelper';
|
||||
import { corsMiddleware } from '@/middlewares';
|
||||
import { send, sendErrorResponse, ServiceUnavailableError } from '@/ResponseHelper';
|
||||
import { rawBody, jsonParser, corsMiddleware } from '@/middlewares';
|
||||
import { TestWebhooks } from '@/TestWebhooks';
|
||||
import { WaitingWebhooks } from '@/WaitingWebhooks';
|
||||
import { WEBHOOK_METHODS } from '@/WebhookHelpers';
|
||||
import { getRedisClusterNodes } from './GenericHelpers';
|
||||
|
||||
const emptyBuffer = Buffer.alloc(0);
|
||||
import { webhookRequestHandler } from '@/WebhookHelpers';
|
||||
|
||||
export abstract class AbstractServer {
|
||||
protected server: Server;
|
||||
|
||||
protected app: express.Application;
|
||||
readonly app: express.Application;
|
||||
|
||||
protected externalHooks: IExternalHooksClass;
|
||||
|
||||
@@ -58,7 +46,9 @@ export abstract class AbstractServer {
|
||||
|
||||
protected instanceId = '';
|
||||
|
||||
abstract configure(): Promise<void>;
|
||||
protected webhooksEnabled = true;
|
||||
|
||||
protected testWebhooksEnabled = false;
|
||||
|
||||
constructor() {
|
||||
this.app = express();
|
||||
@@ -76,6 +66,10 @@ export abstract class AbstractServer {
|
||||
this.endpointWebhookWaiting = config.getEnv('endpoints.webhookWaiting');
|
||||
}
|
||||
|
||||
async configure(): Promise<void> {
|
||||
// Additional configuration in derived classes
|
||||
}
|
||||
|
||||
private async setupErrorHandlers() {
|
||||
const { app } = this;
|
||||
|
||||
@@ -87,66 +81,12 @@ export abstract class AbstractServer {
|
||||
app.use(errorHandler());
|
||||
}
|
||||
|
||||
private async setupCommonMiddlewares() {
|
||||
const { app } = this;
|
||||
|
||||
private setupCommonMiddlewares() {
|
||||
// Compress the response data
|
||||
app.use(compression());
|
||||
this.app.use(compression());
|
||||
|
||||
// Make sure that each request has the "parsedUrl" parameter
|
||||
app.use((req, res, next) => {
|
||||
req.parsedUrl = parseUrl(req)!;
|
||||
req.rawBody = emptyBuffer;
|
||||
next();
|
||||
});
|
||||
|
||||
const payloadSizeMax = config.getEnv('endpoints.payloadSizeMax');
|
||||
|
||||
// Support application/json type post data
|
||||
app.use(
|
||||
bodyParser.json({
|
||||
limit: `${payloadSizeMax}mb`,
|
||||
verify: (req, res, buf) => {
|
||||
req.rawBody = buf;
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
// Support application/xml type post data
|
||||
bodyParserXml(bodyParser);
|
||||
app.use(
|
||||
bodyParser.xml({
|
||||
limit: `${payloadSizeMax}mb`,
|
||||
xmlParseOptions: {
|
||||
normalize: true, // Trim whitespace inside text nodes
|
||||
normalizeTags: true, // Transform tags to lowercase
|
||||
explicitArray: false, // Only put properties in array if length > 1
|
||||
},
|
||||
verify: (req, res, buf) => {
|
||||
req.rawBody = buf;
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
app.use(
|
||||
bodyParser.text({
|
||||
limit: `${payloadSizeMax}mb`,
|
||||
verify: (req, res, buf) => {
|
||||
req.rawBody = buf;
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
// support application/x-www-form-urlencoded post data
|
||||
app.use(
|
||||
bodyParser.urlencoded({
|
||||
limit: `${payloadSizeMax}mb`,
|
||||
extended: false,
|
||||
verify: (req, res, buf) => {
|
||||
req.rawBody = buf;
|
||||
},
|
||||
}),
|
||||
);
|
||||
// Read incoming data into `rawBody`
|
||||
this.app.use(rawBody);
|
||||
}
|
||||
|
||||
private setupDevMiddlewares() {
|
||||
@@ -246,163 +186,6 @@ export abstract class AbstractServer {
|
||||
});
|
||||
}
|
||||
|
||||
// ----------------------------------------
|
||||
// Regular Webhooks
|
||||
// ----------------------------------------
|
||||
protected setupWebhookEndpoint() {
|
||||
const endpoint = this.endpointWebhook;
|
||||
const activeWorkflowRunner = this.activeWorkflowRunner;
|
||||
|
||||
// Register all webhook requests
|
||||
this.app.all(`/${endpoint}/*`, async (req, res) => {
|
||||
// Cut away the "/webhook/" to get the registered part of the url
|
||||
const requestUrl = req.parsedUrl.pathname!.slice(endpoint.length + 2);
|
||||
|
||||
const method = req.method.toUpperCase() as WebhookHttpMethod;
|
||||
if (method === 'OPTIONS') {
|
||||
let allowedMethods: string[];
|
||||
try {
|
||||
allowedMethods = await activeWorkflowRunner.getWebhookMethods(requestUrl);
|
||||
allowedMethods.push('OPTIONS');
|
||||
|
||||
// Add custom "Allow" header to satisfy OPTIONS response.
|
||||
res.append('Allow', allowedMethods);
|
||||
} catch (error) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-argument
|
||||
sendErrorResponse(res, error);
|
||||
return;
|
||||
}
|
||||
|
||||
res.header('Access-Control-Allow-Origin', '*');
|
||||
|
||||
sendSuccessResponse(res, {}, true, 204);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!WEBHOOK_METHODS.includes(method)) {
|
||||
sendErrorResponse(res, new Error(`The method ${method} is not supported.`));
|
||||
return;
|
||||
}
|
||||
|
||||
let response;
|
||||
try {
|
||||
response = await activeWorkflowRunner.executeWebhook(method, requestUrl, req, res);
|
||||
} catch (error) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-argument
|
||||
sendErrorResponse(res, error);
|
||||
return;
|
||||
}
|
||||
|
||||
if (response.noWebhookResponse === true) {
|
||||
// Nothing else to do as the response got already sent
|
||||
return;
|
||||
}
|
||||
|
||||
sendSuccessResponse(res, response.data, true, response.responseCode, response.headers);
|
||||
});
|
||||
}
|
||||
|
||||
// ----------------------------------------
|
||||
// Waiting Webhooks
|
||||
// ----------------------------------------
|
||||
protected setupWaitingWebhookEndpoint() {
|
||||
const endpoint = this.endpointWebhookWaiting;
|
||||
const waitingWebhooks = Container.get(WaitingWebhooks);
|
||||
|
||||
// Register all webhook-waiting requests
|
||||
this.app.all(`/${endpoint}/*`, async (req, res) => {
|
||||
// Cut away the "/webhook-waiting/" to get the registered part of the url
|
||||
const requestUrl = req.parsedUrl.pathname!.slice(endpoint.length + 2);
|
||||
|
||||
const method = req.method.toUpperCase() as WebhookHttpMethod;
|
||||
|
||||
if (!WEBHOOK_METHODS.includes(method)) {
|
||||
sendErrorResponse(res, new Error(`The method ${method} is not supported.`));
|
||||
return;
|
||||
}
|
||||
|
||||
let response;
|
||||
try {
|
||||
response = await waitingWebhooks.executeWebhook(method, requestUrl, req, res);
|
||||
} catch (error) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-argument
|
||||
sendErrorResponse(res, error);
|
||||
return;
|
||||
}
|
||||
|
||||
if (response.noWebhookResponse === true) {
|
||||
// Nothing else to do as the response got already sent
|
||||
return;
|
||||
}
|
||||
|
||||
sendSuccessResponse(res, response.data, true, response.responseCode, response.headers);
|
||||
});
|
||||
}
|
||||
|
||||
// ----------------------------------------
|
||||
// Testing Webhooks
|
||||
// ----------------------------------------
|
||||
protected setupTestWebhookEndpoint() {
|
||||
const endpoint = this.endpointWebhookTest;
|
||||
const testWebhooks = Container.get(TestWebhooks);
|
||||
|
||||
// Register all test webhook requests (for testing via the UI)
|
||||
this.app.all(`/${endpoint}/*`, async (req, res) => {
|
||||
// Cut away the "/webhook-test/" to get the registered part of the url
|
||||
const requestUrl = req.parsedUrl.pathname!.slice(endpoint.length + 2);
|
||||
|
||||
const method = req.method.toUpperCase() as WebhookHttpMethod;
|
||||
|
||||
if (method === 'OPTIONS') {
|
||||
let allowedMethods: string[];
|
||||
try {
|
||||
allowedMethods = await testWebhooks.getWebhookMethods(requestUrl);
|
||||
allowedMethods.push('OPTIONS');
|
||||
|
||||
// Add custom "Allow" header to satisfy OPTIONS response.
|
||||
res.append('Allow', allowedMethods);
|
||||
} catch (error) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-argument
|
||||
sendErrorResponse(res, error);
|
||||
return;
|
||||
}
|
||||
|
||||
res.header('Access-Control-Allow-Origin', '*');
|
||||
|
||||
sendSuccessResponse(res, {}, true, 204);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!WEBHOOK_METHODS.includes(method)) {
|
||||
sendErrorResponse(res, new Error(`The method ${method} is not supported.`));
|
||||
return;
|
||||
}
|
||||
|
||||
let response;
|
||||
try {
|
||||
response = await testWebhooks.callTestWebhook(method, requestUrl, req, res);
|
||||
} catch (error) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-argument
|
||||
sendErrorResponse(res, error);
|
||||
return;
|
||||
}
|
||||
|
||||
if (response.noWebhookResponse === true) {
|
||||
// Nothing else to do as the response got already sent
|
||||
return;
|
||||
}
|
||||
|
||||
sendSuccessResponse(res, response.data, true, response.responseCode, response.headers);
|
||||
});
|
||||
|
||||
// Removes a test webhook
|
||||
// TODO UM: check if this needs validation with user management.
|
||||
this.app.delete(
|
||||
`/${this.restEndpoint}/test-webhook/:id`,
|
||||
send(async (req) => testWebhooks.cancelTestWebhook(req.params.id)),
|
||||
);
|
||||
}
|
||||
|
||||
async init(): Promise<void> {
|
||||
const { app, protocol, sslKey, sslCert } = this;
|
||||
|
||||
@@ -443,27 +226,60 @@ export abstract class AbstractServer {
|
||||
}
|
||||
|
||||
async start(): Promise<void> {
|
||||
await this.setupErrorHandlers();
|
||||
this.setupPushServer();
|
||||
await this.setupCommonMiddlewares();
|
||||
if (!inTest) {
|
||||
await this.setupErrorHandlers();
|
||||
this.setupPushServer();
|
||||
}
|
||||
|
||||
this.setupCommonMiddlewares();
|
||||
|
||||
// Setup webhook handlers before bodyParser, to let the Webhook node handle binary data in requests
|
||||
if (this.webhooksEnabled) {
|
||||
// Register a handler for active webhooks
|
||||
this.app.all(
|
||||
`/${this.endpointWebhook}/:path(*)`,
|
||||
webhookRequestHandler(Container.get(ActiveWorkflowRunner)),
|
||||
);
|
||||
|
||||
// Register a handler for waiting webhooks
|
||||
this.app.all(
|
||||
`/${this.endpointWebhookWaiting}/:path/:suffix?`,
|
||||
webhookRequestHandler(Container.get(WaitingWebhooks)),
|
||||
);
|
||||
}
|
||||
|
||||
if (this.testWebhooksEnabled) {
|
||||
const testWebhooks = Container.get(TestWebhooks);
|
||||
|
||||
// Register a handler for test webhooks
|
||||
this.app.all(`/${this.endpointWebhookTest}/:path(*)`, webhookRequestHandler(testWebhooks));
|
||||
|
||||
// Removes a test webhook
|
||||
// TODO UM: check if this needs validation with user management.
|
||||
this.app.delete(
|
||||
`/${this.restEndpoint}/test-webhook/:id`,
|
||||
send(async (req) => testWebhooks.cancelTestWebhook(req.params.id)),
|
||||
);
|
||||
}
|
||||
|
||||
if (inDevelopment) {
|
||||
this.setupDevMiddlewares();
|
||||
}
|
||||
|
||||
// Setup JSON parsing middleware after the webhook handlers are setup
|
||||
this.app.use(jsonParser);
|
||||
|
||||
await this.configure();
|
||||
console.log(`Version: ${N8N_VERSION}`);
|
||||
|
||||
const defaultLocale = config.getEnv('defaultLocale');
|
||||
if (defaultLocale !== 'en') {
|
||||
console.log(`Locale: ${defaultLocale}`);
|
||||
if (!inTest) {
|
||||
console.log(`Version: ${N8N_VERSION}`);
|
||||
|
||||
const defaultLocale = config.getEnv('defaultLocale');
|
||||
if (defaultLocale !== 'en') {
|
||||
console.log(`Locale: ${defaultLocale}`);
|
||||
}
|
||||
|
||||
await this.externalHooks.run('n8n.ready', [this, config]);
|
||||
}
|
||||
|
||||
await this.externalHooks.run('n8n.ready', [this, config]);
|
||||
}
|
||||
}
|
||||
|
||||
declare module 'http' {
|
||||
export interface IncomingMessage {
|
||||
parsedUrl: Url;
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user