feat: Add global event bus (#4860)

* fix branch * fix deserialize, add filewriter * add catchAll eventGroup/Name * adding simple Redis sender and receiver to eventbus * remove native node threads * improve eventbus * refactor and simplify * more refactoring and syslog client * more refactor, improved endpoints and eventbus * remove local broker and receivers from mvp * destination de/serialization * create MessageEventBusDestinationEntity * db migrations, load destinations at startup * add delete destination endpoint * pnpm merge and circular import fix * delete destination fix * trigger log file shuffle after size reached * add environment variables for eventbus * reworking event messages * serialize to thread fix * some refactor and lint fixing * add emit to eventbus * cleanup and fix sending unsent * quicksave frontend trial * initial EventTree vue component * basic log streaming settings in vue * http request code merge * create destination settings modals * fix eventmessage options types * credentials are loaded * fix and clean up frontend code * move request code to axios * update lock file * merge fix * fix redis build * move destination interfaces into workflow pkg * revive sentry as destination * migration fixes and frontend cleanup * N8N-5777 / N8N-5789 N8N-5788 * N8N-5784 * N8N-5782 removed event levels * N8N-5790 sentry destination cleanup * N8N-5786 and refactoring * N8N-5809 and refactor/cleanup * UI fixes and anonymize renaming * N8N-5837 * N8N-5834 * fix no-items UI issues * remove card / settings label in modal * N8N-5842 fix * disable webhook auth for now and update ui * change sidebar to tabs * remove payload option * extend audit events with more user data * N8N-5853 and UI revert to sidebar * remove redis destination * N8N-5864 / N8N-5868 / N8N-5867 / N8N-5865 * ui and licensing fixes * add node events and info bubbles to frontend * ui wording changes * frontend tests * N8N-5896 and ee rename * improves backend tests * merge fix * fix backend test * make linter happy * remove unnecessary cfg / limit actions to owners * fix multiple sentry DSN and anon bug * eslint fix * more tests and fixes * merge fix * fix workflow audit events * remove 'n8n.workflow.execution.error' event * merge fix * lint fix * lint fix * review fixes * fix merge * prettier fixes * merge * review changes * use loggerproxy * remove catch from internal hook promises * fix tests * lint fix * include review PR changes * review changes * delete duplicate lines from a bad merge * decouple log-streaming UI options from public API * logstreaming -> log-streaming for consistency * do not make unnecessary api calls when log streaming is disabled * prevent sentryClient.close() from being called if init failed * fix the e2e test for log-streaming * review changes * cleanup * use `private` for one last private property * do not use node prefix package names.. just yet * remove unused import * fix the tests because there is a folder called `events`, tsc-alias is messing up all imports for native events module. https://github.com/justkey007/tsc-alias/issues/152 Co-authored-by: कारतोफ्फेलस्क्रिप्ट™ <aditya@netroy.in>
2025-12-17 18:12:04 +00:00 · 2023-01-04 09:47:48 +01:00
parent 0795cdb74c
commit b67f803cbe
104 changed files with 5867 additions and 219 deletions
--- a/packages/cli/src/eventbus/MessageEventBusDestination/MessageEventBusDestinationWebhook.ee.ts
+++ b/packages/cli/src/eventbus/MessageEventBusDestination/MessageEventBusDestinationWebhook.ee.ts
@@ -0,0 +1,372 @@
+/* eslint-disable import/no-cycle */
+/* eslint-disable @typescript-eslint/no-unsafe-call */
+/* eslint-disable @typescript-eslint/no-unsafe-return */
+/* eslint-disable @typescript-eslint/no-unsafe-member-access */
+/* eslint-disable @typescript-eslint/no-unsafe-assignment */
+/* eslint-disable @typescript-eslint/no-unused-vars */
+/* eslint-disable @typescript-eslint/no-unnecessary-boolean-literal-compare */
+import { MessageEventBusDestination } from './MessageEventBusDestination.ee';
+import axios, { AxiosRequestConfig, Method } from 'axios';
+import { eventBus } from '../MessageEventBus/MessageEventBus';
+import { EventMessageTypes } from '../EventMessageClasses';
+import {
+	jsonParse,
+	LoggerProxy,
+	MessageEventBusDestinationOptions,
+	MessageEventBusDestinationTypeNames,
+	MessageEventBusDestinationWebhookOptions,
+	MessageEventBusDestinationWebhookParameterItem,
+	MessageEventBusDestinationWebhookParameterOptions,
+} from 'n8n-workflow';
+import { CredentialsHelper } from '../../CredentialsHelper';
+import { UserSettings } from 'n8n-core';
+import { Agent as HTTPSAgent } from 'https';
+import config from '../../config';
+import { isLogStreamingEnabled } from '../MessageEventBus/MessageEventBusHelper';
+import { eventMessageGenericDestinationTestEvent } from '../EventMessageClasses/EventMessageGeneric';
+
+export const isMessageEventBusDestinationWebhookOptions = (
+	candidate: unknown,
+): candidate is MessageEventBusDestinationWebhookOptions => {
+	const o = candidate as MessageEventBusDestinationWebhookOptions;
+	if (!o) return false;
+	return o.url !== undefined;
+};
+
+export class MessageEventBusDestinationWebhook
+	extends MessageEventBusDestination
+	implements MessageEventBusDestinationWebhookOptions
+{
+	url: string;
+
+	responseCodeMustMatch = false;
+
+	expectedStatusCode = 200;
+
+	method = 'POST';
+
+	authentication: 'predefinedCredentialType' | 'genericCredentialType' | 'none' = 'none';
+
+	sendQuery = false;
+
+	sendHeaders = false;
+
+	genericAuthType = '';
+
+	nodeCredentialType = '';
+
+	specifyHeaders = '';
+
+	specifyQuery = '';
+
+	jsonQuery = '';
+
+	jsonHeaders = '';
+
+	headerParameters: MessageEventBusDestinationWebhookParameterItem = { parameters: [] };
+
+	queryParameters: MessageEventBusDestinationWebhookParameterItem = { parameters: [] };
+
+	options: MessageEventBusDestinationWebhookParameterOptions = {};
+
+	sendPayload = true;
+
+	credentialsHelper?: CredentialsHelper;
+
+	axiosRequestOptions: AxiosRequestConfig;
+
+	constructor(options: MessageEventBusDestinationWebhookOptions) {
+		super(options);
+		this.url = options.url;
+		this.label = options.label ?? 'Webhook Endpoint';
+		this.__type = options.__type ?? MessageEventBusDestinationTypeNames.webhook;
+		if (options.responseCodeMustMatch) this.responseCodeMustMatch = options.responseCodeMustMatch;
+		if (options.expectedStatusCode) this.expectedStatusCode = options.expectedStatusCode;
+		if (options.method) this.method = options.method;
+		if (options.authentication) this.authentication = options.authentication;
+		if (options.sendQuery) this.sendQuery = options.sendQuery;
+		if (options.sendHeaders) this.sendHeaders = options.sendHeaders;
+		if (options.genericAuthType) this.genericAuthType = options.genericAuthType;
+		if (options.nodeCredentialType) this.nodeCredentialType = options.nodeCredentialType;
+		if (options.specifyHeaders) this.specifyHeaders = options.specifyHeaders;
+		if (options.specifyQuery) this.specifyQuery = options.specifyQuery;
+		if (options.jsonQuery) this.jsonQuery = options.jsonQuery;
+		if (options.jsonHeaders) this.jsonHeaders = options.jsonHeaders;
+		if (options.headerParameters) this.headerParameters = options.headerParameters;
+		if (options.queryParameters) this.queryParameters = options.queryParameters;
+		if (options.sendPayload) this.sendPayload = options.sendPayload;
+		if (options.options) this.options = options.options;
+
+		LoggerProxy.debug(`MessageEventBusDestinationWebhook with id ${this.getId()} initialized`);
+	}
+
+	async matchDecryptedCredentialType(credentialType: string) {
+		const foundCredential = Object.entries(this.credentials).find((e) => e[0] === credentialType);
+		if (foundCredential) {
+			const timezone = config.getEnv('generic.timezone');
+			const credentialsDecrypted = await this.credentialsHelper?.getDecrypted(
+				foundCredential[1],
+				foundCredential[0],
+				'internal',
+				timezone,
+				true,
+			);
+			return credentialsDecrypted;
+		}
+		return null;
+	}
+
+	async generateAxiosOptions() {
+		if (this.axiosRequestOptions?.url) {
+			return;
+		}
+
+		this.axiosRequestOptions = {
+			headers: {},
+			method: this.method as Method,
+			url: this.url,
+			maxRedirects: 0,
+		} as AxiosRequestConfig;
+
+		if (this.credentialsHelper === undefined) {
+			let encryptionKey: string | undefined;
+			try {
+				encryptionKey = await UserSettings.getEncryptionKey();
+			} catch (_) {}
+			if (encryptionKey) {
+				this.credentialsHelper = new CredentialsHelper(encryptionKey);
+			}
+		}
+
+		const sendQuery = this.sendQuery;
+		const specifyQuery = this.specifyQuery;
+		const sendPayload = this.sendPayload;
+		const sendHeaders = this.sendHeaders;
+		const specifyHeaders = this.specifyHeaders;
+
+		if (this.options.allowUnauthorizedCerts) {
+			this.axiosRequestOptions.httpsAgent = new HTTPSAgent({ rejectUnauthorized: false });
+		}
+
+		if (this.options.redirect?.followRedirects) {
+			this.axiosRequestOptions.maxRedirects = this.options.redirect?.maxRedirects;
+		}
+
+		if (this.options.proxy) {
+			this.axiosRequestOptions.proxy = this.options.proxy;
+		}
+
+		if (this.options.timeout) {
+			this.axiosRequestOptions.timeout = this.options.timeout;
+		} else {
+			this.axiosRequestOptions.timeout = 10000;
+		}
+
+		if (this.sendQuery && this.options.queryParameterArrays) {
+			Object.assign(this.axiosRequestOptions, {
+				qsStringifyOptions: { arrayFormat: this.options.queryParameterArrays },
+			});
+		}
+
+		const parametersToKeyValue = async (
+			// eslint-disable-next-line @typescript-eslint/no-explicit-any
+			acc: Promise<{ [key: string]: any }>,
+			cur: { name: string; value: string; parameterType?: string; inputDataFieldName?: string },
+		) => {
+			const acumulator = await acc;
+			acumulator[cur.name] = cur.value;
+			return acumulator;
+		};
+
+		// Get parameters defined in the UI
+		if (sendQuery && this.queryParameters.parameters) {
+			if (specifyQuery === 'keypair') {
+				this.axiosRequestOptions.params = this.queryParameters.parameters.reduce(
+					parametersToKeyValue,
+					Promise.resolve({}),
+				);
+			} else if (specifyQuery === 'json') {
+				// query is specified using JSON
+				try {
+					JSON.parse(this.jsonQuery);
+				} catch (_) {
+					console.log('JSON parameter need to be an valid JSON');
+				}
+
+				// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
+				this.axiosRequestOptions.params = jsonParse(this.jsonQuery);
+			}
+		}
+
+		// Get parameters defined in the UI
+		if (sendHeaders && this.headerParameters.parameters) {
+			if (specifyHeaders === 'keypair') {
+				this.axiosRequestOptions.headers = await this.headerParameters.parameters.reduce(
+					parametersToKeyValue,
+					Promise.resolve({}),
+				);
+			} else if (specifyHeaders === 'json') {
+				// body is specified using JSON
+				try {
+					JSON.parse(this.jsonHeaders);
+				} catch (_) {
+					console.log('JSON parameter need to be an valid JSON');
+				}
+
+				// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
+				this.axiosRequestOptions.headers = jsonParse(this.jsonHeaders);
+			}
+		}
+
+		// default for bodyContentType.raw
+		if (this.axiosRequestOptions.headers === undefined) {
+			this.axiosRequestOptions.headers = {};
+		}
+		// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+		this.axiosRequestOptions.headers['Content-Type'] = 'application/json';
+	}
+
+	serialize(): MessageEventBusDestinationWebhookOptions {
+		const abstractSerialized = super.serialize();
+		return {
+			...abstractSerialized,
+			url: this.url,
+			responseCodeMustMatch: this.responseCodeMustMatch,
+			expectedStatusCode: this.expectedStatusCode,
+			method: this.method,
+			authentication: this.authentication,
+			sendQuery: this.sendQuery,
+			sendHeaders: this.sendHeaders,
+			genericAuthType: this.genericAuthType,
+			nodeCredentialType: this.nodeCredentialType,
+			specifyHeaders: this.specifyHeaders,
+			specifyQuery: this.specifyQuery,
+			jsonQuery: this.jsonQuery,
+			jsonHeaders: this.jsonHeaders,
+			headerParameters: this.headerParameters,
+			queryParameters: this.queryParameters,
+			sendPayload: this.sendPayload,
+			options: this.options,
+			credentials: this.credentials,
+		};
+	}
+
+	static deserialize(
+		data: MessageEventBusDestinationOptions,
+	): MessageEventBusDestinationWebhook | null {
+		if (
+			'__type' in data &&
+			data.__type === MessageEventBusDestinationTypeNames.webhook &&
+			isMessageEventBusDestinationWebhookOptions(data)
+		) {
+			return new MessageEventBusDestinationWebhook(data);
+		}
+		return null;
+	}
+
+	async receiveFromEventBus(msg: EventMessageTypes): Promise<boolean> {
+		let sendResult = false;
+		if (msg.eventName !== eventMessageGenericDestinationTestEvent) {
+			if (!isLogStreamingEnabled()) return sendResult;
+			if (!this.hasSubscribedToEvent(msg)) return sendResult;
+		}
+		// at first run, build this.requestOptions with the destination settings
+		await this.generateAxiosOptions();
+
+		const payload = this.anonymizeAuditMessages ? msg.anonymize() : msg.payload;
+
+		if (['PATCH', 'POST', 'PUT', 'GET'].includes(this.method.toUpperCase())) {
+			if (this.sendPayload) {
+				this.axiosRequestOptions.data = {
+					...msg,
+					__type: undefined,
+					payload,
+					ts: msg.ts.toISO(),
+				};
+			} else {
+				this.axiosRequestOptions.data = {
+					...msg,
+					__type: undefined,
+					payload: undefined,
+					ts: msg.ts.toISO(),
+				};
+			}
+		}
+
+		// TODO: implement extra auth requests
+		let httpBasicAuth;
+		let httpDigestAuth;
+		let httpHeaderAuth;
+		let httpQueryAuth;
+		let oAuth1Api;
+		let oAuth2Api;
+
+		if (this.authentication === 'genericCredentialType') {
+			if (this.genericAuthType === 'httpBasicAuth') {
+				try {
+					httpBasicAuth = await this.matchDecryptedCredentialType('httpBasicAuth');
+				} catch (_) {}
+			} else if (this.genericAuthType === 'httpDigestAuth') {
+				try {
+					httpDigestAuth = await this.matchDecryptedCredentialType('httpDigestAuth');
+				} catch (_) {}
+			} else if (this.genericAuthType === 'httpHeaderAuth') {
+				try {
+					httpHeaderAuth = await this.matchDecryptedCredentialType('httpHeaderAuth');
+				} catch (_) {}
+			} else if (this.genericAuthType === 'httpQueryAuth') {
+				try {
+					httpQueryAuth = await this.matchDecryptedCredentialType('httpQueryAuth');
+				} catch (_) {}
+			} else if (this.genericAuthType === 'oAuth1Api') {
+				try {
+					oAuth1Api = await this.matchDecryptedCredentialType('oAuth1Api');
+				} catch (_) {}
+			} else if (this.genericAuthType === 'oAuth2Api') {
+				try {
+					oAuth2Api = await this.matchDecryptedCredentialType('oAuth2Api');
+				} catch (_) {}
+			}
+		}
+
+		if (httpBasicAuth) {
+			// Add credentials if any are set
+			this.axiosRequestOptions.auth = {
+				username: httpBasicAuth.user as string,
+				password: httpBasicAuth.password as string,
+			};
+		} else if (httpHeaderAuth) {
+			// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+			this.axiosRequestOptions.headers[httpHeaderAuth.name as string] = httpHeaderAuth.value;
+		} else if (httpQueryAuth) {
+			// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+			this.axiosRequestOptions.params[httpQueryAuth.name as string] = httpQueryAuth.value;
+		} else if (httpDigestAuth) {
+			this.axiosRequestOptions.auth = {
+				username: httpDigestAuth.user as string,
+				password: httpDigestAuth.password as string,
+			};
+		}
+
+		try {
+			const requestResponse = await axios.request(this.axiosRequestOptions);
+			if (requestResponse) {
+				if (this.responseCodeMustMatch) {
+					if (requestResponse.status === this.expectedStatusCode) {
+						await eventBus.confirmSent(msg, { id: this.id, name: this.label });
+						sendResult = true;
+					} else {
+						sendResult = false;
+					}
+				} else {
+					await eventBus.confirmSent(msg, { id: this.id, name: this.label });
+					sendResult = true;
+				}
+			}
+		} catch (error) {
+			console.error(error);
+		}
+
+		return sendResult;
+	}
+}