feat: Add global event bus (#4860)

* fix branch

* fix deserialize, add filewriter

* add catchAll eventGroup/Name

* adding simple Redis sender and receiver to eventbus

* remove native node threads

* improve eventbus

* refactor and simplify

* more refactoring and syslog client

* more refactor, improved endpoints and eventbus

* remove local broker and receivers from mvp

* destination de/serialization

* create MessageEventBusDestinationEntity

* db migrations, load destinations at startup

* add delete destination endpoint

* pnpm merge and circular import fix

* delete destination fix

* trigger log file shuffle after size reached

* add environment variables for eventbus

* reworking event messages

* serialize to thread fix

* some refactor and lint fixing

* add emit to eventbus

* cleanup and fix sending unsent

* quicksave frontend trial

* initial EventTree vue component

* basic log streaming settings in vue

* http request code merge

* create destination settings modals

* fix eventmessage options types

* credentials are loaded

* fix and clean up frontend code

* move request code to axios

* update lock file

* merge fix

* fix redis build

* move destination interfaces into workflow pkg

* revive sentry as destination

* migration fixes and frontend cleanup

* N8N-5777 / N8N-5789 N8N-5788

* N8N-5784

* N8N-5782 removed event levels

* N8N-5790 sentry destination cleanup

* N8N-5786 and refactoring

* N8N-5809 and refactor/cleanup

* UI fixes and anonymize renaming

* N8N-5837

* N8N-5834

* fix no-items UI issues

* remove card / settings label in modal

* N8N-5842 fix

* disable webhook auth for now and update ui

* change sidebar to tabs

* remove payload option

* extend audit events with more user data

* N8N-5853 and UI revert to sidebar

* remove redis destination

* N8N-5864 / N8N-5868 / N8N-5867 / N8N-5865

* ui and licensing fixes

* add node events and info bubbles to frontend

* ui wording changes

* frontend tests

* N8N-5896 and ee rename

* improves backend tests

* merge fix

* fix backend test

* make linter happy

* remove unnecessary cfg / limit  actions to owners

* fix multiple sentry DSN and anon bug

* eslint fix

* more tests and fixes

* merge fix

* fix workflow audit events

* remove 'n8n.workflow.execution.error' event

* merge fix

* lint fix

* lint fix

* review fixes

* fix merge

* prettier fixes

* merge

* review changes

* use loggerproxy

* remove catch from internal hook promises

* fix tests

* lint fix

* include review PR changes

* review changes

* delete duplicate lines from a bad merge

* decouple log-streaming UI options from public API

* logstreaming -> log-streaming for consistency

* do not make unnecessary api calls when log streaming is disabled

* prevent sentryClient.close() from being called if init failed

* fix the e2e test for log-streaming

* review changes

* cleanup

* use `private` for one last private property

* do not use node prefix package names.. just yet

* remove unused import

* fix the tests

because there is a folder called `events`, tsc-alias is messing up all imports for native events module.
https://github.com/justkey007/tsc-alias/issues/152

Co-authored-by: कारतोफ्फेलस्क्रिप्ट™ <aditya@netroy.in>
This commit is contained in:
Michael Auerswald
2023-01-04 09:47:48 +01:00
committed by GitHub
parent 0795cdb74c
commit b67f803cbe
104 changed files with 5867 additions and 219 deletions

View File

@@ -0,0 +1,137 @@
/* eslint-disable @typescript-eslint/no-unsafe-member-access */
/* eslint-disable @typescript-eslint/no-unsafe-return */
/* eslint-disable @typescript-eslint/no-unsafe-assignment */
import { MessageEventBusDestination } from './MessageEventBusDestination.ee';
import * as Sentry from '@sentry/node';
import { eventBus } from '../MessageEventBus/MessageEventBus';
import {
LoggerProxy,
MessageEventBusDestinationOptions,
MessageEventBusDestinationSentryOptions,
MessageEventBusDestinationTypeNames,
} from 'n8n-workflow';
import { GenericHelpers } from '../..';
import { isLogStreamingEnabled } from '../MessageEventBus/MessageEventBusHelper';
import { EventMessageTypes } from '../EventMessageClasses';
import { eventMessageGenericDestinationTestEvent } from '../EventMessageClasses/EventMessageGeneric';
export const isMessageEventBusDestinationSentryOptions = (
candidate: unknown,
): candidate is MessageEventBusDestinationSentryOptions => {
const o = candidate as MessageEventBusDestinationSentryOptions;
if (!o) return false;
return o.dsn !== undefined;
};
export class MessageEventBusDestinationSentry
extends MessageEventBusDestination
implements MessageEventBusDestinationSentryOptions
{
dsn: string;
tracesSampleRate = 1.0;
sendPayload: boolean;
sentryClient?: Sentry.NodeClient;
constructor(options: MessageEventBusDestinationSentryOptions) {
super(options);
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
this.label = options.label ?? 'Sentry DSN';
this.__type = options.__type ?? MessageEventBusDestinationTypeNames.sentry;
this.dsn = options.dsn;
if (options.sendPayload) this.sendPayload = options.sendPayload;
if (options.tracesSampleRate) this.tracesSampleRate = options.tracesSampleRate;
const { ENVIRONMENT: environment } = process.env;
GenericHelpers.getVersions()
.then((versions) => {
this.sentryClient = new Sentry.NodeClient({
dsn: this.dsn,
tracesSampleRate: this.tracesSampleRate,
environment,
release: versions.cli,
transport: Sentry.makeNodeTransport,
integrations: Sentry.defaultIntegrations,
stackParser: Sentry.defaultStackParser,
});
LoggerProxy.debug(`MessageEventBusDestinationSentry with id ${this.getId()} initialized`);
})
.catch((error) => {
console.error(error);
});
}
async receiveFromEventBus(msg: EventMessageTypes): Promise<boolean> {
let sendResult = false;
if (!this.sentryClient) return sendResult;
if (msg.eventName !== eventMessageGenericDestinationTestEvent) {
if (!isLogStreamingEnabled()) return sendResult;
if (!this.hasSubscribedToEvent(msg)) return sendResult;
}
try {
const payload = this.anonymizeAuditMessages ? msg.anonymize() : msg.payload;
const scope: Sentry.Scope = new Sentry.Scope();
const level = (
msg.eventName.toLowerCase().endsWith('error') ? 'error' : 'log'
) as Sentry.SeverityLevel;
scope.setLevel(level);
scope.setTags({
event: msg.getEventName(),
logger: this.label ?? this.getId(),
app: 'n8n',
});
if (this.sendPayload) {
scope.setExtras(payload);
}
const sentryResult = this.sentryClient.captureMessage(
msg.message ?? msg.eventName,
level,
{ event_id: msg.id, data: payload },
scope,
);
if (sentryResult) {
await eventBus.confirmSent(msg, { id: this.id, name: this.label });
sendResult = true;
}
} catch (error) {
console.log(error);
}
return sendResult;
}
serialize(): MessageEventBusDestinationSentryOptions {
const abstractSerialized = super.serialize();
return {
...abstractSerialized,
dsn: this.dsn,
tracesSampleRate: this.tracesSampleRate,
sendPayload: this.sendPayload,
};
}
static deserialize(
data: MessageEventBusDestinationOptions,
): MessageEventBusDestinationSentry | null {
if (
'__type' in data &&
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
data.__type === MessageEventBusDestinationTypeNames.sentry &&
isMessageEventBusDestinationSentryOptions(data)
) {
return new MessageEventBusDestinationSentry(data);
}
return null;
}
toString() {
return JSON.stringify(this.serialize());
}
async close() {
await super.close();
await this.sentryClient?.close();
}
}