feat: Add global event bus (#4860)

* fix branch

* fix deserialize, add filewriter

* add catchAll eventGroup/Name

* adding simple Redis sender and receiver to eventbus

* remove native node threads

* improve eventbus

* refactor and simplify

* more refactoring and syslog client

* more refactor, improved endpoints and eventbus

* remove local broker and receivers from mvp

* destination de/serialization

* create MessageEventBusDestinationEntity

* db migrations, load destinations at startup

* add delete destination endpoint

* pnpm merge and circular import fix

* delete destination fix

* trigger log file shuffle after size reached

* add environment variables for eventbus

* reworking event messages

* serialize to thread fix

* some refactor and lint fixing

* add emit to eventbus

* cleanup and fix sending unsent

* quicksave frontend trial

* initial EventTree vue component

* basic log streaming settings in vue

* http request code merge

* create destination settings modals

* fix eventmessage options types

* credentials are loaded

* fix and clean up frontend code

* move request code to axios

* update lock file

* merge fix

* fix redis build

* move destination interfaces into workflow pkg

* revive sentry as destination

* migration fixes and frontend cleanup

* N8N-5777 / N8N-5789 N8N-5788

* N8N-5784

* N8N-5782 removed event levels

* N8N-5790 sentry destination cleanup

* N8N-5786 and refactoring

* N8N-5809 and refactor/cleanup

* UI fixes and anonymize renaming

* N8N-5837

* N8N-5834

* fix no-items UI issues

* remove card / settings label in modal

* N8N-5842 fix

* disable webhook auth for now and update ui

* change sidebar to tabs

* remove payload option

* extend audit events with more user data

* N8N-5853 and UI revert to sidebar

* remove redis destination

* N8N-5864 / N8N-5868 / N8N-5867 / N8N-5865

* ui and licensing fixes

* add node events and info bubbles to frontend

* ui wording changes

* frontend tests

* N8N-5896 and ee rename

* improves backend tests

* merge fix

* fix backend test

* make linter happy

* remove unnecessary cfg / limit  actions to owners

* fix multiple sentry DSN and anon bug

* eslint fix

* more tests and fixes

* merge fix

* fix workflow audit events

* remove 'n8n.workflow.execution.error' event

* merge fix

* lint fix

* lint fix

* review fixes

* fix merge

* prettier fixes

* merge

* review changes

* use loggerproxy

* remove catch from internal hook promises

* fix tests

* lint fix

* include review PR changes

* review changes

* delete duplicate lines from a bad merge

* decouple log-streaming UI options from public API

* logstreaming -> log-streaming for consistency

* do not make unnecessary api calls when log streaming is disabled

* prevent sentryClient.close() from being called if init failed

* fix the e2e test for log-streaming

* review changes

* cleanup

* use `private` for one last private property

* do not use node prefix package names.. just yet

* remove unused import

* fix the tests

because there is a folder called `events`, tsc-alias is messing up all imports for native events module.
https://github.com/justkey007/tsc-alias/issues/152

Co-authored-by: कारतोफ्फेलस्क्रिप्ट™ <aditya@netroy.in>
This commit is contained in:
Michael Auerswald
2023-01-04 09:47:48 +01:00
committed by GitHub
parent 0795cdb74c
commit b67f803cbe
104 changed files with 5867 additions and 219 deletions

View File

@@ -6,6 +6,7 @@ module.exports = {
},
globalSetup: '<rootDir>/test/setup.ts',
globalTeardown: '<rootDir>/test/teardown.ts',
setupFilesAfterEnv: ['<rootDir>/test/setup-mocks.ts'],
moduleNameMapper: {
'^@/(.*)$': '<rootDir>/src/$1',
'^@db/(.*)$': '<rootDir>/src/databases/$1',

View File

@@ -75,11 +75,15 @@
"@types/localtunnel": "^1.9.0",
"@types/lodash.get": "^4.4.6",
"@types/lodash.intersection": "^4.4.7",
"@types/lodash.iteratee": "^4.7.7",
"@types/lodash.merge": "^4.6.6",
"@types/lodash.omit": "^4.5.7",
"@types/lodash.pick": "^4.4.7",
"@types/lodash.remove": "^4.7.7",
"@types/lodash.set": "^4.3.6",
"@types/lodash.split": "^4.4.7",
"@types/lodash.unionby": "^4.8.7",
"@types/lodash.uniqby": "^4.7.7",
"@types/lodash.unset": "^4.5.7",
"@types/parseurl": "^1.3.1",
"@types/passport-jwt": "^3.0.6",
@@ -90,6 +94,7 @@
"@types/superagent": "4.1.13",
"@types/supertest": "^2.0.11",
"@types/swagger-ui-express": "^4.1.3",
"@types/syslog-client": "^1.1.2",
"@types/uuid": "^8.3.2",
"@types/validator": "^13.7.0",
"@types/yamljs": "^0.2.31",
@@ -142,12 +147,17 @@
"localtunnel": "^2.0.0",
"lodash.get": "^4.4.2",
"lodash.intersection": "^4.4.0",
"lodash.iteratee": "^4.7.0",
"lodash.merge": "^4.6.2",
"lodash.omit": "^4.5.0",
"lodash.pick": "^4.4.0",
"lodash.remove": "^4.7.0",
"lodash.set": "^4.3.2",
"lodash.split": "^4.4.2",
"lodash.unionby": "^4.8.0",
"lodash.uniqby": "^4.7.0",
"lodash.unset": "^4.5.2",
"luxon": "^3.1.0",
"mysql2": "~2.3.0",
"n8n-core": "~0.149.2",
"n8n-editor-ui": "~0.175.4",
@@ -174,6 +184,8 @@
"sqlite3": "^5.1.2",
"sse-channel": "^4.0.0",
"swagger-ui-express": "^4.3.0",
"syslog-client": "^1.1.1",
"threads": "^1.7.0",
"tslib": "1.14.1",
"typeorm": "0.2.45",
"uuid": "^8.3.2",

View File

@@ -180,6 +180,8 @@ export async function init(
collections.InstalledNodes = linkRepository(entities.InstalledNodes);
collections.WorkflowStatistics = linkRepository(entities.WorkflowStatistics);
collections.EventDestinations = linkRepository(entities.EventDestinations);
isInitialized = true;
return collections;

View File

@@ -41,6 +41,7 @@ import type { User } from '@db/entities/User';
import type { WebhookEntity } from '@db/entities/WebhookEntity';
import type { WorkflowEntity } from '@db/entities/WorkflowEntity';
import type { WorkflowStatistics } from '@db/entities/WorkflowStatistics';
import type { EventDestinations } from '@db/entities/MessageEventBusDestinationEntity';
export interface IActivationError {
time: number;
@@ -82,6 +83,7 @@ export interface IDatabaseCollections {
InstalledPackages: Repository<InstalledPackages>;
InstalledNodes: Repository<InstalledNodes>;
WorkflowStatistics: Repository<WorkflowStatistics>;
EventDestinations: Repository<EventDestinations>;
}
// ----------------------------------
@@ -339,32 +341,102 @@ export interface IInternalHooksClass {
firstWorkflowCreatedAt?: Date,
): Promise<unknown[]>;
onPersonalizationSurveySubmitted(userId: string, answers: Record<string, string>): Promise<void>;
onWorkflowCreated(userId: string, workflow: IWorkflowBase, publicApi: boolean): Promise<void>;
onWorkflowDeleted(userId: string, workflowId: string, publicApi: boolean): Promise<void>;
onWorkflowSaved(userId: string, workflow: IWorkflowBase, publicApi: boolean): Promise<void>;
onWorkflowCreated(user: User, workflow: IWorkflowBase, publicApi: boolean): Promise<void>;
onWorkflowDeleted(user: User, workflowId: string, publicApi: boolean): Promise<void>;
onWorkflowSaved(user: User, workflow: IWorkflowBase, publicApi: boolean): Promise<void>;
onWorkflowBeforeExecute(executionId: string, data: IWorkflowExecutionDataProcess): Promise<void>;
onWorkflowPostExecute(
executionId: string,
workflow: IWorkflowBase,
runData?: IRun,
userId?: string,
): Promise<void>;
onUserDeletion(
userId: string,
userDeletionData: ITelemetryUserDeletionData,
publicApi: boolean,
onNodeBeforeExecute(
executionId: string,
workflow: IWorkflowBase,
nodeName: string,
): Promise<void>;
onUserInvite(userInviteData: { user_id: string; target_user_id: string[] }): Promise<void>;
onUserReinvite(userReinviteData: { user_id: string; target_user_id: string }): Promise<void>;
onUserUpdate(userUpdateData: { user_id: string; fields_changed: string[] }): Promise<void>;
onUserInviteEmailClick(userInviteClickData: { user_id: string }): Promise<void>;
onUserPasswordResetEmailClick(userPasswordResetData: { user_id: string }): Promise<void>;
onUserTransactionalEmail(userTransactionalEmailData: {
user_id: string;
message_type: 'Reset password' | 'New user invite' | 'Resend invite';
onNodePostExecute(executionId: string, workflow: IWorkflowBase, nodeName: string): Promise<void>;
onUserDeletion(userDeletionData: {
user: User;
telemetryData: ITelemetryUserDeletionData;
publicApi: boolean;
}): Promise<void>;
onUserPasswordResetRequestClick(userPasswordResetData: { user_id: string }): Promise<void>;
onInstanceOwnerSetup(instanceOwnerSetupData: { user_id: string }): Promise<void>;
onUserSignup(userSignupData: { user_id: string }): Promise<void>;
onUserInvite(userInviteData: {
user: User;
target_user_id: string[];
public_api: boolean;
}): Promise<void>;
onUserReinvite(userReinviteData: {
user: User;
target_user_id: string;
public_api: boolean;
}): Promise<void>;
onUserUpdate(userUpdateData: { user: User; fields_changed: string[] }): Promise<void>;
onUserInviteEmailClick(userInviteClickData: { inviter: User; invitee: User }): Promise<void>;
onUserPasswordResetEmailClick(userPasswordResetData: { user: User }): Promise<void>;
onUserTransactionalEmail(
userTransactionalEmailData: {
user_id: string;
message_type: 'Reset password' | 'New user invite' | 'Resend invite';
},
user?: User,
): Promise<void>;
onEmailFailed(failedEmailData: {
user: User;
message_type: 'Reset password' | 'New user invite' | 'Resend invite';
public_api: boolean;
}): Promise<void>;
onUserCreatedCredentials(userCreatedCredentialsData: {
user: User;
credential_name: string;
credential_type: string;
credential_id: string;
public_api: boolean;
}): Promise<void>;
onUserSharedCredentials(userSharedCredentialsData: {
user: User;
credential_name: string;
credential_type: string;
credential_id: string;
user_id_sharer: string;
user_ids_sharees_added: string[];
sharees_removed: number | null;
}): Promise<void>;
onUserPasswordResetRequestClick(userPasswordResetData: { user: User }): Promise<void>;
onInstanceOwnerSetup(instanceOwnerSetupData: { user_id: string }, user?: User): Promise<void>;
onUserSignup(userSignupData: { user: User }): Promise<void>;
onCommunityPackageInstallFinished(installationData: {
user: User;
input_string: string;
package_name: string;
success: boolean;
package_version?: string;
package_node_names?: string[];
package_author?: string;
package_author_email?: string;
failure_reason?: string;
}): Promise<void>;
onCommunityPackageUpdateFinished(updateData: {
user: User;
package_name: string;
package_version_current: string;
package_version_new: string;
package_node_names: string[];
package_author?: string;
package_author_email?: string;
}): Promise<void>;
onCommunityPackageDeleteFinished(deleteData: {
user: User;
package_name: string;
package_version?: string;
package_node_names?: string[];
package_author?: string;
package_author_email?: string;
}): Promise<void>;
onApiKeyCreated(apiKeyDeletedData: { user: User; public_api: boolean }): Promise<void>;
onApiKeyDeleted(apiKeyDeletedData: { user: User; public_api: boolean }): Promise<void>;
}
export interface IN8nConfig {
@@ -475,6 +547,7 @@ export interface IN8nUISettings {
};
enterprise: {
sharing: boolean;
logStreaming: boolean;
};
hideUsagePage: boolean;
license: {

View File

@@ -1,3 +1,6 @@
/* eslint-disable @typescript-eslint/no-unsafe-assignment */
/* eslint-disable @typescript-eslint/no-unsafe-call */
/* eslint-disable @typescript-eslint/no-unsafe-member-access */
import { snakeCase } from 'change-case';
import { BinaryDataManager } from 'n8n-core';
import {
@@ -15,9 +18,28 @@ import {
ITelemetryUserDeletionData,
IWorkflowDb,
IExecutionTrackProperties,
IWorkflowExecutionDataProcess,
} from '@/Interfaces';
import { Telemetry } from '@/telemetry';
import { RoleService } from './role/role.service';
import { eventBus } from './eventbus';
import { User } from './databases/entities/User';
function userToPayload(user: User): {
userId: string;
_email: string;
_firstName: string;
_lastName: string;
globalRole?: string;
} {
return {
userId: user.id,
_email: user.email,
_firstName: user.firstName,
_lastName: user.lastName,
globalRole: user.globalRole?.name,
};
}
export class InternalHooksClass implements IInternalHooksClass {
private versionCli: string;
@@ -82,29 +104,44 @@ export class InternalHooksClass implements IInternalHooksClass {
);
}
async onWorkflowCreated(
userId: string,
workflow: IWorkflowBase,
publicApi: boolean,
): Promise<void> {
async onWorkflowCreated(user: User, workflow: IWorkflowBase, publicApi: boolean): Promise<void> {
const { nodeGraph } = TelemetryHelpers.generateNodesGraph(workflow, this.nodeTypes);
return this.telemetry.track('User created workflow', {
user_id: userId,
workflow_id: workflow.id,
node_graph_string: JSON.stringify(nodeGraph),
public_api: publicApi,
});
void Promise.all([
eventBus.sendAuditEvent({
eventName: 'n8n.audit.workflow.created',
payload: {
...userToPayload(user),
workflowId: workflow.id,
workflowName: workflow.name,
},
}),
this.telemetry.track('User created workflow', {
user_id: user.id,
workflow_id: workflow.id,
node_graph_string: JSON.stringify(nodeGraph),
public_api: publicApi,
}),
]);
}
async onWorkflowDeleted(userId: string, workflowId: string, publicApi: boolean): Promise<void> {
return this.telemetry.track('User deleted workflow', {
user_id: userId,
workflow_id: workflowId,
public_api: publicApi,
});
async onWorkflowDeleted(user: User, workflowId: string, publicApi: boolean): Promise<void> {
void Promise.all([
eventBus.sendAuditEvent({
eventName: 'n8n.audit.workflow.deleted',
payload: {
...userToPayload(user),
workflowId,
},
}),
this.telemetry.track('User deleted workflow', {
user_id: user.id,
workflow_id: workflowId,
public_api: publicApi,
}),
]);
}
async onWorkflowSaved(userId: string, workflow: IWorkflowDb, publicApi: boolean): Promise<void> {
async onWorkflowSaved(user: User, workflow: IWorkflowDb, publicApi: boolean): Promise<void> {
const { nodeGraph } = TelemetryHelpers.generateNodesGraph(workflow, this.nodeTypes);
const notesCount = Object.keys(nodeGraph.notes).length;
@@ -113,28 +150,88 @@ export class InternalHooksClass implements IInternalHooksClass {
).length;
let userRole: 'owner' | 'sharee' | undefined = undefined;
if (userId && workflow.id) {
const role = await RoleService.getUserRoleForWorkflow(userId, workflow.id);
if (user.id && workflow.id) {
const role = await RoleService.getUserRoleForWorkflow(user.id, workflow.id);
if (role) {
userRole = role.name === 'owner' ? 'owner' : 'sharee';
}
}
return this.telemetry.track(
'User saved workflow',
{
user_id: userId,
workflow_id: workflow.id,
node_graph_string: JSON.stringify(nodeGraph),
notes_count_overlapping: overlappingCount,
notes_count_non_overlapping: notesCount - overlappingCount,
version_cli: this.versionCli,
num_tags: workflow.tags?.length ?? 0,
public_api: publicApi,
sharing_role: userRole,
void Promise.all([
eventBus.sendAuditEvent({
eventName: 'n8n.audit.workflow.updated',
payload: {
...userToPayload(user),
workflowId: workflow.id,
workflowName: workflow.name,
},
}),
this.telemetry.track(
'User saved workflow',
{
user_id: user.id,
workflow_id: workflow.id,
node_graph_string: JSON.stringify(nodeGraph),
notes_count_overlapping: overlappingCount,
notes_count_non_overlapping: notesCount - overlappingCount,
version_cli: this.versionCli,
num_tags: workflow.tags?.length ?? 0,
public_api: publicApi,
sharing_role: userRole,
},
{ withPostHog: true },
),
]);
}
async onNodeBeforeExecute(
executionId: string,
workflow: IWorkflowBase,
nodeName: string,
): Promise<void> {
void eventBus.sendNodeEvent({
eventName: 'n8n.node.started',
payload: {
executionId,
nodeName,
workflowId: workflow.id?.toString(),
workflowName: workflow.name,
},
{ withPostHog: true },
);
});
}
async onNodePostExecute(
executionId: string,
workflow: IWorkflowBase,
nodeName: string,
): Promise<void> {
void eventBus.sendNodeEvent({
eventName: 'n8n.node.finished',
payload: {
executionId,
nodeName,
workflowId: workflow.id?.toString(),
workflowName: workflow.name,
},
});
}
async onWorkflowBeforeExecute(
executionId: string,
data: IWorkflowExecutionDataProcess,
): Promise<void> {
void Promise.all([
eventBus.sendWorkflowEvent({
eventName: 'n8n.workflow.started',
payload: {
executionId,
userId: data.userId,
workflowId: data.workflowData.id?.toString(),
isManual: data.executionMode === 'manual',
workflowName: data.workflowData.name,
},
}),
]);
}
async onWorkflowPostExecute(
@@ -208,6 +305,7 @@ export class InternalHooksClass implements IInternalHooksClass {
let userRole: 'owner' | 'sharee' | undefined = undefined;
if (userId) {
// eslint-disable-next-line @typescript-eslint/no-unsafe-argument
const role = await RoleService.getUserRoleForWorkflow(userId, workflow.id);
if (role) {
userRole = role.name === 'owner' ? 'owner' : 'sharee';
@@ -266,11 +364,39 @@ export class InternalHooksClass implements IInternalHooksClass {
}
}
return Promise.all([
...promises,
BinaryDataManager.getInstance().persistBinaryDataForExecutionId(executionId),
this.telemetry.trackWorkflowExecution(properties),
]).then(() => {});
promises.push(
properties.success
? eventBus.sendWorkflowEvent({
eventName: 'n8n.workflow.success',
payload: {
executionId,
success: properties.success,
userId: properties.user_id,
workflowId: properties.workflow_id,
isManual: properties.is_manual,
workflowName: workflow.name,
},
})
: eventBus.sendWorkflowEvent({
eventName: 'n8n.workflow.failed',
payload: {
executionId,
success: properties.success,
userId: properties.user_id,
workflowId: properties.workflow_id,
lastNodeExecuted: runData?.data.resultData.lastNodeExecuted,
errorNodeType: properties.error_node_type,
errorNodeId: properties.error_node_id?.toString(),
errorMessage: properties.error_message?.toString(),
isManual: properties.is_manual,
workflowName: workflow.name,
},
}),
);
await BinaryDataManager.getInstance().persistBinaryDataForExecutionId(executionId);
void Promise.all([...promises, this.telemetry.trackWorkflowExecution(properties)]);
}
async onWorkflowSharingUpdate(workflowId: string, userId: string, userList: string[]) {
@@ -293,32 +419,66 @@ export class InternalHooksClass implements IInternalHooksClass {
return Promise.race([timeoutPromise, this.telemetry.trackN8nStop()]);
}
async onUserDeletion(
userId: string,
userDeletionData: ITelemetryUserDeletionData,
publicApi: boolean,
): Promise<void> {
return this.telemetry.track('User deleted user', {
...userDeletionData,
user_id: userId,
public_api: publicApi,
});
async onUserDeletion(userDeletionData: {
user: User;
telemetryData: ITelemetryUserDeletionData;
publicApi: boolean;
}): Promise<void> {
void Promise.all([
eventBus.sendAuditEvent({
eventName: 'n8n.audit.user.deleted',
payload: {
...userToPayload(userDeletionData.user),
},
}),
this.telemetry.track('User deleted user', {
...userDeletionData.telemetryData,
user_id: userDeletionData.user.id,
public_api: userDeletionData.publicApi,
}),
]);
}
async onUserInvite(userInviteData: {
user_id: string;
user: User;
target_user_id: string[];
public_api: boolean;
}): Promise<void> {
return this.telemetry.track('User invited new user', userInviteData);
void Promise.all([
eventBus.sendAuditEvent({
eventName: 'n8n.audit.user.invited',
payload: {
...userToPayload(userInviteData.user),
targetUserId: userInviteData.target_user_id,
},
}),
this.telemetry.track('User invited new user', {
user_id: userInviteData.user.id,
target_user_id: userInviteData.target_user_id,
public_api: userInviteData.public_api,
}),
]);
}
async onUserReinvite(userReinviteData: {
user_id: string;
user: User;
target_user_id: string;
public_api: boolean;
}): Promise<void> {
return this.telemetry.track('User resent new user invite email', userReinviteData);
void Promise.all([
eventBus.sendAuditEvent({
eventName: 'n8n.audit.user.reinvited',
payload: {
...userToPayload(userReinviteData.user),
targetUserId: userReinviteData.target_user_id,
},
}),
this.telemetry.track('User resent new user invite email', {
user_id: userReinviteData.user.id,
target_user_id: userReinviteData.target_user_id,
public_api: userReinviteData.public_api,
}),
]);
}
async onUserRetrievedUser(userRetrievedData: {
@@ -363,19 +523,56 @@ export class InternalHooksClass implements IInternalHooksClass {
return this.telemetry.track('User retrieved all workflows', userRetrievedData);
}
async onUserUpdate(userUpdateData: { user_id: string; fields_changed: string[] }): Promise<void> {
return this.telemetry.track('User changed personal settings', userUpdateData);
async onUserUpdate(userUpdateData: { user: User; fields_changed: string[] }): Promise<void> {
void Promise.all([
eventBus.sendAuditEvent({
eventName: 'n8n.audit.user.updated',
payload: {
...userToPayload(userUpdateData.user),
fieldsChanged: userUpdateData.fields_changed,
},
}),
this.telemetry.track('User changed personal settings', {
user_id: userUpdateData.user.id,
fields_changed: userUpdateData.fields_changed,
}),
]);
}
async onUserInviteEmailClick(userInviteClickData: { user_id: string }): Promise<void> {
return this.telemetry.track('User clicked invite link from email', userInviteClickData);
async onUserInviteEmailClick(userInviteClickData: {
inviter: User;
invitee: User;
}): Promise<void> {
void Promise.all([
eventBus.sendAuditEvent({
eventName: 'n8n.audit.user.invitation.accepted',
payload: {
invitee: {
...userToPayload(userInviteClickData.invitee),
},
inviter: {
...userToPayload(userInviteClickData.inviter),
},
},
}),
this.telemetry.track('User clicked invite link from email', {
user_id: userInviteClickData.invitee.id,
}),
]);
}
async onUserPasswordResetEmailClick(userPasswordResetData: { user_id: string }): Promise<void> {
return this.telemetry.track(
'User clicked password reset link from email',
userPasswordResetData,
);
async onUserPasswordResetEmailClick(userPasswordResetData: { user: User }): Promise<void> {
void Promise.all([
eventBus.sendAuditEvent({
eventName: 'n8n.audit.user.reset',
payload: {
...userToPayload(userPasswordResetData.user),
},
}),
this.telemetry.track('User clicked password reset link from email', {
user_id: userPasswordResetData.user.id,
}),
]);
}
async onUserTransactionalEmail(userTransactionalEmailData: {
@@ -398,44 +595,85 @@ export class InternalHooksClass implements IInternalHooksClass {
return this.telemetry.track('User invoked API', userInvokedApiData);
}
async onApiKeyDeleted(apiKeyDeletedData: {
user_id: string;
public_api: boolean;
}): Promise<void> {
return this.telemetry.track('API key deleted', apiKeyDeletedData);
async onApiKeyDeleted(apiKeyDeletedData: { user: User; public_api: boolean }): Promise<void> {
void Promise.all([
eventBus.sendAuditEvent({
eventName: 'n8n.audit.user.api.deleted',
payload: {
...userToPayload(apiKeyDeletedData.user),
},
}),
this.telemetry.track('API key deleted', {
user_id: apiKeyDeletedData.user.id,
public_api: apiKeyDeletedData.public_api,
}),
]);
}
async onApiKeyCreated(apiKeyCreatedData: {
user_id: string;
public_api: boolean;
}): Promise<void> {
return this.telemetry.track('API key created', apiKeyCreatedData);
async onApiKeyCreated(apiKeyCreatedData: { user: User; public_api: boolean }): Promise<void> {
void Promise.all([
eventBus.sendAuditEvent({
eventName: 'n8n.audit.user.api.created',
payload: {
...userToPayload(apiKeyCreatedData.user),
},
}),
this.telemetry.track('API key created', {
user_id: apiKeyCreatedData.user.id,
public_api: apiKeyCreatedData.public_api,
}),
]);
}
async onUserPasswordResetRequestClick(userPasswordResetData: { user_id: string }): Promise<void> {
return this.telemetry.track(
'User requested password reset while logged out',
userPasswordResetData,
);
async onUserPasswordResetRequestClick(userPasswordResetData: { user: User }): Promise<void> {
void Promise.all([
eventBus.sendAuditEvent({
eventName: 'n8n.audit.user.reset.requested',
payload: {
...userToPayload(userPasswordResetData.user),
},
}),
this.telemetry.track('User requested password reset while logged out', {
user_id: userPasswordResetData.user.id,
}),
]);
}
async onInstanceOwnerSetup(instanceOwnerSetupData: { user_id: string }): Promise<void> {
return this.telemetry.track('Owner finished instance setup', instanceOwnerSetupData);
}
async onUserSignup(userSignupData: { user_id: string }): Promise<void> {
return this.telemetry.track('User signed up', userSignupData);
async onUserSignup(userSignupData: { user: User }): Promise<void> {
void Promise.all([
eventBus.sendAuditEvent({
eventName: 'n8n.audit.user.signedup',
payload: {
...userToPayload(userSignupData.user),
},
}),
this.telemetry.track('User signed up', {
user_id: userSignupData.user.id,
}),
]);
}
async onEmailFailed(failedEmailData: {
user_id: string;
user: User;
message_type: 'Reset password' | 'New user invite' | 'Resend invite';
public_api: boolean;
}): Promise<void> {
return this.telemetry.track(
'Instance failed to send transactional email to user',
failedEmailData,
);
void Promise.all([
eventBus.sendAuditEvent({
eventName: 'n8n.audit.user.email.failed',
payload: {
messageType: failedEmailData.message_type,
...userToPayload(failedEmailData.user),
},
}),
this.telemetry.track('Instance failed to send transactional email to user', {
user_id: failedEmailData.user.id,
}),
]);
}
/**
@@ -443,27 +681,63 @@ export class InternalHooksClass implements IInternalHooksClass {
*/
async onUserCreatedCredentials(userCreatedCredentialsData: {
user: User;
credential_name: string;
credential_type: string;
credential_id: string;
public_api: boolean;
}): Promise<void> {
return this.telemetry.track('User created credentials', {
...userCreatedCredentialsData,
instance_id: this.instanceId,
});
void Promise.all([
eventBus.sendAuditEvent({
eventName: 'n8n.audit.user.credentials.created',
payload: {
...userToPayload(userCreatedCredentialsData.user),
credentialName: userCreatedCredentialsData.credential_name,
credentialType: userCreatedCredentialsData.credential_type,
credentialId: userCreatedCredentialsData.credential_id,
},
}),
this.telemetry.track('User created credentials', {
user_id: userCreatedCredentialsData.user.id,
credential_type: userCreatedCredentialsData.credential_type,
credential_id: userCreatedCredentialsData.credential_id,
instance_id: this.instanceId,
}),
]);
}
async onUserSharedCredentials(userSharedCredentialsData: {
user: User;
credential_name: string;
credential_type: string;
credential_id: string;
user_id_sharer: string;
user_ids_sharees_added: string[];
sharees_removed: number | null;
}): Promise<void> {
return this.telemetry.track('User updated cred sharing', {
...userSharedCredentialsData,
instance_id: this.instanceId,
});
void Promise.all([
eventBus.sendAuditEvent({
eventName: 'n8n.audit.user.credentials.shared',
payload: {
...userToPayload(userSharedCredentialsData.user),
credentialName: userSharedCredentialsData.credential_name,
credentialType: userSharedCredentialsData.credential_type,
credentialId: userSharedCredentialsData.credential_id,
userIdSharer: userSharedCredentialsData.user_id_sharer,
userIdsShareesAdded: userSharedCredentialsData.user_ids_sharees_added,
shareesRemoved: userSharedCredentialsData.sharees_removed,
},
}),
this.telemetry.track('User updated cred sharing', {
user_id: userSharedCredentialsData.user.id,
credential_type: userSharedCredentialsData.credential_type,
credential_id: userSharedCredentialsData.credential_id,
user_id_sharer: userSharedCredentialsData.user_id_sharer,
user_ids_sharees_added: userSharedCredentialsData.user_ids_sharees_added,
sharees_removed: userSharedCredentialsData.sharees_removed,
instance_id: this.instanceId,
}),
]);
}
/**
@@ -471,7 +745,7 @@ export class InternalHooksClass implements IInternalHooksClass {
*/
async onCommunityPackageInstallFinished(installationData: {
user_id: string;
user: User;
input_string: string;
package_name: string;
success: boolean;
@@ -481,11 +755,37 @@ export class InternalHooksClass implements IInternalHooksClass {
package_author_email?: string;
failure_reason?: string;
}): Promise<void> {
return this.telemetry.track('cnr package install finished', installationData);
void Promise.all([
eventBus.sendAuditEvent({
eventName: 'n8n.audit.package.installed',
payload: {
...userToPayload(installationData.user),
inputString: installationData.input_string,
packageName: installationData.package_name,
success: installationData.success,
packageVersion: installationData.package_version,
packageNodeNames: installationData.package_node_names,
packageAuthor: installationData.package_author,
packageAuthorEmail: installationData.package_author_email,
failureReason: installationData.failure_reason,
},
}),
this.telemetry.track('cnr package install finished', {
user_id: installationData.user.id,
input_string: installationData.input_string,
package_name: installationData.package_name,
success: installationData.success,
package_version: installationData.package_version,
package_node_names: installationData.package_node_names,
package_author: installationData.package_author,
package_author_email: installationData.package_author_email,
failure_reason: installationData.failure_reason,
}),
]);
}
async onCommunityPackageUpdateFinished(updateData: {
user_id: string;
user: User;
package_name: string;
package_version_current: string;
package_version_new: string;
@@ -493,18 +793,60 @@ export class InternalHooksClass implements IInternalHooksClass {
package_author?: string;
package_author_email?: string;
}): Promise<void> {
return this.telemetry.track('cnr package updated', updateData);
void Promise.all([
eventBus.sendAuditEvent({
eventName: 'n8n.audit.package.updated',
payload: {
...userToPayload(updateData.user),
packageName: updateData.package_name,
packageVersionCurrent: updateData.package_version_current,
packageVersionNew: updateData.package_version_new,
packageNodeNames: updateData.package_node_names,
packageAuthor: updateData.package_author,
packageAuthorEmail: updateData.package_author_email,
},
}),
this.telemetry.track('cnr package updated', {
user_id: updateData.user.id,
package_name: updateData.package_name,
package_version_current: updateData.package_version_current,
package_version_new: updateData.package_version_new,
package_node_names: updateData.package_node_names,
package_author: updateData.package_author,
package_author_email: updateData.package_author_email,
}),
]);
}
async onCommunityPackageDeleteFinished(updateData: {
user_id: string;
async onCommunityPackageDeleteFinished(deleteData: {
user: User;
package_name: string;
package_version: string;
package_node_names: string[];
package_author?: string;
package_author_email?: string;
}): Promise<void> {
return this.telemetry.track('cnr package deleted', updateData);
void Promise.all([
eventBus.sendAuditEvent({
eventName: 'n8n.audit.package.deleted',
payload: {
...userToPayload(deleteData.user),
packageName: deleteData.package_name,
packageVersion: deleteData.package_version,
packageNodeNames: deleteData.package_node_names,
packageAuthor: deleteData.package_author,
packageAuthorEmail: deleteData.package_author_email,
},
}),
this.telemetry.track('cnr package deleted', {
user_id: deleteData.user.id,
package_name: deleteData.package_name,
package_version: deleteData.package_version,
package_node_names: deleteData.package_node_names,
package_author: deleteData.package_author,
package_author_email: deleteData.package_author_email,
}),
]);
}
/**

View File

@@ -93,6 +93,10 @@ export class License {
return this.isFeatureEnabled(LICENSE_FEATURES.SHARING);
}
isLogStreamingEnabled() {
return this.isFeatureEnabled(LICENSE_FEATURES.LOG_STREAMING);
}
getCurrentEntitlements() {
return this.manager?.getCurrentEntitlements() ?? [];
}

View File

@@ -50,7 +50,7 @@ export = {
const createdWorkflow = await createWorkflow(workflow, req.user, role);
await ExternalHooks().run('workflow.afterCreate', [createdWorkflow]);
void InternalHooksManager.getInstance().onWorkflowCreated(req.user.id, createdWorkflow, true);
void InternalHooksManager.getInstance().onWorkflowCreated(req.user, createdWorkflow, true);
return res.json(createdWorkflow);
},
@@ -75,7 +75,7 @@ export = {
await Db.collections.Workflow.delete(id);
void InternalHooksManager.getInstance().onWorkflowDeleted(req.user.id, id, true);
void InternalHooksManager.getInstance().onWorkflowDeleted(req.user, id, true);
await ExternalHooks().run('workflow.afterDelete', [id]);
return res.json(sharedWorkflow.workflow);
@@ -221,7 +221,7 @@ export = {
const updatedWorkflow = await getWorkflowById(sharedWorkflow.workflowId);
await ExternalHooks().run('workflow.afterUpdate', [updateData]);
void InternalHooksManager.getInstance().onWorkflowSaved(req.user.id, updateData, true);
void InternalHooksManager.getInstance().onWorkflowSaved(req.user, updateData, true);
return res.json(updatedWorkflow);
},

View File

@@ -66,6 +66,9 @@ import {
ErrorReporterProxy as ErrorReporter,
INodeTypes,
ICredentialTypes,
INode,
IWorkflowBase,
IRun,
} from 'n8n-workflow';
import basicAuth from 'basic-auth';
@@ -157,9 +160,12 @@ import * as WebhookServer from '@/WebhookServer';
import * as WorkflowExecuteAdditionalData from '@/WorkflowExecuteAdditionalData';
import { toHttpNodeParameters } from '@/CurlConverterHelper';
import { setupErrorMiddleware } from '@/ErrorReporting';
import { eventBus } from '@/eventbus';
import { eventBusRouter } from '@/eventbus/eventBusRoutes';
import { isLogStreamingEnabled } from '@/eventbus/MessageEventBus/MessageEventBusHelper';
import { getLicense } from '@/License';
import { licenseController } from './license/license.controller';
import { corsMiddleware } from './middlewares/cors';
import { licenseController } from '@/license/license.controller';
import { corsMiddleware } from '@/middlewares/cors';
require('body-parser-xml')(bodyParser);
@@ -359,6 +365,7 @@ class App {
},
enterprise: {
sharing: false,
logStreaming: config.getEnv('enterprise.features.logStreaming'),
},
hideUsagePage: config.getEnv('hideUsagePage'),
license: {
@@ -391,6 +398,7 @@ class App {
// refresh enterprise status
Object.assign(this.frontendSettings.enterprise, {
sharing: isSharingEnabled(),
logStreaming: isLogStreamingEnabled(),
});
if (config.get('nodes.packagesMissing').length > 0) {
@@ -1542,6 +1550,16 @@ class App {
),
);
// ----------------------------------------
// EventBus Setup
// ----------------------------------------
if (!eventBus.isInitialized) {
await eventBus.initialize();
}
// add Event Bus REST endpoints
this.app.use(`/${this.restEndpoint}/eventbus`, eventBusRouter);
// ----------------------------------------
// Webhooks
// ----------------------------------------

View File

@@ -65,7 +65,7 @@ export function meNamespace(this: N8nApp): void {
const updatedkeys = Object.keys(req.body);
void InternalHooksManager.getInstance().onUserUpdate({
user_id: req.user.id,
user,
fields_changed: updatedkeys,
});
await this.externalHooks.run('user.profile.update', [currentEmail, sanitizeUser(user)]);
@@ -106,7 +106,7 @@ export function meNamespace(this: N8nApp): void {
await issueCookie(res, user);
void InternalHooksManager.getInstance().onUserUpdate({
user_id: req.user.id,
user,
fields_changed: ['password'],
});
@@ -162,12 +162,10 @@ export function meNamespace(this: N8nApp): void {
apiKey,
});
const telemetryData = {
user_id: req.user.id,
void InternalHooksManager.getInstance().onApiKeyCreated({
user: req.user,
public_api: false,
};
void InternalHooksManager.getInstance().onApiKeyCreated(telemetryData);
});
return { apiKey };
}),
@@ -183,12 +181,10 @@ export function meNamespace(this: N8nApp): void {
apiKey: null,
});
const telemetryData = {
user_id: req.user.id,
void InternalHooksManager.getInstance().onApiKeyDeleted({
user: req.user,
public_api: false,
};
void InternalHooksManager.getInstance().onApiKeyDeleted(telemetryData);
});
return { success: true };
}),

View File

@@ -86,7 +86,7 @@ export function passwordResetNamespace(this: N8nApp): void {
});
} catch (error) {
void InternalHooksManager.getInstance().onEmailFailed({
user_id: user.id,
user,
message_type: 'Reset password',
public_api: false,
});
@@ -105,7 +105,7 @@ export function passwordResetNamespace(this: N8nApp): void {
});
void InternalHooksManager.getInstance().onUserPasswordResetRequestClick({
user_id: id,
user,
});
}),
);
@@ -152,7 +152,7 @@ export function passwordResetNamespace(this: N8nApp): void {
Logger.info('Reset-password token resolved successfully', { userId: id });
void InternalHooksManager.getInstance().onUserPasswordResetEmailClick({
user_id: id,
user,
});
}),
);
@@ -212,7 +212,7 @@ export function passwordResetNamespace(this: N8nApp): void {
await issueCookie(res, user);
void InternalHooksManager.getInstance().onUserUpdate({
user_id: userId,
user,
fields_changed: ['password'],
});

View File

@@ -145,7 +145,7 @@ export function usersNamespace(this: N8nApp): void {
});
void InternalHooksManager.getInstance().onUserInvite({
user_id: req.user.id,
user: req.user,
target_user_id: Object.values(createUsers) as string[],
public_api: false,
});
@@ -190,7 +190,7 @@ export function usersNamespace(this: N8nApp): void {
});
} else {
void InternalHooksManager.getInstance().onEmailFailed({
user_id: req.user.id,
user: req.user,
message_type: 'New user invite',
public_api: false,
});
@@ -282,7 +282,8 @@ export function usersNamespace(this: N8nApp): void {
}
void InternalHooksManager.getInstance().onUserInviteEmailClick({
user_id: inviteeId,
inviter,
invitee,
});
const { firstName, lastName } = inviter;
@@ -348,7 +349,7 @@ export function usersNamespace(this: N8nApp): void {
await issueCookie(res, updatedUser);
void InternalHooksManager.getInstance().onUserSignup({
user_id: invitee.id,
user: updatedUser,
});
await this.externalHooks.run('user.profile.update', [invitee.email, sanitizeUser(invitee)]);
@@ -479,7 +480,11 @@ export function usersNamespace(this: N8nApp): void {
await transactionManager.delete(User, { id: userToDelete.id });
});
void InternalHooksManager.getInstance().onUserDeletion(req.user.id, telemetryData, false);
void InternalHooksManager.getInstance().onUserDeletion({
user: req.user,
telemetryData,
publicApi: false,
});
await this.externalHooks.run('user.deleted', [sanitizeUser(userToDelete)]);
return { success: true };
}
@@ -512,7 +517,12 @@ export function usersNamespace(this: N8nApp): void {
await transactionManager.delete(User, { id: userToDelete.id });
});
void InternalHooksManager.getInstance().onUserDeletion(req.user.id, telemetryData, false);
void InternalHooksManager.getInstance().onUserDeletion({
user: req.user,
telemetryData,
publicApi: false,
});
await this.externalHooks.run('user.deleted', [sanitizeUser(userToDelete)]);
return { success: true };
}),
@@ -570,7 +580,7 @@ export function usersNamespace(this: N8nApp): void {
if (!result?.success) {
void InternalHooksManager.getInstance().onEmailFailed({
user_id: req.user.id,
user: reinvitee,
message_type: 'Resend invite',
public_api: false,
});
@@ -583,7 +593,7 @@ export function usersNamespace(this: N8nApp): void {
}
void InternalHooksManager.getInstance().onUserReinvite({
user_id: req.user.id,
user: reinvitee,
target_user_id: reinvitee.id,
public_api: false,
});

View File

@@ -64,6 +64,7 @@ import * as WorkflowHelpers from '@/WorkflowHelpers';
import { getWorkflowOwner } from '@/UserManagement/UserManagementHelper';
import { findSubworkflowStart } from '@/utils';
import { PermissionChecker } from './UserManagement/PermissionChecker';
import { eventBus } from './eventbus';
import { WorkflowsService } from './workflows/workflows.services';
const ERROR_TRIGGER_TYPE = config.getEnv('nodes.errorTriggerType');
@@ -632,7 +633,6 @@ function hookFunctionsSave(parentProcessMode?: string): IWorkflowExecuteHooks {
workflowId: this.workflowData.id,
error,
});
if (!isManualMode) {
executeErrorWorkflow(
this.workflowData,
@@ -905,6 +905,8 @@ async function executeWorkflow(
: await ActiveExecutions.getInstance().add(runData);
}
void InternalHooksManager.getInstance().onWorkflowBeforeExecute(executionId || '', runData);
let data;
try {
await PermissionChecker.check(workflow, additionalData.userId);
@@ -1003,12 +1005,8 @@ async function executeWorkflow(
}
await externalHooks.run('workflow.postExecute', [data, workflowData, executionId]);
void InternalHooksManager.getInstance().onWorkflowPostExecute(
executionId,
workflowData,
data,
additionalData.userId,
);
void InternalHooksManager.getInstance().onWorkflowBeforeExecute(executionId || '', runData);
if (data.finished === true) {
// Workflow did finish successfully
@@ -1150,6 +1148,27 @@ export function getWorkflowHooksWorkerMain(
// So to avoid confusion, we are removing other hooks.
hookFunctions.nodeExecuteBefore = [];
hookFunctions.nodeExecuteAfter = [];
hookFunctions.nodeExecuteBefore.push(async function (
this: WorkflowHooks,
nodeName: string,
): Promise<void> {
void InternalHooksManager.getInstance().onNodeBeforeExecute(
this.executionId,
this.workflowData,
nodeName,
);
});
hookFunctions.nodeExecuteAfter.push(async function (
this: WorkflowHooks,
nodeName: string,
): Promise<void> {
void InternalHooksManager.getInstance().onNodePostExecute(
this.executionId,
this.workflowData,
nodeName,
);
});
return new WorkflowHooks(hookFunctions, mode, executionId, workflowData, optionalParameters);
}
@@ -1181,6 +1200,29 @@ export function getWorkflowHooksMain(
}
}
if (!hookFunctions.nodeExecuteBefore) hookFunctions.nodeExecuteBefore = [];
hookFunctions.nodeExecuteBefore?.push(async function (
this: WorkflowHooks,
nodeName: string,
): Promise<void> {
void InternalHooksManager.getInstance().onNodeBeforeExecute(
this.executionId,
this.workflowData,
nodeName,
);
});
if (!hookFunctions.nodeExecuteAfter) hookFunctions.nodeExecuteAfter = [];
hookFunctions.nodeExecuteAfter.push(async function (
this: WorkflowHooks,
nodeName: string,
): Promise<void> {
void InternalHooksManager.getInstance().onNodePostExecute(
this.executionId,
this.workflowData,
nodeName,
);
});
return new WorkflowHooks(hookFunctions, data.executionMode, executionId, data.workflowData, {
sessionId: data.sessionId,
retryOf: data.retryOf as string,

View File

@@ -95,6 +95,7 @@ export async function executeErrorWorkflow(
// 2) if now instance owner, then check if the user has access to the
// triggered workflow.
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
const user = await getWorkflowOwner(workflowErrorData.workflow.id!);
if (user.globalRole.name === 'owner') {

View File

@@ -149,6 +149,8 @@ export class WorkflowRunner {
executionId = await this.runSubprocess(data, loadStaticData, executionId, responsePromise);
}
void InternalHooksManager.getInstance().onWorkflowBeforeExecute(executionId, data);
const postExecutePromise = this.activeExecutions.getPostExecutePromise(executionId);
const externalHooks = ExternalHooks();

View File

@@ -222,6 +222,9 @@ class WorkflowRunnerProcess {
resolve(executionId);
};
});
void InternalHooksManager.getInstance().onWorkflowBeforeExecute(executionId || '', runData);
let result: IRun;
try {
const executeWorkflowFunctionOutput = (await executeWorkflowFunction(

View File

@@ -11,6 +11,7 @@ import config from '@/config';
import * as Db from '@/Db';
import { Role } from '@/databases/entities/Role';
import { hashPassword } from '@/UserManagement/UserManagementHelper';
import { eventBus } from '@/eventbus/MessageEventBus/MessageEventBus';
if (process.env.E2E_TESTS !== 'true') {
console.error('E2E endpoints only allowed during E2E tests');
@@ -18,12 +19,14 @@ if (process.env.E2E_TESTS !== 'true') {
}
const tablesToTruncate = [
'event_destinations',
'shared_workflow',
'shared_credentials',
'webhook_entity',
'workflows_tags',
'credentials_entity',
'tag_entity',
'workflow_statistics',
'workflow_entity',
'execution_entity',
'settings',
@@ -40,7 +43,6 @@ const truncateAll = async () => {
`DELETE FROM ${table}; DELETE FROM sqlite_sequence WHERE name=${table};`,
);
}
config.set('userManagement.isInstanceOwnerSetUp', false);
};
const setupUserManagement = async () => {
@@ -69,11 +71,21 @@ const setupUserManagement = async () => {
await connection.query(
"INSERT INTO \"settings\" (key, value, loadOnStartup) values ('userManagement.isInstanceOwnerSetUp', 'false', true), ('userManagement.skipInstanceOwnerSetup', 'false', true)",
);
config.set('userManagement.isInstanceOwnerSetUp', false);
};
const resetLogStreaming = async () => {
config.set('enterprise.features.logStreaming', false);
for (const id in eventBus.destinations) {
await eventBus.removeDestination(id);
}
};
export const e2eController = Router();
e2eController.post('/db/reset', async (req, res) => {
await resetLogStreaming();
await truncateAll();
await setupUserManagement();
@@ -109,3 +121,8 @@ e2eController.post('/db/setup-owner', bodyParser.json(), async (req, res) => {
res.writeHead(204).end();
});
e2eController.post('/enable-feature/:feature', async (req, res) => {
config.set(`enterprise.features.${req.params.feature}`, true);
res.writeHead(204).end();
});

View File

@@ -124,7 +124,7 @@ nodesController.post(
const errorMessage = error instanceof Error ? error.message : UNKNOWN_FAILURE_REASON;
void InternalHooksManager.getInstance().onCommunityPackageInstallFinished({
user_id: req.user.id,
user: req.user,
input_string: name,
package_name: parsed.packageName,
success: false,
@@ -152,7 +152,7 @@ nodesController.post(
});
void InternalHooksManager.getInstance().onCommunityPackageInstallFinished({
user_id: req.user.id,
user: req.user,
input_string: name,
package_name: parsed.packageName,
success: true,
@@ -259,7 +259,7 @@ nodesController.delete(
});
void InternalHooksManager.getInstance().onCommunityPackageDeleteFinished({
user_id: req.user.id,
user: req.user,
package_name: name,
package_version: installedPackage.installedVersion,
package_node_names: installedPackage.installedNodes.map((node) => node.name),
@@ -313,7 +313,7 @@ nodesController.patch(
});
void InternalHooksManager.getInstance().onCommunityPackageUpdateFinished({
user_id: req.user.id,
user: req.user,
package_name: name,
package_version_current: previouslyInstalledPackage.installedVersion,
package_version_new: newInstalledPackage.installedVersion,

View File

@@ -42,6 +42,7 @@ import { initErrorHandling } from '@/ErrorReporting';
import * as CrashJournal from '@/CrashJournal';
import { createPostHogLoadingScript } from '@/telemetry/scripts';
import { EDITOR_UI_DIST_DIR, GENERATED_STATIC_DIR } from '@/constants';
import { eventBus } from '../eventbus';
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-var-requires
const open = require('open');
@@ -154,6 +155,9 @@ export class Start extends Command {
await sleep(500);
executingWorkflows = activeExecutionsInstance.getActiveExecutions();
}
//finally shut down Event Bus
await eventBus.close();
} catch (error) {
console.error('There was an error shutting down n8n.', error);
}

View File

@@ -916,6 +916,10 @@ export const schema = {
format: Boolean,
default: false,
},
logStreaming: {
format: Boolean,
default: false,
},
},
},
@@ -1044,4 +1048,39 @@ export const schema = {
env: 'N8N_HIDE_USAGE_PAGE',
doc: 'Hide or show the usage page',
},
eventBus: {
checkUnsentInterval: {
doc: 'How often (in ms) to check for unsent event messages. Can in rare cases cause a message to be sent twice. 0=disabled',
format: Number,
default: 0,
env: 'N8N_EVENTBUS_CHECKUNSENTINTERVAL',
},
logWriter: {
syncFileAccess: {
doc: 'Whether all file access happens synchronously within the thread.',
format: Boolean,
default: false,
env: 'N8N_EVENTBUS_LOGWRITER_SYNCFILEACCESS',
},
keepLogCount: {
doc: 'How many event log files to keep.',
format: Number,
default: 3,
env: 'N8N_EVENTBUS_LOGWRITER_KEEPLOGCOUNT',
},
maxFileSizeInKB: {
doc: 'Maximum size of an event log file before a new one is started.',
format: Number,
default: 102400, // 100MB
env: 'N8N_EVENTBUS_LOGWRITER_MAXFILESIZEINKB',
},
logBaseName: {
doc: 'Basename of the event log file.',
format: String,
default: 'n8nEventLog',
env: 'N8N_EVENTBUS_LOGWRITER_LOGBASENAME',
},
},
},
};

View File

@@ -55,6 +55,7 @@ export const SETTINGS_LICENSE_CERT_KEY = 'license.cert';
export enum LICENSE_FEATURES {
SHARING = 'feat:sharing',
LOG_STREAMING = 'feat:logStreaming',
}
export const CREDENTIAL_BLANKING_VALUE = '__n8n_BLANK_VALUE_e5362baf-c777-4d57-a609-6eaf1f9e87f6';

View File

@@ -174,6 +174,8 @@ EECredentialsController.put(
});
void InternalHooksManager.getInstance().onUserSharedCredentials({
user: req.user,
credential_name: credential.name,
credential_type: credential.type,
credential_id: credential.id,
user_id_sharer: req.user.id,

View File

@@ -130,6 +130,8 @@ credentialsController.post(
const credential = await CredentialsService.save(newCredential, encryptedData, req.user);
void InternalHooksManager.getInstance().onUserCreatedCredentials({
user: req.user,
credential_name: newCredential.name,
credential_type: credential.type,
credential_id: credential.id,
public_api: false,

View File

@@ -0,0 +1,12 @@
import { MessageEventBusDestinationOptions } from 'n8n-workflow';
import { Column, Entity, PrimaryColumn } from 'typeorm';
import { AbstractEntity, jsonColumnType } from './AbstractEntity';
@Entity({ name: 'event_destinations' })
export class EventDestinations extends AbstractEntity {
@PrimaryColumn('uuid')
id: string;
@Column(jsonColumnType)
destination: MessageEventBusDestinationOptions;
}

View File

@@ -12,6 +12,7 @@ import { SharedCredentials } from './SharedCredentials';
import { InstalledPackages } from './InstalledPackages';
import { InstalledNodes } from './InstalledNodes';
import { WorkflowStatistics } from './WorkflowStatistics';
import { EventDestinations } from './MessageEventBusDestinationEntity';
export const entities = {
CredentialsEntity,
@@ -27,4 +28,5 @@ export const entities = {
InstalledPackages,
InstalledNodes,
WorkflowStatistics,
EventDestinations,
};

View File

@@ -0,0 +1,27 @@
import { MigrationInterface, QueryRunner } from 'typeorm';
import { getTablePrefix, logMigrationEnd, logMigrationStart } from '../../utils/migrationHelpers';
export class MessageEventBusDestinations1671535397530 implements MigrationInterface {
name = 'MessageEventBusDestinations1671535397530';
async up(queryRunner: QueryRunner) {
logMigrationStart(this.name);
const tablePrefix = getTablePrefix();
await queryRunner.query(
`CREATE TABLE ${tablePrefix}event_destinations (` +
'`id` varchar(36) PRIMARY KEY NOT NULL,' +
'`destination` text NOT NULL,' +
'`createdAt` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, ' +
'`updatedAt` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP' +
") ENGINE='InnoDB';",
);
logMigrationEnd(this.name);
}
async down(queryRunner: QueryRunner) {
logMigrationStart(this.name);
const tablePrefix = getTablePrefix();
await queryRunner.query(`DROP TABLE "${tablePrefix}event_destinations"`);
logMigrationEnd(this.name);
}
}

View File

@@ -26,6 +26,7 @@ import { CreateCredentialUsageTable1665484192213 } from './1665484192213-CreateC
import { RemoveCredentialUsageTable1665754637026 } from './1665754637026-RemoveCredentialUsageTable';
import { AddWorkflowVersionIdColumn1669739707125 } from './1669739707125-AddWorkflowVersionIdColumn';
import { AddTriggerCountColumn1669823906994 } from './1669823906994-AddTriggerCountColumn';
import { MessageEventBusDestinations1671535397530 } from './1671535397530-MessageEventBusDestinations';
export const mysqlMigrations = [
InitialMigration1588157391238,
@@ -56,4 +57,5 @@ export const mysqlMigrations = [
AddWorkflowVersionIdColumn1669739707125,
WorkflowStatistics1664196174002,
AddTriggerCountColumn1669823906994,
MessageEventBusDestinations1671535397530,
];

View File

@@ -0,0 +1,27 @@
import { MigrationInterface, QueryRunner } from 'typeorm';
import { getTablePrefix, logMigrationEnd, logMigrationStart } from '../../utils/migrationHelpers';
export class MessageEventBusDestinations1671535397530 implements MigrationInterface {
name = 'MessageEventBusDestinations1671535397530';
async up(queryRunner: QueryRunner) {
logMigrationStart(this.name);
const tablePrefix = getTablePrefix();
await queryRunner.query(
`CREATE TABLE ${tablePrefix}event_destinations (` +
`"id" UUID PRIMARY KEY NOT NULL,` +
`"destination" JSONB NOT NULL,` +
`"createdAt" TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,` +
`"updatedAt" TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP);`,
);
logMigrationEnd(this.name);
}
async down(queryRunner: QueryRunner) {
logMigrationStart(this.name);
const tablePrefix = getTablePrefix();
await queryRunner.query(`DROP TABLE "${tablePrefix}event_destinations"`);
logMigrationEnd(this.name);
}
}

View File

@@ -24,6 +24,7 @@ import { CreateCredentialUsageTable1665484192212 } from './1665484192212-CreateC
import { RemoveCredentialUsageTable1665754637025 } from './1665754637025-RemoveCredentialUsageTable';
import { AddWorkflowVersionIdColumn1669739707126 } from './1669739707126-AddWorkflowVersionIdColumn';
import { AddTriggerCountColumn1669823906995 } from './1669823906995-AddTriggerCountColumn';
import { MessageEventBusDestinations1671535397530 } from './1671535397530-MessageEventBusDestinations';
export const postgresMigrations = [
InitialMigration1587669153312,
@@ -52,4 +53,5 @@ export const postgresMigrations = [
AddWorkflowVersionIdColumn1669739707126,
WorkflowStatistics1664196174001,
AddTriggerCountColumn1669823906995,
MessageEventBusDestinations1671535397530,
];

View File

@@ -0,0 +1,27 @@
import { MigrationInterface, QueryRunner } from 'typeorm';
import { getTablePrefix, logMigrationEnd, logMigrationStart } from '../../utils/migrationHelpers';
export class MessageEventBusDestinations1671535397530 implements MigrationInterface {
name = 'MessageEventBusDestinations1671535397530';
async up(queryRunner: QueryRunner) {
logMigrationStart(this.name);
const tablePrefix = getTablePrefix();
await queryRunner.query(
`CREATE TABLE "${tablePrefix}event_destinations" (` +
`"id" varchar(36) PRIMARY KEY NOT NULL,` +
`"destination" text NOT NULL,` +
`"createdAt" datetime(3) NOT NULL DEFAULT 'STRFTIME(''%Y-%m-%d %H:%M:%f'', ''NOW'')',` +
`"updatedAt" datetime(3) NOT NULL DEFAULT 'STRFTIME(''%Y-%m-%d %H:%M:%f'', ''NOW'')'` +
`);`,
);
logMigrationEnd(this.name);
}
async down(queryRunner: QueryRunner) {
logMigrationStart(this.name);
const tablePrefix = getTablePrefix();
await queryRunner.query(`DROP TABLE "${tablePrefix}event_destinations"`);
logMigrationEnd(this.name);
}
}

View File

@@ -23,6 +23,7 @@ import { CreateCredentialUsageTable1665484192211 } from './1665484192211-CreateC
import { RemoveCredentialUsageTable1665754637024 } from './1665754637024-RemoveCredentialUsageTable';
import { AddWorkflowVersionIdColumn1669739707124 } from './1669739707124-AddWorkflowVersionIdColumn';
import { AddTriggerCountColumn1669823906993 } from './1669823906993-AddTriggerCountColumn';
import { MessageEventBusDestinations1671535397530 } from './1671535397530-MessageEventBusDestinations';
const sqliteMigrations = [
InitialMigration1588102412422,
@@ -50,6 +51,7 @@ const sqliteMigrations = [
AddWorkflowVersionIdColumn1669739707124,
AddTriggerCountColumn1669823906993,
WorkflowStatistics1664196174000,
MessageEventBusDestinations1671535397530,
];
export { sqliteMigrations };

View File

@@ -0,0 +1,143 @@
/* eslint-disable @typescript-eslint/no-unsafe-argument */
/* eslint-disable @typescript-eslint/no-explicit-any */
import { DateTime } from 'luxon';
import type { EventMessageTypeNames, JsonObject } from 'n8n-workflow';
import { v4 as uuid } from 'uuid';
import type { AbstractEventPayload } from './AbstractEventPayload';
import type { AbstractEventMessageOptions } from './AbstractEventMessageOptions';
function modifyUnderscoredKeys(
input: { [key: string]: any },
modifier: (secret: string) => string | undefined = () => '*',
) {
const result: { [key: string]: any } = {};
if (!input) return input;
Object.keys(input).forEach((key) => {
if (typeof input[key] === 'string') {
if (key.substring(0, 1) === '_') {
const modifierResult = modifier(input[key]);
if (modifierResult !== undefined) {
result[key] = modifier(input[key]);
}
} else {
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
result[key] = input[key];
}
} else if (typeof input[key] === 'object') {
if (Array.isArray(input[key])) {
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call
result[key] = input[key].map((item: any) => {
if (typeof item === 'object' && !Array.isArray(item)) {
return modifyUnderscoredKeys(item, modifier);
} else {
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
return item;
}
});
} else {
result[key] = modifyUnderscoredKeys(input[key], modifier);
}
} else {
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
result[key] = input[key];
}
});
return result;
}
export const isEventMessage = (candidate: unknown): candidate is AbstractEventMessage => {
const o = candidate as AbstractEventMessage;
if (!o) return false;
return (
o.eventName !== undefined &&
o.id !== undefined &&
o.ts !== undefined &&
o.getEventName !== undefined
);
};
export const isEventMessageOptions = (
candidate: unknown,
): candidate is AbstractEventMessageOptions => {
const o = candidate as AbstractEventMessageOptions;
if (!o) return false;
if (o.eventName !== undefined) {
if (o.eventName.match(/^[\w\s]+\.[\w\s]+\.[\w\s]+/)) {
return true;
}
}
return false;
};
export const isEventMessageOptionsWithType = (
candidate: unknown,
expectedType: string,
): candidate is AbstractEventMessageOptions => {
const o = candidate as AbstractEventMessageOptions;
if (!o) return false;
return o.eventName !== undefined && o.__type !== undefined && o.__type === expectedType;
};
export abstract class AbstractEventMessage {
abstract readonly __type: EventMessageTypeNames;
id: string;
ts: DateTime;
eventName: string;
message: string;
abstract payload: AbstractEventPayload;
/**
* Creates a new instance of Event Message
* @param props.eventName The specific events name e.g. "n8n.workflow.workflowStarted"
* @param props.level The log level, defaults to. "info"
* @param props.severity The severity of the event e.g. "normal"
* @returns instance of EventMessage
*/
constructor(options: AbstractEventMessageOptions) {
this.setOptionsOrDefault(options);
}
abstract deserialize(data: JsonObject): this;
abstract setPayload(payload: AbstractEventPayload): this;
anonymize(): AbstractEventPayload {
const anonymizedPayload = modifyUnderscoredKeys(this.payload);
return anonymizedPayload;
}
serialize(): AbstractEventMessageOptions {
return {
__type: this.__type,
id: this.id,
ts: this.ts.toISO(),
eventName: this.eventName,
message: this.message,
payload: this.payload,
};
}
setOptionsOrDefault(options: AbstractEventMessageOptions) {
this.id = options.id ?? uuid();
this.eventName = options.eventName;
this.message = options.message ?? options.eventName;
if (typeof options.ts === 'string') {
this.ts = DateTime.fromISO(options.ts) ?? DateTime.now();
} else {
this.ts = options.ts ?? DateTime.now();
}
}
getEventName(): string {
return this.eventName;
}
toString() {
return JSON.stringify(this.serialize());
}
}

View File

@@ -0,0 +1,13 @@
import type { DateTime } from 'luxon';
import { EventMessageTypeNames } from 'n8n-workflow';
import type { AbstractEventPayload } from './AbstractEventPayload';
export interface AbstractEventMessageOptions {
__type?: EventMessageTypeNames;
id?: string;
ts?: DateTime | string;
eventName: string;
message?: string;
payload?: AbstractEventPayload;
anonymize?: boolean;
}

View File

@@ -0,0 +1,5 @@
import type { IWorkflowBase, JsonValue } from 'n8n-workflow';
export interface AbstractEventPayload {
[key: string]: JsonValue | IWorkflowBase | undefined;
}

View File

@@ -0,0 +1,74 @@
import { AbstractEventMessage, isEventMessageOptionsWithType } from './AbstractEventMessage';
import { EventMessageTypeNames, JsonObject, JsonValue } from 'n8n-workflow';
import { AbstractEventPayload } from './AbstractEventPayload';
import { AbstractEventMessageOptions } from './AbstractEventMessageOptions';
export const eventNamesAudit = [
'n8n.audit.user.signedup',
'n8n.audit.user.updated',
'n8n.audit.user.deleted',
'n8n.audit.user.invited',
'n8n.audit.user.invitation.accepted',
'n8n.audit.user.reinvited',
'n8n.audit.user.email.failed',
'n8n.audit.user.reset.requested',
'n8n.audit.user.reset',
'n8n.audit.user.credentials.created',
'n8n.audit.user.credentials.shared',
'n8n.audit.user.api.created',
'n8n.audit.user.api.deleted',
'n8n.audit.package.installed',
'n8n.audit.package.updated',
'n8n.audit.package.deleted',
'n8n.audit.workflow.created',
'n8n.audit.workflow.deleted',
'n8n.audit.workflow.updated',
] as const;
export type EventNamesAuditType = typeof eventNamesAudit[number];
// --------------------------------------
// EventMessage class for Audit events
// --------------------------------------
export interface EventPayloadAudit extends AbstractEventPayload {
msg?: JsonValue;
userId?: string;
userEmail?: string;
firstName?: string;
lastName?: string;
}
export interface EventMessageAuditOptions extends AbstractEventMessageOptions {
eventName: EventNamesAuditType;
payload?: EventPayloadAudit;
}
export class EventMessageAudit extends AbstractEventMessage {
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access
readonly __type = EventMessageTypeNames.audit;
eventName: EventNamesAuditType;
payload: EventPayloadAudit;
constructor(options: EventMessageAuditOptions) {
super(options);
if (options.payload) this.setPayload(options.payload);
if (options.anonymize) {
this.anonymize();
}
}
setPayload(payload: EventPayloadAudit): this {
this.payload = payload;
return this;
}
deserialize(data: JsonObject): this {
if (isEventMessageOptionsWithType(data, this.__type)) {
this.setOptionsOrDefault(data);
if (data.payload) this.setPayload(data.payload as EventPayloadAudit);
}
return this;
}
}

View File

@@ -0,0 +1,39 @@
import { DateTime } from 'luxon';
import { EventMessageTypeNames, JsonObject, JsonValue } from 'n8n-workflow';
export interface EventMessageConfirmSource extends JsonObject {
id: string;
name: string;
}
export class EventMessageConfirm {
readonly __type = EventMessageTypeNames.confirm;
readonly confirm: string;
readonly source?: EventMessageConfirmSource;
readonly ts: DateTime;
constructor(confirm: string, source?: EventMessageConfirmSource) {
this.confirm = confirm;
this.ts = DateTime.now();
if (source) this.source = source;
}
serialize(): JsonValue {
// TODO: filter payload for sensitive info here?
return {
__type: this.__type,
confirm: this.confirm,
ts: this.ts.toISO(),
source: this.source ?? { name: '', id: '' },
};
}
}
export const isEventMessageConfirm = (candidate: unknown): candidate is EventMessageConfirm => {
const o = candidate as EventMessageConfirm;
if (!o) return false;
return o.confirm !== undefined && o.ts !== undefined;
};

View File

@@ -0,0 +1,41 @@
import { EventMessageTypeNames, JsonObject } from 'n8n-workflow';
import { AbstractEventMessage, isEventMessageOptionsWithType } from './AbstractEventMessage';
import type { AbstractEventPayload } from './AbstractEventPayload';
import type { AbstractEventMessageOptions } from './AbstractEventMessageOptions';
export const eventMessageGenericDestinationTestEvent = 'n8n.destination.test';
export interface EventPayloadGeneric extends AbstractEventPayload {
msg?: string;
}
export interface EventMessageGenericOptions extends AbstractEventMessageOptions {
payload?: EventPayloadGeneric;
}
export class EventMessageGeneric extends AbstractEventMessage {
readonly __type = EventMessageTypeNames.generic;
payload: EventPayloadGeneric;
constructor(options: EventMessageGenericOptions) {
super(options);
if (options.payload) this.setPayload(options.payload);
if (options.anonymize) {
this.anonymize();
}
}
setPayload(payload: EventPayloadGeneric): this {
this.payload = payload;
return this;
}
deserialize(data: JsonObject): this {
if (isEventMessageOptionsWithType(data, this.__type)) {
this.setOptionsOrDefault(data);
if (data.payload) this.setPayload(data.payload as EventPayloadGeneric);
}
return this;
}
}

View File

@@ -0,0 +1,49 @@
import { AbstractEventMessage, isEventMessageOptionsWithType } from './AbstractEventMessage';
import { EventMessageTypeNames, JsonObject } from 'n8n-workflow';
import type { AbstractEventMessageOptions } from './AbstractEventMessageOptions';
import type { AbstractEventPayload } from './AbstractEventPayload';
export const eventNamesNode = ['n8n.node.started', 'n8n.node.finished'] as const;
export type EventNamesNodeType = typeof eventNamesNode[number];
// --------------------------------------
// EventMessage class for Node events
// --------------------------------------
export interface EventPayloadNode extends AbstractEventPayload {
msg?: string;
}
export interface EventMessageNodeOptions extends AbstractEventMessageOptions {
eventName: EventNamesNodeType;
payload?: EventPayloadNode | undefined;
}
export class EventMessageNode extends AbstractEventMessage {
readonly __type = EventMessageTypeNames.node;
eventName: EventNamesNodeType;
payload: EventPayloadNode;
constructor(options: EventMessageNodeOptions) {
super(options);
if (options.payload) this.setPayload(options.payload);
if (options.anonymize) {
this.anonymize();
}
}
setPayload(payload: EventPayloadNode): this {
this.payload = payload;
return this;
}
deserialize(data: JsonObject): this {
if (isEventMessageOptionsWithType(data, this.__type)) {
this.setOptionsOrDefault(data);
if (data.payload) this.setPayload(data.payload as EventPayloadNode);
}
return this;
}
}

View File

@@ -0,0 +1,61 @@
import { AbstractEventMessage, isEventMessageOptionsWithType } from './AbstractEventMessage';
import { EventMessageTypeNames, IWorkflowBase, JsonObject } from 'n8n-workflow';
import type { AbstractEventMessageOptions } from './AbstractEventMessageOptions';
import type { AbstractEventPayload } from './AbstractEventPayload';
import { IExecutionBase } from '@/Interfaces';
export const eventNamesWorkflow = [
'n8n.workflow.started',
'n8n.workflow.success',
'n8n.workflow.failed',
] as const;
export type EventNamesWorkflowType = typeof eventNamesWorkflow[number];
// --------------------------------------
// EventMessage class for Workflow events
// --------------------------------------
interface EventPayloadWorkflow extends AbstractEventPayload {
msg?: string;
workflowData?: IWorkflowBase;
executionId?: IExecutionBase['id'];
workflowId?: IWorkflowBase['id'];
}
export interface EventMessageWorkflowOptions extends AbstractEventMessageOptions {
eventName: EventNamesWorkflowType;
payload?: EventPayloadWorkflow | undefined;
}
export class EventMessageWorkflow extends AbstractEventMessage {
readonly __type = EventMessageTypeNames.workflow;
eventName: EventNamesWorkflowType;
payload: EventPayloadWorkflow;
constructor(options: EventMessageWorkflowOptions) {
super(options);
if (options.payload) this.setPayload(options.payload);
if (options.anonymize) {
this.anonymize();
}
}
setPayload(payload: EventPayloadWorkflow): this {
this.payload = payload;
return this;
}
deserialize(data: JsonObject): this {
if (isEventMessageOptionsWithType(data, this.__type)) {
this.setOptionsOrDefault(data);
if (data.payload) this.setPayload(data.payload as EventPayloadWorkflow);
}
return this;
}
}

View File

@@ -0,0 +1,92 @@
import type { EventMessageTypes } from '.';
import { EventMessageGeneric, EventMessageGenericOptions } from './EventMessageGeneric';
import type { AbstractEventMessageOptions } from './AbstractEventMessageOptions';
import { EventMessageWorkflow, EventMessageWorkflowOptions } from './EventMessageWorkflow';
import { EventMessageTypeNames } from 'n8n-workflow';
export const getEventMessageObjectByType = (
message: AbstractEventMessageOptions,
): EventMessageTypes | null => {
switch (message.__type as EventMessageTypeNames) {
case EventMessageTypeNames.generic:
return new EventMessageGeneric(message as EventMessageGenericOptions);
case EventMessageTypeNames.workflow:
return new EventMessageWorkflow(message as EventMessageWorkflowOptions);
default:
return null;
}
};
interface StringIndexedObject {
[key: string]: StringIndexedObject | string;
}
export function eventGroupFromEventName(eventName: string): string | undefined {
const matches = eventName.match(/^[\w\s]+\.[\w\s]+/);
if (matches && matches?.length > 0) {
return matches[0];
}
return;
}
function dotsToObject2(dottedString: string, o?: StringIndexedObject): StringIndexedObject {
const rootObject: StringIndexedObject = o ?? {};
if (!dottedString) return rootObject;
const parts = dottedString.split('.'); /*?*/
let part: string | undefined;
let obj: StringIndexedObject = rootObject;
while ((part = parts.shift())) {
if (typeof obj[part] !== 'object') {
obj[part] = {
__name: part,
};
}
obj = obj[part] as StringIndexedObject;
}
return rootObject;
}
export function eventListToObject(dottedList: string[]): object {
const result = {};
dottedList.forEach((e) => {
dotsToObject2(e, result);
});
return result;
}
interface StringIndexedChild {
name: string;
children: StringIndexedChild[];
}
export function eventListToObjectTree(dottedList: string[]): StringIndexedChild {
const x: StringIndexedChild = {
name: 'eventTree',
children: [] as unknown as StringIndexedChild[],
};
dottedList.forEach((dottedString: string) => {
const parts = dottedString.split('.');
let part: string | undefined;
let children = x.children;
while ((part = parts.shift())) {
if (part) {
// eslint-disable-next-line @typescript-eslint/no-loop-func
const foundChild = children.find((e) => e.name === part);
if (foundChild) {
children = foundChild.children;
} else {
const newChild: StringIndexedChild = {
name: part,
children: [],
};
children.push(newChild);
children = newChild.children;
}
}
}
});
return x;
}

View File

@@ -0,0 +1,17 @@
import { EventMessageAudit, eventNamesAudit, EventNamesAuditType } from './EventMessageAudit';
import { EventMessageGeneric } from './EventMessageGeneric';
import { EventMessageNode, eventNamesNode, EventNamesNodeType } from './EventMessageNode';
import {
EventMessageWorkflow,
eventNamesWorkflow,
EventNamesWorkflowType,
} from './EventMessageWorkflow';
export type EventNamesTypes = EventNamesAuditType | EventNamesWorkflowType | EventNamesNodeType;
export const eventNamesAll = [...eventNamesAudit, ...eventNamesWorkflow, ...eventNamesNode];
export type EventMessageTypes =
| EventMessageGeneric
| EventMessageWorkflow
| EventMessageAudit
| EventMessageNode;

View File

@@ -0,0 +1,253 @@
import { LoggerProxy, MessageEventBusDestinationOptions } from 'n8n-workflow';
import { DeleteResult } from 'typeorm';
import { EventMessageTypes } from '../EventMessageClasses/';
import type { MessageEventBusDestination } from '../MessageEventBusDestination/MessageEventBusDestination.ee';
import { MessageEventBusLogWriter } from '../MessageEventBusWriter/MessageEventBusLogWriter';
import EventEmitter from 'events';
import config from '@/config';
import * as Db from '@/Db';
import { messageEventBusDestinationFromDb } from '../MessageEventBusDestination/Helpers.ee';
import uniqby from 'lodash.uniqby';
import { EventMessageConfirmSource } from '../EventMessageClasses/EventMessageConfirm';
import {
EventMessageAuditOptions,
EventMessageAudit,
} from '../EventMessageClasses/EventMessageAudit';
import {
EventMessageWorkflowOptions,
EventMessageWorkflow,
} from '../EventMessageClasses/EventMessageWorkflow';
import { isLogStreamingEnabled } from './MessageEventBusHelper';
import { EventMessageNode, EventMessageNodeOptions } from '../EventMessageClasses/EventMessageNode';
import {
EventMessageGeneric,
eventMessageGenericDestinationTestEvent,
} from '../EventMessageClasses/EventMessageGeneric';
export type EventMessageReturnMode = 'sent' | 'unsent' | 'all';
class MessageEventBus extends EventEmitter {
private static instance: MessageEventBus;
isInitialized: boolean;
logWriter: MessageEventBusLogWriter;
destinations: {
[key: string]: MessageEventBusDestination;
} = {};
private pushIntervalTimer: NodeJS.Timer;
constructor() {
super();
this.isInitialized = false;
}
static getInstance(): MessageEventBus {
if (!MessageEventBus.instance) {
MessageEventBus.instance = new MessageEventBus();
}
return MessageEventBus.instance;
}
/**
* Needs to be called once at startup to set the event bus instance up. Will launch the event log writer and,
* if configured to do so, the previously stored event destinations.
*
* Will check for unsent event messages in the previous log files once at startup and try to re-send them.
*
* Sets `isInitialized` to `true` once finished.
*/
async initialize() {
if (this.isInitialized) {
return;
}
LoggerProxy.debug('Initializing event bus...');
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call
const savedEventDestinations = await Db.collections.EventDestinations.find({});
if (savedEventDestinations.length > 0) {
for (const destinationData of savedEventDestinations) {
try {
const destination = messageEventBusDestinationFromDb(destinationData);
if (destination) {
await this.addDestination(destination);
}
} catch (error) {
console.log(error);
}
}
}
LoggerProxy.debug('Initializing event writer');
this.logWriter = await MessageEventBusLogWriter.getInstance();
// unsent event check:
// - find unsent messages in current event log(s)
// - cycle event logs and start the logging to a fresh file
// - retry sending events
LoggerProxy.debug('Checking for unsent event messages');
const unsentMessages = await this.getEventsUnsent();
LoggerProxy.debug(
`Start logging into ${
(await this.logWriter?.getThread()?.getLogFileName()) ?? 'unknown filename'
} `,
);
await this.logWriter?.startLogging();
await this.send(unsentMessages);
// if configured, run this test every n ms
if (config.getEnv('eventBus.checkUnsentInterval') > 0) {
if (this.pushIntervalTimer) {
clearInterval(this.pushIntervalTimer);
}
this.pushIntervalTimer = setInterval(async () => {
await this.trySendingUnsent();
}, config.getEnv('eventBus.checkUnsentInterval'));
}
LoggerProxy.debug('MessageEventBus initialized');
this.isInitialized = true;
}
async addDestination(destination: MessageEventBusDestination) {
await this.removeDestination(destination.getId());
this.destinations[destination.getId()] = destination;
this.destinations[destination.getId()].startListening();
return destination;
}
async findDestination(id?: string): Promise<MessageEventBusDestinationOptions[]> {
let result: MessageEventBusDestinationOptions[];
if (id && Object.keys(this.destinations).includes(id)) {
result = [this.destinations[id].serialize()];
} else {
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
result = Object.keys(this.destinations).map((e) => this.destinations[e].serialize());
}
// eslint-disable-next-line @typescript-eslint/no-unsafe-return, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call
return result.sort((a, b) => (a.__type ?? '').localeCompare(b.__type ?? ''));
}
async removeDestination(id: string): Promise<DeleteResult | undefined> {
let result;
if (Object.keys(this.destinations).includes(id)) {
await this.destinations[id].close();
result = await this.destinations[id].deleteFromDb();
delete this.destinations[id];
}
return result;
}
private async trySendingUnsent(msgs?: EventMessageTypes[]) {
const unsentMessages = msgs ?? (await this.getEventsUnsent());
if (unsentMessages.length > 0) {
LoggerProxy.debug(`Found unsent event messages: ${unsentMessages.length}`);
for (const unsentMsg of unsentMessages) {
LoggerProxy.debug(`Retrying: ${unsentMsg.id} ${unsentMsg.__type}`);
await this.emitMessage(unsentMsg);
}
}
}
async close() {
LoggerProxy.debug('Shutting down event writer...');
await this.logWriter?.close();
for (const destinationName of Object.keys(this.destinations)) {
LoggerProxy.debug(
`Shutting down event destination ${this.destinations[destinationName].getId()}...`,
);
await this.destinations[destinationName].close();
}
LoggerProxy.debug('EventBus shut down.');
}
async send(msgs: EventMessageTypes | EventMessageTypes[]) {
if (!Array.isArray(msgs)) {
msgs = [msgs];
}
for (const msg of msgs) {
await this.logWriter?.putMessage(msg);
await this.emitMessage(msg);
}
}
async testDestination(destinationId: string): Promise<boolean> {
const testMessage = new EventMessageGeneric({
eventName: eventMessageGenericDestinationTestEvent,
});
const destination = await this.findDestination(destinationId);
if (destination.length > 0) {
const sendResult = await this.destinations[destinationId].receiveFromEventBus(testMessage);
return sendResult;
}
return false;
}
async confirmSent(msg: EventMessageTypes, source?: EventMessageConfirmSource) {
await this.logWriter?.confirmMessageSent(msg.id, source);
}
private async emitMessage(msg: EventMessageTypes) {
// generic emit for external modules to capture events
// this is for internal use ONLY and not for use with custom destinations!
this.emit('message', msg);
LoggerProxy.debug(`Listeners: ${this.eventNames().join(',')}`);
// if there are no set up destinations, immediately mark the event as sent
if (!isLogStreamingEnabled() || Object.keys(this.destinations).length === 0) {
await this.confirmSent(msg, { id: '0', name: 'eventBus' });
} else {
for (const destinationName of Object.keys(this.destinations)) {
this.emit(this.destinations[destinationName].getId(), msg);
}
}
}
async getEvents(mode: EventMessageReturnMode = 'all'): Promise<EventMessageTypes[]> {
let queryResult: EventMessageTypes[];
switch (mode) {
case 'all':
queryResult = await this.logWriter?.getMessages();
break;
case 'sent':
queryResult = await this.logWriter?.getMessagesSent();
break;
case 'unsent':
queryResult = await this.logWriter?.getMessagesUnsent();
}
const filtered = uniqby(queryResult, 'id');
return filtered;
}
async getEventsSent(): Promise<EventMessageTypes[]> {
const sentMessages = await this.getEvents('sent');
return sentMessages;
}
async getEventsUnsent(): Promise<EventMessageTypes[]> {
const unSentMessages = await this.getEvents('unsent');
return unSentMessages;
}
/**
* Convenience Methods
*/
async sendAuditEvent(options: EventMessageAuditOptions) {
await this.send(new EventMessageAudit(options));
}
async sendWorkflowEvent(options: EventMessageWorkflowOptions) {
await this.send(new EventMessageWorkflow(options));
}
async sendNodeEvent(options: EventMessageNodeOptions) {
await this.send(new EventMessageNode(options));
}
}
export const eventBus = MessageEventBus.getInstance();

View File

@@ -0,0 +1,7 @@
import config from '@/config';
import { getLicense } from '@/License';
export function isLogStreamingEnabled(): boolean {
const license = getLicense();
return config.getEnv('enterprise.features.logStreaming') || license.isLogStreamingEnabled();
}

View File

@@ -0,0 +1,28 @@
/* eslint-disable import/no-cycle */
/* eslint-disable @typescript-eslint/no-unsafe-member-access */
import { MessageEventBusDestinationTypeNames } from 'n8n-workflow';
import type { EventDestinations } from '@/databases/entities/MessageEventBusDestinationEntity';
import type { MessageEventBusDestination } from './MessageEventBusDestination.ee';
import { MessageEventBusDestinationSentry } from './MessageEventBusDestinationSentry.ee';
import { MessageEventBusDestinationSyslog } from './MessageEventBusDestinationSyslog.ee';
import { MessageEventBusDestinationWebhook } from './MessageEventBusDestinationWebhook.ee';
export function messageEventBusDestinationFromDb(
dbData: EventDestinations,
): MessageEventBusDestination | null {
// eslint-disable-next-line @typescript-eslint/no-unsafe-argument, @typescript-eslint/no-unsafe-assignment
const destinationData = dbData.destination;
if ('__type' in destinationData) {
switch (destinationData.__type) {
case MessageEventBusDestinationTypeNames.sentry:
return MessageEventBusDestinationSentry.deserialize(destinationData);
case MessageEventBusDestinationTypeNames.syslog:
return MessageEventBusDestinationSyslog.deserialize(destinationData);
case MessageEventBusDestinationTypeNames.webhook:
return MessageEventBusDestinationWebhook.deserialize(destinationData);
default:
console.log('MessageEventBusDestination __type unknown');
}
}
return null;
}

View File

@@ -0,0 +1,125 @@
/* eslint-disable @typescript-eslint/no-unsafe-member-access */
/* eslint-disable @typescript-eslint/no-unsafe-assignment */
import { v4 as uuid } from 'uuid';
import {
INodeCredentials,
LoggerProxy,
MessageEventBusDestinationOptions,
MessageEventBusDestinationTypeNames,
} from 'n8n-workflow';
import * as Db from '@/Db';
import { AbstractEventMessage } from '../EventMessageClasses/AbstractEventMessage';
import { EventMessageTypes } from '../EventMessageClasses';
import { eventBus } from '..';
import { DeleteResult, InsertResult } from 'typeorm';
export abstract class MessageEventBusDestination implements MessageEventBusDestinationOptions {
// Since you can't have static abstract functions - this just serves as a reminder that you need to implement these. Please.
// static abstract deserialize(): MessageEventBusDestination | null;
readonly id: string;
__type: MessageEventBusDestinationTypeNames;
label: string;
enabled: boolean;
subscribedEvents: string[];
credentials: INodeCredentials = {};
anonymizeAuditMessages: boolean;
constructor(options: MessageEventBusDestinationOptions) {
this.id = !options.id || options.id.length !== 36 ? uuid() : options.id;
this.__type = options.__type ?? MessageEventBusDestinationTypeNames.abstract;
this.label = options.label ?? 'Log Destination';
this.enabled = options.enabled ?? false;
this.subscribedEvents = options.subscribedEvents ?? [];
this.anonymizeAuditMessages = options.anonymizeAuditMessages ?? false;
if (options.credentials) this.credentials = options.credentials;
LoggerProxy.debug(`${this.__type}(${this.id}) event destination constructed`);
}
startListening() {
if (this.enabled) {
eventBus.on(this.getId(), async (msg: EventMessageTypes) => {
await this.receiveFromEventBus(msg);
});
LoggerProxy.debug(`${this.id} listener started`);
}
}
stopListening() {
eventBus.removeAllListeners(this.getId());
}
enable() {
this.enabled = true;
this.startListening();
}
disable() {
this.enabled = false;
this.stopListening();
}
getId() {
return this.id;
}
hasSubscribedToEvent(msg: AbstractEventMessage) {
if (!this.enabled) return false;
for (const eventName of this.subscribedEvents) {
if (eventName === '*' || msg.eventName.startsWith(eventName)) {
return true;
}
}
return false;
}
async saveToDb() {
const data = {
id: this.getId(),
destination: this.serialize(),
};
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call
const dbResult: InsertResult = await Db.collections.EventDestinations.upsert(data, {
skipUpdateIfNoValuesChanged: true,
conflictPaths: ['id'],
});
Db.collections.EventDestinations.createQueryBuilder().insert().into('something').onConflict('');
return dbResult;
}
async deleteFromDb() {
return MessageEventBusDestination.deleteFromDb(this.getId());
}
static async deleteFromDb(id: string): Promise<DeleteResult> {
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access
const dbResult = await Db.collections.EventDestinations.delete({ id });
return dbResult;
}
serialize(): MessageEventBusDestinationOptions {
return {
__type: this.__type,
id: this.getId(),
label: this.label,
enabled: this.enabled,
subscribedEvents: this.subscribedEvents,
anonymizeAuditMessages: this.anonymizeAuditMessages,
};
}
abstract receiveFromEventBus(msg: AbstractEventMessage): Promise<boolean>;
toString() {
return JSON.stringify(this.serialize());
}
close(): void | Promise<void> {
this.stopListening();
}
}

View File

@@ -0,0 +1,137 @@
/* eslint-disable @typescript-eslint/no-unsafe-member-access */
/* eslint-disable @typescript-eslint/no-unsafe-return */
/* eslint-disable @typescript-eslint/no-unsafe-assignment */
import { MessageEventBusDestination } from './MessageEventBusDestination.ee';
import * as Sentry from '@sentry/node';
import { eventBus } from '../MessageEventBus/MessageEventBus';
import {
LoggerProxy,
MessageEventBusDestinationOptions,
MessageEventBusDestinationSentryOptions,
MessageEventBusDestinationTypeNames,
} from 'n8n-workflow';
import { GenericHelpers } from '../..';
import { isLogStreamingEnabled } from '../MessageEventBus/MessageEventBusHelper';
import { EventMessageTypes } from '../EventMessageClasses';
import { eventMessageGenericDestinationTestEvent } from '../EventMessageClasses/EventMessageGeneric';
export const isMessageEventBusDestinationSentryOptions = (
candidate: unknown,
): candidate is MessageEventBusDestinationSentryOptions => {
const o = candidate as MessageEventBusDestinationSentryOptions;
if (!o) return false;
return o.dsn !== undefined;
};
export class MessageEventBusDestinationSentry
extends MessageEventBusDestination
implements MessageEventBusDestinationSentryOptions
{
dsn: string;
tracesSampleRate = 1.0;
sendPayload: boolean;
sentryClient?: Sentry.NodeClient;
constructor(options: MessageEventBusDestinationSentryOptions) {
super(options);
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
this.label = options.label ?? 'Sentry DSN';
this.__type = options.__type ?? MessageEventBusDestinationTypeNames.sentry;
this.dsn = options.dsn;
if (options.sendPayload) this.sendPayload = options.sendPayload;
if (options.tracesSampleRate) this.tracesSampleRate = options.tracesSampleRate;
const { ENVIRONMENT: environment } = process.env;
GenericHelpers.getVersions()
.then((versions) => {
this.sentryClient = new Sentry.NodeClient({
dsn: this.dsn,
tracesSampleRate: this.tracesSampleRate,
environment,
release: versions.cli,
transport: Sentry.makeNodeTransport,
integrations: Sentry.defaultIntegrations,
stackParser: Sentry.defaultStackParser,
});
LoggerProxy.debug(`MessageEventBusDestinationSentry with id ${this.getId()} initialized`);
})
.catch((error) => {
console.error(error);
});
}
async receiveFromEventBus(msg: EventMessageTypes): Promise<boolean> {
let sendResult = false;
if (!this.sentryClient) return sendResult;
if (msg.eventName !== eventMessageGenericDestinationTestEvent) {
if (!isLogStreamingEnabled()) return sendResult;
if (!this.hasSubscribedToEvent(msg)) return sendResult;
}
try {
const payload = this.anonymizeAuditMessages ? msg.anonymize() : msg.payload;
const scope: Sentry.Scope = new Sentry.Scope();
const level = (
msg.eventName.toLowerCase().endsWith('error') ? 'error' : 'log'
) as Sentry.SeverityLevel;
scope.setLevel(level);
scope.setTags({
event: msg.getEventName(),
logger: this.label ?? this.getId(),
app: 'n8n',
});
if (this.sendPayload) {
scope.setExtras(payload);
}
const sentryResult = this.sentryClient.captureMessage(
msg.message ?? msg.eventName,
level,
{ event_id: msg.id, data: payload },
scope,
);
if (sentryResult) {
await eventBus.confirmSent(msg, { id: this.id, name: this.label });
sendResult = true;
}
} catch (error) {
console.log(error);
}
return sendResult;
}
serialize(): MessageEventBusDestinationSentryOptions {
const abstractSerialized = super.serialize();
return {
...abstractSerialized,
dsn: this.dsn,
tracesSampleRate: this.tracesSampleRate,
sendPayload: this.sendPayload,
};
}
static deserialize(
data: MessageEventBusDestinationOptions,
): MessageEventBusDestinationSentry | null {
if (
'__type' in data &&
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
data.__type === MessageEventBusDestinationTypeNames.sentry &&
isMessageEventBusDestinationSentryOptions(data)
) {
return new MessageEventBusDestinationSentry(data);
}
return null;
}
toString() {
return JSON.stringify(this.serialize());
}
async close() {
await super.close();
await this.sentryClient?.close();
}
}

View File

@@ -0,0 +1,149 @@
/* eslint-disable @typescript-eslint/no-unsafe-assignment */
/* eslint-disable @typescript-eslint/no-non-null-assertion */
/* eslint-disable @typescript-eslint/no-unsafe-member-access */
import syslog from 'syslog-client';
import { eventBus } from '../MessageEventBus/MessageEventBus';
import {
LoggerProxy,
MessageEventBusDestinationOptions,
MessageEventBusDestinationSyslogOptions,
MessageEventBusDestinationTypeNames,
} from 'n8n-workflow';
import { MessageEventBusDestination } from './MessageEventBusDestination.ee';
import { isLogStreamingEnabled } from '../MessageEventBus/MessageEventBusHelper';
import { EventMessageTypes } from '../EventMessageClasses';
import { eventMessageGenericDestinationTestEvent } from '../EventMessageClasses/EventMessageGeneric';
export const isMessageEventBusDestinationSyslogOptions = (
candidate: unknown,
): candidate is MessageEventBusDestinationSyslogOptions => {
const o = candidate as MessageEventBusDestinationSyslogOptions;
if (!o) return false;
return o.host !== undefined;
};
export class MessageEventBusDestinationSyslog
extends MessageEventBusDestination
implements MessageEventBusDestinationSyslogOptions
{
client: syslog.Client;
expectedStatusCode?: number;
host: string;
port: number;
protocol: 'udp' | 'tcp';
facility: syslog.Facility;
app_name: string;
eol: string;
constructor(options: MessageEventBusDestinationSyslogOptions) {
super(options);
this.__type = options.__type ?? MessageEventBusDestinationTypeNames.syslog;
this.label = options.label ?? 'Syslog Server';
this.host = options.host ?? 'localhost';
this.port = options.port ?? 514;
this.protocol = options.protocol ?? 'udp';
this.facility = options.facility ?? syslog.Facility.Local0;
this.app_name = options.app_name ?? 'n8n';
this.eol = options.eol ?? '\n';
this.expectedStatusCode = options.expectedStatusCode ?? 200;
this.client = syslog.createClient(this.host, {
appName: this.app_name,
facility: syslog.Facility.Local0,
// severity: syslog.Severity.Error,
port: this.port,
transport:
options.protocol !== undefined && options.protocol === 'tcp'
? syslog.Transport.Tcp
: syslog.Transport.Udp,
});
LoggerProxy.debug(`MessageEventBusDestinationSyslog with id ${this.getId()} initialized`);
this.client.on('error', function (error) {
console.error(error);
});
}
async receiveFromEventBus(msg: EventMessageTypes): Promise<boolean> {
let sendResult = false;
if (msg.eventName !== eventMessageGenericDestinationTestEvent) {
if (!isLogStreamingEnabled()) return sendResult;
if (!this.hasSubscribedToEvent(msg)) return sendResult;
}
try {
const serializedMessage = msg.serialize();
if (this.anonymizeAuditMessages) {
serializedMessage.payload = msg.anonymize();
}
delete serializedMessage.__type;
this.client.log(
JSON.stringify(serializedMessage),
{
severity: msg.eventName.toLowerCase().endsWith('error')
? syslog.Severity.Error
: syslog.Severity.Debug,
msgid: msg.id,
timestamp: msg.ts.toJSDate(),
},
async (error) => {
if (error) {
console.log(error);
} else {
await eventBus.confirmSent(msg, { id: this.id, name: this.label });
sendResult = true;
}
},
);
} catch (error) {
console.log(error);
}
if (msg.eventName === eventMessageGenericDestinationTestEvent) {
await new Promise((resolve) => setTimeout(resolve, 500));
}
return sendResult;
}
serialize(): MessageEventBusDestinationSyslogOptions {
const abstractSerialized = super.serialize();
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
return {
...abstractSerialized,
expectedStatusCode: this.expectedStatusCode,
host: this.host,
port: this.port,
protocol: this.protocol,
facility: this.facility,
app_name: this.app_name,
eol: this.eol,
};
}
static deserialize(
data: MessageEventBusDestinationOptions,
): MessageEventBusDestinationSyslog | null {
if (
'__type' in data &&
data.__type === MessageEventBusDestinationTypeNames.syslog &&
isMessageEventBusDestinationSyslogOptions(data)
) {
return new MessageEventBusDestinationSyslog(data);
}
return null;
}
toString() {
return JSON.stringify(this.serialize());
}
async close() {
await super.close();
this.client.close();
}
}

View File

@@ -0,0 +1,372 @@
/* eslint-disable import/no-cycle */
/* eslint-disable @typescript-eslint/no-unsafe-call */
/* eslint-disable @typescript-eslint/no-unsafe-return */
/* eslint-disable @typescript-eslint/no-unsafe-member-access */
/* eslint-disable @typescript-eslint/no-unsafe-assignment */
/* eslint-disable @typescript-eslint/no-unused-vars */
/* eslint-disable @typescript-eslint/no-unnecessary-boolean-literal-compare */
import { MessageEventBusDestination } from './MessageEventBusDestination.ee';
import axios, { AxiosRequestConfig, Method } from 'axios';
import { eventBus } from '../MessageEventBus/MessageEventBus';
import { EventMessageTypes } from '../EventMessageClasses';
import {
jsonParse,
LoggerProxy,
MessageEventBusDestinationOptions,
MessageEventBusDestinationTypeNames,
MessageEventBusDestinationWebhookOptions,
MessageEventBusDestinationWebhookParameterItem,
MessageEventBusDestinationWebhookParameterOptions,
} from 'n8n-workflow';
import { CredentialsHelper } from '../../CredentialsHelper';
import { UserSettings } from 'n8n-core';
import { Agent as HTTPSAgent } from 'https';
import config from '../../config';
import { isLogStreamingEnabled } from '../MessageEventBus/MessageEventBusHelper';
import { eventMessageGenericDestinationTestEvent } from '../EventMessageClasses/EventMessageGeneric';
export const isMessageEventBusDestinationWebhookOptions = (
candidate: unknown,
): candidate is MessageEventBusDestinationWebhookOptions => {
const o = candidate as MessageEventBusDestinationWebhookOptions;
if (!o) return false;
return o.url !== undefined;
};
export class MessageEventBusDestinationWebhook
extends MessageEventBusDestination
implements MessageEventBusDestinationWebhookOptions
{
url: string;
responseCodeMustMatch = false;
expectedStatusCode = 200;
method = 'POST';
authentication: 'predefinedCredentialType' | 'genericCredentialType' | 'none' = 'none';
sendQuery = false;
sendHeaders = false;
genericAuthType = '';
nodeCredentialType = '';
specifyHeaders = '';
specifyQuery = '';
jsonQuery = '';
jsonHeaders = '';
headerParameters: MessageEventBusDestinationWebhookParameterItem = { parameters: [] };
queryParameters: MessageEventBusDestinationWebhookParameterItem = { parameters: [] };
options: MessageEventBusDestinationWebhookParameterOptions = {};
sendPayload = true;
credentialsHelper?: CredentialsHelper;
axiosRequestOptions: AxiosRequestConfig;
constructor(options: MessageEventBusDestinationWebhookOptions) {
super(options);
this.url = options.url;
this.label = options.label ?? 'Webhook Endpoint';
this.__type = options.__type ?? MessageEventBusDestinationTypeNames.webhook;
if (options.responseCodeMustMatch) this.responseCodeMustMatch = options.responseCodeMustMatch;
if (options.expectedStatusCode) this.expectedStatusCode = options.expectedStatusCode;
if (options.method) this.method = options.method;
if (options.authentication) this.authentication = options.authentication;
if (options.sendQuery) this.sendQuery = options.sendQuery;
if (options.sendHeaders) this.sendHeaders = options.sendHeaders;
if (options.genericAuthType) this.genericAuthType = options.genericAuthType;
if (options.nodeCredentialType) this.nodeCredentialType = options.nodeCredentialType;
if (options.specifyHeaders) this.specifyHeaders = options.specifyHeaders;
if (options.specifyQuery) this.specifyQuery = options.specifyQuery;
if (options.jsonQuery) this.jsonQuery = options.jsonQuery;
if (options.jsonHeaders) this.jsonHeaders = options.jsonHeaders;
if (options.headerParameters) this.headerParameters = options.headerParameters;
if (options.queryParameters) this.queryParameters = options.queryParameters;
if (options.sendPayload) this.sendPayload = options.sendPayload;
if (options.options) this.options = options.options;
LoggerProxy.debug(`MessageEventBusDestinationWebhook with id ${this.getId()} initialized`);
}
async matchDecryptedCredentialType(credentialType: string) {
const foundCredential = Object.entries(this.credentials).find((e) => e[0] === credentialType);
if (foundCredential) {
const timezone = config.getEnv('generic.timezone');
const credentialsDecrypted = await this.credentialsHelper?.getDecrypted(
foundCredential[1],
foundCredential[0],
'internal',
timezone,
true,
);
return credentialsDecrypted;
}
return null;
}
async generateAxiosOptions() {
if (this.axiosRequestOptions?.url) {
return;
}
this.axiosRequestOptions = {
headers: {},
method: this.method as Method,
url: this.url,
maxRedirects: 0,
} as AxiosRequestConfig;
if (this.credentialsHelper === undefined) {
let encryptionKey: string | undefined;
try {
encryptionKey = await UserSettings.getEncryptionKey();
} catch (_) {}
if (encryptionKey) {
this.credentialsHelper = new CredentialsHelper(encryptionKey);
}
}
const sendQuery = this.sendQuery;
const specifyQuery = this.specifyQuery;
const sendPayload = this.sendPayload;
const sendHeaders = this.sendHeaders;
const specifyHeaders = this.specifyHeaders;
if (this.options.allowUnauthorizedCerts) {
this.axiosRequestOptions.httpsAgent = new HTTPSAgent({ rejectUnauthorized: false });
}
if (this.options.redirect?.followRedirects) {
this.axiosRequestOptions.maxRedirects = this.options.redirect?.maxRedirects;
}
if (this.options.proxy) {
this.axiosRequestOptions.proxy = this.options.proxy;
}
if (this.options.timeout) {
this.axiosRequestOptions.timeout = this.options.timeout;
} else {
this.axiosRequestOptions.timeout = 10000;
}
if (this.sendQuery && this.options.queryParameterArrays) {
Object.assign(this.axiosRequestOptions, {
qsStringifyOptions: { arrayFormat: this.options.queryParameterArrays },
});
}
const parametersToKeyValue = async (
// eslint-disable-next-line @typescript-eslint/no-explicit-any
acc: Promise<{ [key: string]: any }>,
cur: { name: string; value: string; parameterType?: string; inputDataFieldName?: string },
) => {
const acumulator = await acc;
acumulator[cur.name] = cur.value;
return acumulator;
};
// Get parameters defined in the UI
if (sendQuery && this.queryParameters.parameters) {
if (specifyQuery === 'keypair') {
this.axiosRequestOptions.params = this.queryParameters.parameters.reduce(
parametersToKeyValue,
Promise.resolve({}),
);
} else if (specifyQuery === 'json') {
// query is specified using JSON
try {
JSON.parse(this.jsonQuery);
} catch (_) {
console.log('JSON parameter need to be an valid JSON');
}
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
this.axiosRequestOptions.params = jsonParse(this.jsonQuery);
}
}
// Get parameters defined in the UI
if (sendHeaders && this.headerParameters.parameters) {
if (specifyHeaders === 'keypair') {
this.axiosRequestOptions.headers = await this.headerParameters.parameters.reduce(
parametersToKeyValue,
Promise.resolve({}),
);
} else if (specifyHeaders === 'json') {
// body is specified using JSON
try {
JSON.parse(this.jsonHeaders);
} catch (_) {
console.log('JSON parameter need to be an valid JSON');
}
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
this.axiosRequestOptions.headers = jsonParse(this.jsonHeaders);
}
}
// default for bodyContentType.raw
if (this.axiosRequestOptions.headers === undefined) {
this.axiosRequestOptions.headers = {};
}
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
this.axiosRequestOptions.headers['Content-Type'] = 'application/json';
}
serialize(): MessageEventBusDestinationWebhookOptions {
const abstractSerialized = super.serialize();
return {
...abstractSerialized,
url: this.url,
responseCodeMustMatch: this.responseCodeMustMatch,
expectedStatusCode: this.expectedStatusCode,
method: this.method,
authentication: this.authentication,
sendQuery: this.sendQuery,
sendHeaders: this.sendHeaders,
genericAuthType: this.genericAuthType,
nodeCredentialType: this.nodeCredentialType,
specifyHeaders: this.specifyHeaders,
specifyQuery: this.specifyQuery,
jsonQuery: this.jsonQuery,
jsonHeaders: this.jsonHeaders,
headerParameters: this.headerParameters,
queryParameters: this.queryParameters,
sendPayload: this.sendPayload,
options: this.options,
credentials: this.credentials,
};
}
static deserialize(
data: MessageEventBusDestinationOptions,
): MessageEventBusDestinationWebhook | null {
if (
'__type' in data &&
data.__type === MessageEventBusDestinationTypeNames.webhook &&
isMessageEventBusDestinationWebhookOptions(data)
) {
return new MessageEventBusDestinationWebhook(data);
}
return null;
}
async receiveFromEventBus(msg: EventMessageTypes): Promise<boolean> {
let sendResult = false;
if (msg.eventName !== eventMessageGenericDestinationTestEvent) {
if (!isLogStreamingEnabled()) return sendResult;
if (!this.hasSubscribedToEvent(msg)) return sendResult;
}
// at first run, build this.requestOptions with the destination settings
await this.generateAxiosOptions();
const payload = this.anonymizeAuditMessages ? msg.anonymize() : msg.payload;
if (['PATCH', 'POST', 'PUT', 'GET'].includes(this.method.toUpperCase())) {
if (this.sendPayload) {
this.axiosRequestOptions.data = {
...msg,
__type: undefined,
payload,
ts: msg.ts.toISO(),
};
} else {
this.axiosRequestOptions.data = {
...msg,
__type: undefined,
payload: undefined,
ts: msg.ts.toISO(),
};
}
}
// TODO: implement extra auth requests
let httpBasicAuth;
let httpDigestAuth;
let httpHeaderAuth;
let httpQueryAuth;
let oAuth1Api;
let oAuth2Api;
if (this.authentication === 'genericCredentialType') {
if (this.genericAuthType === 'httpBasicAuth') {
try {
httpBasicAuth = await this.matchDecryptedCredentialType('httpBasicAuth');
} catch (_) {}
} else if (this.genericAuthType === 'httpDigestAuth') {
try {
httpDigestAuth = await this.matchDecryptedCredentialType('httpDigestAuth');
} catch (_) {}
} else if (this.genericAuthType === 'httpHeaderAuth') {
try {
httpHeaderAuth = await this.matchDecryptedCredentialType('httpHeaderAuth');
} catch (_) {}
} else if (this.genericAuthType === 'httpQueryAuth') {
try {
httpQueryAuth = await this.matchDecryptedCredentialType('httpQueryAuth');
} catch (_) {}
} else if (this.genericAuthType === 'oAuth1Api') {
try {
oAuth1Api = await this.matchDecryptedCredentialType('oAuth1Api');
} catch (_) {}
} else if (this.genericAuthType === 'oAuth2Api') {
try {
oAuth2Api = await this.matchDecryptedCredentialType('oAuth2Api');
} catch (_) {}
}
}
if (httpBasicAuth) {
// Add credentials if any are set
this.axiosRequestOptions.auth = {
username: httpBasicAuth.user as string,
password: httpBasicAuth.password as string,
};
} else if (httpHeaderAuth) {
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
this.axiosRequestOptions.headers[httpHeaderAuth.name as string] = httpHeaderAuth.value;
} else if (httpQueryAuth) {
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
this.axiosRequestOptions.params[httpQueryAuth.name as string] = httpQueryAuth.value;
} else if (httpDigestAuth) {
this.axiosRequestOptions.auth = {
username: httpDigestAuth.user as string,
password: httpDigestAuth.password as string,
};
}
try {
const requestResponse = await axios.request(this.axiosRequestOptions);
if (requestResponse) {
if (this.responseCodeMustMatch) {
if (requestResponse.status === this.expectedStatusCode) {
await eventBus.confirmSent(msg, { id: this.id, name: this.label });
sendResult = true;
} else {
sendResult = false;
}
} else {
await eventBus.confirmSent(msg, { id: this.id, name: this.label });
sendResult = true;
}
}
} catch (error) {
console.error(error);
}
return sendResult;
}
}

View File

@@ -0,0 +1,221 @@
/* eslint-disable @typescript-eslint/no-unsafe-member-access */
/* eslint-disable @typescript-eslint/no-explicit-any */
import { isEventMessageOptions } from '../EventMessageClasses/AbstractEventMessage';
import { UserSettings } from 'n8n-core';
import path, { parse } from 'path';
import { ModuleThread, spawn, Thread, Worker } from 'threads';
import { MessageEventBusLogWriterWorker } from './MessageEventBusLogWriterWorker';
import { createReadStream, existsSync } from 'fs';
import readline from 'readline';
import { jsonParse, LoggerProxy } from 'n8n-workflow';
import remove from 'lodash.remove';
import config from '@/config';
import { getEventMessageObjectByType } from '../EventMessageClasses/Helpers';
import type { EventMessageReturnMode } from '../MessageEventBus/MessageEventBus';
import type { EventMessageTypes } from '../EventMessageClasses';
import {
EventMessageConfirm,
EventMessageConfirmSource,
isEventMessageConfirm,
} from '../EventMessageClasses/EventMessageConfirm';
import { once as eventOnce } from 'events';
interface MessageEventBusLogWriterOptions {
syncFileAccess?: boolean;
logBaseName?: string;
logBasePath?: string;
keepLogCount?: number;
maxFileSizeInKB?: number;
}
/**
* MessageEventBusWriter for Files
*/
export class MessageEventBusLogWriter {
private static instance: MessageEventBusLogWriter;
static options: Required<MessageEventBusLogWriterOptions>;
private worker: ModuleThread<MessageEventBusLogWriterWorker> | null;
/**
* Instantiates the Writer and the corresponding worker thread.
* To actually start logging, call startLogging() function on the instance.
*
* **Note** that starting to log will archive existing logs, so handle unsent events first before calling startLogging()
*/
static async getInstance(
options?: MessageEventBusLogWriterOptions,
): Promise<MessageEventBusLogWriter> {
if (!MessageEventBusLogWriter.instance) {
MessageEventBusLogWriter.instance = new MessageEventBusLogWriter();
MessageEventBusLogWriter.options = {
logBaseName: options?.logBaseName ?? config.getEnv('eventBus.logWriter.logBaseName'),
logBasePath: options?.logBasePath ?? UserSettings.getUserN8nFolderPath(),
syncFileAccess:
options?.syncFileAccess ?? config.getEnv('eventBus.logWriter.syncFileAccess'),
keepLogCount: options?.keepLogCount ?? config.getEnv('eventBus.logWriter.keepLogCount'),
maxFileSizeInKB:
options?.maxFileSizeInKB ?? config.getEnv('eventBus.logWriter.maxFileSizeInKB'),
};
await MessageEventBusLogWriter.instance.startThread();
}
return MessageEventBusLogWriter.instance;
}
/**
* First archives existing log files one history level upwards,
* then starts logging events into a fresh event log
*/
async startLogging() {
await MessageEventBusLogWriter.instance.getThread()?.startLogging();
}
/**
* Pauses all logging. Events are still received by the worker, they just are not logged any more
*/
async pauseLogging() {
await MessageEventBusLogWriter.instance.getThread()?.pauseLogging();
}
private async startThread() {
if (this.worker) {
await this.close();
}
await MessageEventBusLogWriter.instance.spawnThread();
await MessageEventBusLogWriter.instance
.getThread()
?.initialize(
path.join(
MessageEventBusLogWriter.options.logBasePath,
MessageEventBusLogWriter.options.logBaseName,
),
MessageEventBusLogWriter.options.syncFileAccess,
MessageEventBusLogWriter.options.keepLogCount,
MessageEventBusLogWriter.options.maxFileSizeInKB,
);
}
private async spawnThread(): Promise<boolean> {
this.worker = await spawn<MessageEventBusLogWriterWorker>(
new Worker(`${parse(__filename).name}Worker`),
);
if (this.worker) {
Thread.errors(this.worker).subscribe(async (error) => {
LoggerProxy.error('Event Bus Log Writer thread error', error);
await MessageEventBusLogWriter.instance.startThread();
});
return true;
}
return false;
}
getThread(): ModuleThread<MessageEventBusLogWriterWorker> | undefined {
if (this.worker) {
return this.worker;
}
return;
}
async close(): Promise<void> {
if (this.worker) {
await Thread.terminate(this.worker);
this.worker = null;
}
}
async putMessage(msg: EventMessageTypes): Promise<void> {
if (this.worker) {
await this.worker.appendMessageToLog(msg.serialize());
}
}
async confirmMessageSent(msgId: string, source?: EventMessageConfirmSource): Promise<void> {
if (this.worker) {
await this.worker.confirmMessageSent(new EventMessageConfirm(msgId, source).serialize());
}
}
async getMessages(
mode: EventMessageReturnMode = 'all',
includePreviousLog = true,
): Promise<EventMessageTypes[]> {
const logFileName0 = await MessageEventBusLogWriter.instance.getThread()?.getLogFileName();
const logFileName1 = includePreviousLog
? await MessageEventBusLogWriter.instance.getThread()?.getLogFileName(1)
: undefined;
const results: {
loggedMessages: EventMessageTypes[];
sentMessages: EventMessageTypes[];
} = {
loggedMessages: [],
sentMessages: [],
};
if (logFileName0) {
await this.readLoggedMessagesFromFile(results, mode, logFileName0);
}
if (logFileName1) {
await this.readLoggedMessagesFromFile(results, mode, logFileName1);
}
switch (mode) {
case 'all':
case 'unsent':
return results.loggedMessages;
case 'sent':
return results.sentMessages;
}
}
async readLoggedMessagesFromFile(
results: {
loggedMessages: EventMessageTypes[];
sentMessages: EventMessageTypes[];
},
mode: EventMessageReturnMode,
logFileName: string,
): Promise<{
loggedMessages: EventMessageTypes[];
sentMessages: EventMessageTypes[];
}> {
if (logFileName && existsSync(logFileName)) {
try {
const rl = readline.createInterface({
input: createReadStream(logFileName),
crlfDelay: Infinity,
});
rl.on('line', (line) => {
try {
const json = jsonParse(line);
if (isEventMessageOptions(json) && json.__type !== undefined) {
const msg = getEventMessageObjectByType(json);
if (msg !== null) results.loggedMessages.push(msg);
}
if (isEventMessageConfirm(json) && mode !== 'all') {
const removedMessage = remove(results.loggedMessages, (e) => e.id === json.confirm);
if (mode === 'sent') {
results.sentMessages.push(...removedMessage);
}
}
} catch {
LoggerProxy.error(
`Error reading line messages from file: ${logFileName}, line: ${line}`,
);
}
});
// wait for stream to finish before continue
await eventOnce(rl, 'close');
} catch {
LoggerProxy.error(`Error reading logged messages from file: ${logFileName}`);
}
}
return results;
}
async getMessagesSent(): Promise<EventMessageTypes[]> {
return this.getMessages('sent');
}
async getMessagesUnsent(): Promise<EventMessageTypes[]> {
return this.getMessages('unsent');
}
}

View File

@@ -0,0 +1,145 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
import { appendFileSync, existsSync, rmSync, renameSync, openSync, closeSync } from 'fs';
import { appendFile, stat } from 'fs/promises';
import { expose, isWorkerRuntime } from 'threads/worker';
// -----------------------------------------
// * This part runs in the Worker Thread ! *
// -----------------------------------------
// all references to and imports from classes have been remove to keep memory usage low
let logFileBasePath = '';
let loggingPaused = true;
let syncFileAccess = false;
let keepFiles = 10;
let fileStatTimer: NodeJS.Timer;
let maxLogFileSizeInKB = 102400;
function setLogFileBasePath(basePath: string) {
logFileBasePath = basePath;
}
function setUseSyncFileAccess(useSync: boolean) {
syncFileAccess = useSync;
}
function setMaxLogFileSizeInKB(maxSizeInKB: number) {
maxLogFileSizeInKB = maxSizeInKB;
}
function setKeepFiles(keepNumberOfFiles: number) {
if (keepNumberOfFiles < 1) {
keepNumberOfFiles = 1;
}
keepFiles = keepNumberOfFiles;
}
function buildLogFileNameWithCounter(counter?: number): string {
if (counter) {
return `${logFileBasePath}-${counter}.log`;
} else {
return `${logFileBasePath}.log`;
}
}
function cleanAllLogs() {
for (let i = 0; i <= keepFiles; i++) {
if (existsSync(buildLogFileNameWithCounter(i))) {
rmSync(buildLogFileNameWithCounter(i));
}
}
}
/**
* Runs synchronously and cycles through log files up to the max amount kept
*/
function renameAndCreateLogs() {
if (existsSync(buildLogFileNameWithCounter(keepFiles))) {
rmSync(buildLogFileNameWithCounter(keepFiles));
}
for (let i = keepFiles - 1; i >= 0; i--) {
if (existsSync(buildLogFileNameWithCounter(i))) {
renameSync(buildLogFileNameWithCounter(i), buildLogFileNameWithCounter(i + 1));
}
}
const f = openSync(buildLogFileNameWithCounter(), 'a');
closeSync(f);
}
async function checkFileSize(path: string) {
const fileStat = await stat(path);
if (fileStat.size / 1024 > maxLogFileSizeInKB) {
renameAndCreateLogs();
}
}
function appendMessageSync(msg: any) {
if (loggingPaused) {
return;
}
appendFileSync(buildLogFileNameWithCounter(), JSON.stringify(msg) + '\n');
}
async function appendMessage(msg: any) {
if (loggingPaused) {
return;
}
await appendFile(buildLogFileNameWithCounter(), JSON.stringify(msg) + '\n');
}
const messageEventBusLogWriterWorker = {
async appendMessageToLog(msg: any) {
if (syncFileAccess) {
appendMessageSync(msg);
} else {
await appendMessage(msg);
}
},
async confirmMessageSent(confirm: unknown) {
if (syncFileAccess) {
appendMessageSync(confirm);
} else {
await appendMessage(confirm);
}
},
pauseLogging() {
loggingPaused = true;
clearInterval(fileStatTimer);
},
initialize(
basePath: string,
useSyncFileAccess = false,
keepNumberOfFiles = 10,
maxSizeInKB = 102400,
) {
setLogFileBasePath(basePath);
setUseSyncFileAccess(useSyncFileAccess);
setKeepFiles(keepNumberOfFiles);
setMaxLogFileSizeInKB(maxSizeInKB);
},
startLogging() {
if (logFileBasePath) {
renameAndCreateLogs();
loggingPaused = false;
fileStatTimer = setInterval(async () => {
await checkFileSize(buildLogFileNameWithCounter());
}, 5000);
}
},
getLogFileName(counter?: number) {
if (logFileBasePath) {
return buildLogFileNameWithCounter(counter);
} else {
return undefined;
}
},
cleanLogs() {
cleanAllLogs();
},
};
if (isWorkerRuntime()) {
// Register the serializer on the worker thread
expose(messageEventBusLogWriterWorker);
}
export type MessageEventBusLogWriterWorker = typeof messageEventBusLogWriterWorker;

View File

@@ -0,0 +1,219 @@
/* eslint-disable @typescript-eslint/no-unsafe-call */
/* eslint-disable @typescript-eslint/no-unsafe-member-access */
/* eslint-disable @typescript-eslint/no-explicit-any */
import express from 'express';
import { ResponseHelper } from '..';
import { isEventMessageOptions } from './EventMessageClasses/AbstractEventMessage';
import { EventMessageGeneric } from './EventMessageClasses/EventMessageGeneric';
import {
EventMessageWorkflow,
EventMessageWorkflowOptions,
} from './EventMessageClasses/EventMessageWorkflow';
import { eventBus, EventMessageReturnMode } from './MessageEventBus/MessageEventBus';
import {
isMessageEventBusDestinationSentryOptions,
MessageEventBusDestinationSentry,
} from './MessageEventBusDestination/MessageEventBusDestinationSentry.ee';
import {
isMessageEventBusDestinationSyslogOptions,
MessageEventBusDestinationSyslog,
} from './MessageEventBusDestination/MessageEventBusDestinationSyslog.ee';
import { MessageEventBusDestinationWebhook } from './MessageEventBusDestination/MessageEventBusDestinationWebhook.ee';
import { eventNamesAll } from './EventMessageClasses';
import {
EventMessageAudit,
EventMessageAuditOptions,
} from './EventMessageClasses/EventMessageAudit';
import { BadRequestError } from '../ResponseHelper';
import {
MessageEventBusDestinationTypeNames,
MessageEventBusDestinationWebhookOptions,
EventMessageTypeNames,
MessageEventBusDestinationOptions,
} from 'n8n-workflow';
import { User } from '../databases/entities/User';
export const eventBusRouter = express.Router();
// ----------------------------------------
// TypeGuards
// ----------------------------------------
const isWithIdString = (candidate: unknown): candidate is { id: string } => {
const o = candidate as { id: string };
if (!o) return false;
return o.id !== undefined;
};
const isWithQueryString = (candidate: unknown): candidate is { query: string } => {
const o = candidate as { query: string };
if (!o) return false;
return o.query !== undefined;
};
// TODO: add credentials
const isMessageEventBusDestinationWebhookOptions = (
candidate: unknown,
): candidate is MessageEventBusDestinationWebhookOptions => {
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
const o = candidate as MessageEventBusDestinationWebhookOptions;
if (!o) return false;
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
return o.url !== undefined;
};
const isMessageEventBusDestinationOptions = (
candidate: unknown,
): candidate is MessageEventBusDestinationOptions => {
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
const o = candidate as MessageEventBusDestinationOptions;
if (!o) return false;
return o.__type !== undefined;
};
// ----------------------------------------
// Events
// ----------------------------------------
eventBusRouter.get(
'/event',
ResponseHelper.send(async (req: express.Request): Promise<any> => {
if (isWithQueryString(req.query)) {
switch (req.query.query as EventMessageReturnMode) {
case 'sent':
return eventBus.getEventsSent();
case 'unsent':
return eventBus.getEventsUnsent();
case 'all':
default:
}
}
return eventBus.getEvents();
}),
);
eventBusRouter.post(
'/event',
ResponseHelper.send(async (req: express.Request): Promise<any> => {
if (isEventMessageOptions(req.body)) {
let msg;
switch (req.body.__type) {
case EventMessageTypeNames.workflow:
msg = new EventMessageWorkflow(req.body as EventMessageWorkflowOptions);
break;
case EventMessageTypeNames.audit:
msg = new EventMessageAudit(req.body as EventMessageAuditOptions);
break;
case EventMessageTypeNames.generic:
default:
msg = new EventMessageGeneric(req.body);
}
await eventBus.send(msg);
} else {
throw new BadRequestError(
'Body is not a serialized EventMessage or eventName does not match format {namespace}.{domain}.{event}',
);
}
}),
);
// ----------------------------------------
// Destinations
// ----------------------------------------
eventBusRouter.get(
'/destination',
// eslint-disable-next-line @typescript-eslint/no-unused-vars
ResponseHelper.send(async (req: express.Request, res: express.Response): Promise<any> => {
let result = [];
if (isWithIdString(req.query)) {
result = await eventBus.findDestination(req.query.id);
} else {
result = await eventBus.findDestination();
}
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
return result;
}),
);
eventBusRouter.post(
'/destination',
// eslint-disable-next-line @typescript-eslint/no-unused-vars
ResponseHelper.send(async (req: express.Request, res: express.Response): Promise<any> => {
if (!req.user || (req.user as User).globalRole.name !== 'owner') {
throw new ResponseHelper.UnauthorizedError('Invalid request');
}
if (isMessageEventBusDestinationOptions(req.body)) {
let result;
switch (req.body.__type) {
case MessageEventBusDestinationTypeNames.sentry:
if (isMessageEventBusDestinationSentryOptions(req.body)) {
result = await eventBus.addDestination(new MessageEventBusDestinationSentry(req.body));
}
break;
case MessageEventBusDestinationTypeNames.webhook:
if (isMessageEventBusDestinationWebhookOptions(req.body)) {
result = await eventBus.addDestination(new MessageEventBusDestinationWebhook(req.body));
}
break;
case MessageEventBusDestinationTypeNames.syslog:
if (isMessageEventBusDestinationSyslogOptions(req.body)) {
result = await eventBus.addDestination(new MessageEventBusDestinationSyslog(req.body));
}
break;
default:
throw new BadRequestError(
// eslint-disable-next-line @typescript-eslint/restrict-template-expressions
`Body is missing ${req.body.__type} options or type ${req.body.__type} is unknown`,
);
}
if (result) {
await result.saveToDb();
return result;
}
throw new BadRequestError('There was an error adding the destination');
}
throw new BadRequestError('Body is not configuring MessageEventBusDestinationOptions');
}),
);
eventBusRouter.get(
'/testmessage',
// eslint-disable-next-line @typescript-eslint/no-unused-vars
ResponseHelper.send(async (req: express.Request, res: express.Response): Promise<any> => {
let result = false;
if (isWithIdString(req.query)) {
result = await eventBus.testDestination(req.query.id);
}
return result;
}),
);
eventBusRouter.delete(
'/destination',
// eslint-disable-next-line @typescript-eslint/no-unused-vars
ResponseHelper.send(async (req: express.Request, res: express.Response): Promise<any> => {
if (!req.user || (req.user as User).globalRole.name !== 'owner') {
throw new ResponseHelper.UnauthorizedError('Invalid request');
}
if (isWithIdString(req.query)) {
const result = await eventBus.removeDestination(req.query.id);
if (result) {
return result;
}
} else {
throw new BadRequestError('Query is missing id');
}
}),
);
// ----------------------------------------
// Utilities
// ----------------------------------------
eventBusRouter.get(
'/eventnames',
ResponseHelper.send(async (): Promise<any> => {
return eventNamesAll;
}),
);

View File

@@ -0,0 +1 @@
export { eventBus } from './MessageEventBus/MessageEventBus';

View File

@@ -187,7 +187,7 @@ EEWorkflowController.post(
}
await ExternalHooks().run('workflow.afterCreate', [savedWorkflow]);
void InternalHooksManager.getInstance().onWorkflowCreated(req.user.id, newWorkflow, false);
void InternalHooksManager.getInstance().onWorkflowCreated(req.user, newWorkflow, false);
return savedWorkflow;
}),

View File

@@ -104,7 +104,7 @@ workflowsController.post(
}
await ExternalHooks().run('workflow.afterCreate', [savedWorkflow]);
void InternalHooksManager.getInstance().onWorkflowCreated(req.user.id, newWorkflow, false);
void InternalHooksManager.getInstance().onWorkflowCreated(req.user, newWorkflow, false);
return savedWorkflow;
}),
@@ -285,7 +285,7 @@ workflowsController.delete(
await Db.collections.Workflow.delete(workflowId);
void InternalHooksManager.getInstance().onWorkflowDeleted(req.user.id, workflowId, false);
void InternalHooksManager.getInstance().onWorkflowDeleted(req.user, workflowId, false);
await ExternalHooks().run('workflow.afterDelete', [workflowId]);
return true;

View File

@@ -314,7 +314,7 @@ export class WorkflowsService {
}
await ExternalHooks().run('workflow.afterUpdate', [updatedWorkflow]);
void InternalHooksManager.getInstance().onWorkflowSaved(user.id, updatedWorkflow, false);
void InternalHooksManager.getInstance().onWorkflowSaved(user, updatedWorkflow, false);
if (updatedWorkflow.active) {
// When the workflow is supposed to be active add it again

View File

@@ -10,8 +10,6 @@ import * as testDb from './shared/testDb';
import type { AuthAgent } from './shared/types';
import * as utils from './shared/utils';
jest.mock('@/telemetry');
let app: express.Application;
let testDbName = '';
let globalOwnerRole: Role;

View File

@@ -11,8 +11,6 @@ import * as testDb from './shared/testDb';
import type { AuthAgent } from './shared/types';
import * as utils from './shared/utils';
jest.mock('@/telemetry');
let app: express.Application;
let testDbName = '';
let globalMemberRole: Role;

View File

@@ -13,8 +13,6 @@ import type { AuthAgent, SaveCredentialFunction } from './shared/types';
import * as utils from './shared/utils';
import type { IUser } from 'n8n-workflow';
jest.mock('@/telemetry');
let app: express.Application;
let testDbName = '';
let globalOwnerRole: Role;

View File

@@ -14,8 +14,6 @@ import config from '@/config';
import type { CredentialsEntity } from '@db/entities/CredentialsEntity';
import type { AuthAgent } from './shared/types';
jest.mock('@/telemetry');
// mock that credentialsSharing is not enabled
const mockIsCredentialsSharingEnabled = jest.spyOn(UserManagementHelpers, 'isSharingEnabled');
mockIsCredentialsSharingEnabled.mockReturnValue(false);

View File

@@ -0,0 +1,317 @@
import express from 'express';
import config from '@/config';
import axios from 'axios';
import syslog from 'syslog-client';
import * as utils from './shared/utils';
import * as testDb from './shared/testDb';
import { Role } from '@db/entities/Role';
import { User } from '@db/entities/User';
import {
defaultMessageEventBusDestinationSentryOptions,
defaultMessageEventBusDestinationSyslogOptions,
defaultMessageEventBusDestinationWebhookOptions,
MessageEventBusDestinationSentryOptions,
MessageEventBusDestinationSyslogOptions,
MessageEventBusDestinationWebhookOptions,
} from 'n8n-workflow';
import { eventBus } from '@/eventbus';
import { SuperAgentTest } from 'supertest';
import { EventMessageGeneric } from '../../src/eventbus/EventMessageClasses/EventMessageGeneric';
import { MessageEventBusDestinationSyslog } from '../../src/eventbus/MessageEventBusDestination/MessageEventBusDestinationSyslog.ee';
import { MessageEventBusDestinationWebhook } from '../../src/eventbus/MessageEventBusDestination/MessageEventBusDestinationWebhook.ee';
import { MessageEventBusDestinationSentry } from '../../src/eventbus/MessageEventBusDestination/MessageEventBusDestinationSentry.ee';
import { EventMessageAudit } from '../../src/eventbus/EventMessageClasses/EventMessageAudit';
jest.unmock('@/eventbus/MessageEventBus/MessageEventBus');
jest.mock('axios');
const mockedAxios = axios as jest.Mocked<typeof axios>;
jest.mock('syslog-client');
const mockedSyslog = syslog as jest.Mocked<typeof syslog>;
let app: express.Application;
let testDbName = '';
let globalOwnerRole: Role;
let owner: User;
let unAuthOwnerAgent: SuperAgentTest;
let authOwnerAgent: SuperAgentTest;
const testSyslogDestination: MessageEventBusDestinationSyslogOptions = {
...defaultMessageEventBusDestinationSyslogOptions,
id: 'b88038f4-0a89-4e94-89a9-658dfdb74539',
protocol: 'udp',
label: 'Test Syslog',
enabled: false,
subscribedEvents: ['n8n.test.message', 'n8n.audit.user.updated'],
};
const testWebhookDestination: MessageEventBusDestinationWebhookOptions = {
...defaultMessageEventBusDestinationWebhookOptions,
id: '88be6560-bfb4-455c-8aa1-06971e9e5522',
url: 'http://localhost:3456',
method: `POST`,
label: 'Test Webhook',
enabled: false,
subscribedEvents: ['n8n.test.message', 'n8n.audit.user.updated'],
};
const testSentryDestination: MessageEventBusDestinationSentryOptions = {
...defaultMessageEventBusDestinationSentryOptions,
id: '450ca04b-87dd-4837-a052-ab3a347a00e9',
dsn: 'http://localhost:3000',
label: 'Test Sentry',
enabled: false,
subscribedEvents: ['n8n.test.message', 'n8n.audit.user.updated'],
};
async function cleanLogs() {
await eventBus.logWriter.getThread()?.cleanLogs();
const allMessages = await eventBus.getEvents('all');
expect(allMessages.length).toBe(0);
}
async function confirmIdsSentUnsent() {
const sent = await eventBus.getEvents('sent');
const unsent = await eventBus.getEvents('unsent');
expect(sent.length).toBe(1);
expect(sent[0].id).toBe(testMessage.id);
expect(unsent.length).toBe(1);
expect(unsent[0].id).toBe(testMessageUnsubscribed.id);
}
const testMessage = new EventMessageGeneric({ eventName: 'n8n.test.message' });
const testMessageUnsubscribed = new EventMessageGeneric({ eventName: 'n8n.test.unsub' });
const testAuditMessage = new EventMessageAudit({
eventName: 'n8n.audit.user.updated',
payload: {
_secret: 'secret',
public: 'public',
},
});
beforeAll(async () => {
const initResult = await testDb.init();
testDbName = initResult.testDbName;
globalOwnerRole = await testDb.getGlobalOwnerRole();
owner = await testDb.createUser({ globalRole: globalOwnerRole });
app = await utils.initTestServer({ endpointGroups: ['eventBus'], applyAuth: true });
unAuthOwnerAgent = utils.createAgent(app, {
apiPath: 'internal',
auth: false,
user: owner,
version: 1,
});
authOwnerAgent = utils.createAgent(app, {
apiPath: 'internal',
auth: true,
user: owner,
version: 1,
});
mockedSyslog.createClient.mockImplementation(() => new syslog.Client());
utils.initConfigFile();
utils.initTestLogger();
config.set('eventBus.logWriter.logBaseName', 'n8n-test-logwriter');
config.set('eventBus.logWriter.keepLogCount', '1');
config.set('enterprise.features.logStreaming', true);
await eventBus.initialize();
});
beforeEach(async () => {
// await testDb.truncate(['EventDestinations'], testDbName);
config.set('userManagement.disabled', false);
config.set('userManagement.isInstanceOwnerSetUp', true);
config.set('enterprise.features.logStreaming', false);
});
afterAll(async () => {
await testDb.terminate(testDbName);
await eventBus.close();
});
test('should have a running logwriter process', async () => {
const thread = eventBus.logWriter.getThread();
expect(thread).toBeDefined();
});
test('should have a clean log', async () => {
await eventBus.logWriter.getThread()?.cleanLogs();
const allMessages = await eventBus.getEvents('all');
expect(allMessages.length).toBe(0);
});
test('should have logwriter log messages', async () => {
await eventBus.send(testMessage);
const sent = await eventBus.getEvents('sent');
const unsent = await eventBus.getEvents('unsent');
expect(sent.length).toBeGreaterThan(0);
expect(unsent.length).toBe(0);
expect(sent.find((e) => e.id === testMessage.id)).toEqual(testMessage);
});
test('GET /eventbus/destination should fail due to missing authentication', async () => {
const response = await unAuthOwnerAgent.get('/eventbus/destination');
expect(response.statusCode).toBe(401);
});
test('POST /eventbus/destination create syslog destination', async () => {
const response = await authOwnerAgent.post('/eventbus/destination').send(testSyslogDestination);
expect(response.statusCode).toBe(200);
});
test('POST /eventbus/destination create sentry destination', async () => {
const response = await authOwnerAgent.post('/eventbus/destination').send(testSentryDestination);
expect(response.statusCode).toBe(200);
});
test('POST /eventbus/destination create webhook destination', async () => {
const response = await authOwnerAgent.post('/eventbus/destination').send(testWebhookDestination);
expect(response.statusCode).toBe(200);
});
test('GET /eventbus/destination all returned destinations should exist in eventbus', async () => {
const response = await authOwnerAgent.get('/eventbus/destination');
expect(response.statusCode).toBe(200);
const data = response.body.data;
expect(data).toBeTruthy();
expect(Array.isArray(data)).toBeTruthy();
for (let index = 0; index < data.length; index++) {
const destination = data[index];
const foundDestinations = await eventBus.findDestination(destination.id);
expect(Array.isArray(foundDestinations)).toBeTruthy();
expect(foundDestinations.length).toBe(1);
expect(foundDestinations[0].label).toBe(destination.label);
}
});
test('should send message to syslog ', async () => {
config.set('enterprise.features.logStreaming', true);
await cleanLogs();
const syslogDestination = eventBus.destinations[
testSyslogDestination.id!
] as MessageEventBusDestinationSyslog;
syslogDestination.enable();
const mockedSyslogClientLog = jest.spyOn(syslogDestination.client, 'log');
mockedSyslogClientLog.mockImplementation((_m, _options, _cb) => {
eventBus.confirmSent(testMessage, {
id: syslogDestination.id,
name: syslogDestination.label,
});
return syslogDestination.client;
});
await eventBus.send(testMessage);
await eventBus.send(testMessageUnsubscribed);
await new Promise((resolve) => setTimeout(resolve, 100));
expect(mockedSyslogClientLog).toHaveBeenCalled();
await confirmIdsSentUnsent();
syslogDestination.disable();
});
test('should anonymize audit message to syslog ', async () => {
config.set('enterprise.features.logStreaming', true);
await cleanLogs();
const syslogDestination = eventBus.destinations[
testSyslogDestination.id!
] as MessageEventBusDestinationSyslog;
syslogDestination.enable();
const mockedSyslogClientLog = jest.spyOn(syslogDestination.client, 'log');
mockedSyslogClientLog.mockImplementation((m, _options, _cb) => {
const o = JSON.parse(m);
expect(o).toHaveProperty('payload');
expect(o.payload).toHaveProperty('_secret');
syslogDestination.anonymizeAuditMessages
? expect(o.payload._secret).toBe('*')
: expect(o.payload._secret).toBe('secret');
expect(o.payload).toHaveProperty('public');
expect(o.payload.public).toBe('public');
return syslogDestination.client;
});
syslogDestination.anonymizeAuditMessages = true;
await eventBus.send(testAuditMessage);
expect(mockedSyslogClientLog).toHaveBeenCalled();
syslogDestination.anonymizeAuditMessages = false;
await eventBus.send(testAuditMessage);
expect(mockedSyslogClientLog).toHaveBeenCalled();
syslogDestination.disable();
});
test('should send message to webhook ', async () => {
config.set('enterprise.features.logStreaming', true);
await cleanLogs();
const webhookDestination = eventBus.destinations[
testWebhookDestination.id!
] as MessageEventBusDestinationWebhook;
webhookDestination.enable();
mockedAxios.post.mockResolvedValue({ status: 200, data: { msg: 'OK' } });
mockedAxios.request.mockResolvedValue({ status: 200, data: { msg: 'OK' } });
await eventBus.send(testMessage);
await eventBus.send(testMessageUnsubscribed);
// not elegant, but since communication happens through emitters, we'll wait for a bit
await new Promise((resolve) => setTimeout(resolve, 100));
await confirmIdsSentUnsent();
webhookDestination.disable();
});
test('should send message to sentry ', async () => {
config.set('enterprise.features.logStreaming', true);
await cleanLogs();
const sentryDestination = eventBus.destinations[
testSentryDestination.id!
] as MessageEventBusDestinationSentry;
sentryDestination.enable();
const mockedSentryCaptureMessage = jest.spyOn(sentryDestination.sentryClient, 'captureMessage');
mockedSentryCaptureMessage.mockImplementation((_m, _level, _hint, _scope) => {
eventBus.confirmSent(testMessage, {
id: sentryDestination.id,
name: sentryDestination.label,
});
return testMessage.id;
});
await eventBus.send(testMessage);
await eventBus.send(testMessageUnsubscribed);
// not elegant, but since communication happens through emitters, we'll wait for a bit
await new Promise((resolve) => setTimeout(resolve, 100));
expect(mockedSentryCaptureMessage).toHaveBeenCalled();
await confirmIdsSentUnsent();
sentryDestination.disable();
});
test('DEL /eventbus/destination delete all destinations by id', async () => {
const existingDestinationIds = [...Object.keys(eventBus.destinations)];
await Promise.all(
existingDestinationIds.map(async (id) => {
const response = await authOwnerAgent.del('/eventbus/destination').query({ id });
expect(response.statusCode).toBe(200);
}),
);
expect(Object.keys(eventBus.destinations).length).toBe(0);
});

View File

@@ -9,9 +9,6 @@ import { ILicensePostResponse, ILicenseReadResponse } from '@/Interfaces';
import { LicenseManager } from '@n8n_io/license-sdk';
import { License } from '@/License';
jest.mock('@/telemetry');
jest.mock('@n8n_io/license-sdk');
const MOCK_SERVER_URL = 'https://server.com/v1';
const MOCK_RENEW_OFFSET = 259200;
const MOCK_INSTANCE_ID = 'instance-id';

View File

@@ -17,8 +17,6 @@ import * as testDb from './shared/testDb';
import type { AuthAgent } from './shared/types';
import * as utils from './shared/utils';
jest.mock('@/telemetry');
let app: express.Application;
let testDbName = '';
let globalOwnerRole: Role;

View File

@@ -21,10 +21,6 @@ import type { AuthAgent } from './shared/types';
import type { InstalledNodes } from '@db/entities/InstalledNodes';
import { COMMUNITY_PACKAGE_VERSION } from './shared/constants';
jest.mock('@/telemetry');
jest.mock('@/Push');
jest.mock('@/CommunityNodes/helpers', () => {
return {
...jest.requireActual('@/CommunityNodes/helpers'),

View File

@@ -14,8 +14,6 @@ import * as testDb from './shared/testDb';
import type { AuthAgent } from './shared/types';
import * as utils from './shared/utils';
jest.mock('@/telemetry');
let app: express.Application;
let testDbName = '';
let globalOwnerRole: Role;

View File

@@ -14,7 +14,6 @@ import {
import * as testDb from './shared/testDb';
import type { Role } from '@db/entities/Role';
jest.mock('@/telemetry');
jest.mock('@/UserManagement/email/NodeMailer');
let app: express.Application;

View File

@@ -18,8 +18,6 @@ let credentialOwnerRole: Role;
let saveCredential: SaveCredentialFunction;
jest.mock('@/telemetry');
beforeAll(async () => {
app = await utils.initTestServer({ endpointGroups: ['publicApi'], applyAuth: false });
const initResult = await testDb.init();

View File

@@ -8,8 +8,6 @@ import { randomApiKey } from '../shared/random';
import * as utils from '../shared/utils';
import * as testDb from '../shared/testDb';
jest.mock('@/telemetry');
let app: express.Application;
let testDbName = '';
let globalOwnerRole: Role;

View File

@@ -17,8 +17,6 @@ let globalMemberRole: Role;
let workflowOwnerRole: Role;
let workflowRunner: ActiveWorkflowRunner;
jest.mock('@/telemetry');
beforeAll(async () => {
app = await utils.initTestServer({ endpointGroups: ['publicApi'], applyAuth: false });
const initResult = await testDb.init();

View File

@@ -250,6 +250,7 @@ function toTableName(sourceName: CollectionName | MappingName) {
InstalledPackages: 'installed_packages',
InstalledNodes: 'installed_nodes',
WorkflowStatistics: 'workflow_statistics',
EventDestinations: 'event_destinations',
}[sourceName];
}

View File

@@ -24,6 +24,7 @@ type EndpointGroup =
| 'workflows'
| 'publicApi'
| 'nodes'
| 'eventBus'
| 'license';
export type CredentialPayload = {

View File

@@ -66,6 +66,7 @@ import type {
PostgresSchemaSection,
} from './types';
import { licenseController } from '@/license/license.controller';
import { eventBusRouter } from '@/eventbus/eventBusRoutes';
const loadNodesAndCredentials: INodesAndCredentials = {
loaded: { nodes: {}, credentials: {} },
@@ -125,6 +126,7 @@ export async function initTestServer({
workflows: { controller: workflowsController, path: 'workflows' },
nodes: { controller: nodesController, path: 'nodes' },
license: { controller: licenseController, path: 'license' },
eventBus: { controller: eventBusRouter, path: 'eventbus' },
publicApi: apiRouters,
};
@@ -169,7 +171,7 @@ const classifyEndpointGroups = (endpointGroups: string[]) => {
const routerEndpoints: string[] = [];
const functionEndpoints: string[] = [];
const ROUTER_GROUP = ['credentials', 'nodes', 'workflows', 'publicApi', 'license'];
const ROUTER_GROUP = ['credentials', 'nodes', 'workflows', 'publicApi', 'license', 'eventBus'];
endpointGroups.forEach((group) =>
(ROUTER_GROUP.includes(group) ? routerEndpoints : functionEndpoints).push(group),

View File

@@ -22,7 +22,6 @@ import * as utils from './shared/utils';
import * as UserManagementMailer from '@/UserManagement/email/UserManagementMailer';
import { NodeMailer } from '@/UserManagement/email/NodeMailer';
jest.mock('@/telemetry');
jest.mock('@/UserManagement/email/NodeMailer');
let app: express.Application;

View File

@@ -13,8 +13,6 @@ import { makeWorkflow } from './shared/utils';
import { randomCredentialPayload } from './shared/random';
import { ActiveWorkflowRunner } from '@/ActiveWorkflowRunner';
jest.mock('@/telemetry');
let app: express.Application;
let testDbName = '';
let globalOwnerRole: Role;

View File

@@ -8,8 +8,6 @@ import type { Role } from '@db/entities/Role';
import type { IPinData } from 'n8n-workflow';
import { makeWorkflow, MOCK_PINDATA } from './shared/utils';
jest.mock('@/telemetry');
let app: express.Application;
let testDbName = '';
let globalOwnerRole: Role;

View File

@@ -0,0 +1,5 @@
jest.mock('@sentry/node');
jest.mock('@n8n_io/license-sdk');
jest.mock('@/telemetry');
jest.mock('@/eventbus/MessageEventBus/MessageEventBus');
jest.mock('@/Push');

View File

@@ -2,6 +2,7 @@ import { Telemetry } from '@/telemetry';
import config from '@/config';
import { flushPromises } from './Helpers';
jest.unmock('@/telemetry');
jest.mock('@/license/License.service', () => {
return {
LicenseService: {

View File

@@ -8,5 +8,12 @@
"tsBuildInfoFile": "dist/build.tsbuildinfo"
},
"include": ["src/**/*.ts"],
"exclude": ["test/**"]
"exclude": ["test/**"],
"tsc-alias": {
"replacers": {
"base-url": {
"enabled": false
}
}
}
}