feat(core): Setup backend modules (no-changelog) (#14084)

Co-authored-by: Guillaume Jacquart <jacquart.guillaume@gmail.com>
Co-authored-by: Danny Martini <danny@n8n.io>

packages/cli/.eslintrc.js

@@ -47,7 +47,12 @@ module.exports = {
 			},
 		},
 		{
-			files: ['./src/databases/**/*.ts', './test/**/*.ts', './src/**/__tests__/**/*.ts'],
+			files: [
+				'./src/databases/**/*.ts',
+				'./src/modules/**/*.ts',
+				'./test/**/*.ts',
+				'./src/**/__tests__/**/*.ts',
+			],
 			rules: {
 				'n8n-local-rules/misplaced-n8n-typeorm-import': 'off',
 			},

packages/cli/src/commands/base-command.ts

@@ -33,6 +33,7 @@ import { ExternalHooks } from '@/external-hooks';
 import { ExternalSecretsManager } from '@/external-secrets.ee/external-secrets-manager.ee';
 import { License } from '@/license';
 import { LoadNodesAndCredentials } from '@/load-nodes-and-credentials';
+import { ModulesConfig } from '@/modules/modules.config';
 import { NodeTypes } from '@/node-types';
 import { PostHogClient } from '@/posthog';
 import { ShutdownService } from '@/shutdown/shutdown.service';
@@ -57,6 +58,8 @@ export abstract class BaseCommand extends Command {
 
 	protected readonly globalConfig = Container.get(GlobalConfig);
 
+	protected readonly modulesConfig = Container.get(ModulesConfig);
+
 	/**
 	 * How long to wait for graceful shutdown before force killing the process.
 	 */
@@ -66,6 +69,13 @@ export abstract class BaseCommand extends Command {
 	/** Whether to init community packages (if enabled) */
 	protected needsCommunityPackages = false;
 
+	protected async loadModules() {
+		for (const moduleName of this.modulesConfig.modules) {
+			await import(`../modules/${moduleName}/${moduleName}.module`);
+			this.logger.debug(`Loaded module "${moduleName}"`);
+		}
+	}
+
 	async init(): Promise<void> {
 		this.errorReporter = Container.get(ErrorReporter);
 

packages/cli/src/commands/start.ts

@@ -239,6 +239,8 @@ export class Start extends BaseCommand {
 			const taskRunnerModule = Container.get(TaskRunnerModule);
 			await taskRunnerModule.start();
 		}
+
+		await this.loadModules();
 	}
 
 	async initOrchestration() {

packages/cli/src/commands/webhook.ts

@@ -79,6 +79,8 @@ export class Webhook extends BaseCommand {
 		this.logger.debug('External hooks init complete');
 		await this.initExternalSecrets();
 		this.logger.debug('External secrets init complete');
+
+		await this.loadModules();
 	}
 
 	async run() {

packages/cli/src/commands/worker.ts

@@ -117,6 +117,8 @@ export class Worker extends BaseCommand {
 			const taskRunnerModule = Container.get(TaskRunnerModule);
 			await taskRunnerModule.start();
 		}
+
+		await this.loadModules();
 	}
 
 	async initEventBus() {

packages/cli/src/decorators/__tests__/module.test.ts

@@ -0,0 +1,33 @@
+import { Container } from '@n8n/di';
+import { mock } from 'jest-mock-extended';
+import type { ExecutionLifecycleHooks } from 'n8n-core';
+
+import type { BaseN8nModule } from '../module';
+import { ModuleRegistry, N8nModule } from '../module';
+
+let moduleRegistry: ModuleRegistry;
+
+beforeEach(() => {
+	moduleRegistry = new ModuleRegistry();
+});
+
+describe('registerLifecycleHooks', () => {
+	@N8nModule()
+	class TestModule implements BaseN8nModule {
+		registerLifecycleHooks() {}
+	}
+
+	test('is called when ModuleRegistry.registerLifecycleHooks is called', () => {
+		// ARRANGE
+		const hooks = mock<ExecutionLifecycleHooks>();
+		const instance = Container.get(TestModule);
+		jest.spyOn(instance, 'registerLifecycleHooks');
+
+		// ACT
+		moduleRegistry.registerLifecycleHooks(hooks);
+
+		// ASSERT
+		expect(instance.registerLifecycleHooks).toHaveBeenCalledTimes(1);
+		expect(instance.registerLifecycleHooks).toHaveBeenCalledWith(hooks);
+	});
+});

packages/cli/src/decorators/module.ts

@@ -0,0 +1,29 @@
+import { Container, Service, type Constructable } from '@n8n/di';
+import type { ExecutionLifecycleHooks } from 'n8n-core';
+
+export interface BaseN8nModule {
+	registerLifecycleHooks?(hooks: ExecutionLifecycleHooks): void;
+}
+
+type Module = Constructable<BaseN8nModule>;
+
+export const registry = new Set<Module>();
+
+export const N8nModule = (): ClassDecorator => (target) => {
+	registry.add(target as unknown as Module);
+
+	// eslint-disable-next-line @typescript-eslint/no-unsafe-return
+	return Service()(target);
+};
+
+@Service()
+export class ModuleRegistry {
+	registerLifecycleHooks(hooks: ExecutionLifecycleHooks) {
+		for (const ModuleClass of registry.keys()) {
+			const instance = Container.get(ModuleClass);
+			if (instance.registerLifecycleHooks) {
+				instance.registerLifecycleHooks(hooks);
+			}
+		}
+	}
+}

packages/cli/src/execution-lifecycle/execution-lifecycle-hooks.ts

@@ -8,6 +8,7 @@ import type {
 } from 'n8n-workflow';
 
 import { ExecutionRepository } from '@/databases/repositories/execution.repository';
+import { ModuleRegistry } from '@/decorators/module';
 import { EventService } from '@/events/event.service';
 import { ExternalHooks } from '@/external-hooks';
 import { Push } from '@/push';
@@ -394,11 +395,13 @@ export function getLifecycleHooksForScalingWorker(
 		hookFunctionsPush(hooks, optionalParameters);
 	}
 
+	Container.get(ModuleRegistry).registerLifecycleHooks(hooks);
+
 	return hooks;
 }
 
 /**
- * Returns ExecutionLifecycleHooks instance for main process if workflow runs via worker
+ * Returns ExecutionLifecycleHooks instance for main process in scaling mode.
  */
 export function getLifecycleHooksForScalingMain(
 	data: IWorkflowExecutionDataProcess,
@@ -454,11 +457,13 @@ export function getLifecycleHooksForScalingMain(
 	hooks.handlers.nodeExecuteBefore = [];
 	hooks.handlers.nodeExecuteAfter = [];
 
+	Container.get(ModuleRegistry).registerLifecycleHooks(hooks);
+
 	return hooks;
 }
 
 /**
- * Returns ExecutionLifecycleHooks instance for running the main workflow
+ * Returns ExecutionLifecycleHooks instance for the main process in regular mode
  */
 export function getLifecycleHooksForRegularMain(
 	data: IWorkflowExecutionDataProcess,
@@ -476,5 +481,6 @@ export function getLifecycleHooksForRegularMain(
 	hookFunctionsSaveProgress(hooks, optionalParameters);
 	hookFunctionsStatistics(hooks);
 	hookFunctionsExternalHooks(hooks);
+	Container.get(ModuleRegistry).registerLifecycleHooks(hooks);
 	return hooks;
 }

packages/cli/src/modules/__tests__/modules.config.test.ts

@@ -0,0 +1,33 @@
+import { Container } from '@n8n/di';
+import { UnexpectedError } from 'n8n-workflow';
+
+import { ModulesConfig } from '../modules.config';
+
+describe('ModulesConfig', () => {
+	beforeEach(() => {
+		jest.resetAllMocks();
+		process.env = {};
+		Container.reset();
+	});
+
+	it('should initialize with empty modules if no environment variable is set', () => {
+		const config = Container.get(ModulesConfig);
+		expect(config.modules).toEqual([]);
+	});
+
+	it('should parse valid module names from environment variable', () => {
+		process.env.N8N_ENABLED_MODULES = 'insights';
+		const config = Container.get(ModulesConfig);
+		expect(config.modules).toEqual(['insights']);
+	});
+
+	it('should throw UnexpectedError for invalid module names', () => {
+		process.env.N8N_ENABLED_MODULES = 'invalidModule';
+		expect(() => Container.get(ModulesConfig)).toThrow(UnexpectedError);
+	});
+
+	it('should throw UnexpectedError if any module name is invalid', () => {
+		process.env.N8N_ENABLED_MODULES = 'insights,invalidModule';
+		expect(() => Container.get(ModulesConfig)).toThrow(UnexpectedError);
+	});
+});

packages/cli/src/modules/modules.config.ts

@@ -0,0 +1,24 @@
+import { CommaSeperatedStringArray, Config, Env } from '@n8n/config';
+import { UnexpectedError } from 'n8n-workflow';
+
+const moduleNames = ['insights'] as const;
+type ModuleName = (typeof moduleNames)[number];
+
+class Modules extends CommaSeperatedStringArray<ModuleName> {
+	constructor(str: string) {
+		super(str);
+
+		for (const moduleName of this) {
+			if (!moduleNames.includes(moduleName)) {
+				throw new UnexpectedError(`Unknown module name ${moduleName}`, { level: 'fatal' });
+			}
+		}
+	}
+}
+
+@Config
+export class ModulesConfig {
+	/** Comma-separated list of all enabled modules */
+	@Env('N8N_ENABLED_MODULES')
+	modules: Modules = [];
+}