feat(core): Lazy-load nodes and credentials to reduce baseline memory usage (#4577)

packages/cli/test/integration/shared/utils.ts

@@ -4,7 +4,7 @@ import { existsSync } from 'fs';
 import bodyParser from 'body-parser';
 import { CronJob } from 'cron';
 import express from 'express';
-import { set } from 'lodash';
+import set from 'lodash.set';
 import { BinaryDataManager, UserSettings } from 'n8n-core';
 import {
 	ICredentialType,
@@ -13,8 +13,7 @@ import {
 	INode,
 	INodeExecutionData,
 	INodeParameters,
-	INodeTypeData,
-	INodeTypes,
+	INodesAndCredentials,
 	ITriggerFunctions,
 	ITriggerResponse,
 	LoggerProxy,
@@ -67,6 +66,14 @@ import type {
 	PostgresSchemaSection,
 } from './types';
 
+const loadNodesAndCredentials: INodesAndCredentials = {
+	loaded: { nodes: {}, credentials: {} },
+	known: { nodes: {}, credentials: {} },
+};
+
+const mockNodeTypes = NodeTypes(loadNodesAndCredentials);
+CredentialTypes(loadNodesAndCredentials);
+
 /**
  * Initialize a test server.
  *
@@ -149,8 +156,6 @@ export async function initTestServer({
  * Pre-requisite: Mock the telemetry module before calling.
  */
 export function initTestTelemetry() {
-	const mockNodeTypes = { nodeTypes: {} } as INodeTypes;
-
 	void InternalHooksManager.init('test-instance-id', 'test-version', mockNodeTypes);
 }
 
@@ -217,20 +222,19 @@ export function gitHubCredentialType(): ICredentialType {
  * Initialize node types.
  */
 export async function initCredentialsTypes(): Promise<void> {
-	const credentialTypes = CredentialTypes();
-	await credentialTypes.init({
+	loadNodesAndCredentials.loaded.credentials = {
 		githubApi: {
 			type: gitHubCredentialType(),
 			sourcePath: '',
 		},
-	});
+	};
 }
 
 /**
  * Initialize node types.
  */
 export async function initNodeTypes() {
-	const types: INodeTypeData = {
+	loadNodesAndCredentials.loaded.nodes = {
 		'n8n-nodes-base.start': {
 			sourcePath: '',
 			type: {
@@ -524,8 +528,6 @@ export async function initNodeTypes() {
 			},
 		},
 	};
-
-	await NodeTypes().init(types);
 }
 
 /**

packages/cli/test/unit/CredentialTypes.test.ts

@@ -1,62 +1,46 @@
-import type { ICredentialTypeData, ICredentialTypes } from 'n8n-workflow';
+import type { ICredentialTypes, INodesAndCredentials } from 'n8n-workflow';
 import { CredentialTypes } from '@/CredentialTypes';
 
 describe('ActiveExecutions', () => {
 	let credentialTypes: ICredentialTypes;
 
 	beforeEach(() => {
-		credentialTypes = CredentialTypes();
-	});
-
-	test('Should start with empty credential list', () => {
-		expect(credentialTypes.getAll()).toEqual([]);
-	});
-
-	test('Should initialize credential types', () => {
-		credentialTypes.init(mockCredentialTypes());
-		expect(credentialTypes.getAll()).toHaveLength(2);
-	});
-
-	test('Should return all credential types', () => {
-		credentialTypes.init(mockCredentialTypes());
-		const mockedCredentialTypes = mockCredentialTypes();
-		expect(credentialTypes.getAll()).toStrictEqual([
-			mockedCredentialTypes.fakeFirstCredential.type,
-			mockedCredentialTypes.fakeSecondCredential.type,
-		]);
+		credentialTypes = CredentialTypes(mockNodesAndCredentials());
 	});
 
 	test('Should throw error when calling invalid credential name', () => {
-		credentialTypes.init(mockCredentialTypes());
 		expect(() => credentialTypes.getByName('fakeThirdCredential')).toThrowError();
 	});
 
 	test('Should return correct credential type for valid name', () => {
-		credentialTypes.init(mockCredentialTypes());
-		const mockedCredentialTypes = mockCredentialTypes();
+		const mockedCredentialTypes = mockNodesAndCredentials().loaded.credentials;
 		expect(credentialTypes.getByName('fakeFirstCredential')).toStrictEqual(
 			mockedCredentialTypes.fakeFirstCredential.type,
 		);
 	});
 });
 
-function mockCredentialTypes(): ICredentialTypeData {
-	return {
-		fakeFirstCredential: {
-			type: {
-				name: 'fakeFirstCredential',
-				displayName: 'Fake First Credential',
-				properties: [],
+const mockNodesAndCredentials = (): INodesAndCredentials => ({
+	loaded: {
+		nodes: {},
+		credentials: {
+			fakeFirstCredential: {
+				type: {
+					name: 'fakeFirstCredential',
+					displayName: 'Fake First Credential',
+					properties: [],
+				},
+				sourcePath: '',
 			},
-			sourcePath: '',
-		},
-		fakeSecondCredential: {
-			type: {
-				name: 'fakeSecondCredential',
-				displayName: 'Fake Second Credential',
-				properties: [],
+			fakeSecondCredential: {
+				type: {
+					name: 'fakeSecondCredential',
+					displayName: 'Fake Second Credential',
+					properties: [],
+				},
+				sourcePath: '',
 			},
-			sourcePath: '',
 		},
-	};
-}
+	},
+	known: { nodes: {}, credentials: {} },
+});

packages/cli/test/unit/CredentialsHelper.test.ts

@@ -6,6 +6,7 @@ import {
 	IHttpRequestOptions,
 	INode,
 	INodeProperties,
+	INodesAndCredentials,
 	Workflow,
 } from 'n8n-workflow';
 import { CredentialsHelper } from '@/CredentialsHelper';
@@ -13,6 +14,10 @@ import { CredentialTypes } from '@/CredentialTypes';
 import * as Helpers from './Helpers';
 
 const TEST_ENCRYPTION_KEY = 'test';
+const mockNodesAndCredentials: INodesAndCredentials = {
+	loaded: { nodes: {}, credentials: {} },
+	known: { nodes: {}, credentials: {} },
+};
 
 describe('CredentialsHelper', () => {
 	describe('authenticate', () => {
@@ -222,14 +227,14 @@ describe('CredentialsHelper', () => {
 
 		for (const testData of tests) {
 			test(testData.description, async () => {
-				const credentialTypes: ICredentialTypeData = {
+				mockNodesAndCredentials.loaded.credentials = {
 					[testData.input.credentialType.name]: {
 						type: testData.input.credentialType,
 						sourcePath: '',
 					},
 				};
 
-				await CredentialTypes().init(credentialTypes);
+				CredentialTypes(mockNodesAndCredentials);
 
 				const credentialsHelper = new CredentialsHelper(TEST_ENCRYPTION_KEY);
 

packages/cli/test/unit/Helpers.ts

@@ -1,4 +1,10 @@
-import { INodeType, INodeTypeData, INodeTypes, NodeHelpers } from 'n8n-workflow';
+import {
+	INodesAndCredentials,
+	INodeType,
+	INodeTypeData,
+	INodeTypes,
+	NodeHelpers,
+} from 'n8n-workflow';
 
 class NodeTypesClass implements INodeTypes {
 	nodeTypes: INodeTypeData = {
@@ -36,8 +42,10 @@ class NodeTypesClass implements INodeTypes {
 		},
 	};
 
-	async init(nodeTypes: INodeTypeData): Promise<void> {
-		this.nodeTypes = nodeTypes;
+	constructor(nodesAndCredentials?: INodesAndCredentials) {
+		if (nodesAndCredentials?.loaded?.nodes) {
+			this.nodeTypes = nodesAndCredentials?.loaded?.nodes;
+		}
 	}
 
 	getAll(): INodeType[] {
@@ -55,9 +63,9 @@ class NodeTypesClass implements INodeTypes {
 
 let nodeTypesInstance: NodeTypesClass | undefined;
 
-export function NodeTypes(): NodeTypesClass {
+export function NodeTypes(nodesAndCredentials?: INodesAndCredentials): NodeTypesClass {
 	if (nodeTypesInstance === undefined) {
-		nodeTypesInstance = new NodeTypesClass();
+		nodeTypesInstance = new NodeTypesClass(nodesAndCredentials);
 	}
 
 	return nodeTypesInstance;

packages/cli/test/unit/PermissionChecker.test.ts

@@ -23,8 +23,13 @@ beforeAll(async () => {
 	const initResult = await testDb.init();
 	testDbName = initResult.testDbName;
 
-	mockNodeTypes = MockNodeTypes();
-	await mockNodeTypes.init(MOCK_NODE_TYPES_DATA);
+	mockNodeTypes = MockNodeTypes({
+		loaded: {
+			nodes: MOCK_NODE_TYPES_DATA,
+			credentials: {},
+		},
+		known: { nodes: {}, credentials: {} },
+	});
 
 	credentialOwnerRole = await testDb.getCredentialOwnerRole();
 	workflowOwnerRole = await testDb.getWorkflowOwnerRole();