feat(core): Lazy-load nodes and credentials to reduce baseline memory usage (#4577)

packages/core/bin/common.js

@@ -0,0 +1,19 @@
+const path = require('path');
+const { mkdir, writeFile } = require('fs/promises');
+
+const packageDir = process.cwd();
+const distDir = path.join(packageDir, 'dist');
+
+const writeJSON = async (file, data) => {
+	const filePath = path.resolve(distDir, file);
+	await mkdir(path.dirname(filePath), { recursive: true });
+	const payload = Array.isArray(data)
+		? `[\n${data.map((entry) => JSON.stringify(entry)).join(',\n')}\n]`
+		: JSON.stringify(data, null, 2);
+	await writeFile(filePath, payload, { encoding: 'utf-8' });
+};
+
+module.exports = {
+	packageDir,
+	writeJSON,
+};

packages/core/bin/generate-known

@@ -0,0 +1,47 @@
+#!/usr/bin/env node
+
+const path = require('path');
+const glob = require('fast-glob');
+const { createContext, Script } = require('vm');
+const { LoggerProxy } = require('n8n-workflow');
+const { packageDir, writeJSON } = require('./common');
+
+LoggerProxy.init({
+	log: console.log.bind(console),
+	warn: console.warn.bind(console),
+});
+
+const context = Object.freeze(createContext({ require }));
+const loadClass = (sourcePath) => {
+	try {
+		const [className] = path.parse(sourcePath).name.split('.');
+		const absolutePath = path.resolve(packageDir, sourcePath);
+		const script = new Script(`new (require('${absolutePath}').${className})()`);
+		const instance = script.runInContext(context);
+		return { instance, sourcePath, className };
+	} catch (e) {
+		LoggerProxy.warn('Failed to load %s: %s', sourcePath, e.message);
+	}
+};
+
+const generate = (kind) => {
+	const data = glob
+		.sync(`dist/${kind}/**/*.${kind === 'nodes' ? 'node' : kind}.js`, {
+			cwd: packageDir,
+		})
+		.filter((filePath) => !/[vV]\d.node\.js$/.test(filePath))
+		.map(loadClass)
+		.filter((data) => !!data)
+		.reduce((obj, { className, sourcePath, instance }) => {
+			const name = kind === 'nodes' ? instance.description.name : instance.name;
+			if (name in obj) console.error('already loaded', kind, name, sourcePath);
+			else obj[name] = { className, sourcePath };
+			return obj;
+		}, {});
+	LoggerProxy.info(`Detected ${Object.keys(data).length} ${kind}`);
+	return writeJSON(`known/${kind}.json`, data);
+};
+
+(async () => {
+	await Promise.all([generate('credentials'), generate('nodes')]);
+})();

packages/core/bin/generate-ui-types

@@ -0,0 +1,29 @@
+#!/usr/bin/env node
+
+const { LoggerProxy, NodeHelpers } = require('n8n-workflow');
+const { PackageDirectoryLoader } = require('../dist/DirectoryLoader');
+const { packageDir, writeJSON } = require('./common');
+
+LoggerProxy.init({
+	log: console.log.bind(console),
+	warn: console.warn.bind(console),
+});
+
+(async () => {
+	const loader = new PackageDirectoryLoader(packageDir);
+	await loader.loadAll();
+
+	const credentialTypes = Object.values(loader.credentialTypes).map((data) => data.type);
+
+	const nodeTypes = Object.values(loader.nodeTypes)
+		.map((data) => data.type)
+		.flatMap((nodeData) => {
+			const allNodeTypes = NodeHelpers.getVersionedNodeTypeAll(nodeData);
+			return allNodeTypes.map((element) => element.description);
+		});
+
+	await Promise.all([
+		writeJSON('types/credentials.json', credentialTypes),
+		writeJSON('types/nodes.json', nodeTypes),
+	]);
+})();