feat(core): Live reload node/credential descriptions in development (no-changelog) (#4939)

packages/cli/.npmignore

@@ -0,0 +1 @@
+dist/ReloadNodesAndCredentials.*

packages/cli/package.json

@@ -75,6 +75,7 @@
     "@types/jsonwebtoken": "^9.0.1",
     "@types/localtunnel": "^1.9.0",
     "@types/lodash.get": "^4.4.6",
+    "@types/lodash.debounce": "^4.0.7",
     "@types/lodash.intersection": "^4.4.7",
     "@types/lodash.iteratee": "^4.7.7",
     "@types/lodash.merge": "^4.6.6",
@@ -99,7 +100,9 @@
     "@types/uuid": "^8.3.2",
     "@types/validator": "^13.7.0",
     "@types/yamljs": "^0.2.31",
+    "chokidar": "^3.5.2",
     "concurrently": "^5.1.0",
+    "lodash.debounce": "^4.0.8",
     "mock-jwks": "^1.0.9",
     "nodemon": "^2.0.2",
     "run-script-os": "^1.0.7",
@@ -190,7 +193,6 @@
     "sse-channel": "^4.0.0",
     "swagger-ui-express": "^4.3.0",
     "syslog-client": "^1.1.1",
-    "tslib": "1.14.1",
     "typeorm": "0.3.11",
     "uuid": "^8.3.2",
     "validator": "13.7.0",

packages/cli/src/Interfaces.ts

@@ -615,7 +615,8 @@ export type IPushData =
 	| PushDataConsoleMessage
 	| PushDataReloadNodeType
 	| PushDataRemoveNodeType
-	| PushDataTestWebhook;
+	| PushDataTestWebhook
+	| PushDataNodeDescriptionUpdated;
 
 type PushDataExecutionFinished = {
 	data: IPushDataExecutionFinished;
@@ -657,6 +658,11 @@ type PushDataTestWebhook = {
 	type: 'testWebhookDeleted' | 'testWebhookReceived';
 };
 
+type PushDataNodeDescriptionUpdated = {
+	data: undefined;
+	type: 'nodeDescriptionUpdated';
+};
+
 export interface IPushDataExecutionFinished {
 	data: IRun;
 	executionId: string;

packages/cli/src/NodeTypes.ts

@@ -10,15 +10,11 @@ import type {
 import { NodeHelpers } from 'n8n-workflow';
 import { RESPONSE_ERROR_MESSAGES } from './constants';
 
-class NodeTypesClass implements INodeTypes {
+export class NodeTypesClass implements INodeTypes {
 	constructor(private nodesAndCredentials: INodesAndCredentials) {
 		// Some nodeTypes need to get special parameters applied like the
 		// polling nodes the polling times
-		// eslint-disable-next-line no-restricted-syntax
-		for (const nodeTypeData of Object.values(this.loadedNodes)) {
-			const nodeType = NodeHelpers.getVersionedNodeType(nodeTypeData.type);
-			NodeHelpers.applySpecialNodeParameters(nodeType);
-		}
+		this.applySpecialNodeParameters();
 	}
 
 	/**
@@ -47,6 +43,13 @@ class NodeTypesClass implements INodeTypes {
 		return NodeHelpers.getVersionedNodeType(this.getNode(nodeType).type, version);
 	}
 
+	applySpecialNodeParameters() {
+		for (const nodeTypeData of Object.values(this.loadedNodes)) {
+			const nodeType = NodeHelpers.getVersionedNodeType(nodeTypeData.type);
+			NodeHelpers.applySpecialNodeParameters(nodeType);
+		}
+	}
+
 	private getNode(type: string): LoadedClass<INodeType | IVersionedNodeType> {
 		const loadedNodes = this.loadedNodes;
 		if (type in loadedNodes) {

packages/cli/src/ReloadNodesAndCredentials.ts

@@ -0,0 +1,41 @@
+import path from 'path';
+import { realpath } from 'fs/promises';
+
+import type { LoadNodesAndCredentialsClass } from '@/LoadNodesAndCredentials';
+import type { NodeTypesClass } from '@/NodeTypes';
+import type { Push } from '@/Push';
+
+export const reloadNodesAndCredentials = async (
+	loadNodesAndCredentials: LoadNodesAndCredentialsClass,
+	nodeTypes: NodeTypesClass,
+	push: Push,
+) => {
+	// eslint-disable-next-line import/no-extraneous-dependencies
+	const { default: debounce } = await import('lodash.debounce');
+	// eslint-disable-next-line import/no-extraneous-dependencies
+	const { watch } = await import('chokidar');
+
+	Object.entries(loadNodesAndCredentials.loaders).forEach(async ([dir, loader]) => {
+		const realModulePath = path.join(await realpath(dir), path.sep);
+		const reloader = debounce(async () => {
+			const modulesToUnload = Object.keys(require.cache).filter((filePath) =>
+				filePath.startsWith(realModulePath),
+			);
+			modulesToUnload.forEach((filePath) => {
+				delete require.cache[filePath];
+			});
+
+			loader.reset();
+			await loader.loadAll();
+			await loadNodesAndCredentials.postProcessLoaders();
+			await loadNodesAndCredentials.generateTypesForFrontend();
+			nodeTypes.applySpecialNodeParameters();
+			push.send('nodeDescriptionUpdated', undefined);
+		}, 100);
+
+		const toWatch = loader.isLazyLoaded
+			? ['**/nodes.json', '**/credentials.json']
+			: ['**/*.js', '**/*.json'];
+		watch(toWatch, { cwd: realModulePath }).on('change', reloader);
+	});
+};

packages/cli/src/Server.ts

@@ -81,6 +81,7 @@ import {
 	AUTH_COOKIE_NAME,
 	EDITOR_UI_DIST_DIR,
 	GENERATED_STATIC_DIR,
+	inDevelopment,
 	N8N_VERSION,
 	NODES_BASE_DIR,
 	RESPONSE_ERROR_MESSAGES,
@@ -138,10 +139,11 @@ import {
 } from '@/CredentialsHelper';
 import { CredentialsOverwrites } from '@/CredentialsOverwrites';
 import { CredentialTypes } from '@/CredentialTypes';
-import { NodeTypes } from '@/NodeTypes';
-import * as Push from '@/Push';
 import { LoadNodesAndCredentials } from '@/LoadNodesAndCredentials';
 import type { LoadNodesAndCredentialsClass } from '@/LoadNodesAndCredentials';
+import type { NodeTypesClass } from '@/NodeTypes';
+import { NodeTypes } from '@/NodeTypes';
+import * as Push from '@/Push';
 import * as ResponseHelper from '@/ResponseHelper';
 import type { WaitTrackerClass } from '@/WaitTracker';
 import { WaitTracker } from '@/WaitTracker';
@@ -177,10 +179,12 @@ class Server extends AbstractServer {
 
 	loadNodesAndCredentials: LoadNodesAndCredentialsClass;
 
-	nodeTypes: INodeTypes;
+	nodeTypes: NodeTypesClass;
 
 	credentialTypes: ICredentialTypes;
 
+	push: Push.Push;
+
 	constructor() {
 		super();
 
@@ -198,6 +202,8 @@ class Server extends AbstractServer {
 			this.app.use('/e2e', require('./api/e2e.api').e2eController);
 		}
 
+		this.push = Push.getInstance();
+
 		const urlBaseWebhook = WebhookHelpers.getWebhookBaseUrl();
 		const telemetrySettings: ITelemetrySettings = {
 			enabled: config.getEnv('diagnostics.enabled'),
@@ -429,7 +435,6 @@ class Server extends AbstractServer {
 		this.app.use(cookieParser());
 
 		// Get push connections
-		const push = Push.getInstance();
 		this.app.use(`/${this.restEndpoint}/push`, corsMiddleware, async (req, res, next) => {
 			const { sessionId } = req.query;
 			if (sessionId === undefined) {
@@ -447,7 +452,7 @@ class Server extends AbstractServer {
 				}
 			}
 
-			push.add(sessionId as string, req, res);
+			this.push.add(sessionId as string, req, res);
 		});
 
 		// Make sure that Vue history mode works properly
@@ -1370,6 +1375,11 @@ export async function start(): Promise<void> {
 	// Set up event handling
 	initEvents();
 
+	if (inDevelopment && process.env.N8N_DEV_RELOAD === 'true') {
+		const { reloadNodesAndCredentials } = await import('@/ReloadNodesAndCredentials');
+		await reloadNodesAndCredentials(app.loadNodesAndCredentials, app.nodeTypes, app.push);
+	}
+
 	void Db.collections.Workflow.findOne({
 		select: ['createdAt'],
 		order: { createdAt: 'ASC' },