fix: Fix hot reloading of custom nodes (#18094)

packages/cli/package.json

@@ -49,6 +49,7 @@
   ],
   "devDependencies": {
     "@n8n/typescript-config": "workspace:*",
+    "@parcel/watcher": "^2.5.1",
     "@redocly/cli": "^1.28.5",
     "@types/aws4": "^1.5.1",
     "@types/bcryptjs": "^2.4.2",

packages/cli/src/__tests__/load-nodes-and-credentials.test.ts

@@ -1,9 +1,29 @@
+import fs from 'fs/promises';
 import { mock } from 'jest-mock-extended';
 import type { DirectoryLoader } from 'n8n-core';
 import type { INodeProperties, INodeTypeDescription } from 'n8n-workflow';
 import { NodeConnectionTypes } from 'n8n-workflow';
+import watcher from '@parcel/watcher';
 
 import { LoadNodesAndCredentials } from '../load-nodes-and-credentials';
+import { Service } from '@n8n/di';
+
+jest.mock('lodash/debounce', () => (fn: () => void) => fn);
+
+jest.mock('@parcel/watcher', () => ({
+	subscribe: jest.fn().mockResolvedValue(undefined),
+}));
+
+jest.mock('fs/promises');
+
+jest.mock('@/push', () => {
+	@Service()
+	class Push {
+		broadcast = jest.fn();
+	}
+
+	return { Push };
+});
 
 describe('LoadNodesAndCredentials', () => {
 	describe('resolveIcon', () => {
@@ -422,4 +442,68 @@ describe('LoadNodesAndCredentials', () => {
 			});
 		});
 	});
+
+	describe('setupHotReload', () => {
+		let instance: LoadNodesAndCredentials;
+
+		const mockLoader = mock<DirectoryLoader>({
+			packageName: 'CUSTOM',
+			directory: '/some/custom/path',
+			isLazyLoaded: false,
+			reset: jest.fn(),
+			loadAll: jest.fn(),
+		});
+
+		beforeEach(() => {
+			instance = new LoadNodesAndCredentials(mock(), mock(), mock(), mock());
+			instance.loaders = { CUSTOM: mockLoader };
+
+			// Allow access to directory
+			(fs.access as jest.Mock).mockResolvedValue(undefined);
+
+			// Simulate custom node dir structure
+			(fs.readdir as jest.Mock).mockResolvedValue([
+				{ name: 'test-node', isDirectory: () => true, isSymbolicLink: () => false },
+			]);
+
+			// Simulate symlink resolution
+			(fs.realpath as jest.Mock).mockResolvedValue('/resolved/test-node');
+		});
+
+		afterEach(() => {
+			jest.clearAllMocks();
+		});
+
+		it('should subscribe to file changes and reload on changes', async () => {
+			const postProcessSpy = jest
+				.spyOn(instance, 'postProcessLoaders')
+				.mockResolvedValue(undefined);
+			const subscribe = jest.mocked(watcher.subscribe);
+
+			await instance.setupHotReload();
+
+			console.log(subscribe);
+			expect(subscribe).toHaveBeenCalledTimes(2);
+			expect(subscribe).toHaveBeenCalledWith('/some/custom/path', expect.any(Function), {
+				ignore: ['**/node_modules/**/node_modules/**'],
+			});
+			expect(subscribe).toHaveBeenCalledWith('/resolved/test-node', expect.any(Function), {
+				ignore: ['**/node_modules/**/node_modules/**'],
+			});
+
+			const [watchPath, onFileUpdate] = subscribe.mock.calls[0];
+
+			expect(watchPath).toBe('/some/custom/path');
+
+			// Simulate file change
+			const fakeModule = '/some/custom/path/some-module.js';
+			require.cache[fakeModule] = mock<NodeJS.Module>({ filename: fakeModule });
+			await onFileUpdate(null, [{ type: 'update', path: fakeModule }]);
+
+			expect(require.cache[fakeModule]).toBeUndefined(); // cache should be cleared
+			expect(mockLoader.reset).toHaveBeenCalled();
+			expect(mockLoader.loadAll).toHaveBeenCalled();
+			expect(postProcessSpy).toHaveBeenCalled();
+		});
+	});
 });

packages/cli/src/load-nodes-and-credentials.ts

@@ -1,6 +1,7 @@
 import { inTest, isContainedWithin, Logger } from '@n8n/backend-common';
 import { GlobalConfig } from '@n8n/config';
 import { Container, Service } from '@n8n/di';
+import type ParcelWatcher from '@parcel/watcher';
 import glob from 'fast-glob';
 import fsPromises from 'fs/promises';
 import type { Class, DirectoryLoader, Types } from 'n8n-core';
@@ -26,13 +27,13 @@ import type {
 	LoadedNodesAndCredentials,
 } from 'n8n-workflow';
 import { deepCopy, NodeConnectionTypes, UnexpectedError, UserError } from 'n8n-workflow';
-import { type Stats } from 'node:fs';
 import path from 'path';
 import picocolors from 'picocolors';
 
-import { CUSTOM_API_CALL_KEY, CUSTOM_API_CALL_NAME, CLI_DIR, inE2ETests } from '@/constants';
 import { CommunityPackagesConfig } from './community-packages/community-packages.config';
 
+import { CUSTOM_API_CALL_KEY, CUSTOM_API_CALL_NAME, CLI_DIR, inE2ETests } from '@/constants';
+
 @Service()
 export class LoadNodesAndCredentials {
 	private known: KnownNodesAndCredentials = { nodes: {}, credentials: {} };
@@ -528,18 +529,18 @@ export class LoadNodesAndCredentials {
 	async setupHotReload() {
 		const { default: debounce } = await import('lodash/debounce');
 
-		const { watch } = await import('chokidar');
+		const { subscribe } = await import('@parcel/watcher');
 
 		const { Push } = await import('@/push');
 		const push = Container.get(Push);
 
-		Object.values(this.loaders).forEach(async (loader) => {
+		for (const loader of Object.values(this.loaders)) {
 			const { directory } = loader;
 			try {
 				await fsPromises.access(directory);
 			} catch {
 				// If directory doesn't exist, there is nothing to watch
-				return;
+				continue;
 			}
 
 			const reloader = debounce(async () => {
@@ -555,35 +556,57 @@ export class LoadNodesAndCredentials {
 			}, 100);
 
 			// For lazy loaded packages, we need to watch the dist directory
-			const watchPath = loader.isLazyLoaded ? path.join(directory, 'dist') : directory;
+			const watchPaths = loader.isLazyLoaded ? [path.join(directory, 'dist')] : [directory];
+			const customNodesRoot = path.join(directory, 'node_modules');
 
-			// Watch options for chokidar v4
-			const watchOptions = {
-				ignoreInitial: true,
-				cwd: directory,
-				// Filter which files to watch based on loader type
-				ignored: (filePath: string, stats?: Stats) => {
-					if (!stats) return false;
-					if (stats.isDirectory()) return false;
-					if (filePath.includes('node_modules')) return true;
+			if (loader.packageName === 'CUSTOM') {
+				const customNodeEntries = await fsPromises.readdir(customNodesRoot, {
+					withFileTypes: true,
+				});
 
-					if (loader.isLazyLoaded) {
-						// Only watch nodes.json and credentials.json files
-						const basename = path.basename(filePath);
-						return basename !== 'nodes.json' && basename !== 'credentials.json';
+				// Custom nodes are usually symlinked using npm link. Resolve symlinks to support file watching
+				const realCustomNodesPaths = await Promise.all(
+					customNodeEntries
+						.filter(
+							(entry) =>
+								(entry.isDirectory() || entry.isSymbolicLink()) && !entry.name.startsWith('.'),
+						)
+						.map(
+							async (entry) =>
+								await fsPromises.realpath(path.join(customNodesRoot, entry.name)).catch(() => null),
+						),
+				);
+
+				watchPaths.push.apply(
+					watchPaths,
+					realCustomNodesPaths.filter((path): path is string => !!path),
+				);
+			}
+
+			this.logger.debug('Watching node folders for hot reload', {
+				loader: loader.packageName,
+				paths: watchPaths,
+			});
+
+			for (const watchPath of watchPaths) {
+				const onFileEvent: ParcelWatcher.SubscribeCallback = async (_error, events) => {
+					if (events.some((event) => event.type !== 'delete')) {
+						const modules = Object.keys(require.cache).filter((module) =>
+							module.startsWith(watchPath),
+						);
+
+						for (const module of modules) {
+							delete require.cache[module];
+						}
+						await reloader();
 					}
+				};
 
-					// Watch all .js and .json files
-					return !filePath.endsWith('.js') && !filePath.endsWith('.json');
-				},
-			};
+				// Ignore nested node_modules folders
+				const ignore = ['**/node_modules/**/node_modules/**'];
 
-			const watcher = watch(watchPath, watchOptions);
-
-			// Watch for file changes and additions
-			// Not watching removals to prevent issues during build processes
-			watcher.on('change', reloader);
-			watcher.on('add', reloader);
-		});
+				await subscribe(watchPath, onFileEvent, { ignore });
+			}
+		}
 	}
 }

packages/nodes-base/package.json

@@ -892,7 +892,6 @@
     "basic-auth": "catalog:",
     "change-case": "4.1.2",
     "cheerio": "1.0.0-rc.6",
-    "chokidar": "catalog:",
     "cron": "3.1.7",
     "csv-parse": "5.5.0",
     "currency-codes": "2.1.0",

pnpm-lock.yaml

@@ -1660,6 +1660,9 @@ importers:
       '@n8n/typescript-config':
         specifier: workspace:*
         version: link:../@n8n/typescript-config
+      '@parcel/watcher':
+        specifier: ^2.5.1
+        version: 2.5.1
       '@redocly/cli':
         specifier: ^1.28.5
         version: 1.28.5(encoding@0.1.13)
@@ -2791,9 +2794,6 @@ importers:
       cheerio:
         specifier: 1.0.0-rc.6
         version: 1.0.0-rc.6
-      chokidar:
-        specifier: ^4.0.1
-        version: 4.0.1
       cron:
         specifier: 3.1.7
         version: 3.1.7
@@ -20502,7 +20502,6 @@ snapshots:
       '@parcel/watcher-win32-arm64': 2.5.1
       '@parcel/watcher-win32-ia32': 2.5.1
       '@parcel/watcher-win32-x64': 2.5.1
-    optional: true
 
   '@petamoriken/float16@3.9.2': {}
 
@@ -24798,8 +24797,7 @@ snapshots:
 
   destr@2.0.5: {}
 
-  detect-libc@1.0.3:
-    optional: true
+  detect-libc@1.0.3: {}
 
   detect-libc@2.0.4: {}
 

pnpm-workspace.yaml

@@ -24,7 +24,6 @@ catalog:
   axios: 1.8.3
   basic-auth: 2.0.1
   callsites: 3.1.0
-  chokidar: 4.0.1
   fast-glob: 3.2.12
   flatted: 3.2.7
   form-data: 4.0.0