From a5fa808d4a872b1fbaedf5be62223a4760e544a8 Mon Sep 17 00:00:00 2001 From: Elias Meire Date: Mon, 11 Aug 2025 09:24:05 +0200 Subject: [PATCH] fix: Fix hot reloading of custom nodes (#18094) --- packages/cli/package.json | 1 + .../load-nodes-and-credentials.test.ts | 84 +++++++++++++++++++ .../cli/src/load-nodes-and-credentials.ts | 83 +++++++++++------- packages/nodes-base/package.json | 1 - pnpm-lock.yaml | 10 +-- pnpm-workspace.yaml | 1 - 6 files changed, 142 insertions(+), 38 deletions(-) diff --git a/packages/cli/package.json b/packages/cli/package.json index 82e91417fb..f8b397183a 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -49,6 +49,7 @@ ], "devDependencies": { "@n8n/typescript-config": "workspace:*", + "@parcel/watcher": "^2.5.1", "@redocly/cli": "^1.28.5", "@types/aws4": "^1.5.1", "@types/bcryptjs": "^2.4.2", diff --git a/packages/cli/src/__tests__/load-nodes-and-credentials.test.ts b/packages/cli/src/__tests__/load-nodes-and-credentials.test.ts index 7b88a0aad0..864d3c22fd 100644 --- a/packages/cli/src/__tests__/load-nodes-and-credentials.test.ts +++ b/packages/cli/src/__tests__/load-nodes-and-credentials.test.ts @@ -1,9 +1,29 @@ +import fs from 'fs/promises'; import { mock } from 'jest-mock-extended'; import type { DirectoryLoader } from 'n8n-core'; import type { INodeProperties, INodeTypeDescription } from 'n8n-workflow'; import { NodeConnectionTypes } from 'n8n-workflow'; +import watcher from '@parcel/watcher'; import { LoadNodesAndCredentials } from '../load-nodes-and-credentials'; +import { Service } from '@n8n/di'; + +jest.mock('lodash/debounce', () => (fn: () => void) => fn); + +jest.mock('@parcel/watcher', () => ({ + subscribe: jest.fn().mockResolvedValue(undefined), +})); + +jest.mock('fs/promises'); + +jest.mock('@/push', () => { + @Service() + class Push { + broadcast = jest.fn(); + } + + return { Push }; +}); describe('LoadNodesAndCredentials', () => { describe('resolveIcon', () => { @@ -422,4 +442,68 @@ describe('LoadNodesAndCredentials', () => { }); }); }); + + describe('setupHotReload', () => { + let instance: LoadNodesAndCredentials; + + const mockLoader = mock({ + packageName: 'CUSTOM', + directory: '/some/custom/path', + isLazyLoaded: false, + reset: jest.fn(), + loadAll: jest.fn(), + }); + + beforeEach(() => { + instance = new LoadNodesAndCredentials(mock(), mock(), mock(), mock()); + instance.loaders = { CUSTOM: mockLoader }; + + // Allow access to directory + (fs.access as jest.Mock).mockResolvedValue(undefined); + + // Simulate custom node dir structure + (fs.readdir as jest.Mock).mockResolvedValue([ + { name: 'test-node', isDirectory: () => true, isSymbolicLink: () => false }, + ]); + + // Simulate symlink resolution + (fs.realpath as jest.Mock).mockResolvedValue('/resolved/test-node'); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + it('should subscribe to file changes and reload on changes', async () => { + const postProcessSpy = jest + .spyOn(instance, 'postProcessLoaders') + .mockResolvedValue(undefined); + const subscribe = jest.mocked(watcher.subscribe); + + await instance.setupHotReload(); + + console.log(subscribe); + expect(subscribe).toHaveBeenCalledTimes(2); + expect(subscribe).toHaveBeenCalledWith('/some/custom/path', expect.any(Function), { + ignore: ['**/node_modules/**/node_modules/**'], + }); + expect(subscribe).toHaveBeenCalledWith('/resolved/test-node', expect.any(Function), { + ignore: ['**/node_modules/**/node_modules/**'], + }); + + const [watchPath, onFileUpdate] = subscribe.mock.calls[0]; + + expect(watchPath).toBe('/some/custom/path'); + + // Simulate file change + const fakeModule = '/some/custom/path/some-module.js'; + require.cache[fakeModule] = mock({ filename: fakeModule }); + await onFileUpdate(null, [{ type: 'update', path: fakeModule }]); + + expect(require.cache[fakeModule]).toBeUndefined(); // cache should be cleared + expect(mockLoader.reset).toHaveBeenCalled(); + expect(mockLoader.loadAll).toHaveBeenCalled(); + expect(postProcessSpy).toHaveBeenCalled(); + }); + }); }); diff --git a/packages/cli/src/load-nodes-and-credentials.ts b/packages/cli/src/load-nodes-and-credentials.ts index fa37a34c21..af7fa01731 100644 --- a/packages/cli/src/load-nodes-and-credentials.ts +++ b/packages/cli/src/load-nodes-and-credentials.ts @@ -1,6 +1,7 @@ import { inTest, isContainedWithin, Logger } from '@n8n/backend-common'; import { GlobalConfig } from '@n8n/config'; import { Container, Service } from '@n8n/di'; +import type ParcelWatcher from '@parcel/watcher'; import glob from 'fast-glob'; import fsPromises from 'fs/promises'; import type { Class, DirectoryLoader, Types } from 'n8n-core'; @@ -26,13 +27,13 @@ import type { LoadedNodesAndCredentials, } from 'n8n-workflow'; import { deepCopy, NodeConnectionTypes, UnexpectedError, UserError } from 'n8n-workflow'; -import { type Stats } from 'node:fs'; import path from 'path'; import picocolors from 'picocolors'; -import { CUSTOM_API_CALL_KEY, CUSTOM_API_CALL_NAME, CLI_DIR, inE2ETests } from '@/constants'; import { CommunityPackagesConfig } from './community-packages/community-packages.config'; +import { CUSTOM_API_CALL_KEY, CUSTOM_API_CALL_NAME, CLI_DIR, inE2ETests } from '@/constants'; + @Service() export class LoadNodesAndCredentials { private known: KnownNodesAndCredentials = { nodes: {}, credentials: {} }; @@ -528,18 +529,18 @@ export class LoadNodesAndCredentials { async setupHotReload() { const { default: debounce } = await import('lodash/debounce'); - const { watch } = await import('chokidar'); + const { subscribe } = await import('@parcel/watcher'); const { Push } = await import('@/push'); const push = Container.get(Push); - Object.values(this.loaders).forEach(async (loader) => { + for (const loader of Object.values(this.loaders)) { const { directory } = loader; try { await fsPromises.access(directory); } catch { // If directory doesn't exist, there is nothing to watch - return; + continue; } const reloader = debounce(async () => { @@ -555,35 +556,57 @@ export class LoadNodesAndCredentials { }, 100); // For lazy loaded packages, we need to watch the dist directory - const watchPath = loader.isLazyLoaded ? path.join(directory, 'dist') : directory; + const watchPaths = loader.isLazyLoaded ? [path.join(directory, 'dist')] : [directory]; + const customNodesRoot = path.join(directory, 'node_modules'); - // Watch options for chokidar v4 - const watchOptions = { - ignoreInitial: true, - cwd: directory, - // Filter which files to watch based on loader type - ignored: (filePath: string, stats?: Stats) => { - if (!stats) return false; - if (stats.isDirectory()) return false; - if (filePath.includes('node_modules')) return true; + if (loader.packageName === 'CUSTOM') { + const customNodeEntries = await fsPromises.readdir(customNodesRoot, { + withFileTypes: true, + }); - if (loader.isLazyLoaded) { - // Only watch nodes.json and credentials.json files - const basename = path.basename(filePath); - return basename !== 'nodes.json' && basename !== 'credentials.json'; + // Custom nodes are usually symlinked using npm link. Resolve symlinks to support file watching + const realCustomNodesPaths = await Promise.all( + customNodeEntries + .filter( + (entry) => + (entry.isDirectory() || entry.isSymbolicLink()) && !entry.name.startsWith('.'), + ) + .map( + async (entry) => + await fsPromises.realpath(path.join(customNodesRoot, entry.name)).catch(() => null), + ), + ); + + watchPaths.push.apply( + watchPaths, + realCustomNodesPaths.filter((path): path is string => !!path), + ); + } + + this.logger.debug('Watching node folders for hot reload', { + loader: loader.packageName, + paths: watchPaths, + }); + + for (const watchPath of watchPaths) { + const onFileEvent: ParcelWatcher.SubscribeCallback = async (_error, events) => { + if (events.some((event) => event.type !== 'delete')) { + const modules = Object.keys(require.cache).filter((module) => + module.startsWith(watchPath), + ); + + for (const module of modules) { + delete require.cache[module]; + } + await reloader(); } + }; - // Watch all .js and .json files - return !filePath.endsWith('.js') && !filePath.endsWith('.json'); - }, - }; + // Ignore nested node_modules folders + const ignore = ['**/node_modules/**/node_modules/**']; - const watcher = watch(watchPath, watchOptions); - - // Watch for file changes and additions - // Not watching removals to prevent issues during build processes - watcher.on('change', reloader); - watcher.on('add', reloader); - }); + await subscribe(watchPath, onFileEvent, { ignore }); + } + } } } diff --git a/packages/nodes-base/package.json b/packages/nodes-base/package.json index 6953034524..3c46413e42 100644 --- a/packages/nodes-base/package.json +++ b/packages/nodes-base/package.json @@ -892,7 +892,6 @@ "basic-auth": "catalog:", "change-case": "4.1.2", "cheerio": "1.0.0-rc.6", - "chokidar": "catalog:", "cron": "3.1.7", "csv-parse": "5.5.0", "currency-codes": "2.1.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 720b283179..7e48d6e0d5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1660,6 +1660,9 @@ importers: '@n8n/typescript-config': specifier: workspace:* version: link:../@n8n/typescript-config + '@parcel/watcher': + specifier: ^2.5.1 + version: 2.5.1 '@redocly/cli': specifier: ^1.28.5 version: 1.28.5(encoding@0.1.13) @@ -2791,9 +2794,6 @@ importers: cheerio: specifier: 1.0.0-rc.6 version: 1.0.0-rc.6 - chokidar: - specifier: ^4.0.1 - version: 4.0.1 cron: specifier: 3.1.7 version: 3.1.7 @@ -20502,7 +20502,6 @@ snapshots: '@parcel/watcher-win32-arm64': 2.5.1 '@parcel/watcher-win32-ia32': 2.5.1 '@parcel/watcher-win32-x64': 2.5.1 - optional: true '@petamoriken/float16@3.9.2': {} @@ -24798,8 +24797,7 @@ snapshots: destr@2.0.5: {} - detect-libc@1.0.3: - optional: true + detect-libc@1.0.3: {} detect-libc@2.0.4: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 4980b91686..7afcb5432c 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -24,7 +24,6 @@ catalog: axios: 1.8.3 basic-auth: 2.0.1 callsites: 3.1.0 - chokidar: 4.0.1 fast-glob: 3.2.12 flatted: 3.2.7 form-data: 4.0.0