From 9e420d15c125123e0e67c7c1ef68661047bddb6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Fri, 22 Aug 2025 12:19:01 +0200 Subject: [PATCH] refactor(core): Modularize community packages (#18641) --- .../modules/__tests__/module-registry.test.ts | 6 ++- .../src/modules/module-registry.ts | 8 +++- .../src/modules/modules.config.ts | 7 +++- packages/@n8n/db/src/entities/index.ts | 6 --- packages/@n8n/db/src/repositories/index.ts | 2 - packages/@n8n/decorators/src/module/module.ts | 4 +- .../commands/__tests__/community-node.test.ts | 5 ++- .../commands/__tests__/execute-batch.test.ts | 2 +- .../src/commands/__tests__/execute.test.ts | 2 +- packages/cli/src/commands/base-command.ts | 5 ++- .../cli/src/load-nodes-and-credentials.ts | 10 ----- .../community-node-types.service.test.ts | 0 .../community-packages.controller.test.ts | 8 ++-- .../community-packages.integration.test.ts} | 23 +++++++---- .../community-packages.service.test.ts | 20 +++++----- .../__tests__/npm-utils.test.ts | 0 .../__tests__/strapi-utils.test.ts | 0 .../community-node-types-utils.ts | 0 .../community-node-types.controller.ts | 2 +- .../community-node-types.service.ts | 3 +- .../community-node.command.ts} | 11 ++++-- .../community-packages.config.ts | 0 .../community-packages.controller.ts | 14 +++---- .../community-packages.module.ts | 38 +++++++++++++++++++ .../community-packages.service.ts | 12 +++--- .../community-packages.types.ts | 0 .../installed-nodes.entity.ts} | 6 +-- .../installed-nodes.repository.ts | 2 +- .../installed-packages.entity.ts} | 4 +- .../installed-packages.repository.ts | 2 +- .../community-packages/npm-utils.ts | 0 .../community-packages/strapi-utils.ts | 0 .../risk-reporters/instance-risk-reporter.ts | 2 +- .../security-audit.repository.ts | 3 +- packages/cli/src/server.ts | 6 --- .../__tests__/frontend.service.test.ts | 4 +- packages/cli/src/services/frontend.service.ts | 10 +++-- .../integration/commands/worker.cmd.test.ts | 2 +- .../nodes-risk-reporter.test.ts | 2 +- .../test/integration/security-audit/utils.ts | 3 +- packages/cli/test/integration/shared/types.ts | 2 +- .../shared/utils/community-nodes.ts | 4 +- .../integration/shared/utils/test-server.ts | 2 +- 43 files changed, 144 insertions(+), 98 deletions(-) rename packages/cli/src/{ => modules}/community-packages/__tests__/community-node-types.service.test.ts (100%) rename packages/cli/src/{ => modules}/community-packages/__tests__/community-packages.controller.test.ts (92%) rename packages/cli/{test/integration/community-packages.api.test.ts => src/modules/community-packages/__tests__/community-packages.integration.test.ts} (90%) rename packages/cli/src/{ => modules}/community-packages/__tests__/community-packages.service.test.ts (98%) rename packages/cli/src/{ => modules}/community-packages/__tests__/npm-utils.test.ts (100%) rename packages/cli/src/{ => modules}/community-packages/__tests__/strapi-utils.test.ts (100%) rename packages/cli/src/{ => modules}/community-packages/community-node-types-utils.ts (100%) rename packages/cli/src/{ => modules}/community-packages/community-node-types.controller.ts (86%) rename packages/cli/src/{ => modules}/community-packages/community-node-types.service.ts (97%) rename packages/cli/src/{commands/community-node.ts => modules/community-packages/community-node.command.ts} (90%) rename packages/cli/src/{ => modules}/community-packages/community-packages.config.ts (100%) rename packages/cli/src/{ => modules}/community-packages/community-packages.controller.ts (98%) create mode 100644 packages/cli/src/modules/community-packages/community-packages.module.ts rename packages/cli/src/{ => modules}/community-packages/community-packages.service.ts (98%) rename packages/cli/src/{ => modules}/community-packages/community-packages.types.ts (100%) rename packages/{@n8n/db/src/entities/installed-nodes.ts => cli/src/modules/community-packages/installed-nodes.entity.ts} (55%) rename packages/{@n8n/db/src/repositories => cli/src/modules/community-packages}/installed-nodes.repository.ts (81%) rename packages/{@n8n/db/src/entities/installed-packages.ts => cli/src/modules/community-packages/installed-packages.entity.ts} (79%) rename packages/{@n8n/db/src/repositories => cli/src/modules/community-packages}/installed-packages.repository.ts (95%) rename packages/cli/src/{ => modules}/community-packages/npm-utils.ts (100%) rename packages/cli/src/{ => modules}/community-packages/strapi-utils.ts (100%) diff --git a/packages/@n8n/backend-common/src/modules/__tests__/module-registry.test.ts b/packages/@n8n/backend-common/src/modules/__tests__/module-registry.test.ts index 39c7513d69..8336dcca51 100644 --- a/packages/@n8n/backend-common/src/modules/__tests__/module-registry.test.ts +++ b/packages/@n8n/backend-common/src/modules/__tests__/module-registry.test.ts @@ -22,7 +22,10 @@ describe('eligibleModules', () => { it('should consider a module ineligible if it was disabled via env var', () => { process.env.N8N_DISABLED_MODULES = 'insights'; - expect(Container.get(ModuleRegistry).eligibleModules).toEqual(['external-secrets']); + expect(Container.get(ModuleRegistry).eligibleModules).toEqual([ + 'external-secrets', + 'community-packages', + ]); }); it('should consider a module eligible if it was enabled via env var', () => { @@ -30,6 +33,7 @@ describe('eligibleModules', () => { expect(Container.get(ModuleRegistry).eligibleModules).toEqual([ 'insights', 'external-secrets', + 'community-packages', 'data-store', ]); }); diff --git a/packages/@n8n/backend-common/src/modules/module-registry.ts b/packages/@n8n/backend-common/src/modules/module-registry.ts index dcaeb6fa61..dda93a28e3 100644 --- a/packages/@n8n/backend-common/src/modules/module-registry.ts +++ b/packages/@n8n/backend-common/src/modules/module-registry.ts @@ -26,7 +26,11 @@ export class ModuleRegistry { private readonly modulesConfig: ModulesConfig, ) {} - private readonly defaultModules: ModuleName[] = ['insights', 'external-secrets']; + private readonly defaultModules: ModuleName[] = [ + 'insights', + 'external-secrets', + 'community-packages', + ]; private readonly activeModules: string[] = []; @@ -83,7 +87,7 @@ export class ModuleRegistry { if (entities?.length) this.entities.push(...entities); - const loadDir = Container.get(ModuleClass).loadDir?.(); + const loadDir = await Container.get(ModuleClass).loadDir?.(); if (loadDir) this.loadDirs.push(loadDir); } diff --git a/packages/@n8n/backend-common/src/modules/modules.config.ts b/packages/@n8n/backend-common/src/modules/modules.config.ts index 1c8688d733..6b9006ad5a 100644 --- a/packages/@n8n/backend-common/src/modules/modules.config.ts +++ b/packages/@n8n/backend-common/src/modules/modules.config.ts @@ -2,7 +2,12 @@ import { CommaSeparatedStringArray, Config, Env } from '@n8n/config'; import { UnknownModuleError } from './errors/unknown-module.error'; -export const MODULE_NAMES = ['insights', 'external-secrets', 'data-store'] as const; +export const MODULE_NAMES = [ + 'insights', + 'external-secrets', + 'community-packages', + 'data-store', +] as const; export type ModuleName = (typeof MODULE_NAMES)[number]; diff --git a/packages/@n8n/db/src/entities/index.ts b/packages/@n8n/db/src/entities/index.ts index 8b458a795a..9b423c8744 100644 --- a/packages/@n8n/db/src/entities/index.ts +++ b/packages/@n8n/db/src/entities/index.ts @@ -11,8 +11,6 @@ import { ExecutionEntity } from './execution-entity'; import { ExecutionMetadata } from './execution-metadata'; import { Folder } from './folder'; import { FolderTagMapping } from './folder-tag-mapping'; -import { InstalledNodes } from './installed-nodes'; -import { InstalledPackages } from './installed-packages'; import { InvalidAuthToken } from './invalid-auth-token'; import { ProcessedData } from './processed-data'; import { Project } from './project'; @@ -35,8 +33,6 @@ import { WorkflowTagMapping } from './workflow-tag-mapping'; export { EventDestinations, - InstalledNodes, - InstalledPackages, InvalidAuthToken, ProcessedData, Settings, @@ -72,8 +68,6 @@ export { export const entities = { EventDestinations, - InstalledNodes, - InstalledPackages, InvalidAuthToken, ProcessedData, Settings, diff --git a/packages/@n8n/db/src/repositories/index.ts b/packages/@n8n/db/src/repositories/index.ts index 4f0fb5f3bf..dcff65f754 100644 --- a/packages/@n8n/db/src/repositories/index.ts +++ b/packages/@n8n/db/src/repositories/index.ts @@ -12,8 +12,6 @@ export { EventDestinationsRepository } from './event-destinations.repository'; export { FolderRepository } from './folder.repository'; export { FolderTagMappingRepository } from './folder-tag-mapping.repository'; export { ScopeRepository } from './scope.repository'; -export { InstalledNodesRepository } from './installed-nodes.repository'; -export { InstalledPackagesRepository } from './installed-packages.repository'; export { InvalidAuthTokenRepository } from './invalid-auth-token.repository'; export { LicenseMetricsRepository } from './license-metrics.repository'; export { ProjectRelationRepository } from './project-relation.repository'; diff --git a/packages/@n8n/decorators/src/module/module.ts b/packages/@n8n/decorators/src/module/module.ts index 349c245686..93558caa72 100644 --- a/packages/@n8n/decorators/src/module/module.ts +++ b/packages/@n8n/decorators/src/module/module.ts @@ -33,10 +33,10 @@ export interface ModuleInterface { settings?(): Promise; /** - * @returns Path to a dir to load nodes and credentials from. + * @returns Path to a dir to load nodes and credentials from. `null` to skip. * @example '/Users/nathan/.n8n/nodes/node_modules' */ - loadDir?(): string; + loadDir?(): Promise; } export type ModuleClass = Constructable; diff --git a/packages/cli/src/commands/__tests__/community-node.test.ts b/packages/cli/src/commands/__tests__/community-node.test.ts index 6de4c264e1..dbb32fa305 100644 --- a/packages/cli/src/commands/__tests__/community-node.test.ts +++ b/packages/cli/src/commands/__tests__/community-node.test.ts @@ -1,7 +1,8 @@ -import { type InstalledNodes, type CredentialsEntity, type User } from '@n8n/db'; +import type { CredentialsEntity, User } from '@n8n/db'; import { mock } from 'jest-mock-extended'; -import { CommunityNode } from '../community-node'; +import { CommunityNode } from '@/modules/community-packages/community-node.command'; +import type { InstalledNodes } from '@/modules/community-packages/installed-nodes.entity'; describe('uninstallCredential', () => { const userId = '1234'; diff --git a/packages/cli/src/commands/__tests__/execute-batch.test.ts b/packages/cli/src/commands/__tests__/execute-batch.test.ts index 498f312914..a52bdd3093 100644 --- a/packages/cli/src/commands/__tests__/execute-batch.test.ts +++ b/packages/cli/src/commands/__tests__/execute-batch.test.ts @@ -8,12 +8,12 @@ import { mock } from 'jest-mock-extended'; import type { IRun } from 'n8n-workflow'; import { ActiveExecutions } from '@/active-executions'; -import { CommunityPackagesService } from '@/community-packages/community-packages.service'; import { DeprecationService } from '@/deprecation/deprecation.service'; import { MessageEventBus } from '@/eventbus/message-event-bus/message-event-bus'; import { TelemetryEventRelay } from '@/events/relays/telemetry.event-relay'; import { ExternalHooks } from '@/external-hooks'; import { LoadNodesAndCredentials } from '@/load-nodes-and-credentials'; +import { CommunityPackagesService } from '@/modules/community-packages/community-packages.service'; import { PostHogClient } from '@/posthog'; import { OwnershipService } from '@/services/ownership.service'; import { ShutdownService } from '@/shutdown/shutdown.service'; diff --git a/packages/cli/src/commands/__tests__/execute.test.ts b/packages/cli/src/commands/__tests__/execute.test.ts index db0ed1165f..30896c4b22 100644 --- a/packages/cli/src/commands/__tests__/execute.test.ts +++ b/packages/cli/src/commands/__tests__/execute.test.ts @@ -7,12 +7,12 @@ import { mock } from 'jest-mock-extended'; import type { IRun } from 'n8n-workflow'; import { ActiveExecutions } from '@/active-executions'; -import { CommunityPackagesService } from '@/community-packages/community-packages.service'; import { DeprecationService } from '@/deprecation/deprecation.service'; import { MessageEventBus } from '@/eventbus/message-event-bus/message-event-bus'; import { TelemetryEventRelay } from '@/events/relays/telemetry.event-relay'; import { ExternalHooks } from '@/external-hooks'; import { LoadNodesAndCredentials } from '@/load-nodes-and-credentials'; +import { CommunityPackagesService } from '@/modules/community-packages/community-packages.service'; import { PostHogClient } from '@/posthog'; import { OwnershipService } from '@/services/ownership.service'; import { ShutdownService } from '@/shutdown/shutdown.service'; diff --git a/packages/cli/src/commands/base-command.ts b/packages/cli/src/commands/base-command.ts index 54bb986d85..525c833171 100644 --- a/packages/cli/src/commands/base-command.ts +++ b/packages/cli/src/commands/base-command.ts @@ -33,11 +33,11 @@ import { TelemetryEventRelay } from '@/events/relays/telemetry.event-relay'; import { ExternalHooks } from '@/external-hooks'; import { License } from '@/license'; import { LoadNodesAndCredentials } from '@/load-nodes-and-credentials'; +import { CommunityPackagesConfig } from '@/modules/community-packages/community-packages.config'; import { NodeTypes } from '@/node-types'; import { PostHogClient } from '@/posthog'; import { ShutdownService } from '@/shutdown/shutdown.service'; import { WorkflowHistoryManager } from '@/workflows/workflow-history.ee/workflow-history-manager.ee'; -import { CommunityPackagesConfig } from '@/community-packages/community-packages.config'; export abstract class BaseCommand { readonly flags: F; @@ -133,10 +133,11 @@ export abstract class BaseCommand { ); } + // @TODO: Move to community-packages module const communityPackagesConfig = Container.get(CommunityPackagesConfig); if (communityPackagesConfig.enabled && this.needsCommunityPackages) { const { CommunityPackagesService } = await import( - '@/community-packages/community-packages.service' + '@/modules/community-packages/community-packages.service' ); await Container.get(CommunityPackagesService).init(); } diff --git a/packages/cli/src/load-nodes-and-credentials.ts b/packages/cli/src/load-nodes-and-credentials.ts index b409ed9593..b601945693 100644 --- a/packages/cli/src/load-nodes-and-credentials.ts +++ b/packages/cli/src/load-nodes-and-credentials.ts @@ -30,8 +30,6 @@ import { deepCopy, NodeConnectionTypes, UnexpectedError, UserError } from 'n8n-w import path from 'path'; import picocolors from 'picocolors'; -import { CommunityPackagesConfig } from './community-packages/community-packages.config'; - import { CUSTOM_API_CALL_KEY, CUSTOM_API_CALL_NAME, CLI_DIR, inE2ETests } from '@/constants'; @Service() @@ -91,14 +89,6 @@ export class LoadNodesAndCredentials { await this.loadNodesFromNodeModules(nodeModulesDir, '@n8n/n8n-nodes-langchain'); } - if (!Container.get(CommunityPackagesConfig).preventLoading) { - // Load nodes from any other `n8n-nodes-*` packages in the download directory - // This includes the community nodes - await this.loadNodesFromNodeModules( - path.join(this.instanceSettings.nodesDownloadDir, 'node_modules'), - ); - } - for (const dir of this.moduleRegistry.loadDirs) { await this.loadNodesFromNodeModules(dir); } diff --git a/packages/cli/src/community-packages/__tests__/community-node-types.service.test.ts b/packages/cli/src/modules/community-packages/__tests__/community-node-types.service.test.ts similarity index 100% rename from packages/cli/src/community-packages/__tests__/community-node-types.service.test.ts rename to packages/cli/src/modules/community-packages/__tests__/community-node-types.service.test.ts diff --git a/packages/cli/src/community-packages/__tests__/community-packages.controller.test.ts b/packages/cli/src/modules/community-packages/__tests__/community-packages.controller.test.ts similarity index 92% rename from packages/cli/src/community-packages/__tests__/community-packages.controller.test.ts rename to packages/cli/src/modules/community-packages/__tests__/community-packages.controller.test.ts index e1e44f06d1..c8a9efaeb0 100644 --- a/packages/cli/src/community-packages/__tests__/community-packages.controller.test.ts +++ b/packages/cli/src/modules/community-packages/__tests__/community-packages.controller.test.ts @@ -1,14 +1,14 @@ import type { CommunityNodeType } from '@n8n/api-types'; -import type { InstalledPackages } from '@n8n/db'; import { mock } from 'jest-mock-extended'; -import { CommunityPackagesController } from '@/community-packages/community-packages.controller'; +import { CommunityPackagesController } from '@/modules/community-packages/community-packages.controller'; import type { NodeRequest } from '@/requests'; -import type { EventService } from '../../events/event.service'; -import type { Push } from '../../push'; +import type { EventService } from '../../../events/event.service'; +import type { Push } from '../../../push'; import type { CommunityNodeTypesService } from '../community-node-types.service'; import type { CommunityPackagesService } from '../community-packages.service'; +import type { InstalledPackages } from '../installed-packages.entity'; describe('CommunityPackagesController', () => { const push = mock(); diff --git a/packages/cli/test/integration/community-packages.api.test.ts b/packages/cli/src/modules/community-packages/__tests__/community-packages.integration.test.ts similarity index 90% rename from packages/cli/test/integration/community-packages.api.test.ts rename to packages/cli/src/modules/community-packages/__tests__/community-packages.integration.test.ts index ca929e51e0..436e5cbe0b 100644 --- a/packages/cli/test/integration/community-packages.api.test.ts +++ b/packages/cli/src/modules/community-packages/__tests__/community-packages.integration.test.ts @@ -1,21 +1,30 @@ import { mockInstance } from '@n8n/backend-test-utils'; -import type { InstalledNodes, InstalledPackages } from '@n8n/db'; import path from 'path'; import { LoadNodesAndCredentials } from '@/load-nodes-and-credentials'; -import { CommunityPackagesService } from '@/community-packages/community-packages.service'; +import { CommunityPackagesService } from '@/modules/community-packages/community-packages.service'; +import type { InstalledNodes } from '@/modules/community-packages/installed-nodes.entity'; +import type { InstalledPackages } from '@/modules/community-packages/installed-packages.entity'; -import { COMMUNITY_PACKAGE_VERSION } from './shared/constants'; -import { createOwner } from './shared/db/users'; -import type { SuperAgentTest } from './shared/types'; -import { setupTestServer, mockPackage, mockNode, mockPackageName } from './shared/utils'; +import { COMMUNITY_PACKAGE_VERSION } from '../../../../test/integration/shared/constants'; +import { createOwner } from '../../../../test/integration/shared/db/users'; +import type { SuperAgentTest } from '../../../../test/integration/shared/types'; +import { + setupTestServer, + mockPackage, + mockNode, + mockPackageName, +} from '../../../../test/integration/shared/utils'; const communityPackagesService = mockInstance(CommunityPackagesService, { hasMissingPackages: false, }); mockInstance(LoadNodesAndCredentials); -const testServer = setupTestServer({ endpointGroups: ['community-packages'] }); +const testServer = setupTestServer({ + endpointGroups: ['community-packages'], + modules: ['community-packages'], +}); const commonUpdatesProps = { createdAt: new Date(), diff --git a/packages/cli/src/community-packages/__tests__/community-packages.service.test.ts b/packages/cli/src/modules/community-packages/__tests__/community-packages.service.test.ts similarity index 98% rename from packages/cli/src/community-packages/__tests__/community-packages.service.test.ts rename to packages/cli/src/modules/community-packages/__tests__/community-packages.service.test.ts index 11f92d99c9..9db83414c2 100644 --- a/packages/cli/src/community-packages/__tests__/community-packages.service.test.ts +++ b/packages/cli/src/modules/community-packages/__tests__/community-packages.service.test.ts @@ -1,22 +1,15 @@ import type { Logger } from '@n8n/backend-common'; import { randomName, mockInstance } from '@n8n/backend-test-utils'; import { LICENSE_FEATURES } from '@n8n/constants'; -import { - InstalledNodes, - InstalledPackages, - InstalledNodesRepository, - InstalledPackagesRepository, -} from '@n8n/db'; import axios from 'axios'; -import { exec } from 'child_process'; -import { mkdir, readFile, writeFile, rm, access, constants } from 'fs/promises'; import { mocked } from 'jest-mock'; import { mock } from 'jest-mock-extended'; import type { InstanceSettings, PackageDirectoryLoader } from 'n8n-core'; import type { PublicInstalledPackage } from 'n8n-workflow'; +import { exec } from 'node:child_process'; +import { mkdir, readFile, writeFile, rm, access, constants } from 'node:fs/promises'; import { join } from 'node:path'; -import { CommunityPackagesService } from '@/community-packages/community-packages.service'; import { NODE_PACKAGE_PREFIX, NPM_COMMAND_TOKENS, @@ -31,10 +24,15 @@ import { COMMUNITY_NODE_VERSION, COMMUNITY_PACKAGE_VERSION } from '@test-integra import { mockPackageName, mockPackagePair } from '@test-integration/utils'; import type { CommunityPackagesConfig } from '../community-packages.config'; +import { CommunityPackagesService } from '../community-packages.service'; import type { CommunityPackages } from '../community-packages.types'; +import { InstalledNodes } from '../installed-nodes.entity'; +import { InstalledNodesRepository } from '../installed-nodes.repository'; +import { InstalledPackages } from '../installed-packages.entity'; +import { InstalledPackagesRepository } from '../installed-packages.repository'; -jest.mock('fs/promises'); -jest.mock('child_process'); +jest.mock('node:fs/promises'); +jest.mock('node:child_process'); jest.mock('axios'); type ExecOptions = NonNullable[1]>; diff --git a/packages/cli/src/community-packages/__tests__/npm-utils.test.ts b/packages/cli/src/modules/community-packages/__tests__/npm-utils.test.ts similarity index 100% rename from packages/cli/src/community-packages/__tests__/npm-utils.test.ts rename to packages/cli/src/modules/community-packages/__tests__/npm-utils.test.ts diff --git a/packages/cli/src/community-packages/__tests__/strapi-utils.test.ts b/packages/cli/src/modules/community-packages/__tests__/strapi-utils.test.ts similarity index 100% rename from packages/cli/src/community-packages/__tests__/strapi-utils.test.ts rename to packages/cli/src/modules/community-packages/__tests__/strapi-utils.test.ts diff --git a/packages/cli/src/community-packages/community-node-types-utils.ts b/packages/cli/src/modules/community-packages/community-node-types-utils.ts similarity index 100% rename from packages/cli/src/community-packages/community-node-types-utils.ts rename to packages/cli/src/modules/community-packages/community-node-types-utils.ts diff --git a/packages/cli/src/community-packages/community-node-types.controller.ts b/packages/cli/src/modules/community-packages/community-node-types.controller.ts similarity index 86% rename from packages/cli/src/community-packages/community-node-types.controller.ts rename to packages/cli/src/modules/community-packages/community-node-types.controller.ts index e1f48d696f..e8ea49da03 100644 --- a/packages/cli/src/community-packages/community-node-types.controller.ts +++ b/packages/cli/src/modules/community-packages/community-node-types.controller.ts @@ -2,7 +2,7 @@ import type { CommunityNodeType } from '@n8n/api-types'; import { Get, RestController } from '@n8n/decorators'; import { Request } from 'express'; -import { CommunityNodeTypesService } from '@/community-packages/community-node-types.service'; +import { CommunityNodeTypesService } from './community-node-types.service'; @RestController('/community-node-types') export class CommunityNodeTypesController { diff --git a/packages/cli/src/community-packages/community-node-types.service.ts b/packages/cli/src/modules/community-packages/community-node-types.service.ts similarity index 97% rename from packages/cli/src/community-packages/community-node-types.service.ts rename to packages/cli/src/modules/community-packages/community-node-types.service.ts index 57e586214a..b17a742ba4 100644 --- a/packages/cli/src/community-packages/community-node-types.service.ts +++ b/packages/cli/src/modules/community-packages/community-node-types.service.ts @@ -3,9 +3,8 @@ import { Logger, inProduction } from '@n8n/backend-common'; import { Service } from '@n8n/di'; import { ensureError } from 'n8n-workflow'; -import { CommunityPackagesConfig } from '@/community-packages/community-packages.config'; - import { getCommunityNodeTypes, StrapiCommunityNodeType } from './community-node-types-utils'; +import { CommunityPackagesConfig } from './community-packages.config'; import { CommunityPackagesService } from './community-packages.service'; const UPDATE_INTERVAL = 8 * 60 * 60 * 1000; diff --git a/packages/cli/src/commands/community-node.ts b/packages/cli/src/modules/community-packages/community-node.command.ts similarity index 90% rename from packages/cli/src/commands/community-node.ts rename to packages/cli/src/modules/community-packages/community-node.command.ts index 98373627e3..2868da31e6 100644 --- a/packages/cli/src/commands/community-node.ts +++ b/packages/cli/src/modules/community-packages/community-node.command.ts @@ -1,13 +1,16 @@ -import { type InstalledNodes, type InstalledPackages, type User } from '@n8n/db'; -import { CredentialsRepository, InstalledNodesRepository, UserRepository } from '@n8n/db'; +import type { User } from '@n8n/db'; +import { CredentialsRepository, UserRepository } from '@n8n/db'; import { Command } from '@n8n/decorators'; import { Container } from '@n8n/di'; import { z } from 'zod'; +import { BaseCommand } from '@/commands/base-command'; import { CredentialsService } from '@/credentials/credentials.service'; -import { CommunityPackagesService } from '@/community-packages/community-packages.service'; -import { BaseCommand } from './base-command'; +import { CommunityPackagesService } from './community-packages.service'; +import { InstalledNodes } from './installed-nodes.entity'; +import { InstalledNodesRepository } from './installed-nodes.repository'; +import { InstalledPackages } from './installed-packages.entity'; const flagsSchema = z.object({ uninstall: z.boolean().describe('Uninstalls the node').optional(), diff --git a/packages/cli/src/community-packages/community-packages.config.ts b/packages/cli/src/modules/community-packages/community-packages.config.ts similarity index 100% rename from packages/cli/src/community-packages/community-packages.config.ts rename to packages/cli/src/modules/community-packages/community-packages.config.ts diff --git a/packages/cli/src/community-packages/community-packages.controller.ts b/packages/cli/src/modules/community-packages/community-packages.controller.ts similarity index 98% rename from packages/cli/src/community-packages/community-packages.controller.ts rename to packages/cli/src/modules/community-packages/community-packages.controller.ts index c8f6db3a7c..cf7550096c 100644 --- a/packages/cli/src/community-packages/community-packages.controller.ts +++ b/packages/cli/src/modules/community-packages/community-packages.controller.ts @@ -1,21 +1,21 @@ +import { Delete, Get, Patch, Post, RestController, GlobalScope } from '@n8n/decorators'; + import { RESPONSE_ERROR_MESSAGES, STARTER_TEMPLATE_NAME, UNKNOWN_FAILURE_REASON, } from '@/constants'; -import type { InstalledPackages } from '@n8n/db'; -import { Delete, Get, Patch, Post, RestController, GlobalScope } from '@n8n/decorators'; - -import type { CommunityPackages } from './community-packages.types'; -import { CommunityNodeTypesService } from './community-node-types.service'; - -import { CommunityPackagesService } from '@/community-packages/community-packages.service'; import { BadRequestError } from '@/errors/response-errors/bad-request.error'; import { InternalServerError } from '@/errors/response-errors/internal-server.error'; import { EventService } from '@/events/event.service'; import { Push } from '@/push'; import { NodeRequest } from '@/requests'; +import { CommunityNodeTypesService } from './community-node-types.service'; +import { CommunityPackagesService } from './community-packages.service'; +import type { CommunityPackages } from './community-packages.types'; +import { InstalledPackages } from './installed-packages.entity'; + const { PACKAGE_NOT_INSTALLED, PACKAGE_NAME_NOT_PROVIDED, diff --git a/packages/cli/src/modules/community-packages/community-packages.module.ts b/packages/cli/src/modules/community-packages/community-packages.module.ts new file mode 100644 index 0000000000..db68dc7f70 --- /dev/null +++ b/packages/cli/src/modules/community-packages/community-packages.module.ts @@ -0,0 +1,38 @@ +import type { EntityClass, ModuleInterface } from '@n8n/decorators'; +import { BackendModule } from '@n8n/decorators'; +import { Container } from '@n8n/di'; +import { InstanceSettings } from 'n8n-core'; +import path from 'node:path'; + +@BackendModule({ name: 'community-packages' }) +export class CommunityPackagesModule implements ModuleInterface { + async init() { + await import('./community-packages.controller'); + } + + async entities() { + const { InstalledNodes } = await import('./installed-nodes.entity'); + const { InstalledPackages } = await import('./installed-packages.entity'); + + return [InstalledNodes, InstalledPackages] as EntityClass[]; + } + + async settings() { + const { CommunityPackagesConfig } = await import('./community-packages.config'); + + return { + communityNodesEnabled: Container.get(CommunityPackagesConfig).enabled, + unverifiedCommunityNodesEnabled: Container.get(CommunityPackagesConfig).unverifiedEnabled, + }; + } + + async loadDir() { + const { CommunityPackagesConfig } = await import('./community-packages.config'); + + const { preventLoading } = Container.get(CommunityPackagesConfig); + + if (preventLoading) return null; + + return path.join(Container.get(InstanceSettings).nodesDownloadDir, 'node_modules'); + } +} diff --git a/packages/cli/src/community-packages/community-packages.service.ts b/packages/cli/src/modules/community-packages/community-packages.service.ts similarity index 98% rename from packages/cli/src/community-packages/community-packages.service.ts rename to packages/cli/src/modules/community-packages/community-packages.service.ts index 4d47b4cef0..f5bc1fd0d7 100644 --- a/packages/cli/src/community-packages/community-packages.service.ts +++ b/packages/cli/src/modules/community-packages/community-packages.service.ts @@ -1,17 +1,15 @@ import { Logger } from '@n8n/backend-common'; import { LICENSE_FEATURES } from '@n8n/constants'; -import type { InstalledPackages } from '@n8n/db'; -import { InstalledPackagesRepository } from '@n8n/db'; import { OnPubSubEvent } from '@n8n/decorators'; import { Service } from '@n8n/di'; import axios from 'axios'; -import { exec } from 'child_process'; -import { access, constants, mkdir, readFile, rm, writeFile } from 'fs/promises'; import type { PackageDirectoryLoader } from 'n8n-core'; import { InstanceSettings } from 'n8n-core'; import { jsonParse, UnexpectedError, UserError, type PublicInstalledPackage } from 'n8n-workflow'; -import { join } from 'path'; -import { promisify } from 'util'; +import { exec } from 'node:child_process'; +import { access, constants, mkdir, readFile, rm, writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { promisify } from 'node:util'; import { NODE_PACKAGE_PREFIX, @@ -28,6 +26,8 @@ import { toError } from '@/utils'; import { CommunityPackagesConfig } from './community-packages.config'; import type { CommunityPackages } from './community-packages.types'; +import { InstalledPackages } from './installed-packages.entity'; +import { InstalledPackagesRepository } from './installed-packages.repository'; import { isVersionExists, verifyIntegrity } from './npm-utils'; const DEFAULT_REGISTRY = 'https://registry.npmjs.org'; diff --git a/packages/cli/src/community-packages/community-packages.types.ts b/packages/cli/src/modules/community-packages/community-packages.types.ts similarity index 100% rename from packages/cli/src/community-packages/community-packages.types.ts rename to packages/cli/src/modules/community-packages/community-packages.types.ts diff --git a/packages/@n8n/db/src/entities/installed-nodes.ts b/packages/cli/src/modules/community-packages/installed-nodes.entity.ts similarity index 55% rename from packages/@n8n/db/src/entities/installed-nodes.ts rename to packages/cli/src/modules/community-packages/installed-nodes.entity.ts index 26c0ce75e0..25fc6ec3d2 100644 --- a/packages/@n8n/db/src/entities/installed-nodes.ts +++ b/packages/cli/src/modules/community-packages/installed-nodes.entity.ts @@ -1,9 +1,9 @@ -import { Column, Entity, JoinColumn, ManyToOne, PrimaryColumn } from '@n8n/typeorm'; +import { BaseEntity, Column, Entity, JoinColumn, ManyToOne, PrimaryColumn } from '@n8n/typeorm'; -import { InstalledPackages } from './installed-packages'; +import { InstalledPackages } from './installed-packages.entity'; @Entity() -export class InstalledNodes { +export class InstalledNodes extends BaseEntity { @Column() name: string; diff --git a/packages/@n8n/db/src/repositories/installed-nodes.repository.ts b/packages/cli/src/modules/community-packages/installed-nodes.repository.ts similarity index 81% rename from packages/@n8n/db/src/repositories/installed-nodes.repository.ts rename to packages/cli/src/modules/community-packages/installed-nodes.repository.ts index 240b237f1a..36f124ba68 100644 --- a/packages/@n8n/db/src/repositories/installed-nodes.repository.ts +++ b/packages/cli/src/modules/community-packages/installed-nodes.repository.ts @@ -1,7 +1,7 @@ import { Service } from '@n8n/di'; import { DataSource, Repository } from '@n8n/typeorm'; -import { InstalledNodes } from '../entities'; +import { InstalledNodes } from './installed-nodes.entity'; @Service() export class InstalledNodesRepository extends Repository { diff --git a/packages/@n8n/db/src/entities/installed-packages.ts b/packages/cli/src/modules/community-packages/installed-packages.entity.ts similarity index 79% rename from packages/@n8n/db/src/entities/installed-packages.ts rename to packages/cli/src/modules/community-packages/installed-packages.entity.ts index 830c3c40e2..17fc0a5bf2 100644 --- a/packages/@n8n/db/src/entities/installed-packages.ts +++ b/packages/cli/src/modules/community-packages/installed-packages.entity.ts @@ -1,7 +1,7 @@ +import { WithTimestamps } from '@n8n/db'; import { Column, Entity, JoinColumn, OneToMany, PrimaryColumn } from '@n8n/typeorm'; -import { WithTimestamps } from './abstract-entity'; -import type { InstalledNodes } from './installed-nodes'; +import type { InstalledNodes } from './installed-nodes.entity'; @Entity() export class InstalledPackages extends WithTimestamps { diff --git a/packages/@n8n/db/src/repositories/installed-packages.repository.ts b/packages/cli/src/modules/community-packages/installed-packages.repository.ts similarity index 95% rename from packages/@n8n/db/src/repositories/installed-packages.repository.ts rename to packages/cli/src/modules/community-packages/installed-packages.repository.ts index f614b0445a..a850015366 100644 --- a/packages/@n8n/db/src/repositories/installed-packages.repository.ts +++ b/packages/cli/src/modules/community-packages/installed-packages.repository.ts @@ -3,7 +3,7 @@ import { DataSource, Repository } from '@n8n/typeorm'; import type { PackageDirectoryLoader } from 'n8n-core'; import { InstalledNodesRepository } from './installed-nodes.repository'; -import { InstalledPackages } from '../entities'; +import { InstalledPackages } from './installed-packages.entity'; @Service() export class InstalledPackagesRepository extends Repository { diff --git a/packages/cli/src/community-packages/npm-utils.ts b/packages/cli/src/modules/community-packages/npm-utils.ts similarity index 100% rename from packages/cli/src/community-packages/npm-utils.ts rename to packages/cli/src/modules/community-packages/npm-utils.ts diff --git a/packages/cli/src/community-packages/strapi-utils.ts b/packages/cli/src/modules/community-packages/strapi-utils.ts similarity index 100% rename from packages/cli/src/community-packages/strapi-utils.ts rename to packages/cli/src/modules/community-packages/strapi-utils.ts diff --git a/packages/cli/src/security-audit/risk-reporters/instance-risk-reporter.ts b/packages/cli/src/security-audit/risk-reporters/instance-risk-reporter.ts index d236f0d16e..b396ec0727 100644 --- a/packages/cli/src/security-audit/risk-reporters/instance-risk-reporter.ts +++ b/packages/cli/src/security-audit/risk-reporters/instance-risk-reporter.ts @@ -7,6 +7,7 @@ import { InstanceSettings } from 'n8n-core'; import type { IWorkflowBase } from 'n8n-workflow'; import { N8N_VERSION } from '@/constants'; +import { CommunityPackagesConfig } from '@/modules/community-packages/community-packages.config'; import { isApiEnabled } from '@/public-api'; import { ENV_VARS_DOCS_URL, @@ -16,7 +17,6 @@ import { } from '@/security-audit/constants'; import type { RiskReporter, Risk, n8n } from '@/security-audit/types'; import { toFlaggedNode } from '@/security-audit/utils'; -import { CommunityPackagesConfig } from '@/community-packages/community-packages.config'; @Service() export class InstanceRiskReporter implements RiskReporter { diff --git a/packages/cli/src/security-audit/security-audit.repository.ts b/packages/cli/src/security-audit/security-audit.repository.ts index f9ba4c31c1..84f6de4f90 100644 --- a/packages/cli/src/security-audit/security-audit.repository.ts +++ b/packages/cli/src/security-audit/security-audit.repository.ts @@ -1,8 +1,9 @@ -import { InstalledPackages } from '@n8n/db'; import { Service } from '@n8n/di'; // eslint-disable-next-line n8n-local-rules/misplaced-n8n-typeorm-import import { DataSource, Repository } from '@n8n/typeorm'; +import { InstalledPackages } from '@/modules/community-packages/installed-packages.entity'; + @Service() export class PackagesRepository extends Repository { constructor(dataSource: DataSource) { diff --git a/packages/cli/src/server.ts b/packages/cli/src/server.ts index d67cef7694..67fe755a41 100644 --- a/packages/cli/src/server.ts +++ b/packages/cli/src/server.ts @@ -65,7 +65,6 @@ import '@/webhooks/webhooks.controller'; import { ChatServer } from './chat/chat-server'; import { MfaService } from './mfa/mfa.service'; -import { CommunityPackagesConfig } from './community-packages/community-packages.config'; @Service() export class Server extends AbstractServer { @@ -119,11 +118,6 @@ export class Server extends AbstractServer { await Container.get(LdapService).init(); } - if (Container.get(CommunityPackagesConfig).enabled) { - await import('@/community-packages/community-packages.controller'); - await import('@/community-packages/community-node-types.controller'); - } - if (inE2ETests) { await import('@/controllers/e2e.controller'); } diff --git a/packages/cli/src/services/__tests__/frontend.service.test.ts b/packages/cli/src/services/__tests__/frontend.service.test.ts index 68e6a96518..0d9478daec 100644 --- a/packages/cli/src/services/__tests__/frontend.service.test.ts +++ b/packages/cli/src/services/__tests__/frontend.service.test.ts @@ -1,5 +1,6 @@ import type { Logger, LicenseState, ModuleRegistry } from '@n8n/backend-common'; import type { GlobalConfig, SecurityConfig } from '@n8n/config'; +import { Container } from '@n8n/di'; import { mock } from 'jest-mock-extended'; import type { InstanceSettings, BinaryDataConfig } from 'n8n-core'; @@ -8,12 +9,11 @@ import type { CredentialsOverwrites } from '@/credentials-overwrites'; import type { License } from '@/license'; import type { LoadNodesAndCredentials } from '@/load-nodes-and-credentials'; import type { MfaService } from '@/mfa/mfa.service'; +import { CommunityPackagesConfig } from '@/modules/community-packages/community-packages.config'; import type { PushConfig } from '@/push/push.config'; import { FrontendService } from '@/services/frontend.service'; import type { UrlService } from '@/services/url.service'; import type { UserManagementMailer } from '@/user-management/email'; -import { CommunityPackagesConfig } from '@/community-packages/community-packages.config'; -import { Container } from '@n8n/di'; describe('FrontendService', () => { let originalEnv: NodeJS.ProcessEnv; diff --git a/packages/cli/src/services/frontend.service.ts b/packages/cli/src/services/frontend.service.ts index 1412d5a570..bffeb691ff 100644 --- a/packages/cli/src/services/frontend.service.ts +++ b/packages/cli/src/services/frontend.service.ts @@ -10,8 +10,6 @@ import { BinaryDataConfig, InstanceSettings } from 'n8n-core'; import type { ICredentialType, INodeTypeBaseDescription } from 'n8n-workflow'; import path from 'path'; -import { CommunityPackagesConfig } from '@/community-packages/community-packages.config'; -import type { CommunityPackagesService } from '@/community-packages/community-packages.service'; import config from '@/config'; import { inE2ETests, N8N_VERSION } from '@/constants'; import { CredentialTypes } from '@/credential-types'; @@ -20,6 +18,8 @@ import { getLdapLoginLabel } from '@/ldap.ee/helpers.ee'; import { License } from '@/license'; import { LoadNodesAndCredentials } from '@/load-nodes-and-credentials'; import { MfaService } from '@/mfa/mfa.service'; +import { CommunityPackagesConfig } from '@/modules/community-packages/community-packages.config'; +import type { CommunityPackagesService } from '@/modules/community-packages/community-packages.service'; import { isApiEnabled } from '@/public-api'; import { PushConfig } from '@/push/push.config'; import { getSamlLoginLabel } from '@/sso.ee/saml/saml-helpers'; @@ -60,8 +60,9 @@ export class FrontendService { this.initSettings(); + // @TODO: Move to community-packages module if (Container.get(CommunityPackagesConfig).enabled) { - void import('@/community-packages/community-packages.service').then( + void import('@/modules/community-packages/community-packages.service').then( ({ CommunityPackagesService }) => { this.communityPackagesService = Container.get(CommunityPackagesService); }, @@ -200,8 +201,11 @@ export class FrontendService { executionMode: config.getEnv('executions.mode'), isMultiMain: this.instanceSettings.isMultiMain, pushBackend: this.pushConfig.backend, + + // @TODO: Move to community-packages module communityNodesEnabled: Container.get(CommunityPackagesConfig).enabled, unverifiedCommunityNodesEnabled: Container.get(CommunityPackagesConfig).unverifiedEnabled, + deployment: { type: this.globalConfig.deployment.type, }, diff --git a/packages/cli/test/integration/commands/worker.cmd.test.ts b/packages/cli/test/integration/commands/worker.cmd.test.ts index 520a1fee3f..6a1c1e3175 100644 --- a/packages/cli/test/integration/commands/worker.cmd.test.ts +++ b/packages/cli/test/integration/commands/worker.cmd.test.ts @@ -12,11 +12,11 @@ import { LogStreamingEventRelay } from '@/events/relays/log-streaming.event-rela import { ExternalHooks } from '@/external-hooks'; import { License } from '@/license'; import { LoadNodesAndCredentials } from '@/load-nodes-and-credentials'; +import { CommunityPackagesService } from '@/modules/community-packages/community-packages.service'; import { Push } from '@/push'; import { Publisher } from '@/scaling/pubsub/publisher.service'; import { Subscriber } from '@/scaling/pubsub/subscriber.service'; import { ScalingService } from '@/scaling/scaling.service'; -import { CommunityPackagesService } from '@/community-packages/community-packages.service'; import { TaskBrokerServer } from '@/task-runners/task-broker/task-broker-server'; import { TaskRunnerProcess } from '@/task-runners/task-runner-process'; import { Telemetry } from '@/telemetry'; diff --git a/packages/cli/test/integration/security-audit/nodes-risk-reporter.test.ts b/packages/cli/test/integration/security-audit/nodes-risk-reporter.test.ts index f56d20cba1..0a16470ae6 100644 --- a/packages/cli/test/integration/security-audit/nodes-risk-reporter.test.ts +++ b/packages/cli/test/integration/security-audit/nodes-risk-reporter.test.ts @@ -5,12 +5,12 @@ import { mock } from 'jest-mock-extended'; import { v4 as uuid } from 'uuid'; import { LoadNodesAndCredentials } from '@/load-nodes-and-credentials'; +import { CommunityPackagesService } from '@/modules/community-packages/community-packages.service'; import { NodeTypes } from '@/node-types'; import { OFFICIAL_RISKY_NODE_TYPES, NODES_REPORT } from '@/security-audit/constants'; import { PackagesRepository } from '@/security-audit/security-audit.repository'; import { SecurityAuditService } from '@/security-audit/security-audit.service'; import { toReportTitle } from '@/security-audit/utils'; -import { CommunityPackagesService } from '@/community-packages/community-packages.service'; import { getRiskSection, MOCK_PACKAGE, saveManualTriggerWorkflow } from './utils'; diff --git a/packages/cli/test/integration/security-audit/utils.ts b/packages/cli/test/integration/security-audit/utils.ts index 8d265dccbe..ac5f9e39c9 100644 --- a/packages/cli/test/integration/security-audit/utils.ts +++ b/packages/cli/test/integration/security-audit/utils.ts @@ -1,11 +1,12 @@ import { GlobalConfig } from '@n8n/config'; -import type { InstalledNodes, InstalledPackages } from '@n8n/db'; import { WorkflowRepository } from '@n8n/db'; import { Container } from '@n8n/di'; import nock from 'nock'; import { v4 as uuid } from 'uuid'; import * as constants from '@/constants'; +import type { InstalledNodes } from '@/modules/community-packages/installed-nodes.entity'; +import type { InstalledPackages } from '@/modules/community-packages/installed-packages.entity'; import type { Risk } from '@/security-audit/types'; import { toReportTitle } from '@/security-audit/utils'; diff --git a/packages/cli/test/integration/shared/types.ts b/packages/cli/test/integration/shared/types.ts index cd1947ce9a..f3e08d85cd 100644 --- a/packages/cli/test/integration/shared/types.ts +++ b/packages/cli/test/integration/shared/types.ts @@ -46,7 +46,7 @@ type EndpointGroup = | 'data-store' | 'module-settings'; -type ModuleName = 'insights' | 'external-secrets' | 'data-store'; +type ModuleName = 'insights' | 'external-secrets' | 'community-packages' | 'data-store'; export interface SetupProps { endpointGroups?: EndpointGroup[]; diff --git a/packages/cli/test/integration/shared/utils/community-nodes.ts b/packages/cli/test/integration/shared/utils/community-nodes.ts index 56b58da129..cbb27e4d12 100644 --- a/packages/cli/test/integration/shared/utils/community-nodes.ts +++ b/packages/cli/test/integration/shared/utils/community-nodes.ts @@ -1,8 +1,10 @@ import { randomName } from '@n8n/backend-test-utils'; -import { InstalledPackages, InstalledNodesRepository, InstalledPackagesRepository } from '@n8n/db'; import { Container } from '@n8n/di'; import { NODE_PACKAGE_PREFIX } from '@/constants'; +import { InstalledNodesRepository } from '@/modules/community-packages/installed-nodes.repository'; +import { InstalledPackages } from '@/modules/community-packages/installed-packages.entity'; +import { InstalledPackagesRepository } from '@/modules/community-packages/installed-packages.repository'; import { COMMUNITY_NODE_VERSION, COMMUNITY_PACKAGE_VERSION } from '../constants'; diff --git a/packages/cli/test/integration/shared/utils/test-server.ts b/packages/cli/test/integration/shared/utils/test-server.ts index f50f9cb665..2e5f09e61f 100644 --- a/packages/cli/test/integration/shared/utils/test-server.ts +++ b/packages/cli/test/integration/shared/utils/test-server.ts @@ -232,7 +232,7 @@ export const setupTestServer = ({ break; case 'community-packages': - await import('@/community-packages/community-packages.controller'); + await import('@/modules/community-packages/community-packages.controller'); break; case 'me':