feat: Make it possible to dynamically load community nodes (#2849)

* ✨ Make it possible to dynamically load node packages

* ⚡ Fix comment

* ✨ Make possible to dynamically install nodes from npm

* Created migration for sqlite regarding community nodes

* Saving to db whenever a package with nodes is installed

* Created endpoint to fetch installed packages

* WIP - uninstall package with nodes

* Fix lint issues

* Updating nodes via API

* Lint and improvement fixes

* Created community node helpers and removed packages taht do not contain nodes

* Check for package updates when fetching installed packages

* Blocked access to non-owner and preventing incorrect install of packages

* Added auto healing process

* Unit tests for helpers

* Finishing tests for helpers

* Improved unit tests, refactored more helpers and created integration tests for GET

* Implemented detection of missing packages on init and added warning to frontend settings

* Add check for banned packages and fix broken tests

* Create migrations for other db systems

* Updated with latest changes from master

* Fixed conflict errors

* Improved unit tests, refactored more helpers and created integration tests for GET

* Implemented detection of missing packages on init and added warning to frontend settings

* 🔥 Removing access check for the Settings sidebar item

* ✨ Added inital community nodes settings screen

* ⚡Added executionMode flag to settings

* ✨ Implemented N8N-callout component

* 💄Updating Callout component template propery names

* 💄 Updating Callout component styling.

* 💄Updating Callout component sizing and colors.

* ✔️ Updating Callout component test snapshots after styling changes

* ✨ Updating the `ActionBox` component so it supports callouts and conditional button rendering

* 💄 Removing duplicate callout theme validation in the `ActionBox` component. Adding a selection control for it in the storybook.

* ✨ Added warning message if instance is in the queue mode. Updated colors based on the new design.

* ⚡ Added a custom permission support to router

* 🔨 Implemented UM detection as a custom permission.

* 👌Updating route permission logic.

* ✨ Implemented installed community packages list in the settings view

* 👌 Updating settings routes rules and community nodes setting view.

* Allow installation of packages that failed to load

* 👌 Updating `ActionBox`, `CommuntyPackageCard` components and settings loading logic.

* 👌 Fixing community nodes loading state and sidebar icon spacing.

* ✨ Implemented loading skeletons for community package cards

* 👌 Handling errrors while loading installed package list. Updating spacing.

* 👌 Updating community nodes error messages.

* Added disable flag

* 🐛 Fixing a community nodes update detection bug when there are missing packages. (#3497)

* ✨ Added front-end support for community nodes feature flag

* ✨ Implemented community package installation modal dialog

* 💄 Community nodes installation modal updates: Moved links to constants and used them in translations, disabling inputs in loading state.

* ✨ Implemented community packages install flow

* Standardize error codes (#3501)

* Standardize error: 400 for request issues such as invalid package name and 500 for installation problems

* Fix http status code for when package is not found

* ✨ Implemented community package installation modal dialog

* 💄 Community nodes installation modal updates: Moved links to constants and used them in translations, disabling inputs in loading state.

* ✨ Implemented community packages install flow

* ✨ Updated error handling based on the response codes

* ✨ Implemented community package installation modal dialog

* ✨ Implemented community package uninstall flow.

* ✨ Finished update confirm modal UI

* 💄 Replaced community nodes tooltip image with the one exported from figma.

* ✨ Implemented community package update process

* ✨ Updating community nodes list after successful package update

* 🔒 Updating public API setting route to use new access rules. Updating express app definition in community nodes tests

* ✨ Implemented community package installation modal dialog

* 💄 Community nodes installation modal updates: Moved links to constants and used them in translations, disabling inputs in loading state.

* ✨ Implemented community packages install flow

* ✨ Updated error handling based on the response codes

* Change output for installation request

* Improve payload for update requests

* 👌 Updating community nodes install modal UI

* 👌 Updating community nodes confirm modal logic

* 👌 Refactoring community nodes confirm modal dialog

* 👌 Separating community nodes components loading states

* 💄 Updating community nodes install modal spacing.

* Fix behavior for installing already installed packages

* 💡 Commenting community nodes install process

* 🔥 Removing leftover commits of deleted Vue mutations

* ✨ Updated node list to identify community nodes and handle node name clash

* ✨ Implemented missing community node dialog.

* 💄 Updating n8n-tabs component to support tooltips

* ✨ Updating node details with community node details.

* 🔨 Using back-end response when updating community packages

* 👌 Updating tabs component and refactoring community nodes store mutations

* 👌 Adding community node flag to node type descriptions and using it to identify community nodes

* 👌 Hiding unnecessary elements from missing node details panel.

* 👌 Updating missing node type descriptions for custom and community nodes

* 👌 Updating community node package name detection logic

* 👌 Removing communityNode flag from node description

* ✨ Adding `force` flag to credentials fetching (#3527)

* ✨ Adding `force` flag to credentials fetching which can be used to skip check when loading credentials

* ✨ Forcing credentials loading when opening nodeView

* 👌 Minor updates to community nodes details panel

* tests for post endpoint

* duplicate comments

* Add Patch and Delete enpoints tests

* 🔒 Using `pageCategory`prop to assemble the list of settings routes instead of hard-coded array (#3562)

* 📈 Added front-end telemetry events for community nodes

* 📈 Updating community nodes telemetry events

* 💄 Updating community nodes settings UI elements based on product/design review

* 💄 Updating node view & node details view for community nodes based on product/design feedback

* 💄 Fixing community node text capitalisation

* ✨ Adding community node install error message under the package name input field

* Fixed and improved tests

* Fix lint issue

* feat: Migrated to npm release of riot-tmpl fork.

* 📈 Updating community nodes telemetry events based on the product review

* 💄 Updating community nodes UI based on the design feedback

* 🔀 Merging recent node draggable panels changes

* Implement self healing process

* Improve error messages for package name requirement and disk space

* 💄 Removing front-end error message override since appropriate response is available from the back-end

* Fix lint issues

* Fix installed node name

* 💄 Removed additional node name parsing

* 📈 Updating community nodes telemetry events

* Fix postgres migration for cascading nodes when package is removed

* Remove postman mock for banned packages

* 📈 Adding missing telemetry event for community node documentation click

* 🐛 Fixing community nodes UI bugs reported during the bug bash

* Fix issue with uninstalling packages not reflecting UI

* 🐛 Fixing a missing node type bug when trying to run a workflow.

* Improve error detection for installing packages

* 💄 Updating community nodes components styling and wording based on the product feedback

* Implement telemetry be events

* Add author name and email to packages

* Fix telemetry be events for community packages

* 📈 Updating front-end telemetry events with community nodes author data

* 💄 Updating credentials documentation link logic to handle community nodes credentials

* 🐛 Fixing draggable panels logic

* Fix duplicate wrong import

* 💄 Hiding community nodes credentials documentation links when they don't contain an absolute URL

* Fix issue with detection of missing packages

* 💄 Adding the `Docs` tab to community nodes

* 💄 Adding a failed loading indicator to community nodes list

* Prevent n8n from crashing on startup

* Refactor and improve code quality

* ⚡ Remove not needed depenedency

Co-authored-by: Omar Ajoue <krynble@gmail.com>
Co-authored-by: Milorad Filipović <milorad@n8n.io>
Co-authored-by: Milorad FIlipović <miloradfilipovic19@gmail.com>
Co-authored-by: agobrech <ael.gobrecht@gmail.com>
Co-authored-by: Alex Grozav <alex@grozav.com>

packages/cli/test/integration/shared/constants.ts

@@ -71,6 +71,17 @@ export const BOOTSTRAP_POSTGRES_CONNECTION_NAME: Readonly<string> = 'n8n_bs_post
  */
 export const BOOTSTRAP_MYSQL_CONNECTION_NAME: Readonly<string> = 'n8n_bs_mysql';
 
+/**
+ * Timeout (in milliseconds) to account for fake SMTP service being slow to respond.
+ */
+export const SMTP_TEST_TIMEOUT = 30_000;
+
+/**
+ * Nodes
+ */
+export const CURRENT_PACKAGE_VERSION = '0.1.0';
+export const UPDATED_PACKAGE_VERSION = '0.2.0';
+
 /**
  * Timeout (in milliseconds) to account for DB being slow to initialize.
  */

packages/cli/test/integration/shared/testDb.ts

@@ -24,8 +24,10 @@ import { categorize, getPostgresSchemaSection } from './utils';
 import { createCredentiasFromCredentialsEntity } from '../../../src/CredentialsHelper';
 
 import type { Role } from '../../../src/databases/entities/Role';
+import type { CollectionName, CredentialPayload, InstalledNodePayload, InstalledPackagePayload, MappingName } from './types';
+import { InstalledPackages } from '../../../src/databases/entities/InstalledPackages';
+import { InstalledNodes } from '../../../src/databases/entities/InstalledNodes';
 import { User } from '../../../src/databases/entities/User';
-import type { CollectionName, CredentialPayload, MappingName } from './types';
 import { WorkflowEntity } from '../../../src/databases/entities/WorkflowEntity';
 import { ExecutionEntity } from '../../../src/databases/entities/ExecutionEntity';
 import { TagEntity } from '../../../src/databases/entities/TagEntity';
@@ -258,6 +260,8 @@ function toTableName(sourceName: CollectionName | MappingName) {
 		SharedCredentials: 'shared_credentials',
 		SharedWorkflow: 'shared_workflow',
 		Settings: 'settings',
+		InstalledPackages: 'installed_packages',
+		InstalledNodes: 'installed_nodes',
 	}[sourceName];
 }
 
@@ -338,6 +342,29 @@ export function createUserShell(globalRole: Role): Promise<User> {
 	return Db.collections.User.save(shell);
 }
 
+// --------------------------------------
+// Installed nodes and packages creation
+// --------------------------------------
+
+export async function saveInstalledPackage(installedPackagePayload: InstalledPackagePayload): Promise<InstalledPackages> {
+	const newInstalledPackage = new InstalledPackages();
+
+	Object.assign(newInstalledPackage, installedPackagePayload);
+
+
+	const savedInstalledPackage = await Db.collections.InstalledPackages.save(newInstalledPackage);
+	return savedInstalledPackage;
+}
+
+export async function saveInstalledNode(installedNodePayload: InstalledNodePayload): Promise<InstalledNodes> {
+	const newInstalledNode = new InstalledNodes();
+
+	Object.assign(newInstalledNode, installedNodePayload);
+
+	const savedInstalledNode = await Db.collections.InstalledNodes.save(newInstalledNode);
+	return savedInstalledNode;
+}
+
 export function addApiKey(user: User): Promise<User> {
 	user.apiKey = randomApiKey();
 	return Db.collections.User.save(user);

packages/cli/test/integration/shared/types.d.ts

@@ -17,7 +17,8 @@ type EndpointGroup =
 	| 'owner'
 	| 'passwordReset'
 	| 'credentials'
-	| 'publicApi';
+	| 'publicApi'
+	| 'nodes';
 
 export type CredentialPayload = {
 	name: string;
@@ -43,3 +44,16 @@ export interface TriggerTime {
 	weekeday: number;
 	[key: string]: string | number;
 }
+
+export type InstalledPackagePayload = {
+	packageName: string;
+	installedVersion: string;
+}
+
+export type InstalledNodePayload = {
+	name: string;
+	type: string;
+	latestVersion: string;
+	package: string;
+}
+

packages/cli/test/integration/shared/utils.ts

@@ -23,8 +23,8 @@ import {
 } from 'n8n-workflow';
 
 import config from '../../../config';
-import { AUTHLESS_ENDPOINTS, PUBLIC_API_REST_PATH_SEGMENT, REST_PATH_SEGMENT } from './constants';
-import { AUTH_COOKIE_NAME } from '../../../src/constants';
+import { AUTHLESS_ENDPOINTS, CURRENT_PACKAGE_VERSION, PUBLIC_API_REST_PATH_SEGMENT, REST_PATH_SEGMENT } from './constants';
+import { AUTH_COOKIE_NAME, NODE_PACKAGE_PREFIX } from '../../../src/constants';
 import { addRoutes as authMiddleware } from '../../../src/UserManagement/routes';
 import {
 	ActiveWorkflowRunner,
@@ -44,8 +44,17 @@ import { getLogger } from '../../../src/Logger';
 import { credentialsController } from '../../../src/api/credentials.api';
 import { loadPublicApiVersions } from '../../../src/PublicApi/';
 import type { User } from '../../../src/databases/entities/User';
-import type { ApiPath, EndpointGroup, PostgresSchemaSection, TriggerTime } from './types';
+import type {
+	ApiPath,
+	EndpointGroup,
+	InstalledNodePayload,
+	InstalledPackagePayload,
+	PostgresSchemaSection,
+	TriggerTime,
+} from './types';
 import type { N8nApp } from '../../../src/UserManagement/Interfaces';
+import { nodesController } from '../../../src/api/nodes.api';
+import { randomName } from './random';
 
 /**
  * Initialize a test server.
@@ -89,6 +98,7 @@ export async function initTestServer({
 		const { apiRouters } = await loadPublicApiVersions(testServer.publicApiEndpoint);
 		const map: Record<string, express.Router | express.Router[]> = {
 			credentials: credentialsController,
+			nodes: nodesController,
 			publicApi: apiRouters,
 		};
 
@@ -136,9 +146,7 @@ const classifyEndpointGroups = (endpointGroups: string[]) => {
 	const functionEndpoints: string[] = [];
 
 	endpointGroups.forEach((group) =>
-		(group === 'credentials' || group === 'publicApi' ? routerEndpoints : functionEndpoints).push(
-			group,
-		),
+		(['credentials', 'nodes', 'publicApi'].includes(group) ? routerEndpoints : functionEndpoints).push(group),
 	);
 
 	return [routerEndpoints, functionEndpoints];
@@ -879,3 +887,24 @@ export function getPostgresSchemaSection(
 
 	return null;
 }
+
+// ----------------------------------
+//              nodes
+// ----------------------------------
+
+export function installedPackagePayload(): InstalledPackagePayload {
+	return {
+		packageName: NODE_PACKAGE_PREFIX + randomName(),
+		installedVersion: CURRENT_PACKAGE_VERSION,
+	};
+}
+
+export function installedNodePayload(packageName: string): InstalledNodePayload {
+	const nodeName = randomName();
+	return {
+		name: nodeName,
+		type: nodeName,
+		latestVersion: CURRENT_PACKAGE_VERSION,
+		package: packageName,
+	};
+}