mirror of
https://github.com/Abdulazizzn/n8n-enterprise-unlocked.git
synced 2025-12-20 11:22:15 +00:00
feat: Make it possible to dynamically load community nodes (#2849)
* ✨ Make it possible to dynamically load node packages * ⚡ Fix comment * ✨ Make possible to dynamically install nodes from npm * Created migration for sqlite regarding community nodes * Saving to db whenever a package with nodes is installed * Created endpoint to fetch installed packages * WIP - uninstall package with nodes * Fix lint issues * Updating nodes via API * Lint and improvement fixes * Created community node helpers and removed packages taht do not contain nodes * Check for package updates when fetching installed packages * Blocked access to non-owner and preventing incorrect install of packages * Added auto healing process * Unit tests for helpers * Finishing tests for helpers * Improved unit tests, refactored more helpers and created integration tests for GET * Implemented detection of missing packages on init and added warning to frontend settings * Add check for banned packages and fix broken tests * Create migrations for other db systems * Updated with latest changes from master * Fixed conflict errors * Improved unit tests, refactored more helpers and created integration tests for GET * Implemented detection of missing packages on init and added warning to frontend settings * 🔥 Removing access check for the Settings sidebar item * ✨ Added inital community nodes settings screen * ⚡Added executionMode flag to settings * ✨ Implemented N8N-callout component * 💄Updating Callout component template propery names * 💄 Updating Callout component styling. * 💄Updating Callout component sizing and colors. * ✔️ Updating Callout component test snapshots after styling changes * ✨ Updating the `ActionBox` component so it supports callouts and conditional button rendering * 💄 Removing duplicate callout theme validation in the `ActionBox` component. Adding a selection control for it in the storybook. * ✨ Added warning message if instance is in the queue mode. Updated colors based on the new design. * ⚡ Added a custom permission support to router * 🔨 Implemented UM detection as a custom permission. * 👌Updating route permission logic. * ✨ Implemented installed community packages list in the settings view * 👌 Updating settings routes rules and community nodes setting view. * Allow installation of packages that failed to load * 👌 Updating `ActionBox`, `CommuntyPackageCard` components and settings loading logic. * 👌 Fixing community nodes loading state and sidebar icon spacing. * ✨ Implemented loading skeletons for community package cards * 👌 Handling errrors while loading installed package list. Updating spacing. * 👌 Updating community nodes error messages. * Added disable flag * 🐛 Fixing a community nodes update detection bug when there are missing packages. (#3497) * ✨ Added front-end support for community nodes feature flag * ✨ Implemented community package installation modal dialog * 💄 Community nodes installation modal updates: Moved links to constants and used them in translations, disabling inputs in loading state. * ✨ Implemented community packages install flow * Standardize error codes (#3501) * Standardize error: 400 for request issues such as invalid package name and 500 for installation problems * Fix http status code for when package is not found * ✨ Implemented community package installation modal dialog * 💄 Community nodes installation modal updates: Moved links to constants and used them in translations, disabling inputs in loading state. * ✨ Implemented community packages install flow * ✨ Updated error handling based on the response codes * ✨ Implemented community package installation modal dialog * ✨ Implemented community package uninstall flow. * ✨ Finished update confirm modal UI * 💄 Replaced community nodes tooltip image with the one exported from figma. * ✨ Implemented community package update process * ✨ Updating community nodes list after successful package update * 🔒 Updating public API setting route to use new access rules. Updating express app definition in community nodes tests * ✨ Implemented community package installation modal dialog * 💄 Community nodes installation modal updates: Moved links to constants and used them in translations, disabling inputs in loading state. * ✨ Implemented community packages install flow * ✨ Updated error handling based on the response codes * Change output for installation request * Improve payload for update requests * 👌 Updating community nodes install modal UI * 👌 Updating community nodes confirm modal logic * 👌 Refactoring community nodes confirm modal dialog * 👌 Separating community nodes components loading states * 💄 Updating community nodes install modal spacing. * Fix behavior for installing already installed packages * 💡 Commenting community nodes install process * 🔥 Removing leftover commits of deleted Vue mutations * ✨ Updated node list to identify community nodes and handle node name clash * ✨ Implemented missing community node dialog. * 💄 Updating n8n-tabs component to support tooltips * ✨ Updating node details with community node details. * 🔨 Using back-end response when updating community packages * 👌 Updating tabs component and refactoring community nodes store mutations * 👌 Adding community node flag to node type descriptions and using it to identify community nodes * 👌 Hiding unnecessary elements from missing node details panel. * 👌 Updating missing node type descriptions for custom and community nodes * 👌 Updating community node package name detection logic * 👌 Removing communityNode flag from node description * ✨ Adding `force` flag to credentials fetching (#3527) * ✨ Adding `force` flag to credentials fetching which can be used to skip check when loading credentials * ✨ Forcing credentials loading when opening nodeView * 👌 Minor updates to community nodes details panel * tests for post endpoint * duplicate comments * Add Patch and Delete enpoints tests * 🔒 Using `pageCategory`prop to assemble the list of settings routes instead of hard-coded array (#3562) * 📈 Added front-end telemetry events for community nodes * 📈 Updating community nodes telemetry events * 💄 Updating community nodes settings UI elements based on product/design review * 💄 Updating node view & node details view for community nodes based on product/design feedback * 💄 Fixing community node text capitalisation * ✨ Adding community node install error message under the package name input field * Fixed and improved tests * Fix lint issue * feat: Migrated to npm release of riot-tmpl fork. * 📈 Updating community nodes telemetry events based on the product review * 💄 Updating community nodes UI based on the design feedback * 🔀 Merging recent node draggable panels changes * Implement self healing process * Improve error messages for package name requirement and disk space * 💄 Removing front-end error message override since appropriate response is available from the back-end * Fix lint issues * Fix installed node name * 💄 Removed additional node name parsing * 📈 Updating community nodes telemetry events * Fix postgres migration for cascading nodes when package is removed * Remove postman mock for banned packages * 📈 Adding missing telemetry event for community node documentation click * 🐛 Fixing community nodes UI bugs reported during the bug bash * Fix issue with uninstalling packages not reflecting UI * 🐛 Fixing a missing node type bug when trying to run a workflow. * Improve error detection for installing packages * 💄 Updating community nodes components styling and wording based on the product feedback * Implement telemetry be events * Add author name and email to packages * Fix telemetry be events for community packages * 📈 Updating front-end telemetry events with community nodes author data * 💄 Updating credentials documentation link logic to handle community nodes credentials * 🐛 Fixing draggable panels logic * Fix duplicate wrong import * 💄 Hiding community nodes credentials documentation links when they don't contain an absolute URL * Fix issue with detection of missing packages * 💄 Adding the `Docs` tab to community nodes * 💄 Adding a failed loading indicator to community nodes list * Prevent n8n from crashing on startup * Refactor and improve code quality * ⚡ Remove not needed depenedency Co-authored-by: Omar Ajoue <krynble@gmail.com> Co-authored-by: Milorad Filipović <milorad@n8n.io> Co-authored-by: Milorad FIlipović <miloradfilipovic19@gmail.com> Co-authored-by: agobrech <ael.gobrecht@gmail.com> Co-authored-by: Alex Grozav <alex@grozav.com>
This commit is contained in:
Jan Oberhauser
218
packages/cli/src/CommunityNodes/helpers.ts
Normal file
218
packages/cli/src/CommunityNodes/helpers.ts
Normal file
@@ -0,0 +1,218 @@
|
||||
/* eslint-disable import/no-cycle */
|
||||
/* eslint-disable @typescript-eslint/naming-convention */
|
||||
import { promisify } from 'util';
|
||||
import { exec } from 'child_process';
|
||||
import { access as fsAccess, mkdir as fsMkdir } from 'fs/promises';
|
||||
|
||||
import { UserSettings } from 'n8n-core';
|
||||
import { LoggerProxy, PublicInstalledPackage } from 'n8n-workflow';
|
||||
import axios from 'axios';
|
||||
import {
|
||||
NODE_PACKAGE_PREFIX,
|
||||
NPM_COMMAND_TOKENS,
|
||||
NPM_PACKAGE_STATUS_GOOD,
|
||||
RESPONSE_ERROR_MESSAGES,
|
||||
} from '../constants';
|
||||
import { NpmPackageStatusCheck, NpmUpdatesAvailable, ParsedNpmPackageName } from '../Interfaces';
|
||||
import { InstalledPackages } from '../databases/entities/InstalledPackages';
|
||||
import config from '../../config';
|
||||
|
||||
const execAsync = promisify(exec);
|
||||
|
||||
export const parsePackageName = (originalString: string | undefined): ParsedNpmPackageName => {
|
||||
if (!originalString) {
|
||||
throw new Error('Package name was not provided');
|
||||
}
|
||||
|
||||
if (new RegExp(/[^0-9a-z@\-./]/).test(originalString)) {
|
||||
// Prevent any strings that are not valid npm package names or
|
||||
// could indicate malicous commands
|
||||
throw new Error('Package name must be a single word');
|
||||
}
|
||||
|
||||
const scope = originalString.includes('/') ? originalString.split('/')[0] : undefined;
|
||||
|
||||
const packageNameWithoutScope = scope ? originalString.replace(`${scope}/`, '') : originalString;
|
||||
|
||||
if (!packageNameWithoutScope.startsWith(NODE_PACKAGE_PREFIX)) {
|
||||
throw new Error('Package name must start with n8n-nodes-');
|
||||
}
|
||||
|
||||
const version = packageNameWithoutScope.includes('@')
|
||||
? packageNameWithoutScope.split('@')[1]
|
||||
: undefined;
|
||||
|
||||
const packageName = version ? originalString.replace(`@${version}`, '') : originalString;
|
||||
|
||||
return {
|
||||
packageName,
|
||||
scope,
|
||||
version,
|
||||
originalString,
|
||||
};
|
||||
};
|
||||
|
||||
export const executeCommand = async (
|
||||
command: string,
|
||||
options?: {
|
||||
doNotHandleError?: boolean;
|
||||
},
|
||||
): Promise<string> => {
|
||||
const downloadFolder = UserSettings.getUserN8nFolderDowloadedNodesPath();
|
||||
// Make sure the node-download folder exists
|
||||
try {
|
||||
await fsAccess(downloadFolder);
|
||||
// eslint-disable-next-line no-empty
|
||||
} catch (error) {
|
||||
await fsMkdir(downloadFolder);
|
||||
}
|
||||
const execOptions = {
|
||||
cwd: downloadFolder,
|
||||
env: {
|
||||
NODE_PATH: process.env.NODE_PATH,
|
||||
PATH: process.env.PATH,
|
||||
},
|
||||
};
|
||||
|
||||
try {
|
||||
const commandResult = await execAsync(command, execOptions);
|
||||
return commandResult.stdout;
|
||||
} catch (error) {
|
||||
if (options?.doNotHandleError) {
|
||||
throw error;
|
||||
}
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
|
||||
const errorMessage = error.message as string;
|
||||
|
||||
if (
|
||||
errorMessage.includes(NPM_COMMAND_TOKENS.NPM_PACKAGE_NOT_FOUND_ERROR) ||
|
||||
errorMessage.includes(NPM_COMMAND_TOKENS.NPM_NO_VERSION_AVAILABLE)
|
||||
) {
|
||||
throw new Error(RESPONSE_ERROR_MESSAGES.PACKAGE_NOT_FOUND);
|
||||
}
|
||||
if (errorMessage.includes(NPM_COMMAND_TOKENS.NPM_PACKAGE_VERSION_NOT_FOUND_ERROR)) {
|
||||
throw new Error(RESPONSE_ERROR_MESSAGES.PACKAGE_VERSION_NOT_FOUND);
|
||||
}
|
||||
if (
|
||||
errorMessage.includes(NPM_COMMAND_TOKENS.NPM_DISK_NO_SPACE) ||
|
||||
errorMessage.includes(NPM_COMMAND_TOKENS.NPM_DISK_INSUFFICIENT_SPACE)
|
||||
) {
|
||||
throw new Error(RESPONSE_ERROR_MESSAGES.DISK_IS_FULL);
|
||||
}
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
|
||||
LoggerProxy.warn('npm command failed; see message', { errorMessage });
|
||||
|
||||
throw new Error('Package could not be installed - check logs for details');
|
||||
}
|
||||
};
|
||||
|
||||
export function matchPackagesWithUpdates(
|
||||
installedPackages: InstalledPackages[],
|
||||
availableUpdates?: NpmUpdatesAvailable,
|
||||
): PublicInstalledPackage[] {
|
||||
if (!availableUpdates) {
|
||||
return installedPackages;
|
||||
}
|
||||
const hydratedPackageList = [] as PublicInstalledPackage[];
|
||||
|
||||
for (let i = 0; i < installedPackages.length; i++) {
|
||||
const installedPackage = installedPackages[i];
|
||||
const publicPackage = { ...installedPackage } as PublicInstalledPackage;
|
||||
|
||||
if (availableUpdates[installedPackage.packageName]) {
|
||||
publicPackage.updateAvailable = availableUpdates[installedPackage.packageName].latest;
|
||||
}
|
||||
hydratedPackageList.push(publicPackage);
|
||||
}
|
||||
|
||||
return hydratedPackageList;
|
||||
}
|
||||
|
||||
export function matchMissingPackages(
|
||||
installedPackages: PublicInstalledPackage[],
|
||||
missingPackages: string,
|
||||
): PublicInstalledPackage[] {
|
||||
const missingPackageNames = missingPackages.split(' ');
|
||||
|
||||
const missingPackagesList = missingPackageNames.map((missingPackageName: string) => {
|
||||
// Strip away versions but maintain scope and package name
|
||||
try {
|
||||
const parsedPackageData = parsePackageName(missingPackageName);
|
||||
return parsedPackageData.packageName;
|
||||
|
||||
// eslint-disable-next-line no-empty
|
||||
} catch (_) {}
|
||||
return undefined;
|
||||
});
|
||||
|
||||
const hydratedPackageList = [] as PublicInstalledPackage[];
|
||||
installedPackages.forEach((installedPackage) => {
|
||||
const hydratedInstalledPackage = { ...installedPackage };
|
||||
if (missingPackagesList.includes(hydratedInstalledPackage.packageName)) {
|
||||
hydratedInstalledPackage.failedLoading = true;
|
||||
}
|
||||
hydratedPackageList.push(hydratedInstalledPackage);
|
||||
});
|
||||
|
||||
return hydratedPackageList;
|
||||
}
|
||||
|
||||
export async function checkPackageStatus(packageName: string): Promise<NpmPackageStatusCheck> {
|
||||
// You can change this URL for testing - the default testing url below
|
||||
// is a postman mock service
|
||||
const n8nBackendServiceUrl = 'https://api.n8n.io/api/package';
|
||||
|
||||
try {
|
||||
const output = await axios.post(
|
||||
n8nBackendServiceUrl,
|
||||
{ name: packageName },
|
||||
{
|
||||
method: 'POST',
|
||||
},
|
||||
);
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
|
||||
if (output.data.status !== NPM_PACKAGE_STATUS_GOOD) {
|
||||
return output.data as NpmPackageStatusCheck;
|
||||
}
|
||||
} catch (error) {
|
||||
// Do nothing if service is unreachable
|
||||
}
|
||||
return { status: NPM_PACKAGE_STATUS_GOOD };
|
||||
}
|
||||
|
||||
export function hasPackageLoadedSuccessfully(packageName: string): boolean {
|
||||
try {
|
||||
const failedPackages = (config.get('nodes.packagesMissing') as string).split(' ');
|
||||
|
||||
const packageFailedToLoad = failedPackages.find(
|
||||
(packageNameAndVersion) =>
|
||||
packageNameAndVersion.startsWith(packageName) &&
|
||||
packageNameAndVersion.replace(packageName, '').startsWith('@'),
|
||||
);
|
||||
if (packageFailedToLoad) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
} catch (_error) {
|
||||
// If key doesn't exist it means all packages loaded fine
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
export function removePackageFromMissingList(packageName: string): void {
|
||||
try {
|
||||
const failedPackages = (config.get('nodes.packagesMissing') as string).split(' ');
|
||||
|
||||
const packageFailedToLoad = failedPackages.filter(
|
||||
(packageNameAndVersion) =>
|
||||
!packageNameAndVersion.startsWith(packageName) ||
|
||||
!packageNameAndVersion.replace(packageName, '').startsWith('@'),
|
||||
);
|
||||
|
||||
config.set('nodes.packagesMissing', packageFailedToLoad.join(' '));
|
||||
} catch (_error) {
|
||||
// Do nothing
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user