mirror of
https://github.com/Abdulazizzn/n8n-enterprise-unlocked.git
synced 2025-12-17 18:12:04 +00:00
refactor(core): Refactor nodes loading (no-changelog) (#7283)
fixes PAY-605
This commit is contained in:
कारतोफ्फेलस्क्रिप्ट™
committed by
GitHub
GitHub
parent
789e1e7ed4
commit
c5ee06cc61
@@ -1,5 +1,8 @@
|
||||
import uniq from 'lodash/uniq';
|
||||
import glob from 'fast-glob';
|
||||
import { Container, Service } from 'typedi';
|
||||
import path from 'path';
|
||||
import fsPromises from 'fs/promises';
|
||||
|
||||
import type { DirectoryLoader, Types } from 'n8n-core';
|
||||
import {
|
||||
CUSTOM_EXTENSION_ENV,
|
||||
@@ -9,36 +12,30 @@ import {
|
||||
LazyPackageDirectoryLoader,
|
||||
} from 'n8n-core';
|
||||
import type {
|
||||
ICredentialTypes,
|
||||
ILogger,
|
||||
INodesAndCredentials,
|
||||
KnownNodesAndCredentials,
|
||||
INodeTypeDescription,
|
||||
LoadedNodesAndCredentials,
|
||||
INodeTypeData,
|
||||
ICredentialTypeData,
|
||||
} from 'n8n-workflow';
|
||||
import { LoggerProxy, ErrorReporterProxy as ErrorReporter } from 'n8n-workflow';
|
||||
|
||||
import { createWriteStream } from 'fs';
|
||||
import { mkdir } from 'fs/promises';
|
||||
import path from 'path';
|
||||
import config from '@/config';
|
||||
import type { InstalledPackages } from '@db/entities/InstalledPackages';
|
||||
import { CommunityPackageService } from './services/communityPackage.service';
|
||||
import {
|
||||
GENERATED_STATIC_DIR,
|
||||
RESPONSE_ERROR_MESSAGES,
|
||||
CUSTOM_API_CALL_KEY,
|
||||
CUSTOM_API_CALL_NAME,
|
||||
inTest,
|
||||
CLI_DIR,
|
||||
inE2ETests,
|
||||
} from '@/constants';
|
||||
import { CredentialsOverwrites } from '@/CredentialsOverwrites';
|
||||
import Container, { Service } from 'typedi';
|
||||
|
||||
interface LoadedNodesAndCredentials {
|
||||
nodes: INodeTypeData;
|
||||
credentials: ICredentialTypeData;
|
||||
}
|
||||
|
||||
@Service()
|
||||
export class LoadNodesAndCredentials implements INodesAndCredentials {
|
||||
known: KnownNodesAndCredentials = { nodes: {}, credentials: {} };
|
||||
export class LoadNodesAndCredentials {
|
||||
private known: KnownNodesAndCredentials = { nodes: {}, credentials: {} };
|
||||
|
||||
loaded: LoadedNodesAndCredentials = { nodes: {}, credentials: {} };
|
||||
|
||||
@@ -50,20 +47,20 @@ export class LoadNodesAndCredentials implements INodesAndCredentials {
|
||||
|
||||
includeNodes = config.getEnv('nodes.include');
|
||||
|
||||
credentialTypes: ICredentialTypes;
|
||||
|
||||
logger: ILogger;
|
||||
|
||||
private downloadFolder: string;
|
||||
|
||||
private postProcessors: Array<() => Promise<void>> = [];
|
||||
|
||||
async init() {
|
||||
if (inTest) throw new Error('Not available in tests');
|
||||
|
||||
// Make sure the imported modules can resolve dependencies fine.
|
||||
const delimiter = process.platform === 'win32' ? ';' : ':';
|
||||
process.env.NODE_PATH = module.paths.join(delimiter);
|
||||
|
||||
// @ts-ignore
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-call
|
||||
if (!inTest) module.constructor._initPaths();
|
||||
module.constructor._initPaths();
|
||||
|
||||
if (!inE2ETests) {
|
||||
this.excludeNodes = this.excludeNodes ?? [];
|
||||
@@ -91,48 +88,30 @@ export class LoadNodesAndCredentials implements INodesAndCredentials {
|
||||
|
||||
await this.loadNodesFromCustomDirectories();
|
||||
await this.postProcessLoaders();
|
||||
this.injectCustomApiCallOptions();
|
||||
}
|
||||
|
||||
async generateTypesForFrontend() {
|
||||
const credentialsOverwrites = CredentialsOverwrites().getAll();
|
||||
for (const credential of this.types.credentials) {
|
||||
const overwrittenProperties = [];
|
||||
this.credentialTypes
|
||||
.getParentTypes(credential.name)
|
||||
.reverse()
|
||||
.map((name) => credentialsOverwrites[name])
|
||||
.forEach((overwrite) => {
|
||||
if (overwrite) overwrittenProperties.push(...Object.keys(overwrite));
|
||||
});
|
||||
addPostProcessor(fn: () => Promise<void>) {
|
||||
this.postProcessors.push(fn);
|
||||
}
|
||||
|
||||
if (credential.name in credentialsOverwrites) {
|
||||
overwrittenProperties.push(...Object.keys(credentialsOverwrites[credential.name]));
|
||||
}
|
||||
isKnownNode(type: string) {
|
||||
return type in this.known.nodes;
|
||||
}
|
||||
|
||||
if (overwrittenProperties.length) {
|
||||
credential.__overwrittenProperties = uniq(overwrittenProperties);
|
||||
}
|
||||
}
|
||||
get loadedCredentials() {
|
||||
return this.loaded.credentials;
|
||||
}
|
||||
|
||||
// pre-render all the node and credential types as static json files
|
||||
await mkdir(path.join(GENERATED_STATIC_DIR, 'types'), { recursive: true });
|
||||
get loadedNodes() {
|
||||
return this.loaded.nodes;
|
||||
}
|
||||
|
||||
const writeStaticJSON = async (name: string, data: object[]) => {
|
||||
const filePath = path.join(GENERATED_STATIC_DIR, `types/${name}.json`);
|
||||
const stream = createWriteStream(filePath, 'utf-8');
|
||||
stream.write('[\n');
|
||||
data.forEach((entry, index) => {
|
||||
stream.write(JSON.stringify(entry));
|
||||
if (index !== data.length - 1) stream.write(',');
|
||||
stream.write('\n');
|
||||
});
|
||||
stream.write(']\n');
|
||||
stream.end();
|
||||
};
|
||||
get knownCredentials() {
|
||||
return this.known.credentials;
|
||||
}
|
||||
|
||||
await writeStaticJSON('nodes', this.types.nodes);
|
||||
await writeStaticJSON('credentials', this.types.credentials);
|
||||
get knownNodes() {
|
||||
return this.known.nodes;
|
||||
}
|
||||
|
||||
private async loadNodesFromNodeModules(
|
||||
@@ -163,6 +142,18 @@ export class LoadNodesAndCredentials implements INodesAndCredentials {
|
||||
}
|
||||
}
|
||||
|
||||
resolveIcon(packageName: string, url: string): string | undefined {
|
||||
const loader = this.loaders[packageName];
|
||||
if (loader) {
|
||||
const pathPrefix = `/icons/${packageName}/`;
|
||||
const filePath = path.resolve(loader.directory, url.substring(pathPrefix.length));
|
||||
if (!path.relative(loader.directory, filePath).includes('..')) {
|
||||
return filePath;
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
getCustomDirectories(): string[] {
|
||||
const customDirectories = [UserSettings.getUserN8nFolderCustomExtensionPath()];
|
||||
|
||||
@@ -180,93 +171,16 @@ export class LoadNodesAndCredentials implements INodesAndCredentials {
|
||||
}
|
||||
}
|
||||
|
||||
private async installOrUpdateNpmModule(
|
||||
packageName: string,
|
||||
options: { version?: string } | { installedPackage: InstalledPackages },
|
||||
) {
|
||||
const isUpdate = 'installedPackage' in options;
|
||||
const command = isUpdate
|
||||
? `npm update ${packageName}`
|
||||
: `npm install ${packageName}${options.version ? `@${options.version}` : ''}`;
|
||||
|
||||
const communityPackageService = Container.get(CommunityPackageService);
|
||||
|
||||
try {
|
||||
await communityPackageService.executeNpmCommand(command);
|
||||
} catch (error) {
|
||||
if (error instanceof Error && error.message === RESPONSE_ERROR_MESSAGES.PACKAGE_NOT_FOUND) {
|
||||
throw new Error(`The npm package "${packageName}" could not be found.`);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
|
||||
async loadPackage(packageName: string) {
|
||||
const finalNodeUnpackedPath = path.join(this.downloadFolder, 'node_modules', packageName);
|
||||
|
||||
let loader: PackageDirectoryLoader;
|
||||
try {
|
||||
loader = await this.runDirectoryLoader(PackageDirectoryLoader, finalNodeUnpackedPath);
|
||||
} catch (error) {
|
||||
// Remove this package since loading it failed
|
||||
const removeCommand = `npm remove ${packageName}`;
|
||||
try {
|
||||
await communityPackageService.executeNpmCommand(removeCommand);
|
||||
} catch {}
|
||||
throw new Error(RESPONSE_ERROR_MESSAGES.PACKAGE_LOADING_FAILED, { cause: error });
|
||||
}
|
||||
|
||||
if (loader.loadedNodes.length > 0) {
|
||||
// Save info to DB
|
||||
try {
|
||||
if (isUpdate) {
|
||||
await communityPackageService.removePackageFromDatabase(options.installedPackage);
|
||||
}
|
||||
const installedPackage = await communityPackageService.persistInstalledPackage(loader);
|
||||
await this.postProcessLoaders();
|
||||
await this.generateTypesForFrontend();
|
||||
return installedPackage;
|
||||
} catch (error) {
|
||||
LoggerProxy.error('Failed to save installed packages and nodes', {
|
||||
error: error as Error,
|
||||
packageName,
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
} else {
|
||||
// Remove this package since it contains no loadable nodes
|
||||
const removeCommand = `npm remove ${packageName}`;
|
||||
try {
|
||||
await communityPackageService.executeNpmCommand(removeCommand);
|
||||
} catch {}
|
||||
|
||||
throw new Error(RESPONSE_ERROR_MESSAGES.PACKAGE_DOES_NOT_CONTAIN_NODES);
|
||||
}
|
||||
return this.runDirectoryLoader(PackageDirectoryLoader, finalNodeUnpackedPath);
|
||||
}
|
||||
|
||||
async installNpmModule(packageName: string, version?: string): Promise<InstalledPackages> {
|
||||
return this.installOrUpdateNpmModule(packageName, { version });
|
||||
}
|
||||
|
||||
async removeNpmModule(packageName: string, installedPackage: InstalledPackages): Promise<void> {
|
||||
const communityPackageService = Container.get(CommunityPackageService);
|
||||
|
||||
await communityPackageService.executeNpmCommand(`npm remove ${packageName}`);
|
||||
|
||||
await communityPackageService.removePackageFromDatabase(installedPackage);
|
||||
|
||||
async unloadPackage(packageName: string) {
|
||||
if (packageName in this.loaders) {
|
||||
this.loaders[packageName].reset();
|
||||
delete this.loaders[packageName];
|
||||
}
|
||||
|
||||
await this.postProcessLoaders();
|
||||
await this.generateTypesForFrontend();
|
||||
}
|
||||
|
||||
async updateNpmModule(
|
||||
packageName: string,
|
||||
installedPackage: InstalledPackages,
|
||||
): Promise<InstalledPackages> {
|
||||
return this.installOrUpdateNpmModule(packageName, { installedPackage });
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -382,5 +296,49 @@ export class LoadNodesAndCredentials implements INodesAndCredentials {
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
this.injectCustomApiCallOptions();
|
||||
|
||||
for (const postProcessor of this.postProcessors) {
|
||||
await postProcessor();
|
||||
}
|
||||
}
|
||||
|
||||
async setupHotReload() {
|
||||
const { default: debounce } = await import('lodash/debounce');
|
||||
// eslint-disable-next-line import/no-extraneous-dependencies
|
||||
const { watch } = await import('chokidar');
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
const { Push } = await import('@/push');
|
||||
const push = Container.get(Push);
|
||||
|
||||
Object.values(this.loaders).forEach(async (loader) => {
|
||||
try {
|
||||
await fsPromises.access(loader.directory);
|
||||
} catch {
|
||||
// If directory doesn't exist, there is nothing to watch
|
||||
return;
|
||||
}
|
||||
|
||||
const realModulePath = path.join(await fsPromises.realpath(loader.directory), path.sep);
|
||||
const reloader = debounce(async () => {
|
||||
const modulesToUnload = Object.keys(require.cache).filter((filePath) =>
|
||||
filePath.startsWith(realModulePath),
|
||||
);
|
||||
modulesToUnload.forEach((filePath) => {
|
||||
delete require.cache[filePath];
|
||||
});
|
||||
|
||||
loader.reset();
|
||||
await loader.loadAll();
|
||||
await this.postProcessLoaders();
|
||||
push.send('nodeDescriptionUpdated', undefined);
|
||||
}, 100);
|
||||
|
||||
const toWatch = loader.isLazyLoaded
|
||||
? ['**/nodes.json', '**/credentials.json']
|
||||
: ['**/*.js', '**/*.json'];
|
||||
watch(toWatch, { cwd: realModulePath }).on('change', reloader);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user