mirror of
https://github.com/Abdulazizzn/n8n-enterprise-unlocked.git
synced 2025-12-17 10:02:05 +00:00
✨ Unify execution id + Queue system (#1340)
* Unify execution ID across executions * Fix indentation and improved comments * WIP: saving data after each node execution * Added on/off to save data after each step, saving initial data and retries working * Fixing lint issues * Fixing more lint issues * ✨ Add bull to execute workflows * 👕 Fix lint issue * ⚡ Add graceful shutdown to worker * ⚡ Add loading staticData to worker * 👕 Fix lint issue * ⚡ Fix import * Changed tables metadata to add nullable to stoppedAt * Reload database on migration run * Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration * Added checks to Redis and exiting process if connection is unavailable * Fixing error with new installations * Fix issue with data not being sent back to browser on manual executions with defined destination * Merging bull and unify execution id branch fixes * Main process will now get execution success from database instead of redis * Omit execution duration if execution did not stop * Fix issue with execution list displaying inconsistant information information while a workflow is running * Remove unused hooks to clarify for developers that these wont run in queue mode * Added active pooling to help recover from Redis crashes * Lint issues * Changing default polling interval to 60 seconds * Removed unnecessary attributes from bull job * ⚡ Improved output on worker job start Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
This commit is contained in:
Omar Ajoue
276
packages/cli/commands/worker.ts
Normal file
276
packages/cli/commands/worker.ts
Normal file
@@ -0,0 +1,276 @@
|
||||
import * as PCancelable from 'p-cancelable';
|
||||
|
||||
import { Command, flags } from '@oclif/command';
|
||||
import {
|
||||
UserSettings,
|
||||
WorkflowExecute,
|
||||
} from 'n8n-core';
|
||||
|
||||
import {
|
||||
IDataObject,
|
||||
INodeTypes,
|
||||
IRun,
|
||||
IWorkflowExecuteHooks,
|
||||
Workflow,
|
||||
WorkflowHooks,
|
||||
} from 'n8n-workflow';
|
||||
|
||||
import {
|
||||
FindOneOptions,
|
||||
} from 'typeorm';
|
||||
|
||||
import {
|
||||
ActiveExecutions,
|
||||
CredentialsOverwrites,
|
||||
CredentialTypes,
|
||||
Db,
|
||||
ExternalHooks,
|
||||
GenericHelpers,
|
||||
IBullJobData,
|
||||
IBullJobResponse,
|
||||
IExecutionFlattedDb,
|
||||
IExecutionResponse,
|
||||
LoadNodesAndCredentials,
|
||||
NodeTypes,
|
||||
ResponseHelper,
|
||||
WorkflowCredentials,
|
||||
WorkflowExecuteAdditionalData,
|
||||
} from "../src";
|
||||
|
||||
import * as config from '../config';
|
||||
import * as Bull from 'bull';
|
||||
|
||||
export class Worker extends Command {
|
||||
static description = '\nStarts a n8n worker';
|
||||
|
||||
static examples = [
|
||||
`$ n8n worker --concurrency=5`,
|
||||
];
|
||||
|
||||
static flags = {
|
||||
help: flags.help({ char: 'h' }),
|
||||
concurrency: flags.integer({
|
||||
default: 10,
|
||||
description: 'How many jobs can run in parallel.',
|
||||
}),
|
||||
};
|
||||
|
||||
static runningJobs: {
|
||||
[key: string]: PCancelable<IRun>;
|
||||
} = {};
|
||||
|
||||
static jobQueue: Bull.Queue;
|
||||
|
||||
static processExistCode = 0;
|
||||
// static activeExecutions = ActiveExecutions.getInstance();
|
||||
|
||||
/**
|
||||
* Stoppes the n8n in a graceful way.
|
||||
* Make for example sure that all the webhooks from third party services
|
||||
* get removed.
|
||||
*/
|
||||
static async stopProcess() {
|
||||
console.log(`\nStopping n8n...`);
|
||||
|
||||
// Stop accepting new jobs
|
||||
Worker.jobQueue.pause(true);
|
||||
|
||||
try {
|
||||
const externalHooks = ExternalHooks();
|
||||
await externalHooks.run('n8n.stop', []);
|
||||
|
||||
const maxStopTime = 30000;
|
||||
|
||||
const stopTime = new Date().getTime() + maxStopTime;
|
||||
|
||||
setTimeout(() => {
|
||||
// In case that something goes wrong with shutdown we
|
||||
// kill after max. 30 seconds no matter what
|
||||
process.exit(Worker.processExistCode);
|
||||
}, maxStopTime);
|
||||
|
||||
// Wait for active workflow executions to finish
|
||||
let count = 0;
|
||||
while (Object.keys(Worker.runningJobs).length !== 0) {
|
||||
if (count++ % 4 === 0) {
|
||||
const waitLeft = Math.ceil((stopTime - new Date().getTime()) / 1000);
|
||||
console.log(`Waiting for ${Object.keys(Worker.runningJobs).length} active executions to finish... (wait ${waitLeft} more seconds)`);
|
||||
}
|
||||
await new Promise((resolve) => {
|
||||
setTimeout(resolve, 500);
|
||||
});
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
console.error('There was an error shutting down n8n.', error);
|
||||
}
|
||||
|
||||
process.exit(Worker.processExistCode);
|
||||
}
|
||||
|
||||
async runJob(job: Bull.Job, nodeTypes: INodeTypes): Promise<IBullJobResponse> {
|
||||
const jobData = job.data as IBullJobData;
|
||||
const executionDb = await Db.collections.Execution!.findOne(jobData.executionId) as IExecutionFlattedDb;
|
||||
const currentExecutionDb = ResponseHelper.unflattenExecutionData(executionDb) as IExecutionResponse;
|
||||
|
||||
console.log(`Start job: ${job.id} (Workflow ID: ${currentExecutionDb.workflowData.id} | Execution: ${jobData.executionId})`);
|
||||
|
||||
let staticData = currentExecutionDb.workflowData!.staticData;
|
||||
if (jobData.loadStaticData === true) {
|
||||
const findOptions = {
|
||||
select: ['id', 'staticData'],
|
||||
} as FindOneOptions;
|
||||
const workflowData = await Db.collections!.Workflow!.findOne(currentExecutionDb.workflowData.id, findOptions);
|
||||
if (workflowData === undefined) {
|
||||
throw new Error(`The workflow with the ID "${currentExecutionDb.workflowData.id}" could not be found`);
|
||||
}
|
||||
staticData = workflowData.staticData;
|
||||
}
|
||||
|
||||
const workflow = new Workflow({ id: currentExecutionDb.workflowData.id as string, name: currentExecutionDb.workflowData.name, nodes: currentExecutionDb.workflowData!.nodes, connections: currentExecutionDb.workflowData!.connections, active: currentExecutionDb.workflowData!.active, nodeTypes, staticData, settings: currentExecutionDb.workflowData!.settings });
|
||||
|
||||
const credentials = await WorkflowCredentials(currentExecutionDb.workflowData.nodes);
|
||||
|
||||
const additionalData = await WorkflowExecuteAdditionalData.getBase(credentials);
|
||||
additionalData.hooks = WorkflowExecuteAdditionalData.getWorkflowHooksIntegrated(currentExecutionDb.mode, job.data.executionId, currentExecutionDb.workflowData, { retryOf: currentExecutionDb.retryOf as string });
|
||||
|
||||
let workflowExecute: WorkflowExecute;
|
||||
let workflowRun: PCancelable<IRun>;
|
||||
if (currentExecutionDb.data !== undefined) {
|
||||
workflowExecute = new WorkflowExecute(additionalData, currentExecutionDb.mode, currentExecutionDb.data);
|
||||
workflowRun = workflowExecute.processRunExecutionData(workflow);
|
||||
} else {
|
||||
// Execute all nodes
|
||||
// Can execute without webhook so go on
|
||||
workflowExecute = new WorkflowExecute(additionalData, currentExecutionDb.mode);
|
||||
workflowRun = workflowExecute.run(workflow);
|
||||
}
|
||||
|
||||
Worker.runningJobs[job.id] = workflowRun;
|
||||
|
||||
// Wait till the execution is finished
|
||||
const runData = await workflowRun;
|
||||
|
||||
delete Worker.runningJobs[job.id];
|
||||
|
||||
return {
|
||||
success: true,
|
||||
};
|
||||
}
|
||||
|
||||
async run() {
|
||||
console.log('Starting n8n worker...');
|
||||
|
||||
// Make sure that n8n shuts down gracefully if possible
|
||||
process.on('SIGTERM', Worker.stopProcess);
|
||||
process.on('SIGINT', Worker.stopProcess);
|
||||
|
||||
// Wrap that the process does not close but we can still use async
|
||||
await (async () => {
|
||||
try {
|
||||
const { flags } = this.parse(Worker);
|
||||
|
||||
// Start directly with the init of the database to improve startup time
|
||||
const startDbInitPromise = Db.init().catch(error => {
|
||||
console.error(`There was an error initializing DB: ${error.message}`);
|
||||
|
||||
Worker.processExistCode = 1;
|
||||
// @ts-ignore
|
||||
process.emit('SIGINT');
|
||||
});
|
||||
|
||||
// Make sure the settings exist
|
||||
await UserSettings.prepareUserSettings();
|
||||
|
||||
// Load all node and credential types
|
||||
const loadNodesAndCredentials = LoadNodesAndCredentials();
|
||||
await loadNodesAndCredentials.init();
|
||||
|
||||
// Load the credentials overwrites if any exist
|
||||
const credentialsOverwrites = CredentialsOverwrites();
|
||||
await credentialsOverwrites.init();
|
||||
|
||||
// Load all external hooks
|
||||
const externalHooks = ExternalHooks();
|
||||
await externalHooks.init();
|
||||
|
||||
// Add the found types to an instance other parts of the application can use
|
||||
const nodeTypes = NodeTypes();
|
||||
await nodeTypes.init(loadNodesAndCredentials.nodeTypes);
|
||||
const credentialTypes = CredentialTypes();
|
||||
await credentialTypes.init(loadNodesAndCredentials.credentialTypes);
|
||||
|
||||
// Wait till the database is ready
|
||||
await startDbInitPromise;
|
||||
|
||||
// Connect to bull-queue
|
||||
const prefix = config.get('queue.bull.prefix') as string;
|
||||
const redisOptions = config.get('queue.bull.redis') as IDataObject;
|
||||
const redisConnectionTimeoutLimit = config.get('queue.bull.redis.timeoutThreshold');
|
||||
// Disabling ready check is necessary as it allows worker to
|
||||
// quickly reconnect to Redis if Redis crashes or is unreachable
|
||||
// for some time. With it enabled, worker might take minutes to realize
|
||||
// redis is back up and resume working.
|
||||
redisOptions.enableReadyCheck = false;
|
||||
Worker.jobQueue = new Bull('jobs', { prefix, redis: redisOptions });
|
||||
Worker.jobQueue.process(flags.concurrency, (job) => this.runJob(job, nodeTypes));
|
||||
|
||||
const versions = await GenericHelpers.getVersions();
|
||||
|
||||
console.log('\nn8n worker is now ready');
|
||||
console.log(` * Version: ${versions.cli}`);
|
||||
console.log(` * Concurrency: ${flags.concurrency}`);
|
||||
console.log('');
|
||||
|
||||
Worker.jobQueue.on('global:progress', (jobId, progress) => {
|
||||
// Progress of a job got updated which does get used
|
||||
// to communicate that a job got canceled.
|
||||
|
||||
if (progress === -1) {
|
||||
// Job has to get canceled
|
||||
if (Worker.runningJobs[jobId] !== undefined) {
|
||||
// Job is processed by current worker so cancel
|
||||
Worker.runningJobs[jobId].cancel();
|
||||
delete Worker.runningJobs[jobId];
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
let lastTimer = 0, cumulativeTimeout = 0;
|
||||
Worker.jobQueue.on('error', (error: Error) => {
|
||||
if (error.toString().includes('ECONNREFUSED') === true) {
|
||||
const now = Date.now();
|
||||
if (now - lastTimer > 30000) {
|
||||
// Means we had no timeout at all or last timeout was temporary and we recovered
|
||||
lastTimer = now;
|
||||
cumulativeTimeout = 0;
|
||||
} else {
|
||||
cumulativeTimeout += now - lastTimer;
|
||||
lastTimer = now;
|
||||
if (cumulativeTimeout > redisConnectionTimeoutLimit) {
|
||||
console.error('Unable to connect to Redis after ' + redisConnectionTimeoutLimit + ". Exiting process.");
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
console.warn('Redis unavailable - trying to reconnect...');
|
||||
} else if (error.toString().includes('Error initializing Lua scripts') === true) {
|
||||
// This is a non-recoverable error
|
||||
// Happens when worker starts and Redis is unavailable
|
||||
// Even if Redis comes back online, worker will be zombie
|
||||
console.error('Error initializing worker.');
|
||||
process.exit(2);
|
||||
} else {
|
||||
console.error('Error from queue: ', error);
|
||||
}
|
||||
});
|
||||
} catch (error) {
|
||||
this.error(`There was an error: ${error.message}`);
|
||||
|
||||
Worker.processExistCode = 1;
|
||||
// @ts-ignore
|
||||
process.emit('SIGINT');
|
||||
}
|
||||
})();
|
||||
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user