fix(core): Ensure only leader handles waiting executions (#9014)

This commit is contained in:
Iván Ovejero
2024-04-04 13:28:20 +02:00
committed by GitHub
parent db4f8d49a3
commit 217b07d735
5 changed files with 85 additions and 10 deletions

View File

@@ -7,8 +7,9 @@ import { Container, Service } from 'typedi';
import type { IExecutionsStopData, IWorkflowExecutionDataProcess } from '@/Interfaces';
import { WorkflowRunner } from '@/WorkflowRunner';
import { ExecutionRepository } from '@db/repositories/execution.repository';
import { OwnershipService } from './services/ownership.service';
import { OwnershipService } from '@/services/ownership.service';
import { Logger } from '@/Logger';
import { OrchestrationService } from '@/services/orchestration.service';
@Service()
export class WaitTracker {
@@ -26,7 +27,22 @@ export class WaitTracker {
private readonly executionRepository: ExecutionRepository,
private readonly ownershipService: OwnershipService,
private readonly workflowRunner: WorkflowRunner,
readonly orchestrationService: OrchestrationService,
) {
const { isLeader, isMultiMainSetupEnabled, multiMainSetup } = orchestrationService;
if (isLeader) this.startTracking();
if (isMultiMainSetupEnabled) {
multiMainSetup
.on('leader-takeover', () => this.startTracking())
.on('leader-stepdown', () => this.stopTracking());
}
}
startTracking() {
this.logger.debug('Wait tracker started tracking waiting executions');
// Poll every 60 seconds a list of upcoming executions
this.mainTimer = setInterval(() => {
void this.getWaitingExecutions();
@@ -174,7 +190,9 @@ export class WaitTracker {
});
}
shutdown() {
stopTracking() {
this.logger.debug('Wait tracker shutting down');
clearInterval(this.mainTimer);
Object.keys(this.waitingExecutions).forEach((executionId) => {
clearTimeout(this.waitingExecutions[executionId].timer);

View File

@@ -94,7 +94,7 @@ export class Start extends BaseCommand {
// Stop with trying to activate workflows that could not be activated
this.activeWorkflowRunner.removeAllQueuedWorkflowActivations();
Container.get(WaitTracker).shutdown();
Container.get(WaitTracker).stopTracking();
await this.externalHooks?.run('n8n.stop', []);

View File

@@ -39,6 +39,9 @@ export class OrchestrationService {
return config.getEnv('redis.queueModeId');
}
/**
* Whether this instance is the leader in a multi-main setup. Always `true` in single-main setup.
*/
get isLeader() {
return config.getEnv('multiMainSetup.instanceType') === 'leader';
}

View File

@@ -62,7 +62,7 @@ export class MultiMainSetup extends EventEmitter {
if (config.getEnv('multiMainSetup.instanceType') === 'leader') {
config.set('multiMainSetup.instanceType', 'follower');
this.emit('leader-stepdown'); // lost leadership - stop triggers, pollers, pruning
this.emit('leader-stepdown'); // lost leadership - stop triggers, pollers, pruning, wait-tracking
EventReporter.info('[Multi-main setup] Leader failed to renew leader key');
}
@@ -97,7 +97,7 @@ export class MultiMainSetup extends EventEmitter {
await this.redisPublisher.setExpiration(this.leaderKey, this.leaderKeyTtl);
this.emit('leader-takeover'); // gained leadership - start triggers, pollers, pruning
this.emit('leader-takeover'); // gained leadership - start triggers, pollers, pruning, wait-tracking
} else {
config.set('multiMainSetup.instanceType', 'follower');
}