feat(core): Set up leader selection for multiple main instances (#7527)

https://linear.app/n8n/issue/PAY-933/set-up-leader-selection-for-multiple-main-instances

- [x] Set up new envs
- [x] Add config and license checks
- [x] Implement `MultiMainInstancePublisher`
- [x] Expand `RedisServicePubSubPublisher` to support
`MultiMainInstancePublisher`
- [x] Init `MultiMainInstancePublisher` on startup and destroy on
shutdown
- [x] Add to sandbox plans
- [x] Test manually

Note: This is only for setup - coordinating in reaction to leadership
changes will come in later PRs.
This commit is contained in:
Iván Ovejero
2023-10-30 16:22:32 +01:00
committed by GitHub
parent 3b5e181e66
commit 442c73e63b
15 changed files with 247 additions and 54 deletions

View File

@@ -21,14 +21,14 @@ import { ActiveWorkflowRunner } from '@/ActiveWorkflowRunner';
import * as Db from '@/Db';
import * as GenericHelpers from '@/GenericHelpers';
import { Server } from '@/Server';
import { EDITOR_UI_DIST_DIR, GENERATED_STATIC_DIR } from '@/constants';
import { EDITOR_UI_DIST_DIR, GENERATED_STATIC_DIR, LICENSE_FEATURES } from '@/constants';
import { eventBus } from '@/eventbus';
import { BaseCommand } from './BaseCommand';
import { InternalHooks } from '@/InternalHooks';
import { License } from '@/License';
import { License, FeatureNotLicensedError } from '@/License';
import { ExecutionRepository } from '@/databases/repositories/execution.repository';
import { IConfig } from '@oclif/config';
import { OrchestrationMainService } from '@/services/orchestration/main/orchestration.main.service';
import { SingleMainInstancePublisher } from '@/services/orchestration/main/SingleMainInstance.publisher';
import { OrchestrationHandlerMainService } from '@/services/orchestration/main/orchestration.handler.main.service';
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-var-requires
@@ -112,6 +112,14 @@ export class Start extends BaseCommand {
Container.get(ExecutionRepository).clearTimers();
if (config.getEnv('leaderSelection.enabled')) {
const { MultiMainInstancePublisher } = await import(
'@/services/orchestration/main/MultiMainInstance.publisher.ee'
);
await Container.get(MultiMainInstancePublisher).destroy();
}
await Container.get(InternalHooks).onN8nStop();
// Wait for active workflow executions to finish
@@ -215,10 +223,24 @@ export class Start extends BaseCommand {
}
async initOrchestration() {
if (config.get('executions.mode') === 'queue') {
await Container.get(OrchestrationMainService).init();
if (config.get('executions.mode') !== 'queue') return;
if (!config.get('leaderSelection.enabled')) {
await Container.get(SingleMainInstancePublisher).init();
await Container.get(OrchestrationHandlerMainService).init();
return;
}
if (!Container.get(License).isMultipleMainInstancesLicensed()) {
throw new FeatureNotLicensedError(LICENSE_FEATURES.MULTIPLE_MAIN_INSTANCES);
}
const { MultiMainInstancePublisher } = await import(
'@/services/orchestration/main/MultiMainInstance.publisher.ee'
);
await Container.get(MultiMainInstancePublisher).init();
await Container.get(OrchestrationHandlerMainService).init();
}
async run() {