feat: Workflow History pruning and prune time settings (#7343)

Github issue / Community forum post (link here to close automatically):

packages/cli/test/integration/shared/testDb.ts

@@ -610,6 +610,22 @@ export async function createWorkflowHistoryItem(
 	});
 }
 
+export async function createManyWorkflowHistoryItems(
+	workflowId: string,
+	count: number,
+	time?: Date,
+) {
+	const baseTime = (time ?? new Date()).valueOf();
+	return Promise.all(
+		[...Array(count)].map(async (_, i) =>
+			createWorkflowHistoryItem(workflowId, {
+				createdAt: new Date(baseTime + i),
+				updatedAt: new Date(baseTime + i),
+			}),
+		),
+	);
+}
+
 // ----------------------------------
 //        connection options
 // ----------------------------------

packages/cli/test/integration/workflowHistoryManager.test.ts

@@ -0,0 +1,208 @@
+import { WorkflowHistoryRepository } from '@/databases/repositories';
+import * as testDb from './shared/testDb';
+import { License } from '@/License';
+import { mockInstance } from './shared/utils';
+import { WorkflowHistoryManager } from '@/workflows/workflowHistory/workflowHistoryManager.ee';
+import Container from 'typedi';
+import config from '@/config';
+import { DateTime } from 'luxon';
+import { In } from 'typeorm';
+
+let licenseMock: License;
+let licensePruneTime = -1;
+let licenseEnabled = true;
+let manager: WorkflowHistoryManager;
+
+beforeAll(async () => {
+	await testDb.init();
+
+	licenseMock = mockInstance(License, {
+		isWorkflowHistoryLicensed() {
+			return licenseEnabled;
+		},
+		getWorkflowHistoryPruneLimit() {
+			return licensePruneTime;
+		},
+	});
+});
+
+beforeEach(async () => {
+	await testDb.truncate([WorkflowHistoryRepository]);
+	jest.useRealTimers();
+	jest.clearAllMocks();
+	config.set('workflowHistory.enabled', true);
+	config.set('workflowHistory.pruneTime', -1);
+	licensePruneTime = -1;
+	licenseEnabled = true;
+});
+
+afterEach(() => {
+	manager?.shutdown();
+});
+
+describe('Workflow History Manager', () => {
+	test('should prune on interval', () => {
+		jest.useFakeTimers();
+
+		manager = new WorkflowHistoryManager(Container.get(WorkflowHistoryRepository));
+		manager.init();
+		const pruneSpy = jest.spyOn(manager, 'prune');
+		const currentCount = pruneSpy.mock.calls.length;
+
+		jest.runOnlyPendingTimers();
+
+		expect(pruneSpy).toBeCalledTimes(currentCount + 1);
+
+		jest.runOnlyPendingTimers();
+		expect(pruneSpy).toBeCalledTimes(currentCount + 2);
+	});
+
+	test('should not prune when not licensed', async () => {
+		// Set a prune time just to make sure it gets to the delete
+		config.set('workflowHistory.pruneTime', 24);
+
+		licenseEnabled = false;
+
+		const repo = Container.get(WorkflowHistoryRepository);
+		manager = new WorkflowHistoryManager(repo);
+		manager.init();
+
+		const workflow = await testDb.createWorkflow();
+		await testDb.createManyWorkflowHistoryItems(
+			workflow.id,
+			10,
+			DateTime.now().minus({ days: 2 }).toJSDate(),
+		);
+
+		expect(await repo.count()).toBe(10);
+
+		const deleteSpy = jest.spyOn(repo, 'delete');
+		await manager.prune();
+		expect(deleteSpy).not.toBeCalled();
+		expect(await repo.count()).toBe(10);
+	});
+
+	test('should not prune when licensed but disabled', async () => {
+		// Set a prune time just to make sure it gets to the delete
+		config.set('workflowHistory.pruneTime', 24);
+
+		config.set('workflowHistory.enabled', false);
+
+		const repo = Container.get(WorkflowHistoryRepository);
+		manager = new WorkflowHistoryManager(repo);
+		manager.init();
+
+		const workflow = await testDb.createWorkflow();
+		await testDb.createManyWorkflowHistoryItems(
+			workflow.id,
+			10,
+			DateTime.now().minus({ days: 2 }).toJSDate(),
+		);
+
+		expect(await repo.count()).toBe(10);
+
+		const deleteSpy = jest.spyOn(repo, 'delete');
+		await manager.prune();
+		expect(deleteSpy).not.toBeCalled();
+		expect(await repo.count()).toBe(10);
+	});
+
+	test('should not prune when both prune times are -1 (infinite)', async () => {
+		config.set('workflowHistory.pruneTime', -1);
+		licensePruneTime = -1;
+
+		const repo = Container.get(WorkflowHistoryRepository);
+		manager = new WorkflowHistoryManager(repo);
+		manager.init();
+
+		const workflow = await testDb.createWorkflow();
+		await testDb.createManyWorkflowHistoryItems(
+			workflow.id,
+			10,
+			DateTime.now().minus({ days: 2 }).toJSDate(),
+		);
+
+		expect(await repo.count()).toBe(10);
+
+		const deleteSpy = jest.spyOn(repo, 'delete');
+		await manager.prune();
+		expect(deleteSpy).not.toBeCalled();
+		expect(await repo.count()).toBe(10);
+	});
+
+	test('should prune when config prune time is not -1 (infinite)', async () => {
+		config.set('workflowHistory.pruneTime', 24);
+		licensePruneTime = -1;
+
+		const repo = Container.get(WorkflowHistoryRepository);
+		manager = new WorkflowHistoryManager(repo);
+		manager.init();
+
+		const workflow = await testDb.createWorkflow();
+		await testDb.createManyWorkflowHistoryItems(
+			workflow.id,
+			10,
+			DateTime.now().minus({ days: 2 }).toJSDate(),
+		);
+
+		expect(await repo.count()).toBe(10);
+
+		const deleteSpy = jest.spyOn(repo, 'delete');
+		await manager.prune();
+		expect(deleteSpy).toBeCalled();
+		expect(await repo.count()).toBe(0);
+	});
+
+	test('should prune when license prune time is not -1 (infinite)', async () => {
+		config.set('workflowHistory.pruneTime', -1);
+		licensePruneTime = 24;
+
+		const repo = Container.get(WorkflowHistoryRepository);
+		manager = new WorkflowHistoryManager(repo);
+		manager.init();
+
+		const workflow = await testDb.createWorkflow();
+		await testDb.createManyWorkflowHistoryItems(
+			workflow.id,
+			10,
+			DateTime.now().minus({ days: 2 }).toJSDate(),
+		);
+
+		expect(await repo.count()).toBe(10);
+
+		const deleteSpy = jest.spyOn(repo, 'delete');
+		await manager.prune();
+		expect(deleteSpy).toBeCalled();
+		expect(await repo.count()).toBe(0);
+	});
+
+	test('should only prune versions older than prune time', async () => {
+		config.set('workflowHistory.pruneTime', 24);
+		licensePruneTime = -1;
+
+		const repo = Container.get(WorkflowHistoryRepository);
+		manager = new WorkflowHistoryManager(repo);
+		manager.init();
+
+		const workflow = await testDb.createWorkflow();
+		const recentVersions = await testDb.createManyWorkflowHistoryItems(workflow.id, 10);
+		const oldVersions = await testDb.createManyWorkflowHistoryItems(
+			workflow.id,
+			10,
+			DateTime.now().minus({ days: 2 }).toJSDate(),
+		);
+
+		expect(await repo.count()).toBe(20);
+
+		const deleteSpy = jest.spyOn(repo, 'delete');
+		await manager.prune();
+		expect(deleteSpy).toBeCalled();
+		expect(await repo.count()).toBe(10);
+		expect(
+			await repo.count({ where: { versionId: In(recentVersions.map((i) => i.versionId)) } }),
+		).toBe(10);
+		expect(
+			await repo.count({ where: { versionId: In(oldVersions.map((i) => i.versionId)) } }),
+		).toBe(0);
+	});
+});

packages/cli/test/unit/workflowHistoryHelper.test.ts

@@ -0,0 +1,55 @@
+import { License } from '@/License';
+import { mockInstance } from '../integration/shared/utils';
+import config from '@/config';
+import { getWorkflowHistoryPruneTime } from '@/workflows/workflowHistory/workflowHistoryHelper.ee';
+
+let licenseMock: License;
+let licensePruneTime = -1;
+
+beforeAll(async () => {
+	licenseMock = mockInstance(License, {
+		getWorkflowHistoryPruneLimit() {
+			return licensePruneTime;
+		},
+	});
+});
+
+beforeEach(() => {
+	licensePruneTime = -1;
+	config.set('workflowHistory.pruneTime', -1);
+});
+
+describe('getWorkflowHistoryPruneTime', () => {
+	test('should return -1 (infinite) if config and license are -1', () => {
+		licensePruneTime = -1;
+		config.set('workflowHistory.pruneTime', -1);
+
+		expect(getWorkflowHistoryPruneTime()).toBe(-1);
+	});
+
+	test('should return config time if license is infinite and config is not', () => {
+		licensePruneTime = -1;
+		config.set('workflowHistory.pruneTime', 24);
+
+		expect(getWorkflowHistoryPruneTime()).toBe(24);
+	});
+
+	test('should return license time if config is infinite and license is not', () => {
+		licensePruneTime = 25;
+		config.set('workflowHistory.pruneTime', -1);
+
+		expect(getWorkflowHistoryPruneTime()).toBe(25);
+	});
+
+	test('should return lowest of config and license time if both are not -1', () => {
+		licensePruneTime = 26;
+		config.set('workflowHistory.pruneTime', 100);
+
+		expect(getWorkflowHistoryPruneTime()).toBe(26);
+
+		licensePruneTime = 100;
+		config.set('workflowHistory.pruneTime', 27);
+
+		expect(getWorkflowHistoryPruneTime()).toBe(27);
+	});
+});