fix(core): Add mechanism to prevent concurrent compaction on Insights (#14988)

Co-authored-by: Danny Martini <danny@n8n.io>

packages/cli/src/modules/insights/__tests__/insights-compaction.service.test.ts

@@ -1,8 +1,10 @@
 import { GlobalConfig } from '@n8n/config';
 import { Container } from '@n8n/di';
+import { mock } from 'jest-mock-extended';
 import { DateTime } from 'luxon';
 
 import { InsightsRawRepository } from '@/modules/insights/database/repositories/insights-raw.repository';
+import { mockLogger } from '@test/mocking';
 import { createTeamProject } from '@test-integration/db/projects';
 import { createWorkflow } from '@test-integration/db/workflows';
 import * as testDb from '@test-integration/test-db';
@@ -45,6 +47,7 @@ if (dbType === 'sqlite' && !globalConfig.database.sqlite.poolSize) {
 	afterAll(async () => {
 		await testDb.terminate();
 	});
+
 	describe('compaction', () => {
 		describe('compactRawToHour', () => {
 			type TestData = {
@@ -317,15 +320,22 @@ if (dbType === 'sqlite' && !globalConfig.database.sqlite.poolSize) {
 
 		describe('compactionSchedule', () => {
 			test('compaction is running on schedule', async () => {
-				jest.useFakeTimers();
-				try {
 				// ARRANGE
-					const insightsCompactionService = Container.get(InsightsCompactionService);
-					insightsCompactionService.startCompactionTimer();
-
+				jest.useFakeTimers();
+				const insightsCompactionService = new InsightsCompactionService(
+					mock<InsightsByPeriodRepository>(),
+					mock<InsightsRawRepository>(),
+					mock<InsightsConfig>({
+						compactionIntervalMinutes: 60,
+					}),
+					mockLogger(),
+				);
 				// spy on the compactInsights method to check if it's called
 				const compactInsightsSpy = jest.spyOn(insightsCompactionService, 'compactInsights');
 
+				try {
+					insightsCompactionService.startCompactionTimer();
+
 					// ACT
 					// advance by 1 hour and 1 minute
 					jest.advanceTimersByTime(1000 * 60 * 61);
@@ -333,6 +343,7 @@ if (dbType === 'sqlite' && !globalConfig.database.sqlite.poolSize) {
 					// ASSERT
 					expect(compactInsightsSpy).toHaveBeenCalledTimes(1);
 				} finally {
+					insightsCompactionService.stopCompactionTimer();
 					jest.useRealTimers();
 				}
 			});

packages/cli/src/modules/insights/database/repositories/__tests__/insights-by-period.repository.test.ts

@@ -0,0 +1,77 @@
+import { Container } from '@n8n/di';
+import { DateTime } from 'luxon';
+
+import { InsightsConfig } from '@/modules/insights/insights.config';
+import { createTeamProject } from '@test-integration/db/projects';
+import { createWorkflow } from '@test-integration/db/workflows';
+import * as testDb from '@test-integration/test-db';
+
+import { createCompactedInsightsEvent, createMetadata } from '../../entities/__tests__/db-utils';
+import { InsightsByPeriodRepository } from '../insights-by-period.repository';
+
+describe('InsightsByPeriodRepository', () => {
+	beforeAll(async () => {
+		await testDb.init();
+	});
+
+	describe('Avoid deadlock error', () => {
+		let defaultBatchSize: number;
+		beforeAll(() => {
+			// Store the original config value
+			const insightsConfig = Container.get(InsightsConfig);
+			defaultBatchSize = insightsConfig.compactionBatchSize;
+
+			// Set a smaller batch size to trigger the deadlock error
+			insightsConfig.compactionBatchSize = 3;
+		});
+
+		afterAll(() => {
+			// Reset the config to its original state
+			const insightsConfig = Container.get(InsightsConfig);
+			insightsConfig.compactionBatchSize = defaultBatchSize;
+		});
+
+		test('should not throw deadlock error on concurrent compaction', async () => {
+			// ARRANGE
+			const insightsConfig = Container.get(InsightsConfig);
+			const insightsByPeriodRepository = Container.get(InsightsByPeriodRepository);
+			const transactionSpy = jest.spyOn(insightsByPeriodRepository.manager, 'transaction');
+			const project = await createTeamProject();
+			const workflow = await createWorkflow({}, project);
+			await createMetadata(workflow);
+
+			const batchQuery = insightsByPeriodRepository.getPeriodInsightsBatchQuery({
+				periodUnitToCompactFrom: 'hour',
+				compactionBatchSize: insightsConfig.compactionBatchSize,
+				maxAgeInDays: insightsConfig.compactionHourlyToDailyThresholdDays,
+			});
+
+			// Create test data
+			const promises = [];
+			for (let i = 0; i < 100; i++) {
+				await createCompactedInsightsEvent(workflow, {
+					type: 'success',
+					value: 1,
+					periodUnit: 'hour',
+					periodStart: DateTime.now().minus({ day: 91, hour: i + 1 }),
+				});
+			}
+
+			// ACT
+			for (let i = 0; i < 10; i++) {
+				promises.push(
+					insightsByPeriodRepository.compactSourceDataIntoInsightPeriod({
+						sourceBatchQuery: batchQuery,
+						sourceTableName: insightsByPeriodRepository.metadata.tableName,
+						periodUnitToCompactInto: 'day',
+					}),
+				);
+			}
+
+			// ASSERT
+			// await all promises concurrently
+			await expect(Promise.all(promises)).resolves.toBeDefined();
+			expect(transactionSpy).toHaveBeenCalledTimes(1);
+		});
+	});
+});

packages/cli/src/modules/insights/database/repositories/insights-by-period.repository.ts

@@ -57,6 +57,8 @@ const aggregatedInsightsByTimeParser = z
 
 @Service()
 export class InsightsByPeriodRepository extends Repository<InsightsByPeriod> {
+	private isRunningCompaction = false;
+
 	constructor(dataSource: DataSource) {
 		super(InsightsByPeriod, dataSource.manager);
 	}
@@ -171,6 +173,13 @@ export class InsightsByPeriodRepository extends Repository<InsightsByPeriod> {
 		 */
 		periodUnitToCompactInto: PeriodUnit;
 	}): Promise<number> {
+		// Skip compaction if the process is already running
+		if (this.isRunningCompaction) {
+			return 0;
+		}
+		this.isRunningCompaction = true;
+
+		try {
 			// Create temp table that only exists in this transaction for rows to compact
 			const getBatchAndStoreInTemporaryTable = sql`
 				CREATE TEMPORARY TABLE rows_to_compact AS
@@ -238,6 +247,9 @@ export class InsightsByPeriodRepository extends Repository<InsightsByPeriod> {
 			});
 
 			return result;
+		} finally {
+			this.isRunningCompaction = false;
+		}
 	}
 
 	private getAgeLimitQuery(maxAgeInDays: number) {