feat(API): Add config to set age threshold for insights compaction (#14221)

packages/cli/src/modules/insights/__tests__/insights.service.test.ts

@@ -21,6 +21,7 @@ import {
 	createRawInsightsEvents,
 } from '../database/entities/__tests__/db-utils';
 import { InsightsByPeriodRepository } from '../database/repositories/insights-by-period.repository';
+import { InsightsConfig } from '../insights.config';
 import { InsightsService } from '../insights.service';
 
 // Initialize DB once for all tests
@@ -710,6 +711,91 @@ describe('compaction', () => {
 			expect(compactedRows).toBe(0);
 		});
 	});
+
+	describe('compaction threshold configuration', () => {
+		test('insights by period older than the hourly to daily threshold are not compacted', async () => {
+			// ARRANGE
+			const insightsService = Container.get(InsightsService);
+			const insightsByPeriodRepository = Container.get(InsightsByPeriodRepository);
+			const config = Container.get(InsightsConfig);
+
+			const project = await createTeamProject();
+			const workflow = await createWorkflow({}, project);
+
+			const thresholdDays = config.compactionHourlyToDailyThresholdDays;
+
+			// Create insights by period within and beyond the threshold
+			const withinThresholdTimestamp = DateTime.utc().minus({ days: thresholdDays - 1 });
+			const beyondThresholdTimestamp = DateTime.utc().minus({ days: thresholdDays + 1 });
+
+			await createCompactedInsightsEvent(workflow, {
+				type: 'success',
+				value: 1,
+				periodUnit: 'hour',
+				periodStart: withinThresholdTimestamp,
+			});
+
+			await createCompactedInsightsEvent(workflow, {
+				type: 'success',
+				value: 1,
+				periodUnit: 'hour',
+				periodStart: beyondThresholdTimestamp,
+			});
+
+			// ACT
+			const compactedRows = await insightsService.compactHourToDay();
+
+			// ASSERT
+			expect(compactedRows).toBe(1); // Only the event within the threshold should be compacted
+			const insightsByPeriods = await insightsByPeriodRepository.find();
+			const dailyInsights = insightsByPeriods.filter((insight) => insight.periodUnit === 'day');
+			expect(dailyInsights).toHaveLength(1); // The event beyond the threshold should remain
+			expect(dailyInsights[0].periodStart.toISOString()).toEqual(
+				beyondThresholdTimestamp.startOf('day').toISO(),
+			);
+		});
+
+		test('insights by period older than the daily to weekly threshold are not compacted', async () => {
+			// ARRANGE
+			const insightsService = Container.get(InsightsService);
+			const insightsByPeriodRepository = Container.get(InsightsByPeriodRepository);
+			const config = Container.get(InsightsConfig);
+
+			const project = await createTeamProject();
+			const workflow = await createWorkflow({}, project);
+
+			const thresholdDays = config.compactionDailyToWeeklyThresholdDays;
+
+			// Create insights by period within and beyond the threshold
+			const withinThresholdTimestamp = DateTime.utc().minus({ days: thresholdDays - 1 });
+			const beyondThresholdTimestamp = DateTime.utc().minus({ days: thresholdDays + 1 });
+
+			await createCompactedInsightsEvent(workflow, {
+				type: 'success',
+				value: 1,
+				periodUnit: 'day',
+				periodStart: withinThresholdTimestamp,
+			});
+			await createCompactedInsightsEvent(workflow, {
+				type: 'success',
+				value: 1,
+				periodUnit: 'day',
+				periodStart: beyondThresholdTimestamp,
+			});
+
+			// ACT
+			const compactedRows = await insightsService.compactDayToWeek();
+
+			// ASSERT
+			expect(compactedRows).toBe(1); // Only the event within the threshold should be compacted
+			const insightsByPeriods = await insightsByPeriodRepository.find();
+			const weeklyInsights = insightsByPeriods.filter((insight) => insight.periodUnit === 'week');
+			expect(weeklyInsights).toHaveLength(1); // The event beyond the threshold should remain
+			expect(weeklyInsights[0].periodStart.toISOString()).toEqual(
+				beyondThresholdTimestamp.startOf('week').toISO(),
+			);
+		});
+	});
 });
 
 describe('getInsightsSummary', () => {

packages/cli/src/modules/insights/insights.config.ts

@@ -15,4 +15,18 @@ export class InsightsConfig {
 	 */
 	@Env('N8N_INSIGHTS_COMPACTION_BATCH_SIZE')
 	compactionBatchSize: number = 500;
+
+	/**
+	 * The max age in days of hourly insights data to compact.
+	 * Default: 90
+	 */
+	@Env('N8N_INSIGHTS_COMPACTION_HOURLY_TO_DAILY_THRESHOLD_DAYS')
+	compactionHourlyToDailyThresholdDays: number = 90;
+
+	/**
+	 * The max age in days of daily insights data to compact.
+	 * Default: 180
+	 */
+	@Env('N8N_INSIGHTS_COMPACTION_DAILY_TO_WEEKLY_THRESHOLD_DAYS')
+	compactionDailyToWeeklyThresholdDays: number = 180;
 }

packages/cli/src/modules/insights/insights.service.ts

@@ -49,10 +49,6 @@ const shouldSkipMode: Record<WorkflowExecuteMode, boolean> = {
 
 @Service()
 export class InsightsService {
-	private readonly maxAgeInDaysForHourlyData = 90;
-
-	private readonly maxAgeInDaysForDailyData = 180;
-
 	private compactInsightsTimer: NodeJS.Timer | undefined;
 
 	constructor(
@@ -195,7 +191,7 @@ export class InsightsService {
 		const batchQuery = this.insightsByPeriodRepository.getPeriodInsightsBatchQuery({
 			periodUnitToCompactFrom: 'hour',
 			compactionBatchSize: config.compactionBatchSize,
-			maxAgeInDays: this.maxAgeInDaysForHourlyData,
+			maxAgeInDays: config.compactionHourlyToDailyThresholdDays,
 		});
 
 		return await this.insightsByPeriodRepository.compactSourceDataIntoInsightPeriod({
@@ -210,7 +206,7 @@ export class InsightsService {
 		const batchQuery = this.insightsByPeriodRepository.getPeriodInsightsBatchQuery({
 			periodUnitToCompactFrom: 'day',
 			compactionBatchSize: config.compactionBatchSize,
-			maxAgeInDays: this.maxAgeInDaysForDailyData,
+			maxAgeInDays: config.compactionDailyToWeeklyThresholdDays,
 		});
 
 		return await this.insightsByPeriodRepository.compactSourceDataIntoInsightPeriod({