From db381492a94d664f5e2ccc42f8830b20c8c19852 Mon Sep 17 00:00:00 2001 From: Guillaume Jacquart Date: Wed, 2 Apr 2025 16:34:57 +0200 Subject: [PATCH] feat(API): Implement BE api for insights data (#14064) Co-authored-by: Danny Martini --- .../api-types/src/schemas/insights.schema.ts | 6 +- .../__tests__/insights.service.test.ts | 353 ++++++++++++++++-- .../database/entities/insights-by-period.ts | 14 +- .../insights-by-period.repository.ts | 156 ++++++-- .../modules/insights/insights.controller.ts | 34 +- .../src/modules/insights/insights.service.ts | 51 ++- .../integration/insights/insights.api.test.ts | 78 ++++ .../cli/test/integration/shared/test-db.ts | 30 +- packages/cli/test/integration/shared/types.ts | 3 +- .../integration/shared/utils/test-server.ts | 3 + 10 files changed, 667 insertions(+), 61 deletions(-) create mode 100644 packages/cli/test/integration/insights/insights.api.test.ts diff --git a/packages/@n8n/api-types/src/schemas/insights.schema.ts b/packages/@n8n/api-types/src/schemas/insights.schema.ts index adb58448da..da8959e9e7 100644 --- a/packages/@n8n/api-types/src/schemas/insights.schema.ts +++ b/packages/@n8n/api-types/src/schemas/insights.schema.ts @@ -49,9 +49,9 @@ export const insightsByWorkflowDataSchemas = { z .object({ workflowId: z.string(), - workflowName: z.string().optional(), - projectId: z.string().optional(), - projectName: z.string().optional(), + workflowName: z.string(), + projectId: z.string(), + projectName: z.string(), total: z.number(), succeeded: z.number(), failed: z.number(), diff --git a/packages/cli/src/modules/insights/__tests__/insights.service.test.ts b/packages/cli/src/modules/insights/__tests__/insights.service.test.ts index 26c31224d3..caeb293516 100644 --- a/packages/cli/src/modules/insights/__tests__/insights.service.test.ts +++ b/packages/cli/src/modules/insights/__tests__/insights.service.test.ts @@ -23,22 +23,19 @@ import { import { InsightsByPeriodRepository } from '../database/repositories/insights-by-period.repository'; import { InsightsService } from '../insights.service'; -async function truncateAll() { - const insightsRawRepository = Container.get(InsightsRawRepository); - const insightsMetadataRepository = Container.get(InsightsMetadataRepository); - const insightsByPeriodRepository = Container.get(InsightsByPeriodRepository); - for (const repo of [ - insightsRawRepository, - insightsMetadataRepository, - insightsByPeriodRepository, - ]) { - await repo.delete({}); - } -} - // Initialize DB once for all tests beforeAll(async () => { - await testDb.init(); + await testDb.init(['insights']); +}); + +beforeEach(async () => { + await testDb.truncate([ + 'InsightsRaw', + 'InsightsByPeriod', + 'InsightsMetadata', + 'Workflow', + 'Project', + ]); }); // Terminate DB once after all tests complete @@ -60,8 +57,6 @@ describe('workflowExecuteAfterHandler', () => { let workflow: IWorkflowDb & WorkflowEntity; beforeEach(async () => { - await truncateAll(); - project = await createTeamProject(); workflow = await createWorkflow( { @@ -261,10 +256,6 @@ describe('workflowExecuteAfterHandler', () => { }); describe('compaction', () => { - beforeEach(async () => { - await truncateAll(); - }); - describe('compactRawToHour', () => { type TestData = { name: string; @@ -731,8 +722,6 @@ describe('getInsightsSummary', () => { let workflow: IWorkflowDb & WorkflowEntity; beforeEach(async () => { - await truncateAll(); - project = await createTeamProject(); workflow = await createWorkflow({}, project); }); @@ -848,3 +837,323 @@ describe('getInsightsSummary', () => { }); }); }); + +describe('getInsightsByWorkflow', () => { + let insightsService: InsightsService; + beforeAll(async () => { + insightsService = Container.get(InsightsService); + }); + + let project: Project; + let workflow1: IWorkflowDb & WorkflowEntity; + let workflow2: IWorkflowDb & WorkflowEntity; + let workflow3: IWorkflowDb & WorkflowEntity; + + beforeEach(async () => { + project = await createTeamProject(); + workflow1 = await createWorkflow({}, project); + workflow2 = await createWorkflow({}, project); + workflow3 = await createWorkflow({}, project); + }); + + test('compacted data are are grouped by workflow correctly', async () => { + // ARRANGE + for (const workflow of [workflow1, workflow2]) { + await createCompactedInsightsEvent(workflow, { + type: 'success', + value: workflow === workflow1 ? 1 : 2, + periodUnit: 'day', + periodStart: DateTime.utc(), + }); + await createCompactedInsightsEvent(workflow, { + type: 'success', + value: 1, + periodUnit: 'day', + periodStart: DateTime.utc().minus({ day: 2 }), + }); + await createCompactedInsightsEvent(workflow, { + type: 'failure', + value: 2, + periodUnit: 'day', + periodStart: DateTime.utc(), + }); + // last 14 days + await createCompactedInsightsEvent(workflow, { + type: 'success', + value: 1, + periodUnit: 'day', + periodStart: DateTime.utc().minus({ days: 10 }), + }); + await createCompactedInsightsEvent(workflow, { + type: 'runtime_ms', + value: 123, + periodUnit: 'day', + periodStart: DateTime.utc().minus({ days: 10 }), + }); + + // Barely in range insight (should be included) + // 1 hour before 14 days ago + await createCompactedInsightsEvent(workflow, { + type: 'success', + value: 1, + periodUnit: 'hour', + periodStart: DateTime.utc().minus({ days: 13, hours: 23 }), + }); + + // Out of date range insight (should not be included) + // 14 days ago + await createCompactedInsightsEvent(workflow, { + type: 'success', + value: 1, + periodUnit: 'day', + periodStart: DateTime.utc().minus({ days: 14 }), + }); + } + + // ACT + const byWorkflow = await insightsService.getInsightsByWorkflow({ + maxAgeInDays: 14, + }); + + // ASSERT + expect(byWorkflow.count).toEqual(2); + expect(byWorkflow.data).toHaveLength(2); + + // expect first workflow to be workflow 2, because it has a bigger total (default sorting) + expect(byWorkflow.data[0]).toMatchObject({ + workflowId: workflow2.id, + workflowName: workflow2.name, + projectId: project.id, + projectName: project.name, + total: 7, + failureRate: 2 / 7, + failed: 2, + runTime: 123, + succeeded: 5, + timeSaved: 0, + averageRunTime: 123 / 7, + }); + + expect(byWorkflow.data[1]).toEqual({ + workflowId: workflow1.id, + workflowName: workflow1.name, + projectId: project.id, + projectName: project.name, + total: 6, + failureRate: 2 / 6, + failed: 2, + runTime: 123, + succeeded: 4, + timeSaved: 0, + averageRunTime: 123 / 6, + }); + }); + + test('compacted data are grouped by workflow correctly with sorting', async () => { + // ARRANGE + for (const workflow of [workflow1, workflow2]) { + await createCompactedInsightsEvent(workflow, { + type: 'success', + value: workflow === workflow1 ? 1 : 2, + periodUnit: 'day', + periodStart: DateTime.utc(), + }); + await createCompactedInsightsEvent(workflow, { + type: 'failure', + value: 2, + periodUnit: 'day', + periodStart: DateTime.utc(), + }); + await createCompactedInsightsEvent(workflow, { + type: 'runtime_ms', + value: workflow === workflow1 ? 2 : 1, + periodUnit: 'day', + periodStart: DateTime.utc().minus({ days: 10 }), + }); + } + + // ACT + const byWorkflow = await insightsService.getInsightsByWorkflow({ + maxAgeInDays: 14, + sortBy: 'runTime:desc', + }); + + // ASSERT + expect(byWorkflow.count).toEqual(2); + expect(byWorkflow.data).toHaveLength(2); + expect(byWorkflow.data[0].workflowId).toEqual(workflow1.id); + }); + + test('compacted data are grouped by workflow correctly with pagination', async () => { + // ARRANGE + for (const workflow of [workflow1, workflow2, workflow3]) { + await createCompactedInsightsEvent(workflow, { + type: 'success', + value: workflow === workflow1 ? 1 : workflow === workflow2 ? 2 : 3, + periodUnit: 'day', + periodStart: DateTime.utc(), + }); + } + + // ACT + const byWorkflow = await insightsService.getInsightsByWorkflow({ + maxAgeInDays: 14, + sortBy: 'succeeded:desc', + skip: 1, + take: 1, + }); + + // ASSERT + expect(byWorkflow.count).toEqual(3); + expect(byWorkflow.data).toHaveLength(1); + expect(byWorkflow.data[0].workflowId).toEqual(workflow2.id); + }); + + test('compacted data are grouped by workflow correctly even with 0 data (check division by 0)', async () => { + // ACT + const byWorkflow = await insightsService.getInsightsByWorkflow({ + maxAgeInDays: 14, + }); + + // ASSERT + expect(byWorkflow.count).toEqual(0); + expect(byWorkflow.data).toHaveLength(0); + }); +}); + +describe('getInsightsByTime', () => { + let insightsService: InsightsService; + beforeAll(async () => { + insightsService = Container.get(InsightsService); + }); + + let project: Project; + let workflow1: IWorkflowDb & WorkflowEntity; + let workflow2: IWorkflowDb & WorkflowEntity; + + beforeEach(async () => { + project = await createTeamProject(); + workflow1 = await createWorkflow({}, project); + workflow2 = await createWorkflow({}, project); + }); + + test('returns empty array when no insights exist', async () => { + const byTime = await insightsService.getInsightsByTime({ maxAgeInDays: 14, periodUnit: 'day' }); + expect(byTime).toEqual([]); + }); + + test('returns empty array when no insights in the time range exists', async () => { + await createCompactedInsightsEvent(workflow1, { + type: 'success', + value: 2, + periodUnit: 'day', + periodStart: DateTime.utc().minus({ days: 30 }), + }); + + const byTime = await insightsService.getInsightsByTime({ maxAgeInDays: 14, periodUnit: 'day' }); + expect(byTime).toEqual([]); + }); + + test('compacted data are are grouped by time correctly', async () => { + // ARRANGE + for (const workflow of [workflow1, workflow2]) { + await createCompactedInsightsEvent(workflow, { + type: 'success', + value: workflow === workflow1 ? 1 : 2, + periodUnit: 'day', + periodStart: DateTime.utc(), + }); + // Check that hourly data is grouped together with the previous daily data + await createCompactedInsightsEvent(workflow, { + type: 'failure', + value: 2, + periodUnit: 'hour', + periodStart: DateTime.utc(), + }); + await createCompactedInsightsEvent(workflow, { + type: 'success', + value: 1, + periodUnit: 'day', + periodStart: DateTime.utc().minus({ day: 2 }), + }); + await createCompactedInsightsEvent(workflow, { + type: 'success', + value: 1, + periodUnit: 'day', + periodStart: DateTime.utc().minus({ days: 10 }), + }); + await createCompactedInsightsEvent(workflow, { + type: 'runtime_ms', + value: workflow === workflow1 ? 10 : 20, + periodUnit: 'day', + periodStart: DateTime.utc().minus({ days: 10 }), + }); + + // Barely in range insight (should be included) + // 1 hour before 14 days ago + await createCompactedInsightsEvent(workflow, { + type: workflow === workflow1 ? 'success' : 'failure', + value: 1, + periodUnit: 'hour', + periodStart: DateTime.utc().minus({ days: 13, hours: 23 }), + }); + + // Out of date range insight (should not be included) + // 14 days ago + await createCompactedInsightsEvent(workflow, { + type: 'success', + value: 1, + periodUnit: 'day', + periodStart: DateTime.utc().minus({ days: 14 }), + }); + } + + // ACT + const byTime = await insightsService.getInsightsByTime({ maxAgeInDays: 14, periodUnit: 'day' }); + + // ASSERT + expect(byTime).toHaveLength(4); + + // expect date to be sorted by oldest first + expect(byTime[0].date).toEqual(DateTime.utc().minus({ days: 14 }).startOf('day').toISO()); + expect(byTime[1].date).toEqual(DateTime.utc().minus({ days: 10 }).startOf('day').toISO()); + expect(byTime[2].date).toEqual(DateTime.utc().minus({ days: 2 }).startOf('day').toISO()); + expect(byTime[3].date).toEqual(DateTime.utc().startOf('day').toISO()); + + expect(byTime[0].values).toEqual({ + total: 2, + succeeded: 1, + failed: 1, + failureRate: 0.5, + averageRunTime: 0, + timeSaved: 0, + }); + + expect(byTime[1].values).toEqual({ + total: 2, + succeeded: 2, + failed: 0, + failureRate: 0, + averageRunTime: 15, + timeSaved: 0, + }); + + expect(byTime[2].values).toEqual({ + total: 2, + succeeded: 2, + failed: 0, + failureRate: 0, + averageRunTime: 0, + timeSaved: 0, + }); + + expect(byTime[3].values).toEqual({ + total: 7, + succeeded: 3, + failed: 4, + failureRate: 4 / 7, + averageRunTime: 0, + timeSaved: 0, + }); + }); +}); diff --git a/packages/cli/src/modules/insights/database/entities/insights-by-period.ts b/packages/cli/src/modules/insights/database/entities/insights-by-period.ts index 25c8ae6cc0..ef04da6c7e 100644 --- a/packages/cli/src/modules/insights/database/entities/insights-by-period.ts +++ b/packages/cli/src/modules/insights/database/entities/insights-by-period.ts @@ -1,6 +1,14 @@ -import { BaseEntity, Column, Entity, PrimaryGeneratedColumn } from '@n8n/typeorm'; +import { + BaseEntity, + Column, + Entity, + JoinColumn, + ManyToOne, + PrimaryGeneratedColumn, +} from '@n8n/typeorm'; import { UnexpectedError } from 'n8n-workflow'; +import { InsightsMetadata } from './insights-metadata'; import type { PeriodUnit } from './insights-shared'; import { isValidPeriodNumber, @@ -20,6 +28,10 @@ export class InsightsByPeriod extends BaseEntity { @Column() metaId: number; + @ManyToOne(() => InsightsMetadata) + @JoinColumn({ name: 'metaId' }) + metadata: InsightsMetadata; + @Column({ name: 'type', type: 'int' }) private type_: number; diff --git a/packages/cli/src/modules/insights/database/repositories/insights-by-period.repository.ts b/packages/cli/src/modules/insights/database/repositories/insights-by-period.repository.ts index dc41605424..9d662df10e 100644 --- a/packages/cli/src/modules/insights/database/repositories/insights-by-period.repository.ts +++ b/packages/cli/src/modules/insights/database/repositories/insights-by-period.repository.ts @@ -2,13 +2,14 @@ import { GlobalConfig } from '@n8n/config'; import { Container, Service } from '@n8n/di'; import type { SelectQueryBuilder } from '@n8n/typeorm'; import { DataSource, Repository } from '@n8n/typeorm'; +import { DateTime } from 'luxon'; import { z } from 'zod'; import { sql } from '@/utils/sql'; import { InsightsByPeriod } from '../entities/insights-by-period'; import type { PeriodUnit } from '../entities/insights-shared'; -import { PeriodUnitToNumber } from '../entities/insights-shared'; +import { PeriodUnitToNumber, TypeToNumber } from '../entities/insights-shared'; const dbType = Container.get(GlobalConfig).database.type; @@ -22,6 +23,38 @@ const summaryParser = z }) .array(); +const aggregatedInsightsByWorkflowParser = z + .object({ + workflowId: z.string(), + workflowName: z.string(), + projectId: z.string(), + projectName: z.string(), + total: z.union([z.number(), z.string()]).transform((value) => Number(value)), + succeeded: z.union([z.number(), z.string()]).transform((value) => Number(value)), + failed: z.union([z.number(), z.string()]).transform((value) => Number(value)), + failureRate: z.union([z.number(), z.string()]).transform((value) => Number(value)), + runTime: z.union([z.number(), z.string()]).transform((value) => Number(value)), + averageRunTime: z.union([z.number(), z.string()]).transform((value) => Number(value)), + timeSaved: z.union([z.number(), z.string()]).transform((value) => Number(value)), + }) + .array(); + +const aggregatedInsightsByTimeParser = z + .object({ + periodStart: z + .union([z.date(), z.string()]) + .transform((value) => + value instanceof Date + ? value.toISOString() + : DateTime.fromSQL(value.toString(), { zone: 'utc' }).toISO(), + ), + runTime: z.union([z.number(), z.string()]).transform((value) => Number(value)), + succeeded: z.union([z.number(), z.string()]).transform((value) => Number(value)), + failed: z.union([z.number(), z.string()]).transform((value) => Number(value)), + timeSaved: z.union([z.number(), z.string()]).transform((value) => Number(value)), + }) + .array(); + @Service() export class InsightsByPeriodRepository extends Repository { constructor(dataSource: DataSource) { @@ -207,6 +240,18 @@ export class InsightsByPeriodRepository extends Repository { return result; } + private getAgeLimitQuery(maxAgeInDays: number) { + if (maxAgeInDays === 0) { + return dbType === 'sqlite' ? "datetime('now')" : 'NOW()'; + } + + return dbType === 'sqlite' + ? `datetime('now', '-${maxAgeInDays} days')` + : dbType === 'postgresdb' + ? `NOW() - INTERVAL '${maxAgeInDays} days'` + : `DATE_SUB(NOW(), INTERVAL ${maxAgeInDays} DAY)`; + } + async getPreviousAndCurrentPeriodTypeAggregates(): Promise< Array<{ period: 'previous' | 'current'; @@ -214,27 +259,12 @@ export class InsightsByPeriodRepository extends Repository { total_value: string | number; }> > { - const cte = - dbType === 'sqlite' - ? sql` - SELECT - datetime('now', '-7 days') AS current_start, - datetime('now') AS current_end, - datetime('now', '-14 days') AS previous_start - ` - : dbType === 'postgresdb' - ? sql` - SELECT - (NOW() - INTERVAL '7 days')::timestamptz AS current_start, - NOW()::timestamptz AS current_end, - (NOW() - INTERVAL '14 days')::timestamptz AS previous_start - ` - : sql` - SELECT - DATE_SUB(NOW(), INTERVAL 7 DAY) AS current_start, - NOW() AS current_end, - DATE_SUB(NOW(), INTERVAL 14 DAY) AS previous_start - `; + const cte = sql` + SELECT + ${this.getAgeLimitQuery(7)} AS current_start, + ${this.getAgeLimitQuery(0)} AS current_end, + ${this.getAgeLimitQuery(14)} AS previous_start + `; const rawRows = await this.createQueryBuilder('insights') .addCommonTableExpression(cte, 'date_ranges') @@ -262,4 +292,86 @@ export class InsightsByPeriodRepository extends Repository { return summaryParser.parse(rawRows); } + + private parseSortingParams(sortBy: string): [string, 'ASC' | 'DESC'] { + const [column, order] = sortBy.split(':'); + return [column, order.toUpperCase() as 'ASC' | 'DESC']; + } + + async getInsightsByWorkflow({ + maxAgeInDays, + skip = 0, + take = 20, + sortBy = 'total:desc', + }: { + maxAgeInDays: number; + skip?: number; + take?: number; + sortBy?: string; + }) { + const [sortField, sortOrder] = this.parseSortingParams(sortBy); + const sumOfExecutions = sql`SUM(CASE WHEN insights.type IN (${TypeToNumber.success.toString()}, ${TypeToNumber.failure.toString()}) THEN value ELSE 0 END)`; + + const cte = sql`SELECT ${this.getAgeLimitQuery(maxAgeInDays)} AS start_date`; + + const rawRowsQuery = this.createQueryBuilder('insights') + .addCommonTableExpression(cte, 'date_range') + .select([ + 'metadata.workflowId AS "workflowId"', + 'metadata.workflowName AS "workflowName"', + 'metadata.projectId AS "projectId"', + 'metadata.projectName AS "projectName"', + `SUM(CASE WHEN insights.type = ${TypeToNumber.success} THEN value ELSE 0 END) AS "succeeded"`, + `SUM(CASE WHEN insights.type = ${TypeToNumber.failure} THEN value ELSE 0 END) AS "failed"`, + `SUM(CASE WHEN insights.type IN (${TypeToNumber.success}, ${TypeToNumber.failure}) THEN value ELSE 0 END) AS "total"`, + sql`CASE + WHEN ${sumOfExecutions} = 0 THEN 0 + ELSE 1.0 * SUM(CASE WHEN insights.type = ${TypeToNumber.failure.toString()} THEN value ELSE 0 END) / ${sumOfExecutions} + END AS "failureRate"`, + `SUM(CASE WHEN insights.type = ${TypeToNumber.runtime_ms} THEN value ELSE 0 END) AS "runTime"`, + `SUM(CASE WHEN insights.type = ${TypeToNumber.time_saved_min} THEN value ELSE 0 END) AS "timeSaved"`, + sql`CASE + WHEN ${sumOfExecutions} = 0 THEN 0 + ELSE 1.0 * SUM(CASE WHEN insights.type = ${TypeToNumber.runtime_ms.toString()} THEN value ELSE 0 END) / ${sumOfExecutions} + END AS "averageRunTime"`, + ]) + .innerJoin('insights.metadata', 'metadata') + // Use a cross join with the CTE + .innerJoin('date_range', 'date_range', '1=1') + .where('insights.periodStart >= date_range.start_date') + .groupBy('metadata.workflowId') + .addGroupBy('metadata.workflowName') + .addGroupBy('metadata.projectId') + .addGroupBy('metadata.projectName') + .orderBy(this.escapeField(sortField), sortOrder); + + const count = (await rawRowsQuery.getRawMany()).length; + const rawRows = await rawRowsQuery.offset(skip).limit(take).getRawMany(); + + return { count, rows: aggregatedInsightsByWorkflowParser.parse(rawRows) }; + } + + async getInsightsByTime({ + maxAgeInDays, + periodUnit, + }: { maxAgeInDays: number; periodUnit: PeriodUnit }) { + const cte = sql`SELECT ${this.getAgeLimitQuery(maxAgeInDays)} AS start_date`; + const rawRowsQuery = this.createQueryBuilder() + .addCommonTableExpression(cte, 'date_range') + .select([ + `${this.getPeriodStartExpr(periodUnit)} as "periodStart"`, + `SUM(CASE WHEN type = ${TypeToNumber.runtime_ms} THEN value ELSE 0 END) AS "runTime"`, + `SUM(CASE WHEN type = ${TypeToNumber.success} THEN value ELSE 0 END) AS "succeeded"`, + `SUM(CASE WHEN type = ${TypeToNumber.failure} THEN value ELSE 0 END) AS "failed"`, + `SUM(CASE WHEN type = ${TypeToNumber.time_saved_min} THEN value ELSE 0 END) AS "timeSaved"`, + ]) + .innerJoin('date_range', 'date_range', '1=1') + .where(`${this.escapeField('periodStart')} >= date_range.start_date`) + .addGroupBy(this.getPeriodStartExpr(periodUnit)) + .orderBy(this.getPeriodStartExpr(periodUnit), 'ASC'); + + const rawRows = await rawRowsQuery.getRawMany(); + + return aggregatedInsightsByTimeParser.parse(rawRows); + } } diff --git a/packages/cli/src/modules/insights/insights.controller.ts b/packages/cli/src/modules/insights/insights.controller.ts index 95cfa72836..806de0c754 100644 --- a/packages/cli/src/modules/insights/insights.controller.ts +++ b/packages/cli/src/modules/insights/insights.controller.ts @@ -1,11 +1,17 @@ -import type { InsightsSummary } from '@n8n/api-types'; +import { ListInsightsWorkflowQueryDto } from '@n8n/api-types'; +import type { InsightsSummary, InsightsByTime, InsightsByWorkflow } from '@n8n/api-types'; -import { Get, GlobalScope, RestController } from '@/decorators'; +import { Get, GlobalScope, Query, RestController } from '@/decorators'; +import { paginationListQueryMiddleware } from '@/middlewares/list-query/pagination'; +import { sortByQueryMiddleware } from '@/middlewares/list-query/sort-by'; +import { AuthenticatedRequest } from '@/requests'; import { InsightsService } from './insights.service'; @RestController('/insights') export class InsightsController { + private readonly maxAgeInDaysFilteredInsights = 14; + constructor(private readonly insightsService: InsightsService) {} @Get('/summary') @@ -13,4 +19,28 @@ export class InsightsController { async getInsightsSummary(): Promise { return await this.insightsService.getInsightsSummary(); } + + @Get('/by-workflow', { middlewares: [paginationListQueryMiddleware, sortByQueryMiddleware] }) + @GlobalScope('insights:list') + async getInsightsByWorkflow( + _req: AuthenticatedRequest, + _res: Response, + @Query payload: ListInsightsWorkflowQueryDto, + ): Promise { + return await this.insightsService.getInsightsByWorkflow({ + maxAgeInDays: this.maxAgeInDaysFilteredInsights, + skip: payload.skip, + take: payload.take, + sortBy: payload.sortBy, + }); + } + + @Get('/by-time') + @GlobalScope('insights:list') + async getInsightsByTime(): Promise { + return await this.insightsService.getInsightsByTime({ + maxAgeInDays: this.maxAgeInDaysFilteredInsights, + periodUnit: 'day', + }); + } } diff --git a/packages/cli/src/modules/insights/insights.service.ts b/packages/cli/src/modules/insights/insights.service.ts index 1445998ec4..cdafe8a2b4 100644 --- a/packages/cli/src/modules/insights/insights.service.ts +++ b/packages/cli/src/modules/insights/insights.service.ts @@ -14,7 +14,7 @@ import { OnShutdown } from '@/decorators/on-shutdown'; import { InsightsMetadata } from '@/modules/insights/database/entities/insights-metadata'; import { InsightsRaw } from '@/modules/insights/database/entities/insights-raw'; -import type { TypeUnit } from './database/entities/insights-shared'; +import type { PeriodUnit, TypeUnit } from './database/entities/insights-shared'; import { NumberToType } from './database/entities/insights-shared'; import { InsightsByPeriodRepository } from './database/repositories/insights-by-period.repository'; import { InsightsRawRepository } from './database/repositories/insights-raw.repository'; @@ -300,4 +300,53 @@ export class InsightsService { return result; } + + async getInsightsByWorkflow({ + maxAgeInDays, + skip = 0, + take = 10, + sortBy = 'total:desc', + }: { + maxAgeInDays: number; + skip?: number; + take?: number; + sortBy?: string; + }) { + const { count, rows } = await this.insightsByPeriodRepository.getInsightsByWorkflow({ + maxAgeInDays, + skip, + take, + sortBy, + }); + + return { + count, + data: rows, + }; + } + + async getInsightsByTime({ + maxAgeInDays, + periodUnit, + }: { maxAgeInDays: number; periodUnit: PeriodUnit }) { + const rows = await this.insightsByPeriodRepository.getInsightsByTime({ + maxAgeInDays, + periodUnit, + }); + + return rows.map((r) => { + const total = r.succeeded + r.failed; + return { + date: r.periodStart, + values: { + total, + succeeded: r.succeeded, + failed: r.failed, + failureRate: r.failed / total, + averageRunTime: r.runTime / total, + timeSaved: r.timeSaved, + }, + }; + }); + } } diff --git a/packages/cli/test/integration/insights/insights.api.test.ts b/packages/cli/test/integration/insights/insights.api.test.ts new file mode 100644 index 0000000000..c8b633d438 --- /dev/null +++ b/packages/cli/test/integration/insights/insights.api.test.ts @@ -0,0 +1,78 @@ +import type { User } from '@/databases/entities/user'; +import { Telemetry } from '@/telemetry'; +import { mockInstance } from '@test/mocking'; + +import { createUser } from '../shared/db/users'; +import type { SuperAgentTest } from '../shared/types'; +import * as utils from '../shared/utils'; + +let authOwnerAgent: SuperAgentTest; +let owner: User; +let admin: User; +let member: User; +mockInstance(Telemetry); + +let agents: Record = {}; + +const testServer = utils.setupTestServer({ + endpointGroups: ['insights', 'license', 'auth'], + enabledFeatures: [], +}); + +beforeAll(async () => { + owner = await createUser({ role: 'global:owner' }); + admin = await createUser({ role: 'global:admin' }); + member = await createUser({ role: 'global:member' }); + authOwnerAgent = testServer.authAgentFor(owner); + agents.owner = authOwnerAgent; + agents.admin = testServer.authAgentFor(admin); + agents.member = testServer.authAgentFor(member); +}); + +describe('GET /insights routes work for owner and admins', () => { + test.each(['owner', 'member', 'admin'])( + 'Call should work and return empty summary for user %s', + async (agentName: string) => { + const authAgent = agents[agentName]; + await authAgent.get('/insights/summary').expect(agentName === 'member' ? 403 : 200); + await authAgent.get('/insights/by-time').expect(agentName === 'member' ? 403 : 200); + await authAgent.get('/insights/by-workflow').expect(agentName === 'member' ? 403 : 200); + }, + ); +}); + +describe('GET /insights/by-worklow', () => { + test('Call should work with valid query parameters', async () => { + await authOwnerAgent + .get('/insights/by-workflow') + .query({ skip: '10', take: '20', sortBy: 'total:desc' }) + .expect(200); + }); + + test.each<{ skip: string; take?: string; sortBy?: string }>([ + { + skip: 'not_a_number', + take: '20', + }, + { + skip: '1', + take: 'not_a_number', + }, + ])( + 'Call should return internal server error with invalid pagination query parameters', + async (queryParams) => { + await authOwnerAgent.get('/insights/by-workflow').query(queryParams).expect(500); + }, + ); + + test('Call should return bad request with invalid sortby query parameters', async () => { + await authOwnerAgent + .get('/insights/by-workflow') + .query({ + skip: '1', + take: '20', + sortBy: 'not_a_sortby', + }) + .expect(400); + }); +}); diff --git a/packages/cli/test/integration/shared/test-db.ts b/packages/cli/test/integration/shared/test-db.ts index 8e50f384d7..7cfe6017b6 100644 --- a/packages/cli/test/integration/shared/test-db.ts +++ b/packages/cli/test/integration/shared/test-db.ts @@ -11,13 +11,18 @@ import * as Db from '@/db'; export const testDbPrefix = 'n8n_test_'; +type Extensions = 'insights'; + +let loadedExtensions: Extensions[] = []; + /** * Initialize one test DB per suite run, with bootstrap connection if needed. */ -export async function init() { +export async function init(extensionNames: Extensions[] = []) { const globalConfig = Container.get(GlobalConfig); const dbType = globalConfig.database.type; const testDbName = `${testDbPrefix}${randomString(6, 10).toLowerCase()}_${Date.now()}`; + loadedExtensions = extensionNames; if (dbType === 'postgresdb') { const bootstrapPostgres = await new Connection( @@ -98,16 +103,23 @@ export async function truncate(names: Array<(typeof repositories)[number]>) { for (const name of names) { let RepositoryClass: Class>; - try { - RepositoryClass = (await import(`@/databases/repositories/${kebabCase(name)}.repository`))[ - `${name}Repository` - ]; - } catch (e) { - RepositoryClass = (await import(`@/databases/repositories/${kebabCase(name)}.repository.ee`))[ - `${name}Repository` - ]; + const fileName = `${kebabCase(name)}.repository`; + const paths = [ + `@/databases/repositories/${fileName}.ee`, + `@/databases/repositories/${fileName}`, + ]; + + for (const extension of loadedExtensions) { + paths.push( + `@/modules/${extension}/database/repositories/${fileName}`, + `@/modules/${extension}/database/repositories/${fileName}.ee`, + ); } + RepositoryClass = (await Promise.any(paths.map(async (path) => await import(path))))[ + `${name}Repository` + ]; + await Container.get(RepositoryClass).delete({}); } } diff --git a/packages/cli/test/integration/shared/types.ts b/packages/cli/test/integration/shared/types.ts index d1110f8692..a280f40148 100644 --- a/packages/cli/test/integration/shared/types.ts +++ b/packages/cli/test/integration/shared/types.ts @@ -44,7 +44,8 @@ type EndpointGroup = | 'apiKeys' | 'evaluation' | 'ai' - | 'folder'; + | 'folder' + | 'insights'; export interface SetupProps { endpointGroups?: EndpointGroup[]; diff --git a/packages/cli/test/integration/shared/utils/test-server.ts b/packages/cli/test/integration/shared/utils/test-server.ts index 739a89305c..02ea4c7892 100644 --- a/packages/cli/test/integration/shared/utils/test-server.ts +++ b/packages/cli/test/integration/shared/utils/test-server.ts @@ -290,6 +290,9 @@ export const setupTestServer = ({ case 'folder': await import('@/controllers/folder.controller'); + + case 'insights': + await import('@/modules/insights/insights.controller'); } }