admin/n8n-enterprise-unlocked
1
0
Fork 0
mirror of https://github.com/Abdulazizzn/n8n-enterprise-unlocked.git synced 2025-12-17 01:56:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(core): Update data model for Evaluations (no-changelog) (#15520)

Browse Source 
Co-authored-by: Yiorgis Gozadinos <yiorgis@n8n.io>
Co-authored-by: JP van Oosten <jp@n8n.io>
This commit is contained in:
Eugene
2025-05-22 12:55:31 +02:00
committed by GitHub
parent 07d526e9d6
commit 8152f8c6a7
42 changed files with 512 additions and 3327 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
packages/@n8n/db/src/entities/index.ts
View File

@@ -22,8 +22,6 @@ import { SharedCredentials } from './shared-credentials';
import { SharedWorkflow } from './shared-workflow';
import { TagEntity } from './tag-entity';
import { TestCaseExecution } from './test-case-execution.ee';
import { TestDefinition } from './test-definition.ee';
import { TestMetric } from './test-metric.ee';
import { TestRun } from './test-run.ee';
import { User } from './user';
import { Variables } from './variables';
@@ -63,8 +61,6 @@ export {
AnnotationTagEntity,
ExecutionAnnotation,
AnnotationTagMapping,
TestDefinition,
TestMetric,
TestRun,
TestCaseExecution,
ExecutionEntity,
@@ -100,8 +96,6 @@ export const entities = {
AnnotationTagEntity,
ExecutionAnnotation,
AnnotationTagMapping,
TestDefinition,
TestMetric,
TestRun,
TestCaseExecution,
ExecutionEntity,

20
packages/@n8n/db/src/entities/test-case-execution.ee.ts
View File

@@ -19,7 +19,7 @@ export type TestCaseExecutionStatus =
/**
* This entity represents the linking between the test runs and individual executions.
* It stores status, links to past, new and evaluation executions, and metrics produced by individual evaluation wf executions
* It stores status, link to the evaluation execution, and metrics produced by individual test case
* Entries in this table are meant to outlive the execution entities, which might be pruned over time.
* This allows us to keep track of the details of test runs' status and metrics even after the executions are deleted.
*/
@@ -28,15 +28,6 @@ export class TestCaseExecution extends WithStringId {
@ManyToOne('TestRun')
testRun: TestRun;
@ManyToOne('ExecutionEntity', {
onDelete: 'SET NULL',
nullable: true,
})
pastExecution: ExecutionEntity | null;
@Column({ type: 'varchar', nullable: true })
pastExecutionId: string | null;
@OneToOne('ExecutionEntity', {
onDelete: 'SET NULL',
nullable: true,
@@ -46,15 +37,6 @@ export class TestCaseExecution extends WithStringId {
@Column({ type: 'varchar', nullable: true })
executionId: string | null;
@OneToOne('ExecutionEntity', {
onDelete: 'SET NULL',
nullable: true,
})
evaluationExecution: ExecutionEntity | null;
@Column({ type: 'varchar', nullable: true })
evaluationExecutionId: string | null;
@Column()
status: TestCaseExecutionStatus;

63
packages/@n8n/db/src/entities/test-definition.ee.ts
View File

@@ -1,63 +0,0 @@
import { Column, Entity, Index, ManyToOne, OneToMany, RelationId } from '@n8n/typeorm';
import { Length } from 'class-validator';
import { JsonColumn, WithTimestampsAndStringId } from './abstract-entity';
import { AnnotationTagEntity } from './annotation-tag-entity.ee';
import type { TestMetric } from './test-metric.ee';
import type { MockedNodeItem } from './types-db';
import { WorkflowEntity } from './workflow-entity';
/**
* Entity representing a Test Definition
* It combines:
* - the workflow under test
* - the workflow used to evaluate the results of test execution
* - the filter used to select test cases from previous executions of the workflow under test - annotation tag
*/
@Entity()
@Index(['workflow'])
@Index(['evaluationWorkflow'])
export class TestDefinition extends WithTimestampsAndStringId {
@Column({ length: 255 })
@Length(1, 255, {
message: 'Test definition name must be $constraint1 to $constraint2 characters long.',
})
name: string;
@Column('text')
description: string;
@JsonColumn({ default: '[]' })
mockedNodes: MockedNodeItem[];
/**
* Relation to the workflow under test
*/
@ManyToOne('WorkflowEntity', 'tests')
workflow: WorkflowEntity;
@RelationId((test: TestDefinition) => test.workflow)
workflowId: string;
/**
* Relation to the workflow used to evaluate the results of test execution
*/
@ManyToOne('WorkflowEntity', 'evaluationTests')
evaluationWorkflow: WorkflowEntity;
@RelationId((test: TestDefinition) => test.evaluationWorkflow)
evaluationWorkflowId: string;
/**
* Relation to the annotation tag associated with the test
* This tag will be used to select the test cases to run from previous executions
*/
@ManyToOne('AnnotationTagEntity', 'test')
annotationTag: AnnotationTagEntity;
@RelationId((test: TestDefinition) => test.annotationTag)
annotationTagId: string;
@OneToMany('TestMetric', 'testDefinition')
metrics: TestMetric[];
}

29
packages/@n8n/db/src/entities/test-metric.ee.ts
View File

@@ -1,29 +0,0 @@
import { Column, Entity, Index, ManyToOne } from '@n8n/typeorm';
import { Length } from 'class-validator';
import { WithTimestampsAndStringId } from './abstract-entity';
import { TestDefinition } from './test-definition.ee';
/**
* Entity representing a Test Metric
* It represents a single metric that can be retrieved from evaluation workflow execution result
*/
@Entity()
@Index(['testDefinition'])
export class TestMetric extends WithTimestampsAndStringId {
/**
* Name of the metric.
* This will be used as a property name to extract metric value from the evaluation workflow execution result object
*/
@Column({ length: 255 })
@Length(1, 255, {
message: 'Metric name must be $constraint1 to $constraint2 characters long.',
})
name: string;
/**
* Relation to test definition
*/
@ManyToOne('TestDefinition', 'metrics')
testDefinition: TestDefinition;
}

38
packages/@n8n/db/src/entities/test-run.ee.ts
View File

@@ -1,27 +1,20 @@
import { Column, Entity, Index, ManyToOne, OneToMany, RelationId } from '@n8n/typeorm';
import { Column, Entity, OneToMany, ManyToOne } from '@n8n/typeorm';
import type { IDataObject } from 'n8n-workflow';
import { DateTimeColumn, JsonColumn, WithTimestampsAndStringId } from './abstract-entity';
import type { TestCaseExecution } from './test-case-execution.ee';
import { TestDefinition } from './test-definition.ee';
import { AggregatedTestRunMetrics } from './types-db';
import type { TestRunErrorCode, TestRunFinalResult } from './types-db';
import { WorkflowEntity } from './workflow-entity';
export type TestRunStatus = 'new' | 'running' | 'completed' | 'error' | 'cancelled';
/**
* Entity representing a Test Run.
* It stores info about a specific run of a test, combining the test definition with the status and collected metrics
* It stores info about a specific run of a test, including the status and collected metrics
*/
@Entity()
@Index(['testDefinition'])
export class TestRun extends WithTimestampsAndStringId {
@ManyToOne('TestDefinition', 'runs')
testDefinition: TestDefinition;
@RelationId((testRun: TestRun) => testRun.testDefinition)
testDefinitionId: string;
@Column('varchar')
status: TestRunStatus;
@@ -34,25 +27,6 @@ export class TestRun extends WithTimestampsAndStringId {
@JsonColumn({ nullable: true })
metrics: AggregatedTestRunMetrics;
/**
* Total number of the test cases, matching the filter condition of the test definition (specified annotationTag)
*/
@Column('integer', { nullable: true })
totalCases: number;
/**
* Number of test cases that passed (evaluation workflow was executed successfully)
*/
@Column('integer', { nullable: true })
passedCases: number;
/**
* Number of failed test cases
* (any unexpected exception happened during the execution or evaluation workflow ended with an error)
*/
@Column('integer', { nullable: true })
failedCases: number;
/**
* This will contain the error code if the test run failed.
* This is used for test run level errors, not for individual test case errors.
@@ -69,6 +43,12 @@ export class TestRun extends WithTimestampsAndStringId {
@OneToMany('TestCaseExecution', 'testRun')
testCaseExecutions: TestCaseExecution[];
@ManyToOne('WorkflowEntity')
workflow: WorkflowEntity;
@Column('varchar', { length: 255 })
workflowId: string;
/**
* Calculated property to determine the final result of the test run
* depending on the statuses of test case executions

4
packages/@n8n/db/src/entities/workflow-entity.ts
View File

@@ -16,6 +16,7 @@ import { JsonColumn, WithTimestampsAndStringId, dbType } from './abstract-entity
import { type Folder } from './folder';
import type { SharedWorkflow } from './shared-workflow';
import type { TagEntity } from './tag-entity';
import type { TestRun } from './test-run.ee';
import type { IWorkflowDb } from './types-db';
import type { WorkflowStatistics } from './workflow-statistics';
import type { WorkflowTagMapping } from './workflow-tag-mapping';
@@ -108,6 +109,9 @@ export class WorkflowEntity extends WithTimestampsAndStringId implements IWorkfl
})
@JoinColumn({ name: 'parentFolderId' })
parentFolder: Folder | null;
@OneToMany('TestRun', 'workflow')
testRuns: TestRun[];
}
/**

68
packages/@n8n/db/src/migrations/common/1745322634000-CleanEvaluations.ts Normal file
View File

@@ -0,0 +1,68 @@
import type { MigrationContext, IrreversibleMigration } from '../migration-types';
const testRunTableName = 'test_run';
const testCaseExecutionTableName = 'test_case_execution';
export class ClearEvaluation1745322634000 implements IrreversibleMigration {
async up({
schemaBuilder: { dropTable, column, createTable },
queryRunner,
tablePrefix,
isSqlite,
isPostgres,
isMysql,
}: MigrationContext) {
// Drop test_metric, test_definition
await dropTable(testCaseExecutionTableName);
await dropTable(testRunTableName);
await dropTable('test_metric');
if (isSqlite) {
await queryRunner.query(`DROP TABLE IF EXISTS ${tablePrefix}test_definition;`);
} else if (isPostgres) {
await queryRunner.query(`DROP TABLE IF EXISTS ${tablePrefix}test_definition CASCADE;`);
} else if (isMysql) {
await queryRunner.query(`DROP TABLE IF EXISTS ${tablePrefix}test_definition CASCADE;`);
}
await createTable(testRunTableName)
.withColumns(
column('id').varchar(36).primary.notNull,
column('workflowId').varchar(36).notNull,
column('status').varchar().notNull,
column('errorCode').varchar(),
column('errorDetails').json,
column('runAt').timestamp(),
column('completedAt').timestamp(),
column('metrics').json,
)
.withIndexOn('workflowId')
.withForeignKey('workflowId', {
tableName: 'workflow_entity',
columnName: 'id',
onDelete: 'CASCADE',
}).withTimestamps;
await createTable(testCaseExecutionTableName)
.withColumns(
column('id').varchar(36).primary.notNull,
column('testRunId').varchar(36).notNull,
column('executionId').int, // Execution of the workflow under test. Might be null if execution was deleted after the test run
column('status').varchar().notNull,
column('runAt').timestamp(),
column('completedAt').timestamp(),
column('errorCode').varchar(),
column('errorDetails').json,
column('metrics').json,
)
.withIndexOn('testRunId')
.withForeignKey('testRunId', {
tableName: 'test_run',
columnName: 'id',
onDelete: 'CASCADE',
})
.withForeignKey('executionId', {
tableName: 'execution_entity',
columnName: 'id',
onDelete: 'SET NULL',
}).withTimestamps;
}
}

2
packages/@n8n/db/src/migrations/mysqldb/index.ts
View File

@@ -82,6 +82,7 @@ import { CreateFolderTable1738709609940 } from '../common/1738709609940-CreateFo
import { CreateAnalyticsTables1739549398681 } from '../common/1739549398681-CreateAnalyticsTables';
import { RenameAnalyticsToInsights1741167584277 } from '../common/1741167584277-RenameAnalyticsToInsights';
import { AddScopesColumnToApiKeys1742918400000 } from '../common/1742918400000-AddScopesColumnToApiKeys';
import { ClearEvaluation1745322634000 } from '../common/1745322634000-CleanEvaluations';
import { AddWorkflowStatisticsRootCount1745587087521 } from '../common/1745587087521-AddWorkflowStatisticsRootCount';
import { AddWorkflowArchivedColumn1745934666076 } from '../common/1745934666076-AddWorkflowArchivedColumn';
import { DropRoleTable1745934666077 } from '../common/1745934666077-DropRoleTable';
@@ -177,4 +178,5 @@ export const mysqlMigrations: Migration[] = [
AddWorkflowStatisticsRootCount1745587087521,
AddWorkflowArchivedColumn1745934666076,
DropRoleTable1745934666077,
ClearEvaluation1745322634000,
];

2
packages/@n8n/db/src/migrations/postgresdb/index.ts
View File

@@ -82,6 +82,7 @@ import { CreateFolderTable1738709609940 } from '../common/1738709609940-CreateFo
import { CreateAnalyticsTables1739549398681 } from '../common/1739549398681-CreateAnalyticsTables';
import { RenameAnalyticsToInsights1741167584277 } from '../common/1741167584277-RenameAnalyticsToInsights';
import { AddScopesColumnToApiKeys1742918400000 } from '../common/1742918400000-AddScopesColumnToApiKeys';
import { ClearEvaluation1745322634000 } from '../common/1745322634000-CleanEvaluations';
import { AddWorkflowStatisticsRootCount1745587087521 } from '../common/1745587087521-AddWorkflowStatisticsRootCount';
import { AddWorkflowArchivedColumn1745934666076 } from '../common/1745934666076-AddWorkflowArchivedColumn';
import { DropRoleTable1745934666077 } from '../common/1745934666077-DropRoleTable';
@@ -175,4 +176,5 @@ export const postgresMigrations: Migration[] = [
AddWorkflowStatisticsRootCount1745587087521,
AddWorkflowArchivedColumn1745934666076,
DropRoleTable1745934666077,
ClearEvaluation1745322634000,
];

2
packages/@n8n/db/src/migrations/sqlite/index.ts
View File

@@ -79,6 +79,7 @@ import { CreateTestCaseExecutionTable1736947513045 } from '../common/17369475130
import { AddErrorColumnsToTestRuns1737715421462 } from '../common/1737715421462-AddErrorColumnsToTestRuns';
import { CreateAnalyticsTables1739549398681 } from '../common/1739549398681-CreateAnalyticsTables';
import { RenameAnalyticsToInsights1741167584277 } from '../common/1741167584277-RenameAnalyticsToInsights';
import { ClearEvaluation1745322634000 } from '../common/1745322634000-CleanEvaluations';
import { AddWorkflowStatisticsRootCount1745587087521 } from '../common/1745587087521-AddWorkflowStatisticsRootCount';
import { AddWorkflowArchivedColumn1745934666076 } from '../common/1745934666076-AddWorkflowArchivedColumn';
import { DropRoleTable1745934666077 } from '../common/1745934666077-DropRoleTable';
@@ -169,6 +170,7 @@ const sqliteMigrations: Migration[] = [
AddWorkflowStatisticsRootCount1745587087521,
AddWorkflowArchivedColumn1745934666076,
DropRoleTable1745934666077,
ClearEvaluation1745322634000,
];
export { sqliteMigrations };

2
packages/@n8n/db/src/repositories/index.ts
View File

@@ -21,8 +21,6 @@ export { ProcessedDataRepository } from './processed-data.repository';
export { SettingsRepository } from './settings.repository';
export { TagRepository } from './tag.repository';
export { TestCaseExecutionRepository } from './test-case-execution.repository.ee';
export { TestDefinitionRepository } from './test-definition.repository.ee';
export { TestMetricRepository } from './test-metric.repository.ee';
export { TestRunRepository } from './test-run.repository.ee';
export { VariablesRepository } from './variables.repository';
export { WorkflowHistoryRepository } from './workflow-history.repository';

48
packages/@n8n/db/src/repositories/test-case-execution.repository.ee.ts
View File

@@ -18,16 +18,10 @@ type MarkAsFailedOptions = StatusUpdateOptions & {
errorDetails?: IDataObject;
};
type MarkAsWarningOptions = MarkAsFailedOptions;
type MarkAsRunningOptions = StatusUpdateOptions & {
executionId: string;
};
type MarkAsEvaluationRunningOptions = StatusUpdateOptions & {
evaluationExecutionId: string;
};
type MarkAsCompletedOptions = StatusUpdateOptions & {
metrics: Record<string, number>;
};
@@ -38,15 +32,12 @@ export class TestCaseExecutionRepository extends Repository<TestCaseExecution> {
super(TestCaseExecution, dataSource.manager);
}
async createBatch(testRunId: string, pastExecutionIds: string[]) {
async createBatch(testRunId: string, testCases: string[]) {
const mappings = this.create(
pastExecutionIds.map<DeepPartial<TestCaseExecution>>((id) => ({
testCases.map<DeepPartial<TestCaseExecution>>(() => ({
testRun: {
id: testRunId,
},
pastExecution: {
id,
},
status: 'new',
})),
);
@@ -68,24 +59,6 @@ export class TestCaseExecutionRepository extends Repository<TestCaseExecution> {
);
}
async markAsEvaluationRunning({
testRunId,
pastExecutionId,
evaluationExecutionId,
trx,
}: MarkAsEvaluationRunningOptions) {
trx = trx ?? this.manager;
return await trx.update(
TestCaseExecution,
{ testRun: { id: testRunId }, pastExecutionId },
{
status: 'evaluation_running',
evaluationExecutionId,
},
);
}
async markAsCompleted({ testRunId, pastExecutionId, metrics, trx }: MarkAsCompletedOptions) {
trx = trx ?? this.manager;
@@ -133,21 +106,4 @@ export class TestCaseExecutionRepository extends Repository<TestCaseExecution> {
},
);
}
async markAsWarning({
testRunId,
pastExecutionId,
errorCode,
errorDetails,
}: MarkAsWarningOptions) {
return await this.update(
{ testRun: { id: testRunId }, pastExecutionId },
{
status: 'warning',
completedAt: new Date(),
errorCode,
errorDetails,
},
);
}
}

70
packages/@n8n/db/src/repositories/test-definition.repository.ee.ts
View File

@@ -1,70 +0,0 @@
import { Service } from '@n8n/di';
import type { FindManyOptions, FindOptionsWhere } from '@n8n/typeorm';
import { DataSource, In, Repository } from '@n8n/typeorm';
import { UserError } from 'n8n-workflow';
import { TestDefinition } from '../entities';
import type { ListQuery } from '../entities/types-db';
@Service()
export class TestDefinitionRepository extends Repository<TestDefinition> {
constructor(dataSource: DataSource) {
super(TestDefinition, dataSource.manager);
}
async getMany(accessibleWorkflowIds: string[], options?: ListQuery.Options) {
if (accessibleWorkflowIds.length === 0) return { tests: [], count: 0 };
const where: FindOptionsWhere<TestDefinition> = {};
if (options?.filter?.workflowId) {
if (!accessibleWorkflowIds.includes(options.filter.workflowId as string)) {
throw new UserError('User does not have access to the workflow');
}
where.workflow = {
id: options.filter.workflowId as string,
};
} else {
where.workflow = {
id: In(accessibleWorkflowIds),
};
}
const findManyOptions: FindManyOptions<TestDefinition> = {
where,
relations: ['annotationTag'],
order: { createdAt: 'DESC' },
};
if (options?.take) {
findManyOptions.skip = options.skip;
findManyOptions.take = options.take;
}
const [testDefinitions, count] = await this.findAndCount(findManyOptions);
return { testDefinitions, count };
}
async getOne(id: string, accessibleWorkflowIds: string[]) {
return await this.findOne({
where: {
id,
workflow: {
id: In(accessibleWorkflowIds),
},
},
relations: ['annotationTag', 'metrics'],
});
}
async deleteById(id: string, accessibleWorkflowIds: string[]) {
return await this.delete({
id,
workflow: {
id: In(accessibleWorkflowIds),
},
});
}
}

11
packages/@n8n/db/src/repositories/test-metric.repository.ee.ts
View File

@@ -1,11 +0,0 @@
import { Service } from '@n8n/di';
import { DataSource, Repository } from '@n8n/typeorm';
import { TestMetric } from '../entities';
@Service()
export class TestMetricRepository extends Repository<TestMetric> {
constructor(dataSource: DataSource) {
super(TestMetric, dataSource.manager);
}
}

32
packages/@n8n/db/src/repositories/test-run.repository.ee.ts
View File

@@ -22,22 +22,21 @@ export class TestRunRepository extends Repository<TestRun> {
super(TestRun, dataSource.manager);
}
async createTestRun(testDefinitionId: string) {
async createTestRun(workflowId: string) {
const testRun = this.create({
status: 'new',
testDefinition: { id: testDefinitionId },
workflow: {
id: workflowId,
},
});
return await this.save(testRun);
}
async markAsRunning(id: string, totalCases: number) {
async markAsRunning(id: string) {
return await this.update(id, {
status: 'running',
runAt: new Date(),
totalCases,
passedCases: 0,
failedCases: 0,
});
}
@@ -65,20 +64,10 @@ export class TestRunRepository extends Repository<TestRun> {
);
}
async incrementPassed(id: string, trx?: EntityManager) {
trx = trx ?? this.manager;
return await trx.increment(TestRun, { id }, 'passedCases', 1);
}
async incrementFailed(id: string, trx?: EntityManager) {
trx = trx ?? this.manager;
return await trx.increment(TestRun, { id }, 'failedCases', 1);
}
async getMany(testDefinitionId: string, options: ListQuery.Options) {
async getMany(workflowId: string, options: ListQuery.Options) {
// FIXME: optimize fetching final result of each test run
const findManyOptions: FindManyOptions<TestRun> = {
where: { testDefinition: { id: testDefinitionId } },
where: { workflow: { id: workflowId } },
order: { createdAt: 'DESC' },
relations: ['testCaseExecutions'],
};
@@ -103,12 +92,9 @@ export class TestRunRepository extends Repository<TestRun> {
* E.g. Test Run is considered successful if all test case executions are successful.
* Test Run is considered failed if at least one test case execution is failed.
*/
async getTestRunSummaryById(
testDefinitionId: string,
testRunId: string,
): Promise<TestRunSummary> {
async getTestRunSummaryById(testRunId: string): Promise<TestRunSummary> {
const testRun = await this.findOne({
where: { id: testRunId, testDefinition: { id: testDefinitionId } },
where: { id: testRunId },
relations: ['testCaseExecutions'],
});