feat(core): Keep track of test case executions during test run (no-changelog) (#12787)

packages/cli/src/databases/entities/index.ts

@@ -20,6 +20,7 @@ import { Settings } from './settings';
 import { SharedCredentials } from './shared-credentials';
 import { SharedWorkflow } from './shared-workflow';
 import { TagEntity } from './tag-entity';
+import { TestCaseExecution } from './test-case-execution.ee';
 import { TestDefinition } from './test-definition.ee';
 import { TestMetric } from './test-metric.ee';
 import { TestRun } from './test-run.ee';
@@ -64,4 +65,5 @@ export const entities = {
 	TestDefinition,
 	TestMetric,
 	TestRun,
+	TestCaseExecution,
 };

packages/cli/src/databases/entities/test-case-execution.ee.ts

@@ -0,0 +1,68 @@
+import { Column, Entity, ManyToOne, OneToOne } from '@n8n/typeorm';
+
+import {
+	datetimeColumnType,
+	jsonColumnType,
+	WithStringId,
+} from '@/databases/entities/abstract-entity';
+import type { ExecutionEntity } from '@/databases/entities/execution-entity';
+import { TestRun } from '@/databases/entities/test-run.ee';
+
+export type TestCaseRunMetrics = Record<string, number | boolean>;
+
+/**
+ * This entity represents the linking between the test runs and individual executions.
+ * It stores status, links to past, new and evaluation executions, and metrics produced by individual evaluation wf executions
+ * Entries in this table are meant to outlive the execution entities, which might be pruned over time.
+ * This allows us to keep track of the details of test runs' status and metrics even after the executions are deleted.
+ */
+@Entity({ name: 'test_case_execution' })
+export class TestCaseExecution extends WithStringId {
+	@ManyToOne('TestRun')
+	testRun: TestRun;
+
+	@ManyToOne('ExecutionEntity', {
+		onDelete: 'SET NULL',
+		nullable: true,
+	})
+	pastExecution: ExecutionEntity | null;
+
+	@Column({ type: 'varchar', nullable: true })
+	pastExecutionId: string | null;
+
+	@OneToOne('ExecutionEntity', {
+		onDelete: 'SET NULL',
+		nullable: true,
+	})
+	execution: ExecutionEntity | null;
+
+	@Column({ type: 'varchar', nullable: true })
+	executionId: string | null;
+
+	@OneToOne('ExecutionEntity', {
+		onDelete: 'SET NULL',
+		nullable: true,
+	})
+	evaluationExecution: ExecutionEntity | null;
+
+	@Column({ type: 'varchar', nullable: true })
+	evaluationExecutionId: string | null;
+
+	@Column()
+	status: 'new' | 'running' | 'evaluation_running' | 'success' | 'error' | 'cancelled';
+
+	@Column({ type: datetimeColumnType, nullable: true })
+	runAt: Date | null;
+
+	@Column({ type: datetimeColumnType, nullable: true })
+	completedAt: Date | null;
+
+	@Column('varchar', { nullable: true })
+	errorCode: string | null;
+
+	@Column(jsonColumnType, { nullable: true })
+	errorDetails: Record<string, unknown>;
+
+	@Column(jsonColumnType, { nullable: true })
+	metrics: TestCaseRunMetrics;
+}

packages/cli/src/databases/migrations/common/1736947513045-CreateTestCaseExecutionTable.ts

@@ -0,0 +1,47 @@
+import type { MigrationContext, ReversibleMigration } from '@/databases/types';
+
+const testCaseExecutionTableName = 'test_case_execution';
+
+export class CreateTestCaseExecutionTable1736947513045 implements ReversibleMigration {
+	async up({ schemaBuilder: { createTable, column } }: MigrationContext) {
+		await createTable(testCaseExecutionTableName)
+			.withColumns(
+				column('id').varchar(36).primary.notNull,
+				column('testRunId').varchar(36).notNull,
+				column('pastExecutionId').int, // Might be null if execution was deleted after the test run
+				column('executionId').int, // Execution of the workflow under test. Might be null if execution was deleted after the test run
+				column('evaluationExecutionId').int, // Execution of the evaluation workflow. Might be null if execution was deleted after the test run, or if the test run was cancelled
+				column('status').varchar().notNull,
+				column('runAt').timestamp(),
+				column('completedAt').timestamp(),
+				column('errorCode').varchar(),
+				column('errorDetails').json,
+				column('metrics').json,
+			)
+			.withIndexOn('testRunId')
+			.withForeignKey('testRunId', {
+				tableName: 'test_run',
+				columnName: 'id',
+				onDelete: 'CASCADE',
+			})
+			.withForeignKey('pastExecutionId', {
+				tableName: 'execution_entity',
+				columnName: 'id',
+				onDelete: 'SET NULL',
+			})
+			.withForeignKey('executionId', {
+				tableName: 'execution_entity',
+				columnName: 'id',
+				onDelete: 'SET NULL',
+			})
+			.withForeignKey('evaluationExecutionId', {
+				tableName: 'execution_entity',
+				columnName: 'id',
+				onDelete: 'SET NULL',
+			}).withTimestamps;
+	}
+
+	async down({ schemaBuilder: { dropTable } }: MigrationContext) {
+		await dropTable(testCaseExecutionTableName);
+	}
+}

packages/cli/src/databases/migrations/mysqldb/index.ts

@@ -77,6 +77,7 @@ import { CreateTestRun1732549866705 } from '../common/1732549866705-CreateTestRu
 import { AddMockedNodesColumnToTestDefinition1733133775640 } from '../common/1733133775640-AddMockedNodesColumnToTestDefinition';
 import { AddManagedColumnToCredentialsTable1734479635324 } from '../common/1734479635324-AddManagedColumnToCredentialsTable';
 import { AddStatsColumnsToTestRun1736172058779 } from '../common/1736172058779-AddStatsColumnsToTestRun';
+import { CreateTestCaseExecutionTable1736947513045 } from '../common/1736947513045-CreateTestCaseExecutionTable';
 
 export const mysqlMigrations: Migration[] = [
 	InitialMigration1588157391238,
@@ -156,4 +157,5 @@ export const mysqlMigrations: Migration[] = [
 	AddManagedColumnToCredentialsTable1734479635324,
 	AddProjectIcons1729607673469,
 	AddStatsColumnsToTestRun1736172058779,
+	CreateTestCaseExecutionTable1736947513045,
 ];

packages/cli/src/databases/migrations/postgresdb/index.ts

@@ -77,6 +77,7 @@ import { CreateTestRun1732549866705 } from '../common/1732549866705-CreateTestRu
 import { AddMockedNodesColumnToTestDefinition1733133775640 } from '../common/1733133775640-AddMockedNodesColumnToTestDefinition';
 import { AddManagedColumnToCredentialsTable1734479635324 } from '../common/1734479635324-AddManagedColumnToCredentialsTable';
 import { AddStatsColumnsToTestRun1736172058779 } from '../common/1736172058779-AddStatsColumnsToTestRun';
+import { CreateTestCaseExecutionTable1736947513045 } from '../common/1736947513045-CreateTestCaseExecutionTable';
 
 export const postgresMigrations: Migration[] = [
 	InitialMigration1587669153312,
@@ -156,4 +157,5 @@ export const postgresMigrations: Migration[] = [
 	AddManagedColumnToCredentialsTable1734479635324,
 	AddProjectIcons1729607673469,
 	AddStatsColumnsToTestRun1736172058779,
+	CreateTestCaseExecutionTable1736947513045,
 ];

packages/cli/src/databases/migrations/sqlite/index.ts

@@ -74,6 +74,7 @@ import { CreateTestRun1732549866705 } from '../common/1732549866705-CreateTestRu
 import { AddMockedNodesColumnToTestDefinition1733133775640 } from '../common/1733133775640-AddMockedNodesColumnToTestDefinition';
 import { AddManagedColumnToCredentialsTable1734479635324 } from '../common/1734479635324-AddManagedColumnToCredentialsTable';
 import { AddStatsColumnsToTestRun1736172058779 } from '../common/1736172058779-AddStatsColumnsToTestRun';
+import { CreateTestCaseExecutionTable1736947513045 } from '../common/1736947513045-CreateTestCaseExecutionTable';
 
 const sqliteMigrations: Migration[] = [
 	InitialMigration1588102412422,
@@ -150,6 +151,7 @@ const sqliteMigrations: Migration[] = [
 	AddManagedColumnToCredentialsTable1734479635324,
 	AddProjectIcons1729607673469,
 	AddStatsColumnsToTestRun1736172058779,
+	CreateTestCaseExecutionTable1736947513045,
 ];
 
 export { sqliteMigrations };

packages/cli/src/databases/repositories/test-case-execution.repository.ee.ts

@@ -0,0 +1,99 @@
+import { Service } from '@n8n/di';
+import type { EntityManager } from '@n8n/typeorm';
+import { DataSource, In, Not, Repository } from '@n8n/typeorm';
+import type { DeepPartial } from '@n8n/typeorm/common/DeepPartial';
+
+import { TestCaseExecution } from '@/databases/entities/test-case-execution.ee';
+
+@Service()
+export class TestCaseExecutionRepository extends Repository<TestCaseExecution> {
+	constructor(dataSource: DataSource) {
+		super(TestCaseExecution, dataSource.manager);
+	}
+
+	async createBatch(testRunId: string, pastExecutionIds: string[]) {
+		const mappings = this.create(
+			pastExecutionIds.map<DeepPartial<TestCaseExecution>>((id) => ({
+				testRun: {
+					id: testRunId,
+				},
+				pastExecution: {
+					id,
+				},
+				status: 'new',
+			})),
+		);
+
+		return await this.save(mappings);
+	}
+
+	async markAsRunning(testRunId: string, pastExecutionId: string, executionId: string) {
+		return await this.update(
+			{ testRun: { id: testRunId }, pastExecutionId },
+			{
+				status: 'running',
+				executionId,
+				runAt: new Date(),
+			},
+		);
+	}
+
+	async markAsEvaluationRunning(
+		testRunId: string,
+		pastExecutionId: string,
+		evaluationExecutionId: string,
+	) {
+		return await this.update(
+			{ testRun: { id: testRunId }, pastExecutionId },
+			{
+				status: 'evaluation_running',
+				evaluationExecutionId,
+			},
+		);
+	}
+
+	async markAsCompleted(
+		testRunId: string,
+		pastExecutionId: string,
+		metrics: Record<string, number>,
+		trx?: EntityManager,
+	) {
+		trx = trx ?? this.manager;
+
+		return await trx.update(
+			TestCaseExecution,
+			{ testRun: { id: testRunId }, pastExecutionId },
+			{
+				status: 'success',
+				completedAt: new Date(),
+				metrics,
+			},
+		);
+	}
+
+	async markAllPendingAsCancelled(testRunId: string, trx?: EntityManager) {
+		trx = trx ?? this.manager;
+
+		return await trx.update(
+			TestCaseExecution,
+			{ testRun: { id: testRunId }, status: Not(In(['success', 'error', 'cancelled'])) },
+			{
+				status: 'cancelled',
+				completedAt: new Date(),
+			},
+		);
+	}
+
+	async markAsFailed(testRunId: string, pastExecutionId: string, trx?: EntityManager) {
+		trx = trx ?? this.manager;
+
+		return await trx.update(
+			TestCaseExecution,
+			{ testRun: { id: testRunId }, pastExecutionId },
+			{
+				status: 'error',
+				completedAt: new Date(),
+			},
+		);
+	}
+}

packages/cli/src/databases/repositories/test-run.repository.ee.ts

@@ -1,5 +1,5 @@
 import { Service } from '@n8n/di';
-import type { FindManyOptions } from '@n8n/typeorm';
+import type { EntityManager, FindManyOptions } from '@n8n/typeorm';
 import { DataSource, Repository } from '@n8n/typeorm';
 
 import type { AggregatedTestRunMetrics } from '@/databases/entities/test-run.ee';
@@ -35,16 +35,19 @@ export class TestRunRepository extends Repository<TestRun> {
 		return await this.update(id, { status: 'completed', completedAt: new Date(), metrics });
 	}
 
-	async markAsCancelled(id: string) {
-		return await this.update(id, { status: 'cancelled' });
+	async markAsCancelled(id: string, trx?: EntityManager) {
+		trx = trx ?? this.manager;
+		return await trx.update(TestRun, id, { status: 'cancelled' });
 	}
 
-	async incrementPassed(id: string) {
-		return await this.increment({ id }, 'passedCases', 1);
+	async incrementPassed(id: string, trx?: EntityManager) {
+		trx = trx ?? this.manager;
+		return await trx.increment(TestRun, { id }, 'passedCases', 1);
 	}
 
-	async incrementFailed(id: string) {
-		return await this.increment({ id }, 'failedCases', 1);
+	async incrementFailed(id: string, trx?: EntityManager) {
+		trx = trx ?? this.manager;
+		return await trx.increment(TestRun, { id }, 'failedCases', 1);
 	}
 
 	async getMany(testDefinitionId: string, options: ListQuery.Options) {

packages/cli/src/evaluation.ee/test-definitions.types.ee.ts

@@ -94,4 +94,6 @@ export declare namespace TestRunsRequest {
 	type Delete = AuthenticatedRequest<RouteParams.TestId & RouteParams.TestRunId>;
 
 	type Cancel = AuthenticatedRequest<RouteParams.TestId & RouteParams.TestRunId>;
+
+	type GetCases = AuthenticatedRequest<RouteParams.TestId & RouteParams.TestRunId>;
 }

packages/cli/src/evaluation.ee/test-runner/__tests__/test-runner.service.ee.test.ts

@@ -13,6 +13,7 @@ import type { TestMetric } from '@/databases/entities/test-metric.ee';
 import type { TestRun } from '@/databases/entities/test-run.ee';
 import type { User } from '@/databases/entities/user';
 import type { ExecutionRepository } from '@/databases/repositories/execution.repository';
+import type { TestCaseExecutionRepository } from '@/databases/repositories/test-case-execution.repository.ee';
 import type { TestMetricRepository } from '@/databases/repositories/test-metric.repository.ee';
 import type { TestRunRepository } from '@/databases/repositories/test-run.repository.ee';
 import type { WorkflowRepository } from '@/databases/repositories/workflow.repository';
@@ -25,6 +26,10 @@ import { mockNodeTypesData } from '@test-integration/utils/node-types-data';
 
 import { TestRunnerService } from '../test-runner.service.ee';
 
+jest.mock('@/db', () => ({
+	transaction: (cb: any) => cb(),
+}));
+
 const wfUnderTestJson = JSON.parse(
 	readFileSync(path.join(__dirname, './mock-data/workflow.under-test.json'), { encoding: 'utf-8' }),
 );
@@ -147,6 +152,7 @@ describe('TestRunnerService', () => {
 	const activeExecutions = mock<ActiveExecutions>();
 	const testRunRepository = mock<TestRunRepository>();
 	const testMetricRepository = mock<TestMetricRepository>();
+	const testCaseExecutionRepository = mock<TestCaseExecutionRepository>();
 
 	const mockNodeTypes = mockInstance(NodeTypes);
 	mockInstance(LoadNodesAndCredentials, {
@@ -190,6 +196,7 @@ describe('TestRunnerService', () => {
 			executionRepository,
 			activeExecutions,
 			testRunRepository,
+			testCaseExecutionRepository,
 			testMetricRepository,
 			mockNodeTypes,
 			errorReporter,
@@ -207,6 +214,7 @@ describe('TestRunnerService', () => {
 			executionRepository,
 			activeExecutions,
 			testRunRepository,
+			testCaseExecutionRepository,
 			testMetricRepository,
 			mockNodeTypes,
 			errorReporter,
@@ -247,6 +255,7 @@ describe('TestRunnerService', () => {
 			executionRepository,
 			activeExecutions,
 			testRunRepository,
+			testCaseExecutionRepository,
 			testMetricRepository,
 			mockNodeTypes,
 			errorReporter,
@@ -350,6 +359,7 @@ describe('TestRunnerService', () => {
 			executionRepository,
 			activeExecutions,
 			testRunRepository,
+			testCaseExecutionRepository,
 			testMetricRepository,
 			mockNodeTypes,
 			errorReporter,
@@ -410,6 +420,7 @@ describe('TestRunnerService', () => {
 			executionRepository,
 			activeExecutions,
 			testRunRepository,
+			testCaseExecutionRepository,
 			testMetricRepository,
 			mockNodeTypes,
 			errorReporter,
@@ -466,6 +477,7 @@ describe('TestRunnerService', () => {
 			executionRepository,
 			activeExecutions,
 			testRunRepository,
+			testCaseExecutionRepository,
 			testMetricRepository,
 			mockNodeTypes,
 			errorReporter,
@@ -526,6 +538,7 @@ describe('TestRunnerService', () => {
 			executionRepository,
 			activeExecutions,
 			testRunRepository,
+			testCaseExecutionRepository,
 			testMetricRepository,
 			mockNodeTypes,
 			errorReporter,
@@ -602,6 +615,7 @@ describe('TestRunnerService', () => {
 			executionRepository,
 			activeExecutions,
 			testRunRepository,
+			testCaseExecutionRepository,
 			testMetricRepository,
 			mockNodeTypes,
 			errorReporter,
@@ -629,6 +643,7 @@ describe('TestRunnerService', () => {
 			executionRepository,
 			activeExecutions,
 			testRunRepository,
+			testCaseExecutionRepository,
 			testMetricRepository,
 			mockNodeTypes,
 			errorReporter,
@@ -661,9 +676,10 @@ describe('TestRunnerService', () => {
 				executionRepository,
 				activeExecutions,
 				testRunRepository,
+				testCaseExecutionRepository,
 				testMetricRepository,
 				mockNodeTypes,
-				mock<ErrorReporter>(),
+				errorReporter,
 			);
 
 			workflowRepository.findById.calledWith('workflow-under-test-id').mockResolvedValueOnce({

packages/cli/src/evaluation.ee/test-runner/evaluation-metrics.ee.ts

@@ -9,12 +9,17 @@ export class EvaluationMetrics {
 		}
 	}
 
-	addResults(result: IDataObject) {
+	addResults(result: IDataObject): Record<string, number> {
+		const addedMetrics: Record<string, number> = {};
+
 		for (const [metricName, metricValue] of Object.entries(result)) {
 			if (typeof metricValue === 'number' && this.metricNames.has(metricName)) {
+				addedMetrics[metricName] = metricValue;
 				this.rawMetricsByName.get(metricName)!.push(metricValue);
 			}
 		}
+
+		return addedMetrics;
 	}
 
 	getAggregatedMetrics() {

packages/cli/src/evaluation.ee/test-runner/test-runner.service.ee.ts

@@ -19,9 +19,11 @@ import type { TestRun } from '@/databases/entities/test-run.ee';
 import type { User } from '@/databases/entities/user';
 import type { WorkflowEntity } from '@/databases/entities/workflow-entity';
 import { ExecutionRepository } from '@/databases/repositories/execution.repository';
+import { TestCaseExecutionRepository } from '@/databases/repositories/test-case-execution.repository.ee';
 import { TestMetricRepository } from '@/databases/repositories/test-metric.repository.ee';
 import { TestRunRepository } from '@/databases/repositories/test-run.repository.ee';
 import { WorkflowRepository } from '@/databases/repositories/workflow.repository';
+import * as Db from '@/db';
 import { NodeTypes } from '@/node-types';
 import { Telemetry } from '@/telemetry';
 import { getRunData } from '@/workflow-execute-additional-data';
@@ -30,6 +32,15 @@ import { WorkflowRunner } from '@/workflow-runner';
 import { EvaluationMetrics } from './evaluation-metrics.ee';
 import { createPinData, getPastExecutionTriggerNode } from './utils.ee';
 
+interface TestRunMetadata {
+	testRunId: string;
+	userId: string;
+}
+
+interface TestCaseRunMetadata extends TestRunMetadata {
+	pastExecutionId: string;
+}
+
 /**
  * This service orchestrates the running of test cases.
  * It uses the test definitions to find
@@ -50,6 +61,7 @@ export class TestRunnerService {
 		private readonly executionRepository: ExecutionRepository,
 		private readonly activeExecutions: ActiveExecutions,
 		private readonly testRunRepository: TestRunRepository,
+		private readonly testCaseExecutionRepository: TestCaseExecutionRepository,
 		private readonly testMetricRepository: TestMetricRepository,
 		private readonly nodeTypes: NodeTypes,
 		private readonly errorReporter: ErrorReporter,
@@ -105,7 +117,7 @@ export class TestRunnerService {
 		pastExecutionData: IRunExecutionData,
 		pastExecutionWorkflowData: IWorkflowBase,
 		mockedNodes: MockedNodeItem[],
-		userId: string,
+		metadata: TestCaseRunMetadata,
 		abortSignal: AbortSignal,
 	): Promise<IRun | undefined> {
 		// Do not run if the test run is cancelled
@@ -128,7 +140,7 @@ export class TestRunnerService {
 			runData: {},
 			pinData,
 			workflowData: { ...workflow, pinData },
-			userId,
+			userId: metadata.userId,
 			partialExecutionVersion: '1',
 		};
 
@@ -141,6 +153,13 @@ export class TestRunnerService {
 			this.activeExecutions.stopExecution(executionId);
 		});
 
+		// Update status of the test run execution mapping
+		await this.testCaseExecutionRepository.markAsRunning(
+			metadata.testRunId,
+			metadata.pastExecutionId,
+			executionId,
+		);
+
 		// Wait for the execution to finish
 		const executePromise = this.activeExecutions.getPostExecutePromise(executionId);
 
@@ -155,7 +174,7 @@ export class TestRunnerService {
 		expectedData: IRunData,
 		actualData: IRunData,
 		abortSignal: AbortSignal,
-		testRunId?: string,
+		metadata: TestCaseRunMetadata,
 	) {
 		// Do not run if the test run is cancelled
 		if (abortSignal.aborted) {
@@ -173,13 +192,6 @@ export class TestRunnerService {
 
 		// Prepare the data to run the evaluation workflow
 		const data = await getRunData(evaluationWorkflow, [evaluationInputData]);
-		// FIXME: This is a hack to add the testRunId to the evaluation workflow execution data
-		// So that we can fetch all execution runs for a test run
-		if (testRunId && data.executionData) {
-			data.executionData.resultData.metadata = {
-				testRunId,
-			};
-		}
 		data.executionMode = 'evaluation';
 
 		// Trigger the evaluation workflow
@@ -191,6 +203,13 @@ export class TestRunnerService {
 			this.activeExecutions.stopExecution(executionId);
 		});
 
+		// Update status of the test run execution mapping
+		await this.testCaseExecutionRepository.markAsEvaluationRunning(
+			metadata.testRunId,
+			metadata.pastExecutionId,
+			executionId,
+		);
+
 		// Wait for the execution to finish
 		const executePromise = this.activeExecutions.getPostExecutePromise(executionId);
 
@@ -248,9 +267,18 @@ export class TestRunnerService {
 		const abortController = new AbortController();
 		this.abortControllers.set(testRun.id, abortController);
 
+		// 0.2 Initialize metadata
+		// This will be passed to the test case executions
+		const testRunMetadata = {
+			testRunId: testRun.id,
+			userId: user.id,
+		};
+
 		const abortSignal = abortController.signal;
 		try {
+			///
 			// 1. Make test cases from previous executions
+			///
 
 			// Select executions with the annotation tag and workflow ID of the test.
 			// Fetch only ids to reduce the data transfer.
@@ -266,13 +294,22 @@ export class TestRunnerService {
 
 			this.logger.debug('Found past executions', { count: pastExecutions.length });
 
+			// Add all past executions mappings to the test run.
+			// This will be used to track the status of each test case and keep the connection between test run and all related executions (past, current, and evaluation).
+			await this.testCaseExecutionRepository.createBatch(
+				testRun.id,
+				pastExecutions.map((e) => e.id),
+			);
+
 			// Get the metrics to collect from the evaluation workflow
 			const testMetricNames = await this.getTestMetricNames(test.id);
 
 			// 2. Run over all the test cases
 			const pastExecutionIds = pastExecutions.map((e) => e.id);
 
+			// Update test run status
 			await this.testRunRepository.markAsRunning(testRun.id, pastExecutions.length);
+
 			this.telemetry.track('User runs test', {
 				user_id: user.id,
 				test_id: test.id,
@@ -282,9 +319,13 @@ export class TestRunnerService {
 				evaluation_workflow_id: test.evaluationWorkflowId,
 			});
 
-			// Object to collect the results of the evaluation workflow executions
+			// Initialize object to collect the results of the evaluation workflow executions
 			const metrics = new EvaluationMetrics(testMetricNames);
 
+			///
+			// 2. Run over all the test cases
+			///
+
 			for (const pastExecutionId of pastExecutionIds) {
 				if (abortSignal.aborted) {
 					this.logger.debug('Test run was cancelled', {
@@ -306,13 +347,18 @@ export class TestRunnerService {
 
 					const executionData = parse(pastExecution.executionData.data) as IRunExecutionData;
 
+					const testCaseMetadata = {
+						...testRunMetadata,
+						pastExecutionId,
+					};
+
 					// Run the test case and wait for it to finish
 					const testCaseExecution = await this.runTestCase(
 						workflow,
 						executionData,
 						pastExecution.executionData.workflowData,
 						test.mockedNodes,
-						user.id,
+						testCaseMetadata,
 						abortSignal,
 					);
 
@@ -321,10 +367,18 @@ export class TestRunnerService {
 					// In case of a permission check issue, the test case execution will be undefined.
 					// Skip them, increment the failed count and continue with the next test case
 					if (!testCaseExecution) {
-						await this.testRunRepository.incrementFailed(testRun.id);
+						await Db.transaction(async (trx) => {
+							await this.testRunRepository.incrementFailed(testRun.id, trx);
+							await this.testCaseExecutionRepository.markAsFailed(testRun.id, pastExecutionId, trx);
+						});
 						continue;
 					}
 
+					// Update status of the test case execution mapping entry in case of an error
+					if (testCaseExecution.data.resultData.error) {
+						await this.testCaseExecutionRepository.markAsFailed(testRun.id, pastExecutionId);
+					}
+
 					// Collect the results of the test case execution
 					const testCaseRunData = testCaseExecution.data.resultData.runData;
 
@@ -337,23 +391,37 @@ export class TestRunnerService {
 						originalRunData,
 						testCaseRunData,
 						abortSignal,
-						testRun.id,
+						testCaseMetadata,
 					);
 					assert(evalExecution);
 
 					this.logger.debug('Evaluation execution finished', { pastExecutionId });
 
 					// Extract the output of the last node executed in the evaluation workflow
-					metrics.addResults(this.extractEvaluationResult(evalExecution));
+					const addedMetrics = metrics.addResults(this.extractEvaluationResult(evalExecution));
 
 					if (evalExecution.data.resultData.error) {
-						await this.testRunRepository.incrementFailed(testRun.id);
+						await Db.transaction(async (trx) => {
+							await this.testRunRepository.incrementFailed(testRun.id, trx);
+							await this.testCaseExecutionRepository.markAsFailed(testRun.id, pastExecutionId, trx);
+						});
 					} else {
-						await this.testRunRepository.incrementPassed(testRun.id);
+						await Db.transaction(async (trx) => {
+							await this.testRunRepository.incrementPassed(testRun.id, trx);
+							await this.testCaseExecutionRepository.markAsCompleted(
+								testRun.id,
+								pastExecutionId,
+								addedMetrics,
+								trx,
+							);
+						});
 					}
 				} catch (e) {
 					// In case of an unexpected error, increment the failed count and continue with the next test case
-					await this.testRunRepository.incrementFailed(testRun.id);
+					await Db.transaction(async (trx) => {
+						await this.testRunRepository.incrementFailed(testRun.id, trx);
+						await this.testCaseExecutionRepository.markAsFailed(testRun.id, pastExecutionId, trx);
+					});
 
 					this.errorReporter.error(e);
 				}
@@ -361,7 +429,10 @@ export class TestRunnerService {
 
 			// Mark the test run as completed or cancelled
 			if (abortSignal.aborted) {
-				await this.testRunRepository.markAsCancelled(testRun.id);
+				await Db.transaction(async (trx) => {
+					await this.testRunRepository.markAsCancelled(testRun.id, trx);
+					await this.testCaseExecutionRepository.markAllPendingAsCancelled(testRun.id, trx);
+				});
 			} else {
 				const aggregatedMetrics = metrics.getAggregatedMetrics();
 				await this.testRunRepository.markAsCompleted(testRun.id, aggregatedMetrics);
@@ -375,7 +446,10 @@ export class TestRunnerService {
 					stoppedOn: e.extra?.executionId,
 				});
 
-				await this.testRunRepository.markAsCancelled(testRun.id);
+				await Db.transaction(async (trx) => {
+					await this.testRunRepository.markAsCancelled(testRun.id, trx);
+					await this.testCaseExecutionRepository.markAllPendingAsCancelled(testRun.id, trx);
+				});
 			} else {
 				throw e;
 			}
@@ -402,8 +476,11 @@ export class TestRunnerService {
 			abortController.abort();
 			this.abortControllers.delete(testRunId);
 		} else {
-			// If there is no abort controller - just mark the test run as cancelled
-			await this.testRunRepository.markAsCancelled(testRunId);
+			// If there is no abort controller - just mark the test run and all its' pending test case executions as cancelled
+			await Db.transaction(async (trx) => {
+				await this.testRunRepository.markAsCancelled(testRunId, trx);
+				await this.testCaseExecutionRepository.markAllPendingAsCancelled(testRunId, trx);
+			});
 		}
 	}
 }

packages/cli/src/evaluation.ee/test-runs.controller.ee.ts

@@ -1,6 +1,7 @@
 import express from 'express';
 import { InstanceSettings } from 'n8n-core';
 
+import { TestCaseExecutionRepository } from '@/databases/repositories/test-case-execution.repository.ee';
 import { TestRunRepository } from '@/databases/repositories/test-run.repository.ee';
 import { Delete, Get, Post, RestController } from '@/decorators';
 import { ConflictError } from '@/errors/response-errors/conflict.error';
@@ -18,6 +19,7 @@ export class TestRunsController {
 	constructor(
 		private readonly testDefinitionService: TestDefinitionService,
 		private readonly testRunRepository: TestRunRepository,
+		private readonly testCaseExecutionRepository: TestCaseExecutionRepository,
 		private readonly testRunnerService: TestRunnerService,
 		private readonly instanceSettings: InstanceSettings,
 	) {}
@@ -76,6 +78,16 @@ export class TestRunsController {
 		return await this.getTestRun(req);
 	}
 
+	@Get('/:testDefinitionId/runs/:id/cases')
+	async getTestCases(req: TestRunsRequest.GetCases) {
+		await this.getTestDefinition(req);
+		await this.getTestRun(req);
+
+		return await this.testCaseExecutionRepository.find({
+			where: { testRun: { id: req.params.id } },
+		});
+	}
+
 	@Delete('/:testDefinitionId/runs/:id')
 	async delete(req: TestRunsRequest.Delete) {
 		const { id: testRunId } = req.params;