admin/n8n-enterprise-unlocked
1
0
Fork 0
mirror of https://github.com/Abdulazizzn/n8n-enterprise-unlocked.git synced 2025-12-19 11:01:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(core): Automatically extract evaluation metrics (no-changelog) (#14051)

Browse Source
This commit is contained in:
oleg
2025-03-25 15:43:19 +01:00
committed by GitHub
parent 22e6569f7e
commit 53e11b19ad
28 changed files with 460 additions and 1286 deletions
Download Patch File Download Diff File Expand all files Collapse all files

82
packages/frontend/editor-ui/src/api/testDefinition.ee.ts
View File

@@ -83,8 +83,6 @@ export interface TestCaseExecutionRecord {
}
const endpoint = '/evaluation/test-definitions';
const getMetricsEndpoint = (testDefinitionId: string, metricId?: string) =>
`${endpoint}/${testDefinitionId}/metrics${metricId ? `/${metricId}` : ''}`;
export async function getTestDefinitions(
context: IRestApiContext,
@@ -141,86 +139,6 @@ export async function getExampleEvaluationInput(
);
}
// Metrics
export interface TestMetricRecord {
id: string;
name: string;
testDefinitionId: string;
createdAt?: string;
updatedAt?: string;
}
export interface CreateTestMetricParams {
testDefinitionId: string;
name: string;
}
export interface UpdateTestMetricParams {
name: string;
id: string;
testDefinitionId: string;
}
export interface DeleteTestMetricParams {
testDefinitionId: string;
id: string;
}
export const getTestMetrics = async (context: IRestApiContext, testDefinitionId: string) => {
return await makeRestApiRequest<TestMetricRecord[]>(
context,
'GET',
getMetricsEndpoint(testDefinitionId),
);
};
export const getTestMetric = async (
context: IRestApiContext,
testDefinitionId: string,
id: string,
) => {
return await makeRestApiRequest<TestMetricRecord>(
context,
'GET',
getMetricsEndpoint(testDefinitionId, id),
);
};
export const createTestMetric = async (
context: IRestApiContext,
params: CreateTestMetricParams,
) => {
return await makeRestApiRequest<TestMetricRecord>(
context,
'POST',
getMetricsEndpoint(params.testDefinitionId),
{ name: params.name },
);
};
export const updateTestMetric = async (
context: IRestApiContext,
params: UpdateTestMetricParams,
) => {
return await makeRestApiRequest<TestMetricRecord>(
context,
'PATCH',
getMetricsEndpoint(params.testDefinitionId, params.id),
{ name: params.name },
);
};
export const deleteTestMetric = async (
context: IRestApiContext,
params: DeleteTestMetricParams,
) => {
return await makeRestApiRequest(
context,
'DELETE',
getMetricsEndpoint(params.testDefinitionId, params.id),
);
};
const getRunsEndpoint = (testDefinitionId: string, runId?: string) =>
`${endpoint}/${testDefinitionId}/runs${runId ? `/${runId}` : ''}`;

70
packages/frontend/editor-ui/src/components/TestDefinition/EditDefinition/MetricsInput.vue
View File

@@ -1,70 +0,0 @@
<script setup lang="ts">
import { useTemplateRef, nextTick } from 'vue';
import type { TestMetricRecord } from '@/api/testDefinition.ee';
import { useI18n } from '@/composables/useI18n';
import { N8nInput, N8nButton, N8nIconButton } from '@n8n/design-system';
export interface MetricsInputProps {
modelValue: Array<Partial<TestMetricRecord>>;
}
const props = defineProps<MetricsInputProps>();
const emit = defineEmits<{
'update:modelValue': [value: MetricsInputProps['modelValue']];
deleteMetric: [metric: TestMetricRecord];
}>();
const locale = useI18n();
const metricsRefs = useTemplateRef<Array<InstanceType<typeof N8nInput>>>('metric');
function addNewMetric() {
emit('update:modelValue', [...props.modelValue, { name: '' }]);
void nextTick(() => metricsRefs.value?.at(-1)?.focus());
}
function updateMetric(index: number, name: string) {
const newMetrics = [...props.modelValue];
newMetrics[index].name = name;
emit('update:modelValue', newMetrics);
}
function onDeleteMetric(metric: Partial<TestMetricRecord>, index: number) {
if (!metric.id) {
const newMetrics = [...props.modelValue];
newMetrics.splice(index, 1);
emit('update:modelValue', newMetrics);
} else {
emit('deleteMetric', metric as TestMetricRecord);
}
}
</script>
<template>
<div>
<div
v-for="(metric, index) in modelValue"
:key="index"
:class="$style.metricItem"
class="mb-xs"
>
<N8nInput
ref="metric"
data-test-id="evaluation-metric-item"
:model-value="metric.name"
:placeholder="locale.baseText('testDefinition.edit.metricsPlaceholder')"
@update:model-value="(value: string) => updateMetric(index, value)"
/>
<N8nIconButton icon="trash" type="secondary" text @click="onDeleteMetric(metric, index)" />
</div>
<N8nButton
type="secondary"
:label="locale.baseText('testDefinition.edit.metricsNew')"
@click="addNewMetric"
/>
</div>
</template>
<style module lang="scss">
.metricItem {
display: flex;
align-items: center;
}
</style>

23
packages/frontend/editor-ui/src/components/TestDefinition/EditDefinition/sections/ConfigSection.vue
View File

@@ -1,8 +1,6 @@
<script setup lang="ts">
import type { TestMetricRecord } from '@/api/testDefinition.ee';
import BlockArrow from '@/components/TestDefinition/EditDefinition/BlockArrow.vue';
import EvaluationStep from '@/components/TestDefinition/EditDefinition/EvaluationStep.vue';
import MetricsInput from '@/components/TestDefinition/EditDefinition/MetricsInput.vue';
import NodesPinning from '@/components/TestDefinition/EditDefinition/NodesPinning.vue';
import WorkflowSelector from '@/components/TestDefinition/EditDefinition/WorkflowSelector.vue';
import type { EditableFormState, EvaluationFormState } from '@/components/TestDefinition/types';
@@ -27,7 +25,6 @@ const props = defineProps<{
}>();
const emit = defineEmits<{
openPinningModal: [];
deleteMetric: [metric: TestMetricRecord];
openExecutionsViewForTag: [];
renameTag: [tag: string];
evaluationWorkflowCreated: [workflowId: string];
@@ -64,7 +61,6 @@ const evaluationWorkflow = defineModel<EvaluationFormState['evaluationWorkflow']
'evaluationWorkflow',
{ required: true },
);
const metrics = defineModel<EvaluationFormState['metrics']>('metrics', { required: true });
const mockedNodes = defineModel<EvaluationFormState['mockedNodes']>('mockedNodes', {
required: true,
});
@@ -177,25 +173,6 @@ function openExecutionsView() {
/>
</template>
</EvaluationStep>
<BlockArrow class="mt-5xs mb-5xs" />
<!-- Metrics -->
<EvaluationStep
:title="locale.baseText('testDefinition.edit.step.metrics')"
:issues="getFieldIssues('metrics')"
:description="locale.baseText('testDefinition.edit.step.metrics.description')"
:tooltip="locale.baseText('testDefinition.edit.step.metrics.tooltip')"
:external-tooltip="!hasRuns"
>
<template #cardContent>
<MetricsInput
v-model="metrics"
:class="{ 'has-issues': getFieldIssues('metrics').length > 0 }"
class="mt-xs"
@delete-metric="(metric) => emit('deleteMetric', metric)"
/>
</template>
</EvaluationStep>
</div>
<Modal
width="calc(100% - (48px * 2))"

37
packages/frontend/editor-ui/src/components/TestDefinition/composables/useTestDefinitionForm.ts
View File

@@ -36,7 +36,6 @@ export function useTestDefinitionForm() {
value: '',
__rl: true,
},
metrics: [],
mockedNodes: [],
});
@@ -62,8 +61,6 @@ export function useTestDefinitionForm() {
const testDefinition = evaluationsStore.testDefinitionsById[testId];
if (testDefinition) {
const metrics = await evaluationsStore.fetchMetrics(testId);
state.value.description = {
value: testDefinition.description ?? '',
isEditing: false,
@@ -84,7 +81,6 @@ export function useTestDefinitionForm() {
value: testDefinition.evaluationWorkflowId ?? '',
__rl: true,
};
state.value.metrics = metrics;
state.value.mockedNodes = testDefinition.mockedNodes ?? [];
evaluationsStore.updateRunFieldIssues(testDefinition.id);
}
@@ -110,37 +106,6 @@ export function useTestDefinitionForm() {
}
};
const deleteMetric = async (metricId: string, testId: string) => {
await evaluationsStore.deleteMetric({ id: metricId, testDefinitionId: testId });
state.value.metrics = state.value.metrics.filter((metric) => metric.id !== metricId);
};
/**
* This method would perform unnecessary updates on the BE
* it's a performance degradation candidate if metrics reach certain amount
*/
const updateMetrics = async (testId: string) => {
const promises = state.value.metrics.map(async (metric) => {
if (!metric.name) return;
if (!metric.id) {
const createdMetric = await evaluationsStore.createMetric({
name: metric.name,
testDefinitionId: testId,
});
metric.id = createdMetric.id;
} else {
await evaluationsStore.updateMetric({
name: metric.name,
id: metric.id,
testDefinitionId: testId,
});
}
});
isSaving.value = true;
await Promise.all(promises);
isSaving.value = false;
};
const updateTest = async (testId: string) => {
if (isSaving.value) return;
@@ -230,8 +195,6 @@ export function useTestDefinitionForm() {
state,
fields,
isSaving: computed(() => isSaving.value),
deleteMetric,
updateMetrics,
loadTestData,
createTest,
updateTest,

2
packages/frontend/editor-ui/src/components/TestDefinition/shared/TableStatusCell.vue
View File

@@ -27,8 +27,6 @@ const errorTooltipMap: Record<string, BaseTextKey> = {
FAILED_TO_EXECUTE_EVALUATION_WORKFLOW: 'testDefinition.runDetail.error.evaluationFailed',
FAILED_TO_EXECUTE_WORKFLOW: 'testDefinition.runDetail.error.executionFailed',
TRIGGER_NO_LONGER_EXISTS: 'testDefinition.runDetail.error.triggerNoLongerExists',
METRICS_MISSING: 'testDefinition.runDetail.error.metricsMissing',
UNKNOWN_METRICS: 'testDefinition.runDetail.error.unknownMetrics',
INVALID_METRICS: 'testDefinition.runDetail.error.invalidMetrics',
// Test run errors

142
packages/frontend/editor-ui/src/components/TestDefinition/tests/MetricsInput.test.ts
View File

@@ -1,142 +0,0 @@
import { describe, it, expect, beforeEach } from 'vitest';
import { createComponentRenderer } from '@/__tests__/render';
import MetricsInput from '../EditDefinition/MetricsInput.vue';
import userEvent from '@testing-library/user-event';
const renderComponent = createComponentRenderer(MetricsInput);
describe('MetricsInput', () => {
let props: { modelValue: Array<{ id?: string; name: string }> };
beforeEach(() => {
props = {
modelValue: [
{ name: 'Metric 1', id: 'metric-1' },
{ name: 'Metric 2', id: 'metric-2' },
],
};
});
it('should render correctly with initial metrics', () => {
const { getAllByPlaceholderText } = renderComponent({ props });
const inputs = getAllByPlaceholderText('e.g. latency');
expect(inputs).toHaveLength(2);
expect(inputs[0]).toHaveValue('Metric 1');
expect(inputs[1]).toHaveValue('Metric 2');
});
it('should update a metric when typing in the input', async () => {
const { getAllByPlaceholderText, emitted } = renderComponent({
props: {
modelValue: [{ name: '' }],
},
});
const inputs = getAllByPlaceholderText('e.g. latency');
await userEvent.type(inputs[0], 'Updated Metric 1');
// Every character typed triggers an update event. Let's check the last emission.
const allEmits = emitted('update:modelValue');
expect(allEmits).toBeTruthy();
// The last emission should contain the fully updated name
const lastEmission = allEmits[allEmits.length - 1];
expect(lastEmission).toEqual([[{ name: 'Updated Metric 1' }]]);
});
it('should render correctly with no initial metrics', () => {
props.modelValue = [];
const { queryAllByRole, getByText } = renderComponent({ props });
const inputs = queryAllByRole('textbox');
expect(inputs).toHaveLength(0);
expect(getByText('New metric')).toBeInTheDocument();
});
it('should handle adding multiple metrics', async () => {
const { getByText, emitted } = renderComponent({ props });
const addButton = getByText('New metric');
await userEvent.click(addButton);
await userEvent.click(addButton);
await userEvent.click(addButton);
// Each click adds a new metric
const updateEvents = emitted('update:modelValue');
expect(updateEvents).toHaveLength(3);
// Check the structure of one of the emissions
// Initial: [{ name: 'Metric 1' }, { name: 'Metric 2' }]
// After first click: [{ name: 'Metric 1' }, { name: 'Metric 2' }, { name: '' }]
expect(updateEvents[0]).toEqual([[...props.modelValue, { name: '' }]]);
});
it('should emit "deleteMetric" event when a delete button is clicked', async () => {
const { getAllByRole, emitted } = renderComponent({ props });
// Each metric row has a delete button, identified by "button"
const deleteButtons = getAllByRole('button', { name: '' });
expect(deleteButtons).toHaveLength(props.modelValue.length);
// Click on the delete button for the second metric
await userEvent.click(deleteButtons[1]);
expect(emitted('deleteMetric')).toBeTruthy();
expect(emitted('deleteMetric')[0]).toEqual([props.modelValue[1]]);
});
it('should emit multiple update events as the user types and reflect the final name correctly', async () => {
const { getAllByPlaceholderText, emitted } = renderComponent({
props: {
modelValue: [{ name: '' }],
},
});
const inputs = getAllByPlaceholderText('e.g. latency');
await userEvent.type(inputs[0], 'ABC');
const allEmits = emitted('update:modelValue');
expect(allEmits).toBeTruthy();
// Each character typed should emit a new value
expect(allEmits.length).toBe(3);
expect(allEmits[2]).toEqual([[{ name: 'ABC' }]]);
});
it('should not break if metrics are empty and still allow adding a new metric', async () => {
props.modelValue = [];
const { queryAllByRole, getByText, emitted } = renderComponent({ props });
// No metrics initially
const inputs = queryAllByRole('textbox');
expect(inputs).toHaveLength(0);
const addButton = getByText('New metric');
await userEvent.click(addButton);
const updates = emitted('update:modelValue');
expect(updates).toBeTruthy();
expect(updates[0]).toEqual([[{ name: '' }]]);
// After adding one metric, we should now have an input
const { getAllByPlaceholderText } = renderComponent({
props: { modelValue: [{ name: '' }] },
});
const updatedInputs = getAllByPlaceholderText('e.g. latency');
expect(updatedInputs).toHaveLength(1);
});
it('should handle deleting the first metric and still display remaining metrics correctly', async () => {
const { getAllByPlaceholderText, getAllByRole, rerender, emitted } = renderComponent({
props,
});
const inputs = getAllByPlaceholderText('e.g. latency');
expect(inputs).toHaveLength(2);
const deleteButtons = getAllByRole('button', { name: '' });
await userEvent.click(deleteButtons[0]);
expect(emitted('deleteMetric')).toBeTruthy();
expect(emitted('deleteMetric')[0]).toEqual([props.modelValue[0]]);
await rerender({ modelValue: [{ name: 'Metric 2' }] });
const updatedInputs = getAllByPlaceholderText('e.g. latency');
expect(updatedInputs).toHaveLength(1);
expect(updatedInputs[0]).toHaveValue('Metric 2');
});
});

75
packages/frontend/editor-ui/src/components/TestDefinition/tests/useTestDefinitionForm.test.ts
View File

@@ -48,20 +48,12 @@ describe('useTestDefinitionForm', () => {
expect(state.value.description.value).toBe('');
expect(state.value.name.value).toContain('My Test');
expect(state.value.tags.value).toEqual([]);
expect(state.value.metrics).toEqual([]);
expect(state.value.evaluationWorkflow.value).toBe('');
});
it('should load test data', async () => {
const { loadTestData, state } = useTestDefinitionForm();
const fetchSpy = vi.spyOn(useTestDefinitionStore(), 'fetchAll');
const fetchMetricsSpy = vi.spyOn(useTestDefinitionStore(), 'fetchMetrics').mockResolvedValue([
{
id: 'metric1',
name: 'Metric 1',
testDefinitionId: TEST_DEF_A.id,
},
]);
const evaluationsStore = mockedStore(useTestDefinitionStore);
evaluationsStore.testDefinitionsById = {
@@ -71,14 +63,10 @@ describe('useTestDefinitionForm', () => {
await loadTestData(TEST_DEF_A.id, '123');
expect(fetchSpy).toBeCalled();
expect(fetchMetricsSpy).toBeCalledWith(TEST_DEF_A.id);
expect(state.value.name.value).toEqual(TEST_DEF_A.name);
expect(state.value.description.value).toEqual(TEST_DEF_A.description);
expect(state.value.tags.value).toEqual([TEST_DEF_A.annotationTagId]);
expect(state.value.evaluationWorkflow.value).toEqual(TEST_DEF_A.evaluationWorkflowId);
expect(state.value.metrics).toEqual([
{ id: 'metric1', name: 'Metric 1', testDefinitionId: TEST_DEF_A.id },
]);
});
it('should gracefully handle loadTestData when no test definition found', async () => {
@@ -94,7 +82,6 @@ describe('useTestDefinitionForm', () => {
expect(state.value.description.value).toBe('');
expect(state.value.name.value).toContain('My Test');
expect(state.value.tags.value).toEqual([]);
expect(state.value.metrics).toEqual([]);
});
it('should handle errors while loading test data', async () => {
@@ -176,68 +163,6 @@ describe('useTestDefinitionForm', () => {
expect(updateSpy).toBeCalled();
});
it('should delete a metric', async () => {
const { state, deleteMetric } = useTestDefinitionForm();
const evaluationsStore = mockedStore(useTestDefinitionStore);
const deleteMetricSpy = vi.spyOn(evaluationsStore, 'deleteMetric');
state.value.metrics = [
{
id: 'metric1',
name: 'Metric 1',
testDefinitionId: '1',
},
{
id: 'metric2',
name: 'Metric 2',
testDefinitionId: '1',
},
];
await deleteMetric('metric1', TEST_DEF_A.id);
expect(deleteMetricSpy).toBeCalledWith({ id: 'metric1', testDefinitionId: TEST_DEF_A.id });
expect(state.value.metrics).toEqual([
{ id: 'metric2', name: 'Metric 2', testDefinitionId: '1' },
]);
});
it('should update metrics', async () => {
const { state, updateMetrics } = useTestDefinitionForm();
const evaluationsStore = mockedStore(useTestDefinitionStore);
const updateMetricSpy = vi.spyOn(evaluationsStore, 'updateMetric');
const createMetricSpy = vi
.spyOn(evaluationsStore, 'createMetric')
.mockResolvedValue({ id: 'metric_new', name: 'Metric 2', testDefinitionId: TEST_DEF_A.id });
state.value.metrics = [
{
id: 'metric1',
name: 'Metric 1',
testDefinitionId: TEST_DEF_A.id,
},
{
id: '',
name: 'Metric 2',
testDefinitionId: TEST_DEF_A.id,
}, // New metric that needs creation
];
await updateMetrics(TEST_DEF_A.id);
expect(createMetricSpy).toHaveBeenCalledWith({
name: 'Metric 2',
testDefinitionId: TEST_DEF_A.id,
});
expect(updateMetricSpy).toHaveBeenCalledWith({
name: 'Metric 1',
id: 'metric1',
testDefinitionId: TEST_DEF_A.id,
});
expect(state.value.metrics).toEqual([
{ id: 'metric1', name: 'Metric 1', testDefinitionId: TEST_DEF_A.id },
{ id: 'metric_new', name: 'Metric 2', testDefinitionId: TEST_DEF_A.id },
]);
});
it('should start editing a field', () => {
const { state, startEditing } = useTestDefinitionForm();

2
packages/frontend/editor-ui/src/components/TestDefinition/types.ts
View File

@@ -1,4 +1,3 @@
import type { TestMetricRecord } from '@/api/testDefinition.ee';
import type { INodeParameterResourceLocator } from 'n8n-workflow';
export interface EditableField<T = string> {
@@ -14,6 +13,5 @@ export interface EditableFormState {
export interface EvaluationFormState extends EditableFormState {
evaluationWorkflow: INodeParameterResourceLocator;
metrics: TestMetricRecord[];
mockedNodes: Array<{ name: string; id: string }>;
}

9
packages/frontend/editor-ui/src/constants.workflows.ts
View File

@@ -62,23 +62,22 @@ export const SAMPLE_EVALUATION_WORKFLOW: IWorkflowDataCreate = {
},
{
parameters: {
assignments: {
metrics: {
assignments: [
{
id: 'a748051d-ebdb-4fcf-aaed-02756130ce2a',
name: 'latency',
value:
'={{(() => {\n const newExecutionRuns = Object.values($json.newExecution)\n .reduce((acc, node) => {\n acc.push(node.runs.filter(run => run.output.main !== undefined))\n return acc\n }, []).flat()\n\n const latency = newExecutionRuns.reduce((acc, run) => acc + run.executionTime, 0)\n\n return latency\n})()}}',
type: 'number',
id: '1ebc15e9-f079-4d1f-a08d-d4880ea0ddb5',
},
],
},
options: {},
},
type: 'n8n-nodes-base.evaluationMetrics',
id: '33e2e94a-ec48-4e7b-b750-f56718d5105c',
name: 'Return metric(s)',
type: 'n8n-nodes-base.set',
typeVersion: 3.4,
typeVersion: 1,
position: [600, 440],
},
{

12
packages/frontend/editor-ui/src/plugins/i18n/locales/en.json
View File

@@ -2918,11 +2918,6 @@
"testDefinition.edit.hideConfig": "Hide config",
"testDefinition.edit.backButtonTitle": "Back to Workflow Evaluation",
"testDefinition.edit.namePlaceholder": "Enter test name",
"testDefinition.edit.metricsTitle": "Metrics",
"testDefinition.edit.metricsHelpText": "The output field of the last node in the evaluation workflow. Metrics will be averaged across all test cases.",
"testDefinition.edit.metricsFields": "Output fields to use as metrics",
"testDefinition.edit.metricsPlaceholder": "e.g. latency",
"testDefinition.edit.metricsNew": "New metric",
"testDefinition.edit.selectTag": "Select tag...",
"testDefinition.edit.tagsHelpText": "Executions with this tag will be added as test cases to this test.",
"testDefinition.edit.workflowSelectorLabel": "Use a second workflow to make the comparison",
@@ -2952,9 +2947,6 @@
"testDefinition.edit.step.reRunExecutions.tooltip": "Each past execution is re-run using the latest version of the workflow being tested",
"testDefinition.edit.step.compareExecutions": "4. Compare each past and new execution",
"testDefinition.edit.step.compareExecutions.tooltip": "Each past execution is compared with its new equivalent to check how similar they are. This is done using a separate evaluation workflow: it receives the two execution versions as input, and outputs metrics.",
"testDefinition.edit.step.metrics": "5. Summarise metrics",
"testDefinition.edit.step.metrics.tooltip": "Metrics returned by the evaluation workflow (defined above). If included in this section, they are displayed in the test run results and averaged to give a score for the entire test run.",
"testDefinition.edit.step.metrics.description": "The names of fields output by your evaluation workflow in the step above.",
"testDefinition.edit.step.collapse": "Collapse",
"testDefinition.edit.step.configure": "Configure",
"testDefinition.edit.selectNodes": "Pin nodes to mock them",
@@ -3025,10 +3017,6 @@
"testDefinition.runDetail.error.evaluationFailed.solution": "View evaluation execution",
"testDefinition.runDetail.error.triggerNoLongerExists": "Trigger in benchmark execution no longer exists in workflow.{link}.",
"testDefinition.runDetail.error.triggerNoLongerExists.solution": "View benchmark",
"testDefinition.runDetail.error.metricsMissing": "Metrics defined in test were not returned by evaluation workflow {link}.",
"testDefinition.runDetail.error.metricsMissing.solution": "Fix test configuration",
"testDefinition.runDetail.error.unknownMetrics": "Evaluation workflow defined metrics that are not defined in the test. {link}.",
"testDefinition.runDetail.error.unknownMetrics.solution": "Fix test configuration",
"testDefinition.runDetail.error.invalidMetrics": "Evaluation workflow returned invalid metrics. Only numeric values are expected. View evaluation execution. {link}.",
"testDefinition.runDetail.error.invalidMetrics.solution": "View evaluation execution",
"testDefinition.runTest": "Run Test",

94
packages/frontend/editor-ui/src/stores/testDefinition.store.ee.test.ts
View File

@@ -11,10 +11,6 @@ const {
deleteTestDefinition,
getTestDefinitions,
updateTestDefinition,
getTestMetrics,
createTestMetric,
updateTestMetric,
deleteTestMetric,
getTestRuns,
getTestRun,
startTestRun,
@@ -24,10 +20,6 @@ const {
createTestDefinition: vi.fn(),
updateTestDefinition: vi.fn(),
deleteTestDefinition: vi.fn(),
getTestMetrics: vi.fn(),
createTestMetric: vi.fn(),
updateTestMetric: vi.fn(),
deleteTestMetric: vi.fn(),
getTestRuns: vi.fn(),
getTestRun: vi.fn(),
startTestRun: vi.fn(),
@@ -39,10 +31,6 @@ vi.mock('@/api/testDefinition.ee', () => ({
deleteTestDefinition,
getTestDefinitions,
updateTestDefinition,
getTestMetrics,
createTestMetric,
updateTestMetric,
deleteTestMetric,
getTestRuns,
getTestRun,
startTestRun,
@@ -77,13 +65,6 @@ const TEST_DEF_NEW: TestDefinitionRecord = {
createdAt: '2023-01-01T00:00:00.000Z',
};
const TEST_METRIC = {
id: 'metric1',
name: 'Test Metric',
testDefinitionId: '1',
createdAt: '2023-01-01T00:00:00.000Z',
};
const TEST_RUN: TestRunRecord = {
id: 'run1',
testDefinitionId: '1',
@@ -124,7 +105,6 @@ describe('testDefinition.store.ee', () => {
getTestRun.mockResolvedValue(TEST_RUN);
startTestRun.mockResolvedValue({ success: true });
deleteTestRun.mockResolvedValue({ success: true });
getTestMetrics.mockResolvedValue([TEST_METRIC]);
});
test('Initialization', () => {
@@ -280,80 +260,6 @@ describe('testDefinition.store.ee', () => {
});
});
describe('Metrics', () => {
test('Fetching Metrics for a Test Definition', async () => {
const metrics = await store.fetchMetrics('1');
expect(getTestMetrics).toHaveBeenCalledWith(rootStoreMock.restApiContext, '1');
expect(store.metricsById).toEqual({
metric1: TEST_METRIC,
});
expect(metrics).toEqual([TEST_METRIC]);
});
test('Creating a Metric', async () => {
createTestMetric.mockResolvedValue(TEST_METRIC);
const params = {
name: 'Test Metric',
testDefinitionId: '1',
};
const result = await store.createMetric(params);
expect(createTestMetric).toHaveBeenCalledWith(rootStoreMock.restApiContext, params);
expect(store.metricsById).toEqual({
metric1: TEST_METRIC,
});
expect(result).toEqual(TEST_METRIC);
});
test('Updating a Metric', async () => {
const updatedMetric = { ...TEST_METRIC, name: 'Updated Metric' };
updateTestMetric.mockResolvedValue(updatedMetric);
const result = await store.updateMetric(updatedMetric);
expect(updateTestMetric).toHaveBeenCalledWith(rootStoreMock.restApiContext, updatedMetric);
expect(store.metricsById).toEqual({
metric1: updatedMetric,
});
expect(result).toEqual(updatedMetric);
});
test('Deleting a Metric', async () => {
store.metricsById = {
metric1: TEST_METRIC,
};
const params = { id: 'metric1', testDefinitionId: '1' };
deleteTestMetric.mockResolvedValue(undefined);
await store.deleteMetric(params);
expect(deleteTestMetric).toHaveBeenCalledWith(rootStoreMock.restApiContext, params);
expect(store.metricsById).toEqual({});
});
test('Getting Metrics by Test ID', () => {
const metric1 = { ...TEST_METRIC, id: 'metric1', testDefinitionId: '1' };
const metric2 = { ...TEST_METRIC, id: 'metric2', testDefinitionId: '1' };
const metric3 = { ...TEST_METRIC, id: 'metric3', testDefinitionId: '2' };
store.metricsById = {
metric1,
metric2,
metric3,
};
const metricsForTest1 = store.metricsByTestId['1'];
expect(metricsForTest1).toEqual([metric1, metric2]);
const metricsForTest2 = store.metricsByTestId['2'];
expect(metricsForTest2).toEqual([metric3]);
});
});
describe('Computed Properties', () => {
test('hasTestDefinitions', () => {
store.testDefinitionsById = {};

76
packages/frontend/editor-ui/src/stores/testDefinition.store.ee.ts
View File

@@ -21,7 +21,6 @@ export const useTestDefinitionStore = defineStore(
const testDefinitionsById = ref<Record<string, TestDefinitionRecord>>({});
const loading = ref(false);
const fetchedAll = ref(false);
const metricsById = ref<Record<string, testDefinitionsApi.TestMetricRecord>>({});
const testRunsById = ref<Record<string, TestRunRecord>>({});
const testCaseExecutionsById = ref<Record<string, TestCaseExecutionRecord>>({});
const pollingTimeouts = ref<Record<string, NodeJS.Timeout>>({});
@@ -61,19 +60,6 @@ export const useTestDefinitionStore = defineStore(
const hasTestDefinitions = computed(() => Object.keys(testDefinitionsById.value).length > 0);
const metricsByTestId = computed(() => {
return Object.values(metricsById.value).reduce(
(acc: Record<string, testDefinitionsApi.TestMetricRecord[]>, metric) => {
if (!acc[metric.testDefinitionId]) {
acc[metric.testDefinitionId] = [];
}
acc[metric.testDefinitionId].push(metric);
return acc;
},
{},
);
});
const testRunsByTestId = computed(() => {
return Object.values(testRunsById.value).reduce(
(acc: Record<string, TestRunRecord[]>, run) => {
@@ -157,11 +143,6 @@ export const useTestDefinitionStore = defineStore(
}
};
const fetchMetricsForAllTests = async () => {
const testDefinitions = Object.values(testDefinitionsById.value);
await Promise.all(testDefinitions.map(async (testDef) => await fetchMetrics(testDef.id)));
};
const fetchTestDefinition = async (id: string) => {
const testDefinition = await testDefinitionsApi.getTestDefinition(
rootStore.restApiContext,
@@ -221,7 +202,6 @@ export const useTestDefinitionStore = defineStore(
await Promise.all([
tagsStore.fetchAll({ force: true, withUsageCount: true }),
fetchRunsForAllTests(),
fetchMetricsForAllTests(),
]);
return retrievedDefinitions;
} finally {
@@ -289,48 +269,6 @@ export const useTestDefinitionStore = defineStore(
return result.success;
};
const fetchMetrics = async (testId: string) => {
loading.value = true;
try {
const metrics = await testDefinitionsApi.getTestMetrics(rootStore.restApiContext, testId);
metrics.forEach((metric) => {
metricsById.value[metric.id] = { ...metric, testDefinitionId: testId };
});
return metrics.map((metric) => ({ ...metric, testDefinitionId: testId }));
} finally {
loading.value = false;
}
};
const createMetric = async (params: {
name: string;
testDefinitionId: string;
}): Promise<testDefinitionsApi.TestMetricRecord> => {
const metric = await testDefinitionsApi.createTestMetric(rootStore.restApiContext, params);
metricsById.value[metric.id] = { ...metric, testDefinitionId: params.testDefinitionId };
return metric;
};
const updateMetric = async (
params: testDefinitionsApi.TestMetricRecord,
): Promise<testDefinitionsApi.TestMetricRecord> => {
const metric = await testDefinitionsApi.updateTestMetric(rootStore.restApiContext, params);
metricsById.value[metric.id] = { ...metric, testDefinitionId: params.testDefinitionId };
updateRunFieldIssues(params.testDefinitionId);
return metric;
};
const deleteMetric = async (
params: testDefinitionsApi.DeleteTestMetricParams,
): Promise<void> => {
await testDefinitionsApi.deleteTestMetric(rootStore.restApiContext, params);
const { [params.id]: deleted, ...rest } = metricsById.value;
metricsById.value = rest;
updateRunFieldIssues(params.testDefinitionId);
};
// Test Runs Methods
const fetchTestRuns = async (testDefinitionId: string) => {
loading.value = true;
@@ -436,14 +374,6 @@ export const useTestDefinitionStore = defineStore(
});
}
const metrics = metricsByTestId.value[testId] || [];
if (metrics.filter((metric) => metric.name).length === 0) {
issues.push({
field: 'metrics',
message: locale.baseText('testDefinition.configError.noMetrics'),
});
}
fieldsIssues.value = {
...fieldsIssues.value,
[testId]: issues,
@@ -464,8 +394,6 @@ export const useTestDefinitionStore = defineStore(
isLoading,
hasTestDefinitions,
isFeatureEnabled,
metricsById,
metricsByTestId,
testRunsByTestId,
lastRunByTestId,
@@ -480,10 +408,6 @@ export const useTestDefinitionStore = defineStore(
deleteById,
upsertTestDefinitions,
deleteTestDefinition,
fetchMetrics,
createMetric,
updateMetric,
deleteMetric,
fetchTestRuns,
getTestRun,
startTestRun,

29
packages/frontend/editor-ui/src/views/TestDefinition/TestDefinitionEditView.vue
View File

@@ -9,7 +9,7 @@ import { useAnnotationTagsStore } from '@/stores/tags.store';
import { computed, ref, watch } from 'vue';
import { useRouter } from 'vue-router';
import type { TestMetricRecord, TestRunRecord } from '@/api/testDefinition.ee';
import type { TestRunRecord } from '@/api/testDefinition.ee';
import InlineNameEdit from '@/components/InlineNameEdit.vue';
import ConfigSection from '@/components/TestDefinition/EditDefinition/sections/ConfigSection.vue';
import RunsSection from '@/components/TestDefinition/EditDefinition/sections/RunsSection.vue';
@@ -44,17 +44,8 @@ watch(visibility, async () => {
testDefinitionStore.updateRunFieldIssues(props.testId);
});
const {
state,
isSaving,
cancelEditing,
loadTestData,
updateTest,
startEditing,
saveChanges,
deleteMetric,
updateMetrics,
} = useTestDefinitionForm();
const { state, isSaving, cancelEditing, loadTestData, updateTest, startEditing, saveChanges } =
useTestDefinitionForm();
const isLoading = computed(() => tagsStore.isLoading);
const tagsById = computed(() => tagsStore.tagsById);
@@ -79,22 +70,11 @@ const handleUpdateTest = async () => {
};
const handleUpdateTestDebounced = debounce(handleUpdateTest, { debounceTime: 400, trailing: true });
const handleUpdateMetricsDebounced = debounce(
async (testId: string) => {
await updateMetrics(testId);
testDefinitionStore.updateRunFieldIssues(testId);
},
{ debounceTime: 400, trailing: true },
);
function getFieldIssues(key: string) {
return fieldsIssues.value.filter((issue) => issue.field === key);
}
async function onDeleteMetric(deletedMetric: TestMetricRecord) {
await deleteMetric(deletedMetric.id, props.testId);
}
async function openPinningModal() {
uiStore.openModal(NODE_PINNING_MODAL_KEY);
}
@@ -253,7 +233,6 @@ function onEvaluationWorkflowCreated(workflowId: string) {
v-if="showConfig"
v-model:tags="state.tags"
v-model:evaluationWorkflow="state.evaluationWorkflow"
v-model:metrics="state.metrics"
v-model:mockedNodes="state.mockedNodes"
:class="$style.config"
:cancel-editing="cancelEditing"
@@ -266,11 +245,9 @@ function onEvaluationWorkflowCreated(workflowId: string) {
:example-pinned-data="examplePinnedData"
:sample-workflow-name="workflowName"
@rename-tag="renameTag"
@update:metrics="() => handleUpdateMetricsDebounced(testId)"
@update:evaluation-workflow="handleUpdateTestDebounced"
@update:mocked-nodes="handleUpdateTestDebounced"
@open-pinning-modal="openPinningModal"
@delete-metric="onDeleteMetric"
@open-executions-view-for-tag="openExecutionsViewForTag"
@evaluation-workflow-created="onEvaluationWorkflowCreated($event)"
/>

18
packages/frontend/editor-ui/src/views/TestDefinition/TestDefinitionRunDetailView.vue
View File

@@ -19,8 +19,6 @@ const TEST_CASE_EXECUTION_ERROR_CODE = {
FAILED_TO_EXECUTE_WORKFLOW: 'FAILED_TO_EXECUTE_WORKFLOW',
EVALUATION_WORKFLOW_DOES_NOT_EXIST: 'EVALUATION_WORKFLOW_DOES_NOT_EXIST',
FAILED_TO_EXECUTE_EVALUATION_WORKFLOW: 'FAILED_TO_EXECUTE_EVALUATION_WORKFLOW',
METRICS_MISSING: 'METRICS_MISSING',
UNKNOWN_METRICS: 'UNKNOWN_METRICS',
INVALID_METRICS: 'INVALID_METRICS',
PAYLOAD_LIMIT_EXCEEDED: 'PAYLOAD_LIMIT_EXCEEDED',
UNKNOWN_ERROR: 'UNKNOWN_ERROR',
@@ -97,8 +95,6 @@ const testCaseErrorDictionary: Partial<Record<TestCaseExecutionErrorCodes, BaseT
FAILED_TO_EXECUTE_EVALUATION_WORKFLOW: 'testDefinition.runDetail.error.evaluationFailed',
FAILED_TO_EXECUTE_WORKFLOW: 'testDefinition.runDetail.error.executionFailed',
TRIGGER_NO_LONGER_EXISTS: 'testDefinition.runDetail.error.triggerNoLongerExists',
METRICS_MISSING: 'testDefinition.runDetail.error.metricsMissing',
UNKNOWN_METRICS: 'testDefinition.runDetail.error.unknownMetrics',
INVALID_METRICS: 'testDefinition.runDetail.error.invalidMetrics',
} as const;
@@ -143,20 +139,6 @@ const getErrorTooltipLinkRoute = (row: TestCaseExecutionRecord) => {
executionId: row.pastExecutionId,
},
};
} else if (row.errorCode === TEST_CASE_EXECUTION_ERROR_CODE.METRICS_MISSING) {
return {
name: VIEWS.TEST_DEFINITION_EDIT,
params: {
testId: testId.value,
},
};
} else if (row.errorCode === TEST_CASE_EXECUTION_ERROR_CODE.UNKNOWN_METRICS) {
return {
name: VIEWS.TEST_DEFINITION_EDIT,
params: {
testId: testId.value,
},
};
} else if (row.errorCode === TEST_CASE_EXECUTION_ERROR_CODE.INVALID_METRICS) {
return {
name: VIEWS.EXECUTION_PREVIEW,

3
packages/frontend/editor-ui/src/views/TestDefinition/tests/TestDefinitionEditView.test.ts
View File

@@ -14,7 +14,6 @@ const form: Partial<ReturnType<typeof useTestDefinitionForm>> = {
description: { value: '', isEditing: false, tempValue: '' },
tags: { value: [], tempValue: [], isEditing: false },
evaluationWorkflow: { mode: 'list', value: '', __rl: true },
metrics: [],
mockedNodes: [],
}),
loadTestData: vi.fn(),
@@ -22,8 +21,6 @@ const form: Partial<ReturnType<typeof useTestDefinitionForm>> = {
updateTest: vi.fn(),
startEditing: vi.fn(),
saveChanges: vi.fn(),
deleteMetric: vi.fn(),
updateMetrics: vi.fn(),
createTest: vi.fn(),
};
vi.mock('@/components/TestDefinition/composables/useTestDefinitionForm', () => ({