diff --git a/packages/cli/src/events/relays/telemetry.event-relay.ts b/packages/cli/src/events/relays/telemetry.event-relay.ts index b59c23c55c..acaeca20cf 100644 --- a/packages/cli/src/events/relays/telemetry.event-relay.ts +++ b/packages/cli/src/events/relays/telemetry.event-relay.ts @@ -731,6 +731,7 @@ export class TelemetryEventRelay extends EventRelay { is_managed: false, eval_rows_left: null, ...TelemetryHelpers.resolveAIMetrics(workflow.nodes, this.nodeTypes), + ...TelemetryHelpers.resolveVectorStoreMetrics(workflow.nodes, this.nodeTypes, runData), }; if (!manualExecEventProperties.node_graph_string) { diff --git a/packages/workflow/src/telemetry-helpers.ts b/packages/workflow/src/telemetry-helpers.ts index b4120a2a1b..00a4433bad 100644 --- a/packages/workflow/src/telemetry-helpers.ts +++ b/packages/workflow/src/telemetry-helpers.ts @@ -604,3 +604,43 @@ export function resolveAIMetrics(nodes: INode[], nodeTypes: INodeTypes): FromAIC fromAIExpressionCount, }; } + +export type VectorStoreMetrics = { + insertedIntoVectorStore: boolean; + queriedDataFromVectorStore: boolean; +}; + +export function resolveVectorStoreMetrics( + nodes: INode[], + nodeTypes: INodeTypes, + run: IRun, +): VectorStoreMetrics | {} { + const resolvedNodes = nodes + .map((x) => [x, nodeTypes.getByNameAndVersion(x.type, x.typeVersion)] as const) + .filter((x) => !!x[1]?.description); + + const vectorStores = resolvedNodes.filter( + (x) => + x[1].description.codex?.categories?.includes('AI') && + x[1].description.codex?.subcategories?.AI?.includes('Vector Stores'), + ); + + if (vectorStores.length === 0) return {}; + + const runData = run?.data?.resultData?.runData; + const succeededVectorStores = vectorStores.filter((x) => + runData?.[x[0].name]?.some((execution) => execution.executionStatus === 'success'), + ); + + const insertingVectorStores = succeededVectorStores.filter( + (x) => x[0].parameters?.mode === 'insert', + ); + const retrievingVectorStores = succeededVectorStores.filter((x) => + ['retrieve-as-tool', 'retrieve', 'load'].find((y) => y === x[0].parameters?.mode), + ); + + return { + insertedIntoVectorStore: insertingVectorStores.length > 0, + queriedDataFromVectorStore: retrievingVectorStores.length > 0, + }; +} diff --git a/packages/workflow/test/telemetry-helpers.test.ts b/packages/workflow/test/telemetry-helpers.test.ts index 9ab741cc90..6468bad227 100644 --- a/packages/workflow/test/telemetry-helpers.test.ts +++ b/packages/workflow/test/telemetry-helpers.test.ts @@ -20,6 +20,7 @@ import { getDomainBase, getDomainPath, resolveAIMetrics, + resolveVectorStoreMetrics, userInInstanceRanOutOfFreeAiCredits, } from '@/telemetry-helpers'; import { randomInt } from '@/utils'; @@ -1808,3 +1809,221 @@ describe('makeAIMetrics', () => { }); }); }); + +describe('resolveVectorStoreMetrics', () => { + const makeNode = (parameters: object, type: string) => + ({ + parameters, + type, + typeVersion: 1, + id: '7cb0b373-715c-4a89-8bbb-3f238907bc86', + name: 'a name', + position: [0, 0], + }) as INode; + + it('should return empty object if no vector store nodes are present', () => { + const nodes = [ + makeNode( + { + mode: 'insert', + }, + 'n8n-nodes-base.nonVectorStoreNode', + ), + ]; + + const nodeTypes = mock({ + getByNameAndVersion: () => ({ + description: { + codex: { + categories: ['Non-AI'], + }, + } as unknown as INodeTypeDescription, + }), + }); + + const run = mock({ + data: { + resultData: { + runData: {}, + }, + }, + }); + + const result = resolveVectorStoreMetrics(nodes, nodeTypes, run); + expect(result).toMatchObject({}); + }); + + it('should detect vector store nodes that inserted data', () => { + const nodes = [ + makeNode( + { + mode: 'insert', + }, + 'n8n-nodes-base.vectorStoreNode', + ), + ]; + + const nodeTypes = mock({ + getByNameAndVersion: () => ({ + description: { + codex: { + categories: ['AI'], + subcategories: { AI: ['Vector Stores'] }, + }, + } as unknown as INodeTypeDescription, + }), + }); + + const run = mock({ + data: { + resultData: { + runData: { + 'a name': [ + { + executionStatus: 'success', + }, + ], + }, + }, + }, + }); + + const result = resolveVectorStoreMetrics(nodes, nodeTypes, run); + expect(result).toMatchObject({ + insertedIntoVectorStore: true, + queriedDataFromVectorStore: false, + }); + }); + + it('should detect vector store nodes that queried data', () => { + const nodes = [ + makeNode( + { + mode: 'retrieve', + }, + 'n8n-nodes-base.vectorStoreNode', + ), + ]; + + const nodeTypes = mock({ + getByNameAndVersion: () => ({ + description: { + codex: { + categories: ['AI'], + subcategories: { AI: ['Vector Stores'] }, + }, + } as unknown as INodeTypeDescription, + }), + }); + + const run = mock({ + data: { + resultData: { + runData: { + 'a name': [ + { + executionStatus: 'success', + }, + ], + }, + }, + }, + }); + + const result = resolveVectorStoreMetrics(nodes, nodeTypes, run); + expect(result).toMatchObject({ + insertedIntoVectorStore: false, + queriedDataFromVectorStore: true, + }); + }); + + it('should detect vector store nodes that both inserted and queried data', () => { + const nodes = [ + makeNode( + { + mode: 'insert', + }, + 'n8n-nodes-base.vectorStoreNode', + ), + makeNode( + { + mode: 'retrieve', + }, + 'n8n-nodes-base.vectorStoreNode', + ), + ]; + + const nodeTypes = mock({ + getByNameAndVersion: () => ({ + description: { + codex: { + categories: ['AI'], + subcategories: { AI: ['Vector Stores'] }, + }, + } as unknown as INodeTypeDescription, + }), + }); + + const run = mock({ + data: { + resultData: { + runData: { + 'a name': [ + { + executionStatus: 'success', + }, + ], + }, + }, + }, + }); + + const result = resolveVectorStoreMetrics(nodes, nodeTypes, run); + expect(result).toMatchObject({ + insertedIntoVectorStore: true, + queriedDataFromVectorStore: true, + }); + }); + + it('should return empty object if no successful executions are found', () => { + const nodes = [ + makeNode( + { + mode: 'insert', + }, + 'n8n-nodes-base.vectorStoreNode', + ), + ]; + + const nodeTypes = mock({ + getByNameAndVersion: () => ({ + description: { + codex: { + categories: ['AI'], + subcategories: { AI: ['Vector Stores'] }, + }, + } as unknown as INodeTypeDescription, + }), + }); + + const run = mock({ + data: { + resultData: { + runData: { + 'a name': [ + { + executionStatus: 'error', + }, + ], + }, + }, + }, + }); + + const result = resolveVectorStoreMetrics(nodes, nodeTypes, run); + expect(result).toMatchObject({ + insertedIntoVectorStore: false, + queriedDataFromVectorStore: false, + }); + }); +});