From 7e1eeb4c31d3f25ec31baa7390b11a7e3280ce01 Mon Sep 17 00:00:00 2001 From: Eugene Date: Mon, 22 Jul 2024 16:15:43 +0200 Subject: [PATCH] feat(Pinecone Vector Store Node, Supabase Vector Store Node): Add update operation to vector store nodes (#10060) --- .../VectorStorePinecone.node.ts | 32 ++-- .../VectorStoreSupabase.node.ts | 35 ++-- .../shared/createVectorStoreNode.ts | 154 ++++++++++++++---- .../@n8n/nodes-langchain/utils/helpers.ts | 2 +- packages/workflow/src/Interfaces.ts | 1 + 5 files changed, 156 insertions(+), 68 deletions(-) diff --git a/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStorePinecone/VectorStorePinecone.node.ts b/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStorePinecone/VectorStorePinecone.node.ts index 08ad2b18dc..85509a6287 100644 --- a/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStorePinecone/VectorStorePinecone.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStorePinecone/VectorStorePinecone.node.ts @@ -9,6 +9,15 @@ import { pineconeIndexSearch } from '../shared/methods/listSearch'; const sharedFields: INodeProperties[] = [pineconeIndexRLC]; +const pineconeNamespaceField: INodeProperties = { + displayName: 'Pinecone Namespace', + name: 'pineconeNamespace', + type: 'string', + description: + 'Partition the records in an index into namespaces. Queries and other operations are then limited to one namespace, so different requests can search different subsets of your index.', + default: '', +}; + const retrieveFields: INodeProperties[] = [ { displayName: 'Options', @@ -16,17 +25,7 @@ const retrieveFields: INodeProperties[] = [ type: 'collection', placeholder: 'Add Option', default: {}, - options: [ - { - displayName: 'Pinecone Namespace', - name: 'pineconeNamespace', - type: 'string', - description: - 'Partition the records in an index into namespaces. Queries and other operations are then limited to one namespace, so different requests can search different subsets of your index.', - default: '', - }, - metadataFilterField, - ], + options: [pineconeNamespaceField, metadataFilterField], }, ]; @@ -45,17 +44,11 @@ const insertFields: INodeProperties[] = [ default: false, description: 'Whether to clear the namespace before inserting new data', }, - { - displayName: 'Pinecone Namespace', - name: 'pineconeNamespace', - type: 'string', - description: - 'Partition the records in an index into namespaces. Queries and other operations are then limited to one namespace, so different requests can search different subsets of your index.', - default: '', - }, + pineconeNamespaceField, ], }, ]; + export const VectorStorePinecone = createVectorStoreNode({ meta: { displayName: 'Pinecone Vector Store', @@ -70,6 +63,7 @@ export const VectorStorePinecone = createVectorStoreNode({ required: true, }, ], + operationModes: ['load', 'insert', 'retrieve', 'update'], }, methods: { listSearch: { pineconeIndexSearch } }, retrieveFields, diff --git a/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreSupabase/VectorStoreSupabase.node.ts b/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreSupabase/VectorStoreSupabase.node.ts index ebeacd9a33..0269a0cef2 100644 --- a/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreSupabase/VectorStoreSupabase.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreSupabase/VectorStoreSupabase.node.ts @@ -6,6 +6,14 @@ import { metadataFilterField } from '../../../utils/sharedFields'; import { supabaseTableNameRLC } from '../shared/descriptions'; import { supabaseTableNameSearch } from '../shared/methods/listSearch'; +const queryNameField: INodeProperties = { + displayName: 'Query Name', + name: 'queryName', + type: 'string', + default: 'match_documents', + description: 'Name of the query to use for matching documents', +}; + const sharedFields: INodeProperties[] = [supabaseTableNameRLC]; const insertFields: INodeProperties[] = [ { @@ -14,17 +22,10 @@ const insertFields: INodeProperties[] = [ type: 'collection', placeholder: 'Add Option', default: {}, - options: [ - { - displayName: 'Query Name', - name: 'queryName', - type: 'string', - default: 'match_documents', - description: 'Name of the query to use for matching documents', - }, - ], + options: [queryNameField], }, ]; + const retrieveFields: INodeProperties[] = [ { displayName: 'Options', @@ -32,18 +33,12 @@ const retrieveFields: INodeProperties[] = [ type: 'collection', placeholder: 'Add Option', default: {}, - options: [ - { - displayName: 'Query Name', - name: 'queryName', - type: 'string', - default: 'match_documents', - description: 'Name of the query to use for matching documents', - }, - metadataFilterField, - ], + options: [queryNameField, metadataFilterField], }, ]; + +const updateFields: INodeProperties[] = [...insertFields]; + export const VectorStoreSupabase = createVectorStoreNode({ meta: { description: 'Work with your data in Supabase Vector Store', @@ -58,6 +53,7 @@ export const VectorStoreSupabase = createVectorStoreNode({ required: true, }, ], + operationModes: ['load', 'insert', 'retrieve', 'update'], }, methods: { listSearch: { supabaseTableNameSearch }, @@ -66,6 +62,7 @@ export const VectorStoreSupabase = createVectorStoreNode({ insertFields, loadFields: retrieveFields, retrieveFields, + updateFields, async getVectorStoreClient(context, filter, embeddings, itemIndex) { const tableName = context.getNodeParameter('tableName', itemIndex, '', { extractValue: true, diff --git a/packages/@n8n/nodes-langchain/nodes/vector_store/shared/createVectorStoreNode.ts b/packages/@n8n/nodes-langchain/nodes/vector_store/shared/createVectorStoreNode.ts index 8994c6ac2b..489c17c976 100644 --- a/packages/@n8n/nodes-langchain/nodes/vector_store/shared/createVectorStoreNode.ts +++ b/packages/@n8n/nodes-langchain/nodes/vector_store/shared/createVectorStoreNode.ts @@ -13,16 +13,21 @@ import type { ILoadOptionsFunctions, INodeListSearchResult, Icon, + INodePropertyOptions, } from 'n8n-workflow'; import type { Embeddings } from '@langchain/core/embeddings'; import type { Document } from '@langchain/core/documents'; import { logWrapper } from '../../../utils/logWrapper'; -import type { N8nJsonLoader } from '../../../utils/N8nJsonLoader'; +import { N8nJsonLoader } from '../../../utils/N8nJsonLoader'; import type { N8nBinaryLoader } from '../../../utils/N8nBinaryLoader'; import { getMetadataFiltersValues, logAiEvent } from '../../../utils/helpers'; import { getConnectionHintNoticeField } from '../../../utils/sharedFields'; import { processDocument } from './processDocuments'; +type NodeOperationMode = 'insert' | 'load' | 'retrieve' | 'update'; + +const DEFAULT_OPERATION_MODES: NodeOperationMode[] = ['load', 'insert', 'retrieve']; + interface NodeMeta { displayName: string; name: string; @@ -30,7 +35,9 @@ interface NodeMeta { docsUrl: string; icon: Icon; credentials?: INodeCredentialDescription[]; + operationModes?: NodeOperationMode[]; } + interface VectorStoreNodeConstructorArgs { meta: NodeMeta; methods?: { @@ -42,10 +49,12 @@ interface VectorStoreNodeConstructorArgs { ) => Promise; }; }; + sharedFields: INodeProperties[]; insertFields?: INodeProperties[]; loadFields?: INodeProperties[]; retrieveFields?: INodeProperties[]; + updateFields?: INodeProperties[]; populateVectorStore: ( context: IExecuteFunctions, embeddings: Embeddings, @@ -60,15 +69,52 @@ interface VectorStoreNodeConstructorArgs { ) => Promise; } -function transformDescriptionForOperationMode( - fields: INodeProperties[], - mode: 'insert' | 'load' | 'retrieve', -) { +function transformDescriptionForOperationMode(fields: INodeProperties[], mode: NodeOperationMode) { return fields.map((field) => ({ ...field, displayOptions: { show: { mode: [mode] } }, })); } + +function isUpdateSupported(args: VectorStoreNodeConstructorArgs): boolean { + return args.meta.operationModes?.includes('update') ?? false; +} + +function getOperationModeOptions(args: VectorStoreNodeConstructorArgs): INodePropertyOptions[] { + const enabledOperationModes = args.meta.operationModes ?? DEFAULT_OPERATION_MODES; + + const allOptions = [ + { + name: 'Get Many', + value: 'load', + description: 'Get many ranked documents from vector store for query', + action: 'Get many ranked documents from vector store for query', + }, + { + name: 'Insert Documents', + value: 'insert', + description: 'Insert documents into vector store', + action: 'Insert documents into vector store', + }, + { + name: 'Retrieve Documents (For Agent/Chain)', + value: 'retrieve', + description: 'Retrieve documents from vector store to be used with AI nodes', + action: 'Retrieve documents from vector store to be used with AI nodes', + }, + { + name: 'Update Documents', + value: 'update', + description: 'Update documents in vector store by ID', + action: 'Update documents in vector store by ID', + }, + ]; + + return allOptions.filter(({ value }) => + enabledOperationModes.includes(value as NodeOperationMode), + ); +} + export const createVectorStoreNode = (args: VectorStoreNodeConstructorArgs) => class VectorStoreNodeType implements INodeType { description: INodeTypeDescription = { @@ -101,11 +147,11 @@ export const createVectorStoreNode = (args: VectorStoreNodeConstructorArgs) => const mode = parameters?.mode; const inputs = [{ displayName: "Embedding", type: "${NodeConnectionType.AiEmbedding}", required: true, maxConnections: 1}] - if (['insert', 'load'].includes(mode)) { + if (['insert', 'load', 'update'].includes(mode)) { inputs.push({ displayName: "", type: "${NodeConnectionType.Main}"}) } - if (mode === 'insert') { + if (['insert'].includes(mode)) { inputs.push({ displayName: "Document", type: "${NodeConnectionType.AiDocument}", required: true, maxConnections: 1}) } return inputs @@ -127,26 +173,7 @@ export const createVectorStoreNode = (args: VectorStoreNodeConstructorArgs) => type: 'options', noDataExpression: true, default: 'retrieve', - options: [ - { - name: 'Get Many', - value: 'load', - description: 'Get many ranked documents from vector store for query', - action: 'Get many ranked documents from vector store for query', - }, - { - name: 'Insert Documents', - value: 'insert', - description: 'Insert documents into vector store', - action: 'Insert documents into vector store', - }, - { - name: 'Retrieve Documents (For Agent/Chain)', - value: 'retrieve', - description: 'Retrieve documents from vector store to be used with AI nodes', - action: 'Retrieve documents from vector store to be used with AI nodes', - }, - ], + options: getOperationModeOptions(args), }, { ...getConnectionHintNoticeField([NodeConnectionType.AiRetriever]), @@ -185,15 +212,30 @@ export const createVectorStoreNode = (args: VectorStoreNodeConstructorArgs) => }, }, }, + // ID is always used for update operation + { + displayName: 'ID', + name: 'id', + type: 'string', + default: '', + required: true, + description: 'ID of an embedding entry', + displayOptions: { + show: { + mode: ['update'], + }, + }, + }, ...transformDescriptionForOperationMode(args.loadFields ?? [], 'load'), ...transformDescriptionForOperationMode(args.retrieveFields ?? [], 'retrieve'), + ...transformDescriptionForOperationMode(args.updateFields ?? [], 'update'), ], }; methods = args.methods; async execute(this: IExecuteFunctions): Promise { - const mode = this.getNodeParameter('mode', 0) as 'load' | 'insert' | 'retrieve'; + const mode = this.getNodeParameter('mode', 0) as NodeOperationMode; const embeddings = (await this.getInputConnectionData( NodeConnectionType.AiEmbedding, @@ -208,7 +250,7 @@ export const createVectorStoreNode = (args: VectorStoreNodeConstructorArgs) => const filter = getMetadataFiltersValues(this, itemIndex); const vectorStore = await args.getVectorStoreClient( this, - // We'll pass filter to similaritySearchVectorWithScore instaed of getVectorStoreClient + // We'll pass filter to similaritySearchVectorWithScore instead of getVectorStoreClient undefined, embeddings, itemIndex, @@ -274,6 +316,60 @@ export const createVectorStoreNode = (args: VectorStoreNodeConstructorArgs) => return [resultData]; } + if (mode === 'update') { + if (!isUpdateSupported(args)) { + throw new NodeOperationError( + this.getNode(), + 'Update operation is not implemented for this Vector Store', + ); + } + + const items = this.getInputData(); + + const loader = new N8nJsonLoader(this); + + const resultData = []; + for (let itemIndex = 0; itemIndex < items.length; itemIndex++) { + const itemData = items[itemIndex]; + + const documentId = this.getNodeParameter('id', itemIndex, '', { + extractValue: true, + }) as string; + + const vectorStore = await args.getVectorStoreClient( + this, + undefined, + embeddings, + itemIndex, + ); + + const { processedDocuments, serializedDocuments } = await processDocument( + loader, + itemData, + itemIndex, + ); + + if (processedDocuments?.length !== 1) { + throw new NodeOperationError(this.getNode(), 'Single document per item expected'); + } + + resultData.push(...serializedDocuments); + + try { + // Use ids option to upsert instead of insert + await vectorStore.addDocuments(processedDocuments, { + ids: [documentId], + }); + + void logAiEvent(this, 'n8n.ai.vector.store.updated'); + } catch (error) { + throw error; + } + } + + return [resultData]; + } + throw new NodeOperationError( this.getNode(), 'Only the "load" and "insert" operation modes are supported with execute', diff --git a/packages/@n8n/nodes-langchain/utils/helpers.ts b/packages/@n8n/nodes-langchain/utils/helpers.ts index 57c54e648c..c6d27ee2f6 100644 --- a/packages/@n8n/nodes-langchain/utils/helpers.ts +++ b/packages/@n8n/nodes-langchain/utils/helpers.ts @@ -27,7 +27,7 @@ export function getMetadataFiltersValues( ctx: IExecuteFunctions, itemIndex: number, ): Record | undefined { - const options = ctx.getNodeParameter('options', itemIndex); + const options = ctx.getNodeParameter('options', itemIndex, {}); if (options.metadata) { const { metadataValues: metadata } = options.metadata as { diff --git a/packages/workflow/src/Interfaces.ts b/packages/workflow/src/Interfaces.ts index 966563748f..057c500b0b 100644 --- a/packages/workflow/src/Interfaces.ts +++ b/packages/workflow/src/Interfaces.ts @@ -2145,6 +2145,7 @@ export const eventNamesAiNodes = [ 'n8n.ai.llm.generated', 'n8n.ai.llm.error', 'n8n.ai.vector.store.populated', + 'n8n.ai.vector.store.updated', ] as const; export type EventNamesAiNodesType = (typeof eventNamesAiNodes)[number];