From 048b9d75890bf27e1dbfbd2862d7377a35e15434 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Tue, 8 Apr 2025 14:34:40 +0200 Subject: [PATCH] feat(Milvus Vector Store Node): Add support for the Milvus vector db (#14404) --- .../credentials/MilvusApi.credentials.ts | 54 +++++++++++ .../VectorStoreMilvus.node.ts | 96 +++++++++++++++++++ .../VectorStoreMilvus/milvus-icon-black.svg | 1 + .../VectorStoreMilvus/milvus-icon-white.svg | 1 + .../methods/listSearch.ts | 23 +++++ .../nodes/vector_store/shared/descriptions.ts | 23 +++++ packages/@n8n/nodes-langchain/package.json | 3 + pnpm-lock.yaml | 53 ++++++++-- 8 files changed, 244 insertions(+), 10 deletions(-) create mode 100644 packages/@n8n/nodes-langchain/credentials/MilvusApi.credentials.ts create mode 100644 packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreMilvus/VectorStoreMilvus.node.ts create mode 100644 packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreMilvus/milvus-icon-black.svg create mode 100644 packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreMilvus/milvus-icon-white.svg diff --git a/packages/@n8n/nodes-langchain/credentials/MilvusApi.credentials.ts b/packages/@n8n/nodes-langchain/credentials/MilvusApi.credentials.ts new file mode 100644 index 0000000000..12c3c66489 --- /dev/null +++ b/packages/@n8n/nodes-langchain/credentials/MilvusApi.credentials.ts @@ -0,0 +1,54 @@ +import type { + ICredentialTestRequest, + ICredentialType, + INodeProperties, + IAuthenticateGeneric, +} from 'n8n-workflow'; + +export class MilvusApi implements ICredentialType { + name = 'milvusApi'; + + displayName = 'Milvus'; + + documentationUrl = 'milvus'; + + properties: INodeProperties[] = [ + { + displayName: 'Base URL', + name: 'baseUrl', + required: true, + type: 'string', + default: 'http://localhost:19530', + }, + { + displayName: 'Username', + name: 'username', + type: 'string', + default: '', + }, + { + displayName: 'Password', + name: 'password', + type: 'string', + typeOptions: { password: true }, + default: '', + }, + ]; + + authenticate: IAuthenticateGeneric = { + type: 'generic', + properties: { + headers: { + Authorization: '=Bearer {{$credentials.username}}:{{$credentials.password}}', + }, + }, + }; + + test: ICredentialTestRequest = { + request: { + baseURL: '={{ $credentials.baseUrl }}', + url: '/v1/vector/collections', + method: 'GET', + }, + }; +} diff --git a/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreMilvus/VectorStoreMilvus.node.ts b/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreMilvus/VectorStoreMilvus.node.ts new file mode 100644 index 0000000000..ff451f4f02 --- /dev/null +++ b/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreMilvus/VectorStoreMilvus.node.ts @@ -0,0 +1,96 @@ +import { Milvus } from '@langchain/community/vectorstores/milvus'; +import type { MilvusLibArgs } from '@langchain/community/vectorstores/milvus'; +import { MilvusClient } from '@zilliz/milvus2-sdk-node'; +import type { INodeProperties } from 'n8n-workflow'; + +import { createVectorStoreNode } from '../shared/createVectorStoreNode/createVectorStoreNode'; +import { milvusCollectionsSearch } from '../shared/createVectorStoreNode/methods/listSearch'; +import { milvusCollectionRLC } from '../shared/descriptions'; + +const sharedFields: INodeProperties[] = [milvusCollectionRLC]; +const insertFields: INodeProperties[] = [ + { + displayName: 'Options', + name: 'options', + type: 'collection', + placeholder: 'Add Option', + default: {}, + options: [ + { + displayName: 'Clear Collection', + name: 'clearCollection', + type: 'boolean', + default: false, + description: 'Whether to clear the collection before inserting new data', + }, + ], + }, +]; + +export class VectorStoreMilvus extends createVectorStoreNode({ + meta: { + displayName: 'Milvus Vector Store', + name: 'vectorStoreMilvus', + description: 'Work with your data in Milvus Vector Store', + icon: { light: 'file:milvus-icon-black.svg', dark: 'file:milvus-icon-white.svg' }, + docsUrl: + 'https://docs.n8n.io/integrations/builtin/cluster-nodes/root-nodes/n8n-nodes-langchain.vectorstoremilvus/', + credentials: [ + { + name: 'milvusApi', + required: true, + }, + ], + operationModes: ['load', 'insert', 'retrieve', 'retrieve-as-tool'], + }, + methods: { listSearch: { milvusCollectionsSearch } }, + sharedFields, + insertFields, + async getVectorStoreClient(context, _filter, embeddings, itemIndex): Promise { + const collection = context.getNodeParameter('milvusCollection', itemIndex, '', { + extractValue: true, + }) as string; + const credentials = await context.getCredentials<{ + baseUrl: string; + username: string; + password: string; + }>('milvusApi'); + const config: MilvusLibArgs = { + url: credentials.baseUrl, + username: credentials.username, + password: credentials.password, + collectionName: collection, + }; + + return await Milvus.fromExistingCollection(embeddings, config); + }, + async populateVectorStore(context, embeddings, documents, itemIndex): Promise { + const collection = context.getNodeParameter('milvusCollection', itemIndex, '', { + extractValue: true, + }) as string; + const options = context.getNodeParameter('options', itemIndex, {}) as { + clearCollection?: boolean; + }; + const credentials = await context.getCredentials<{ + baseUrl: string; + username: string; + password: string; + }>('milvusApi'); + const config: MilvusLibArgs = { + url: credentials.baseUrl, + username: credentials.username, + password: credentials.password, + collectionName: collection, + }; + + if (options.clearCollection) { + const client = new MilvusClient({ + address: credentials.baseUrl, + token: `${credentials.username}:${credentials.password}`, + }); + await client.dropCollection({ collection_name: collection }); + } + + await Milvus.fromDocuments(documents, embeddings, config); + }, +}) {} diff --git a/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreMilvus/milvus-icon-black.svg b/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreMilvus/milvus-icon-black.svg new file mode 100644 index 0000000000..eed02f904a --- /dev/null +++ b/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreMilvus/milvus-icon-black.svg @@ -0,0 +1 @@ +milvus-icon-black \ No newline at end of file diff --git a/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreMilvus/milvus-icon-white.svg b/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreMilvus/milvus-icon-white.svg new file mode 100644 index 0000000000..70df855d61 --- /dev/null +++ b/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreMilvus/milvus-icon-white.svg @@ -0,0 +1 @@ +milvus-icon-white \ No newline at end of file diff --git a/packages/@n8n/nodes-langchain/nodes/vector_store/shared/createVectorStoreNode/methods/listSearch.ts b/packages/@n8n/nodes-langchain/nodes/vector_store/shared/createVectorStoreNode/methods/listSearch.ts index 278d879f90..0a09793772 100644 --- a/packages/@n8n/nodes-langchain/nodes/vector_store/shared/createVectorStoreNode/methods/listSearch.ts +++ b/packages/@n8n/nodes-langchain/nodes/vector_store/shared/createVectorStoreNode/methods/listSearch.ts @@ -1,5 +1,6 @@ import { Pinecone } from '@pinecone-database/pinecone'; import { QdrantClient } from '@qdrant/js-client-rest'; +import { MilvusClient } from '@zilliz/milvus2-sdk-node'; import { ApplicationError, type IDataObject, type ILoadOptionsFunctions } from 'n8n-workflow'; export async function pineconeIndexSearch(this: ILoadOptionsFunctions) { @@ -67,3 +68,25 @@ export async function qdrantCollectionsSearch(this: ILoadOptionsFunctions) { return { results }; } + +export async function milvusCollectionsSearch(this: ILoadOptionsFunctions) { + const credentials = await this.getCredentials<{ + baseUrl: string; + username: string; + password: string; + }>('milvusApi'); + + const client = new MilvusClient({ + address: credentials.baseUrl, + token: `${credentials.username}:${credentials.password}`, + }); + + const response = await client.listCollections(); + + const results = response.data.map((collection) => ({ + name: collection.name, + value: collection.name, + })); + + return { results }; +} diff --git a/packages/@n8n/nodes-langchain/nodes/vector_store/shared/descriptions.ts b/packages/@n8n/nodes-langchain/nodes/vector_store/shared/descriptions.ts index 483e73c1fe..d496c9e39d 100644 --- a/packages/@n8n/nodes-langchain/nodes/vector_store/shared/descriptions.ts +++ b/packages/@n8n/nodes-langchain/nodes/vector_store/shared/descriptions.ts @@ -68,3 +68,26 @@ export const qdrantCollectionRLC: INodeProperties = { }, ], }; + +export const milvusCollectionRLC: INodeProperties = { + displayName: 'Milvus Collection', + name: 'milvusCollection', + type: 'resourceLocator', + default: { mode: 'list', value: '' }, + required: true, + modes: [ + { + displayName: 'From List', + name: 'list', + type: 'list', + typeOptions: { + searchListMethod: 'milvusCollectionsSearch', + }, + }, + { + displayName: 'ID', + name: 'id', + type: 'string', + }, + ], +}; diff --git a/packages/@n8n/nodes-langchain/package.json b/packages/@n8n/nodes-langchain/package.json index c22fc3827c..976e9c7018 100644 --- a/packages/@n8n/nodes-langchain/package.json +++ b/packages/@n8n/nodes-langchain/package.json @@ -31,6 +31,7 @@ "dist/credentials/GroqApi.credentials.js", "dist/credentials/HuggingFaceApi.credentials.js", "dist/credentials/MotorheadApi.credentials.js", + "dist/credentials/MilvusApi.credentials.js", "dist/credentials/MistralCloudApi.credentials.js", "dist/credentials/OllamaApi.credentials.js", "dist/credentials/OpenRouterApi.credentials.js", @@ -115,6 +116,7 @@ "dist/nodes/vector_store/VectorStoreInMemory/VectorStoreInMemory.node.js", "dist/nodes/vector_store/VectorStoreInMemoryInsert/VectorStoreInMemoryInsert.node.js", "dist/nodes/vector_store/VectorStoreInMemoryLoad/VectorStoreInMemoryLoad.node.js", + "dist/nodes/vector_store/VectorStoreMilvus/VectorStoreMilvus.node.js", "dist/nodes/vector_store/VectorStoreMongoDBAtlas/VectorStoreMongoDBAtlas.node.js", "dist/nodes/vector_store/VectorStorePGVector/VectorStorePGVector.node.js", "dist/nodes/vector_store/VectorStorePinecone/VectorStorePinecone.node.js", @@ -174,6 +176,7 @@ "@qdrant/js-client-rest": "1.11.0", "@supabase/supabase-js": "2.45.4", "@xata.io/client": "0.28.4", + "@zilliz/milvus2-sdk-node": "^2.5.7", "basic-auth": "catalog:", "cheerio": "1.0.0", "cohere-ai": "7.14.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a5ff1a8880..7301392df4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -576,7 +576,7 @@ importers: version: 0.3.2(@aws-sdk/client-sso-oidc@3.666.0(@aws-sdk/client-sts@3.666.0))(@langchain/core@0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.24.1)))(encoding@0.1.13) '@langchain/community': specifier: 0.3.24 - version: 0.3.24(a04a643b073c1ba3ea97b63ddf599935) + version: 0.3.24(c5fc7e11d6e6167a46cb8d3fd9b490a5) '@langchain/core': specifier: 'catalog:' version: 0.3.30(openai@4.78.1(encoding@0.1.13)(zod@3.24.1)) @@ -640,6 +640,9 @@ importers: '@xata.io/client': specifier: 0.28.4 version: 0.28.4(typescript@5.8.2) + '@zilliz/milvus2-sdk-node': + specifier: ^2.5.7 + version: 2.5.7 basic-auth: specifier: 'catalog:' version: 2.0.1 @@ -3726,6 +3729,10 @@ packages: resolution: {integrity: sha512-vYVqYzHicDqyKB+NQhAc54I1QWCBLCrYG6unqOIcBTHx+7x8C9lcoLj3KVJXs2VB4lUbpWY+Kk9NipcbXYWmvg==} engines: {node: '>=12.10.0'} + '@grpc/grpc-js@1.13.2': + resolution: {integrity: sha512-nnR5nmL6lxF8YBqb6gWvEgLdLh/Fn+kvAdX5hUOnt48sNSb0riz/93ASd2E5gvanPA41X6Yp25bIfGRp1SMb2g==} + engines: {node: '>=12.10.0'} + '@grpc/proto-loader@0.7.13': resolution: {integrity: sha512-AiXO/bfe9bmxBjxxtYxFAXGZvMaN5s8kO+jBHAJCON8rJoB5YS/D6X7ZNc6XQkuHNmyl4CYaMI1fJ/Gn27RGGw==} engines: {node: '>=6'} @@ -4970,6 +4977,9 @@ packages: '@otplib/preset-v11@12.0.1': resolution: {integrity: sha512-9hSetMI7ECqbFiKICrNa4w70deTUfArtwXykPUvSHWOdzOlfa9ajglu7mNCntlvxycTiOAXkQGwjQCzzDEMRMg==} + '@petamoriken/float16@3.9.2': + resolution: {integrity: sha512-VgffxawQde93xKxT3qap3OH+meZf7VaSB5Sqd4Rqc+FP5alWbpOyan/7tRbOAvynjpG3GpdtAuGU/NdhQpmrog==} + '@pinecone-database/pinecone@4.0.0': resolution: {integrity: sha512-INYS+GBys9v5BRTyn0tv8srVsPTlSRvE3BPE4Wkc/lOEyAIyB9F7DEMXbeF19FOLEgRwCuHTLjzm1niENl+4FA==} engines: {node: '>=18.0.0'} @@ -6640,6 +6650,9 @@ packages: resolution: {integrity: sha512-2WALfTl4xo2SkGCYRt6rDTFfk9R1czmBvUQy12gK2KuRKIpWEhcbbzy8EZXtz/jkRqHX8bFEc6FC1HjX4TUWYw==} engines: {node: '>=10.0.0'} + '@zilliz/milvus2-sdk-node@2.5.7': + resolution: {integrity: sha512-5gvIlllZCDkbcy1O9WjV4bZWgq+mlNekl5W4JcyO0HpyPcovoGF6cgWksl38kjXSl7WUSt1jWm2Q71Ua3r6foA==} + abab@2.0.6: resolution: {integrity: sha512-j2afSsaIENvHZN2B8GOpF566vZ5WVk5opAiMTvWgaQT8DkbOqsTfvNAvHoRGU2zzP8cPoqys+xHTRDWW8L+/BA==} deprecated: Use your platform's native atob() and btoa() methods instead @@ -16195,11 +16208,16 @@ snapshots: '@grpc/proto-loader': 0.7.13 '@js-sdsl/ordered-map': 4.4.2 + '@grpc/grpc-js@1.13.2': + dependencies: + '@grpc/proto-loader': 0.7.13 + '@js-sdsl/ordered-map': 4.4.2 + '@grpc/proto-loader@0.7.13': dependencies: lodash.camelcase: 4.3.0 long: 5.2.3 - protobufjs: 7.3.0 + protobufjs: 7.4.0 yargs: 17.7.2 '@hapi/hoek@9.3.0': {} @@ -16637,7 +16655,7 @@ snapshots: - aws-crt - encoding - '@langchain/community@0.3.24(a04a643b073c1ba3ea97b63ddf599935)': + '@langchain/community@0.3.24(c5fc7e11d6e6167a46cb8d3fd9b490a5)': dependencies: '@browserbasehq/stagehand': 1.9.0(@playwright/test@1.49.1)(deepmerge@4.3.1)(dotenv@16.4.5)(encoding@0.1.13)(openai@4.78.1(encoding@0.1.13)(zod@3.24.1))(zod@3.24.1) '@ibm-cloud/watsonx-ai': 1.1.2 @@ -16677,6 +16695,7 @@ snapshots: '@smithy/util-utf8': 2.3.0 '@supabase/supabase-js': 2.45.4 '@xata.io/client': 0.28.4(typescript@5.8.2) + '@zilliz/milvus2-sdk-node': 2.5.7 cheerio: 1.0.0 cohere-ai: 7.14.0(@aws-sdk/client-sso-oidc@3.666.0(@aws-sdk/client-sts@3.666.0))(encoding@0.1.13) crypto-js: 4.2.0 @@ -17438,6 +17457,8 @@ snapshots: '@otplib/plugin-crypto': 12.0.1 '@otplib/plugin-thirty-two': 12.0.1 + '@petamoriken/float16@3.9.2': {} + '@pinecone-database/pinecone@4.0.0': dependencies: encoding: 0.1.13 @@ -19656,6 +19677,18 @@ snapshots: '@xmldom/xmldom@0.8.10': {} + '@zilliz/milvus2-sdk-node@2.5.7': + dependencies: + '@grpc/grpc-js': 1.13.2 + '@grpc/proto-loader': 0.7.13 + '@opentelemetry/api': 1.9.0 + '@petamoriken/float16': 3.9.2 + dayjs: 1.11.10 + generic-pool: 3.9.0 + lru-cache: 9.1.2 + protobufjs: 7.4.0 + winston: 3.14.2 + abab@2.0.6: {} abbrev@1.1.1: {} @@ -21664,7 +21697,7 @@ snapshots: eslint-import-resolver-node@0.3.9: dependencies: - debug: 3.2.7(supports-color@5.5.0) + debug: 3.2.7(supports-color@8.1.1) is-core-module: 2.13.1 resolve: 1.22.8 transitivePeerDependencies: @@ -21689,7 +21722,7 @@ snapshots: eslint-module-utils@2.8.0(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.8.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0): dependencies: - debug: 3.2.7(supports-color@5.5.0) + debug: 3.2.7(supports-color@8.1.1) optionalDependencies: '@typescript-eslint/parser': 7.2.0(eslint@8.57.0)(typescript@5.8.2) eslint: 8.57.0 @@ -21709,7 +21742,7 @@ snapshots: array.prototype.findlastindex: 1.2.3 array.prototype.flat: 1.3.2 array.prototype.flatmap: 1.3.2 - debug: 3.2.7(supports-color@5.5.0) + debug: 3.2.7(supports-color@8.1.1) doctrine: 2.1.0 eslint: 8.57.0 eslint-import-resolver-node: 0.3.9 @@ -22522,7 +22555,7 @@ snapshots: array-parallel: 0.1.3 array-series: 0.1.5 cross-spawn: 4.0.2 - debug: 3.2.7(supports-color@5.5.0) + debug: 3.2.7(supports-color@8.1.1) transitivePeerDependencies: - supports-color @@ -25394,7 +25427,7 @@ snapshots: pdf-parse@1.1.1: dependencies: - debug: 3.2.7(supports-color@5.5.0) + debug: 3.2.7(supports-color@8.1.1) node-ensure: 0.0.0 transitivePeerDependencies: - supports-color @@ -25721,7 +25754,7 @@ snapshots: proto3-json-serializer@2.0.2: dependencies: - protobufjs: 7.3.0 + protobufjs: 7.4.0 protobufjs@7.3.0: dependencies: @@ -26238,7 +26271,7 @@ snapshots: rhea@1.0.24: dependencies: - debug: 3.2.7(supports-color@5.5.0) + debug: 3.2.7(supports-color@8.1.1) transitivePeerDependencies: - supports-color