feat(editor): Use resource locator at Simple Vector Store memory key, allow cross workflow use (#15421)

Remove workflow isolation from in-memory Simple Vector Store, making it possible to use vector stores created on other workflows. Display all current in-memory vector stores with a resource locator at Memory Key picker.

Note that these vector stores are still intended for non-production development use. Any users of an instance can access data in all in-memory vector stores as they aren't bound to workflows.

packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreInMemory/VectorStoreInMemory.node.ts

@@ -1,17 +1,28 @@
+import type { Embeddings } from '@langchain/core/embeddings';
 import type { MemoryVectorStore } from 'langchain/vectorstores/memory';
-import type { INodeProperties } from 'n8n-workflow';
+import {
+	type INodeProperties,
+	type ILoadOptionsFunctions,
+	type INodeListSearchResult,
+	type IDataObject,
+	type NodeParameterValueType,
+	type IExecuteFunctions,
+	type ISupplyDataFunctions,
+	ApplicationError,
+} from 'n8n-workflow';
 
 import { createVectorStoreNode } from '../shared/createVectorStoreNode/createVectorStoreNode';
 import { MemoryVectorStoreManager } from '../shared/MemoryManager/MemoryVectorStoreManager';
 
+const warningBanner: INodeProperties = {
+	displayName:
+		'<strong>For experimental use only</strong>: Data is stored in memory and will be lost if n8n restarts. Data may also be cleared if available memory gets low, and is accessible to all users of this instance. <a href="https://docs.n8n.io/integrations/builtin/cluster-nodes/root-nodes/n8n-nodes-langchain.vectorstoreinmemory/">More info</a>',
+	name: 'notice',
+	type: 'notice',
+	default: '',
+};
+
 const insertFields: INodeProperties[] = [
-	{
-		displayName:
-			'<strong>For experimental use only</strong>: Data is stored in memory and will be lost if n8n restarts. Data may also be cleared if available memory gets low. <a href="https://docs.n8n.io/integrations/builtin/cluster-nodes/root-nodes/n8n-nodes-langchain.vectorstoreinmemory/">More info</a>',
-		name: 'notice',
-		type: 'notice',
-		default: '',
-	},
 	{
 		displayName: 'Clear Store',
 		name: 'clearStore',
@@ -19,8 +30,28 @@ const insertFields: INodeProperties[] = [
 		default: false,
 		description: 'Whether to clear the store before inserting new data',
 	},
+	warningBanner,
 ];
 
+const DEFAULT_MEMORY_KEY = 'vector_store_key';
+
+function getMemoryKey(context: IExecuteFunctions | ISupplyDataFunctions, itemIndex: number) {
+	const node = context.getNode();
+	if (node.typeVersion <= 1.1) {
+		const memoryKeyParam = context.getNodeParameter('memoryKey', itemIndex) as string;
+		const workflowId = context.getWorkflow().id;
+
+		return `${workflowId}__${memoryKeyParam}`;
+	} else {
+		const memoryKeyParam = context.getNodeParameter('memoryKey', itemIndex) as {
+			mode: string;
+			value: string;
+		};
+
+		return memoryKeyParam.value;
+	}
+}
+
 export class VectorStoreInMemory extends createVectorStoreNode<MemoryVectorStore>({
 	meta: {
 		displayName: 'Simple Vector Store',
@@ -37,27 +68,119 @@ export class VectorStoreInMemory extends createVectorStoreNode<MemoryVectorStore
 			displayName: 'Memory Key',
 			name: 'memoryKey',
 			type: 'string',
-			default: 'vector_store_key',
+			default: DEFAULT_MEMORY_KEY,
 			description:
 				'The key to use to store the vector memory in the workflow data. The key will be prefixed with the workflow ID to avoid collisions.',
+			displayOptions: {
+				show: {
+					'@version': [{ _cnd: { lte: 1.1 } }],
+				},
+			},
+		},
+		{
+			displayName: 'Memory Key',
+			name: 'memoryKey',
+			type: 'resourceLocator',
+			required: true,
+			default: { mode: 'list', value: DEFAULT_MEMORY_KEY },
+			description:
+				'The key to use to store the vector memory in the workflow data. These keys are shared between workflows.',
+			displayOptions: {
+				show: {
+					'@version': [{ _cnd: { gte: 1.2 } }],
+				},
+			},
+			modes: [
+				{
+					displayName: 'From List',
+					name: 'list',
+					type: 'list',
+					typeOptions: {
+						searchListMethod: 'vectorStoresSearch',
+						searchable: true,
+						allowNewResource: {
+							label: 'resourceLocator.mode.list.addNewResource.vectorStoreInMemory',
+							defaultName: DEFAULT_MEMORY_KEY,
+							method: 'createVectorStore',
+						},
+					},
+				},
+				{
+					displayName: 'Manual',
+					name: 'id',
+					type: 'string',
+					placeholder: DEFAULT_MEMORY_KEY,
+				},
+			],
 		},
 	],
+	methods: {
+		listSearch: {
+			async vectorStoresSearch(
+				this: ILoadOptionsFunctions,
+				filter?: string,
+			): Promise<INodeListSearchResult> {
+				const vectorStoreSingleton = MemoryVectorStoreManager.getInstance(
+					{} as Embeddings, // Real Embeddings are provided when executing the node
+					this.logger,
+				);
+
+				const searchOptions: INodeListSearchResult['results'] = vectorStoreSingleton
+					.getMemoryKeysList()
+					.map((key) => {
+						return {
+							name: key,
+							value: key,
+						};
+					});
+
+				let results = searchOptions;
+				if (filter) {
+					results = results.filter((option) => option.name.includes(filter));
+				}
+
+				return {
+					results,
+				};
+			},
+		},
+		actionHandler: {
+			async createVectorStore(
+				this: ILoadOptionsFunctions,
+				payload: string | IDataObject | undefined,
+			): Promise<NodeParameterValueType> {
+				if (!payload || typeof payload === 'string') {
+					throw new ApplicationError('Invalid payload type');
+				}
+
+				const { name } = payload;
+
+				const vectorStoreSingleton = MemoryVectorStoreManager.getInstance(
+					{} as Embeddings, // Real Embeddings are provided when executing the node
+					this.logger,
+				);
+
+				const memoryKey = !!name ? (name as string) : DEFAULT_MEMORY_KEY;
+				await vectorStoreSingleton.getVectorStore(memoryKey);
+
+				return memoryKey;
+			},
+		},
+	},
 	insertFields,
-	loadFields: [],
-	retrieveFields: [],
+	loadFields: [warningBanner],
+	retrieveFields: [warningBanner],
 	async getVectorStoreClient(context, _filter, embeddings, itemIndex) {
-		const workflowId = context.getWorkflow().id;
-		const memoryKey = context.getNodeParameter('memoryKey', itemIndex) as string;
+		const memoryKey = getMemoryKey(context, itemIndex);
 		const vectorStoreSingleton = MemoryVectorStoreManager.getInstance(embeddings, context.logger);
 
-		return await vectorStoreSingleton.getVectorStore(`${workflowId}__${memoryKey}`);
+		return await vectorStoreSingleton.getVectorStore(memoryKey);
 	},
 	async populateVectorStore(context, embeddings, documents, itemIndex) {
-		const memoryKey = context.getNodeParameter('memoryKey', itemIndex) as string;
+		const memoryKey = getMemoryKey(context, itemIndex);
 		const clearStore = context.getNodeParameter('clearStore', itemIndex) as boolean;
-		const workflowId = context.getWorkflow().id;
 		const vectorStoreInstance = MemoryVectorStoreManager.getInstance(embeddings, context.logger);
 
-		await vectorStoreInstance.addDocuments(`${workflowId}__${memoryKey}`, documents, clearStore);
+		await vectorStoreInstance.addDocuments(memoryKey, documents, clearStore);
 	},
 }) {}

packages/@n8n/nodes-langchain/nodes/vector_store/shared/MemoryManager/MemoryVectorStoreManager.ts

@@ -124,6 +124,10 @@ export class MemoryVectorStoreManager {
 		}
 	}
 
+	getMemoryKeysList(): string[] {
+		return Array.from(this.vectorStoreBuffer.keys());
+	}
+
 	/**
 	 * Get or create a vector store by key
 	 */

packages/@n8n/nodes-langchain/nodes/vector_store/shared/MemoryManager/test/MemoryVectorStoreManager.test.ts

@@ -246,4 +246,34 @@ describe('MemoryVectorStoreManager', () => {
 		expect(stats.stores.store1.vectors).toBe(50);
 		expect(stats.stores.store2.vectors).toBe(30);
 	});
+
+	it('should list all vector stores', async () => {
+		const embeddings = mock<OpenAIEmbeddings>();
+		const instance = MemoryVectorStoreManager.getInstance(embeddings, logger);
+
+		const mockVectorStore1 = mock<MemoryVectorStore>();
+		// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
+		mockVectorStore1.memoryVectors = new Array(50).fill({
+			embedding: createTestEmbedding(),
+			content: 'test1',
+			metadata: {},
+		});
+
+		const mockVectorStore2 = mock<MemoryVectorStore>();
+		// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
+		mockVectorStore2.memoryVectors = new Array(30).fill({
+			embedding: createTestEmbedding(),
+			content: 'test2',
+			metadata: {},
+		});
+
+		// Mock internal state
+		instance['vectorStoreBuffer'].set('store1', mockVectorStore1);
+		instance['vectorStoreBuffer'].set('store2', mockVectorStore2);
+
+		const list = instance.getMemoryKeysList();
+		expect(list).toHaveLength(2);
+		expect(list[0]).toBe('store1');
+		expect(list[1]).toBe('store2');
+	});
 });

packages/@n8n/nodes-langchain/nodes/vector_store/shared/createVectorStoreNode/__snapshots__/createVectorStoreNode.test.ts.snap

@@ -253,6 +253,7 @@ exports[`createVectorStoreNode retrieve mode supplies vector store as data 1`] =
   "version": [
     1,
     1.1,
+    1.2,
   ],
 }
 `;

packages/@n8n/nodes-langchain/nodes/vector_store/shared/createVectorStoreNode/createVectorStoreNode.ts

@@ -43,7 +43,8 @@ export const createVectorStoreNode = <T extends VectorStore = VectorStore>(
 			icon: args.meta.icon,
 			iconColor: args.meta.iconColor,
 			group: ['transform'],
-			version: [1, 1.1],
+			// 1.2 has changes to VectorStoreInMemory node.
+			version: [1, 1.1, 1.2],
 			defaults: {
 				name: args.meta.displayName,
 			},

packages/@n8n/nodes-langchain/nodes/vector_store/shared/createVectorStoreNode/types.ts

@@ -10,6 +10,8 @@ import type {
 	Icon,
 	ISupplyDataFunctions,
 	ThemeIconColor,
+	IDataObject,
+	NodeParameterValueType,
 } from 'n8n-workflow';
 
 export type NodeOperationMode = 'insert' | 'load' | 'retrieve' | 'update' | 'retrieve-as-tool';
@@ -35,6 +37,12 @@ export interface VectorStoreNodeConstructorArgs<T extends VectorStore = VectorSt
 				paginationToken?: string,
 			) => Promise<INodeListSearchResult>;
 		};
+		actionHandler?: {
+			[functionName: string]: (
+				this: ILoadOptionsFunctions,
+				payload: IDataObject | string | undefined,
+			) => Promise<NodeParameterValueType>;
+		};
 	};
 
 	sharedFields: INodeProperties[];