feat(editor): Use resource locator at Simple Vector Store memory key, allow cross workflow use (#15421)

Remove workflow isolation from in-memory Simple Vector Store, making it possible to use vector stores created on other workflows. Display all current in-memory vector stores with a resource locator at Memory Key picker.

Note that these vector stores are still intended for non-production development use. Any users of an instance can access data in all in-memory vector stores as they aren't bound to workflows.
This commit is contained in:
Jaakko Husso
2025-05-22 23:34:59 +03:00
committed by GitHub
parent a86bc43f50
commit e5c2aea6fe
16 changed files with 392 additions and 36 deletions

View File

@@ -208,6 +208,21 @@ export function typeIntoFixedCollectionItem(collectionName: string, index: numbe
);
}
export function selectResourceLocatorAddResourceItem(
resourceLocator: string,
expectedText: string,
) {
clickResourceLocatorInput(resourceLocator);
// getVisiblePopper().findChildByTestId('rlc-item-add-resource').eq(0).should('exist');
getVisiblePopper()
.findChildByTestId('rlc-item-add-resource')
.eq(0)
.find('span')
.should('contain.text', expectedText)
.click();
}
export function selectResourceLocatorItem(
resourceLocator: string,
index: number,

View File

@@ -3,7 +3,7 @@ import { setCredentialValues } from '../composables/modals/credential-modal';
import {
clickCreateNewCredential,
getNdvContainer,
selectResourceLocatorItem,
selectResourceLocatorAddResourceItem,
} from '../composables/ndv';
import * as projects from '../composables/projects';
import {
@@ -299,9 +299,11 @@ describe('Projects', { disableAutoLogin: true }, () => {
workflowPage.actions.saveWorkflowOnButtonClick();
workflowPage.actions.addNodeToCanvas('Execute Workflow', true, true);
// This mock fails when running with `test:e2e:dev` but works with `test:e2e:ui`,
// at least on macOS at version 1.94.0. ¯\_(ツ)_/¯
cy.window().then((win) => cy.stub(win, 'open').callsFake((url) => cy.visit(url)));
selectResourceLocatorItem('workflowId', 0, 'Create a');
selectResourceLocatorAddResourceItem('workflowId', 'Create a');
// Need to wait for the trigger node to auto-open after a delay
getNdvContainer().should('be.visible');
cy.get('body').type('{esc}');

View File

@@ -27,7 +27,9 @@ describe('Workflow Selector Parameter', () => {
getVisiblePopper()
.should('have.length', 1)
.findChildByTestId('rlc-item')
.should('have.length', 3);
.should('have.length', 2);
getVisiblePopper().findChildByTestId('rlc-item-add-resource').should('have.length', 1);
});
it('should show required parameter warning', () => {
@@ -44,8 +46,8 @@ describe('Workflow Selector Parameter', () => {
getVisiblePopper()
.should('have.length', 1)
.findChildByTestId('rlc-item')
.should('have.length', 2)
.eq(1)
.should('have.length', 1)
.eq(0)
.click();
ndv.getters
@@ -91,14 +93,14 @@ describe('Workflow Selector Parameter', () => {
ndv.getters.resourceLocator('workflowId').should('be.visible');
ndv.getters.resourceLocatorInput('workflowId').click();
getVisiblePopper().findChildByTestId('rlc-item').eq(0).should('exist');
getVisiblePopper().findChildByTestId('rlc-item-add-resource').eq(0).should('exist');
getVisiblePopper()
.findChildByTestId('rlc-item')
.findChildByTestId('rlc-item-add-resource')
.eq(0)
.find('span')
.should('contain.text', 'Create a'); // Due to some inconsistency we're sometimes in a project and sometimes not, this covers both cases
getVisiblePopper().findChildByTestId('rlc-item').eq(0).click();
getVisiblePopper().findChildByTestId('rlc-item-add-resource').eq(0).click();
cy.wait('@createSubworkflow').then((interception) => {
expect(interception.request.body).to.have.property('name').that.includes('Sub-Workflow');

View File

@@ -8,6 +8,7 @@ import {
getParameterInputByName,
populateFixedCollection,
selectResourceLocatorItem,
selectResourceLocatorAddResourceItem,
typeIntoFixedCollectionItem,
clickWorkflowCardContent,
assertOutputTableContent,
@@ -63,7 +64,7 @@ describe('Sub-workflow creation and typed usage', () => {
openedUrl = url;
});
});
selectResourceLocatorItem('workflowId', 0, 'Create a');
selectResourceLocatorAddResourceItem('workflowId', 'Create a');
cy.then(() => cy.visit(openedUrl));
// **************************
// NAVIGATE TO CHILD WORKFLOW
@@ -142,7 +143,7 @@ describe('Sub-workflow creation and typed usage', () => {
cy.window().then((win) => {
cy.stub(win, 'open').callsFake((url) => {
cy.visit(url);
selectResourceLocatorItem('workflowId', 0, 'Create a');
selectResourceLocatorAddResourceItem('workflowId', 'Create a');
openNode('When Executed by Another Workflow');
@@ -218,7 +219,7 @@ function validateAndReturnToParent(targetChild: string, offset: number, fields:
// Note that outside of e2e tests this will be pre-selected correctly.
// Due to our workaround to remain in the same tab we need to select the correct tab manually
selectResourceLocatorItem('workflowId', offset, targetChild);
selectResourceLocatorItem('workflowId', offset - 1, targetChild);
clickExecuteNode();

View File

@@ -1,17 +1,28 @@
import type { Embeddings } from '@langchain/core/embeddings';
import type { MemoryVectorStore } from 'langchain/vectorstores/memory';
import type { INodeProperties } from 'n8n-workflow';
import {
type INodeProperties,
type ILoadOptionsFunctions,
type INodeListSearchResult,
type IDataObject,
type NodeParameterValueType,
type IExecuteFunctions,
type ISupplyDataFunctions,
ApplicationError,
} from 'n8n-workflow';
import { createVectorStoreNode } from '../shared/createVectorStoreNode/createVectorStoreNode';
import { MemoryVectorStoreManager } from '../shared/MemoryManager/MemoryVectorStoreManager';
const warningBanner: INodeProperties = {
displayName:
'<strong>For experimental use only</strong>: Data is stored in memory and will be lost if n8n restarts. Data may also be cleared if available memory gets low, and is accessible to all users of this instance. <a href="https://docs.n8n.io/integrations/builtin/cluster-nodes/root-nodes/n8n-nodes-langchain.vectorstoreinmemory/">More info</a>',
name: 'notice',
type: 'notice',
default: '',
};
const insertFields: INodeProperties[] = [
{
displayName:
'<strong>For experimental use only</strong>: Data is stored in memory and will be lost if n8n restarts. Data may also be cleared if available memory gets low. <a href="https://docs.n8n.io/integrations/builtin/cluster-nodes/root-nodes/n8n-nodes-langchain.vectorstoreinmemory/">More info</a>',
name: 'notice',
type: 'notice',
default: '',
},
{
displayName: 'Clear Store',
name: 'clearStore',
@@ -19,8 +30,28 @@ const insertFields: INodeProperties[] = [
default: false,
description: 'Whether to clear the store before inserting new data',
},
warningBanner,
];
const DEFAULT_MEMORY_KEY = 'vector_store_key';
function getMemoryKey(context: IExecuteFunctions | ISupplyDataFunctions, itemIndex: number) {
const node = context.getNode();
if (node.typeVersion <= 1.1) {
const memoryKeyParam = context.getNodeParameter('memoryKey', itemIndex) as string;
const workflowId = context.getWorkflow().id;
return `${workflowId}__${memoryKeyParam}`;
} else {
const memoryKeyParam = context.getNodeParameter('memoryKey', itemIndex) as {
mode: string;
value: string;
};
return memoryKeyParam.value;
}
}
export class VectorStoreInMemory extends createVectorStoreNode<MemoryVectorStore>({
meta: {
displayName: 'Simple Vector Store',
@@ -37,27 +68,119 @@ export class VectorStoreInMemory extends createVectorStoreNode<MemoryVectorStore
displayName: 'Memory Key',
name: 'memoryKey',
type: 'string',
default: 'vector_store_key',
default: DEFAULT_MEMORY_KEY,
description:
'The key to use to store the vector memory in the workflow data. The key will be prefixed with the workflow ID to avoid collisions.',
displayOptions: {
show: {
'@version': [{ _cnd: { lte: 1.1 } }],
},
},
},
{
displayName: 'Memory Key',
name: 'memoryKey',
type: 'resourceLocator',
required: true,
default: { mode: 'list', value: DEFAULT_MEMORY_KEY },
description:
'The key to use to store the vector memory in the workflow data. These keys are shared between workflows.',
displayOptions: {
show: {
'@version': [{ _cnd: { gte: 1.2 } }],
},
},
modes: [
{
displayName: 'From List',
name: 'list',
type: 'list',
typeOptions: {
searchListMethod: 'vectorStoresSearch',
searchable: true,
allowNewResource: {
label: 'resourceLocator.mode.list.addNewResource.vectorStoreInMemory',
defaultName: DEFAULT_MEMORY_KEY,
method: 'createVectorStore',
},
},
},
{
displayName: 'Manual',
name: 'id',
type: 'string',
placeholder: DEFAULT_MEMORY_KEY,
},
],
},
],
methods: {
listSearch: {
async vectorStoresSearch(
this: ILoadOptionsFunctions,
filter?: string,
): Promise<INodeListSearchResult> {
const vectorStoreSingleton = MemoryVectorStoreManager.getInstance(
{} as Embeddings, // Real Embeddings are provided when executing the node
this.logger,
);
const searchOptions: INodeListSearchResult['results'] = vectorStoreSingleton
.getMemoryKeysList()
.map((key) => {
return {
name: key,
value: key,
};
});
let results = searchOptions;
if (filter) {
results = results.filter((option) => option.name.includes(filter));
}
return {
results,
};
},
},
actionHandler: {
async createVectorStore(
this: ILoadOptionsFunctions,
payload: string | IDataObject | undefined,
): Promise<NodeParameterValueType> {
if (!payload || typeof payload === 'string') {
throw new ApplicationError('Invalid payload type');
}
const { name } = payload;
const vectorStoreSingleton = MemoryVectorStoreManager.getInstance(
{} as Embeddings, // Real Embeddings are provided when executing the node
this.logger,
);
const memoryKey = !!name ? (name as string) : DEFAULT_MEMORY_KEY;
await vectorStoreSingleton.getVectorStore(memoryKey);
return memoryKey;
},
},
},
insertFields,
loadFields: [],
retrieveFields: [],
loadFields: [warningBanner],
retrieveFields: [warningBanner],
async getVectorStoreClient(context, _filter, embeddings, itemIndex) {
const workflowId = context.getWorkflow().id;
const memoryKey = context.getNodeParameter('memoryKey', itemIndex) as string;
const memoryKey = getMemoryKey(context, itemIndex);
const vectorStoreSingleton = MemoryVectorStoreManager.getInstance(embeddings, context.logger);
return await vectorStoreSingleton.getVectorStore(`${workflowId}__${memoryKey}`);
return await vectorStoreSingleton.getVectorStore(memoryKey);
},
async populateVectorStore(context, embeddings, documents, itemIndex) {
const memoryKey = context.getNodeParameter('memoryKey', itemIndex) as string;
const memoryKey = getMemoryKey(context, itemIndex);
const clearStore = context.getNodeParameter('clearStore', itemIndex) as boolean;
const workflowId = context.getWorkflow().id;
const vectorStoreInstance = MemoryVectorStoreManager.getInstance(embeddings, context.logger);
await vectorStoreInstance.addDocuments(`${workflowId}__${memoryKey}`, documents, clearStore);
await vectorStoreInstance.addDocuments(memoryKey, documents, clearStore);
},
}) {}

View File

@@ -124,6 +124,10 @@ export class MemoryVectorStoreManager {
}
}
getMemoryKeysList(): string[] {
return Array.from(this.vectorStoreBuffer.keys());
}
/**
* Get or create a vector store by key
*/

View File

@@ -246,4 +246,34 @@ describe('MemoryVectorStoreManager', () => {
expect(stats.stores.store1.vectors).toBe(50);
expect(stats.stores.store2.vectors).toBe(30);
});
it('should list all vector stores', async () => {
const embeddings = mock<OpenAIEmbeddings>();
const instance = MemoryVectorStoreManager.getInstance(embeddings, logger);
const mockVectorStore1 = mock<MemoryVectorStore>();
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
mockVectorStore1.memoryVectors = new Array(50).fill({
embedding: createTestEmbedding(),
content: 'test1',
metadata: {},
});
const mockVectorStore2 = mock<MemoryVectorStore>();
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
mockVectorStore2.memoryVectors = new Array(30).fill({
embedding: createTestEmbedding(),
content: 'test2',
metadata: {},
});
// Mock internal state
instance['vectorStoreBuffer'].set('store1', mockVectorStore1);
instance['vectorStoreBuffer'].set('store2', mockVectorStore2);
const list = instance.getMemoryKeysList();
expect(list).toHaveLength(2);
expect(list[0]).toBe('store1');
expect(list[1]).toBe('store2');
});
});

View File

@@ -253,6 +253,7 @@ exports[`createVectorStoreNode retrieve mode supplies vector store as data 1`] =
"version": [
1,
1.1,
1.2,
],
}
`;

View File

@@ -43,7 +43,8 @@ export const createVectorStoreNode = <T extends VectorStore = VectorStore>(
icon: args.meta.icon,
iconColor: args.meta.iconColor,
group: ['transform'],
version: [1, 1.1],
// 1.2 has changes to VectorStoreInMemory node.
version: [1, 1.1, 1.2],
defaults: {
name: args.meta.displayName,
},

View File

@@ -10,6 +10,8 @@ import type {
Icon,
ISupplyDataFunctions,
ThemeIconColor,
IDataObject,
NodeParameterValueType,
} from 'n8n-workflow';
export type NodeOperationMode = 'insert' | 'load' | 'retrieve' | 'update' | 'retrieve-as-tool';
@@ -35,6 +37,12 @@ export interface VectorStoreNodeConstructorArgs<T extends VectorStore = VectorSt
paginationToken?: string,
) => Promise<INodeListSearchResult>;
};
actionHandler?: {
[functionName: string]: (
this: ILoadOptionsFunctions,
payload: IDataObject | string | undefined,
) => Promise<NodeParameterValueType>;
};
};
sharedFields: INodeProperties[];

View File

@@ -49,6 +49,27 @@ export const TEST_PARAMETER_SINGLE_MODE: INodeProperties = {
],
};
export const TEST_PARAMETER_ADD_RESOURCE: INodeProperties = {
...TEST_PARAMETER_MULTI_MODE,
name: 'testParameterAddResource',
modes: [
{
displayName: 'From List',
name: 'list',
type: 'list',
typeOptions: {
searchListMethod: 'testSearch',
searchable: true,
allowNewResource: {
label: 'resourceLocator.mode.list.addNewResource.vectorStoreInMemory',
method: 'testAddResource',
defaultName: 'test',
},
},
},
],
};
export const TEST_NODE_MULTI_MODE: INode = {
type: 'n8n-nodes-base.airtable',
typeVersion: 2.1,

View File

@@ -9,6 +9,7 @@ import {
TEST_MODEL_VALUE,
TEST_NODE_MULTI_MODE,
TEST_NODE_SINGLE_MODE,
TEST_PARAMETER_ADD_RESOURCE,
TEST_PARAMETER_MULTI_MODE,
TEST_PARAMETER_SINGLE_MODE,
} from './ResourceLocator.test.constants';
@@ -133,6 +134,69 @@ describe('ResourceLocator', () => {
});
});
it('renders add resource button', async () => {
nodeTypesStore.getResourceLocatorResults.mockResolvedValue({
results: [],
paginationToken: null,
});
const { getByTestId } = renderComponent({
props: {
modelValue: TEST_MODEL_VALUE,
parameter: TEST_PARAMETER_ADD_RESOURCE,
path: `parameters.${TEST_PARAMETER_ADD_RESOURCE.name}`,
node: TEST_NODE_SINGLE_MODE,
displayTitle: 'Test Resource Locator',
expressionComputedValue: '',
},
});
expect(getByTestId(`resource-locator-${TEST_PARAMETER_ADD_RESOURCE.name}`)).toBeInTheDocument();
// Click on the input to open it
await userEvent.click(getByTestId('rlc-input'));
// Expect the button to create a new resource to be rendered
expect(getByTestId('rlc-item-add-resource')).toBeInTheDocument();
});
it('creates new resource passing search filter as name', async () => {
nodeTypesStore.getResourceLocatorResults.mockResolvedValue({
results: [],
paginationToken: null,
});
nodeTypesStore.getNodeParameterActionResult.mockResolvedValue('new-resource');
const { getByTestId } = renderComponent({
props: {
modelValue: TEST_MODEL_VALUE,
parameter: TEST_PARAMETER_ADD_RESOURCE,
path: `parameters.${TEST_PARAMETER_ADD_RESOURCE.name}`,
node: TEST_NODE_SINGLE_MODE,
displayTitle: 'Test Resource Locator',
expressionComputedValue: '',
},
});
// Click on the input to open it
await userEvent.click(getByTestId('rlc-input'));
// Type in the input to name the resource
await userEvent.type(getByTestId('rlc-search'), 'Test Resource');
// Click on the add resource button
await userEvent.click(getByTestId('rlc-item-add-resource'));
expect(nodeTypesStore.getNodeParameterActionResult).toHaveBeenCalledWith({
nodeTypeAndVersion: {
name: TEST_NODE_SINGLE_MODE.type,
version: TEST_NODE_SINGLE_MODE.typeVersion,
},
path: `parameters.${TEST_PARAMETER_ADD_RESOURCE.name}`,
currentNodeParameters: expect.any(Object),
credentials: TEST_NODE_SINGLE_MODE.credentials,
handler: 'testAddResource',
payload: {
name: 'Test Resource',
},
});
});
// Testing error message deduplication
describe('ResourceLocator credentials error handling', () => {
it.each([

View File

@@ -1,5 +1,5 @@
<script setup lang="ts">
import type { ResourceLocatorRequestDto } from '@n8n/api-types';
import type { ResourceLocatorRequestDto, ActionResultRequestDto } from '@n8n/api-types';
import type { IResourceLocatorResultExpanded, IUpdateInformation } from '@/Interface';
import DraggableTarget from '@/components/DraggableTarget.vue';
import ExpressionParameterInput from '@/components/ExpressionParameterInput.vue';
@@ -55,6 +55,7 @@ import {
} from '../../utils/fromAIOverrideUtils';
import { N8nNotice } from '@n8n/design-system';
import { completeExpressionSyntax } from '@/utils/expressions';
import type { BaseTextKey } from '@/plugins/i18n';
/**
* Regular expression to check if the error message contains credential-related phrases.
@@ -346,6 +347,70 @@ const showOverrideButton = computed(
() => canBeContentOverride.value && !isContentOverride.value && !props.isReadOnly,
);
const allowNewResources = computed(() => {
if (!props.node) {
return undefined;
}
const addNewResourceOptions = getPropertyArgument(currentMode.value, 'allowNewResource');
if (!addNewResourceOptions || typeof addNewResourceOptions !== 'object') {
return undefined;
}
return {
label: i18n.baseText(addNewResourceOptions.label as BaseTextKey, {
interpolate: {
resourceName: !!searchFilter.value ? searchFilter.value : addNewResourceOptions.defaultName,
},
}),
method: addNewResourceOptions.method,
};
});
const handleAddResourceClick = async () => {
if (!props.node || !allowNewResources.value) {
return;
}
const { method: addNewResourceMethodName } = allowNewResources.value;
const resolvedNodeParameters = workflowHelpers.resolveRequiredParameters(
props.parameter,
currentRequestParams.value.parameters,
);
if (!resolvedNodeParameters || !addNewResourceMethodName) {
return;
}
const requestParams: ActionResultRequestDto = {
nodeTypeAndVersion: {
name: props.node.type,
version: props.node.typeVersion,
},
path: props.path,
currentNodeParameters: resolvedNodeParameters,
credentials: props.node.credentials,
handler: addNewResourceMethodName,
payload: {
name: searchFilter.value,
},
};
const newResource = (await nodeTypesStore.getNodeParameterActionResult(
requestParams,
)) as NodeParameterValue;
refreshList();
await loadResources();
searchFilter.value = '';
onListItemSelected(newResource);
};
const onAddResourceClicked = computed(() =>
allowNewResources.value ? handleAddResourceClick : undefined,
);
watch(currentQueryError, (curr, prev) => {
if (resourceDropdownVisible.value && curr && !prev) {
if (inputRef.value) {
@@ -453,7 +518,7 @@ function openResource(url: string) {
function getPropertyArgument(
parameter: INodePropertyMode,
argumentName: keyof INodePropertyModeTypeOptions,
): string | number | boolean | undefined {
): string | number | boolean | INodePropertyModeTypeOptions['allowNewResource'] | undefined {
return parameter.typeOptions?.[argumentName];
}
@@ -838,9 +903,11 @@ function removeOverride() {
:error-view="currentQueryError"
:width="width"
:event-bus="eventBus"
:allow-new-resources="allowNewResources"
@update:model-value="onListItemSelected"
@filter="onSearchFilter"
@load-more="loadResourcesDebounced"
@add-resource-click="onAddResourceClicked"
>
<template #error>
<div :class="$style.errorContainer" data-test-id="rlc-error-container">

View File

@@ -128,7 +128,8 @@ function openUrl(event: MouseEvent, url: string) {
function onKeyDown(e: KeyboardEvent) {
if (e.key === 'ArrowDown') {
if (hoverIndex.value < sortedResources.value.length - 1) {
// hoverIndex 0 is reserved for the "add new resource" item
if (hoverIndex.value < sortedResources.value.length) {
hoverIndex.value++;
if (resultsContainerRef.value && itemsRef.value.length === 1) {
@@ -155,7 +156,12 @@ function onKeyDown(e: KeyboardEvent) {
}
}
} else if (e.key === 'Enter') {
const selected = sortedResources.value[hoverIndex.value]?.value;
if (hoverIndex.value === 0 && props.allowNewResources.label) {
emit('addResourceClick');
return;
}
const selected = sortedResources.value[hoverIndex.value - 1]?.value;
// Selected resource can be empty when loading or empty results
if (selected) {
@@ -225,7 +231,11 @@ defineExpose({ isWithinDropdown });
ref="searchRef"
:model-value="filter"
:clearable="true"
:placeholder="i18n.baseText('resourceLocator.search.placeholder')"
:placeholder="
allowNewResources.label
? i18n.baseText('resourceLocator.placeholder.searchOrCreate')
: i18n.baseText('resourceLocator.placeholder.search')
"
data-test-id="rlc-search"
@update:model-value="onFilterInput"
>
@@ -253,7 +263,7 @@ defineExpose({ isWithinDropdown });
v-if="allowNewResources.label"
key="addResourceKey"
ref="itemsRef"
data-test-id="rlc-item"
data-test-id="rlc-item-add-resource"
:class="{
[$style.resourceItem]: true,
[$style.hovering]: hoverIndex === 0,

View File

@@ -1722,10 +1722,12 @@
"resourceLocator.mode.list.openUrl": "Open URL",
"resourceLocator.mode.list.placeholder": "Choose...",
"resourceLocator.mode.list.searchRequired": "Enter a search term to show results",
"resourceLocator.mode.list.addNewResource.vectorStoreInMemory": "Create key '{resourceName}'",
"resourceLocator.modeSelector.placeholder": "Mode...",
"resourceLocator.openSpecificResource": "Open {entity} in {appName}",
"resourceLocator.openResource": "Open in {appName}",
"resourceLocator.search.placeholder": "Search...",
"resourceLocator.placeholder.searchOrCreate": "Search or create...",
"resourceLocator.placeholder.search": "Search...",
"resourceLocator.url.placeholder": "Enter URL...",
"resourceMapper.autoMappingNotice": "In this mode, make sure the incoming data fields are named the same as the {fieldWord} in {serviceName}. (Use an 'Edit Fields' node before this node to change them if required.)",
"resourceMapper.mappingMode.label": "Mapping Column Mode",

View File

@@ -1444,6 +1444,11 @@ export interface INodePropertyModeTypeOptions {
searchListMethod?: string; // Supported by: options
searchFilterRequired?: boolean;
searchable?: boolean;
allowNewResource?: {
label: string;
defaultName: string;
method: string;
};
}
export interface INodePropertyMode {