feat(core): Add Data Store Backend API (no-changelog) (#17824)

This commit is contained in:
Charlie Kolb
2025-08-12 14:54:24 +02:00
committed by GitHub
parent d06581ef3f
commit 98dc71e6a7
48 changed files with 3491 additions and 30 deletions

View File

@@ -0,0 +1,5 @@
import { Z } from 'zod-class';
import { dataStoreCreateColumnSchema } from '../../schemas/data-store.schema';
export class AddDataStoreColumnDto extends Z.class(dataStoreCreateColumnSchema.shape) {}

View File

@@ -0,0 +1,8 @@
import { z } from 'zod';
import { Z } from 'zod-class';
import { dataStoreColumnNameSchema } from '../../schemas/data-store.schema';
export class AddDataStoreRowsDto extends Z.class({
data: z.array(z.record(dataStoreColumnNameSchema, z.any())),
}) {}

View File

@@ -0,0 +1,11 @@
import { Z } from 'zod-class';
import {
dataStoreColumnNameSchema,
dataStoreColumnTypeSchema,
} from '../../schemas/data-store.schema';
export class CreateDataStoreColumnDto extends Z.class({
name: dataStoreColumnNameSchema,
type: dataStoreColumnTypeSchema,
}) {}

View File

@@ -0,0 +1,10 @@
import { z } from 'zod';
import { Z } from 'zod-class';
import { CreateDataStoreColumnDto } from './create-data-store-column.dto';
import { dataStoreNameSchema } from '../../schemas/data-store.schema';
export class CreateDataStoreDto extends Z.class({
name: dataStoreNameSchema,
columns: z.array(CreateDataStoreColumnDto),
}) {}

View File

@@ -0,0 +1,105 @@
import { jsonParse } from 'n8n-workflow';
import { z } from 'zod';
import { Z } from 'zod-class';
import { dataStoreColumnNameSchema } from '../../schemas/data-store.schema';
import { paginationSchema } from '../pagination/pagination.dto';
const FilterConditionSchema = z.union([z.literal('eq'), z.literal('neq')]);
export type ListDataStoreContentFilterConditionType = z.infer<typeof FilterConditionSchema>;
const filterRecord = z.object({
columnName: dataStoreColumnNameSchema,
condition: FilterConditionSchema.default('eq'),
value: z.union([z.string(), z.number(), z.boolean(), z.date()]),
});
const chainedFilterSchema = z.union([z.literal('and'), z.literal('or')]);
export type ListDataStoreContentFilter = z.infer<typeof filterSchema>;
// ---------------------
// Parameter Validators
// ---------------------
const filterSchema = z.object({
type: chainedFilterSchema.default('and'),
filters: z.array(filterRecord).default([]),
});
// Filter parameter validation
const filterValidator = z
.string()
.optional()
.transform((val, ctx) => {
if (!val) return undefined;
try {
const parsed: unknown = jsonParse(val);
try {
return filterSchema.parse(parsed);
} catch (e) {
ctx.addIssue({
code: z.ZodIssueCode.custom,
message: 'Invalid filter fields',
path: ['filter'],
});
return z.NEVER;
}
} catch (e) {
ctx.addIssue({
code: z.ZodIssueCode.custom,
message: 'Invalid filter format',
path: ['filter'],
});
return z.NEVER;
}
});
// SortBy parameter validation
const sortByValidator = z
.string()
.optional()
.transform((val, ctx) => {
if (val === undefined) return val;
if (!val.includes(':')) {
ctx.addIssue({
code: z.ZodIssueCode.custom,
message: 'Invalid sort format, expected <columnName>:<asc/desc>',
path: ['sort'],
});
return z.NEVER;
}
let [column, direction] = val.split(':');
try {
column = dataStoreColumnNameSchema.parse(column);
} catch {
ctx.addIssue({
code: z.ZodIssueCode.custom,
message: 'Invalid sort columnName',
path: ['sort'],
});
return z.NEVER;
}
direction = direction?.toUpperCase();
if (direction !== 'ASC' && direction !== 'DESC') {
ctx.addIssue({
code: z.ZodIssueCode.custom,
message: 'Invalid sort direction',
path: ['sort'],
});
return z.NEVER;
}
return [column, direction] as const;
});
export class ListDataStoreContentQueryDto extends Z.class({
take: paginationSchema.take.optional(),
skip: paginationSchema.skip.optional(),
filter: filterValidator.optional(),
sortBy: sortByValidator.optional(),
}) {}

View File

@@ -0,0 +1,70 @@
import { jsonParse } from 'n8n-workflow';
import { z } from 'zod';
import { Z } from 'zod-class';
import { paginationSchema } from '../pagination/pagination.dto';
const VALID_SORT_OPTIONS = [
'name:asc',
'name:desc',
'createdAt:asc',
'createdAt:desc',
'updatedAt:asc',
'updatedAt:desc',
'sizeBytes:asc',
'sizeBytes:desc',
] as const;
export type ListDataStoreQuerySortOptions = (typeof VALID_SORT_OPTIONS)[number];
const FILTER_OPTIONS = {
id: z.union([z.string(), z.array(z.string())]).optional(),
name: z.union([z.string(), z.array(z.string())]).optional(),
projectId: z.union([z.string(), z.array(z.string())]).optional(),
// todo: can probably include others here as well?
};
// Filter schema - only allow specific properties
const filterSchema = z.object(FILTER_OPTIONS).strict();
// ---------------------
// Parameter Validators
// ---------------------
// Filter parameter validation
const filterValidator = z
.string()
.optional()
.transform((val, ctx) => {
if (!val) return undefined;
try {
const parsed: unknown = jsonParse(val);
try {
return filterSchema.parse(parsed);
} catch (e) {
ctx.addIssue({
code: z.ZodIssueCode.custom,
message: 'Invalid filter fields',
path: ['filter'],
});
return z.NEVER;
}
} catch (e) {
ctx.addIssue({
code: z.ZodIssueCode.custom,
message: 'Invalid filter format',
path: ['filter'],
});
return z.NEVER;
}
});
// SortBy parameter validation
const sortByValidator = z
.enum(VALID_SORT_OPTIONS, { message: `sortBy must be one of: ${VALID_SORT_OPTIONS.join(', ')}` })
.optional();
export class ListDataStoreQueryDto extends Z.class({
...paginationSchema,
filter: filterValidator,
sortBy: sortByValidator,
}) {}

View File

@@ -0,0 +1,6 @@
import { z } from 'zod';
import { Z } from 'zod-class';
export class MoveDataStoreColumnDto extends Z.class({
targetIndex: z.number(),
}) {}

View File

@@ -0,0 +1,7 @@
import { Z } from 'zod-class';
import { dataStoreNameSchema } from '../../schemas/data-store.schema';
export class UpdateDataStoreDto extends Z.class({
name: dataStoreNameSchema,
}) {}

View File

@@ -0,0 +1,13 @@
import { z } from 'zod';
import { Z } from 'zod-class';
import { dataStoreColumnNameSchema } from '../../schemas/data-store.schema';
const dataStoreValueSchema = z.union([z.string(), z.number(), z.boolean(), z.date(), z.null()]);
const upsertDataStoreRowsShape = {
rows: z.array(z.record(dataStoreValueSchema)),
matchFields: z.array(dataStoreColumnNameSchema).min(1),
};
export class UpsertDataStoreRowsDto extends Z.class(upsertDataStoreRowsShape) {}

View File

@@ -79,3 +79,16 @@ export {
} from './user/users-list-filter.dto';
export { OidcConfigDto } from './oidc/config.dto';
export { CreateDataStoreDto } from './data-store/create-data-store.dto';
export { UpdateDataStoreDto } from './data-store/update-data-store.dto';
export { UpsertDataStoreRowsDto } from './data-store/upsert-data-store-rows.dto';
export { ListDataStoreQueryDto } from './data-store/list-data-store-query.dto';
export {
ListDataStoreContentQueryDto,
ListDataStoreContentFilter,
} from './data-store/list-data-store-content-query.dto';
export { CreateDataStoreColumnDto } from './data-store/create-data-store-column.dto';
export { AddDataStoreRowsDto } from './data-store/add-data-store-rows.dto';
export { AddDataStoreColumnDto } from './data-store/add-data-store-column.dto';
export { MoveDataStoreColumnDto } from './data-store/move-data-store-column.dto';

View File

@@ -45,3 +45,15 @@ export {
type UsersList,
usersListSchema,
} from './schemas/user.schema';
export {
DATA_STORE_COLUMN_REGEX,
type DataStore,
type DataStoreColumn,
type DataStoreCreateColumnSchema,
type DataStoreColumnJsType,
type DataStoreListFilter,
type DataStoreRows,
type DataStoreListOptions,
type DataStoreUserTableName,
} from './schemas/data-store.schema';

View File

@@ -0,0 +1,56 @@
import { z } from 'zod';
import type { ListDataStoreQueryDto } from '../dto';
export const dataStoreNameSchema = z.string().trim().min(1).max(128);
export const dataStoreIdSchema = z.string().max(36);
export const DATA_STORE_COLUMN_REGEX = /^[a-zA-Z0-9][a-zA-Z0-9-]*$/;
export const dataStoreColumnNameSchema = z
.string()
.trim()
.min(1)
.max(128)
.regex(
DATA_STORE_COLUMN_REGEX,
'Only alphanumeric characters and non-leading dashes are allowed for column names',
);
export const dataStoreColumnTypeSchema = z.enum(['string', 'number', 'boolean', 'date']);
export const dataStoreCreateColumnSchema = z.object({
name: dataStoreColumnNameSchema,
type: dataStoreColumnTypeSchema,
index: z.number().optional(),
});
export type DataStoreCreateColumnSchema = z.infer<typeof dataStoreCreateColumnSchema>;
export const dataStoreColumnSchema = dataStoreCreateColumnSchema.extend({
dataStoreId: dataStoreIdSchema,
});
export const dataStoreSchema = z.object({
id: dataStoreIdSchema,
name: dataStoreNameSchema,
columns: z.array(dataStoreColumnSchema),
createdAt: z.string().datetime(),
updatedAt: z.string().datetime(),
});
export type DataStore = z.infer<typeof dataStoreSchema>;
export type DataStoreColumn = z.infer<typeof dataStoreColumnSchema>;
export type DataStoreUserTableName = `data_store_user_${string}`;
export type DataStoreListFilter = {
id?: string | string[];
projectId?: string | string[];
name?: string;
};
export type DataStoreListOptions = Partial<ListDataStoreQueryDto> & {
filter: { projectId: string };
};
export type DataStoreColumnJsType = string | number | boolean | Date;
export type DataStoreRows = Array<Record<string, DataStoreColumnJsType | null>>;

View File

@@ -15,7 +15,9 @@ beforeEach(() => {
describe('eligibleModules', () => {
it('should consider all default modules eligible', () => {
expect(Container.get(ModuleRegistry).eligibleModules).toEqual(MODULE_NAMES);
// 'data-store' isn't (yet) eligible module by default
const expectedModules = MODULE_NAMES.filter((name) => name !== 'data-store');
expect(Container.get(ModuleRegistry).eligibleModules).toEqual(expectedModules);
});
it('should consider a module ineligible if it was disabled via env var', () => {
@@ -23,6 +25,15 @@ describe('eligibleModules', () => {
expect(Container.get(ModuleRegistry).eligibleModules).toEqual(['external-secrets']);
});
it('should consider a module eligible if it was enabled via env var', () => {
process.env.N8N_ENABLED_MODULES = 'data-store';
expect(Container.get(ModuleRegistry).eligibleModules).toEqual([
'insights',
'external-secrets',
'data-store',
]);
});
it('should throw `ModuleConfusionError` if a module is both enabled and disabled', () => {
process.env.N8N_ENABLED_MODULES = 'insights';
process.env.N8N_DISABLED_MODULES = 'insights';

View File

@@ -2,7 +2,7 @@ import { CommaSeparatedStringArray, Config, Env } from '@n8n/config';
import { UnknownModuleError } from './errors/unknown-module.error';
export const MODULE_NAMES = ['insights', 'external-secrets'] as const;
export const MODULE_NAMES = ['insights', 'external-secrets', 'data-store'] as const;
export type ModuleName = (typeof MODULE_NAMES)[number];

View File

@@ -66,7 +66,13 @@ export async function terminate() {
dbConnection.connectionState.connected = false;
}
type EntityName = keyof typeof entities | 'InsightsRaw' | 'InsightsByPeriod' | 'InsightsMetadata';
type EntityName =
| keyof typeof entities
| 'InsightsRaw'
| 'InsightsByPeriod'
| 'InsightsMetadata'
| 'DataStore'
| 'DataStoreColumn';
/**
* Truncate specific DB tables in a test DB.

View File

@@ -19,6 +19,7 @@ export const LOG_SCOPES = [
'insights',
'workflow-activation',
'ssh-client',
'data-store',
'cron',
'community-nodes',
'legacy-sqlite-execution-recovery',

View File

@@ -23,6 +23,8 @@ export { NoUrl } from './utils/validators/no-url.validator';
export * from './repositories';
export * from './subscribers';
export { Column as DslColumn } from './migrations/dsl/column';
export { CreateTable } from './migrations/dsl/table';
export { sqliteMigrations } from './migrations/sqlite';
export { mysqlMigrations } from './migrations/mysqldb';
export { postgresMigrations } from './migrations/postgresdb';

View File

@@ -26,6 +26,7 @@ export const RESOURCES = {
folder: [...DEFAULT_OPERATIONS, 'move'] as const,
insights: ['list'] as const,
oidc: ['manage'] as const,
dataStore: [...DEFAULT_OPERATIONS, 'readRow', 'writeRow'] as const,
} as const;
export const API_KEY_RESOURCES = {

View File

@@ -79,6 +79,13 @@ export const GLOBAL_OWNER_SCOPES: Scope[] = [
'insights:list',
'folder:move',
'oidc:manage',
'dataStore:create',
'dataStore:delete',
'dataStore:read',
'dataStore:update',
'dataStore:list',
'dataStore:readRow',
'dataStore:writeRow',
];
export const GLOBAL_ADMIN_SCOPES = GLOBAL_OWNER_SCOPES.concat();
@@ -98,4 +105,7 @@ export const GLOBAL_MEMBER_SCOPES: Scope[] = [
'user:list',
'variable:list',
'variable:read',
'dataStore:read',
'dataStore:list',
'dataStore:readRow',
];

View File

@@ -32,6 +32,13 @@ export const REGULAR_PROJECT_ADMIN_SCOPES: Scope[] = [
'folder:list',
'folder:move',
'sourceControl:push',
'dataStore:create',
'dataStore:delete',
'dataStore:read',
'dataStore:update',
'dataStore:list',
'dataStore:readRow',
'dataStore:writeRow',
];
export const PERSONAL_PROJECT_OWNER_SCOPES: Scope[] = [
@@ -58,6 +65,13 @@ export const PERSONAL_PROJECT_OWNER_SCOPES: Scope[] = [
'folder:delete',
'folder:list',
'folder:move',
'dataStore:create',
'dataStore:delete',
'dataStore:read',
'dataStore:update',
'dataStore:list',
'dataStore:readRow',
'dataStore:writeRow',
];
export const PROJECT_EDITOR_SCOPES: Scope[] = [
@@ -79,6 +93,13 @@ export const PROJECT_EDITOR_SCOPES: Scope[] = [
'folder:update',
'folder:delete',
'folder:list',
'dataStore:create',
'dataStore:delete',
'dataStore:read',
'dataStore:update',
'dataStore:list',
'dataStore:readRow',
'dataStore:writeRow',
];
export const PROJECT_VIEWER_SCOPES: Scope[] = [
@@ -90,4 +111,7 @@ export const PROJECT_VIEWER_SCOPES: Scope[] = [
'workflow:read',
'folder:read',
'folder:list',
'dataStore:list',
'dataStore:read',
'dataStore:readRow',
];

View File

@@ -31,6 +31,7 @@ describe('permissions', () => {
workflow: {},
folder: {},
insights: {},
dataStore: {},
});
});
it('getResourcePermissions', () => {
@@ -128,6 +129,7 @@ describe('permissions', () => {
insights: {
list: true,
},
dataStore: {},
};
expect(getResourcePermissions(scopes)).toEqual(permissionRecord);

View File

@@ -0,0 +1,215 @@
import { createTeamProject, testDb, testModules } from '@n8n/backend-test-utils';
import { ProjectRelationRepository, type Project, type User } from '@n8n/db';
import { Container } from '@n8n/di';
import type { EntityManager } from '@n8n/typeorm';
import { mock } from 'jest-mock-extended';
import { createUser } from '@test-integration/db/users';
import { DataStoreAggregateService } from '../data-store-aggregate.service';
import { DataStoreService } from '../data-store.service';
beforeAll(async () => {
await testModules.loadModules(['data-store']);
await testDb.init();
});
beforeEach(async () => {
await testDb.truncate(['DataStore', 'DataStoreColumn']);
});
afterAll(async () => {
await testDb.terminate();
});
describe('dataStoreAggregate', () => {
let dataStoreService: DataStoreService;
let dataStoreAggregateService: DataStoreAggregateService;
const manager = mock<EntityManager>();
const projectRelationRepository = mock<ProjectRelationRepository>({ manager });
beforeAll(() => {
Container.set(ProjectRelationRepository, projectRelationRepository);
dataStoreAggregateService = Container.get(DataStoreAggregateService);
dataStoreService = Container.get(DataStoreService);
});
let user: User;
let project1: Project;
let project2: Project;
beforeEach(async () => {
project1 = await createTeamProject();
project2 = await createTeamProject();
user = await createUser({ role: 'global:owner' });
});
afterEach(async () => {
// Clean up any created user data stores
await dataStoreService.deleteDataStoreAll();
});
describe('getManyAndCount', () => {
it('should return the correct data stores for the user', async () => {
// ARRANGE
const ds1 = await dataStoreService.createDataStore(project1.id, {
name: 'store1',
columns: [],
});
const ds2 = await dataStoreService.createDataStore(project1.id, {
name: 'store2',
columns: [],
});
projectRelationRepository.find.mockResolvedValueOnce([
{
userId: user.id,
projectId: project1.id,
role: 'project:admin',
user,
project: project1,
createdAt: new Date(),
updatedAt: new Date(),
setUpdateDate: jest.fn(),
},
{
userId: user.id,
projectId: project2.id,
role: 'project:viewer',
user,
project: project2,
createdAt: new Date(),
updatedAt: new Date(),
setUpdateDate: jest.fn(),
},
]);
await dataStoreService.createDataStore(project2.id, {
name: 'store3',
columns: [],
});
// ACT
const result = await dataStoreAggregateService.getManyAndCount(user, {
filter: { projectId: project1.id },
skip: 0,
take: 10,
});
// ASSERT
expect(result.data).toEqual(
expect.arrayContaining([
expect.objectContaining({ id: ds1!.id, name: ds1!.name }),
expect.objectContaining({ id: ds2!.id, name: ds2!.name }),
]),
);
expect(result.count).toBe(2);
});
it('should return an empty array if user has no access to any project', async () => {
// ARRANGE
const currentUser = await createUser({ role: 'global:member' });
await dataStoreService.createDataStore(project1.id, {
name: 'store1',
columns: [],
});
projectRelationRepository.find.mockResolvedValueOnce([]);
// ACT
const result = await dataStoreAggregateService.getManyAndCount(currentUser, {
skip: 0,
take: 10,
});
// ASSERT
expect(result.data).toEqual([]);
expect(result.count).toBe(0);
});
it('should return only the data store matching the given data store id filter', async () => {
// ARRANGE
await dataStoreService.createDataStore(project1.id, {
name: 'store1',
columns: [],
});
const ds2 = await dataStoreService.createDataStore(project1.id, {
name: 'store2',
columns: [],
});
projectRelationRepository.find.mockResolvedValueOnce([
{
userId: user.id,
projectId: project1.id,
role: 'project:admin',
user,
project: project1,
createdAt: new Date(),
updatedAt: new Date(),
setUpdateDate: jest.fn(),
},
{
userId: user.id,
projectId: project2.id,
role: 'project:viewer',
user,
project: project2,
createdAt: new Date(),
updatedAt: new Date(),
setUpdateDate: jest.fn(),
},
]);
// ACT
const result = await dataStoreAggregateService.getManyAndCount(user, {
filter: { id: ds2!.id },
skip: 0,
take: 10,
});
// ASSERT
expect(result.data).toEqual([expect.objectContaining({ id: ds2!.id, name: ds2!.name })]);
expect(result.count).toBe(1);
});
it('should respect pagination (skip/take)', async () => {
// ARRANGE
const ds1 = await dataStoreService.createDataStore(project1.id, {
name: 'store1',
columns: [],
});
const ds2 = await dataStoreService.createDataStore(project1.id, {
name: 'store2',
columns: [],
});
const ds3 = await dataStoreService.createDataStore(project1.id, {
name: 'store3',
columns: [],
});
projectRelationRepository.find.mockResolvedValueOnce([
{
userId: user.id,
projectId: project1.id,
role: 'project:admin',
user,
project: project1,
createdAt: new Date(),
updatedAt: new Date(),
setUpdateDate: jest.fn(),
},
]);
// ACT
const result = await dataStoreAggregateService.getManyAndCount(user, {
filter: { projectId: project1.id },
skip: 1,
take: 1,
});
// ASSERT
expect(result.data.length).toBe(1);
expect([ds1!.id, ds2!.id, ds3!.id]).toContain(result.data[0].id);
expect(result.count).toBe(3);
});
});
});

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,228 @@
import type { DataStoreRows } from '@n8n/api-types';
import {
addColumnQuery,
deleteColumnQuery,
buildInsertQuery,
buildUpdateQuery,
splitRowsByExistence,
} from '../utils/sql-utils';
describe('sql-utils', () => {
describe('addColumnQuery', () => {
it('should generate a valid SQL query for adding columns to a table', () => {
const tableName = 'data_store_user_abc';
const column = { name: 'email', type: 'number' as const };
const query = addColumnQuery(tableName, column, 'sqlite');
expect(query).toBe('ALTER TABLE "data_store_user_abc" ADD "email" FLOAT');
});
});
describe('deleteColumnQuery', () => {
it('should generate a valid SQL query for deleting columns from a table', () => {
const tableName = 'data_store_user_abc';
const column = 'email';
const query = deleteColumnQuery(tableName, column, 'sqlite');
expect(query).toBe('ALTER TABLE "data_store_user_abc" DROP COLUMN "email"');
});
});
describe('buildInsertQuery', () => {
it('should generate a valid SQL query for inserting rows into a table', () => {
const tableName = 'data_store_user_abc';
const columns = [
{ name: 'name', type: 'string' },
{ name: 'age', type: 'number' },
];
const rows = [
{ name: 'Alice', age: 30 },
{ name: 'Bob', age: 25 },
];
const [query, parameters] = buildInsertQuery(tableName, rows, columns, 'postgres');
expect(query).toBe(
'INSERT INTO "data_store_user_abc" ("name", "age") VALUES ($1, $2), ($3, $4)',
);
expect(parameters).toEqual(['Alice', 30, 'Bob', 25]);
});
it('should return an empty query and parameters when rows are empty', () => {
const tableName = 'data_store_user_abc';
const rows: [] = [];
const [query, parameters] = buildInsertQuery(tableName, rows, []);
expect(query).toBe('');
expect(parameters).toEqual([]);
});
it('should return an empty query and parameters when rows have no keys', () => {
const tableName = 'data_store_user_abc';
const rows = [{}];
const [query, parameters] = buildInsertQuery(tableName, rows, []);
expect(query).toBe('');
expect(parameters).toEqual([]);
});
it('should replace T and Z for MySQL', () => {
const tableName = 'data_store_user_abc';
const columns = [{ name: 'participatedAt', type: 'date' }];
const rows = [
{ participatedAt: new Date('2021-01-01') },
{ participatedAt: new Date('2021-01-02') },
];
const [query, parameters] = buildInsertQuery(tableName, rows, columns, 'mysql');
expect(query).toBe('INSERT INTO `data_store_user_abc` (`participatedAt`) VALUES (?), (?)');
expect(parameters).toEqual(['2021-01-01 00:00:00.000', '2021-01-02 00:00:00.000']);
});
});
describe('buildUpdateQuery', () => {
it('should generate a valid SQL update query with one match field', () => {
const tableName = 'data_store_user_abc';
const row = { name: 'Alice', age: 30, city: 'Paris' };
const columns = [
{ id: 1, name: 'name', type: 'string' },
{ id: 2, name: 'age', type: 'number' },
{ id: 3, name: 'city', type: 'string' },
];
const matchFields = ['name'];
const [query, parameters] = buildUpdateQuery(tableName, row, columns, matchFields);
expect(query).toBe('UPDATE "data_store_user_abc" SET "age" = ?, "city" = ? WHERE "name" = ?');
expect(parameters).toEqual([30, 'Paris', 'Alice']);
});
it('should generate a valid SQL update query with multiple match fields', () => {
const tableName = 'data_store_user_abc';
const row = { name: 'Alice', age: 30, city: 'Paris' };
const columns = [
{ id: 1, name: 'name', type: 'string' },
{ id: 2, name: 'age', type: 'number' },
{ id: 3, name: 'city', type: 'string' },
];
const matchFields = ['name', 'city'];
const [query, parameters] = buildUpdateQuery(tableName, row, columns, matchFields);
expect(query).toBe(
'UPDATE "data_store_user_abc" SET "age" = ? WHERE "name" = ? AND "city" = ?',
);
expect(parameters).toEqual([30, 'Alice', 'Paris']);
});
it('should return empty query and parameters if row is empty', () => {
const tableName = 'data_store_user_abc';
const row = {};
const matchFields = ['id'];
const [query, parameters] = buildUpdateQuery(tableName, row, [], matchFields);
expect(query).toBe('');
expect(parameters).toEqual([]);
});
it('should return empty query and parameters if matchFields is empty', () => {
const tableName = 'data_store_user_abc';
const row = { name: 'Alice', age: 30 };
const columns = [
{ id: 1, name: 'name', type: 'string' },
{ id: 2, name: 'age', type: 'number' },
];
const matchFields: string[] = [];
const [query, parameters] = buildUpdateQuery(tableName, row, columns, matchFields);
expect(query).toBe('');
expect(parameters).toEqual([]);
});
it('should return empty query and parameters if only matchFields are present in row', () => {
const tableName = 'data_store_user_abc';
const row = { id: 1 };
const matchFields = ['id'];
const [query, parameters] = buildUpdateQuery(tableName, row, [], matchFields);
expect(query).toBe('');
expect(parameters).toEqual([]);
});
});
describe('splitRowsByExistence', () => {
it('should correctly separate rows into insert and update based on matchFields', () => {
const existing = [
{ name: 'Alice', age: 30 },
{ name: 'Bob', age: 25 },
];
const matchFields = ['name'];
const rows: DataStoreRows = [
{ name: 'Alice', age: 30 },
{ name: 'Bob', age: 26 },
{ name: 'Charlie', age: 35 },
];
const { rowsToInsert, rowsToUpdate } = splitRowsByExistence(existing, matchFields, rows);
expect(rowsToUpdate).toEqual([
{ name: 'Alice', age: 30 },
{ name: 'Bob', age: 26 },
]);
expect(rowsToInsert).toEqual([{ name: 'Charlie', age: 35 }]);
});
it('should treat rows as new if matchFields combination does not exist', () => {
const existing = [{ name: 'Bob', city: 'Berlin' }];
const matchFields = ['name', 'city'];
const rows: DataStoreRows = [{ name: 'Alice', city: 'Berlin' }];
const { rowsToInsert, rowsToUpdate } = splitRowsByExistence(existing, matchFields, rows);
expect(rowsToUpdate).toEqual([]);
expect(rowsToInsert).toEqual([{ name: 'Alice', city: 'Berlin' }]);
});
it('should insert all rows if existing set is empty', () => {
const existing: Array<Record<string, unknown>> = [];
const matchFields = ['name'];
const rows: DataStoreRows = [{ name: 'Alice' }, { name: 'Bob' }];
const { rowsToInsert, rowsToUpdate } = splitRowsByExistence(existing, matchFields, rows);
expect(rowsToUpdate).toEqual([]);
expect(rowsToInsert).toEqual(rows);
});
it('should update all rows if all keys match existing', () => {
const existing = [{ name: 'Alice' }, { name: 'Bob' }];
const matchFields = ['name'];
const rows: DataStoreRows = [{ name: 'Alice' }, { name: 'Bob' }];
const { rowsToInsert, rowsToUpdate } = splitRowsByExistence(existing, matchFields, rows);
expect(rowsToInsert).toEqual([]);
expect(rowsToUpdate).toEqual(rows);
});
it('should handle empty input rows', () => {
const existing = [{ name: 'Alice' }];
const matchFields = ['name'];
const rows: DataStoreRows = [];
const { rowsToInsert, rowsToUpdate } = splitRowsByExistence(existing, matchFields, rows);
expect(rowsToInsert).toEqual([]);
expect(rowsToUpdate).toEqual([]);
});
});
});

View File

@@ -0,0 +1,20 @@
import { ListDataStoreQueryDto } from '@n8n/api-types';
import { AuthenticatedRequest } from '@n8n/db';
import { Get, ProjectScope, Query, RestController } from '@n8n/decorators';
import { DataStoreAggregateService } from './data-store-aggregate.service';
@RestController('/data-stores-global')
export class DataStoreAggregateController {
constructor(private readonly dataStoreAggregateService: DataStoreAggregateService) {}
@Get('/')
@ProjectScope('dataStore:list')
async listDataStores(
req: AuthenticatedRequest,
_res: Response,
@Query payload: ListDataStoreQueryDto,
) {
return await this.dataStoreAggregateService.getManyAndCount(req.user, payload);
}
}

View File

@@ -0,0 +1,42 @@
import type { ListDataStoreQueryDto } from '@n8n/api-types';
import { Logger } from '@n8n/backend-common';
import { User } from '@n8n/db';
import { Service } from '@n8n/di';
import { ProjectService } from '@/services/project.service.ee';
import { DataStoreRepository } from './data-store.repository';
@Service()
export class DataStoreAggregateService {
constructor(
private readonly dataStoreRepository: DataStoreRepository,
private readonly projectService: ProjectService,
private readonly logger: Logger,
) {
this.logger = this.logger.scoped('data-store');
}
async start() {}
async shutdown() {}
async getManyAndCount(user: User, options: ListDataStoreQueryDto) {
const projects = await this.projectService.getProjectRelationsForUser(user);
let projectIds = projects.map((x) => x.projectId);
if (options.filter?.projectId) {
const mask = [options.filter?.projectId].flat();
projectIds = projectIds.filter((x) => mask.includes(x));
}
if (projectIds.length === 0) {
return { count: 0, data: [] };
}
return await this.dataStoreRepository.getManyAndCount({
...options,
filter: {
...options.filter,
projectId: projectIds,
},
});
}
}

View File

@@ -0,0 +1,24 @@
import { WithTimestampsAndStringId } from '@n8n/db';
import { Column, Entity, Index, JoinColumn, ManyToOne } from '@n8n/typeorm';
import { type DataStore } from './data-store.entity';
@Entity()
@Index(['dataStoreId', 'name'], { unique: true })
export class DataStoreColumn extends WithTimestampsAndStringId {
@Column()
dataStoreId: string;
@Column()
name: string;
@Column({ type: 'varchar' })
type: 'string' | 'number' | 'boolean' | 'date';
@Column({ type: 'int' })
index: number;
@ManyToOne('DataStore', 'columns')
@JoinColumn({ name: 'dataStoreId' })
dataStore: DataStore;
}

View File

@@ -0,0 +1,135 @@
import { DataStoreCreateColumnSchema } from '@n8n/api-types';
import { Service } from '@n8n/di';
import { DataSource, EntityManager, Repository } from '@n8n/typeorm';
import { UserError } from 'n8n-workflow';
import { DataStoreColumn } from './data-store-column.entity';
import { DataStoreRowsRepository } from './data-store-rows.repository';
@Service()
export class DataStoreColumnRepository extends Repository<DataStoreColumn> {
constructor(
dataSource: DataSource,
private dataStoreRowsRepository: DataStoreRowsRepository,
) {
super(DataStoreColumn, dataSource.manager);
}
async getColumns(rawDataStoreId: string, em?: EntityManager) {
const executor = em ?? this.manager;
const columns = await executor
.createQueryBuilder(DataStoreColumn, 'dsc')
.where('dsc.dataStoreId = :dataStoreId', { dataStoreId: rawDataStoreId })
.getMany();
// Ensure columns are always returned in the correct order by index,
// since the database does not guarantee ordering and TypeORM does not preserve
// join order in @OneToMany relations.
columns.sort((a, b) => a.index - b.index);
return columns;
}
async addColumn(dataStoreId: string, schema: DataStoreCreateColumnSchema) {
return await this.manager.transaction(async (em) => {
const existingColumnMatch = await em.existsBy(DataStoreColumn, {
name: schema.name,
dataStoreId,
});
if (existingColumnMatch) {
throw new UserError(
`column name '${schema.name}' already taken in data store '${dataStoreId}'`,
);
}
if (schema.index === undefined) {
const columns = await this.getColumns(dataStoreId, em);
schema.index = columns.length;
} else {
await this.shiftColumns(dataStoreId, schema.index, 1, em);
}
const column = em.create(DataStoreColumn, {
...schema,
dataStoreId,
});
await em.insert(DataStoreColumn, column);
const queryRunner = em.queryRunner;
if (!queryRunner) {
throw new Error('QueryRunner is not available');
}
await this.dataStoreRowsRepository.ensureTableAndAddColumn(
dataStoreId,
column,
queryRunner,
em.connection.options.type,
);
return column;
});
}
async deleteColumn(dataStoreId: string, column: DataStoreColumn) {
await this.manager.transaction(async (em) => {
await em.remove(DataStoreColumn, column);
await this.dataStoreRowsRepository.dropColumnFromTable(
dataStoreId,
column.name,
em,
em.connection.options.type,
);
await this.shiftColumns(dataStoreId, column.index, -1, em);
});
}
async moveColumn(rawDataStoreId: string, columnId: string, targetIndex: number) {
await this.manager.transaction(async (em) => {
const columnCount = await em.countBy(DataStoreColumn, { dataStoreId: rawDataStoreId });
if (targetIndex >= columnCount) {
throw new UserError('tried to move column to index larger than column count');
}
if (targetIndex < 0) {
throw new UserError('tried to move column to negative index');
}
const existingColumn = await em.findOneBy(DataStoreColumn, {
id: columnId,
dataStoreId: rawDataStoreId,
});
if (existingColumn === null) {
throw new UserError(`tried to move column not present in data store '${rawDataStoreId}'`);
}
await this.shiftColumns(rawDataStoreId, existingColumn.index, -1, em);
await this.shiftColumns(rawDataStoreId, targetIndex, 1, em);
await em.update(DataStoreColumn, { id: existingColumn.id }, { index: targetIndex });
});
}
async shiftColumns(
rawDataStoreId: string,
lowestIndex: number,
delta: -1 | 1,
em?: EntityManager,
) {
const executor = em ?? this.manager;
await executor
.createQueryBuilder()
.update(DataStoreColumn)
.set({
index: () => `index + ${delta}`,
})
.where('dataStoreId = :dataStoreId AND index >= :thresholdValue', {
dataStoreId: rawDataStoreId,
thresholdValue: lowestIndex,
})
.execute();
}
}

View File

@@ -0,0 +1,238 @@
import type {
ListDataStoreContentQueryDto,
ListDataStoreContentFilter,
DataStoreUserTableName,
DataStoreRows,
UpsertDataStoreRowsDto,
} from '@n8n/api-types';
import { CreateTable, DslColumn } from '@n8n/db';
import { Service } from '@n8n/di';
import {
DataSource,
DataSourceOptions,
EntityManager,
QueryRunner,
SelectQueryBuilder,
} from '@n8n/typeorm';
import { DataStoreColumn } from './data-store-column.entity';
import {
addColumnQuery,
buildInsertQuery,
buildUpdateQuery,
deleteColumnQuery,
getPlaceholder,
quoteIdentifier,
splitRowsByExistence,
toDslColumns,
toTableName,
} from './utils/sql-utils';
// eslint-disable-next-line @typescript-eslint/no-explicit-any
type QueryBuilder = SelectQueryBuilder<any>;
function getConditionAndParams(
filter: ListDataStoreContentFilter['filters'][number],
index: number,
dbType: DataSourceOptions['type'],
): [string, Record<string, unknown>] {
const paramName = `filter_${index}`;
const column = `dataStore.${quoteIdentifier(filter.columnName, dbType)}`;
switch (filter.condition) {
case 'eq':
return [`${column} = :${paramName}`, { [paramName]: filter.value }];
case 'neq':
return [`${column} != :${paramName}`, { [paramName]: filter.value }];
}
}
@Service()
export class DataStoreRowsRepository {
constructor(private dataSource: DataSource) {}
// TypeORM cannot infer the columns for a dynamic table name, so we use a raw query
async insertRows(
tableName: DataStoreUserTableName,
rows: DataStoreRows,
columns: DataStoreColumn[],
) {
const dbType = this.dataSource.options.type;
await this.dataSource.query.apply(
this.dataSource,
buildInsertQuery(tableName, rows, columns, dbType),
);
return true;
}
async upsertRows(
tableName: DataStoreUserTableName,
dto: UpsertDataStoreRowsDto,
columns: DataStoreColumn[],
) {
const dbType = this.dataSource.options.type;
const { rows, matchFields } = dto;
if (rows.length === 0) {
return false;
}
const { rowsToInsert, rowsToUpdate } = await this.fetchAndSplitRowsByExistence(
tableName,
matchFields,
rows,
);
if (rowsToInsert.length > 0) {
await this.insertRows(tableName, rowsToInsert, columns);
}
if (rowsToUpdate.length > 0) {
for (const row of rowsToUpdate) {
// TypeORM cannot infer the columns for a dynamic table name, so we use a raw query
const [query, parameters] = buildUpdateQuery(tableName, row, columns, matchFields, dbType);
await this.dataSource.query(query, parameters);
}
}
return true;
}
async createTableWithColumns(
tableName: string,
columns: DataStoreColumn[],
queryRunner: QueryRunner,
) {
const dslColumns = [new DslColumn('id').int.autoGenerate2.primary, ...toDslColumns(columns)];
const createTable = new CreateTable(tableName, '', queryRunner);
createTable.withColumns.apply(createTable, dslColumns);
await createTable.execute(queryRunner);
}
async ensureTableAndAddColumn(
dataStoreId: string,
column: DataStoreColumn,
queryRunner: QueryRunner,
dbType: DataSourceOptions['type'],
) {
const tableName = toTableName(dataStoreId);
const tableExists = await queryRunner.hasTable(tableName);
if (!tableExists) {
await this.createTableWithColumns(tableName, [column], queryRunner);
} else {
await queryRunner.manager.query(addColumnQuery(tableName, column, dbType));
}
}
async dropColumnFromTable(
dataStoreId: string,
columnName: string,
em: EntityManager,
dbType: DataSourceOptions['type'],
) {
await em.query(deleteColumnQuery(toTableName(dataStoreId), columnName, dbType));
}
async getManyAndCount(dataStoreId: DataStoreUserTableName, dto: ListDataStoreContentQueryDto) {
const [countQuery, query] = this.getManyQuery(dataStoreId, dto);
const data: Array<Record<string, unknown>> = await query.select('*').getRawMany();
const countResult = await countQuery.select('COUNT(*) as count').getRawOne<{
count: number | string | null;
}>();
const count =
typeof countResult?.count === 'number' ? countResult.count : Number(countResult?.count) || 0;
return { count: count ?? -1, data };
}
async getRowIds(dataStoreId: DataStoreUserTableName, dto: ListDataStoreContentQueryDto) {
const [_, query] = this.getManyQuery(dataStoreId, dto);
const result = await query.select('dataStore.id').getRawMany<number>();
return result;
}
private getManyQuery(
dataStoreTableName: DataStoreUserTableName,
dto: ListDataStoreContentQueryDto,
): [QueryBuilder, QueryBuilder] {
const query = this.dataSource.createQueryBuilder();
query.from(dataStoreTableName, 'dataStore');
this.applyFilters(query, dto);
const countQuery = query.clone().select('COUNT(*)');
this.applySorting(query, dto);
this.applyPagination(query, dto);
return [countQuery, query];
}
private applyFilters(query: QueryBuilder, dto: ListDataStoreContentQueryDto): void {
const filters = dto.filter?.filters ?? [];
const filterType = dto.filter?.type ?? 'and';
const dbType = this.dataSource.options.type;
const conditionsAndParams = filters.map((filter, i) =>
getConditionAndParams(filter, i, dbType),
);
for (const [condition, params] of conditionsAndParams) {
if (filterType === 'or') {
query.orWhere(condition, params);
} else {
query.andWhere(condition, params);
}
}
}
private applySorting(query: QueryBuilder, dto: ListDataStoreContentQueryDto): void {
if (!dto.sortBy) {
// query.orderBy('dataStore.', 'DESC');
return;
}
const [field, order] = dto.sortBy;
this.applySortingByField(query, field, order);
}
private applySortingByField(query: QueryBuilder, field: string, direction: 'DESC' | 'ASC'): void {
const dbType = this.dataSource.options.type;
const quotedField = `dataStore.${quoteIdentifier(field, dbType)}`;
query.orderBy(quotedField, direction);
}
private applyPagination(query: QueryBuilder, dto: ListDataStoreContentQueryDto): void {
query.skip(dto.skip);
query.take(dto.take);
}
private async fetchAndSplitRowsByExistence(
tableName: string,
matchFields: string[],
rows: DataStoreRows,
): Promise<{ rowsToInsert: DataStoreRows; rowsToUpdate: DataStoreRows }> {
const dbType = this.dataSource.options.type;
const whereClauses: string[] = [];
const params: unknown[] = [];
for (const row of rows) {
const clause = matchFields
.map((field) => {
params.push(row[field]);
return `${quoteIdentifier(field, dbType)} = ${getPlaceholder(params.length, dbType)}`;
})
.join(' AND ');
whereClauses.push(`(${clause})`);
}
const quotedFields = matchFields.map((field) => quoteIdentifier(field, dbType)).join(', ');
const quotedTableName = quoteIdentifier(tableName, dbType);
const query = `
SELECT ${quotedFields}
FROM ${quotedTableName}
WHERE ${whereClauses.join(' OR ')}
`;
const existing: Array<Record<string, unknown>> = await this.dataSource.query(query, params);
return splitRowsByExistence(existing, matchFields, rows);
}
}

View File

@@ -0,0 +1,151 @@
import {
AddDataStoreRowsDto,
AddDataStoreColumnDto,
CreateDataStoreDto,
ListDataStoreContentQueryDto,
ListDataStoreQueryDto,
MoveDataStoreColumnDto,
UpdateDataStoreDto,
UpsertDataStoreRowsDto,
} from '@n8n/api-types';
import { AuthenticatedRequest } from '@n8n/db';
import {
Body,
Delete,
Get,
Param,
Patch,
Post,
ProjectScope,
Query,
RestController,
} from '@n8n/decorators';
import { DataStoreService } from './data-store.service';
@RestController('/projects/:projectId/data-stores')
export class DataStoreController {
constructor(private readonly dataStoreService: DataStoreService) {}
@Post('/')
@ProjectScope('dataStore:create')
async createDataStore(
req: AuthenticatedRequest<{ projectId: string }>,
_res: Response,
@Body dto: CreateDataStoreDto,
) {
return await this.dataStoreService.createDataStore(req.params.projectId, dto);
}
@Get('/')
@ProjectScope('dataStore:list')
async listDataStores(
req: AuthenticatedRequest<{ projectId: string }>,
_res: Response,
@Query payload: ListDataStoreQueryDto,
) {
const providedFilter = payload?.filter ?? {};
return await this.dataStoreService.getManyAndCount({
...payload,
filter: { ...providedFilter, projectId: req.params.projectId },
});
}
@Patch('/:dataStoreId')
@ProjectScope('dataStore:update')
async updateDataStore(
_req: AuthenticatedRequest<{ projectId: string }>,
_res: Response,
@Param('dataStoreId') dataStoreId: string,
@Body dto: UpdateDataStoreDto,
) {
return await this.dataStoreService.updateDataStore(dataStoreId, dto);
}
@Delete('/:dataStoreId')
@ProjectScope('dataStore:delete')
async deleteDataStore(
_req: AuthenticatedRequest<{ projectId: string }>,
_res: Response,
@Param('dataStoreId') dataStoreId: string,
) {
return await this.dataStoreService.deleteDataStore(dataStoreId);
}
@Get('/:dataStoreId/columns')
@ProjectScope('dataStore:read')
async getColumns(
_req: AuthenticatedRequest<{ projectId: string }>,
_res: Response,
@Param('dataStoreId') dataStoreId: string,
) {
return await this.dataStoreService.getColumns(dataStoreId);
}
@Post('/:dataStoreId/columns')
@ProjectScope('dataStore:update')
async addColumn(
_req: AuthenticatedRequest<{ projectId: string }>,
_res: Response,
@Param('dataStoreId') dataStoreId: string,
@Body dto: AddDataStoreColumnDto,
) {
return await this.dataStoreService.addColumn(dataStoreId, dto);
}
@Delete('/:dataStoreId/columns/:columnId')
@ProjectScope('dataStore:update')
async deleteColumn(
_req: AuthenticatedRequest<{ projectId: string }>,
_res: Response,
@Param('dataStoreId') dataStoreId: string,
@Param('columnId') columnId: string,
) {
return await this.dataStoreService.deleteColumn(dataStoreId, columnId);
}
@Patch('/:dataStoreId/columns/:columnId/move')
@ProjectScope('dataStore:update')
async moveColumn(
_req: AuthenticatedRequest<{ projectId: string }>,
_res: Response,
@Param('dataStoreId') dataStoreId: string,
@Param('columnId') columnId: string,
@Body dto: MoveDataStoreColumnDto,
) {
return await this.dataStoreService.moveColumn(dataStoreId, columnId, dto);
}
@Get('/:dataStoreId/rows')
@ProjectScope('dataStore:readRow')
async getDataStoreRows(
_req: AuthenticatedRequest<{ projectId: string }>,
_res: Response,
@Param('dataStoreId') dataStoreId: string,
@Query dto: ListDataStoreContentQueryDto,
) {
return await this.dataStoreService.getManyRowsAndCount(dataStoreId, dto);
}
@Post('/:dataStoreId/insert')
@ProjectScope('dataStore:writeRow')
async appendDataStoreRows(
_req: AuthenticatedRequest<{ projectId: string }>,
_res: Response,
@Param('dataStoreId') dataStoreId: string,
@Body dto: AddDataStoreRowsDto,
) {
return await this.dataStoreService.insertRows(dataStoreId, dto.data);
}
@Post('/:dataStoreId/upsert')
@ProjectScope('dataStore:writeRow')
async upsertDataStoreRows(
_req: AuthenticatedRequest<{ projectId: string }>,
_res: Response,
@Param('dataStoreId') dataStoreId: string,
@Body dto: UpsertDataStoreRowsDto,
) {
return await this.dataStoreService.upsertRows(dataStoreId, dto);
}
}

View File

@@ -0,0 +1,34 @@
import { Project, WithTimestampsAndStringId } from '@n8n/db';
import { Column, Entity, Index, JoinColumn, ManyToOne, OneToMany } from '@n8n/typeorm';
import { DataStoreColumn } from './data-store-column.entity';
@Entity()
@Index(['name', 'projectId'], { unique: true })
export class DataStore extends WithTimestampsAndStringId {
constructor() {
super();
}
@Column()
name: string;
@OneToMany(
() => DataStoreColumn,
(dataStoreColumn) => dataStoreColumn.dataStore,
{
cascade: true,
},
)
columns: DataStoreColumn[];
@ManyToOne(() => Project)
@JoinColumn({ name: 'projectId' })
project: Project;
@Column()
projectId: string;
@Column({ type: 'int', default: 0 })
sizeBytes: number;
}

View File

@@ -0,0 +1,34 @@
import type { ModuleInterface } from '@n8n/decorators';
import { BackendModule, OnShutdown } from '@n8n/decorators';
import { Container } from '@n8n/di';
import { BaseEntity } from '@n8n/typeorm';
@BackendModule({ name: 'data-store' })
export class DataStoreModule implements ModuleInterface {
async init() {
await import('./data-store.controller');
await import('./data-store-aggregate.controller');
const { DataStoreService } = await import('./data-store.service');
await Container.get(DataStoreService).start();
const { DataStoreAggregateService } = await import('./data-store-aggregate.service');
await Container.get(DataStoreAggregateService).start();
}
@OnShutdown()
async shutdown() {
const { DataStoreService } = await import('./data-store.service');
await Container.get(DataStoreService).shutdown();
const { DataStoreAggregateService } = await import('./data-store-aggregate.service');
await Container.get(DataStoreAggregateService).start();
}
async entities() {
const { DataStore } = await import('./data-store.entity');
const { DataStoreColumn } = await import('./data-store-column.entity');
return [DataStore, DataStoreColumn] as unknown as Array<new () => BaseEntity>;
}
}

View File

@@ -0,0 +1,235 @@
import {
DATA_STORE_COLUMN_REGEX,
type DataStoreCreateColumnSchema,
type ListDataStoreQueryDto,
} from '@n8n/api-types';
import { Service } from '@n8n/di';
import { DataSource, EntityManager, Repository, SelectQueryBuilder } from '@n8n/typeorm';
import { UnexpectedError } from 'n8n-workflow';
import { DataStoreColumn } from './data-store-column.entity';
import { DataStoreRowsRepository } from './data-store-rows.repository';
import { DataStore } from './data-store.entity';
import { toTableName } from './utils/sql-utils';
@Service()
export class DataStoreRepository extends Repository<DataStore> {
constructor(
dataSource: DataSource,
private dataStoreRowsRepository: DataStoreRowsRepository,
) {
super(DataStore, dataSource.manager);
}
async createDataStore(projectId: string, name: string, columns: DataStoreCreateColumnSchema[]) {
if (columns.some((c) => !DATA_STORE_COLUMN_REGEX.test(c.name))) {
throw new UnexpectedError('bad column name');
}
let dataStoreId: string | undefined;
await this.manager.transaction(async (em) => {
const dataStore = em.create(DataStore, { name, columns, projectId });
await em.insert(DataStore, dataStore);
dataStoreId = dataStore.id;
const tableName = toTableName(dataStore.id);
const queryRunner = em.queryRunner;
if (!queryRunner) {
throw new UnexpectedError('QueryRunner is not available');
}
if (columns.length === 0) {
return;
}
// insert columns
const columnEntities = columns.map((col, index) =>
em.create(DataStoreColumn, {
name: col.name,
type: col.type,
dataStoreId: dataStore.id,
index: col.index ?? index,
}),
);
await em.insert(DataStoreColumn, columnEntities);
// create user table
await this.dataStoreRowsRepository.createTableWithColumns(
tableName,
columnEntities,
queryRunner,
);
});
if (!dataStoreId) {
throw new UnexpectedError('Data store creation failed');
}
return await this.findOne({
where: { id: dataStoreId },
relations: ['project', 'columns'],
});
}
async deleteDataStore(dataStoreId: string, entityManager?: EntityManager) {
const executor = entityManager ?? this.manager;
return await executor.transaction(async (em) => {
const queryRunner = em.queryRunner;
if (!queryRunner) {
throw new UnexpectedError('QueryRunner is not available');
}
await em.delete(DataStore, { id: dataStoreId });
await queryRunner.dropTable(toTableName(dataStoreId), true);
return true;
});
}
async deleteDataStoreByProjectId(projectId: string) {
return await this.manager.transaction(async (em) => {
const existingTables = await em.findBy(DataStore, { projectId });
let changed = false;
for (const match of existingTables) {
const result = await this.deleteDataStore(match.id, em);
changed = changed || result;
}
return changed;
});
}
async deleteDataStoreAll() {
return await this.manager.transaction(async (em) => {
const queryRunner = em.queryRunner;
if (!queryRunner) {
throw new UnexpectedError('QueryRunner is not available');
}
const existingTables = await em.findBy(DataStore, {});
let changed = false;
for (const match of existingTables) {
const result = await em.delete(DataStore, { id: match.id });
await queryRunner.dropTable(toTableName(match.id), true);
changed = changed || (result.affected ?? 0) > 0;
}
return changed;
});
}
async getManyAndCount(options: Partial<ListDataStoreQueryDto>) {
const query = this.getManyQuery(options);
const [data, count] = await query.getManyAndCount();
return { count, data };
}
async getMany(options: Partial<ListDataStoreQueryDto>) {
const query = this.getManyQuery(options);
return await query.getMany();
}
private getManyQuery(options: Partial<ListDataStoreQueryDto>): SelectQueryBuilder<DataStore> {
const query = this.createQueryBuilder('dataStore');
this.applySelections(query);
this.applyFilters(query, options.filter);
this.applySorting(query, options.sortBy);
this.applyPagination(query, options);
return query;
}
private applySelections(query: SelectQueryBuilder<DataStore>): void {
this.applyDefaultSelect(query);
}
private applyFilters(
query: SelectQueryBuilder<DataStore>,
filter: Partial<ListDataStoreQueryDto>['filter'],
): void {
for (const x of ['id', 'projectId'] as const) {
const content = [filter?.[x]].flat().filter((x) => x !== undefined);
if (content.length === 0) continue;
query.andWhere(`dataStore.${x} IN (:...${x}s)`, {
/*
* If list is empty, add a dummy value to prevent an error
* when using the IN operator with an empty array.
*/
[x + 's']: content.length > 0 ? content : [''],
});
}
if (filter?.name && typeof filter.name === 'string') {
query.andWhere('LOWER(dataStore.name) LIKE LOWER(:name)', {
name: `%${filter.name}%`,
});
}
}
private applySorting(query: SelectQueryBuilder<DataStore>, sortBy?: string): void {
if (!sortBy) {
query.orderBy('dataStore.updatedAt', 'DESC');
return;
}
const [field, order] = this.parseSortingParams(sortBy);
this.applySortingByField(query, field, order);
}
private parseSortingParams(sortBy: string): [string, 'DESC' | 'ASC'] {
const [field, order] = sortBy.split(':');
return [field, order?.toLowerCase() === 'desc' ? 'DESC' : 'ASC'];
}
private applySortingByField(
query: SelectQueryBuilder<DataStore>,
field: string,
direction: 'DESC' | 'ASC',
): void {
if (field === 'name') {
query.orderBy('LOWER(dataStore.name)', direction);
} else if (['createdAt', 'updatedAt'].includes(field)) {
query.orderBy(`dataStore.${field}`, direction);
}
}
private applyPagination(
query: SelectQueryBuilder<DataStore>,
options: Partial<ListDataStoreQueryDto>,
): void {
query.skip(options.skip ?? 0);
if (options?.take) {
query.skip(options.skip ?? 0).take(options.take);
}
}
private applyDefaultSelect(query: SelectQueryBuilder<DataStore>): void {
query
.leftJoinAndSelect('dataStore.project', 'project')
.leftJoinAndSelect('dataStore.columns', 'data_store_column')
.select([
'dataStore',
...this.getDataStoreColumnFields('data_store_column'),
...this.getProjectFields('project'),
])
.addOrderBy('data_store_column.index', 'ASC');
}
private getDataStoreColumnFields(alias: string): string[] {
return [
`${alias}.id`,
`${alias}.name`,
`${alias}.type`,
`${alias}.createdAt`,
`${alias}.updatedAt`,
];
}
private getProjectFields(alias: string): string[] {
return [`${alias}.id`, `${alias}.name`, `${alias}.type`, `${alias}.icon`];
}
}

View File

@@ -0,0 +1,265 @@
import type {
AddDataStoreColumnDto,
CreateDataStoreDto,
ListDataStoreContentQueryDto,
MoveDataStoreColumnDto,
DataStoreListOptions,
DataStoreRows,
UpsertDataStoreRowsDto,
} from '@n8n/api-types';
import { UpdateDataStoreDto } from '@n8n/api-types/src/dto/data-store/update-data-store.dto';
import { Logger } from '@n8n/backend-common';
import { Service } from '@n8n/di';
import { UserError } from 'n8n-workflow';
import { DataStoreColumn } from './data-store-column.entity';
import { DataStoreColumnRepository } from './data-store-column.repository';
import { DataStoreRowsRepository } from './data-store-rows.repository';
import { DataStoreRepository } from './data-store.repository';
import { toTableName } from './utils/sql-utils';
@Service()
export class DataStoreService {
constructor(
private readonly dataStoreRepository: DataStoreRepository,
private readonly dataStoreColumnRepository: DataStoreColumnRepository,
private readonly dataStoreRowsRepository: DataStoreRowsRepository,
private readonly logger: Logger,
) {
this.logger = this.logger.scoped('data-store');
}
async start() {}
async shutdown() {}
async createDataStore(projectId: string, dto: CreateDataStoreDto) {
const existingTable = await this.dataStoreRepository.findOneBy({
name: dto.name,
projectId,
});
if (existingTable !== null) {
throw new UserError(`Data store with name '${dto.name}' already exists in this project`);
}
return await this.dataStoreRepository.createDataStore(projectId, dto.name, dto.columns);
}
// Currently only renames data stores
async updateDataStore(dataStoreId: string, dto: UpdateDataStoreDto) {
const name = dto.name.trim();
if (!name) {
throw new UserError('Data store name must not be empty');
}
const existingTable = await this.dataStoreRepository.findOneBy({
id: dataStoreId,
});
if (existingTable === null) {
throw new UserError(`Tried to rename non-existent data store '${dataStoreId}'`);
}
const hasNameClash = await this.dataStoreRepository.existsBy({
name,
projectId: existingTable.projectId,
});
if (hasNameClash) {
throw new UserError(`The name '${name}' is already taken within this project`);
}
await this.dataStoreRepository.update({ id: dataStoreId }, { name });
return true;
}
async deleteDataStoreByProjectId(projectId: string) {
return await this.dataStoreRepository.deleteDataStoreByProjectId(projectId);
}
async deleteDataStoreAll() {
return await this.dataStoreRepository.deleteDataStoreAll();
}
async deleteDataStore(dataStoreId: string) {
await this.validateDataStoreExists(
dataStoreId,
`Tried to delete non-existent data store '${dataStoreId}'`,
);
await this.dataStoreRepository.deleteDataStore(dataStoreId);
return true;
}
async addColumn(dataStoreId: string, dto: AddDataStoreColumnDto) {
await this.validateDataStoreExists(
dataStoreId,
`Tried to add column to non-existent data store '${dataStoreId}'`,
);
return await this.dataStoreColumnRepository.addColumn(dataStoreId, dto);
}
async moveColumn(dataStoreId: string, columnId: string, dto: MoveDataStoreColumnDto) {
await this.validateDataStoreExists(
dataStoreId,
`Tried to move column from non-existent data store '${dataStoreId}'`,
);
await this.dataStoreColumnRepository.moveColumn(dataStoreId, columnId, dto.targetIndex);
return true;
}
async deleteColumn(dataStoreId: string, columnId: string) {
await this.validateDataStoreExists(
dataStoreId,
`Tried to delete column from non-existent data store '${dataStoreId}'`,
);
const existingColumnMatch = await this.dataStoreColumnRepository.findOneBy({
id: columnId,
dataStoreId,
});
if (existingColumnMatch === null) {
throw new UserError(
`Tried to delete column with name not present in data store '${dataStoreId}'`,
);
}
await this.dataStoreColumnRepository.deleteColumn(dataStoreId, existingColumnMatch);
return true;
}
async getManyAndCount(options: DataStoreListOptions) {
return await this.dataStoreRepository.getManyAndCount(options);
}
async getManyRowsAndCount(dataStoreId: string, dto: ListDataStoreContentQueryDto) {
// unclear if we should validate here, only use case would be to reduce the chance of
// a renamed/removed column appearing here (or added column missing) if the store was
// modified between when the frontend sent the request and we received it
const columns = await this.dataStoreColumnRepository.getColumns(dataStoreId);
const result = await this.dataStoreRowsRepository.getManyAndCount(
toTableName(dataStoreId),
dto,
);
return {
count: result.count,
data: this.normalizeRows(result.data, columns),
};
}
async getColumns(dataStoreId: string) {
return await this.dataStoreColumnRepository.getColumns(dataStoreId);
}
// TODO: move to utils and test
private normalizeRows(rows: Array<Record<string, unknown>>, columns: DataStoreColumn[]) {
const typeMap = new Map(columns.map((col) => [col.name, col.type]));
return rows.map((row) => {
const normalized = { ...row };
for (const [key, value] of Object.entries(row)) {
const type = typeMap.get(key);
if (type === 'boolean') {
// Convert boolean values to true/false
if (typeof value === 'boolean') {
normalized[key] = value;
} else if (value === 1 || value === '1') {
normalized[key] = true;
} else if (value === 0 || value === '0') {
normalized[key] = false;
}
}
if (type === 'date' && value !== null && value !== undefined) {
// Convert date objects or strings to ISO string
let dateObj: Date | null = null;
if (value instanceof Date) {
dateObj = value;
} else if (typeof value === 'string' || typeof value === 'number') {
const parsed = new Date(value);
if (!isNaN(parsed.getTime())) {
dateObj = parsed;
}
}
normalized[key] = dateObj ? dateObj.toISOString() : value;
}
}
return normalized;
});
}
private async validateRows(dataStoreId: string, rows: DataStoreRows): Promise<void> {
const columns = await this.dataStoreColumnRepository.getColumns(dataStoreId);
if (columns.length === 0) {
throw new UserError('No columns found for this data store or data store not found');
}
const columnNames = new Set(columns.map((x) => x.name));
const columnTypeMap = new Map(columns.map((x) => [x.name, x.type]));
for (const row of rows) {
const keys = Object.keys(row);
if (columns.length !== keys.length) {
throw new UserError('mismatched key count');
}
for (const key of keys) {
if (!columnNames.has(key)) {
throw new UserError('unknown column name');
}
const cell = row[key];
if (cell === null) continue;
switch (columnTypeMap.get(key)) {
case 'boolean':
if (typeof cell !== 'boolean')
throw new UserError(
`value '${cell.toString()}' does not match column type 'boolean'`,
);
break;
case 'date':
if (!(cell instanceof Date))
throw new UserError(`value '${cell}' does not match column type 'date'`);
row[key] = cell.toISOString();
break;
case 'string':
if (typeof cell !== 'string')
throw new UserError(`value '${cell.toString()}' does not match column type 'string'`);
break;
case 'number':
if (typeof cell !== 'number')
throw new UserError(`value '${cell.toString()}' does not match column type 'number'`);
break;
}
}
}
}
async insertRows(dataStoreId: string, rows: DataStoreRows) {
await this.validateRows(dataStoreId, rows);
const columns = await this.dataStoreColumnRepository.getColumns(dataStoreId);
return await this.dataStoreRowsRepository.insertRows(toTableName(dataStoreId), rows, columns);
}
async upsertRows(dataStoreId: string, dto: UpsertDataStoreRowsDto) {
await this.validateRows(dataStoreId, dto.rows);
const columns = await this.dataStoreColumnRepository.getColumns(dataStoreId);
return await this.dataStoreRowsRepository.upsertRows(toTableName(dataStoreId), dto, columns);
}
private async validateDataStoreExists(dataStoreId: string, msg?: string) {
const existingTable = await this.dataStoreRepository.findOneBy({
id: dataStoreId,
});
if (!existingTable) {
throw new UserError(msg ?? `Data Store '${dataStoreId}' does not exist.`);
}
}
}

View File

@@ -0,0 +1 @@
export type DataStoreUserTableName = `data_store_user_${string}`;

View File

@@ -0,0 +1,242 @@
import {
DATA_STORE_COLUMN_REGEX,
type DataStoreRows,
type DataStoreCreateColumnSchema,
} from '@n8n/api-types';
import { DslColumn } from '@n8n/db';
import type { DataSourceOptions } from '@n8n/typeorm';
import { UnexpectedError } from 'n8n-workflow';
import type { DataStoreUserTableName } from '../data-store.types';
import { NotFoundError } from '@/errors/response-errors/not-found.error';
export function toDslColumns(columns: DataStoreCreateColumnSchema[]): DslColumn[] {
return columns.map((col) => {
const name = new DslColumn(col.name.trim());
switch (col.type) {
case 'number':
return name.int;
case 'boolean':
return name.bool;
case 'string':
return name.text;
case 'date':
return name.timestampTimezone();
default:
return name.text;
}
});
}
function dataStoreColumnTypeToSql(
type: DataStoreCreateColumnSchema['type'],
dbType: DataSourceOptions['type'],
) {
switch (type) {
case 'string':
return 'TEXT';
case 'number':
return 'FLOAT';
case 'boolean':
return 'BOOLEAN';
case 'date':
if (dbType === 'postgres') {
return 'TIMESTAMP';
}
return 'DATETIME';
default:
throw new NotFoundError(`Unsupported field type: ${type as string}`);
}
}
function columnToWildcardAndType(
column: DataStoreCreateColumnSchema,
dbType: DataSourceOptions['type'],
) {
return `${quoteIdentifier(column.name, dbType)} ${dataStoreColumnTypeToSql(column.type, dbType)}`;
}
function isValidColumnName(name: string) {
// Only allow alphanumeric and underscore
return DATA_STORE_COLUMN_REGEX.test(name);
}
export function addColumnQuery(
tableName: DataStoreUserTableName,
column: DataStoreCreateColumnSchema,
dbType: DataSourceOptions['type'],
) {
// API requests should already conform to this, but better safe than sorry
if (!isValidColumnName(column.name)) {
throw new UnexpectedError('bad column name');
}
const quotedTableName = quoteIdentifier(tableName, dbType);
return `ALTER TABLE ${quotedTableName} ADD ${columnToWildcardAndType(column, dbType)}`;
}
export function deleteColumnQuery(
tableName: DataStoreUserTableName,
column: string,
dbType: DataSourceOptions['type'],
): string {
const quotedTableName = quoteIdentifier(tableName, dbType);
return `ALTER TABLE ${quotedTableName} DROP COLUMN ${quoteIdentifier(column, dbType)}`;
}
export function buildInsertQuery(
tableName: DataStoreUserTableName,
rows: DataStoreRows,
columns: Array<{ name: string; type: string }>,
dbType: DataSourceOptions['type'] = 'sqlite',
): [string, unknown[]] {
if (rows.length === 0 || Object.keys(rows[0]).length === 0) {
return ['', []];
}
const keys = Object.keys(rows[0]);
const quotedKeys = keys.map((key) => quoteIdentifier(key, dbType)).join(', ');
const quotedTableName = quoteIdentifier(tableName, dbType);
const columnTypeMap = buildColumnTypeMap(columns);
const parameters: unknown[] = [];
const valuePlaceholders: string[] = [];
let placeholderIndex = 1;
for (const row of rows) {
const rowPlaceholders = keys.map((key) => {
const value = normalizeValue(row[key], columnTypeMap[key], dbType);
parameters.push(value);
return getPlaceholder(placeholderIndex++, dbType);
});
valuePlaceholders.push(`(${rowPlaceholders.join(', ')})`);
}
const query = `INSERT INTO ${quotedTableName} (${quotedKeys}) VALUES ${valuePlaceholders.join(', ')}`;
return [query, parameters];
}
export function buildUpdateQuery(
tableName: DataStoreUserTableName,
row: Record<string, unknown>,
columns: Array<{ name: string; type: string }>,
matchFields: string[],
dbType: DataSourceOptions['type'] = 'sqlite',
): [string, unknown[]] {
if (Object.keys(row).length === 0 || matchFields.length === 0) {
return ['', []];
}
const updateKeys = Object.keys(row).filter((key) => !matchFields.includes(key));
if (updateKeys.length === 0) {
return ['', []];
}
const quotedTableName = quoteIdentifier(tableName, dbType);
const columnTypeMap = buildColumnTypeMap(columns);
const parameters: unknown[] = [];
let placeholderIndex = 1;
const setClause = updateKeys
.map((key) => {
const value = normalizeValue(row[key], columnTypeMap[key], dbType);
parameters.push(value);
return `${quoteIdentifier(key, dbType)} = ${getPlaceholder(placeholderIndex++, dbType)}`;
})
.join(', ');
const whereClause = matchFields
.map((key) => {
const value = normalizeValue(row[key], columnTypeMap[key], dbType);
parameters.push(value);
return `${quoteIdentifier(key, dbType)} = ${getPlaceholder(placeholderIndex++, dbType)}`;
})
.join(' AND ');
const query = `UPDATE ${quotedTableName} SET ${setClause} WHERE ${whereClause}`;
return [query, parameters];
}
export function splitRowsByExistence(
existing: Array<Record<string, unknown>>,
matchFields: string[],
rows: DataStoreRows,
): { rowsToInsert: DataStoreRows; rowsToUpdate: DataStoreRows } {
// Extracts only the fields relevant to matching and serializes them for comparison
const getMatchKey = (row: Record<string, unknown>): string =>
JSON.stringify(Object.fromEntries(matchFields.map((field) => [field, row[field]])));
const existingSet = new Set(existing.map((row) => getMatchKey(row)));
const rowsToUpdate: DataStoreRows = [];
const rowsToInsert: DataStoreRows = [];
for (const row of rows) {
const key = getMatchKey(row);
if (existingSet.has(key)) {
rowsToUpdate.push(row);
} else {
rowsToInsert.push(row);
}
}
return { rowsToInsert, rowsToUpdate };
}
export function quoteIdentifier(name: string, dbType: DataSourceOptions['type']): string {
switch (dbType) {
case 'mysql':
case 'mariadb':
return `\`${name}\``;
case 'postgres':
case 'sqlite':
default:
return `"${name}"`;
}
}
export function toTableName(dataStoreId: string): DataStoreUserTableName {
return `data_store_user_${dataStoreId}`;
}
function normalizeValue(
value: unknown,
columnType: string | undefined,
dbType: DataSourceOptions['type'],
): unknown {
if (['mysql', 'mariadb'].includes(dbType)) {
if (columnType === 'date') {
if (
value instanceof Date ||
(typeof value === 'string' && value.match(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}/))
) {
return toMySQLDateTimeString(value);
}
}
}
return value;
}
function toMySQLDateTimeString(date: Date | string, convertFromDate = true): string {
const dateString = convertFromDate
? date instanceof Date
? date.toISOString()
: date
: (date as string);
return dateString.replace('T', ' ').replace('Z', '');
}
export function getPlaceholder(index: number, dbType: DataSourceOptions['type']): string {
return dbType.includes('postgres') ? `$${index}` : '?';
}
function buildColumnTypeMap(
columns: Array<{ name: string; type: string }>,
): Record<string, string> {
return Object.fromEntries(columns.map((col) => [col.name, col.type]));
}

View File

@@ -42,9 +42,10 @@ type EndpointGroup =
| 'evaluation'
| 'ai'
| 'folder'
| 'insights';
| 'insights'
| 'data-store';
type ModuleName = 'insights' | 'external-secrets';
type ModuleName = 'insights' | 'external-secrets' | 'data-store';
export interface SetupProps {
endpointGroups?: EndpointGroup[];

View File

@@ -1,6 +1,6 @@
<script setup lang="ts">
import { onMounted, ref } from 'vue';
import type { DataStoreEntity } from '@/features/dataStore/datastore.types';
import type { DataStore } from '@/features/dataStore/datastore.types';
import { useDataStoreStore } from '@/features/dataStore/dataStore.store';
import { useToast } from '@/composables/useToast';
import { useI18n } from '@n8n/i18n';
@@ -24,7 +24,7 @@ const documentTitle = useDocumentTitle();
const dataStoreStore = useDataStoreStore();
const loading = ref(false);
const dataStore = ref<DataStoreEntity | null>(null);
const dataStore = ref<DataStore | null>(null);
const showErrorAndGoBackToList = async (error: unknown) => {
if (!(error instanceof Error)) {

View File

@@ -9,6 +9,7 @@ import { createTestingPinia } from '@pinia/testing';
import { createRouter, createWebHistory } from 'vue-router';
import type { DataStoreResource } from '@/features/dataStore/types';
import { useDataStoreStore } from '@/features/dataStore/dataStore.store';
vi.mock('@/composables/useProjectPages', () => ({
useProjectPages: vi.fn().mockReturnValue({
isOverviewSubPage: false,

View File

@@ -5,7 +5,7 @@ import { vi } from 'vitest';
import DataStoreActions from '@/features/dataStore/components/DataStoreActions.vue';
import { DATA_STORE_CARD_ACTIONS } from '@/features/dataStore/constants';
import { MODAL_CONFIRM } from '@/constants';
import type { DataStoreEntity } from '@/features/dataStore/datastore.types';
import type { DataStore } from '@/features/dataStore/datastore.types';
const mockMessage = {
confirm: vi.fn(),
@@ -49,7 +49,7 @@ vi.mock('@n8n/i18n', async (importOriginal) => ({
}),
}));
const mockDataStore: DataStoreEntity = {
const mockDataStore: DataStore = {
id: '1',
name: 'Test DataStore',
sizeBytes: 1024,

View File

@@ -1,5 +1,5 @@
<script setup lang="ts">
import type { DataStoreEntity } from '@/features/dataStore/datastore.types';
import type { DataStore } from '@/features/dataStore/datastore.types';
import type { IUser, UserAction } from '@n8n/design-system';
import { DATA_STORE_CARD_ACTIONS } from '@/features/dataStore/constants';
import { useI18n } from '@n8n/i18n';
@@ -10,7 +10,7 @@ import { useDataStoreStore } from '@/features/dataStore/dataStore.store';
import { useToast } from '@/composables/useToast';
type Props = {
dataStore: DataStoreEntity;
dataStore: DataStore;
isReadOnly?: boolean;
};
@@ -21,7 +21,7 @@ const props = withDefaults(defineProps<Props>(), {
const emit = defineEmits<{
rename: [
value: {
dataStore: DataStoreEntity;
dataStore: DataStore;
action: string;
},
];

View File

@@ -3,7 +3,7 @@ import { createComponentRenderer } from '@/__tests__/render';
import { createTestingPinia } from '@pinia/testing';
import { vi } from 'vitest';
import DataStoreBreadcrumbs from '@/features/dataStore/components/DataStoreBreadcrumbs.vue';
import type { DataStoreEntity } from '@/features/dataStore/datastore.types';
import type { DataStore } from '@/features/dataStore/datastore.types';
const mockRouter = {
push: vi.fn(),
@@ -48,7 +48,7 @@ vi.mock('@n8n/i18n', async (importOriginal) => ({
}),
}));
const mockDataStore: DataStoreEntity = {
const mockDataStore: DataStore = {
id: '1',
name: 'Test DataStore',
sizeBytes: 1024,
@@ -69,7 +69,7 @@ const mockDataStore: DataStoreEntity = {
},
};
const mockDataStoreWithoutProject: DataStoreEntity = {
const mockDataStoreWithoutProject: DataStore = {
...mockDataStore,
project: undefined,
};

View File

@@ -1,6 +1,6 @@
<script setup lang="ts">
import { computed, nextTick, ref, useTemplateRef, watch } from 'vue';
import type { DataStoreEntity } from '@/features/dataStore/datastore.types';
import type { DataStore } from '@/features/dataStore/datastore.types';
import { useI18n } from '@n8n/i18n';
import type { PathItem } from '@n8n/design-system/components/N8nBreadcrumbs/Breadcrumbs.vue';
import { useRouter } from 'vue-router';
@@ -12,7 +12,7 @@ import { useToast } from '@/composables/useToast';
const BREADCRUMBS_SEPARATOR = '';
type Props = {
dataStore: DataStoreEntity;
dataStore: DataStore;
};
const props = defineProps<Props>();

View File

@@ -1,11 +1,11 @@
import { makeRestApiRequest } from '@n8n/rest-api-client';
import type { IRestApiContext } from '@n8n/rest-api-client';
import { type DataStoreEntity } from '@/features/dataStore/datastore.types';
import { type DataStore } from '@/features/dataStore/datastore.types';
export const fetchDataStoresApi = async (
context: IRestApiContext,
projectId?: string,
projectId: string,
options?: {
skip?: number;
take?: number;
@@ -17,7 +17,7 @@ export const fetchDataStoresApi = async (
},
) => {
const apiEndpoint = projectId ? `/projects/${projectId}/data-stores` : '/data-stores-global';
return await makeRestApiRequest<{ count: number; data: DataStoreEntity[] }>(
return await makeRestApiRequest<{ count: number; data: DataStore[] }>(
context,
'GET',
apiEndpoint,
@@ -33,7 +33,7 @@ export const createDataStoreApi = async (
name: string,
projectId?: string,
) => {
return await makeRestApiRequest<DataStoreEntity>(
return await makeRestApiRequest<DataStore>(
context,
'POST',
`/projects/${projectId}/data-stores`,
@@ -66,7 +66,7 @@ export const updateDataStoreApi = async (
name: string,
projectId?: string,
) => {
return await makeRestApiRequest<DataStoreEntity>(
return await makeRestApiRequest<DataStore>(
context,
'PATCH',
`/projects/${projectId}/data-stores/${dataStoreId}`,

View File

@@ -7,15 +7,15 @@ import {
createDataStoreApi,
deleteDataStoreApi,
updateDataStoreApi,
} from '@/features/dataStore/datastore.api';
import type { DataStoreEntity } from '@/features/dataStore/datastore.types';
} from '@/features/dataStore/dataStore.api';
import type { DataStore } from '@/features/dataStore/datastore.types';
import { useProjectsStore } from '@/stores/projects.store';
export const useDataStoreStore = defineStore(DATA_STORE_STORE, () => {
const rootStore = useRootStore();
const projectStore = useProjectsStore();
const dataStores = ref<DataStoreEntity[]>([]);
const dataStores = ref<DataStore[]>([]);
const totalCount = ref(0);
const fetchDataStores = async (projectId: string, page: number, pageSize: number) => {

View File

@@ -1,18 +1,18 @@
import type { Project } from '@/types/projects.types';
export type DataStoreEntity = {
export type DataStore = {
id: string;
name: string;
sizeBytes: number;
recordCount: number;
columns: DataStoreColumnEntity[];
columns: DataStoreColumn[];
createdAt: string;
updatedAt: string;
projectId?: string;
project?: Project;
};
export type DataStoreColumnEntity = {
export type DataStoreColumn = {
id: string;
name: string;
type: string;

View File

@@ -1,12 +1,12 @@
import type { BaseResource } from '@/Interface';
import type { DataStoreEntity } from '@/features/dataStore/datastore.types';
import type { DataStore } from '@/features/dataStore/datastore.types';
/**
* Data Store resource type definition
* This extends the ModuleResources interface to add DataStore as a resource type
*/
export type DataStoreResource = BaseResource &
DataStoreEntity & {
DataStore & {
resourceType: 'datastore';
};

View File

@@ -37,6 +37,7 @@ export const useRBACStore = defineStore(STORES.RBAC, () => {
securityAudit: {},
folder: {},
insights: {},
dataStore: {},
});
function addGlobalRole(role: Role) {