mirror of
https://github.com/Abdulazizzn/n8n-enterprise-unlocked.git
synced 2025-12-16 17:46:45 +00:00
feat(core): Add Data Store Backend API (no-changelog) (#17824)
This commit is contained in:
@@ -0,0 +1,5 @@
|
||||
import { Z } from 'zod-class';
|
||||
|
||||
import { dataStoreCreateColumnSchema } from '../../schemas/data-store.schema';
|
||||
|
||||
export class AddDataStoreColumnDto extends Z.class(dataStoreCreateColumnSchema.shape) {}
|
||||
@@ -0,0 +1,8 @@
|
||||
import { z } from 'zod';
|
||||
import { Z } from 'zod-class';
|
||||
|
||||
import { dataStoreColumnNameSchema } from '../../schemas/data-store.schema';
|
||||
|
||||
export class AddDataStoreRowsDto extends Z.class({
|
||||
data: z.array(z.record(dataStoreColumnNameSchema, z.any())),
|
||||
}) {}
|
||||
@@ -0,0 +1,11 @@
|
||||
import { Z } from 'zod-class';
|
||||
|
||||
import {
|
||||
dataStoreColumnNameSchema,
|
||||
dataStoreColumnTypeSchema,
|
||||
} from '../../schemas/data-store.schema';
|
||||
|
||||
export class CreateDataStoreColumnDto extends Z.class({
|
||||
name: dataStoreColumnNameSchema,
|
||||
type: dataStoreColumnTypeSchema,
|
||||
}) {}
|
||||
@@ -0,0 +1,10 @@
|
||||
import { z } from 'zod';
|
||||
import { Z } from 'zod-class';
|
||||
|
||||
import { CreateDataStoreColumnDto } from './create-data-store-column.dto';
|
||||
import { dataStoreNameSchema } from '../../schemas/data-store.schema';
|
||||
|
||||
export class CreateDataStoreDto extends Z.class({
|
||||
name: dataStoreNameSchema,
|
||||
columns: z.array(CreateDataStoreColumnDto),
|
||||
}) {}
|
||||
@@ -0,0 +1,105 @@
|
||||
import { jsonParse } from 'n8n-workflow';
|
||||
import { z } from 'zod';
|
||||
import { Z } from 'zod-class';
|
||||
|
||||
import { dataStoreColumnNameSchema } from '../../schemas/data-store.schema';
|
||||
import { paginationSchema } from '../pagination/pagination.dto';
|
||||
|
||||
const FilterConditionSchema = z.union([z.literal('eq'), z.literal('neq')]);
|
||||
export type ListDataStoreContentFilterConditionType = z.infer<typeof FilterConditionSchema>;
|
||||
|
||||
const filterRecord = z.object({
|
||||
columnName: dataStoreColumnNameSchema,
|
||||
condition: FilterConditionSchema.default('eq'),
|
||||
value: z.union([z.string(), z.number(), z.boolean(), z.date()]),
|
||||
});
|
||||
|
||||
const chainedFilterSchema = z.union([z.literal('and'), z.literal('or')]);
|
||||
|
||||
export type ListDataStoreContentFilter = z.infer<typeof filterSchema>;
|
||||
|
||||
// ---------------------
|
||||
// Parameter Validators
|
||||
// ---------------------
|
||||
|
||||
const filterSchema = z.object({
|
||||
type: chainedFilterSchema.default('and'),
|
||||
filters: z.array(filterRecord).default([]),
|
||||
});
|
||||
|
||||
// Filter parameter validation
|
||||
const filterValidator = z
|
||||
.string()
|
||||
.optional()
|
||||
.transform((val, ctx) => {
|
||||
if (!val) return undefined;
|
||||
try {
|
||||
const parsed: unknown = jsonParse(val);
|
||||
try {
|
||||
return filterSchema.parse(parsed);
|
||||
} catch (e) {
|
||||
ctx.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
message: 'Invalid filter fields',
|
||||
path: ['filter'],
|
||||
});
|
||||
return z.NEVER;
|
||||
}
|
||||
} catch (e) {
|
||||
ctx.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
message: 'Invalid filter format',
|
||||
path: ['filter'],
|
||||
});
|
||||
return z.NEVER;
|
||||
}
|
||||
});
|
||||
|
||||
// SortBy parameter validation
|
||||
const sortByValidator = z
|
||||
.string()
|
||||
.optional()
|
||||
.transform((val, ctx) => {
|
||||
if (val === undefined) return val;
|
||||
|
||||
if (!val.includes(':')) {
|
||||
ctx.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
message: 'Invalid sort format, expected <columnName>:<asc/desc>',
|
||||
path: ['sort'],
|
||||
});
|
||||
return z.NEVER;
|
||||
}
|
||||
|
||||
let [column, direction] = val.split(':');
|
||||
|
||||
try {
|
||||
column = dataStoreColumnNameSchema.parse(column);
|
||||
} catch {
|
||||
ctx.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
message: 'Invalid sort columnName',
|
||||
path: ['sort'],
|
||||
});
|
||||
return z.NEVER;
|
||||
}
|
||||
|
||||
direction = direction?.toUpperCase();
|
||||
if (direction !== 'ASC' && direction !== 'DESC') {
|
||||
ctx.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
message: 'Invalid sort direction',
|
||||
path: ['sort'],
|
||||
});
|
||||
|
||||
return z.NEVER;
|
||||
}
|
||||
return [column, direction] as const;
|
||||
});
|
||||
|
||||
export class ListDataStoreContentQueryDto extends Z.class({
|
||||
take: paginationSchema.take.optional(),
|
||||
skip: paginationSchema.skip.optional(),
|
||||
filter: filterValidator.optional(),
|
||||
sortBy: sortByValidator.optional(),
|
||||
}) {}
|
||||
@@ -0,0 +1,70 @@
|
||||
import { jsonParse } from 'n8n-workflow';
|
||||
import { z } from 'zod';
|
||||
import { Z } from 'zod-class';
|
||||
|
||||
import { paginationSchema } from '../pagination/pagination.dto';
|
||||
|
||||
const VALID_SORT_OPTIONS = [
|
||||
'name:asc',
|
||||
'name:desc',
|
||||
'createdAt:asc',
|
||||
'createdAt:desc',
|
||||
'updatedAt:asc',
|
||||
'updatedAt:desc',
|
||||
'sizeBytes:asc',
|
||||
'sizeBytes:desc',
|
||||
] as const;
|
||||
|
||||
export type ListDataStoreQuerySortOptions = (typeof VALID_SORT_OPTIONS)[number];
|
||||
|
||||
const FILTER_OPTIONS = {
|
||||
id: z.union([z.string(), z.array(z.string())]).optional(),
|
||||
name: z.union([z.string(), z.array(z.string())]).optional(),
|
||||
projectId: z.union([z.string(), z.array(z.string())]).optional(),
|
||||
// todo: can probably include others here as well?
|
||||
};
|
||||
|
||||
// Filter schema - only allow specific properties
|
||||
const filterSchema = z.object(FILTER_OPTIONS).strict();
|
||||
// ---------------------
|
||||
// Parameter Validators
|
||||
// ---------------------
|
||||
|
||||
// Filter parameter validation
|
||||
const filterValidator = z
|
||||
.string()
|
||||
.optional()
|
||||
.transform((val, ctx) => {
|
||||
if (!val) return undefined;
|
||||
try {
|
||||
const parsed: unknown = jsonParse(val);
|
||||
try {
|
||||
return filterSchema.parse(parsed);
|
||||
} catch (e) {
|
||||
ctx.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
message: 'Invalid filter fields',
|
||||
path: ['filter'],
|
||||
});
|
||||
return z.NEVER;
|
||||
}
|
||||
} catch (e) {
|
||||
ctx.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
message: 'Invalid filter format',
|
||||
path: ['filter'],
|
||||
});
|
||||
return z.NEVER;
|
||||
}
|
||||
});
|
||||
|
||||
// SortBy parameter validation
|
||||
const sortByValidator = z
|
||||
.enum(VALID_SORT_OPTIONS, { message: `sortBy must be one of: ${VALID_SORT_OPTIONS.join(', ')}` })
|
||||
.optional();
|
||||
|
||||
export class ListDataStoreQueryDto extends Z.class({
|
||||
...paginationSchema,
|
||||
filter: filterValidator,
|
||||
sortBy: sortByValidator,
|
||||
}) {}
|
||||
@@ -0,0 +1,6 @@
|
||||
import { z } from 'zod';
|
||||
import { Z } from 'zod-class';
|
||||
|
||||
export class MoveDataStoreColumnDto extends Z.class({
|
||||
targetIndex: z.number(),
|
||||
}) {}
|
||||
@@ -0,0 +1,7 @@
|
||||
import { Z } from 'zod-class';
|
||||
|
||||
import { dataStoreNameSchema } from '../../schemas/data-store.schema';
|
||||
|
||||
export class UpdateDataStoreDto extends Z.class({
|
||||
name: dataStoreNameSchema,
|
||||
}) {}
|
||||
@@ -0,0 +1,13 @@
|
||||
import { z } from 'zod';
|
||||
import { Z } from 'zod-class';
|
||||
|
||||
import { dataStoreColumnNameSchema } from '../../schemas/data-store.schema';
|
||||
|
||||
const dataStoreValueSchema = z.union([z.string(), z.number(), z.boolean(), z.date(), z.null()]);
|
||||
|
||||
const upsertDataStoreRowsShape = {
|
||||
rows: z.array(z.record(dataStoreValueSchema)),
|
||||
matchFields: z.array(dataStoreColumnNameSchema).min(1),
|
||||
};
|
||||
|
||||
export class UpsertDataStoreRowsDto extends Z.class(upsertDataStoreRowsShape) {}
|
||||
@@ -79,3 +79,16 @@ export {
|
||||
} from './user/users-list-filter.dto';
|
||||
|
||||
export { OidcConfigDto } from './oidc/config.dto';
|
||||
|
||||
export { CreateDataStoreDto } from './data-store/create-data-store.dto';
|
||||
export { UpdateDataStoreDto } from './data-store/update-data-store.dto';
|
||||
export { UpsertDataStoreRowsDto } from './data-store/upsert-data-store-rows.dto';
|
||||
export { ListDataStoreQueryDto } from './data-store/list-data-store-query.dto';
|
||||
export {
|
||||
ListDataStoreContentQueryDto,
|
||||
ListDataStoreContentFilter,
|
||||
} from './data-store/list-data-store-content-query.dto';
|
||||
export { CreateDataStoreColumnDto } from './data-store/create-data-store-column.dto';
|
||||
export { AddDataStoreRowsDto } from './data-store/add-data-store-rows.dto';
|
||||
export { AddDataStoreColumnDto } from './data-store/add-data-store-column.dto';
|
||||
export { MoveDataStoreColumnDto } from './data-store/move-data-store-column.dto';
|
||||
|
||||
@@ -45,3 +45,15 @@ export {
|
||||
type UsersList,
|
||||
usersListSchema,
|
||||
} from './schemas/user.schema';
|
||||
|
||||
export {
|
||||
DATA_STORE_COLUMN_REGEX,
|
||||
type DataStore,
|
||||
type DataStoreColumn,
|
||||
type DataStoreCreateColumnSchema,
|
||||
type DataStoreColumnJsType,
|
||||
type DataStoreListFilter,
|
||||
type DataStoreRows,
|
||||
type DataStoreListOptions,
|
||||
type DataStoreUserTableName,
|
||||
} from './schemas/data-store.schema';
|
||||
|
||||
56
packages/@n8n/api-types/src/schemas/data-store.schema.ts
Normal file
56
packages/@n8n/api-types/src/schemas/data-store.schema.ts
Normal file
@@ -0,0 +1,56 @@
|
||||
import { z } from 'zod';
|
||||
|
||||
import type { ListDataStoreQueryDto } from '../dto';
|
||||
|
||||
export const dataStoreNameSchema = z.string().trim().min(1).max(128);
|
||||
export const dataStoreIdSchema = z.string().max(36);
|
||||
|
||||
export const DATA_STORE_COLUMN_REGEX = /^[a-zA-Z0-9][a-zA-Z0-9-]*$/;
|
||||
|
||||
export const dataStoreColumnNameSchema = z
|
||||
.string()
|
||||
.trim()
|
||||
.min(1)
|
||||
.max(128)
|
||||
.regex(
|
||||
DATA_STORE_COLUMN_REGEX,
|
||||
'Only alphanumeric characters and non-leading dashes are allowed for column names',
|
||||
);
|
||||
export const dataStoreColumnTypeSchema = z.enum(['string', 'number', 'boolean', 'date']);
|
||||
|
||||
export const dataStoreCreateColumnSchema = z.object({
|
||||
name: dataStoreColumnNameSchema,
|
||||
type: dataStoreColumnTypeSchema,
|
||||
index: z.number().optional(),
|
||||
});
|
||||
export type DataStoreCreateColumnSchema = z.infer<typeof dataStoreCreateColumnSchema>;
|
||||
|
||||
export const dataStoreColumnSchema = dataStoreCreateColumnSchema.extend({
|
||||
dataStoreId: dataStoreIdSchema,
|
||||
});
|
||||
|
||||
export const dataStoreSchema = z.object({
|
||||
id: dataStoreIdSchema,
|
||||
name: dataStoreNameSchema,
|
||||
columns: z.array(dataStoreColumnSchema),
|
||||
createdAt: z.string().datetime(),
|
||||
updatedAt: z.string().datetime(),
|
||||
});
|
||||
export type DataStore = z.infer<typeof dataStoreSchema>;
|
||||
export type DataStoreColumn = z.infer<typeof dataStoreColumnSchema>;
|
||||
|
||||
export type DataStoreUserTableName = `data_store_user_${string}`;
|
||||
|
||||
export type DataStoreListFilter = {
|
||||
id?: string | string[];
|
||||
projectId?: string | string[];
|
||||
name?: string;
|
||||
};
|
||||
|
||||
export type DataStoreListOptions = Partial<ListDataStoreQueryDto> & {
|
||||
filter: { projectId: string };
|
||||
};
|
||||
|
||||
export type DataStoreColumnJsType = string | number | boolean | Date;
|
||||
|
||||
export type DataStoreRows = Array<Record<string, DataStoreColumnJsType | null>>;
|
||||
@@ -15,7 +15,9 @@ beforeEach(() => {
|
||||
|
||||
describe('eligibleModules', () => {
|
||||
it('should consider all default modules eligible', () => {
|
||||
expect(Container.get(ModuleRegistry).eligibleModules).toEqual(MODULE_NAMES);
|
||||
// 'data-store' isn't (yet) eligible module by default
|
||||
const expectedModules = MODULE_NAMES.filter((name) => name !== 'data-store');
|
||||
expect(Container.get(ModuleRegistry).eligibleModules).toEqual(expectedModules);
|
||||
});
|
||||
|
||||
it('should consider a module ineligible if it was disabled via env var', () => {
|
||||
@@ -23,6 +25,15 @@ describe('eligibleModules', () => {
|
||||
expect(Container.get(ModuleRegistry).eligibleModules).toEqual(['external-secrets']);
|
||||
});
|
||||
|
||||
it('should consider a module eligible if it was enabled via env var', () => {
|
||||
process.env.N8N_ENABLED_MODULES = 'data-store';
|
||||
expect(Container.get(ModuleRegistry).eligibleModules).toEqual([
|
||||
'insights',
|
||||
'external-secrets',
|
||||
'data-store',
|
||||
]);
|
||||
});
|
||||
|
||||
it('should throw `ModuleConfusionError` if a module is both enabled and disabled', () => {
|
||||
process.env.N8N_ENABLED_MODULES = 'insights';
|
||||
process.env.N8N_DISABLED_MODULES = 'insights';
|
||||
|
||||
@@ -2,7 +2,7 @@ import { CommaSeparatedStringArray, Config, Env } from '@n8n/config';
|
||||
|
||||
import { UnknownModuleError } from './errors/unknown-module.error';
|
||||
|
||||
export const MODULE_NAMES = ['insights', 'external-secrets'] as const;
|
||||
export const MODULE_NAMES = ['insights', 'external-secrets', 'data-store'] as const;
|
||||
|
||||
export type ModuleName = (typeof MODULE_NAMES)[number];
|
||||
|
||||
|
||||
@@ -66,7 +66,13 @@ export async function terminate() {
|
||||
dbConnection.connectionState.connected = false;
|
||||
}
|
||||
|
||||
type EntityName = keyof typeof entities | 'InsightsRaw' | 'InsightsByPeriod' | 'InsightsMetadata';
|
||||
type EntityName =
|
||||
| keyof typeof entities
|
||||
| 'InsightsRaw'
|
||||
| 'InsightsByPeriod'
|
||||
| 'InsightsMetadata'
|
||||
| 'DataStore'
|
||||
| 'DataStoreColumn';
|
||||
|
||||
/**
|
||||
* Truncate specific DB tables in a test DB.
|
||||
|
||||
@@ -19,6 +19,7 @@ export const LOG_SCOPES = [
|
||||
'insights',
|
||||
'workflow-activation',
|
||||
'ssh-client',
|
||||
'data-store',
|
||||
'cron',
|
||||
'community-nodes',
|
||||
'legacy-sqlite-execution-recovery',
|
||||
|
||||
@@ -23,6 +23,8 @@ export { NoUrl } from './utils/validators/no-url.validator';
|
||||
export * from './repositories';
|
||||
export * from './subscribers';
|
||||
|
||||
export { Column as DslColumn } from './migrations/dsl/column';
|
||||
export { CreateTable } from './migrations/dsl/table';
|
||||
export { sqliteMigrations } from './migrations/sqlite';
|
||||
export { mysqlMigrations } from './migrations/mysqldb';
|
||||
export { postgresMigrations } from './migrations/postgresdb';
|
||||
|
||||
@@ -26,6 +26,7 @@ export const RESOURCES = {
|
||||
folder: [...DEFAULT_OPERATIONS, 'move'] as const,
|
||||
insights: ['list'] as const,
|
||||
oidc: ['manage'] as const,
|
||||
dataStore: [...DEFAULT_OPERATIONS, 'readRow', 'writeRow'] as const,
|
||||
} as const;
|
||||
|
||||
export const API_KEY_RESOURCES = {
|
||||
|
||||
@@ -79,6 +79,13 @@ export const GLOBAL_OWNER_SCOPES: Scope[] = [
|
||||
'insights:list',
|
||||
'folder:move',
|
||||
'oidc:manage',
|
||||
'dataStore:create',
|
||||
'dataStore:delete',
|
||||
'dataStore:read',
|
||||
'dataStore:update',
|
||||
'dataStore:list',
|
||||
'dataStore:readRow',
|
||||
'dataStore:writeRow',
|
||||
];
|
||||
|
||||
export const GLOBAL_ADMIN_SCOPES = GLOBAL_OWNER_SCOPES.concat();
|
||||
@@ -98,4 +105,7 @@ export const GLOBAL_MEMBER_SCOPES: Scope[] = [
|
||||
'user:list',
|
||||
'variable:list',
|
||||
'variable:read',
|
||||
'dataStore:read',
|
||||
'dataStore:list',
|
||||
'dataStore:readRow',
|
||||
];
|
||||
|
||||
@@ -32,6 +32,13 @@ export const REGULAR_PROJECT_ADMIN_SCOPES: Scope[] = [
|
||||
'folder:list',
|
||||
'folder:move',
|
||||
'sourceControl:push',
|
||||
'dataStore:create',
|
||||
'dataStore:delete',
|
||||
'dataStore:read',
|
||||
'dataStore:update',
|
||||
'dataStore:list',
|
||||
'dataStore:readRow',
|
||||
'dataStore:writeRow',
|
||||
];
|
||||
|
||||
export const PERSONAL_PROJECT_OWNER_SCOPES: Scope[] = [
|
||||
@@ -58,6 +65,13 @@ export const PERSONAL_PROJECT_OWNER_SCOPES: Scope[] = [
|
||||
'folder:delete',
|
||||
'folder:list',
|
||||
'folder:move',
|
||||
'dataStore:create',
|
||||
'dataStore:delete',
|
||||
'dataStore:read',
|
||||
'dataStore:update',
|
||||
'dataStore:list',
|
||||
'dataStore:readRow',
|
||||
'dataStore:writeRow',
|
||||
];
|
||||
|
||||
export const PROJECT_EDITOR_SCOPES: Scope[] = [
|
||||
@@ -79,6 +93,13 @@ export const PROJECT_EDITOR_SCOPES: Scope[] = [
|
||||
'folder:update',
|
||||
'folder:delete',
|
||||
'folder:list',
|
||||
'dataStore:create',
|
||||
'dataStore:delete',
|
||||
'dataStore:read',
|
||||
'dataStore:update',
|
||||
'dataStore:list',
|
||||
'dataStore:readRow',
|
||||
'dataStore:writeRow',
|
||||
];
|
||||
|
||||
export const PROJECT_VIEWER_SCOPES: Scope[] = [
|
||||
@@ -90,4 +111,7 @@ export const PROJECT_VIEWER_SCOPES: Scope[] = [
|
||||
'workflow:read',
|
||||
'folder:read',
|
||||
'folder:list',
|
||||
'dataStore:list',
|
||||
'dataStore:read',
|
||||
'dataStore:readRow',
|
||||
];
|
||||
|
||||
@@ -31,6 +31,7 @@ describe('permissions', () => {
|
||||
workflow: {},
|
||||
folder: {},
|
||||
insights: {},
|
||||
dataStore: {},
|
||||
});
|
||||
});
|
||||
it('getResourcePermissions', () => {
|
||||
@@ -128,6 +129,7 @@ describe('permissions', () => {
|
||||
insights: {
|
||||
list: true,
|
||||
},
|
||||
dataStore: {},
|
||||
};
|
||||
|
||||
expect(getResourcePermissions(scopes)).toEqual(permissionRecord);
|
||||
|
||||
@@ -0,0 +1,215 @@
|
||||
import { createTeamProject, testDb, testModules } from '@n8n/backend-test-utils';
|
||||
import { ProjectRelationRepository, type Project, type User } from '@n8n/db';
|
||||
import { Container } from '@n8n/di';
|
||||
import type { EntityManager } from '@n8n/typeorm';
|
||||
import { mock } from 'jest-mock-extended';
|
||||
|
||||
import { createUser } from '@test-integration/db/users';
|
||||
|
||||
import { DataStoreAggregateService } from '../data-store-aggregate.service';
|
||||
import { DataStoreService } from '../data-store.service';
|
||||
|
||||
beforeAll(async () => {
|
||||
await testModules.loadModules(['data-store']);
|
||||
await testDb.init();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
await testDb.truncate(['DataStore', 'DataStoreColumn']);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await testDb.terminate();
|
||||
});
|
||||
|
||||
describe('dataStoreAggregate', () => {
|
||||
let dataStoreService: DataStoreService;
|
||||
let dataStoreAggregateService: DataStoreAggregateService;
|
||||
const manager = mock<EntityManager>();
|
||||
const projectRelationRepository = mock<ProjectRelationRepository>({ manager });
|
||||
|
||||
beforeAll(() => {
|
||||
Container.set(ProjectRelationRepository, projectRelationRepository);
|
||||
dataStoreAggregateService = Container.get(DataStoreAggregateService);
|
||||
dataStoreService = Container.get(DataStoreService);
|
||||
});
|
||||
|
||||
let user: User;
|
||||
let project1: Project;
|
||||
let project2: Project;
|
||||
|
||||
beforeEach(async () => {
|
||||
project1 = await createTeamProject();
|
||||
project2 = await createTeamProject();
|
||||
user = await createUser({ role: 'global:owner' });
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
// Clean up any created user data stores
|
||||
await dataStoreService.deleteDataStoreAll();
|
||||
});
|
||||
|
||||
describe('getManyAndCount', () => {
|
||||
it('should return the correct data stores for the user', async () => {
|
||||
// ARRANGE
|
||||
const ds1 = await dataStoreService.createDataStore(project1.id, {
|
||||
name: 'store1',
|
||||
columns: [],
|
||||
});
|
||||
const ds2 = await dataStoreService.createDataStore(project1.id, {
|
||||
name: 'store2',
|
||||
columns: [],
|
||||
});
|
||||
|
||||
projectRelationRepository.find.mockResolvedValueOnce([
|
||||
{
|
||||
userId: user.id,
|
||||
projectId: project1.id,
|
||||
role: 'project:admin',
|
||||
user,
|
||||
project: project1,
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
setUpdateDate: jest.fn(),
|
||||
},
|
||||
{
|
||||
userId: user.id,
|
||||
projectId: project2.id,
|
||||
role: 'project:viewer',
|
||||
user,
|
||||
project: project2,
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
setUpdateDate: jest.fn(),
|
||||
},
|
||||
]);
|
||||
|
||||
await dataStoreService.createDataStore(project2.id, {
|
||||
name: 'store3',
|
||||
columns: [],
|
||||
});
|
||||
|
||||
// ACT
|
||||
const result = await dataStoreAggregateService.getManyAndCount(user, {
|
||||
filter: { projectId: project1.id },
|
||||
skip: 0,
|
||||
take: 10,
|
||||
});
|
||||
|
||||
// ASSERT
|
||||
expect(result.data).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({ id: ds1!.id, name: ds1!.name }),
|
||||
expect.objectContaining({ id: ds2!.id, name: ds2!.name }),
|
||||
]),
|
||||
);
|
||||
expect(result.count).toBe(2);
|
||||
});
|
||||
|
||||
it('should return an empty array if user has no access to any project', async () => {
|
||||
// ARRANGE
|
||||
const currentUser = await createUser({ role: 'global:member' });
|
||||
|
||||
await dataStoreService.createDataStore(project1.id, {
|
||||
name: 'store1',
|
||||
columns: [],
|
||||
});
|
||||
projectRelationRepository.find.mockResolvedValueOnce([]);
|
||||
|
||||
// ACT
|
||||
const result = await dataStoreAggregateService.getManyAndCount(currentUser, {
|
||||
skip: 0,
|
||||
take: 10,
|
||||
});
|
||||
|
||||
// ASSERT
|
||||
expect(result.data).toEqual([]);
|
||||
expect(result.count).toBe(0);
|
||||
});
|
||||
|
||||
it('should return only the data store matching the given data store id filter', async () => {
|
||||
// ARRANGE
|
||||
await dataStoreService.createDataStore(project1.id, {
|
||||
name: 'store1',
|
||||
columns: [],
|
||||
});
|
||||
const ds2 = await dataStoreService.createDataStore(project1.id, {
|
||||
name: 'store2',
|
||||
columns: [],
|
||||
});
|
||||
projectRelationRepository.find.mockResolvedValueOnce([
|
||||
{
|
||||
userId: user.id,
|
||||
projectId: project1.id,
|
||||
role: 'project:admin',
|
||||
user,
|
||||
project: project1,
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
setUpdateDate: jest.fn(),
|
||||
},
|
||||
{
|
||||
userId: user.id,
|
||||
projectId: project2.id,
|
||||
role: 'project:viewer',
|
||||
user,
|
||||
project: project2,
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
setUpdateDate: jest.fn(),
|
||||
},
|
||||
]);
|
||||
|
||||
// ACT
|
||||
const result = await dataStoreAggregateService.getManyAndCount(user, {
|
||||
filter: { id: ds2!.id },
|
||||
skip: 0,
|
||||
take: 10,
|
||||
});
|
||||
|
||||
// ASSERT
|
||||
expect(result.data).toEqual([expect.objectContaining({ id: ds2!.id, name: ds2!.name })]);
|
||||
expect(result.count).toBe(1);
|
||||
});
|
||||
|
||||
it('should respect pagination (skip/take)', async () => {
|
||||
// ARRANGE
|
||||
const ds1 = await dataStoreService.createDataStore(project1.id, {
|
||||
name: 'store1',
|
||||
columns: [],
|
||||
});
|
||||
const ds2 = await dataStoreService.createDataStore(project1.id, {
|
||||
name: 'store2',
|
||||
columns: [],
|
||||
});
|
||||
const ds3 = await dataStoreService.createDataStore(project1.id, {
|
||||
name: 'store3',
|
||||
columns: [],
|
||||
});
|
||||
projectRelationRepository.find.mockResolvedValueOnce([
|
||||
{
|
||||
userId: user.id,
|
||||
projectId: project1.id,
|
||||
role: 'project:admin',
|
||||
user,
|
||||
project: project1,
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
setUpdateDate: jest.fn(),
|
||||
},
|
||||
]);
|
||||
|
||||
// ACT
|
||||
const result = await dataStoreAggregateService.getManyAndCount(user, {
|
||||
filter: { projectId: project1.id },
|
||||
skip: 1,
|
||||
take: 1,
|
||||
});
|
||||
|
||||
// ASSERT
|
||||
expect(result.data.length).toBe(1);
|
||||
expect([ds1!.id, ds2!.id, ds3!.id]).toContain(result.data[0].id);
|
||||
expect(result.count).toBe(3);
|
||||
});
|
||||
});
|
||||
});
|
||||
File diff suppressed because it is too large
Load Diff
228
packages/cli/src/modules/data-store/__tests__/sql-utils.test.ts
Normal file
228
packages/cli/src/modules/data-store/__tests__/sql-utils.test.ts
Normal file
@@ -0,0 +1,228 @@
|
||||
import type { DataStoreRows } from '@n8n/api-types';
|
||||
|
||||
import {
|
||||
addColumnQuery,
|
||||
deleteColumnQuery,
|
||||
buildInsertQuery,
|
||||
buildUpdateQuery,
|
||||
splitRowsByExistence,
|
||||
} from '../utils/sql-utils';
|
||||
|
||||
describe('sql-utils', () => {
|
||||
describe('addColumnQuery', () => {
|
||||
it('should generate a valid SQL query for adding columns to a table', () => {
|
||||
const tableName = 'data_store_user_abc';
|
||||
const column = { name: 'email', type: 'number' as const };
|
||||
|
||||
const query = addColumnQuery(tableName, column, 'sqlite');
|
||||
|
||||
expect(query).toBe('ALTER TABLE "data_store_user_abc" ADD "email" FLOAT');
|
||||
});
|
||||
});
|
||||
|
||||
describe('deleteColumnQuery', () => {
|
||||
it('should generate a valid SQL query for deleting columns from a table', () => {
|
||||
const tableName = 'data_store_user_abc';
|
||||
const column = 'email';
|
||||
|
||||
const query = deleteColumnQuery(tableName, column, 'sqlite');
|
||||
|
||||
expect(query).toBe('ALTER TABLE "data_store_user_abc" DROP COLUMN "email"');
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildInsertQuery', () => {
|
||||
it('should generate a valid SQL query for inserting rows into a table', () => {
|
||||
const tableName = 'data_store_user_abc';
|
||||
const columns = [
|
||||
{ name: 'name', type: 'string' },
|
||||
{ name: 'age', type: 'number' },
|
||||
];
|
||||
const rows = [
|
||||
{ name: 'Alice', age: 30 },
|
||||
{ name: 'Bob', age: 25 },
|
||||
];
|
||||
|
||||
const [query, parameters] = buildInsertQuery(tableName, rows, columns, 'postgres');
|
||||
|
||||
expect(query).toBe(
|
||||
'INSERT INTO "data_store_user_abc" ("name", "age") VALUES ($1, $2), ($3, $4)',
|
||||
);
|
||||
expect(parameters).toEqual(['Alice', 30, 'Bob', 25]);
|
||||
});
|
||||
|
||||
it('should return an empty query and parameters when rows are empty', () => {
|
||||
const tableName = 'data_store_user_abc';
|
||||
const rows: [] = [];
|
||||
|
||||
const [query, parameters] = buildInsertQuery(tableName, rows, []);
|
||||
|
||||
expect(query).toBe('');
|
||||
expect(parameters).toEqual([]);
|
||||
});
|
||||
|
||||
it('should return an empty query and parameters when rows have no keys', () => {
|
||||
const tableName = 'data_store_user_abc';
|
||||
const rows = [{}];
|
||||
|
||||
const [query, parameters] = buildInsertQuery(tableName, rows, []);
|
||||
|
||||
expect(query).toBe('');
|
||||
expect(parameters).toEqual([]);
|
||||
});
|
||||
|
||||
it('should replace T and Z for MySQL', () => {
|
||||
const tableName = 'data_store_user_abc';
|
||||
const columns = [{ name: 'participatedAt', type: 'date' }];
|
||||
const rows = [
|
||||
{ participatedAt: new Date('2021-01-01') },
|
||||
{ participatedAt: new Date('2021-01-02') },
|
||||
];
|
||||
|
||||
const [query, parameters] = buildInsertQuery(tableName, rows, columns, 'mysql');
|
||||
|
||||
expect(query).toBe('INSERT INTO `data_store_user_abc` (`participatedAt`) VALUES (?), (?)');
|
||||
expect(parameters).toEqual(['2021-01-01 00:00:00.000', '2021-01-02 00:00:00.000']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildUpdateQuery', () => {
|
||||
it('should generate a valid SQL update query with one match field', () => {
|
||||
const tableName = 'data_store_user_abc';
|
||||
const row = { name: 'Alice', age: 30, city: 'Paris' };
|
||||
const columns = [
|
||||
{ id: 1, name: 'name', type: 'string' },
|
||||
{ id: 2, name: 'age', type: 'number' },
|
||||
{ id: 3, name: 'city', type: 'string' },
|
||||
];
|
||||
const matchFields = ['name'];
|
||||
|
||||
const [query, parameters] = buildUpdateQuery(tableName, row, columns, matchFields);
|
||||
|
||||
expect(query).toBe('UPDATE "data_store_user_abc" SET "age" = ?, "city" = ? WHERE "name" = ?');
|
||||
expect(parameters).toEqual([30, 'Paris', 'Alice']);
|
||||
});
|
||||
|
||||
it('should generate a valid SQL update query with multiple match fields', () => {
|
||||
const tableName = 'data_store_user_abc';
|
||||
const row = { name: 'Alice', age: 30, city: 'Paris' };
|
||||
const columns = [
|
||||
{ id: 1, name: 'name', type: 'string' },
|
||||
{ id: 2, name: 'age', type: 'number' },
|
||||
{ id: 3, name: 'city', type: 'string' },
|
||||
];
|
||||
const matchFields = ['name', 'city'];
|
||||
|
||||
const [query, parameters] = buildUpdateQuery(tableName, row, columns, matchFields);
|
||||
|
||||
expect(query).toBe(
|
||||
'UPDATE "data_store_user_abc" SET "age" = ? WHERE "name" = ? AND "city" = ?',
|
||||
);
|
||||
expect(parameters).toEqual([30, 'Alice', 'Paris']);
|
||||
});
|
||||
|
||||
it('should return empty query and parameters if row is empty', () => {
|
||||
const tableName = 'data_store_user_abc';
|
||||
const row = {};
|
||||
const matchFields = ['id'];
|
||||
|
||||
const [query, parameters] = buildUpdateQuery(tableName, row, [], matchFields);
|
||||
|
||||
expect(query).toBe('');
|
||||
expect(parameters).toEqual([]);
|
||||
});
|
||||
|
||||
it('should return empty query and parameters if matchFields is empty', () => {
|
||||
const tableName = 'data_store_user_abc';
|
||||
const row = { name: 'Alice', age: 30 };
|
||||
const columns = [
|
||||
{ id: 1, name: 'name', type: 'string' },
|
||||
{ id: 2, name: 'age', type: 'number' },
|
||||
];
|
||||
const matchFields: string[] = [];
|
||||
|
||||
const [query, parameters] = buildUpdateQuery(tableName, row, columns, matchFields);
|
||||
|
||||
expect(query).toBe('');
|
||||
expect(parameters).toEqual([]);
|
||||
});
|
||||
|
||||
it('should return empty query and parameters if only matchFields are present in row', () => {
|
||||
const tableName = 'data_store_user_abc';
|
||||
const row = { id: 1 };
|
||||
const matchFields = ['id'];
|
||||
|
||||
const [query, parameters] = buildUpdateQuery(tableName, row, [], matchFields);
|
||||
|
||||
expect(query).toBe('');
|
||||
expect(parameters).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('splitRowsByExistence', () => {
|
||||
it('should correctly separate rows into insert and update based on matchFields', () => {
|
||||
const existing = [
|
||||
{ name: 'Alice', age: 30 },
|
||||
{ name: 'Bob', age: 25 },
|
||||
];
|
||||
const matchFields = ['name'];
|
||||
const rows: DataStoreRows = [
|
||||
{ name: 'Alice', age: 30 },
|
||||
{ name: 'Bob', age: 26 },
|
||||
{ name: 'Charlie', age: 35 },
|
||||
];
|
||||
|
||||
const { rowsToInsert, rowsToUpdate } = splitRowsByExistence(existing, matchFields, rows);
|
||||
|
||||
expect(rowsToUpdate).toEqual([
|
||||
{ name: 'Alice', age: 30 },
|
||||
{ name: 'Bob', age: 26 },
|
||||
]);
|
||||
expect(rowsToInsert).toEqual([{ name: 'Charlie', age: 35 }]);
|
||||
});
|
||||
|
||||
it('should treat rows as new if matchFields combination does not exist', () => {
|
||||
const existing = [{ name: 'Bob', city: 'Berlin' }];
|
||||
const matchFields = ['name', 'city'];
|
||||
const rows: DataStoreRows = [{ name: 'Alice', city: 'Berlin' }];
|
||||
|
||||
const { rowsToInsert, rowsToUpdate } = splitRowsByExistence(existing, matchFields, rows);
|
||||
|
||||
expect(rowsToUpdate).toEqual([]);
|
||||
expect(rowsToInsert).toEqual([{ name: 'Alice', city: 'Berlin' }]);
|
||||
});
|
||||
|
||||
it('should insert all rows if existing set is empty', () => {
|
||||
const existing: Array<Record<string, unknown>> = [];
|
||||
const matchFields = ['name'];
|
||||
const rows: DataStoreRows = [{ name: 'Alice' }, { name: 'Bob' }];
|
||||
|
||||
const { rowsToInsert, rowsToUpdate } = splitRowsByExistence(existing, matchFields, rows);
|
||||
|
||||
expect(rowsToUpdate).toEqual([]);
|
||||
expect(rowsToInsert).toEqual(rows);
|
||||
});
|
||||
|
||||
it('should update all rows if all keys match existing', () => {
|
||||
const existing = [{ name: 'Alice' }, { name: 'Bob' }];
|
||||
const matchFields = ['name'];
|
||||
const rows: DataStoreRows = [{ name: 'Alice' }, { name: 'Bob' }];
|
||||
|
||||
const { rowsToInsert, rowsToUpdate } = splitRowsByExistence(existing, matchFields, rows);
|
||||
|
||||
expect(rowsToInsert).toEqual([]);
|
||||
expect(rowsToUpdate).toEqual(rows);
|
||||
});
|
||||
|
||||
it('should handle empty input rows', () => {
|
||||
const existing = [{ name: 'Alice' }];
|
||||
const matchFields = ['name'];
|
||||
const rows: DataStoreRows = [];
|
||||
|
||||
const { rowsToInsert, rowsToUpdate } = splitRowsByExistence(existing, matchFields, rows);
|
||||
|
||||
expect(rowsToInsert).toEqual([]);
|
||||
expect(rowsToUpdate).toEqual([]);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,20 @@
|
||||
import { ListDataStoreQueryDto } from '@n8n/api-types';
|
||||
import { AuthenticatedRequest } from '@n8n/db';
|
||||
import { Get, ProjectScope, Query, RestController } from '@n8n/decorators';
|
||||
|
||||
import { DataStoreAggregateService } from './data-store-aggregate.service';
|
||||
|
||||
@RestController('/data-stores-global')
|
||||
export class DataStoreAggregateController {
|
||||
constructor(private readonly dataStoreAggregateService: DataStoreAggregateService) {}
|
||||
|
||||
@Get('/')
|
||||
@ProjectScope('dataStore:list')
|
||||
async listDataStores(
|
||||
req: AuthenticatedRequest,
|
||||
_res: Response,
|
||||
@Query payload: ListDataStoreQueryDto,
|
||||
) {
|
||||
return await this.dataStoreAggregateService.getManyAndCount(req.user, payload);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
import type { ListDataStoreQueryDto } from '@n8n/api-types';
|
||||
import { Logger } from '@n8n/backend-common';
|
||||
import { User } from '@n8n/db';
|
||||
import { Service } from '@n8n/di';
|
||||
|
||||
import { ProjectService } from '@/services/project.service.ee';
|
||||
|
||||
import { DataStoreRepository } from './data-store.repository';
|
||||
|
||||
@Service()
|
||||
export class DataStoreAggregateService {
|
||||
constructor(
|
||||
private readonly dataStoreRepository: DataStoreRepository,
|
||||
private readonly projectService: ProjectService,
|
||||
private readonly logger: Logger,
|
||||
) {
|
||||
this.logger = this.logger.scoped('data-store');
|
||||
}
|
||||
async start() {}
|
||||
async shutdown() {}
|
||||
|
||||
async getManyAndCount(user: User, options: ListDataStoreQueryDto) {
|
||||
const projects = await this.projectService.getProjectRelationsForUser(user);
|
||||
let projectIds = projects.map((x) => x.projectId);
|
||||
if (options.filter?.projectId) {
|
||||
const mask = [options.filter?.projectId].flat();
|
||||
projectIds = projectIds.filter((x) => mask.includes(x));
|
||||
}
|
||||
|
||||
if (projectIds.length === 0) {
|
||||
return { count: 0, data: [] };
|
||||
}
|
||||
|
||||
return await this.dataStoreRepository.getManyAndCount({
|
||||
...options,
|
||||
filter: {
|
||||
...options.filter,
|
||||
projectId: projectIds,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
import { WithTimestampsAndStringId } from '@n8n/db';
|
||||
import { Column, Entity, Index, JoinColumn, ManyToOne } from '@n8n/typeorm';
|
||||
|
||||
import { type DataStore } from './data-store.entity';
|
||||
|
||||
@Entity()
|
||||
@Index(['dataStoreId', 'name'], { unique: true })
|
||||
export class DataStoreColumn extends WithTimestampsAndStringId {
|
||||
@Column()
|
||||
dataStoreId: string;
|
||||
|
||||
@Column()
|
||||
name: string;
|
||||
|
||||
@Column({ type: 'varchar' })
|
||||
type: 'string' | 'number' | 'boolean' | 'date';
|
||||
|
||||
@Column({ type: 'int' })
|
||||
index: number;
|
||||
|
||||
@ManyToOne('DataStore', 'columns')
|
||||
@JoinColumn({ name: 'dataStoreId' })
|
||||
dataStore: DataStore;
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
import { DataStoreCreateColumnSchema } from '@n8n/api-types';
|
||||
import { Service } from '@n8n/di';
|
||||
import { DataSource, EntityManager, Repository } from '@n8n/typeorm';
|
||||
import { UserError } from 'n8n-workflow';
|
||||
|
||||
import { DataStoreColumn } from './data-store-column.entity';
|
||||
import { DataStoreRowsRepository } from './data-store-rows.repository';
|
||||
|
||||
@Service()
|
||||
export class DataStoreColumnRepository extends Repository<DataStoreColumn> {
|
||||
constructor(
|
||||
dataSource: DataSource,
|
||||
private dataStoreRowsRepository: DataStoreRowsRepository,
|
||||
) {
|
||||
super(DataStoreColumn, dataSource.manager);
|
||||
}
|
||||
|
||||
async getColumns(rawDataStoreId: string, em?: EntityManager) {
|
||||
const executor = em ?? this.manager;
|
||||
const columns = await executor
|
||||
.createQueryBuilder(DataStoreColumn, 'dsc')
|
||||
.where('dsc.dataStoreId = :dataStoreId', { dataStoreId: rawDataStoreId })
|
||||
.getMany();
|
||||
|
||||
// Ensure columns are always returned in the correct order by index,
|
||||
// since the database does not guarantee ordering and TypeORM does not preserve
|
||||
// join order in @OneToMany relations.
|
||||
columns.sort((a, b) => a.index - b.index);
|
||||
|
||||
return columns;
|
||||
}
|
||||
|
||||
async addColumn(dataStoreId: string, schema: DataStoreCreateColumnSchema) {
|
||||
return await this.manager.transaction(async (em) => {
|
||||
const existingColumnMatch = await em.existsBy(DataStoreColumn, {
|
||||
name: schema.name,
|
||||
dataStoreId,
|
||||
});
|
||||
|
||||
if (existingColumnMatch) {
|
||||
throw new UserError(
|
||||
`column name '${schema.name}' already taken in data store '${dataStoreId}'`,
|
||||
);
|
||||
}
|
||||
|
||||
if (schema.index === undefined) {
|
||||
const columns = await this.getColumns(dataStoreId, em);
|
||||
schema.index = columns.length;
|
||||
} else {
|
||||
await this.shiftColumns(dataStoreId, schema.index, 1, em);
|
||||
}
|
||||
|
||||
const column = em.create(DataStoreColumn, {
|
||||
...schema,
|
||||
dataStoreId,
|
||||
});
|
||||
|
||||
await em.insert(DataStoreColumn, column);
|
||||
|
||||
const queryRunner = em.queryRunner;
|
||||
if (!queryRunner) {
|
||||
throw new Error('QueryRunner is not available');
|
||||
}
|
||||
|
||||
await this.dataStoreRowsRepository.ensureTableAndAddColumn(
|
||||
dataStoreId,
|
||||
column,
|
||||
queryRunner,
|
||||
em.connection.options.type,
|
||||
);
|
||||
|
||||
return column;
|
||||
});
|
||||
}
|
||||
|
||||
async deleteColumn(dataStoreId: string, column: DataStoreColumn) {
|
||||
await this.manager.transaction(async (em) => {
|
||||
await em.remove(DataStoreColumn, column);
|
||||
await this.dataStoreRowsRepository.dropColumnFromTable(
|
||||
dataStoreId,
|
||||
column.name,
|
||||
em,
|
||||
em.connection.options.type,
|
||||
);
|
||||
await this.shiftColumns(dataStoreId, column.index, -1, em);
|
||||
});
|
||||
}
|
||||
|
||||
async moveColumn(rawDataStoreId: string, columnId: string, targetIndex: number) {
|
||||
await this.manager.transaction(async (em) => {
|
||||
const columnCount = await em.countBy(DataStoreColumn, { dataStoreId: rawDataStoreId });
|
||||
|
||||
if (targetIndex >= columnCount) {
|
||||
throw new UserError('tried to move column to index larger than column count');
|
||||
}
|
||||
|
||||
if (targetIndex < 0) {
|
||||
throw new UserError('tried to move column to negative index');
|
||||
}
|
||||
|
||||
const existingColumn = await em.findOneBy(DataStoreColumn, {
|
||||
id: columnId,
|
||||
dataStoreId: rawDataStoreId,
|
||||
});
|
||||
|
||||
if (existingColumn === null) {
|
||||
throw new UserError(`tried to move column not present in data store '${rawDataStoreId}'`);
|
||||
}
|
||||
|
||||
await this.shiftColumns(rawDataStoreId, existingColumn.index, -1, em);
|
||||
await this.shiftColumns(rawDataStoreId, targetIndex, 1, em);
|
||||
await em.update(DataStoreColumn, { id: existingColumn.id }, { index: targetIndex });
|
||||
});
|
||||
}
|
||||
|
||||
async shiftColumns(
|
||||
rawDataStoreId: string,
|
||||
lowestIndex: number,
|
||||
delta: -1 | 1,
|
||||
em?: EntityManager,
|
||||
) {
|
||||
const executor = em ?? this.manager;
|
||||
await executor
|
||||
.createQueryBuilder()
|
||||
.update(DataStoreColumn)
|
||||
.set({
|
||||
index: () => `index + ${delta}`,
|
||||
})
|
||||
.where('dataStoreId = :dataStoreId AND index >= :thresholdValue', {
|
||||
dataStoreId: rawDataStoreId,
|
||||
thresholdValue: lowestIndex,
|
||||
})
|
||||
.execute();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,238 @@
|
||||
import type {
|
||||
ListDataStoreContentQueryDto,
|
||||
ListDataStoreContentFilter,
|
||||
DataStoreUserTableName,
|
||||
DataStoreRows,
|
||||
UpsertDataStoreRowsDto,
|
||||
} from '@n8n/api-types';
|
||||
import { CreateTable, DslColumn } from '@n8n/db';
|
||||
import { Service } from '@n8n/di';
|
||||
import {
|
||||
DataSource,
|
||||
DataSourceOptions,
|
||||
EntityManager,
|
||||
QueryRunner,
|
||||
SelectQueryBuilder,
|
||||
} from '@n8n/typeorm';
|
||||
|
||||
import { DataStoreColumn } from './data-store-column.entity';
|
||||
import {
|
||||
addColumnQuery,
|
||||
buildInsertQuery,
|
||||
buildUpdateQuery,
|
||||
deleteColumnQuery,
|
||||
getPlaceholder,
|
||||
quoteIdentifier,
|
||||
splitRowsByExistence,
|
||||
toDslColumns,
|
||||
toTableName,
|
||||
} from './utils/sql-utils';
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
type QueryBuilder = SelectQueryBuilder<any>;
|
||||
|
||||
function getConditionAndParams(
|
||||
filter: ListDataStoreContentFilter['filters'][number],
|
||||
index: number,
|
||||
dbType: DataSourceOptions['type'],
|
||||
): [string, Record<string, unknown>] {
|
||||
const paramName = `filter_${index}`;
|
||||
const column = `dataStore.${quoteIdentifier(filter.columnName, dbType)}`;
|
||||
|
||||
switch (filter.condition) {
|
||||
case 'eq':
|
||||
return [`${column} = :${paramName}`, { [paramName]: filter.value }];
|
||||
case 'neq':
|
||||
return [`${column} != :${paramName}`, { [paramName]: filter.value }];
|
||||
}
|
||||
}
|
||||
|
||||
@Service()
|
||||
export class DataStoreRowsRepository {
|
||||
constructor(private dataSource: DataSource) {}
|
||||
|
||||
// TypeORM cannot infer the columns for a dynamic table name, so we use a raw query
|
||||
async insertRows(
|
||||
tableName: DataStoreUserTableName,
|
||||
rows: DataStoreRows,
|
||||
columns: DataStoreColumn[],
|
||||
) {
|
||||
const dbType = this.dataSource.options.type;
|
||||
await this.dataSource.query.apply(
|
||||
this.dataSource,
|
||||
buildInsertQuery(tableName, rows, columns, dbType),
|
||||
);
|
||||
return true;
|
||||
}
|
||||
|
||||
async upsertRows(
|
||||
tableName: DataStoreUserTableName,
|
||||
dto: UpsertDataStoreRowsDto,
|
||||
columns: DataStoreColumn[],
|
||||
) {
|
||||
const dbType = this.dataSource.options.type;
|
||||
const { rows, matchFields } = dto;
|
||||
|
||||
if (rows.length === 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const { rowsToInsert, rowsToUpdate } = await this.fetchAndSplitRowsByExistence(
|
||||
tableName,
|
||||
matchFields,
|
||||
rows,
|
||||
);
|
||||
|
||||
if (rowsToInsert.length > 0) {
|
||||
await this.insertRows(tableName, rowsToInsert, columns);
|
||||
}
|
||||
|
||||
if (rowsToUpdate.length > 0) {
|
||||
for (const row of rowsToUpdate) {
|
||||
// TypeORM cannot infer the columns for a dynamic table name, so we use a raw query
|
||||
const [query, parameters] = buildUpdateQuery(tableName, row, columns, matchFields, dbType);
|
||||
await this.dataSource.query(query, parameters);
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
async createTableWithColumns(
|
||||
tableName: string,
|
||||
columns: DataStoreColumn[],
|
||||
queryRunner: QueryRunner,
|
||||
) {
|
||||
const dslColumns = [new DslColumn('id').int.autoGenerate2.primary, ...toDslColumns(columns)];
|
||||
const createTable = new CreateTable(tableName, '', queryRunner);
|
||||
createTable.withColumns.apply(createTable, dslColumns);
|
||||
await createTable.execute(queryRunner);
|
||||
}
|
||||
|
||||
async ensureTableAndAddColumn(
|
||||
dataStoreId: string,
|
||||
column: DataStoreColumn,
|
||||
queryRunner: QueryRunner,
|
||||
dbType: DataSourceOptions['type'],
|
||||
) {
|
||||
const tableName = toTableName(dataStoreId);
|
||||
const tableExists = await queryRunner.hasTable(tableName);
|
||||
if (!tableExists) {
|
||||
await this.createTableWithColumns(tableName, [column], queryRunner);
|
||||
} else {
|
||||
await queryRunner.manager.query(addColumnQuery(tableName, column, dbType));
|
||||
}
|
||||
}
|
||||
|
||||
async dropColumnFromTable(
|
||||
dataStoreId: string,
|
||||
columnName: string,
|
||||
em: EntityManager,
|
||||
dbType: DataSourceOptions['type'],
|
||||
) {
|
||||
await em.query(deleteColumnQuery(toTableName(dataStoreId), columnName, dbType));
|
||||
}
|
||||
|
||||
async getManyAndCount(dataStoreId: DataStoreUserTableName, dto: ListDataStoreContentQueryDto) {
|
||||
const [countQuery, query] = this.getManyQuery(dataStoreId, dto);
|
||||
const data: Array<Record<string, unknown>> = await query.select('*').getRawMany();
|
||||
const countResult = await countQuery.select('COUNT(*) as count').getRawOne<{
|
||||
count: number | string | null;
|
||||
}>();
|
||||
const count =
|
||||
typeof countResult?.count === 'number' ? countResult.count : Number(countResult?.count) || 0;
|
||||
return { count: count ?? -1, data };
|
||||
}
|
||||
|
||||
async getRowIds(dataStoreId: DataStoreUserTableName, dto: ListDataStoreContentQueryDto) {
|
||||
const [_, query] = this.getManyQuery(dataStoreId, dto);
|
||||
const result = await query.select('dataStore.id').getRawMany<number>();
|
||||
return result;
|
||||
}
|
||||
|
||||
private getManyQuery(
|
||||
dataStoreTableName: DataStoreUserTableName,
|
||||
dto: ListDataStoreContentQueryDto,
|
||||
): [QueryBuilder, QueryBuilder] {
|
||||
const query = this.dataSource.createQueryBuilder();
|
||||
|
||||
query.from(dataStoreTableName, 'dataStore');
|
||||
this.applyFilters(query, dto);
|
||||
const countQuery = query.clone().select('COUNT(*)');
|
||||
this.applySorting(query, dto);
|
||||
this.applyPagination(query, dto);
|
||||
|
||||
return [countQuery, query];
|
||||
}
|
||||
|
||||
private applyFilters(query: QueryBuilder, dto: ListDataStoreContentQueryDto): void {
|
||||
const filters = dto.filter?.filters ?? [];
|
||||
const filterType = dto.filter?.type ?? 'and';
|
||||
|
||||
const dbType = this.dataSource.options.type;
|
||||
const conditionsAndParams = filters.map((filter, i) =>
|
||||
getConditionAndParams(filter, i, dbType),
|
||||
);
|
||||
|
||||
for (const [condition, params] of conditionsAndParams) {
|
||||
if (filterType === 'or') {
|
||||
query.orWhere(condition, params);
|
||||
} else {
|
||||
query.andWhere(condition, params);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private applySorting(query: QueryBuilder, dto: ListDataStoreContentQueryDto): void {
|
||||
if (!dto.sortBy) {
|
||||
// query.orderBy('dataStore.', 'DESC');
|
||||
return;
|
||||
}
|
||||
|
||||
const [field, order] = dto.sortBy;
|
||||
this.applySortingByField(query, field, order);
|
||||
}
|
||||
|
||||
private applySortingByField(query: QueryBuilder, field: string, direction: 'DESC' | 'ASC'): void {
|
||||
const dbType = this.dataSource.options.type;
|
||||
const quotedField = `dataStore.${quoteIdentifier(field, dbType)}`;
|
||||
query.orderBy(quotedField, direction);
|
||||
}
|
||||
|
||||
private applyPagination(query: QueryBuilder, dto: ListDataStoreContentQueryDto): void {
|
||||
query.skip(dto.skip);
|
||||
query.take(dto.take);
|
||||
}
|
||||
|
||||
private async fetchAndSplitRowsByExistence(
|
||||
tableName: string,
|
||||
matchFields: string[],
|
||||
rows: DataStoreRows,
|
||||
): Promise<{ rowsToInsert: DataStoreRows; rowsToUpdate: DataStoreRows }> {
|
||||
const dbType = this.dataSource.options.type;
|
||||
const whereClauses: string[] = [];
|
||||
const params: unknown[] = [];
|
||||
|
||||
for (const row of rows) {
|
||||
const clause = matchFields
|
||||
.map((field) => {
|
||||
params.push(row[field]);
|
||||
return `${quoteIdentifier(field, dbType)} = ${getPlaceholder(params.length, dbType)}`;
|
||||
})
|
||||
.join(' AND ');
|
||||
whereClauses.push(`(${clause})`);
|
||||
}
|
||||
|
||||
const quotedFields = matchFields.map((field) => quoteIdentifier(field, dbType)).join(', ');
|
||||
const quotedTableName = quoteIdentifier(tableName, dbType);
|
||||
|
||||
const query = `
|
||||
SELECT ${quotedFields}
|
||||
FROM ${quotedTableName}
|
||||
WHERE ${whereClauses.join(' OR ')}
|
||||
`;
|
||||
const existing: Array<Record<string, unknown>> = await this.dataSource.query(query, params);
|
||||
|
||||
return splitRowsByExistence(existing, matchFields, rows);
|
||||
}
|
||||
}
|
||||
151
packages/cli/src/modules/data-store/data-store.controller.ts
Normal file
151
packages/cli/src/modules/data-store/data-store.controller.ts
Normal file
@@ -0,0 +1,151 @@
|
||||
import {
|
||||
AddDataStoreRowsDto,
|
||||
AddDataStoreColumnDto,
|
||||
CreateDataStoreDto,
|
||||
ListDataStoreContentQueryDto,
|
||||
ListDataStoreQueryDto,
|
||||
MoveDataStoreColumnDto,
|
||||
UpdateDataStoreDto,
|
||||
UpsertDataStoreRowsDto,
|
||||
} from '@n8n/api-types';
|
||||
import { AuthenticatedRequest } from '@n8n/db';
|
||||
import {
|
||||
Body,
|
||||
Delete,
|
||||
Get,
|
||||
Param,
|
||||
Patch,
|
||||
Post,
|
||||
ProjectScope,
|
||||
Query,
|
||||
RestController,
|
||||
} from '@n8n/decorators';
|
||||
|
||||
import { DataStoreService } from './data-store.service';
|
||||
|
||||
@RestController('/projects/:projectId/data-stores')
|
||||
export class DataStoreController {
|
||||
constructor(private readonly dataStoreService: DataStoreService) {}
|
||||
|
||||
@Post('/')
|
||||
@ProjectScope('dataStore:create')
|
||||
async createDataStore(
|
||||
req: AuthenticatedRequest<{ projectId: string }>,
|
||||
_res: Response,
|
||||
@Body dto: CreateDataStoreDto,
|
||||
) {
|
||||
return await this.dataStoreService.createDataStore(req.params.projectId, dto);
|
||||
}
|
||||
|
||||
@Get('/')
|
||||
@ProjectScope('dataStore:list')
|
||||
async listDataStores(
|
||||
req: AuthenticatedRequest<{ projectId: string }>,
|
||||
_res: Response,
|
||||
@Query payload: ListDataStoreQueryDto,
|
||||
) {
|
||||
const providedFilter = payload?.filter ?? {};
|
||||
return await this.dataStoreService.getManyAndCount({
|
||||
...payload,
|
||||
filter: { ...providedFilter, projectId: req.params.projectId },
|
||||
});
|
||||
}
|
||||
|
||||
@Patch('/:dataStoreId')
|
||||
@ProjectScope('dataStore:update')
|
||||
async updateDataStore(
|
||||
_req: AuthenticatedRequest<{ projectId: string }>,
|
||||
_res: Response,
|
||||
@Param('dataStoreId') dataStoreId: string,
|
||||
@Body dto: UpdateDataStoreDto,
|
||||
) {
|
||||
return await this.dataStoreService.updateDataStore(dataStoreId, dto);
|
||||
}
|
||||
|
||||
@Delete('/:dataStoreId')
|
||||
@ProjectScope('dataStore:delete')
|
||||
async deleteDataStore(
|
||||
_req: AuthenticatedRequest<{ projectId: string }>,
|
||||
_res: Response,
|
||||
@Param('dataStoreId') dataStoreId: string,
|
||||
) {
|
||||
return await this.dataStoreService.deleteDataStore(dataStoreId);
|
||||
}
|
||||
|
||||
@Get('/:dataStoreId/columns')
|
||||
@ProjectScope('dataStore:read')
|
||||
async getColumns(
|
||||
_req: AuthenticatedRequest<{ projectId: string }>,
|
||||
_res: Response,
|
||||
@Param('dataStoreId') dataStoreId: string,
|
||||
) {
|
||||
return await this.dataStoreService.getColumns(dataStoreId);
|
||||
}
|
||||
|
||||
@Post('/:dataStoreId/columns')
|
||||
@ProjectScope('dataStore:update')
|
||||
async addColumn(
|
||||
_req: AuthenticatedRequest<{ projectId: string }>,
|
||||
_res: Response,
|
||||
@Param('dataStoreId') dataStoreId: string,
|
||||
@Body dto: AddDataStoreColumnDto,
|
||||
) {
|
||||
return await this.dataStoreService.addColumn(dataStoreId, dto);
|
||||
}
|
||||
|
||||
@Delete('/:dataStoreId/columns/:columnId')
|
||||
@ProjectScope('dataStore:update')
|
||||
async deleteColumn(
|
||||
_req: AuthenticatedRequest<{ projectId: string }>,
|
||||
_res: Response,
|
||||
@Param('dataStoreId') dataStoreId: string,
|
||||
@Param('columnId') columnId: string,
|
||||
) {
|
||||
return await this.dataStoreService.deleteColumn(dataStoreId, columnId);
|
||||
}
|
||||
|
||||
@Patch('/:dataStoreId/columns/:columnId/move')
|
||||
@ProjectScope('dataStore:update')
|
||||
async moveColumn(
|
||||
_req: AuthenticatedRequest<{ projectId: string }>,
|
||||
_res: Response,
|
||||
@Param('dataStoreId') dataStoreId: string,
|
||||
@Param('columnId') columnId: string,
|
||||
@Body dto: MoveDataStoreColumnDto,
|
||||
) {
|
||||
return await this.dataStoreService.moveColumn(dataStoreId, columnId, dto);
|
||||
}
|
||||
|
||||
@Get('/:dataStoreId/rows')
|
||||
@ProjectScope('dataStore:readRow')
|
||||
async getDataStoreRows(
|
||||
_req: AuthenticatedRequest<{ projectId: string }>,
|
||||
_res: Response,
|
||||
@Param('dataStoreId') dataStoreId: string,
|
||||
@Query dto: ListDataStoreContentQueryDto,
|
||||
) {
|
||||
return await this.dataStoreService.getManyRowsAndCount(dataStoreId, dto);
|
||||
}
|
||||
|
||||
@Post('/:dataStoreId/insert')
|
||||
@ProjectScope('dataStore:writeRow')
|
||||
async appendDataStoreRows(
|
||||
_req: AuthenticatedRequest<{ projectId: string }>,
|
||||
_res: Response,
|
||||
@Param('dataStoreId') dataStoreId: string,
|
||||
@Body dto: AddDataStoreRowsDto,
|
||||
) {
|
||||
return await this.dataStoreService.insertRows(dataStoreId, dto.data);
|
||||
}
|
||||
|
||||
@Post('/:dataStoreId/upsert')
|
||||
@ProjectScope('dataStore:writeRow')
|
||||
async upsertDataStoreRows(
|
||||
_req: AuthenticatedRequest<{ projectId: string }>,
|
||||
_res: Response,
|
||||
@Param('dataStoreId') dataStoreId: string,
|
||||
@Body dto: UpsertDataStoreRowsDto,
|
||||
) {
|
||||
return await this.dataStoreService.upsertRows(dataStoreId, dto);
|
||||
}
|
||||
}
|
||||
34
packages/cli/src/modules/data-store/data-store.entity.ts
Normal file
34
packages/cli/src/modules/data-store/data-store.entity.ts
Normal file
@@ -0,0 +1,34 @@
|
||||
import { Project, WithTimestampsAndStringId } from '@n8n/db';
|
||||
import { Column, Entity, Index, JoinColumn, ManyToOne, OneToMany } from '@n8n/typeorm';
|
||||
|
||||
import { DataStoreColumn } from './data-store-column.entity';
|
||||
|
||||
@Entity()
|
||||
@Index(['name', 'projectId'], { unique: true })
|
||||
export class DataStore extends WithTimestampsAndStringId {
|
||||
constructor() {
|
||||
super();
|
||||
}
|
||||
|
||||
@Column()
|
||||
name: string;
|
||||
|
||||
@OneToMany(
|
||||
() => DataStoreColumn,
|
||||
(dataStoreColumn) => dataStoreColumn.dataStore,
|
||||
{
|
||||
cascade: true,
|
||||
},
|
||||
)
|
||||
columns: DataStoreColumn[];
|
||||
|
||||
@ManyToOne(() => Project)
|
||||
@JoinColumn({ name: 'projectId' })
|
||||
project: Project;
|
||||
|
||||
@Column()
|
||||
projectId: string;
|
||||
|
||||
@Column({ type: 'int', default: 0 })
|
||||
sizeBytes: number;
|
||||
}
|
||||
34
packages/cli/src/modules/data-store/data-store.module.ts
Normal file
34
packages/cli/src/modules/data-store/data-store.module.ts
Normal file
@@ -0,0 +1,34 @@
|
||||
import type { ModuleInterface } from '@n8n/decorators';
|
||||
import { BackendModule, OnShutdown } from '@n8n/decorators';
|
||||
import { Container } from '@n8n/di';
|
||||
import { BaseEntity } from '@n8n/typeorm';
|
||||
|
||||
@BackendModule({ name: 'data-store' })
|
||||
export class DataStoreModule implements ModuleInterface {
|
||||
async init() {
|
||||
await import('./data-store.controller');
|
||||
await import('./data-store-aggregate.controller');
|
||||
|
||||
const { DataStoreService } = await import('./data-store.service');
|
||||
await Container.get(DataStoreService).start();
|
||||
|
||||
const { DataStoreAggregateService } = await import('./data-store-aggregate.service');
|
||||
await Container.get(DataStoreAggregateService).start();
|
||||
}
|
||||
|
||||
@OnShutdown()
|
||||
async shutdown() {
|
||||
const { DataStoreService } = await import('./data-store.service');
|
||||
await Container.get(DataStoreService).shutdown();
|
||||
|
||||
const { DataStoreAggregateService } = await import('./data-store-aggregate.service');
|
||||
await Container.get(DataStoreAggregateService).start();
|
||||
}
|
||||
|
||||
async entities() {
|
||||
const { DataStore } = await import('./data-store.entity');
|
||||
const { DataStoreColumn } = await import('./data-store-column.entity');
|
||||
|
||||
return [DataStore, DataStoreColumn] as unknown as Array<new () => BaseEntity>;
|
||||
}
|
||||
}
|
||||
235
packages/cli/src/modules/data-store/data-store.repository.ts
Normal file
235
packages/cli/src/modules/data-store/data-store.repository.ts
Normal file
@@ -0,0 +1,235 @@
|
||||
import {
|
||||
DATA_STORE_COLUMN_REGEX,
|
||||
type DataStoreCreateColumnSchema,
|
||||
type ListDataStoreQueryDto,
|
||||
} from '@n8n/api-types';
|
||||
import { Service } from '@n8n/di';
|
||||
import { DataSource, EntityManager, Repository, SelectQueryBuilder } from '@n8n/typeorm';
|
||||
import { UnexpectedError } from 'n8n-workflow';
|
||||
|
||||
import { DataStoreColumn } from './data-store-column.entity';
|
||||
import { DataStoreRowsRepository } from './data-store-rows.repository';
|
||||
import { DataStore } from './data-store.entity';
|
||||
import { toTableName } from './utils/sql-utils';
|
||||
|
||||
@Service()
|
||||
export class DataStoreRepository extends Repository<DataStore> {
|
||||
constructor(
|
||||
dataSource: DataSource,
|
||||
private dataStoreRowsRepository: DataStoreRowsRepository,
|
||||
) {
|
||||
super(DataStore, dataSource.manager);
|
||||
}
|
||||
|
||||
async createDataStore(projectId: string, name: string, columns: DataStoreCreateColumnSchema[]) {
|
||||
if (columns.some((c) => !DATA_STORE_COLUMN_REGEX.test(c.name))) {
|
||||
throw new UnexpectedError('bad column name');
|
||||
}
|
||||
|
||||
let dataStoreId: string | undefined;
|
||||
await this.manager.transaction(async (em) => {
|
||||
const dataStore = em.create(DataStore, { name, columns, projectId });
|
||||
await em.insert(DataStore, dataStore);
|
||||
dataStoreId = dataStore.id;
|
||||
|
||||
const tableName = toTableName(dataStore.id);
|
||||
const queryRunner = em.queryRunner;
|
||||
if (!queryRunner) {
|
||||
throw new UnexpectedError('QueryRunner is not available');
|
||||
}
|
||||
|
||||
if (columns.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
// insert columns
|
||||
const columnEntities = columns.map((col, index) =>
|
||||
em.create(DataStoreColumn, {
|
||||
name: col.name,
|
||||
type: col.type,
|
||||
dataStoreId: dataStore.id,
|
||||
index: col.index ?? index,
|
||||
}),
|
||||
);
|
||||
await em.insert(DataStoreColumn, columnEntities);
|
||||
|
||||
// create user table
|
||||
await this.dataStoreRowsRepository.createTableWithColumns(
|
||||
tableName,
|
||||
columnEntities,
|
||||
queryRunner,
|
||||
);
|
||||
});
|
||||
|
||||
if (!dataStoreId) {
|
||||
throw new UnexpectedError('Data store creation failed');
|
||||
}
|
||||
|
||||
return await this.findOne({
|
||||
where: { id: dataStoreId },
|
||||
relations: ['project', 'columns'],
|
||||
});
|
||||
}
|
||||
|
||||
async deleteDataStore(dataStoreId: string, entityManager?: EntityManager) {
|
||||
const executor = entityManager ?? this.manager;
|
||||
return await executor.transaction(async (em) => {
|
||||
const queryRunner = em.queryRunner;
|
||||
if (!queryRunner) {
|
||||
throw new UnexpectedError('QueryRunner is not available');
|
||||
}
|
||||
|
||||
await em.delete(DataStore, { id: dataStoreId });
|
||||
await queryRunner.dropTable(toTableName(dataStoreId), true);
|
||||
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
async deleteDataStoreByProjectId(projectId: string) {
|
||||
return await this.manager.transaction(async (em) => {
|
||||
const existingTables = await em.findBy(DataStore, { projectId });
|
||||
|
||||
let changed = false;
|
||||
for (const match of existingTables) {
|
||||
const result = await this.deleteDataStore(match.id, em);
|
||||
changed = changed || result;
|
||||
}
|
||||
|
||||
return changed;
|
||||
});
|
||||
}
|
||||
|
||||
async deleteDataStoreAll() {
|
||||
return await this.manager.transaction(async (em) => {
|
||||
const queryRunner = em.queryRunner;
|
||||
if (!queryRunner) {
|
||||
throw new UnexpectedError('QueryRunner is not available');
|
||||
}
|
||||
|
||||
const existingTables = await em.findBy(DataStore, {});
|
||||
|
||||
let changed = false;
|
||||
for (const match of existingTables) {
|
||||
const result = await em.delete(DataStore, { id: match.id });
|
||||
await queryRunner.dropTable(toTableName(match.id), true);
|
||||
changed = changed || (result.affected ?? 0) > 0;
|
||||
}
|
||||
|
||||
return changed;
|
||||
});
|
||||
}
|
||||
|
||||
async getManyAndCount(options: Partial<ListDataStoreQueryDto>) {
|
||||
const query = this.getManyQuery(options);
|
||||
const [data, count] = await query.getManyAndCount();
|
||||
return { count, data };
|
||||
}
|
||||
|
||||
async getMany(options: Partial<ListDataStoreQueryDto>) {
|
||||
const query = this.getManyQuery(options);
|
||||
return await query.getMany();
|
||||
}
|
||||
|
||||
private getManyQuery(options: Partial<ListDataStoreQueryDto>): SelectQueryBuilder<DataStore> {
|
||||
const query = this.createQueryBuilder('dataStore');
|
||||
|
||||
this.applySelections(query);
|
||||
this.applyFilters(query, options.filter);
|
||||
this.applySorting(query, options.sortBy);
|
||||
this.applyPagination(query, options);
|
||||
|
||||
return query;
|
||||
}
|
||||
|
||||
private applySelections(query: SelectQueryBuilder<DataStore>): void {
|
||||
this.applyDefaultSelect(query);
|
||||
}
|
||||
|
||||
private applyFilters(
|
||||
query: SelectQueryBuilder<DataStore>,
|
||||
filter: Partial<ListDataStoreQueryDto>['filter'],
|
||||
): void {
|
||||
for (const x of ['id', 'projectId'] as const) {
|
||||
const content = [filter?.[x]].flat().filter((x) => x !== undefined);
|
||||
if (content.length === 0) continue;
|
||||
|
||||
query.andWhere(`dataStore.${x} IN (:...${x}s)`, {
|
||||
/*
|
||||
* If list is empty, add a dummy value to prevent an error
|
||||
* when using the IN operator with an empty array.
|
||||
*/
|
||||
[x + 's']: content.length > 0 ? content : [''],
|
||||
});
|
||||
}
|
||||
|
||||
if (filter?.name && typeof filter.name === 'string') {
|
||||
query.andWhere('LOWER(dataStore.name) LIKE LOWER(:name)', {
|
||||
name: `%${filter.name}%`,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private applySorting(query: SelectQueryBuilder<DataStore>, sortBy?: string): void {
|
||||
if (!sortBy) {
|
||||
query.orderBy('dataStore.updatedAt', 'DESC');
|
||||
return;
|
||||
}
|
||||
|
||||
const [field, order] = this.parseSortingParams(sortBy);
|
||||
this.applySortingByField(query, field, order);
|
||||
}
|
||||
|
||||
private parseSortingParams(sortBy: string): [string, 'DESC' | 'ASC'] {
|
||||
const [field, order] = sortBy.split(':');
|
||||
return [field, order?.toLowerCase() === 'desc' ? 'DESC' : 'ASC'];
|
||||
}
|
||||
|
||||
private applySortingByField(
|
||||
query: SelectQueryBuilder<DataStore>,
|
||||
field: string,
|
||||
direction: 'DESC' | 'ASC',
|
||||
): void {
|
||||
if (field === 'name') {
|
||||
query.orderBy('LOWER(dataStore.name)', direction);
|
||||
} else if (['createdAt', 'updatedAt'].includes(field)) {
|
||||
query.orderBy(`dataStore.${field}`, direction);
|
||||
}
|
||||
}
|
||||
|
||||
private applyPagination(
|
||||
query: SelectQueryBuilder<DataStore>,
|
||||
options: Partial<ListDataStoreQueryDto>,
|
||||
): void {
|
||||
query.skip(options.skip ?? 0);
|
||||
if (options?.take) {
|
||||
query.skip(options.skip ?? 0).take(options.take);
|
||||
}
|
||||
}
|
||||
|
||||
private applyDefaultSelect(query: SelectQueryBuilder<DataStore>): void {
|
||||
query
|
||||
.leftJoinAndSelect('dataStore.project', 'project')
|
||||
.leftJoinAndSelect('dataStore.columns', 'data_store_column')
|
||||
.select([
|
||||
'dataStore',
|
||||
...this.getDataStoreColumnFields('data_store_column'),
|
||||
...this.getProjectFields('project'),
|
||||
])
|
||||
.addOrderBy('data_store_column.index', 'ASC');
|
||||
}
|
||||
|
||||
private getDataStoreColumnFields(alias: string): string[] {
|
||||
return [
|
||||
`${alias}.id`,
|
||||
`${alias}.name`,
|
||||
`${alias}.type`,
|
||||
`${alias}.createdAt`,
|
||||
`${alias}.updatedAt`,
|
||||
];
|
||||
}
|
||||
|
||||
private getProjectFields(alias: string): string[] {
|
||||
return [`${alias}.id`, `${alias}.name`, `${alias}.type`, `${alias}.icon`];
|
||||
}
|
||||
}
|
||||
265
packages/cli/src/modules/data-store/data-store.service.ts
Normal file
265
packages/cli/src/modules/data-store/data-store.service.ts
Normal file
@@ -0,0 +1,265 @@
|
||||
import type {
|
||||
AddDataStoreColumnDto,
|
||||
CreateDataStoreDto,
|
||||
ListDataStoreContentQueryDto,
|
||||
MoveDataStoreColumnDto,
|
||||
DataStoreListOptions,
|
||||
DataStoreRows,
|
||||
UpsertDataStoreRowsDto,
|
||||
} from '@n8n/api-types';
|
||||
import { UpdateDataStoreDto } from '@n8n/api-types/src/dto/data-store/update-data-store.dto';
|
||||
import { Logger } from '@n8n/backend-common';
|
||||
import { Service } from '@n8n/di';
|
||||
import { UserError } from 'n8n-workflow';
|
||||
|
||||
import { DataStoreColumn } from './data-store-column.entity';
|
||||
import { DataStoreColumnRepository } from './data-store-column.repository';
|
||||
import { DataStoreRowsRepository } from './data-store-rows.repository';
|
||||
import { DataStoreRepository } from './data-store.repository';
|
||||
import { toTableName } from './utils/sql-utils';
|
||||
|
||||
@Service()
|
||||
export class DataStoreService {
|
||||
constructor(
|
||||
private readonly dataStoreRepository: DataStoreRepository,
|
||||
private readonly dataStoreColumnRepository: DataStoreColumnRepository,
|
||||
private readonly dataStoreRowsRepository: DataStoreRowsRepository,
|
||||
private readonly logger: Logger,
|
||||
) {
|
||||
this.logger = this.logger.scoped('data-store');
|
||||
}
|
||||
|
||||
async start() {}
|
||||
async shutdown() {}
|
||||
|
||||
async createDataStore(projectId: string, dto: CreateDataStoreDto) {
|
||||
const existingTable = await this.dataStoreRepository.findOneBy({
|
||||
name: dto.name,
|
||||
projectId,
|
||||
});
|
||||
if (existingTable !== null) {
|
||||
throw new UserError(`Data store with name '${dto.name}' already exists in this project`);
|
||||
}
|
||||
return await this.dataStoreRepository.createDataStore(projectId, dto.name, dto.columns);
|
||||
}
|
||||
|
||||
// Currently only renames data stores
|
||||
async updateDataStore(dataStoreId: string, dto: UpdateDataStoreDto) {
|
||||
const name = dto.name.trim();
|
||||
|
||||
if (!name) {
|
||||
throw new UserError('Data store name must not be empty');
|
||||
}
|
||||
|
||||
const existingTable = await this.dataStoreRepository.findOneBy({
|
||||
id: dataStoreId,
|
||||
});
|
||||
|
||||
if (existingTable === null) {
|
||||
throw new UserError(`Tried to rename non-existent data store '${dataStoreId}'`);
|
||||
}
|
||||
|
||||
const hasNameClash = await this.dataStoreRepository.existsBy({
|
||||
name,
|
||||
projectId: existingTable.projectId,
|
||||
});
|
||||
|
||||
if (hasNameClash) {
|
||||
throw new UserError(`The name '${name}' is already taken within this project`);
|
||||
}
|
||||
|
||||
await this.dataStoreRepository.update({ id: dataStoreId }, { name });
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
async deleteDataStoreByProjectId(projectId: string) {
|
||||
return await this.dataStoreRepository.deleteDataStoreByProjectId(projectId);
|
||||
}
|
||||
|
||||
async deleteDataStoreAll() {
|
||||
return await this.dataStoreRepository.deleteDataStoreAll();
|
||||
}
|
||||
|
||||
async deleteDataStore(dataStoreId: string) {
|
||||
await this.validateDataStoreExists(
|
||||
dataStoreId,
|
||||
`Tried to delete non-existent data store '${dataStoreId}'`,
|
||||
);
|
||||
|
||||
await this.dataStoreRepository.deleteDataStore(dataStoreId);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
async addColumn(dataStoreId: string, dto: AddDataStoreColumnDto) {
|
||||
await this.validateDataStoreExists(
|
||||
dataStoreId,
|
||||
`Tried to add column to non-existent data store '${dataStoreId}'`,
|
||||
);
|
||||
|
||||
return await this.dataStoreColumnRepository.addColumn(dataStoreId, dto);
|
||||
}
|
||||
|
||||
async moveColumn(dataStoreId: string, columnId: string, dto: MoveDataStoreColumnDto) {
|
||||
await this.validateDataStoreExists(
|
||||
dataStoreId,
|
||||
`Tried to move column from non-existent data store '${dataStoreId}'`,
|
||||
);
|
||||
|
||||
await this.dataStoreColumnRepository.moveColumn(dataStoreId, columnId, dto.targetIndex);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
async deleteColumn(dataStoreId: string, columnId: string) {
|
||||
await this.validateDataStoreExists(
|
||||
dataStoreId,
|
||||
`Tried to delete column from non-existent data store '${dataStoreId}'`,
|
||||
);
|
||||
|
||||
const existingColumnMatch = await this.dataStoreColumnRepository.findOneBy({
|
||||
id: columnId,
|
||||
dataStoreId,
|
||||
});
|
||||
|
||||
if (existingColumnMatch === null) {
|
||||
throw new UserError(
|
||||
`Tried to delete column with name not present in data store '${dataStoreId}'`,
|
||||
);
|
||||
}
|
||||
|
||||
await this.dataStoreColumnRepository.deleteColumn(dataStoreId, existingColumnMatch);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
async getManyAndCount(options: DataStoreListOptions) {
|
||||
return await this.dataStoreRepository.getManyAndCount(options);
|
||||
}
|
||||
|
||||
async getManyRowsAndCount(dataStoreId: string, dto: ListDataStoreContentQueryDto) {
|
||||
// unclear if we should validate here, only use case would be to reduce the chance of
|
||||
// a renamed/removed column appearing here (or added column missing) if the store was
|
||||
// modified between when the frontend sent the request and we received it
|
||||
const columns = await this.dataStoreColumnRepository.getColumns(dataStoreId);
|
||||
const result = await this.dataStoreRowsRepository.getManyAndCount(
|
||||
toTableName(dataStoreId),
|
||||
dto,
|
||||
);
|
||||
return {
|
||||
count: result.count,
|
||||
data: this.normalizeRows(result.data, columns),
|
||||
};
|
||||
}
|
||||
|
||||
async getColumns(dataStoreId: string) {
|
||||
return await this.dataStoreColumnRepository.getColumns(dataStoreId);
|
||||
}
|
||||
|
||||
// TODO: move to utils and test
|
||||
private normalizeRows(rows: Array<Record<string, unknown>>, columns: DataStoreColumn[]) {
|
||||
const typeMap = new Map(columns.map((col) => [col.name, col.type]));
|
||||
return rows.map((row) => {
|
||||
const normalized = { ...row };
|
||||
for (const [key, value] of Object.entries(row)) {
|
||||
const type = typeMap.get(key);
|
||||
|
||||
if (type === 'boolean') {
|
||||
// Convert boolean values to true/false
|
||||
if (typeof value === 'boolean') {
|
||||
normalized[key] = value;
|
||||
} else if (value === 1 || value === '1') {
|
||||
normalized[key] = true;
|
||||
} else if (value === 0 || value === '0') {
|
||||
normalized[key] = false;
|
||||
}
|
||||
}
|
||||
if (type === 'date' && value !== null && value !== undefined) {
|
||||
// Convert date objects or strings to ISO string
|
||||
let dateObj: Date | null = null;
|
||||
|
||||
if (value instanceof Date) {
|
||||
dateObj = value;
|
||||
} else if (typeof value === 'string' || typeof value === 'number') {
|
||||
const parsed = new Date(value);
|
||||
if (!isNaN(parsed.getTime())) {
|
||||
dateObj = parsed;
|
||||
}
|
||||
}
|
||||
|
||||
normalized[key] = dateObj ? dateObj.toISOString() : value;
|
||||
}
|
||||
}
|
||||
return normalized;
|
||||
});
|
||||
}
|
||||
|
||||
private async validateRows(dataStoreId: string, rows: DataStoreRows): Promise<void> {
|
||||
const columns = await this.dataStoreColumnRepository.getColumns(dataStoreId);
|
||||
if (columns.length === 0) {
|
||||
throw new UserError('No columns found for this data store or data store not found');
|
||||
}
|
||||
|
||||
const columnNames = new Set(columns.map((x) => x.name));
|
||||
const columnTypeMap = new Map(columns.map((x) => [x.name, x.type]));
|
||||
for (const row of rows) {
|
||||
const keys = Object.keys(row);
|
||||
if (columns.length !== keys.length) {
|
||||
throw new UserError('mismatched key count');
|
||||
}
|
||||
for (const key of keys) {
|
||||
if (!columnNames.has(key)) {
|
||||
throw new UserError('unknown column name');
|
||||
}
|
||||
const cell = row[key];
|
||||
if (cell === null) continue;
|
||||
switch (columnTypeMap.get(key)) {
|
||||
case 'boolean':
|
||||
if (typeof cell !== 'boolean')
|
||||
throw new UserError(
|
||||
`value '${cell.toString()}' does not match column type 'boolean'`,
|
||||
);
|
||||
break;
|
||||
case 'date':
|
||||
if (!(cell instanceof Date))
|
||||
throw new UserError(`value '${cell}' does not match column type 'date'`);
|
||||
row[key] = cell.toISOString();
|
||||
break;
|
||||
case 'string':
|
||||
if (typeof cell !== 'string')
|
||||
throw new UserError(`value '${cell.toString()}' does not match column type 'string'`);
|
||||
break;
|
||||
case 'number':
|
||||
if (typeof cell !== 'number')
|
||||
throw new UserError(`value '${cell.toString()}' does not match column type 'number'`);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async insertRows(dataStoreId: string, rows: DataStoreRows) {
|
||||
await this.validateRows(dataStoreId, rows);
|
||||
const columns = await this.dataStoreColumnRepository.getColumns(dataStoreId);
|
||||
|
||||
return await this.dataStoreRowsRepository.insertRows(toTableName(dataStoreId), rows, columns);
|
||||
}
|
||||
|
||||
async upsertRows(dataStoreId: string, dto: UpsertDataStoreRowsDto) {
|
||||
await this.validateRows(dataStoreId, dto.rows);
|
||||
const columns = await this.dataStoreColumnRepository.getColumns(dataStoreId);
|
||||
|
||||
return await this.dataStoreRowsRepository.upsertRows(toTableName(dataStoreId), dto, columns);
|
||||
}
|
||||
|
||||
private async validateDataStoreExists(dataStoreId: string, msg?: string) {
|
||||
const existingTable = await this.dataStoreRepository.findOneBy({
|
||||
id: dataStoreId,
|
||||
});
|
||||
|
||||
if (!existingTable) {
|
||||
throw new UserError(msg ?? `Data Store '${dataStoreId}' does not exist.`);
|
||||
}
|
||||
}
|
||||
}
|
||||
1
packages/cli/src/modules/data-store/data-store.types.ts
Normal file
1
packages/cli/src/modules/data-store/data-store.types.ts
Normal file
@@ -0,0 +1 @@
|
||||
export type DataStoreUserTableName = `data_store_user_${string}`;
|
||||
242
packages/cli/src/modules/data-store/utils/sql-utils.ts
Normal file
242
packages/cli/src/modules/data-store/utils/sql-utils.ts
Normal file
@@ -0,0 +1,242 @@
|
||||
import {
|
||||
DATA_STORE_COLUMN_REGEX,
|
||||
type DataStoreRows,
|
||||
type DataStoreCreateColumnSchema,
|
||||
} from '@n8n/api-types';
|
||||
import { DslColumn } from '@n8n/db';
|
||||
import type { DataSourceOptions } from '@n8n/typeorm';
|
||||
import { UnexpectedError } from 'n8n-workflow';
|
||||
|
||||
import type { DataStoreUserTableName } from '../data-store.types';
|
||||
|
||||
import { NotFoundError } from '@/errors/response-errors/not-found.error';
|
||||
|
||||
export function toDslColumns(columns: DataStoreCreateColumnSchema[]): DslColumn[] {
|
||||
return columns.map((col) => {
|
||||
const name = new DslColumn(col.name.trim());
|
||||
|
||||
switch (col.type) {
|
||||
case 'number':
|
||||
return name.int;
|
||||
case 'boolean':
|
||||
return name.bool;
|
||||
case 'string':
|
||||
return name.text;
|
||||
case 'date':
|
||||
return name.timestampTimezone();
|
||||
default:
|
||||
return name.text;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function dataStoreColumnTypeToSql(
|
||||
type: DataStoreCreateColumnSchema['type'],
|
||||
dbType: DataSourceOptions['type'],
|
||||
) {
|
||||
switch (type) {
|
||||
case 'string':
|
||||
return 'TEXT';
|
||||
case 'number':
|
||||
return 'FLOAT';
|
||||
case 'boolean':
|
||||
return 'BOOLEAN';
|
||||
case 'date':
|
||||
if (dbType === 'postgres') {
|
||||
return 'TIMESTAMP';
|
||||
}
|
||||
return 'DATETIME';
|
||||
default:
|
||||
throw new NotFoundError(`Unsupported field type: ${type as string}`);
|
||||
}
|
||||
}
|
||||
|
||||
function columnToWildcardAndType(
|
||||
column: DataStoreCreateColumnSchema,
|
||||
dbType: DataSourceOptions['type'],
|
||||
) {
|
||||
return `${quoteIdentifier(column.name, dbType)} ${dataStoreColumnTypeToSql(column.type, dbType)}`;
|
||||
}
|
||||
|
||||
function isValidColumnName(name: string) {
|
||||
// Only allow alphanumeric and underscore
|
||||
return DATA_STORE_COLUMN_REGEX.test(name);
|
||||
}
|
||||
|
||||
export function addColumnQuery(
|
||||
tableName: DataStoreUserTableName,
|
||||
column: DataStoreCreateColumnSchema,
|
||||
dbType: DataSourceOptions['type'],
|
||||
) {
|
||||
// API requests should already conform to this, but better safe than sorry
|
||||
if (!isValidColumnName(column.name)) {
|
||||
throw new UnexpectedError('bad column name');
|
||||
}
|
||||
|
||||
const quotedTableName = quoteIdentifier(tableName, dbType);
|
||||
|
||||
return `ALTER TABLE ${quotedTableName} ADD ${columnToWildcardAndType(column, dbType)}`;
|
||||
}
|
||||
|
||||
export function deleteColumnQuery(
|
||||
tableName: DataStoreUserTableName,
|
||||
column: string,
|
||||
dbType: DataSourceOptions['type'],
|
||||
): string {
|
||||
const quotedTableName = quoteIdentifier(tableName, dbType);
|
||||
return `ALTER TABLE ${quotedTableName} DROP COLUMN ${quoteIdentifier(column, dbType)}`;
|
||||
}
|
||||
|
||||
export function buildInsertQuery(
|
||||
tableName: DataStoreUserTableName,
|
||||
rows: DataStoreRows,
|
||||
columns: Array<{ name: string; type: string }>,
|
||||
dbType: DataSourceOptions['type'] = 'sqlite',
|
||||
): [string, unknown[]] {
|
||||
if (rows.length === 0 || Object.keys(rows[0]).length === 0) {
|
||||
return ['', []];
|
||||
}
|
||||
|
||||
const keys = Object.keys(rows[0]);
|
||||
const quotedKeys = keys.map((key) => quoteIdentifier(key, dbType)).join(', ');
|
||||
const quotedTableName = quoteIdentifier(tableName, dbType);
|
||||
|
||||
const columnTypeMap = buildColumnTypeMap(columns);
|
||||
const parameters: unknown[] = [];
|
||||
const valuePlaceholders: string[] = [];
|
||||
let placeholderIndex = 1;
|
||||
|
||||
for (const row of rows) {
|
||||
const rowPlaceholders = keys.map((key) => {
|
||||
const value = normalizeValue(row[key], columnTypeMap[key], dbType);
|
||||
parameters.push(value);
|
||||
return getPlaceholder(placeholderIndex++, dbType);
|
||||
});
|
||||
valuePlaceholders.push(`(${rowPlaceholders.join(', ')})`);
|
||||
}
|
||||
|
||||
const query = `INSERT INTO ${quotedTableName} (${quotedKeys}) VALUES ${valuePlaceholders.join(', ')}`;
|
||||
return [query, parameters];
|
||||
}
|
||||
|
||||
export function buildUpdateQuery(
|
||||
tableName: DataStoreUserTableName,
|
||||
row: Record<string, unknown>,
|
||||
columns: Array<{ name: string; type: string }>,
|
||||
matchFields: string[],
|
||||
dbType: DataSourceOptions['type'] = 'sqlite',
|
||||
): [string, unknown[]] {
|
||||
if (Object.keys(row).length === 0 || matchFields.length === 0) {
|
||||
return ['', []];
|
||||
}
|
||||
|
||||
const updateKeys = Object.keys(row).filter((key) => !matchFields.includes(key));
|
||||
if (updateKeys.length === 0) {
|
||||
return ['', []];
|
||||
}
|
||||
|
||||
const quotedTableName = quoteIdentifier(tableName, dbType);
|
||||
const columnTypeMap = buildColumnTypeMap(columns);
|
||||
|
||||
const parameters: unknown[] = [];
|
||||
let placeholderIndex = 1;
|
||||
|
||||
const setClause = updateKeys
|
||||
.map((key) => {
|
||||
const value = normalizeValue(row[key], columnTypeMap[key], dbType);
|
||||
parameters.push(value);
|
||||
return `${quoteIdentifier(key, dbType)} = ${getPlaceholder(placeholderIndex++, dbType)}`;
|
||||
})
|
||||
.join(', ');
|
||||
|
||||
const whereClause = matchFields
|
||||
.map((key) => {
|
||||
const value = normalizeValue(row[key], columnTypeMap[key], dbType);
|
||||
parameters.push(value);
|
||||
return `${quoteIdentifier(key, dbType)} = ${getPlaceholder(placeholderIndex++, dbType)}`;
|
||||
})
|
||||
.join(' AND ');
|
||||
|
||||
const query = `UPDATE ${quotedTableName} SET ${setClause} WHERE ${whereClause}`;
|
||||
return [query, parameters];
|
||||
}
|
||||
|
||||
export function splitRowsByExistence(
|
||||
existing: Array<Record<string, unknown>>,
|
||||
matchFields: string[],
|
||||
rows: DataStoreRows,
|
||||
): { rowsToInsert: DataStoreRows; rowsToUpdate: DataStoreRows } {
|
||||
// Extracts only the fields relevant to matching and serializes them for comparison
|
||||
const getMatchKey = (row: Record<string, unknown>): string =>
|
||||
JSON.stringify(Object.fromEntries(matchFields.map((field) => [field, row[field]])));
|
||||
|
||||
const existingSet = new Set(existing.map((row) => getMatchKey(row)));
|
||||
|
||||
const rowsToUpdate: DataStoreRows = [];
|
||||
const rowsToInsert: DataStoreRows = [];
|
||||
|
||||
for (const row of rows) {
|
||||
const key = getMatchKey(row);
|
||||
|
||||
if (existingSet.has(key)) {
|
||||
rowsToUpdate.push(row);
|
||||
} else {
|
||||
rowsToInsert.push(row);
|
||||
}
|
||||
}
|
||||
|
||||
return { rowsToInsert, rowsToUpdate };
|
||||
}
|
||||
|
||||
export function quoteIdentifier(name: string, dbType: DataSourceOptions['type']): string {
|
||||
switch (dbType) {
|
||||
case 'mysql':
|
||||
case 'mariadb':
|
||||
return `\`${name}\``;
|
||||
case 'postgres':
|
||||
case 'sqlite':
|
||||
default:
|
||||
return `"${name}"`;
|
||||
}
|
||||
}
|
||||
|
||||
export function toTableName(dataStoreId: string): DataStoreUserTableName {
|
||||
return `data_store_user_${dataStoreId}`;
|
||||
}
|
||||
|
||||
function normalizeValue(
|
||||
value: unknown,
|
||||
columnType: string | undefined,
|
||||
dbType: DataSourceOptions['type'],
|
||||
): unknown {
|
||||
if (['mysql', 'mariadb'].includes(dbType)) {
|
||||
if (columnType === 'date') {
|
||||
if (
|
||||
value instanceof Date ||
|
||||
(typeof value === 'string' && value.match(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}/))
|
||||
) {
|
||||
return toMySQLDateTimeString(value);
|
||||
}
|
||||
}
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function toMySQLDateTimeString(date: Date | string, convertFromDate = true): string {
|
||||
const dateString = convertFromDate
|
||||
? date instanceof Date
|
||||
? date.toISOString()
|
||||
: date
|
||||
: (date as string);
|
||||
return dateString.replace('T', ' ').replace('Z', '');
|
||||
}
|
||||
|
||||
export function getPlaceholder(index: number, dbType: DataSourceOptions['type']): string {
|
||||
return dbType.includes('postgres') ? `$${index}` : '?';
|
||||
}
|
||||
|
||||
function buildColumnTypeMap(
|
||||
columns: Array<{ name: string; type: string }>,
|
||||
): Record<string, string> {
|
||||
return Object.fromEntries(columns.map((col) => [col.name, col.type]));
|
||||
}
|
||||
@@ -42,9 +42,10 @@ type EndpointGroup =
|
||||
| 'evaluation'
|
||||
| 'ai'
|
||||
| 'folder'
|
||||
| 'insights';
|
||||
| 'insights'
|
||||
| 'data-store';
|
||||
|
||||
type ModuleName = 'insights' | 'external-secrets';
|
||||
type ModuleName = 'insights' | 'external-secrets' | 'data-store';
|
||||
|
||||
export interface SetupProps {
|
||||
endpointGroups?: EndpointGroup[];
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
<script setup lang="ts">
|
||||
import { onMounted, ref } from 'vue';
|
||||
import type { DataStoreEntity } from '@/features/dataStore/datastore.types';
|
||||
import type { DataStore } from '@/features/dataStore/datastore.types';
|
||||
import { useDataStoreStore } from '@/features/dataStore/dataStore.store';
|
||||
import { useToast } from '@/composables/useToast';
|
||||
import { useI18n } from '@n8n/i18n';
|
||||
@@ -24,7 +24,7 @@ const documentTitle = useDocumentTitle();
|
||||
const dataStoreStore = useDataStoreStore();
|
||||
|
||||
const loading = ref(false);
|
||||
const dataStore = ref<DataStoreEntity | null>(null);
|
||||
const dataStore = ref<DataStore | null>(null);
|
||||
|
||||
const showErrorAndGoBackToList = async (error: unknown) => {
|
||||
if (!(error instanceof Error)) {
|
||||
|
||||
@@ -9,6 +9,7 @@ import { createTestingPinia } from '@pinia/testing';
|
||||
import { createRouter, createWebHistory } from 'vue-router';
|
||||
import type { DataStoreResource } from '@/features/dataStore/types';
|
||||
import { useDataStoreStore } from '@/features/dataStore/dataStore.store';
|
||||
|
||||
vi.mock('@/composables/useProjectPages', () => ({
|
||||
useProjectPages: vi.fn().mockReturnValue({
|
||||
isOverviewSubPage: false,
|
||||
|
||||
@@ -5,7 +5,7 @@ import { vi } from 'vitest';
|
||||
import DataStoreActions from '@/features/dataStore/components/DataStoreActions.vue';
|
||||
import { DATA_STORE_CARD_ACTIONS } from '@/features/dataStore/constants';
|
||||
import { MODAL_CONFIRM } from '@/constants';
|
||||
import type { DataStoreEntity } from '@/features/dataStore/datastore.types';
|
||||
import type { DataStore } from '@/features/dataStore/datastore.types';
|
||||
|
||||
const mockMessage = {
|
||||
confirm: vi.fn(),
|
||||
@@ -49,7 +49,7 @@ vi.mock('@n8n/i18n', async (importOriginal) => ({
|
||||
}),
|
||||
}));
|
||||
|
||||
const mockDataStore: DataStoreEntity = {
|
||||
const mockDataStore: DataStore = {
|
||||
id: '1',
|
||||
name: 'Test DataStore',
|
||||
sizeBytes: 1024,
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
<script setup lang="ts">
|
||||
import type { DataStoreEntity } from '@/features/dataStore/datastore.types';
|
||||
import type { DataStore } from '@/features/dataStore/datastore.types';
|
||||
import type { IUser, UserAction } from '@n8n/design-system';
|
||||
import { DATA_STORE_CARD_ACTIONS } from '@/features/dataStore/constants';
|
||||
import { useI18n } from '@n8n/i18n';
|
||||
@@ -10,7 +10,7 @@ import { useDataStoreStore } from '@/features/dataStore/dataStore.store';
|
||||
import { useToast } from '@/composables/useToast';
|
||||
|
||||
type Props = {
|
||||
dataStore: DataStoreEntity;
|
||||
dataStore: DataStore;
|
||||
isReadOnly?: boolean;
|
||||
};
|
||||
|
||||
@@ -21,7 +21,7 @@ const props = withDefaults(defineProps<Props>(), {
|
||||
const emit = defineEmits<{
|
||||
rename: [
|
||||
value: {
|
||||
dataStore: DataStoreEntity;
|
||||
dataStore: DataStore;
|
||||
action: string;
|
||||
},
|
||||
];
|
||||
|
||||
@@ -3,7 +3,7 @@ import { createComponentRenderer } from '@/__tests__/render';
|
||||
import { createTestingPinia } from '@pinia/testing';
|
||||
import { vi } from 'vitest';
|
||||
import DataStoreBreadcrumbs from '@/features/dataStore/components/DataStoreBreadcrumbs.vue';
|
||||
import type { DataStoreEntity } from '@/features/dataStore/datastore.types';
|
||||
import type { DataStore } from '@/features/dataStore/datastore.types';
|
||||
|
||||
const mockRouter = {
|
||||
push: vi.fn(),
|
||||
@@ -48,7 +48,7 @@ vi.mock('@n8n/i18n', async (importOriginal) => ({
|
||||
}),
|
||||
}));
|
||||
|
||||
const mockDataStore: DataStoreEntity = {
|
||||
const mockDataStore: DataStore = {
|
||||
id: '1',
|
||||
name: 'Test DataStore',
|
||||
sizeBytes: 1024,
|
||||
@@ -69,7 +69,7 @@ const mockDataStore: DataStoreEntity = {
|
||||
},
|
||||
};
|
||||
|
||||
const mockDataStoreWithoutProject: DataStoreEntity = {
|
||||
const mockDataStoreWithoutProject: DataStore = {
|
||||
...mockDataStore,
|
||||
project: undefined,
|
||||
};
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
<script setup lang="ts">
|
||||
import { computed, nextTick, ref, useTemplateRef, watch } from 'vue';
|
||||
import type { DataStoreEntity } from '@/features/dataStore/datastore.types';
|
||||
import type { DataStore } from '@/features/dataStore/datastore.types';
|
||||
import { useI18n } from '@n8n/i18n';
|
||||
import type { PathItem } from '@n8n/design-system/components/N8nBreadcrumbs/Breadcrumbs.vue';
|
||||
import { useRouter } from 'vue-router';
|
||||
@@ -12,7 +12,7 @@ import { useToast } from '@/composables/useToast';
|
||||
const BREADCRUMBS_SEPARATOR = '›';
|
||||
|
||||
type Props = {
|
||||
dataStore: DataStoreEntity;
|
||||
dataStore: DataStore;
|
||||
};
|
||||
|
||||
const props = defineProps<Props>();
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
import { makeRestApiRequest } from '@n8n/rest-api-client';
|
||||
import type { IRestApiContext } from '@n8n/rest-api-client';
|
||||
|
||||
import { type DataStoreEntity } from '@/features/dataStore/datastore.types';
|
||||
import { type DataStore } from '@/features/dataStore/datastore.types';
|
||||
|
||||
export const fetchDataStoresApi = async (
|
||||
context: IRestApiContext,
|
||||
projectId?: string,
|
||||
projectId: string,
|
||||
options?: {
|
||||
skip?: number;
|
||||
take?: number;
|
||||
@@ -17,7 +17,7 @@ export const fetchDataStoresApi = async (
|
||||
},
|
||||
) => {
|
||||
const apiEndpoint = projectId ? `/projects/${projectId}/data-stores` : '/data-stores-global';
|
||||
return await makeRestApiRequest<{ count: number; data: DataStoreEntity[] }>(
|
||||
return await makeRestApiRequest<{ count: number; data: DataStore[] }>(
|
||||
context,
|
||||
'GET',
|
||||
apiEndpoint,
|
||||
@@ -33,7 +33,7 @@ export const createDataStoreApi = async (
|
||||
name: string,
|
||||
projectId?: string,
|
||||
) => {
|
||||
return await makeRestApiRequest<DataStoreEntity>(
|
||||
return await makeRestApiRequest<DataStore>(
|
||||
context,
|
||||
'POST',
|
||||
`/projects/${projectId}/data-stores`,
|
||||
@@ -66,7 +66,7 @@ export const updateDataStoreApi = async (
|
||||
name: string,
|
||||
projectId?: string,
|
||||
) => {
|
||||
return await makeRestApiRequest<DataStoreEntity>(
|
||||
return await makeRestApiRequest<DataStore>(
|
||||
context,
|
||||
'PATCH',
|
||||
`/projects/${projectId}/data-stores/${dataStoreId}`,
|
||||
@@ -7,15 +7,15 @@ import {
|
||||
createDataStoreApi,
|
||||
deleteDataStoreApi,
|
||||
updateDataStoreApi,
|
||||
} from '@/features/dataStore/datastore.api';
|
||||
import type { DataStoreEntity } from '@/features/dataStore/datastore.types';
|
||||
} from '@/features/dataStore/dataStore.api';
|
||||
import type { DataStore } from '@/features/dataStore/datastore.types';
|
||||
import { useProjectsStore } from '@/stores/projects.store';
|
||||
|
||||
export const useDataStoreStore = defineStore(DATA_STORE_STORE, () => {
|
||||
const rootStore = useRootStore();
|
||||
const projectStore = useProjectsStore();
|
||||
|
||||
const dataStores = ref<DataStoreEntity[]>([]);
|
||||
const dataStores = ref<DataStore[]>([]);
|
||||
const totalCount = ref(0);
|
||||
|
||||
const fetchDataStores = async (projectId: string, page: number, pageSize: number) => {
|
||||
|
||||
@@ -1,18 +1,18 @@
|
||||
import type { Project } from '@/types/projects.types';
|
||||
|
||||
export type DataStoreEntity = {
|
||||
export type DataStore = {
|
||||
id: string;
|
||||
name: string;
|
||||
sizeBytes: number;
|
||||
recordCount: number;
|
||||
columns: DataStoreColumnEntity[];
|
||||
columns: DataStoreColumn[];
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
projectId?: string;
|
||||
project?: Project;
|
||||
};
|
||||
|
||||
export type DataStoreColumnEntity = {
|
||||
export type DataStoreColumn = {
|
||||
id: string;
|
||||
name: string;
|
||||
type: string;
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
import type { BaseResource } from '@/Interface';
|
||||
import type { DataStoreEntity } from '@/features/dataStore/datastore.types';
|
||||
import type { DataStore } from '@/features/dataStore/datastore.types';
|
||||
|
||||
/**
|
||||
* Data Store resource type definition
|
||||
* This extends the ModuleResources interface to add DataStore as a resource type
|
||||
*/
|
||||
export type DataStoreResource = BaseResource &
|
||||
DataStoreEntity & {
|
||||
DataStore & {
|
||||
resourceType: 'datastore';
|
||||
};
|
||||
|
||||
|
||||
@@ -37,6 +37,7 @@ export const useRBACStore = defineStore(STORES.RBAC, () => {
|
||||
securityAudit: {},
|
||||
folder: {},
|
||||
insights: {},
|
||||
dataStore: {},
|
||||
});
|
||||
|
||||
function addGlobalRole(role: Role) {
|
||||
|
||||
Reference in New Issue
Block a user