From 98dc71e6a7391b2cf58fd69e5238dcb6dea4ae6e Mon Sep 17 00:00:00 2001 From: Charlie Kolb Date: Tue, 12 Aug 2025 14:54:24 +0200 Subject: [PATCH] feat(core): Add Data Store Backend API (no-changelog) (#17824) --- .../data-store/add-data-store-column.dto.ts | 5 + .../dto/data-store/add-data-store-rows.dto.ts | 8 + .../create-data-store-column.dto.ts | 11 + .../dto/data-store/create-data-store.dto.ts | 10 + .../list-data-store-content-query.dto.ts | 105 ++ .../data-store/list-data-store-query.dto.ts | 70 + .../data-store/move-data-store-column.dto.ts | 6 + .../dto/data-store/update-data-store.dto.ts | 7 + .../data-store/upsert-data-store-rows.dto.ts | 13 + packages/@n8n/api-types/src/dto/index.ts | 13 + packages/@n8n/api-types/src/index.ts | 12 + .../src/schemas/data-store.schema.ts | 56 + .../modules/__tests__/module-registry.test.ts | 13 +- .../src/modules/modules.config.ts | 2 +- .../@n8n/backend-test-utils/src/test-db.ts | 8 +- .../@n8n/config/src/configs/logging.config.ts | 1 + packages/@n8n/db/src/index.ts | 2 + packages/@n8n/permissions/src/constants.ee.ts | 1 + .../src/roles/scopes/global-scopes.ee.ts | 10 + .../src/roles/scopes/project-scopes.ee.ts | 24 + .../get-resource-permissions.test.ts | 2 + .../data-store-aggregate.service.test.ts | 215 +++ .../__tests__/data-store.service.test.ts | 1221 +++++++++++++++++ .../data-store/__tests__/sql-utils.test.ts | 228 +++ .../data-store-aggregate.controller.ts | 20 + .../data-store-aggregate.service.ts | 42 + .../data-store/data-store-column.entity.ts | 24 + .../data-store-column.repository.ts | 135 ++ .../data-store/data-store-rows.repository.ts | 238 ++++ .../data-store/data-store.controller.ts | 151 ++ .../modules/data-store/data-store.entity.ts | 34 + .../modules/data-store/data-store.module.ts | 34 + .../data-store/data-store.repository.ts | 235 ++++ .../modules/data-store/data-store.service.ts | 265 ++++ .../modules/data-store/data-store.types.ts | 1 + .../src/modules/data-store/utils/sql-utils.ts | 242 ++++ packages/cli/test/integration/shared/types.ts | 5 +- .../dataStore/DataStoreDetailsView.vue | 4 +- .../features/dataStore/DataStoreView.test.ts | 1 + .../components/DataStoreActions.test.ts | 4 +- .../dataStore/components/DataStoreActions.vue | 6 +- .../components/DataStoreBreadcrumbs.test.ts | 6 +- .../components/DataStoreBreadcrumbs.vue | 4 +- .../{datastore.api.ts => dataStore.api.ts} | 10 +- .../src/features/dataStore/dataStore.store.ts | 6 +- .../src/features/dataStore/datastore.types.ts | 6 +- .../editor-ui/src/features/dataStore/types.ts | 4 +- .../editor-ui/src/stores/rbac.store.ts | 1 + 48 files changed, 3491 insertions(+), 30 deletions(-) create mode 100644 packages/@n8n/api-types/src/dto/data-store/add-data-store-column.dto.ts create mode 100644 packages/@n8n/api-types/src/dto/data-store/add-data-store-rows.dto.ts create mode 100644 packages/@n8n/api-types/src/dto/data-store/create-data-store-column.dto.ts create mode 100644 packages/@n8n/api-types/src/dto/data-store/create-data-store.dto.ts create mode 100644 packages/@n8n/api-types/src/dto/data-store/list-data-store-content-query.dto.ts create mode 100644 packages/@n8n/api-types/src/dto/data-store/list-data-store-query.dto.ts create mode 100644 packages/@n8n/api-types/src/dto/data-store/move-data-store-column.dto.ts create mode 100644 packages/@n8n/api-types/src/dto/data-store/update-data-store.dto.ts create mode 100644 packages/@n8n/api-types/src/dto/data-store/upsert-data-store-rows.dto.ts create mode 100644 packages/@n8n/api-types/src/schemas/data-store.schema.ts create mode 100644 packages/cli/src/modules/data-store/__tests__/data-store-aggregate.service.test.ts create mode 100644 packages/cli/src/modules/data-store/__tests__/data-store.service.test.ts create mode 100644 packages/cli/src/modules/data-store/__tests__/sql-utils.test.ts create mode 100644 packages/cli/src/modules/data-store/data-store-aggregate.controller.ts create mode 100644 packages/cli/src/modules/data-store/data-store-aggregate.service.ts create mode 100644 packages/cli/src/modules/data-store/data-store-column.entity.ts create mode 100644 packages/cli/src/modules/data-store/data-store-column.repository.ts create mode 100644 packages/cli/src/modules/data-store/data-store-rows.repository.ts create mode 100644 packages/cli/src/modules/data-store/data-store.controller.ts create mode 100644 packages/cli/src/modules/data-store/data-store.entity.ts create mode 100644 packages/cli/src/modules/data-store/data-store.module.ts create mode 100644 packages/cli/src/modules/data-store/data-store.repository.ts create mode 100644 packages/cli/src/modules/data-store/data-store.service.ts create mode 100644 packages/cli/src/modules/data-store/data-store.types.ts create mode 100644 packages/cli/src/modules/data-store/utils/sql-utils.ts rename packages/frontend/editor-ui/src/features/dataStore/{datastore.api.ts => dataStore.api.ts} (86%) diff --git a/packages/@n8n/api-types/src/dto/data-store/add-data-store-column.dto.ts b/packages/@n8n/api-types/src/dto/data-store/add-data-store-column.dto.ts new file mode 100644 index 0000000000..37e089b4d5 --- /dev/null +++ b/packages/@n8n/api-types/src/dto/data-store/add-data-store-column.dto.ts @@ -0,0 +1,5 @@ +import { Z } from 'zod-class'; + +import { dataStoreCreateColumnSchema } from '../../schemas/data-store.schema'; + +export class AddDataStoreColumnDto extends Z.class(dataStoreCreateColumnSchema.shape) {} diff --git a/packages/@n8n/api-types/src/dto/data-store/add-data-store-rows.dto.ts b/packages/@n8n/api-types/src/dto/data-store/add-data-store-rows.dto.ts new file mode 100644 index 0000000000..a752b0d115 --- /dev/null +++ b/packages/@n8n/api-types/src/dto/data-store/add-data-store-rows.dto.ts @@ -0,0 +1,8 @@ +import { z } from 'zod'; +import { Z } from 'zod-class'; + +import { dataStoreColumnNameSchema } from '../../schemas/data-store.schema'; + +export class AddDataStoreRowsDto extends Z.class({ + data: z.array(z.record(dataStoreColumnNameSchema, z.any())), +}) {} diff --git a/packages/@n8n/api-types/src/dto/data-store/create-data-store-column.dto.ts b/packages/@n8n/api-types/src/dto/data-store/create-data-store-column.dto.ts new file mode 100644 index 0000000000..2246589ad5 --- /dev/null +++ b/packages/@n8n/api-types/src/dto/data-store/create-data-store-column.dto.ts @@ -0,0 +1,11 @@ +import { Z } from 'zod-class'; + +import { + dataStoreColumnNameSchema, + dataStoreColumnTypeSchema, +} from '../../schemas/data-store.schema'; + +export class CreateDataStoreColumnDto extends Z.class({ + name: dataStoreColumnNameSchema, + type: dataStoreColumnTypeSchema, +}) {} diff --git a/packages/@n8n/api-types/src/dto/data-store/create-data-store.dto.ts b/packages/@n8n/api-types/src/dto/data-store/create-data-store.dto.ts new file mode 100644 index 0000000000..f6b1298ebc --- /dev/null +++ b/packages/@n8n/api-types/src/dto/data-store/create-data-store.dto.ts @@ -0,0 +1,10 @@ +import { z } from 'zod'; +import { Z } from 'zod-class'; + +import { CreateDataStoreColumnDto } from './create-data-store-column.dto'; +import { dataStoreNameSchema } from '../../schemas/data-store.schema'; + +export class CreateDataStoreDto extends Z.class({ + name: dataStoreNameSchema, + columns: z.array(CreateDataStoreColumnDto), +}) {} diff --git a/packages/@n8n/api-types/src/dto/data-store/list-data-store-content-query.dto.ts b/packages/@n8n/api-types/src/dto/data-store/list-data-store-content-query.dto.ts new file mode 100644 index 0000000000..ccb05e974c --- /dev/null +++ b/packages/@n8n/api-types/src/dto/data-store/list-data-store-content-query.dto.ts @@ -0,0 +1,105 @@ +import { jsonParse } from 'n8n-workflow'; +import { z } from 'zod'; +import { Z } from 'zod-class'; + +import { dataStoreColumnNameSchema } from '../../schemas/data-store.schema'; +import { paginationSchema } from '../pagination/pagination.dto'; + +const FilterConditionSchema = z.union([z.literal('eq'), z.literal('neq')]); +export type ListDataStoreContentFilterConditionType = z.infer; + +const filterRecord = z.object({ + columnName: dataStoreColumnNameSchema, + condition: FilterConditionSchema.default('eq'), + value: z.union([z.string(), z.number(), z.boolean(), z.date()]), +}); + +const chainedFilterSchema = z.union([z.literal('and'), z.literal('or')]); + +export type ListDataStoreContentFilter = z.infer; + +// --------------------- +// Parameter Validators +// --------------------- + +const filterSchema = z.object({ + type: chainedFilterSchema.default('and'), + filters: z.array(filterRecord).default([]), +}); + +// Filter parameter validation +const filterValidator = z + .string() + .optional() + .transform((val, ctx) => { + if (!val) return undefined; + try { + const parsed: unknown = jsonParse(val); + try { + return filterSchema.parse(parsed); + } catch (e) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: 'Invalid filter fields', + path: ['filter'], + }); + return z.NEVER; + } + } catch (e) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: 'Invalid filter format', + path: ['filter'], + }); + return z.NEVER; + } + }); + +// SortBy parameter validation +const sortByValidator = z + .string() + .optional() + .transform((val, ctx) => { + if (val === undefined) return val; + + if (!val.includes(':')) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: 'Invalid sort format, expected :', + path: ['sort'], + }); + return z.NEVER; + } + + let [column, direction] = val.split(':'); + + try { + column = dataStoreColumnNameSchema.parse(column); + } catch { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: 'Invalid sort columnName', + path: ['sort'], + }); + return z.NEVER; + } + + direction = direction?.toUpperCase(); + if (direction !== 'ASC' && direction !== 'DESC') { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: 'Invalid sort direction', + path: ['sort'], + }); + + return z.NEVER; + } + return [column, direction] as const; + }); + +export class ListDataStoreContentQueryDto extends Z.class({ + take: paginationSchema.take.optional(), + skip: paginationSchema.skip.optional(), + filter: filterValidator.optional(), + sortBy: sortByValidator.optional(), +}) {} diff --git a/packages/@n8n/api-types/src/dto/data-store/list-data-store-query.dto.ts b/packages/@n8n/api-types/src/dto/data-store/list-data-store-query.dto.ts new file mode 100644 index 0000000000..36296d46ad --- /dev/null +++ b/packages/@n8n/api-types/src/dto/data-store/list-data-store-query.dto.ts @@ -0,0 +1,70 @@ +import { jsonParse } from 'n8n-workflow'; +import { z } from 'zod'; +import { Z } from 'zod-class'; + +import { paginationSchema } from '../pagination/pagination.dto'; + +const VALID_SORT_OPTIONS = [ + 'name:asc', + 'name:desc', + 'createdAt:asc', + 'createdAt:desc', + 'updatedAt:asc', + 'updatedAt:desc', + 'sizeBytes:asc', + 'sizeBytes:desc', +] as const; + +export type ListDataStoreQuerySortOptions = (typeof VALID_SORT_OPTIONS)[number]; + +const FILTER_OPTIONS = { + id: z.union([z.string(), z.array(z.string())]).optional(), + name: z.union([z.string(), z.array(z.string())]).optional(), + projectId: z.union([z.string(), z.array(z.string())]).optional(), + // todo: can probably include others here as well? +}; + +// Filter schema - only allow specific properties +const filterSchema = z.object(FILTER_OPTIONS).strict(); +// --------------------- +// Parameter Validators +// --------------------- + +// Filter parameter validation +const filterValidator = z + .string() + .optional() + .transform((val, ctx) => { + if (!val) return undefined; + try { + const parsed: unknown = jsonParse(val); + try { + return filterSchema.parse(parsed); + } catch (e) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: 'Invalid filter fields', + path: ['filter'], + }); + return z.NEVER; + } + } catch (e) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: 'Invalid filter format', + path: ['filter'], + }); + return z.NEVER; + } + }); + +// SortBy parameter validation +const sortByValidator = z + .enum(VALID_SORT_OPTIONS, { message: `sortBy must be one of: ${VALID_SORT_OPTIONS.join(', ')}` }) + .optional(); + +export class ListDataStoreQueryDto extends Z.class({ + ...paginationSchema, + filter: filterValidator, + sortBy: sortByValidator, +}) {} diff --git a/packages/@n8n/api-types/src/dto/data-store/move-data-store-column.dto.ts b/packages/@n8n/api-types/src/dto/data-store/move-data-store-column.dto.ts new file mode 100644 index 0000000000..03541193c3 --- /dev/null +++ b/packages/@n8n/api-types/src/dto/data-store/move-data-store-column.dto.ts @@ -0,0 +1,6 @@ +import { z } from 'zod'; +import { Z } from 'zod-class'; + +export class MoveDataStoreColumnDto extends Z.class({ + targetIndex: z.number(), +}) {} diff --git a/packages/@n8n/api-types/src/dto/data-store/update-data-store.dto.ts b/packages/@n8n/api-types/src/dto/data-store/update-data-store.dto.ts new file mode 100644 index 0000000000..7821f9dd43 --- /dev/null +++ b/packages/@n8n/api-types/src/dto/data-store/update-data-store.dto.ts @@ -0,0 +1,7 @@ +import { Z } from 'zod-class'; + +import { dataStoreNameSchema } from '../../schemas/data-store.schema'; + +export class UpdateDataStoreDto extends Z.class({ + name: dataStoreNameSchema, +}) {} diff --git a/packages/@n8n/api-types/src/dto/data-store/upsert-data-store-rows.dto.ts b/packages/@n8n/api-types/src/dto/data-store/upsert-data-store-rows.dto.ts new file mode 100644 index 0000000000..1626588f7e --- /dev/null +++ b/packages/@n8n/api-types/src/dto/data-store/upsert-data-store-rows.dto.ts @@ -0,0 +1,13 @@ +import { z } from 'zod'; +import { Z } from 'zod-class'; + +import { dataStoreColumnNameSchema } from '../../schemas/data-store.schema'; + +const dataStoreValueSchema = z.union([z.string(), z.number(), z.boolean(), z.date(), z.null()]); + +const upsertDataStoreRowsShape = { + rows: z.array(z.record(dataStoreValueSchema)), + matchFields: z.array(dataStoreColumnNameSchema).min(1), +}; + +export class UpsertDataStoreRowsDto extends Z.class(upsertDataStoreRowsShape) {} diff --git a/packages/@n8n/api-types/src/dto/index.ts b/packages/@n8n/api-types/src/dto/index.ts index 88c1875e8f..84028f6474 100644 --- a/packages/@n8n/api-types/src/dto/index.ts +++ b/packages/@n8n/api-types/src/dto/index.ts @@ -79,3 +79,16 @@ export { } from './user/users-list-filter.dto'; export { OidcConfigDto } from './oidc/config.dto'; + +export { CreateDataStoreDto } from './data-store/create-data-store.dto'; +export { UpdateDataStoreDto } from './data-store/update-data-store.dto'; +export { UpsertDataStoreRowsDto } from './data-store/upsert-data-store-rows.dto'; +export { ListDataStoreQueryDto } from './data-store/list-data-store-query.dto'; +export { + ListDataStoreContentQueryDto, + ListDataStoreContentFilter, +} from './data-store/list-data-store-content-query.dto'; +export { CreateDataStoreColumnDto } from './data-store/create-data-store-column.dto'; +export { AddDataStoreRowsDto } from './data-store/add-data-store-rows.dto'; +export { AddDataStoreColumnDto } from './data-store/add-data-store-column.dto'; +export { MoveDataStoreColumnDto } from './data-store/move-data-store-column.dto'; diff --git a/packages/@n8n/api-types/src/index.ts b/packages/@n8n/api-types/src/index.ts index d3fdf99b2b..618c8bec50 100644 --- a/packages/@n8n/api-types/src/index.ts +++ b/packages/@n8n/api-types/src/index.ts @@ -45,3 +45,15 @@ export { type UsersList, usersListSchema, } from './schemas/user.schema'; + +export { + DATA_STORE_COLUMN_REGEX, + type DataStore, + type DataStoreColumn, + type DataStoreCreateColumnSchema, + type DataStoreColumnJsType, + type DataStoreListFilter, + type DataStoreRows, + type DataStoreListOptions, + type DataStoreUserTableName, +} from './schemas/data-store.schema'; diff --git a/packages/@n8n/api-types/src/schemas/data-store.schema.ts b/packages/@n8n/api-types/src/schemas/data-store.schema.ts new file mode 100644 index 0000000000..7325397f95 --- /dev/null +++ b/packages/@n8n/api-types/src/schemas/data-store.schema.ts @@ -0,0 +1,56 @@ +import { z } from 'zod'; + +import type { ListDataStoreQueryDto } from '../dto'; + +export const dataStoreNameSchema = z.string().trim().min(1).max(128); +export const dataStoreIdSchema = z.string().max(36); + +export const DATA_STORE_COLUMN_REGEX = /^[a-zA-Z0-9][a-zA-Z0-9-]*$/; + +export const dataStoreColumnNameSchema = z + .string() + .trim() + .min(1) + .max(128) + .regex( + DATA_STORE_COLUMN_REGEX, + 'Only alphanumeric characters and non-leading dashes are allowed for column names', + ); +export const dataStoreColumnTypeSchema = z.enum(['string', 'number', 'boolean', 'date']); + +export const dataStoreCreateColumnSchema = z.object({ + name: dataStoreColumnNameSchema, + type: dataStoreColumnTypeSchema, + index: z.number().optional(), +}); +export type DataStoreCreateColumnSchema = z.infer; + +export const dataStoreColumnSchema = dataStoreCreateColumnSchema.extend({ + dataStoreId: dataStoreIdSchema, +}); + +export const dataStoreSchema = z.object({ + id: dataStoreIdSchema, + name: dataStoreNameSchema, + columns: z.array(dataStoreColumnSchema), + createdAt: z.string().datetime(), + updatedAt: z.string().datetime(), +}); +export type DataStore = z.infer; +export type DataStoreColumn = z.infer; + +export type DataStoreUserTableName = `data_store_user_${string}`; + +export type DataStoreListFilter = { + id?: string | string[]; + projectId?: string | string[]; + name?: string; +}; + +export type DataStoreListOptions = Partial & { + filter: { projectId: string }; +}; + +export type DataStoreColumnJsType = string | number | boolean | Date; + +export type DataStoreRows = Array>; diff --git a/packages/@n8n/backend-common/src/modules/__tests__/module-registry.test.ts b/packages/@n8n/backend-common/src/modules/__tests__/module-registry.test.ts index cdb761cc34..97078ddd04 100644 --- a/packages/@n8n/backend-common/src/modules/__tests__/module-registry.test.ts +++ b/packages/@n8n/backend-common/src/modules/__tests__/module-registry.test.ts @@ -15,7 +15,9 @@ beforeEach(() => { describe('eligibleModules', () => { it('should consider all default modules eligible', () => { - expect(Container.get(ModuleRegistry).eligibleModules).toEqual(MODULE_NAMES); + // 'data-store' isn't (yet) eligible module by default + const expectedModules = MODULE_NAMES.filter((name) => name !== 'data-store'); + expect(Container.get(ModuleRegistry).eligibleModules).toEqual(expectedModules); }); it('should consider a module ineligible if it was disabled via env var', () => { @@ -23,6 +25,15 @@ describe('eligibleModules', () => { expect(Container.get(ModuleRegistry).eligibleModules).toEqual(['external-secrets']); }); + it('should consider a module eligible if it was enabled via env var', () => { + process.env.N8N_ENABLED_MODULES = 'data-store'; + expect(Container.get(ModuleRegistry).eligibleModules).toEqual([ + 'insights', + 'external-secrets', + 'data-store', + ]); + }); + it('should throw `ModuleConfusionError` if a module is both enabled and disabled', () => { process.env.N8N_ENABLED_MODULES = 'insights'; process.env.N8N_DISABLED_MODULES = 'insights'; diff --git a/packages/@n8n/backend-common/src/modules/modules.config.ts b/packages/@n8n/backend-common/src/modules/modules.config.ts index 49f7b343bf..1c8688d733 100644 --- a/packages/@n8n/backend-common/src/modules/modules.config.ts +++ b/packages/@n8n/backend-common/src/modules/modules.config.ts @@ -2,7 +2,7 @@ import { CommaSeparatedStringArray, Config, Env } from '@n8n/config'; import { UnknownModuleError } from './errors/unknown-module.error'; -export const MODULE_NAMES = ['insights', 'external-secrets'] as const; +export const MODULE_NAMES = ['insights', 'external-secrets', 'data-store'] as const; export type ModuleName = (typeof MODULE_NAMES)[number]; diff --git a/packages/@n8n/backend-test-utils/src/test-db.ts b/packages/@n8n/backend-test-utils/src/test-db.ts index 925d8b7649..0f5798461a 100644 --- a/packages/@n8n/backend-test-utils/src/test-db.ts +++ b/packages/@n8n/backend-test-utils/src/test-db.ts @@ -66,7 +66,13 @@ export async function terminate() { dbConnection.connectionState.connected = false; } -type EntityName = keyof typeof entities | 'InsightsRaw' | 'InsightsByPeriod' | 'InsightsMetadata'; +type EntityName = + | keyof typeof entities + | 'InsightsRaw' + | 'InsightsByPeriod' + | 'InsightsMetadata' + | 'DataStore' + | 'DataStoreColumn'; /** * Truncate specific DB tables in a test DB. diff --git a/packages/@n8n/config/src/configs/logging.config.ts b/packages/@n8n/config/src/configs/logging.config.ts index a51d959603..924f8e9342 100644 --- a/packages/@n8n/config/src/configs/logging.config.ts +++ b/packages/@n8n/config/src/configs/logging.config.ts @@ -19,6 +19,7 @@ export const LOG_SCOPES = [ 'insights', 'workflow-activation', 'ssh-client', + 'data-store', 'cron', 'community-nodes', 'legacy-sqlite-execution-recovery', diff --git a/packages/@n8n/db/src/index.ts b/packages/@n8n/db/src/index.ts index e7551a3342..2856310188 100644 --- a/packages/@n8n/db/src/index.ts +++ b/packages/@n8n/db/src/index.ts @@ -23,6 +23,8 @@ export { NoUrl } from './utils/validators/no-url.validator'; export * from './repositories'; export * from './subscribers'; +export { Column as DslColumn } from './migrations/dsl/column'; +export { CreateTable } from './migrations/dsl/table'; export { sqliteMigrations } from './migrations/sqlite'; export { mysqlMigrations } from './migrations/mysqldb'; export { postgresMigrations } from './migrations/postgresdb'; diff --git a/packages/@n8n/permissions/src/constants.ee.ts b/packages/@n8n/permissions/src/constants.ee.ts index b24b4328da..c4085cd71b 100644 --- a/packages/@n8n/permissions/src/constants.ee.ts +++ b/packages/@n8n/permissions/src/constants.ee.ts @@ -26,6 +26,7 @@ export const RESOURCES = { folder: [...DEFAULT_OPERATIONS, 'move'] as const, insights: ['list'] as const, oidc: ['manage'] as const, + dataStore: [...DEFAULT_OPERATIONS, 'readRow', 'writeRow'] as const, } as const; export const API_KEY_RESOURCES = { diff --git a/packages/@n8n/permissions/src/roles/scopes/global-scopes.ee.ts b/packages/@n8n/permissions/src/roles/scopes/global-scopes.ee.ts index 6574339187..e7cfd059e5 100644 --- a/packages/@n8n/permissions/src/roles/scopes/global-scopes.ee.ts +++ b/packages/@n8n/permissions/src/roles/scopes/global-scopes.ee.ts @@ -79,6 +79,13 @@ export const GLOBAL_OWNER_SCOPES: Scope[] = [ 'insights:list', 'folder:move', 'oidc:manage', + 'dataStore:create', + 'dataStore:delete', + 'dataStore:read', + 'dataStore:update', + 'dataStore:list', + 'dataStore:readRow', + 'dataStore:writeRow', ]; export const GLOBAL_ADMIN_SCOPES = GLOBAL_OWNER_SCOPES.concat(); @@ -98,4 +105,7 @@ export const GLOBAL_MEMBER_SCOPES: Scope[] = [ 'user:list', 'variable:list', 'variable:read', + 'dataStore:read', + 'dataStore:list', + 'dataStore:readRow', ]; diff --git a/packages/@n8n/permissions/src/roles/scopes/project-scopes.ee.ts b/packages/@n8n/permissions/src/roles/scopes/project-scopes.ee.ts index d843d2dec8..f618e9f376 100644 --- a/packages/@n8n/permissions/src/roles/scopes/project-scopes.ee.ts +++ b/packages/@n8n/permissions/src/roles/scopes/project-scopes.ee.ts @@ -32,6 +32,13 @@ export const REGULAR_PROJECT_ADMIN_SCOPES: Scope[] = [ 'folder:list', 'folder:move', 'sourceControl:push', + 'dataStore:create', + 'dataStore:delete', + 'dataStore:read', + 'dataStore:update', + 'dataStore:list', + 'dataStore:readRow', + 'dataStore:writeRow', ]; export const PERSONAL_PROJECT_OWNER_SCOPES: Scope[] = [ @@ -58,6 +65,13 @@ export const PERSONAL_PROJECT_OWNER_SCOPES: Scope[] = [ 'folder:delete', 'folder:list', 'folder:move', + 'dataStore:create', + 'dataStore:delete', + 'dataStore:read', + 'dataStore:update', + 'dataStore:list', + 'dataStore:readRow', + 'dataStore:writeRow', ]; export const PROJECT_EDITOR_SCOPES: Scope[] = [ @@ -79,6 +93,13 @@ export const PROJECT_EDITOR_SCOPES: Scope[] = [ 'folder:update', 'folder:delete', 'folder:list', + 'dataStore:create', + 'dataStore:delete', + 'dataStore:read', + 'dataStore:update', + 'dataStore:list', + 'dataStore:readRow', + 'dataStore:writeRow', ]; export const PROJECT_VIEWER_SCOPES: Scope[] = [ @@ -90,4 +111,7 @@ export const PROJECT_VIEWER_SCOPES: Scope[] = [ 'workflow:read', 'folder:read', 'folder:list', + 'dataStore:list', + 'dataStore:read', + 'dataStore:readRow', ]; diff --git a/packages/@n8n/permissions/src/utilities/__tests__/get-resource-permissions.test.ts b/packages/@n8n/permissions/src/utilities/__tests__/get-resource-permissions.test.ts index ce54f858ce..2df39ccd31 100644 --- a/packages/@n8n/permissions/src/utilities/__tests__/get-resource-permissions.test.ts +++ b/packages/@n8n/permissions/src/utilities/__tests__/get-resource-permissions.test.ts @@ -31,6 +31,7 @@ describe('permissions', () => { workflow: {}, folder: {}, insights: {}, + dataStore: {}, }); }); it('getResourcePermissions', () => { @@ -128,6 +129,7 @@ describe('permissions', () => { insights: { list: true, }, + dataStore: {}, }; expect(getResourcePermissions(scopes)).toEqual(permissionRecord); diff --git a/packages/cli/src/modules/data-store/__tests__/data-store-aggregate.service.test.ts b/packages/cli/src/modules/data-store/__tests__/data-store-aggregate.service.test.ts new file mode 100644 index 0000000000..fbfbe77571 --- /dev/null +++ b/packages/cli/src/modules/data-store/__tests__/data-store-aggregate.service.test.ts @@ -0,0 +1,215 @@ +import { createTeamProject, testDb, testModules } from '@n8n/backend-test-utils'; +import { ProjectRelationRepository, type Project, type User } from '@n8n/db'; +import { Container } from '@n8n/di'; +import type { EntityManager } from '@n8n/typeorm'; +import { mock } from 'jest-mock-extended'; + +import { createUser } from '@test-integration/db/users'; + +import { DataStoreAggregateService } from '../data-store-aggregate.service'; +import { DataStoreService } from '../data-store.service'; + +beforeAll(async () => { + await testModules.loadModules(['data-store']); + await testDb.init(); +}); + +beforeEach(async () => { + await testDb.truncate(['DataStore', 'DataStoreColumn']); +}); + +afterAll(async () => { + await testDb.terminate(); +}); + +describe('dataStoreAggregate', () => { + let dataStoreService: DataStoreService; + let dataStoreAggregateService: DataStoreAggregateService; + const manager = mock(); + const projectRelationRepository = mock({ manager }); + + beforeAll(() => { + Container.set(ProjectRelationRepository, projectRelationRepository); + dataStoreAggregateService = Container.get(DataStoreAggregateService); + dataStoreService = Container.get(DataStoreService); + }); + + let user: User; + let project1: Project; + let project2: Project; + + beforeEach(async () => { + project1 = await createTeamProject(); + project2 = await createTeamProject(); + user = await createUser({ role: 'global:owner' }); + }); + + afterEach(async () => { + // Clean up any created user data stores + await dataStoreService.deleteDataStoreAll(); + }); + + describe('getManyAndCount', () => { + it('should return the correct data stores for the user', async () => { + // ARRANGE + const ds1 = await dataStoreService.createDataStore(project1.id, { + name: 'store1', + columns: [], + }); + const ds2 = await dataStoreService.createDataStore(project1.id, { + name: 'store2', + columns: [], + }); + + projectRelationRepository.find.mockResolvedValueOnce([ + { + userId: user.id, + projectId: project1.id, + role: 'project:admin', + user, + project: project1, + createdAt: new Date(), + updatedAt: new Date(), + setUpdateDate: jest.fn(), + }, + { + userId: user.id, + projectId: project2.id, + role: 'project:viewer', + user, + project: project2, + createdAt: new Date(), + updatedAt: new Date(), + setUpdateDate: jest.fn(), + }, + ]); + + await dataStoreService.createDataStore(project2.id, { + name: 'store3', + columns: [], + }); + + // ACT + const result = await dataStoreAggregateService.getManyAndCount(user, { + filter: { projectId: project1.id }, + skip: 0, + take: 10, + }); + + // ASSERT + expect(result.data).toEqual( + expect.arrayContaining([ + expect.objectContaining({ id: ds1!.id, name: ds1!.name }), + expect.objectContaining({ id: ds2!.id, name: ds2!.name }), + ]), + ); + expect(result.count).toBe(2); + }); + + it('should return an empty array if user has no access to any project', async () => { + // ARRANGE + const currentUser = await createUser({ role: 'global:member' }); + + await dataStoreService.createDataStore(project1.id, { + name: 'store1', + columns: [], + }); + projectRelationRepository.find.mockResolvedValueOnce([]); + + // ACT + const result = await dataStoreAggregateService.getManyAndCount(currentUser, { + skip: 0, + take: 10, + }); + + // ASSERT + expect(result.data).toEqual([]); + expect(result.count).toBe(0); + }); + + it('should return only the data store matching the given data store id filter', async () => { + // ARRANGE + await dataStoreService.createDataStore(project1.id, { + name: 'store1', + columns: [], + }); + const ds2 = await dataStoreService.createDataStore(project1.id, { + name: 'store2', + columns: [], + }); + projectRelationRepository.find.mockResolvedValueOnce([ + { + userId: user.id, + projectId: project1.id, + role: 'project:admin', + user, + project: project1, + createdAt: new Date(), + updatedAt: new Date(), + setUpdateDate: jest.fn(), + }, + { + userId: user.id, + projectId: project2.id, + role: 'project:viewer', + user, + project: project2, + createdAt: new Date(), + updatedAt: new Date(), + setUpdateDate: jest.fn(), + }, + ]); + + // ACT + const result = await dataStoreAggregateService.getManyAndCount(user, { + filter: { id: ds2!.id }, + skip: 0, + take: 10, + }); + + // ASSERT + expect(result.data).toEqual([expect.objectContaining({ id: ds2!.id, name: ds2!.name })]); + expect(result.count).toBe(1); + }); + + it('should respect pagination (skip/take)', async () => { + // ARRANGE + const ds1 = await dataStoreService.createDataStore(project1.id, { + name: 'store1', + columns: [], + }); + const ds2 = await dataStoreService.createDataStore(project1.id, { + name: 'store2', + columns: [], + }); + const ds3 = await dataStoreService.createDataStore(project1.id, { + name: 'store3', + columns: [], + }); + projectRelationRepository.find.mockResolvedValueOnce([ + { + userId: user.id, + projectId: project1.id, + role: 'project:admin', + user, + project: project1, + createdAt: new Date(), + updatedAt: new Date(), + setUpdateDate: jest.fn(), + }, + ]); + + // ACT + const result = await dataStoreAggregateService.getManyAndCount(user, { + filter: { projectId: project1.id }, + skip: 1, + take: 1, + }); + + // ASSERT + expect(result.data.length).toBe(1); + expect([ds1!.id, ds2!.id, ds3!.id]).toContain(result.data[0].id); + expect(result.count).toBe(3); + }); + }); +}); diff --git a/packages/cli/src/modules/data-store/__tests__/data-store.service.test.ts b/packages/cli/src/modules/data-store/__tests__/data-store.service.test.ts new file mode 100644 index 0000000000..6deb6b4c6a --- /dev/null +++ b/packages/cli/src/modules/data-store/__tests__/data-store.service.test.ts @@ -0,0 +1,1221 @@ +import type { AddDataStoreColumnDto, CreateDataStoreColumnDto } from '@n8n/api-types'; +import { createTeamProject, testDb, testModules } from '@n8n/backend-test-utils'; +import { Project } from '@n8n/db'; +import { Container } from '@n8n/di'; + +import { DataStoreRowsRepository } from '../data-store-rows.repository'; +import { DataStoreRepository } from '../data-store.repository'; +import { DataStoreService } from '../data-store.service'; +import { toTableName } from '../utils/sql-utils'; + +beforeAll(async () => { + await testModules.loadModules(['data-store']); + await testDb.init(); +}); + +beforeEach(async () => { + await testDb.truncate(['DataStore', 'DataStoreColumn']); +}); + +afterAll(async () => { + await testDb.terminate(); +}); + +describe('dataStore', () => { + let dataStoreService: DataStoreService; + let dataStoreRepository: DataStoreRepository; + let dataStoreRowsRepository: DataStoreRowsRepository; + + beforeAll(() => { + dataStoreService = Container.get(DataStoreService); + dataStoreRepository = Container.get(DataStoreRepository); + dataStoreRowsRepository = Container.get(DataStoreRowsRepository); + }); + + let project1: Project; + let project2: Project; + + beforeEach(async () => { + project1 = await createTeamProject(); + project2 = await createTeamProject(); + }); + + afterEach(async () => { + // Clean up any created user data stores + await dataStoreService.deleteDataStoreAll(); + }); + + describe('createDataStore', () => { + it('should succeed and not create a user table if columns are not provided', async () => { + const name = 'dataStore'; + + // ACT + const result = await dataStoreService.createDataStore(project1.id, { + name, + columns: [], + }); + const { id: dataStoreId } = result!; + + // ASSERT + expect(result).toEqual({ + columns: [], + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + createdAt: expect.any(Date), + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + id: expect.any(String), + name, + projectId: project1.id, + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + project: expect.any(Project), + sizeBytes: 0, + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + updatedAt: expect.any(Date), + }); + + const created = await dataStoreRepository.findOneBy({ name, projectId: project1.id }); + expect(created?.id).toBe(dataStoreId); + + const columns = await dataStoreRepository.manager.getRepository('DataStoreColumn').find({ + where: { dataStoreId }, + }); + expect(columns).toEqual([]); + + const userTableName = toTableName(dataStoreId); + await expect( + dataStoreRepository.manager + .createQueryBuilder() + .select() + .from(userTableName, userTableName) + .limit(1) + .getRawMany(), + ).rejects.toThrow(); + }); + + it('should succeed even if the name exists in a different project', async () => { + const name = 'dataStore'; + + await dataStoreService.createDataStore(project2.id, { + name, + columns: [], + }); + + // ACT + const result = await dataStoreService.createDataStore(project1.id, { + name, + columns: [], + }); + const { id: dataStoreId } = result!; + + const created = await dataStoreRepository.findOneBy({ name, projectId: project1.id }); + expect(created?.id).toBe(dataStoreId); + }); + + it('should create the user table and columns entity immediately if columns are provided', async () => { + const dataStore = await dataStoreService.createDataStore(project1.id, { + name: 'dataStoreWithColumns', + columns: [{ name: 'foo', type: 'string' }], + }); + const { id: dataStoreId } = dataStore!; + + await expect(dataStoreService.getColumns(dataStoreId)).resolves.toEqual([ + { + name: 'foo', + type: 'string', + index: 0, + dataStoreId, + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + id: expect.any(String), + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + createdAt: expect.any(Date), + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + updatedAt: expect.any(Date), + }, + ]); + + // Select the column from user table to check for its existence + const userTableName = toTableName(dataStoreId); + const rows = await dataStoreRepository.manager + .createQueryBuilder() + .select('foo') + .from(userTableName, userTableName) + .limit(1) + .getRawMany(); + + expect(rows).toEqual([]); + }); + + it('should populate the project relation when creating a data store', async () => { + const name = 'dataStore'; + + // ACT + const result = await dataStoreService.createDataStore(project1.id, { + name, + columns: [], + }); + + // ASSERT + const { project } = result!; + expect(project.id).toBe(project1.id); + expect(project.name).toBe(project1.name); + }); + + it('should return an error if name/project combination already exists', async () => { + const name = 'dataStore'; + + // ARRANGE + await dataStoreService.createDataStore(project1.id, { + name, + columns: [], + }); + + // ACT + const result = dataStoreService.createDataStore(project1.id, { + name, + columns: [], + }); + + // ASSERT + await expect(result).rejects.toThrow( + `Data store with name '${name}' already exists in this project`, + ); + }); + }); + + describe('updateDataStore', () => { + it('should succeed when renaming to an available name', async () => { + // ARRANGE + const dataStore = await dataStoreService.createDataStore(project1.id, { + name: 'myDataStore1', + columns: [], + }); + const { id: dataStoreId, updatedAt } = dataStore!; + + // ACT + // Wait to get second difference + await new Promise((resolve) => setTimeout(resolve, 1001)); + + const result = await dataStoreService.updateDataStore(dataStoreId, { name: 'aNewName' }); + + // ASSERT + expect(result).toEqual(true); + + const updated = await dataStoreRepository.findOneBy({ id: dataStoreId }); + expect(updated?.name).toBe('aNewName'); + expect(updated?.updatedAt.getTime()).toBeGreaterThan(updatedAt.getTime()); + }); + + it('should fail when renaming a non-existent data store', async () => { + // ACT + const result = dataStoreService.updateDataStore('this is not an id', { + name: 'aNewName', + }); + + // ASSERT + await expect(result).rejects.toThrow( + "Tried to rename non-existent data store 'this is not an id'", + ); + }); + + it('should fail when renaming to an empty name', async () => { + // ARRANGE + const dataStore = await dataStoreService.createDataStore(project1.id, { + name: 'myDataStore', + columns: [], + }); + const { id: dataStoreId } = dataStore!; + + // ACT + const result = dataStoreService.updateDataStore(dataStoreId, { name: '' }); + + // ASSERT + await expect(result).rejects.toThrow('Data store name must not be empty'); + }); + + it('should trim the name', async () => { + // ARRANGE + const dataStore = await dataStoreService.createDataStore(project1.id, { + name: 'myDataStore1', + columns: [], + }); + const { id: dataStoreId } = dataStore!; + + // ACT + const result = dataStoreService.updateDataStore(dataStoreId, { name: ' aNewName ' }); + + // ASSERT + await expect(result).resolves.toEqual(true); + + const updated = await dataStoreRepository.findOneBy({ id: dataStoreId }); + expect(updated?.name).toBe('aNewName'); + }); + + it('should fail when renaming to a taken name', async () => { + // ARRANGE + const name = 'myDataStoreOld'; + await dataStoreService.createDataStore(project1.id, { + name, + columns: [], + }); + + const dataStoreNew = await dataStoreService.createDataStore(project1.id, { + name: 'myDataStoreNew', + columns: [], + }); + const { id: dataStoreNewId } = dataStoreNew!; + + // ACT + const result = dataStoreService.updateDataStore(dataStoreNewId, { name }); + + // ASSERT + await expect(result).rejects.toThrow( + `The name '${name}' is already taken within this project`, + ); + }); + }); + + describe('deleteDataStore', () => { + it('should succeed with deleting a store', async () => { + // ARRANGE + const dataStore = await dataStoreService.createDataStore(project1.id, { + name: 'myDataStore1', + columns: [], + }); + const { id: dataStoreId } = dataStore!; + + // ACT + const result = await dataStoreService.deleteDataStore(dataStoreId); + const userTableName = toTableName(dataStoreId); + + // ASSERT + expect(result).toEqual(true); + + const deletedDatastore = await dataStoreRepository.findOneBy({ id: dataStoreId }); + expect(deletedDatastore).toBeNull(); + + const queryUserTable = dataStoreRepository.manager + .createQueryBuilder() + .select() + .from(userTableName, userTableName) + .getMany(); + await expect(queryUserTable).rejects.toBeDefined(); + }); + + it('should fail when deleting a non-existent id', async () => { + // ACT + const result = dataStoreService.deleteDataStore('this is not an id'); + + // ASSERT + await expect(result).rejects.toThrow( + "Tried to delete non-existent data store 'this is not an id'", + ); + }); + }); + + describe('addColumn', () => { + it('should succeed with adding columns to a non-empty table', async () => { + const existingColumns: CreateDataStoreColumnDto[] = [{ name: 'myColumn0', type: 'string' }]; + + const dataStore = await dataStoreService.createDataStore(project1.id, { + name: 'dataStoreWithColumns', + columns: existingColumns, + }); + const { id: dataStoreId } = dataStore!; + + const columns: AddDataStoreColumnDto[] = [ + { name: 'myColumn1', type: 'string' }, + { name: 'myColumn2', type: 'number' }, + { name: 'myColumn3', type: 'number' }, + { name: 'myColumn4', type: 'date' }, + ]; + for (const column of columns) { + // ACT + const result = await dataStoreService.addColumn(dataStoreId, column); + // ASSERT + expect(result).toMatchObject(column); + } + const columnResult = await dataStoreService.getColumns(dataStoreId); + expect(columnResult).toEqual([ + { + index: 0, + dataStoreId, + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + id: expect.any(String), + name: 'myColumn0', + type: 'string', + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + createdAt: expect.any(Date), + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + updatedAt: expect.any(Date), + }, + { + index: 1, + dataStoreId, + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + id: expect.any(String), + name: 'myColumn1', + type: 'string', + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + createdAt: expect.any(Date), + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + updatedAt: expect.any(Date), + }, + { + index: 2, + dataStoreId, + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + id: expect.any(String), + name: 'myColumn2', + type: 'number', + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + createdAt: expect.any(Date), + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + updatedAt: expect.any(Date), + }, + { + index: 3, + dataStoreId, + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + id: expect.any(String), + name: 'myColumn3', + type: 'number', + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + createdAt: expect.any(Date), + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + updatedAt: expect.any(Date), + }, + { + index: 4, + dataStoreId, + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + id: expect.any(String), + name: 'myColumn4', + type: 'date', + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + createdAt: expect.any(Date), + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + updatedAt: expect.any(Date), + }, + ]); + }); + + it('should create the user table on first addColumn if it does not exist', async () => { + // ARRANGE + const dataStore = await dataStoreService.createDataStore(project1.id, { + name: 'dataStore', + columns: [], + }); + const { id: dataStoreId } = dataStore!; + + // ACT + const result = await dataStoreService.addColumn(dataStoreId, { + name: 'foo', + type: 'string', + }); + + // ASSERT + expect(result).toMatchObject({ name: 'foo', type: 'string' }); + + const userTableName = toTableName(dataStoreId); + const rows = await dataStoreRepository.manager + .createQueryBuilder() + .select('foo') + .from(userTableName, userTableName) + .limit(1) + .getRawMany(); + + expect(rows).toEqual([]); + }); + + it('should fail with adding two columns of the same name', async () => { + // ARRANGE + const dataStore = await dataStoreService.createDataStore(project1.id, { + name: 'myDataStore', + columns: [ + { + name: 'myColumn1', + type: 'string', + }, + ], + }); + const { id: dataStoreId } = dataStore!; + + // ACT + const result = dataStoreService.addColumn(dataStoreId, { + name: 'myColumn1', + type: 'number', + }); + + // ASSERT + await expect(result).rejects.toThrow( + `column name 'myColumn1' already taken in data store '${dataStoreId}'`, + ); + }); + + it('should fail with adding column of non-existent table', async () => { + // ACT + const result = dataStoreService.addColumn('this is not an id', { + name: 'myColumn1', + type: 'number', + }); + + // ASSERT + await expect(result).rejects.toThrow( + "Tried to add column to non-existent data store 'this is not an id'", + ); + }); + }); + + describe('deleteColumn', () => { + it('should succeed with deleting a column', async () => { + // ARRANGE + const dataStore = await dataStoreService.createDataStore(project1.id, { + name: 'dataStore', + columns: [], + }); + const { id: dataStoreId } = dataStore!; + + const c1 = await dataStoreService.addColumn(dataStoreId, { + name: 'myColumn1', + type: 'string', + }); + const c2 = await dataStoreService.addColumn(dataStoreId, { + name: 'myColumn2', + type: 'number', + }); + + // ACT + const result = await dataStoreService.deleteColumn(dataStoreId, c1.id); + + // ASSERT + expect(result).toEqual(true); + + const columns = await dataStoreService.getColumns(dataStoreId); + expect(columns).toEqual([ + { + index: 0, + dataStoreId, + id: c2.id, + name: 'myColumn2', + type: 'number', + createdAt: c2.createdAt, + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + updatedAt: expect.any(Date), + }, + ]); + }); + + it('should fail when deleting unknown column', async () => { + // ARRANGE + const dataStore = await dataStoreService.createDataStore(project1.id, { + name: 'dataStore', + columns: [ + { + name: 'myColumn1', + type: 'string', + }, + ], + }); + const { id: dataStoreId } = dataStore!; + + // ACT + const result = dataStoreService.deleteColumn(dataStoreId, 'thisIsNotAnId'); + + // ASSERT + await expect(result).rejects.toThrow( + `Tried to delete column with name not present in data store '${dataStoreId}'`, + ); + }); + + it('should fail when deleting column from unknown table', async () => { + // ARRANGE + const dataStore = await dataStoreService.createDataStore(project1.id, { + name: 'dataStore', + columns: [], + }); + const c1 = await dataStoreService.addColumn(dataStore!.id, { + name: 'myColumn1', + type: 'string', + }); + + // ACT + const result = dataStoreService.deleteColumn('this is not an id', c1.id); + + // ASSERT + await expect(result).rejects.toThrow( + "Tried to delete column from non-existent data store 'this is not an id'", + ); + }); + }); + + describe('moveColumn', () => { + it('should succeed with moving a column', async () => { + // ARRANGE + const dataStore = await dataStoreService.createDataStore(project1.id, { + name: 'dataStore', + columns: [], + }); + const { id: dataStoreId } = dataStore!; + + const c1 = await dataStoreService.addColumn(dataStoreId, { + name: 'myColumn1', + type: 'string', + }); + const c2 = await dataStoreService.addColumn(dataStoreId, { + name: 'myColumn2', + type: 'number', + }); + + // ACT + const result = await dataStoreService.moveColumn(dataStoreId, c2.id, { + targetIndex: 0, + }); + + // ASSERT + expect(result).toEqual(true); + + const columns = await dataStoreService.getColumns(dataStoreId); + expect(columns).toMatchObject([ + { + index: 0, + dataStoreId, + id: c2.id, + name: 'myColumn2', + type: 'number', + }, + { + index: 1, + dataStoreId, + id: c1.id, + name: 'myColumn1', + type: 'string', + }, + ]); + }); + }); + + describe('getManyAndCount', () => { + it('should retrieve by name', async () => { + // ARRANGE + const dataStore = await dataStoreService.createDataStore(project1.id, { + name: 'dataStore', + columns: [], + }); + const { name } = dataStore!; + + // ACT + const result = await dataStoreService.getManyAndCount({ + filter: { projectId: project1.id, name }, + }); + + // ASSERT + expect(result.data).toHaveLength(1); + expect(result.data[0]).toEqual({ + ...dataStore, + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + project: expect.any(Project), + }); + expect(result.data[0].project).toEqual({ + icon: null, + id: project1.id, + name: project1.name, + type: project1.type, + }); + expect(result.count).toEqual(1); + }); + + it('should retrieve by ids', async () => { + // ARRANGE + const dataStore1 = await dataStoreService.createDataStore(project1.id, { + name: 'myDataStore1', + columns: [], + }); + const { id: dataStoreId1 } = dataStore1!; + + const dataStore2 = await dataStoreService.createDataStore(project1.id, { + name: 'myDataStore2', + columns: [], + }); + const { id: dataStoreId2 } = dataStore2!; + + // ACT + const result = await dataStoreService.getManyAndCount({ + filter: { projectId: project1.id, id: [dataStoreId1, dataStoreId2] }, + }); + + // ASSERT + expect(result.data).toHaveLength(2); + expect(result.data).toContainEqual({ + ...dataStore2, + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + project: expect.any(Project), + }); + expect(result.data).toContainEqual({ + ...dataStore1, + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + project: expect.any(Project), + }); + expect(result.count).toEqual(2); + }); + + it('should retrieve by projectId', async () => { + // ARRANGE + const dataStore = await dataStoreService.createDataStore(project1.id, { + name: 'myDataStore', + columns: [], + }); + const { name } = dataStore!; + const names = [name]; + for (let i = 0; i < 10; ++i) { + const ds = await dataStoreService.createDataStore(project1.id, { + name: `anotherDataStore${i}`, + columns: [], + }); + names.push(ds!.name); + } + + // ACT + const result = await dataStoreService.getManyAndCount({ + filter: { projectId: project1.id }, + }); + + // ASSERT + expect(result.data.map((x) => x.name).sort()).toEqual(names.sort()); + expect(result.count).toEqual(11); + }); + + it('should retrieve by id with pagination', async () => { + // ARRANGE + const dataStore = await dataStoreService.createDataStore(project1.id, { + name: 'myDataStore', + columns: [], + }); + const { name } = dataStore!; + const names = [name]; + + for (let i = 0; i < 10; ++i) { + const ds = await dataStoreService.createDataStore(project1.id, { + name: `anotherDataStore${i}`, + columns: [], + }); + names.push(ds!.name); + } + + // ACT + const p0 = await dataStoreService.getManyAndCount({ + filter: { projectId: project1.id }, + skip: 0, + take: 3, + }); + const p1 = await dataStoreService.getManyAndCount({ + filter: { projectId: project1.id }, + skip: 3, + take: 3, + }); + const rest = await dataStoreService.getManyAndCount({ + filter: { projectId: project1.id }, + skip: 6, + take: 10, + }); + + // ASSERT + expect(p0.count).toBe(11); + expect(p0.data).toHaveLength(3); + + expect(p1.count).toBe(11); + expect(p1.data).toHaveLength(3); + + expect(rest.count).toBe(11); + expect(rest.data).toHaveLength(5); + + expect( + p0.data + .concat(p1.data) + .concat(rest.data) + .map((x) => x.name) + .sort(), + ).toEqual(names.sort()); + }); + + it('correctly joins columns', async () => { + // ARRANGE + const columns: CreateDataStoreColumnDto[] = [ + { name: 'myColumn1', type: 'string' }, + { name: 'myColumn2', type: 'number' }, + { name: 'myColumn3', type: 'number' }, + { name: 'myColumn4', type: 'date' }, + ]; + const dataStore = await dataStoreService.createDataStore(project1.id, { + name: 'myDataStore', + columns, + }); + const { id: dataStoreId } = dataStore!; + + // ACT + const result = await dataStoreService.getManyAndCount({ + filter: { projectId: project1.id, id: dataStoreId }, + }); + + // ASSERT + expect(result.count).toEqual(1); + + const resultColumns = result.data[0].columns; + + expect(resultColumns).toEqual( + expect.arrayContaining([ + { + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + id: expect.any(String), + name: 'myColumn1', + type: 'string', + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + createdAt: expect.any(Date), + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + updatedAt: expect.any(Date), + }, + { + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + id: expect.any(String), + name: 'myColumn2', + type: 'number', + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + createdAt: expect.any(Date), + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + updatedAt: expect.any(Date), + }, + { + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + id: expect.any(String), + name: 'myColumn3', + type: 'number', + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + createdAt: expect.any(Date), + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + updatedAt: expect.any(Date), + }, + { + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + id: expect.any(String), + name: 'myColumn4', + type: 'date', + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + createdAt: expect.any(Date), + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + updatedAt: expect.any(Date), + }, + ]), + ); + + for (let i = 1; i < resultColumns.length; i++) { + expect(new Date(resultColumns[i].updatedAt).getTime()).toBeGreaterThanOrEqual( + new Date(resultColumns[i - 1].updatedAt).getTime(), + ); + } + }); + + describe('sorts as expected', () => { + it('sorts by name', async () => { + // ARRANGE + await dataStoreService.createDataStore(project1.id, { + name: 'ds2', + columns: [], + }); + + await dataStoreService.createDataStore(project1.id, { + name: 'ds1', + columns: [], + }); + + await dataStoreService.createDataStore(project1.id, { + name: 'ds3', + columns: [], + }); + + // ACT + const nameAsc = await dataStoreService.getManyAndCount({ + filter: { projectId: project1.id }, + sortBy: 'name:asc', + }); + const nameDesc = await dataStoreService.getManyAndCount({ + filter: { projectId: project1.id }, + sortBy: 'name:desc', + }); + + // ASSERT + expect(nameAsc.data.map((x) => x.name)).toEqual(['ds1', 'ds2', 'ds3']); + expect(nameDesc.data.map((x) => x.name)).toEqual(['ds3', 'ds2', 'ds1']); + }); + + it('sorts by createdAt', async () => { + // ARRANGE + await dataStoreService.createDataStore(project1.id, { + name: 'ds0', + columns: [], + }); + + // Wait to get seconds difference + await new Promise((resolve) => setTimeout(resolve, 1001)); + await dataStoreService.createDataStore(project1.id, { + name: 'ds1', + columns: [], + }); + + // Wait to get seconds difference + await new Promise((resolve) => setTimeout(resolve, 1001)); + await dataStoreService.createDataStore(project1.id, { + name: 'ds2', + columns: [], + }); + + // ACT + const createdAsc = await dataStoreService.getManyAndCount({ + filter: { projectId: project1.id }, + sortBy: 'createdAt:asc', + }); + const createdDesc = await dataStoreService.getManyAndCount({ + filter: { projectId: project1.id }, + sortBy: 'createdAt:desc', + }); + + // ASSERT + expect(createdAsc.data.map((x) => x.name)).toEqual(['ds0', 'ds1', 'ds2']); + expect(createdDesc.data.map((x) => x.name)).toEqual(['ds2', 'ds1', 'ds0']); + }); + + it('sorts by updatedAt', async () => { + // ARRANGE + const ds1 = await dataStoreService.createDataStore(project1.id, { + name: 'ds1', + columns: [], + }); + const { id: ds1Id } = ds1!; + + // Wait to get seconds difference + await new Promise((resolve) => setTimeout(resolve, 1001)); + await dataStoreService.createDataStore(project1.id, { + name: 'ds2', + columns: [], + }); + + // Wait to get seconds difference + await new Promise((resolve) => setTimeout(resolve, 1001)); + await dataStoreService.updateDataStore(ds1Id, { name: 'ds1Updated' }); + + // ACT + const updatedAsc = await dataStoreService.getManyAndCount({ + filter: { projectId: project1.id }, + sortBy: 'updatedAt:asc', + }); + + const updatedDesc = await dataStoreService.getManyAndCount({ + filter: { projectId: project1.id }, + sortBy: 'updatedAt:desc', + }); + + // ASSERT + expect(updatedAsc.data.map((x) => x.name)).toEqual(['ds2', 'ds1Updated']); + expect(updatedDesc.data.map((x) => x.name)).toEqual(['ds1Updated', 'ds2']); + }); + }); + }); + + describe('insertRows', () => { + it('inserts rows into an existing table', async () => { + // ARRANGE + const dataStore = await dataStoreService.createDataStore(project1.id, { + name: 'dataStore', + columns: [ + { name: 'c1', type: 'number' }, + { name: 'c2', type: 'boolean' }, + { name: 'c3', type: 'date' }, + { name: 'c4', type: 'string' }, + ], + }); + const { id: dataStoreId } = dataStore!; + + // ACT + const rows = [ + { c1: 3, c2: true, c3: new Date(), c4: 'hello?' }, + { c1: 4, c2: false, c3: new Date(), c4: 'hello!' }, + { c1: 5, c2: true, c3: new Date(), c4: 'hello.' }, + ]; + const result = await dataStoreService.insertRows(dataStoreId, rows); + + // ASSERT + expect(result).toBe(true); + + const { count, data } = await dataStoreService.getManyRowsAndCount(dataStoreId, {}); + expect(count).toEqual(3); + expect(data).toEqual( + rows.map((row, i) => ({ + ...row, + id: i + 1, + c1: row.c1, + c2: row.c2, + c3: row.c3 instanceof Date ? row.c3.toISOString() : row.c3, + c4: row.c4, + })), + ); + }); + + it('inserts a row even if it matches with the existing one', async () => { + // ARRANGE + const dataStore = await dataStoreService.createDataStore(project1.id, { + name: 'myDataStore', + columns: [ + { name: 'c1', type: 'number' }, + { name: 'c2', type: 'string' }, + ], + }); + const { id: dataStoreId } = dataStore!; + + // Insert initial row + await dataStoreService.insertRows(dataStoreId, [{ c1: 1, c2: 'foo' }]); + + // Attempt to insert a row with the same primary key + const result = await dataStoreService.insertRows(dataStoreId, [{ c1: 1, c2: 'foo' }]); + + // ASSERT + expect(result).toBe(true); + + const { count, data } = await dataStoreRowsRepository.getManyAndCount( + toTableName(dataStoreId), + {}, + ); + + expect(count).toEqual(2); + expect(data).toEqual([ + { c1: 1, c2: 'foo', id: 1 }, + { c1: 1, c2: 'foo', id: 2 }, + ]); + }); + + it('rejects a mismatched row with extra column', async () => { + // ARRANGE + const dataStore = await dataStoreService.createDataStore(project1.id, { + name: 'dataStore', + columns: [ + { name: 'c1', type: 'number' }, + { name: 'c2', type: 'boolean' }, + { name: 'c3', type: 'date' }, + { name: 'c4', type: 'string' }, + ], + }); + const { id: dataStoreId } = dataStore!; + + // ACT + const result = dataStoreService.insertRows(dataStoreId, [ + { c1: 3, c2: true, c3: new Date(), c4: 'hello?' }, + { cWrong: 3, c1: 4, c2: true, c3: new Date(), c4: 'hello?' }, + ]); + + // ASSERT + await expect(result).rejects.toThrow('mismatched key count'); + }); + + it('rejects a mismatched row with missing column', async () => { + // ARRANGE + const dataStore = await dataStoreService.createDataStore(project1.id, { + name: 'dataStore', + columns: [ + { name: 'c1', type: 'number' }, + { name: 'c2', type: 'boolean' }, + { name: 'c3', type: 'date' }, + { name: 'c4', type: 'string' }, + ], + }); + const { id: dataStoreId } = dataStore!; + + // ACT + const result = dataStoreService.insertRows(dataStoreId, [ + { c1: 3, c2: true, c3: new Date(), c4: 'hello?' }, + { c2: true, c3: new Date(), c4: 'hello?' }, + ]); + + // ASSERT + await expect(result).rejects.toThrow('mismatched key count'); + }); + + it('rejects a mismatched row with replaced column', async () => { + // ARRANGE + const dataStore = await dataStoreService.createDataStore(project1.id, { + name: 'dataStore', + columns: [ + { name: 'c1', type: 'number' }, + { name: 'c2', type: 'boolean' }, + { name: 'c3', type: 'date' }, + { name: 'c4', type: 'string' }, + ], + }); + const { id: dataStoreId } = dataStore!; + + // ACT + const result = dataStoreService.insertRows(dataStoreId, [ + { c1: 3, c2: true, c3: new Date(), c4: 'hello?' }, + { cWrong: 3, c2: true, c3: new Date(), c4: 'hello?' }, + ]); + + // ASSERT + await expect(result).rejects.toThrow('unknown column name'); + }); + + it('rejects unknown data store id', async () => { + // ARRANGE + await dataStoreService.createDataStore(project1.id, { + name: 'dataStore', + columns: [ + { name: 'c1', type: 'number' }, + { name: 'c2', type: 'boolean' }, + { name: 'c3', type: 'date' }, + { name: 'c4', type: 'string' }, + ], + }); + + // ACT + const result = dataStoreService.insertRows('this is not an id', [ + { c1: 3, c2: true, c3: new Date(), c4: 'hello?' }, + { cWrong: 3, c2: true, c3: new Date(), c4: 'hello?' }, + ]); + + // ASSERT + await expect(result).rejects.toThrow( + 'No columns found for this data store or data store not found', + ); + }); + + it('rejects on empty column list', async () => { + // ACT + const result = dataStoreService.insertRows('this is not an id', [{}, {}]); + + // ASSERT + await expect(result).rejects.toThrow( + 'No columns found for this data store or data store not found', + ); + }); + + it('fails on type mismatch', async () => { + // ARRANGE + const dataStore = await dataStoreService.createDataStore(project1.id, { + name: 'dataStore', + columns: [{ name: 'c1', type: 'number' }], + }); + const { id: dataStoreId } = dataStore!; + + // ACT + const result = dataStoreService.insertRows(dataStoreId, [{ c1: 3 }, { c1: true }]); + + // ASSERT + await expect(result).rejects.toThrow("value 'true' does not match column type 'number'"); + }); + }); + + describe('upsertRows', () => { + it('updates a row if filter matches', async () => { + // ARRANGE + const dataStore = await dataStoreService.createDataStore(project1.id, { + name: 'dataStore', + columns: [ + { name: 'pid', type: 'string' }, + { name: 'fullName', type: 'string' }, + { name: 'age', type: 'number' }, + ], + }); + const { id: dataStoreId } = dataStore!; + + // Insert initial row + await dataStoreService.insertRows(dataStoreId, [ + { pid: '1995-111a', fullName: 'Alice', age: 30 }, + ]); + + // ACT + const result = await dataStoreService.upsertRows(dataStoreId, { + rows: [{ pid: '1995-111a', fullName: 'Alicia', age: 31 }], + matchFields: ['pid'], + }); + + // ASSERT + expect(result).toBe(true); + + const { count, data } = await dataStoreRowsRepository.getManyAndCount( + toTableName(dataStoreId), + {}, + ); + + expect(count).toEqual(1); + expect(data).toEqual([{ fullName: 'Alicia', age: 31, id: 1, pid: '1995-111a' }]); + }); + + it('inserts a row if filter does not match', async () => { + // ARRANGE + const dataStore = await dataStoreService.createDataStore(project1.id, { + name: 'dataStore', + columns: [ + { name: 'pid', type: 'string' }, + { name: 'fullName', type: 'string' }, + { name: 'age', type: 'number' }, + ], + }); + const { id: dataStoreId } = dataStore!; + + // Insert initial row + await dataStoreService.insertRows(dataStoreId, [ + { pid: '1995-111a', fullName: 'Alice', age: 30 }, + ]); + + // ACT + const result = await dataStoreService.upsertRows(dataStoreId, { + rows: [{ pid: '1992-222b', fullName: 'Alice', age: 30 }], + matchFields: ['pid'], + }); + + // ASSERT + expect(result).toBe(true); + + const { count, data } = await dataStoreRowsRepository.getManyAndCount( + toTableName(dataStoreId), + {}, + ); + + expect(count).toEqual(2); + expect(data).toEqual([ + { fullName: 'Alice', age: 30, id: 1, pid: '1995-111a' }, + { fullName: 'Alice', age: 30, id: 2, pid: '1992-222b' }, + ]); + }); + }); + + describe('getManyRowsAndCount', () => { + it('retrieves rows correctly', async () => { + // ARRANGE + const dataStore = await dataStoreService.createDataStore(project1.id, { + name: 'dataStore', + columns: [ + { name: 'c1', type: 'number' }, + { name: 'c2', type: 'boolean' }, + { name: 'c3', type: 'date' }, + { name: 'c4', type: 'string' }, + ], + }); + const { id: dataStoreId } = dataStore!; + + const rows = [ + { c1: 3, c2: true, c3: new Date(0), c4: 'hello?' }, + { c1: 4, c2: false, c3: new Date(1), c4: 'hello!' }, + { c1: 5, c2: true, c3: new Date(2), c4: 'hello.' }, + ]; + + await dataStoreService.insertRows(dataStoreId, rows); + + // ACT + const result = await dataStoreService.getManyRowsAndCount(dataStoreId, {}); + + // ASSERT + expect(result.count).toEqual(3); + // Assuming IDs are auto-incremented starting from 1 + expect(result.data).toEqual([ + { c1: rows[0].c1, c2: rows[0].c2, c3: '1970-01-01T00:00:00.000Z', c4: rows[0].c4, id: 1 }, + { c1: rows[1].c1, c2: rows[1].c2, c3: '1970-01-01T00:00:00.001Z', c4: rows[1].c4, id: 2 }, + { c1: rows[2].c1, c2: rows[2].c2, c3: '1970-01-01T00:00:00.002Z', c4: rows[2].c4, id: 3 }, + ]); + }); + }); +}); diff --git a/packages/cli/src/modules/data-store/__tests__/sql-utils.test.ts b/packages/cli/src/modules/data-store/__tests__/sql-utils.test.ts new file mode 100644 index 0000000000..6f90885a5c --- /dev/null +++ b/packages/cli/src/modules/data-store/__tests__/sql-utils.test.ts @@ -0,0 +1,228 @@ +import type { DataStoreRows } from '@n8n/api-types'; + +import { + addColumnQuery, + deleteColumnQuery, + buildInsertQuery, + buildUpdateQuery, + splitRowsByExistence, +} from '../utils/sql-utils'; + +describe('sql-utils', () => { + describe('addColumnQuery', () => { + it('should generate a valid SQL query for adding columns to a table', () => { + const tableName = 'data_store_user_abc'; + const column = { name: 'email', type: 'number' as const }; + + const query = addColumnQuery(tableName, column, 'sqlite'); + + expect(query).toBe('ALTER TABLE "data_store_user_abc" ADD "email" FLOAT'); + }); + }); + + describe('deleteColumnQuery', () => { + it('should generate a valid SQL query for deleting columns from a table', () => { + const tableName = 'data_store_user_abc'; + const column = 'email'; + + const query = deleteColumnQuery(tableName, column, 'sqlite'); + + expect(query).toBe('ALTER TABLE "data_store_user_abc" DROP COLUMN "email"'); + }); + }); + + describe('buildInsertQuery', () => { + it('should generate a valid SQL query for inserting rows into a table', () => { + const tableName = 'data_store_user_abc'; + const columns = [ + { name: 'name', type: 'string' }, + { name: 'age', type: 'number' }, + ]; + const rows = [ + { name: 'Alice', age: 30 }, + { name: 'Bob', age: 25 }, + ]; + + const [query, parameters] = buildInsertQuery(tableName, rows, columns, 'postgres'); + + expect(query).toBe( + 'INSERT INTO "data_store_user_abc" ("name", "age") VALUES ($1, $2), ($3, $4)', + ); + expect(parameters).toEqual(['Alice', 30, 'Bob', 25]); + }); + + it('should return an empty query and parameters when rows are empty', () => { + const tableName = 'data_store_user_abc'; + const rows: [] = []; + + const [query, parameters] = buildInsertQuery(tableName, rows, []); + + expect(query).toBe(''); + expect(parameters).toEqual([]); + }); + + it('should return an empty query and parameters when rows have no keys', () => { + const tableName = 'data_store_user_abc'; + const rows = [{}]; + + const [query, parameters] = buildInsertQuery(tableName, rows, []); + + expect(query).toBe(''); + expect(parameters).toEqual([]); + }); + + it('should replace T and Z for MySQL', () => { + const tableName = 'data_store_user_abc'; + const columns = [{ name: 'participatedAt', type: 'date' }]; + const rows = [ + { participatedAt: new Date('2021-01-01') }, + { participatedAt: new Date('2021-01-02') }, + ]; + + const [query, parameters] = buildInsertQuery(tableName, rows, columns, 'mysql'); + + expect(query).toBe('INSERT INTO `data_store_user_abc` (`participatedAt`) VALUES (?), (?)'); + expect(parameters).toEqual(['2021-01-01 00:00:00.000', '2021-01-02 00:00:00.000']); + }); + }); + + describe('buildUpdateQuery', () => { + it('should generate a valid SQL update query with one match field', () => { + const tableName = 'data_store_user_abc'; + const row = { name: 'Alice', age: 30, city: 'Paris' }; + const columns = [ + { id: 1, name: 'name', type: 'string' }, + { id: 2, name: 'age', type: 'number' }, + { id: 3, name: 'city', type: 'string' }, + ]; + const matchFields = ['name']; + + const [query, parameters] = buildUpdateQuery(tableName, row, columns, matchFields); + + expect(query).toBe('UPDATE "data_store_user_abc" SET "age" = ?, "city" = ? WHERE "name" = ?'); + expect(parameters).toEqual([30, 'Paris', 'Alice']); + }); + + it('should generate a valid SQL update query with multiple match fields', () => { + const tableName = 'data_store_user_abc'; + const row = { name: 'Alice', age: 30, city: 'Paris' }; + const columns = [ + { id: 1, name: 'name', type: 'string' }, + { id: 2, name: 'age', type: 'number' }, + { id: 3, name: 'city', type: 'string' }, + ]; + const matchFields = ['name', 'city']; + + const [query, parameters] = buildUpdateQuery(tableName, row, columns, matchFields); + + expect(query).toBe( + 'UPDATE "data_store_user_abc" SET "age" = ? WHERE "name" = ? AND "city" = ?', + ); + expect(parameters).toEqual([30, 'Alice', 'Paris']); + }); + + it('should return empty query and parameters if row is empty', () => { + const tableName = 'data_store_user_abc'; + const row = {}; + const matchFields = ['id']; + + const [query, parameters] = buildUpdateQuery(tableName, row, [], matchFields); + + expect(query).toBe(''); + expect(parameters).toEqual([]); + }); + + it('should return empty query and parameters if matchFields is empty', () => { + const tableName = 'data_store_user_abc'; + const row = { name: 'Alice', age: 30 }; + const columns = [ + { id: 1, name: 'name', type: 'string' }, + { id: 2, name: 'age', type: 'number' }, + ]; + const matchFields: string[] = []; + + const [query, parameters] = buildUpdateQuery(tableName, row, columns, matchFields); + + expect(query).toBe(''); + expect(parameters).toEqual([]); + }); + + it('should return empty query and parameters if only matchFields are present in row', () => { + const tableName = 'data_store_user_abc'; + const row = { id: 1 }; + const matchFields = ['id']; + + const [query, parameters] = buildUpdateQuery(tableName, row, [], matchFields); + + expect(query).toBe(''); + expect(parameters).toEqual([]); + }); + }); + + describe('splitRowsByExistence', () => { + it('should correctly separate rows into insert and update based on matchFields', () => { + const existing = [ + { name: 'Alice', age: 30 }, + { name: 'Bob', age: 25 }, + ]; + const matchFields = ['name']; + const rows: DataStoreRows = [ + { name: 'Alice', age: 30 }, + { name: 'Bob', age: 26 }, + { name: 'Charlie', age: 35 }, + ]; + + const { rowsToInsert, rowsToUpdate } = splitRowsByExistence(existing, matchFields, rows); + + expect(rowsToUpdate).toEqual([ + { name: 'Alice', age: 30 }, + { name: 'Bob', age: 26 }, + ]); + expect(rowsToInsert).toEqual([{ name: 'Charlie', age: 35 }]); + }); + + it('should treat rows as new if matchFields combination does not exist', () => { + const existing = [{ name: 'Bob', city: 'Berlin' }]; + const matchFields = ['name', 'city']; + const rows: DataStoreRows = [{ name: 'Alice', city: 'Berlin' }]; + + const { rowsToInsert, rowsToUpdate } = splitRowsByExistence(existing, matchFields, rows); + + expect(rowsToUpdate).toEqual([]); + expect(rowsToInsert).toEqual([{ name: 'Alice', city: 'Berlin' }]); + }); + + it('should insert all rows if existing set is empty', () => { + const existing: Array> = []; + const matchFields = ['name']; + const rows: DataStoreRows = [{ name: 'Alice' }, { name: 'Bob' }]; + + const { rowsToInsert, rowsToUpdate } = splitRowsByExistence(existing, matchFields, rows); + + expect(rowsToUpdate).toEqual([]); + expect(rowsToInsert).toEqual(rows); + }); + + it('should update all rows if all keys match existing', () => { + const existing = [{ name: 'Alice' }, { name: 'Bob' }]; + const matchFields = ['name']; + const rows: DataStoreRows = [{ name: 'Alice' }, { name: 'Bob' }]; + + const { rowsToInsert, rowsToUpdate } = splitRowsByExistence(existing, matchFields, rows); + + expect(rowsToInsert).toEqual([]); + expect(rowsToUpdate).toEqual(rows); + }); + + it('should handle empty input rows', () => { + const existing = [{ name: 'Alice' }]; + const matchFields = ['name']; + const rows: DataStoreRows = []; + + const { rowsToInsert, rowsToUpdate } = splitRowsByExistence(existing, matchFields, rows); + + expect(rowsToInsert).toEqual([]); + expect(rowsToUpdate).toEqual([]); + }); + }); +}); diff --git a/packages/cli/src/modules/data-store/data-store-aggregate.controller.ts b/packages/cli/src/modules/data-store/data-store-aggregate.controller.ts new file mode 100644 index 0000000000..0e060c9157 --- /dev/null +++ b/packages/cli/src/modules/data-store/data-store-aggregate.controller.ts @@ -0,0 +1,20 @@ +import { ListDataStoreQueryDto } from '@n8n/api-types'; +import { AuthenticatedRequest } from '@n8n/db'; +import { Get, ProjectScope, Query, RestController } from '@n8n/decorators'; + +import { DataStoreAggregateService } from './data-store-aggregate.service'; + +@RestController('/data-stores-global') +export class DataStoreAggregateController { + constructor(private readonly dataStoreAggregateService: DataStoreAggregateService) {} + + @Get('/') + @ProjectScope('dataStore:list') + async listDataStores( + req: AuthenticatedRequest, + _res: Response, + @Query payload: ListDataStoreQueryDto, + ) { + return await this.dataStoreAggregateService.getManyAndCount(req.user, payload); + } +} diff --git a/packages/cli/src/modules/data-store/data-store-aggregate.service.ts b/packages/cli/src/modules/data-store/data-store-aggregate.service.ts new file mode 100644 index 0000000000..af89fb2c66 --- /dev/null +++ b/packages/cli/src/modules/data-store/data-store-aggregate.service.ts @@ -0,0 +1,42 @@ +import type { ListDataStoreQueryDto } from '@n8n/api-types'; +import { Logger } from '@n8n/backend-common'; +import { User } from '@n8n/db'; +import { Service } from '@n8n/di'; + +import { ProjectService } from '@/services/project.service.ee'; + +import { DataStoreRepository } from './data-store.repository'; + +@Service() +export class DataStoreAggregateService { + constructor( + private readonly dataStoreRepository: DataStoreRepository, + private readonly projectService: ProjectService, + private readonly logger: Logger, + ) { + this.logger = this.logger.scoped('data-store'); + } + async start() {} + async shutdown() {} + + async getManyAndCount(user: User, options: ListDataStoreQueryDto) { + const projects = await this.projectService.getProjectRelationsForUser(user); + let projectIds = projects.map((x) => x.projectId); + if (options.filter?.projectId) { + const mask = [options.filter?.projectId].flat(); + projectIds = projectIds.filter((x) => mask.includes(x)); + } + + if (projectIds.length === 0) { + return { count: 0, data: [] }; + } + + return await this.dataStoreRepository.getManyAndCount({ + ...options, + filter: { + ...options.filter, + projectId: projectIds, + }, + }); + } +} diff --git a/packages/cli/src/modules/data-store/data-store-column.entity.ts b/packages/cli/src/modules/data-store/data-store-column.entity.ts new file mode 100644 index 0000000000..1d9dd2945d --- /dev/null +++ b/packages/cli/src/modules/data-store/data-store-column.entity.ts @@ -0,0 +1,24 @@ +import { WithTimestampsAndStringId } from '@n8n/db'; +import { Column, Entity, Index, JoinColumn, ManyToOne } from '@n8n/typeorm'; + +import { type DataStore } from './data-store.entity'; + +@Entity() +@Index(['dataStoreId', 'name'], { unique: true }) +export class DataStoreColumn extends WithTimestampsAndStringId { + @Column() + dataStoreId: string; + + @Column() + name: string; + + @Column({ type: 'varchar' }) + type: 'string' | 'number' | 'boolean' | 'date'; + + @Column({ type: 'int' }) + index: number; + + @ManyToOne('DataStore', 'columns') + @JoinColumn({ name: 'dataStoreId' }) + dataStore: DataStore; +} diff --git a/packages/cli/src/modules/data-store/data-store-column.repository.ts b/packages/cli/src/modules/data-store/data-store-column.repository.ts new file mode 100644 index 0000000000..b500aab710 --- /dev/null +++ b/packages/cli/src/modules/data-store/data-store-column.repository.ts @@ -0,0 +1,135 @@ +import { DataStoreCreateColumnSchema } from '@n8n/api-types'; +import { Service } from '@n8n/di'; +import { DataSource, EntityManager, Repository } from '@n8n/typeorm'; +import { UserError } from 'n8n-workflow'; + +import { DataStoreColumn } from './data-store-column.entity'; +import { DataStoreRowsRepository } from './data-store-rows.repository'; + +@Service() +export class DataStoreColumnRepository extends Repository { + constructor( + dataSource: DataSource, + private dataStoreRowsRepository: DataStoreRowsRepository, + ) { + super(DataStoreColumn, dataSource.manager); + } + + async getColumns(rawDataStoreId: string, em?: EntityManager) { + const executor = em ?? this.manager; + const columns = await executor + .createQueryBuilder(DataStoreColumn, 'dsc') + .where('dsc.dataStoreId = :dataStoreId', { dataStoreId: rawDataStoreId }) + .getMany(); + + // Ensure columns are always returned in the correct order by index, + // since the database does not guarantee ordering and TypeORM does not preserve + // join order in @OneToMany relations. + columns.sort((a, b) => a.index - b.index); + + return columns; + } + + async addColumn(dataStoreId: string, schema: DataStoreCreateColumnSchema) { + return await this.manager.transaction(async (em) => { + const existingColumnMatch = await em.existsBy(DataStoreColumn, { + name: schema.name, + dataStoreId, + }); + + if (existingColumnMatch) { + throw new UserError( + `column name '${schema.name}' already taken in data store '${dataStoreId}'`, + ); + } + + if (schema.index === undefined) { + const columns = await this.getColumns(dataStoreId, em); + schema.index = columns.length; + } else { + await this.shiftColumns(dataStoreId, schema.index, 1, em); + } + + const column = em.create(DataStoreColumn, { + ...schema, + dataStoreId, + }); + + await em.insert(DataStoreColumn, column); + + const queryRunner = em.queryRunner; + if (!queryRunner) { + throw new Error('QueryRunner is not available'); + } + + await this.dataStoreRowsRepository.ensureTableAndAddColumn( + dataStoreId, + column, + queryRunner, + em.connection.options.type, + ); + + return column; + }); + } + + async deleteColumn(dataStoreId: string, column: DataStoreColumn) { + await this.manager.transaction(async (em) => { + await em.remove(DataStoreColumn, column); + await this.dataStoreRowsRepository.dropColumnFromTable( + dataStoreId, + column.name, + em, + em.connection.options.type, + ); + await this.shiftColumns(dataStoreId, column.index, -1, em); + }); + } + + async moveColumn(rawDataStoreId: string, columnId: string, targetIndex: number) { + await this.manager.transaction(async (em) => { + const columnCount = await em.countBy(DataStoreColumn, { dataStoreId: rawDataStoreId }); + + if (targetIndex >= columnCount) { + throw new UserError('tried to move column to index larger than column count'); + } + + if (targetIndex < 0) { + throw new UserError('tried to move column to negative index'); + } + + const existingColumn = await em.findOneBy(DataStoreColumn, { + id: columnId, + dataStoreId: rawDataStoreId, + }); + + if (existingColumn === null) { + throw new UserError(`tried to move column not present in data store '${rawDataStoreId}'`); + } + + await this.shiftColumns(rawDataStoreId, existingColumn.index, -1, em); + await this.shiftColumns(rawDataStoreId, targetIndex, 1, em); + await em.update(DataStoreColumn, { id: existingColumn.id }, { index: targetIndex }); + }); + } + + async shiftColumns( + rawDataStoreId: string, + lowestIndex: number, + delta: -1 | 1, + em?: EntityManager, + ) { + const executor = em ?? this.manager; + await executor + .createQueryBuilder() + .update(DataStoreColumn) + .set({ + index: () => `index + ${delta}`, + }) + .where('dataStoreId = :dataStoreId AND index >= :thresholdValue', { + dataStoreId: rawDataStoreId, + thresholdValue: lowestIndex, + }) + .execute(); + } +} diff --git a/packages/cli/src/modules/data-store/data-store-rows.repository.ts b/packages/cli/src/modules/data-store/data-store-rows.repository.ts new file mode 100644 index 0000000000..5a8f30930c --- /dev/null +++ b/packages/cli/src/modules/data-store/data-store-rows.repository.ts @@ -0,0 +1,238 @@ +import type { + ListDataStoreContentQueryDto, + ListDataStoreContentFilter, + DataStoreUserTableName, + DataStoreRows, + UpsertDataStoreRowsDto, +} from '@n8n/api-types'; +import { CreateTable, DslColumn } from '@n8n/db'; +import { Service } from '@n8n/di'; +import { + DataSource, + DataSourceOptions, + EntityManager, + QueryRunner, + SelectQueryBuilder, +} from '@n8n/typeorm'; + +import { DataStoreColumn } from './data-store-column.entity'; +import { + addColumnQuery, + buildInsertQuery, + buildUpdateQuery, + deleteColumnQuery, + getPlaceholder, + quoteIdentifier, + splitRowsByExistence, + toDslColumns, + toTableName, +} from './utils/sql-utils'; + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +type QueryBuilder = SelectQueryBuilder; + +function getConditionAndParams( + filter: ListDataStoreContentFilter['filters'][number], + index: number, + dbType: DataSourceOptions['type'], +): [string, Record] { + const paramName = `filter_${index}`; + const column = `dataStore.${quoteIdentifier(filter.columnName, dbType)}`; + + switch (filter.condition) { + case 'eq': + return [`${column} = :${paramName}`, { [paramName]: filter.value }]; + case 'neq': + return [`${column} != :${paramName}`, { [paramName]: filter.value }]; + } +} + +@Service() +export class DataStoreRowsRepository { + constructor(private dataSource: DataSource) {} + + // TypeORM cannot infer the columns for a dynamic table name, so we use a raw query + async insertRows( + tableName: DataStoreUserTableName, + rows: DataStoreRows, + columns: DataStoreColumn[], + ) { + const dbType = this.dataSource.options.type; + await this.dataSource.query.apply( + this.dataSource, + buildInsertQuery(tableName, rows, columns, dbType), + ); + return true; + } + + async upsertRows( + tableName: DataStoreUserTableName, + dto: UpsertDataStoreRowsDto, + columns: DataStoreColumn[], + ) { + const dbType = this.dataSource.options.type; + const { rows, matchFields } = dto; + + if (rows.length === 0) { + return false; + } + + const { rowsToInsert, rowsToUpdate } = await this.fetchAndSplitRowsByExistence( + tableName, + matchFields, + rows, + ); + + if (rowsToInsert.length > 0) { + await this.insertRows(tableName, rowsToInsert, columns); + } + + if (rowsToUpdate.length > 0) { + for (const row of rowsToUpdate) { + // TypeORM cannot infer the columns for a dynamic table name, so we use a raw query + const [query, parameters] = buildUpdateQuery(tableName, row, columns, matchFields, dbType); + await this.dataSource.query(query, parameters); + } + } + + return true; + } + + async createTableWithColumns( + tableName: string, + columns: DataStoreColumn[], + queryRunner: QueryRunner, + ) { + const dslColumns = [new DslColumn('id').int.autoGenerate2.primary, ...toDslColumns(columns)]; + const createTable = new CreateTable(tableName, '', queryRunner); + createTable.withColumns.apply(createTable, dslColumns); + await createTable.execute(queryRunner); + } + + async ensureTableAndAddColumn( + dataStoreId: string, + column: DataStoreColumn, + queryRunner: QueryRunner, + dbType: DataSourceOptions['type'], + ) { + const tableName = toTableName(dataStoreId); + const tableExists = await queryRunner.hasTable(tableName); + if (!tableExists) { + await this.createTableWithColumns(tableName, [column], queryRunner); + } else { + await queryRunner.manager.query(addColumnQuery(tableName, column, dbType)); + } + } + + async dropColumnFromTable( + dataStoreId: string, + columnName: string, + em: EntityManager, + dbType: DataSourceOptions['type'], + ) { + await em.query(deleteColumnQuery(toTableName(dataStoreId), columnName, dbType)); + } + + async getManyAndCount(dataStoreId: DataStoreUserTableName, dto: ListDataStoreContentQueryDto) { + const [countQuery, query] = this.getManyQuery(dataStoreId, dto); + const data: Array> = await query.select('*').getRawMany(); + const countResult = await countQuery.select('COUNT(*) as count').getRawOne<{ + count: number | string | null; + }>(); + const count = + typeof countResult?.count === 'number' ? countResult.count : Number(countResult?.count) || 0; + return { count: count ?? -1, data }; + } + + async getRowIds(dataStoreId: DataStoreUserTableName, dto: ListDataStoreContentQueryDto) { + const [_, query] = this.getManyQuery(dataStoreId, dto); + const result = await query.select('dataStore.id').getRawMany(); + return result; + } + + private getManyQuery( + dataStoreTableName: DataStoreUserTableName, + dto: ListDataStoreContentQueryDto, + ): [QueryBuilder, QueryBuilder] { + const query = this.dataSource.createQueryBuilder(); + + query.from(dataStoreTableName, 'dataStore'); + this.applyFilters(query, dto); + const countQuery = query.clone().select('COUNT(*)'); + this.applySorting(query, dto); + this.applyPagination(query, dto); + + return [countQuery, query]; + } + + private applyFilters(query: QueryBuilder, dto: ListDataStoreContentQueryDto): void { + const filters = dto.filter?.filters ?? []; + const filterType = dto.filter?.type ?? 'and'; + + const dbType = this.dataSource.options.type; + const conditionsAndParams = filters.map((filter, i) => + getConditionAndParams(filter, i, dbType), + ); + + for (const [condition, params] of conditionsAndParams) { + if (filterType === 'or') { + query.orWhere(condition, params); + } else { + query.andWhere(condition, params); + } + } + } + + private applySorting(query: QueryBuilder, dto: ListDataStoreContentQueryDto): void { + if (!dto.sortBy) { + // query.orderBy('dataStore.', 'DESC'); + return; + } + + const [field, order] = dto.sortBy; + this.applySortingByField(query, field, order); + } + + private applySortingByField(query: QueryBuilder, field: string, direction: 'DESC' | 'ASC'): void { + const dbType = this.dataSource.options.type; + const quotedField = `dataStore.${quoteIdentifier(field, dbType)}`; + query.orderBy(quotedField, direction); + } + + private applyPagination(query: QueryBuilder, dto: ListDataStoreContentQueryDto): void { + query.skip(dto.skip); + query.take(dto.take); + } + + private async fetchAndSplitRowsByExistence( + tableName: string, + matchFields: string[], + rows: DataStoreRows, + ): Promise<{ rowsToInsert: DataStoreRows; rowsToUpdate: DataStoreRows }> { + const dbType = this.dataSource.options.type; + const whereClauses: string[] = []; + const params: unknown[] = []; + + for (const row of rows) { + const clause = matchFields + .map((field) => { + params.push(row[field]); + return `${quoteIdentifier(field, dbType)} = ${getPlaceholder(params.length, dbType)}`; + }) + .join(' AND '); + whereClauses.push(`(${clause})`); + } + + const quotedFields = matchFields.map((field) => quoteIdentifier(field, dbType)).join(', '); + const quotedTableName = quoteIdentifier(tableName, dbType); + + const query = ` + SELECT ${quotedFields} + FROM ${quotedTableName} + WHERE ${whereClauses.join(' OR ')} + `; + const existing: Array> = await this.dataSource.query(query, params); + + return splitRowsByExistence(existing, matchFields, rows); + } +} diff --git a/packages/cli/src/modules/data-store/data-store.controller.ts b/packages/cli/src/modules/data-store/data-store.controller.ts new file mode 100644 index 0000000000..0901638dcc --- /dev/null +++ b/packages/cli/src/modules/data-store/data-store.controller.ts @@ -0,0 +1,151 @@ +import { + AddDataStoreRowsDto, + AddDataStoreColumnDto, + CreateDataStoreDto, + ListDataStoreContentQueryDto, + ListDataStoreQueryDto, + MoveDataStoreColumnDto, + UpdateDataStoreDto, + UpsertDataStoreRowsDto, +} from '@n8n/api-types'; +import { AuthenticatedRequest } from '@n8n/db'; +import { + Body, + Delete, + Get, + Param, + Patch, + Post, + ProjectScope, + Query, + RestController, +} from '@n8n/decorators'; + +import { DataStoreService } from './data-store.service'; + +@RestController('/projects/:projectId/data-stores') +export class DataStoreController { + constructor(private readonly dataStoreService: DataStoreService) {} + + @Post('/') + @ProjectScope('dataStore:create') + async createDataStore( + req: AuthenticatedRequest<{ projectId: string }>, + _res: Response, + @Body dto: CreateDataStoreDto, + ) { + return await this.dataStoreService.createDataStore(req.params.projectId, dto); + } + + @Get('/') + @ProjectScope('dataStore:list') + async listDataStores( + req: AuthenticatedRequest<{ projectId: string }>, + _res: Response, + @Query payload: ListDataStoreQueryDto, + ) { + const providedFilter = payload?.filter ?? {}; + return await this.dataStoreService.getManyAndCount({ + ...payload, + filter: { ...providedFilter, projectId: req.params.projectId }, + }); + } + + @Patch('/:dataStoreId') + @ProjectScope('dataStore:update') + async updateDataStore( + _req: AuthenticatedRequest<{ projectId: string }>, + _res: Response, + @Param('dataStoreId') dataStoreId: string, + @Body dto: UpdateDataStoreDto, + ) { + return await this.dataStoreService.updateDataStore(dataStoreId, dto); + } + + @Delete('/:dataStoreId') + @ProjectScope('dataStore:delete') + async deleteDataStore( + _req: AuthenticatedRequest<{ projectId: string }>, + _res: Response, + @Param('dataStoreId') dataStoreId: string, + ) { + return await this.dataStoreService.deleteDataStore(dataStoreId); + } + + @Get('/:dataStoreId/columns') + @ProjectScope('dataStore:read') + async getColumns( + _req: AuthenticatedRequest<{ projectId: string }>, + _res: Response, + @Param('dataStoreId') dataStoreId: string, + ) { + return await this.dataStoreService.getColumns(dataStoreId); + } + + @Post('/:dataStoreId/columns') + @ProjectScope('dataStore:update') + async addColumn( + _req: AuthenticatedRequest<{ projectId: string }>, + _res: Response, + @Param('dataStoreId') dataStoreId: string, + @Body dto: AddDataStoreColumnDto, + ) { + return await this.dataStoreService.addColumn(dataStoreId, dto); + } + + @Delete('/:dataStoreId/columns/:columnId') + @ProjectScope('dataStore:update') + async deleteColumn( + _req: AuthenticatedRequest<{ projectId: string }>, + _res: Response, + @Param('dataStoreId') dataStoreId: string, + @Param('columnId') columnId: string, + ) { + return await this.dataStoreService.deleteColumn(dataStoreId, columnId); + } + + @Patch('/:dataStoreId/columns/:columnId/move') + @ProjectScope('dataStore:update') + async moveColumn( + _req: AuthenticatedRequest<{ projectId: string }>, + _res: Response, + @Param('dataStoreId') dataStoreId: string, + @Param('columnId') columnId: string, + @Body dto: MoveDataStoreColumnDto, + ) { + return await this.dataStoreService.moveColumn(dataStoreId, columnId, dto); + } + + @Get('/:dataStoreId/rows') + @ProjectScope('dataStore:readRow') + async getDataStoreRows( + _req: AuthenticatedRequest<{ projectId: string }>, + _res: Response, + @Param('dataStoreId') dataStoreId: string, + @Query dto: ListDataStoreContentQueryDto, + ) { + return await this.dataStoreService.getManyRowsAndCount(dataStoreId, dto); + } + + @Post('/:dataStoreId/insert') + @ProjectScope('dataStore:writeRow') + async appendDataStoreRows( + _req: AuthenticatedRequest<{ projectId: string }>, + _res: Response, + @Param('dataStoreId') dataStoreId: string, + @Body dto: AddDataStoreRowsDto, + ) { + return await this.dataStoreService.insertRows(dataStoreId, dto.data); + } + + @Post('/:dataStoreId/upsert') + @ProjectScope('dataStore:writeRow') + async upsertDataStoreRows( + _req: AuthenticatedRequest<{ projectId: string }>, + _res: Response, + @Param('dataStoreId') dataStoreId: string, + @Body dto: UpsertDataStoreRowsDto, + ) { + return await this.dataStoreService.upsertRows(dataStoreId, dto); + } +} diff --git a/packages/cli/src/modules/data-store/data-store.entity.ts b/packages/cli/src/modules/data-store/data-store.entity.ts new file mode 100644 index 0000000000..4af77dd02e --- /dev/null +++ b/packages/cli/src/modules/data-store/data-store.entity.ts @@ -0,0 +1,34 @@ +import { Project, WithTimestampsAndStringId } from '@n8n/db'; +import { Column, Entity, Index, JoinColumn, ManyToOne, OneToMany } from '@n8n/typeorm'; + +import { DataStoreColumn } from './data-store-column.entity'; + +@Entity() +@Index(['name', 'projectId'], { unique: true }) +export class DataStore extends WithTimestampsAndStringId { + constructor() { + super(); + } + + @Column() + name: string; + + @OneToMany( + () => DataStoreColumn, + (dataStoreColumn) => dataStoreColumn.dataStore, + { + cascade: true, + }, + ) + columns: DataStoreColumn[]; + + @ManyToOne(() => Project) + @JoinColumn({ name: 'projectId' }) + project: Project; + + @Column() + projectId: string; + + @Column({ type: 'int', default: 0 }) + sizeBytes: number; +} diff --git a/packages/cli/src/modules/data-store/data-store.module.ts b/packages/cli/src/modules/data-store/data-store.module.ts new file mode 100644 index 0000000000..87e4419bf0 --- /dev/null +++ b/packages/cli/src/modules/data-store/data-store.module.ts @@ -0,0 +1,34 @@ +import type { ModuleInterface } from '@n8n/decorators'; +import { BackendModule, OnShutdown } from '@n8n/decorators'; +import { Container } from '@n8n/di'; +import { BaseEntity } from '@n8n/typeorm'; + +@BackendModule({ name: 'data-store' }) +export class DataStoreModule implements ModuleInterface { + async init() { + await import('./data-store.controller'); + await import('./data-store-aggregate.controller'); + + const { DataStoreService } = await import('./data-store.service'); + await Container.get(DataStoreService).start(); + + const { DataStoreAggregateService } = await import('./data-store-aggregate.service'); + await Container.get(DataStoreAggregateService).start(); + } + + @OnShutdown() + async shutdown() { + const { DataStoreService } = await import('./data-store.service'); + await Container.get(DataStoreService).shutdown(); + + const { DataStoreAggregateService } = await import('./data-store-aggregate.service'); + await Container.get(DataStoreAggregateService).start(); + } + + async entities() { + const { DataStore } = await import('./data-store.entity'); + const { DataStoreColumn } = await import('./data-store-column.entity'); + + return [DataStore, DataStoreColumn] as unknown as Array BaseEntity>; + } +} diff --git a/packages/cli/src/modules/data-store/data-store.repository.ts b/packages/cli/src/modules/data-store/data-store.repository.ts new file mode 100644 index 0000000000..2d3b5a5e59 --- /dev/null +++ b/packages/cli/src/modules/data-store/data-store.repository.ts @@ -0,0 +1,235 @@ +import { + DATA_STORE_COLUMN_REGEX, + type DataStoreCreateColumnSchema, + type ListDataStoreQueryDto, +} from '@n8n/api-types'; +import { Service } from '@n8n/di'; +import { DataSource, EntityManager, Repository, SelectQueryBuilder } from '@n8n/typeorm'; +import { UnexpectedError } from 'n8n-workflow'; + +import { DataStoreColumn } from './data-store-column.entity'; +import { DataStoreRowsRepository } from './data-store-rows.repository'; +import { DataStore } from './data-store.entity'; +import { toTableName } from './utils/sql-utils'; + +@Service() +export class DataStoreRepository extends Repository { + constructor( + dataSource: DataSource, + private dataStoreRowsRepository: DataStoreRowsRepository, + ) { + super(DataStore, dataSource.manager); + } + + async createDataStore(projectId: string, name: string, columns: DataStoreCreateColumnSchema[]) { + if (columns.some((c) => !DATA_STORE_COLUMN_REGEX.test(c.name))) { + throw new UnexpectedError('bad column name'); + } + + let dataStoreId: string | undefined; + await this.manager.transaction(async (em) => { + const dataStore = em.create(DataStore, { name, columns, projectId }); + await em.insert(DataStore, dataStore); + dataStoreId = dataStore.id; + + const tableName = toTableName(dataStore.id); + const queryRunner = em.queryRunner; + if (!queryRunner) { + throw new UnexpectedError('QueryRunner is not available'); + } + + if (columns.length === 0) { + return; + } + + // insert columns + const columnEntities = columns.map((col, index) => + em.create(DataStoreColumn, { + name: col.name, + type: col.type, + dataStoreId: dataStore.id, + index: col.index ?? index, + }), + ); + await em.insert(DataStoreColumn, columnEntities); + + // create user table + await this.dataStoreRowsRepository.createTableWithColumns( + tableName, + columnEntities, + queryRunner, + ); + }); + + if (!dataStoreId) { + throw new UnexpectedError('Data store creation failed'); + } + + return await this.findOne({ + where: { id: dataStoreId }, + relations: ['project', 'columns'], + }); + } + + async deleteDataStore(dataStoreId: string, entityManager?: EntityManager) { + const executor = entityManager ?? this.manager; + return await executor.transaction(async (em) => { + const queryRunner = em.queryRunner; + if (!queryRunner) { + throw new UnexpectedError('QueryRunner is not available'); + } + + await em.delete(DataStore, { id: dataStoreId }); + await queryRunner.dropTable(toTableName(dataStoreId), true); + + return true; + }); + } + + async deleteDataStoreByProjectId(projectId: string) { + return await this.manager.transaction(async (em) => { + const existingTables = await em.findBy(DataStore, { projectId }); + + let changed = false; + for (const match of existingTables) { + const result = await this.deleteDataStore(match.id, em); + changed = changed || result; + } + + return changed; + }); + } + + async deleteDataStoreAll() { + return await this.manager.transaction(async (em) => { + const queryRunner = em.queryRunner; + if (!queryRunner) { + throw new UnexpectedError('QueryRunner is not available'); + } + + const existingTables = await em.findBy(DataStore, {}); + + let changed = false; + for (const match of existingTables) { + const result = await em.delete(DataStore, { id: match.id }); + await queryRunner.dropTable(toTableName(match.id), true); + changed = changed || (result.affected ?? 0) > 0; + } + + return changed; + }); + } + + async getManyAndCount(options: Partial) { + const query = this.getManyQuery(options); + const [data, count] = await query.getManyAndCount(); + return { count, data }; + } + + async getMany(options: Partial) { + const query = this.getManyQuery(options); + return await query.getMany(); + } + + private getManyQuery(options: Partial): SelectQueryBuilder { + const query = this.createQueryBuilder('dataStore'); + + this.applySelections(query); + this.applyFilters(query, options.filter); + this.applySorting(query, options.sortBy); + this.applyPagination(query, options); + + return query; + } + + private applySelections(query: SelectQueryBuilder): void { + this.applyDefaultSelect(query); + } + + private applyFilters( + query: SelectQueryBuilder, + filter: Partial['filter'], + ): void { + for (const x of ['id', 'projectId'] as const) { + const content = [filter?.[x]].flat().filter((x) => x !== undefined); + if (content.length === 0) continue; + + query.andWhere(`dataStore.${x} IN (:...${x}s)`, { + /* + * If list is empty, add a dummy value to prevent an error + * when using the IN operator with an empty array. + */ + [x + 's']: content.length > 0 ? content : [''], + }); + } + + if (filter?.name && typeof filter.name === 'string') { + query.andWhere('LOWER(dataStore.name) LIKE LOWER(:name)', { + name: `%${filter.name}%`, + }); + } + } + + private applySorting(query: SelectQueryBuilder, sortBy?: string): void { + if (!sortBy) { + query.orderBy('dataStore.updatedAt', 'DESC'); + return; + } + + const [field, order] = this.parseSortingParams(sortBy); + this.applySortingByField(query, field, order); + } + + private parseSortingParams(sortBy: string): [string, 'DESC' | 'ASC'] { + const [field, order] = sortBy.split(':'); + return [field, order?.toLowerCase() === 'desc' ? 'DESC' : 'ASC']; + } + + private applySortingByField( + query: SelectQueryBuilder, + field: string, + direction: 'DESC' | 'ASC', + ): void { + if (field === 'name') { + query.orderBy('LOWER(dataStore.name)', direction); + } else if (['createdAt', 'updatedAt'].includes(field)) { + query.orderBy(`dataStore.${field}`, direction); + } + } + + private applyPagination( + query: SelectQueryBuilder, + options: Partial, + ): void { + query.skip(options.skip ?? 0); + if (options?.take) { + query.skip(options.skip ?? 0).take(options.take); + } + } + + private applyDefaultSelect(query: SelectQueryBuilder): void { + query + .leftJoinAndSelect('dataStore.project', 'project') + .leftJoinAndSelect('dataStore.columns', 'data_store_column') + .select([ + 'dataStore', + ...this.getDataStoreColumnFields('data_store_column'), + ...this.getProjectFields('project'), + ]) + .addOrderBy('data_store_column.index', 'ASC'); + } + + private getDataStoreColumnFields(alias: string): string[] { + return [ + `${alias}.id`, + `${alias}.name`, + `${alias}.type`, + `${alias}.createdAt`, + `${alias}.updatedAt`, + ]; + } + + private getProjectFields(alias: string): string[] { + return [`${alias}.id`, `${alias}.name`, `${alias}.type`, `${alias}.icon`]; + } +} diff --git a/packages/cli/src/modules/data-store/data-store.service.ts b/packages/cli/src/modules/data-store/data-store.service.ts new file mode 100644 index 0000000000..ee2bea41d4 --- /dev/null +++ b/packages/cli/src/modules/data-store/data-store.service.ts @@ -0,0 +1,265 @@ +import type { + AddDataStoreColumnDto, + CreateDataStoreDto, + ListDataStoreContentQueryDto, + MoveDataStoreColumnDto, + DataStoreListOptions, + DataStoreRows, + UpsertDataStoreRowsDto, +} from '@n8n/api-types'; +import { UpdateDataStoreDto } from '@n8n/api-types/src/dto/data-store/update-data-store.dto'; +import { Logger } from '@n8n/backend-common'; +import { Service } from '@n8n/di'; +import { UserError } from 'n8n-workflow'; + +import { DataStoreColumn } from './data-store-column.entity'; +import { DataStoreColumnRepository } from './data-store-column.repository'; +import { DataStoreRowsRepository } from './data-store-rows.repository'; +import { DataStoreRepository } from './data-store.repository'; +import { toTableName } from './utils/sql-utils'; + +@Service() +export class DataStoreService { + constructor( + private readonly dataStoreRepository: DataStoreRepository, + private readonly dataStoreColumnRepository: DataStoreColumnRepository, + private readonly dataStoreRowsRepository: DataStoreRowsRepository, + private readonly logger: Logger, + ) { + this.logger = this.logger.scoped('data-store'); + } + + async start() {} + async shutdown() {} + + async createDataStore(projectId: string, dto: CreateDataStoreDto) { + const existingTable = await this.dataStoreRepository.findOneBy({ + name: dto.name, + projectId, + }); + if (existingTable !== null) { + throw new UserError(`Data store with name '${dto.name}' already exists in this project`); + } + return await this.dataStoreRepository.createDataStore(projectId, dto.name, dto.columns); + } + + // Currently only renames data stores + async updateDataStore(dataStoreId: string, dto: UpdateDataStoreDto) { + const name = dto.name.trim(); + + if (!name) { + throw new UserError('Data store name must not be empty'); + } + + const existingTable = await this.dataStoreRepository.findOneBy({ + id: dataStoreId, + }); + + if (existingTable === null) { + throw new UserError(`Tried to rename non-existent data store '${dataStoreId}'`); + } + + const hasNameClash = await this.dataStoreRepository.existsBy({ + name, + projectId: existingTable.projectId, + }); + + if (hasNameClash) { + throw new UserError(`The name '${name}' is already taken within this project`); + } + + await this.dataStoreRepository.update({ id: dataStoreId }, { name }); + + return true; + } + + async deleteDataStoreByProjectId(projectId: string) { + return await this.dataStoreRepository.deleteDataStoreByProjectId(projectId); + } + + async deleteDataStoreAll() { + return await this.dataStoreRepository.deleteDataStoreAll(); + } + + async deleteDataStore(dataStoreId: string) { + await this.validateDataStoreExists( + dataStoreId, + `Tried to delete non-existent data store '${dataStoreId}'`, + ); + + await this.dataStoreRepository.deleteDataStore(dataStoreId); + + return true; + } + + async addColumn(dataStoreId: string, dto: AddDataStoreColumnDto) { + await this.validateDataStoreExists( + dataStoreId, + `Tried to add column to non-existent data store '${dataStoreId}'`, + ); + + return await this.dataStoreColumnRepository.addColumn(dataStoreId, dto); + } + + async moveColumn(dataStoreId: string, columnId: string, dto: MoveDataStoreColumnDto) { + await this.validateDataStoreExists( + dataStoreId, + `Tried to move column from non-existent data store '${dataStoreId}'`, + ); + + await this.dataStoreColumnRepository.moveColumn(dataStoreId, columnId, dto.targetIndex); + + return true; + } + + async deleteColumn(dataStoreId: string, columnId: string) { + await this.validateDataStoreExists( + dataStoreId, + `Tried to delete column from non-existent data store '${dataStoreId}'`, + ); + + const existingColumnMatch = await this.dataStoreColumnRepository.findOneBy({ + id: columnId, + dataStoreId, + }); + + if (existingColumnMatch === null) { + throw new UserError( + `Tried to delete column with name not present in data store '${dataStoreId}'`, + ); + } + + await this.dataStoreColumnRepository.deleteColumn(dataStoreId, existingColumnMatch); + + return true; + } + + async getManyAndCount(options: DataStoreListOptions) { + return await this.dataStoreRepository.getManyAndCount(options); + } + + async getManyRowsAndCount(dataStoreId: string, dto: ListDataStoreContentQueryDto) { + // unclear if we should validate here, only use case would be to reduce the chance of + // a renamed/removed column appearing here (or added column missing) if the store was + // modified between when the frontend sent the request and we received it + const columns = await this.dataStoreColumnRepository.getColumns(dataStoreId); + const result = await this.dataStoreRowsRepository.getManyAndCount( + toTableName(dataStoreId), + dto, + ); + return { + count: result.count, + data: this.normalizeRows(result.data, columns), + }; + } + + async getColumns(dataStoreId: string) { + return await this.dataStoreColumnRepository.getColumns(dataStoreId); + } + + // TODO: move to utils and test + private normalizeRows(rows: Array>, columns: DataStoreColumn[]) { + const typeMap = new Map(columns.map((col) => [col.name, col.type])); + return rows.map((row) => { + const normalized = { ...row }; + for (const [key, value] of Object.entries(row)) { + const type = typeMap.get(key); + + if (type === 'boolean') { + // Convert boolean values to true/false + if (typeof value === 'boolean') { + normalized[key] = value; + } else if (value === 1 || value === '1') { + normalized[key] = true; + } else if (value === 0 || value === '0') { + normalized[key] = false; + } + } + if (type === 'date' && value !== null && value !== undefined) { + // Convert date objects or strings to ISO string + let dateObj: Date | null = null; + + if (value instanceof Date) { + dateObj = value; + } else if (typeof value === 'string' || typeof value === 'number') { + const parsed = new Date(value); + if (!isNaN(parsed.getTime())) { + dateObj = parsed; + } + } + + normalized[key] = dateObj ? dateObj.toISOString() : value; + } + } + return normalized; + }); + } + + private async validateRows(dataStoreId: string, rows: DataStoreRows): Promise { + const columns = await this.dataStoreColumnRepository.getColumns(dataStoreId); + if (columns.length === 0) { + throw new UserError('No columns found for this data store or data store not found'); + } + + const columnNames = new Set(columns.map((x) => x.name)); + const columnTypeMap = new Map(columns.map((x) => [x.name, x.type])); + for (const row of rows) { + const keys = Object.keys(row); + if (columns.length !== keys.length) { + throw new UserError('mismatched key count'); + } + for (const key of keys) { + if (!columnNames.has(key)) { + throw new UserError('unknown column name'); + } + const cell = row[key]; + if (cell === null) continue; + switch (columnTypeMap.get(key)) { + case 'boolean': + if (typeof cell !== 'boolean') + throw new UserError( + `value '${cell.toString()}' does not match column type 'boolean'`, + ); + break; + case 'date': + if (!(cell instanceof Date)) + throw new UserError(`value '${cell}' does not match column type 'date'`); + row[key] = cell.toISOString(); + break; + case 'string': + if (typeof cell !== 'string') + throw new UserError(`value '${cell.toString()}' does not match column type 'string'`); + break; + case 'number': + if (typeof cell !== 'number') + throw new UserError(`value '${cell.toString()}' does not match column type 'number'`); + break; + } + } + } + } + + async insertRows(dataStoreId: string, rows: DataStoreRows) { + await this.validateRows(dataStoreId, rows); + const columns = await this.dataStoreColumnRepository.getColumns(dataStoreId); + + return await this.dataStoreRowsRepository.insertRows(toTableName(dataStoreId), rows, columns); + } + + async upsertRows(dataStoreId: string, dto: UpsertDataStoreRowsDto) { + await this.validateRows(dataStoreId, dto.rows); + const columns = await this.dataStoreColumnRepository.getColumns(dataStoreId); + + return await this.dataStoreRowsRepository.upsertRows(toTableName(dataStoreId), dto, columns); + } + + private async validateDataStoreExists(dataStoreId: string, msg?: string) { + const existingTable = await this.dataStoreRepository.findOneBy({ + id: dataStoreId, + }); + + if (!existingTable) { + throw new UserError(msg ?? `Data Store '${dataStoreId}' does not exist.`); + } + } +} diff --git a/packages/cli/src/modules/data-store/data-store.types.ts b/packages/cli/src/modules/data-store/data-store.types.ts new file mode 100644 index 0000000000..a648000a81 --- /dev/null +++ b/packages/cli/src/modules/data-store/data-store.types.ts @@ -0,0 +1 @@ +export type DataStoreUserTableName = `data_store_user_${string}`; diff --git a/packages/cli/src/modules/data-store/utils/sql-utils.ts b/packages/cli/src/modules/data-store/utils/sql-utils.ts new file mode 100644 index 0000000000..6e91ecba48 --- /dev/null +++ b/packages/cli/src/modules/data-store/utils/sql-utils.ts @@ -0,0 +1,242 @@ +import { + DATA_STORE_COLUMN_REGEX, + type DataStoreRows, + type DataStoreCreateColumnSchema, +} from '@n8n/api-types'; +import { DslColumn } from '@n8n/db'; +import type { DataSourceOptions } from '@n8n/typeorm'; +import { UnexpectedError } from 'n8n-workflow'; + +import type { DataStoreUserTableName } from '../data-store.types'; + +import { NotFoundError } from '@/errors/response-errors/not-found.error'; + +export function toDslColumns(columns: DataStoreCreateColumnSchema[]): DslColumn[] { + return columns.map((col) => { + const name = new DslColumn(col.name.trim()); + + switch (col.type) { + case 'number': + return name.int; + case 'boolean': + return name.bool; + case 'string': + return name.text; + case 'date': + return name.timestampTimezone(); + default: + return name.text; + } + }); +} + +function dataStoreColumnTypeToSql( + type: DataStoreCreateColumnSchema['type'], + dbType: DataSourceOptions['type'], +) { + switch (type) { + case 'string': + return 'TEXT'; + case 'number': + return 'FLOAT'; + case 'boolean': + return 'BOOLEAN'; + case 'date': + if (dbType === 'postgres') { + return 'TIMESTAMP'; + } + return 'DATETIME'; + default: + throw new NotFoundError(`Unsupported field type: ${type as string}`); + } +} + +function columnToWildcardAndType( + column: DataStoreCreateColumnSchema, + dbType: DataSourceOptions['type'], +) { + return `${quoteIdentifier(column.name, dbType)} ${dataStoreColumnTypeToSql(column.type, dbType)}`; +} + +function isValidColumnName(name: string) { + // Only allow alphanumeric and underscore + return DATA_STORE_COLUMN_REGEX.test(name); +} + +export function addColumnQuery( + tableName: DataStoreUserTableName, + column: DataStoreCreateColumnSchema, + dbType: DataSourceOptions['type'], +) { + // API requests should already conform to this, but better safe than sorry + if (!isValidColumnName(column.name)) { + throw new UnexpectedError('bad column name'); + } + + const quotedTableName = quoteIdentifier(tableName, dbType); + + return `ALTER TABLE ${quotedTableName} ADD ${columnToWildcardAndType(column, dbType)}`; +} + +export function deleteColumnQuery( + tableName: DataStoreUserTableName, + column: string, + dbType: DataSourceOptions['type'], +): string { + const quotedTableName = quoteIdentifier(tableName, dbType); + return `ALTER TABLE ${quotedTableName} DROP COLUMN ${quoteIdentifier(column, dbType)}`; +} + +export function buildInsertQuery( + tableName: DataStoreUserTableName, + rows: DataStoreRows, + columns: Array<{ name: string; type: string }>, + dbType: DataSourceOptions['type'] = 'sqlite', +): [string, unknown[]] { + if (rows.length === 0 || Object.keys(rows[0]).length === 0) { + return ['', []]; + } + + const keys = Object.keys(rows[0]); + const quotedKeys = keys.map((key) => quoteIdentifier(key, dbType)).join(', '); + const quotedTableName = quoteIdentifier(tableName, dbType); + + const columnTypeMap = buildColumnTypeMap(columns); + const parameters: unknown[] = []; + const valuePlaceholders: string[] = []; + let placeholderIndex = 1; + + for (const row of rows) { + const rowPlaceholders = keys.map((key) => { + const value = normalizeValue(row[key], columnTypeMap[key], dbType); + parameters.push(value); + return getPlaceholder(placeholderIndex++, dbType); + }); + valuePlaceholders.push(`(${rowPlaceholders.join(', ')})`); + } + + const query = `INSERT INTO ${quotedTableName} (${quotedKeys}) VALUES ${valuePlaceholders.join(', ')}`; + return [query, parameters]; +} + +export function buildUpdateQuery( + tableName: DataStoreUserTableName, + row: Record, + columns: Array<{ name: string; type: string }>, + matchFields: string[], + dbType: DataSourceOptions['type'] = 'sqlite', +): [string, unknown[]] { + if (Object.keys(row).length === 0 || matchFields.length === 0) { + return ['', []]; + } + + const updateKeys = Object.keys(row).filter((key) => !matchFields.includes(key)); + if (updateKeys.length === 0) { + return ['', []]; + } + + const quotedTableName = quoteIdentifier(tableName, dbType); + const columnTypeMap = buildColumnTypeMap(columns); + + const parameters: unknown[] = []; + let placeholderIndex = 1; + + const setClause = updateKeys + .map((key) => { + const value = normalizeValue(row[key], columnTypeMap[key], dbType); + parameters.push(value); + return `${quoteIdentifier(key, dbType)} = ${getPlaceholder(placeholderIndex++, dbType)}`; + }) + .join(', '); + + const whereClause = matchFields + .map((key) => { + const value = normalizeValue(row[key], columnTypeMap[key], dbType); + parameters.push(value); + return `${quoteIdentifier(key, dbType)} = ${getPlaceholder(placeholderIndex++, dbType)}`; + }) + .join(' AND '); + + const query = `UPDATE ${quotedTableName} SET ${setClause} WHERE ${whereClause}`; + return [query, parameters]; +} + +export function splitRowsByExistence( + existing: Array>, + matchFields: string[], + rows: DataStoreRows, +): { rowsToInsert: DataStoreRows; rowsToUpdate: DataStoreRows } { + // Extracts only the fields relevant to matching and serializes them for comparison + const getMatchKey = (row: Record): string => + JSON.stringify(Object.fromEntries(matchFields.map((field) => [field, row[field]]))); + + const existingSet = new Set(existing.map((row) => getMatchKey(row))); + + const rowsToUpdate: DataStoreRows = []; + const rowsToInsert: DataStoreRows = []; + + for (const row of rows) { + const key = getMatchKey(row); + + if (existingSet.has(key)) { + rowsToUpdate.push(row); + } else { + rowsToInsert.push(row); + } + } + + return { rowsToInsert, rowsToUpdate }; +} + +export function quoteIdentifier(name: string, dbType: DataSourceOptions['type']): string { + switch (dbType) { + case 'mysql': + case 'mariadb': + return `\`${name}\``; + case 'postgres': + case 'sqlite': + default: + return `"${name}"`; + } +} + +export function toTableName(dataStoreId: string): DataStoreUserTableName { + return `data_store_user_${dataStoreId}`; +} + +function normalizeValue( + value: unknown, + columnType: string | undefined, + dbType: DataSourceOptions['type'], +): unknown { + if (['mysql', 'mariadb'].includes(dbType)) { + if (columnType === 'date') { + if ( + value instanceof Date || + (typeof value === 'string' && value.match(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}/)) + ) { + return toMySQLDateTimeString(value); + } + } + } + return value; +} + +function toMySQLDateTimeString(date: Date | string, convertFromDate = true): string { + const dateString = convertFromDate + ? date instanceof Date + ? date.toISOString() + : date + : (date as string); + return dateString.replace('T', ' ').replace('Z', ''); +} + +export function getPlaceholder(index: number, dbType: DataSourceOptions['type']): string { + return dbType.includes('postgres') ? `$${index}` : '?'; +} + +function buildColumnTypeMap( + columns: Array<{ name: string; type: string }>, +): Record { + return Object.fromEntries(columns.map((col) => [col.name, col.type])); +} diff --git a/packages/cli/test/integration/shared/types.ts b/packages/cli/test/integration/shared/types.ts index 9d6c58e757..ad20066445 100644 --- a/packages/cli/test/integration/shared/types.ts +++ b/packages/cli/test/integration/shared/types.ts @@ -42,9 +42,10 @@ type EndpointGroup = | 'evaluation' | 'ai' | 'folder' - | 'insights'; + | 'insights' + | 'data-store'; -type ModuleName = 'insights' | 'external-secrets'; +type ModuleName = 'insights' | 'external-secrets' | 'data-store'; export interface SetupProps { endpointGroups?: EndpointGroup[]; diff --git a/packages/frontend/editor-ui/src/features/dataStore/DataStoreDetailsView.vue b/packages/frontend/editor-ui/src/features/dataStore/DataStoreDetailsView.vue index 4a3c590b90..271e601267 100644 --- a/packages/frontend/editor-ui/src/features/dataStore/DataStoreDetailsView.vue +++ b/packages/frontend/editor-ui/src/features/dataStore/DataStoreDetailsView.vue @@ -1,6 +1,6 @@