diff --git a/packages/cli/src/modules/data-table/__tests__/data-store-proxy.service.test.ts b/packages/cli/src/modules/data-table/__tests__/data-store-proxy.service.test.ts index 225991f01a..0a261b6f49 100644 --- a/packages/cli/src/modules/data-table/__tests__/data-store-proxy.service.test.ts +++ b/packages/cli/src/modules/data-table/__tests__/data-store-proxy.service.test.ts @@ -209,7 +209,7 @@ describe('DataStoreProxyService', () => { ); await dataStoreOperations.insertRows(rows); - expect(dataStoreServiceMock.insertRows).toBeCalledWith('dataStore-id', PROJECT_ID, rows); + expect(dataStoreServiceMock.insertRows).toBeCalledWith('dataStore-id', PROJECT_ID, rows, true); }); it('should call upsertRows with correct parameters', async () => { @@ -225,6 +225,11 @@ describe('DataStoreProxyService', () => { ); await dataStoreOperations.upsertRows(options); - expect(dataStoreServiceMock.upsertRows).toBeCalledWith('dataStore-id', PROJECT_ID, options); + expect(dataStoreServiceMock.upsertRows).toBeCalledWith( + 'dataStore-id', + PROJECT_ID, + options, + true, + ); }); }); diff --git a/packages/cli/src/modules/data-table/__tests__/data-store.controller.test.ts b/packages/cli/src/modules/data-table/__tests__/data-store.controller.test.ts index 3d32814e5d..8417d05917 100644 --- a/packages/cli/src/modules/data-table/__tests__/data-store.controller.test.ts +++ b/packages/cli/src/modules/data-table/__tests__/data-store.controller.test.ts @@ -2048,11 +2048,15 @@ describe('POST /projects/:projectId/data-stores/:dataStoreId/insert', () => { id: 1, first: 'first row', second: 'some value', + createdAt: expect.any(String), + updatedAt: expect.any(String), }, { id: 2, first: 'another row', second: 'another value', + createdAt: expect.any(String), + updatedAt: expect.any(String), }, ], }); @@ -3038,9 +3042,27 @@ describe('POST /projects/:projectId/data-stores/:dataStoreId/upsert', () => { expect(result.body.data).toEqual( expect.arrayContaining([ - { id: 1, first: 'test row', second: 'updated value' }, - { id: 2, first: 'test row', second: 'updated value' }, - { id: 3, first: 'new row', second: 'new value' }, + { + id: 1, + first: 'test row', + second: 'updated value', + createdAt: expect.any(String), + updatedAt: expect.any(String), + }, + { + id: 2, + first: 'test row', + second: 'updated value', + createdAt: expect.any(String), + updatedAt: expect.any(String), + }, + { + id: 3, + first: 'new row', + second: 'new value', + createdAt: expect.any(String), + updatedAt: expect.any(String), + }, ]), ); }); diff --git a/packages/cli/src/modules/data-table/__tests__/data-store.service.test.ts b/packages/cli/src/modules/data-table/__tests__/data-store.service.test.ts index f3ff44e717..3f4d75669b 100644 --- a/packages/cli/src/modules/data-table/__tests__/data-store.service.test.ts +++ b/packages/cli/src/modules/data-table/__tests__/data-store.service.test.ts @@ -1111,9 +1111,33 @@ describe('dataStore', () => { true, ); expect(ids).toEqual([ - { id: 1, c1: 1, c2: 'foo', c3: true, c4: now }, - { id: 2, c1: 2, c2: 'bar', c3: false, c4: now }, - { id: 3, c1: null, c2: null, c3: null, c4: null }, + { + id: 1, + c1: 1, + c2: 'foo', + c3: true, + c4: now, + createdAt: expect.any(Date), + updatedAt: expect.any(Date), + }, + { + id: 2, + c1: 2, + c2: 'bar', + c3: false, + c4: now, + createdAt: expect.any(Date), + updatedAt: expect.any(Date), + }, + { + id: 3, + c1: null, + c2: null, + c3: null, + c4: null, + createdAt: expect.any(Date), + updatedAt: expect.any(Date), + }, ]); }); @@ -1489,6 +1513,8 @@ describe('dataStore', () => { age: 31, pid: '1995-111a', birthday: new Date('1995-01-01'), + createdAt: expect.any(Date), + updatedAt: expect.any(Date), }, { id: 2, @@ -1496,6 +1522,8 @@ describe('dataStore', () => { age: 30, pid: '1992-222b', birthday: new Date('1992-01-01'), + createdAt: expect.any(Date), + updatedAt: expect.any(Date), }, ]), ); @@ -1725,11 +1753,11 @@ describe('dataStore', () => { expect(updatedRows[0].createdAt).not.toBeNull(); expect(updatedRows[0].updatedAt).not.toBeNull(); expect(initialRows[0].updatedAt).not.toBeNull(); - expect(new Date(updatedRows[0].updatedAt as string).getTime()).toBeGreaterThan( - new Date(initialRows[0].updatedAt as string).getTime(), + expect(new Date(updatedRows[0].updatedAt).getTime()).toBeGreaterThan( + new Date(initialRows[0].updatedAt).getTime(), ); - expect(new Date(updatedRows[0].updatedAt as string).getTime()).toBeGreaterThan( - new Date(updatedRows[0].createdAt as string).getTime(), + expect(new Date(updatedRows[0].updatedAt).getTime()).toBeGreaterThan( + new Date(updatedRows[0].createdAt).getTime(), ); }); diff --git a/packages/cli/src/modules/data-table/data-store-proxy.service.ts b/packages/cli/src/modules/data-table/data-store-proxy.service.ts index 4bd4bad436..f1be6ba87b 100644 --- a/packages/cli/src/modules/data-table/data-store-proxy.service.ts +++ b/packages/cli/src/modules/data-table/data-store-proxy.service.ts @@ -125,11 +125,15 @@ export class DataStoreProxyService implements DataStoreProxyProvider { }, async insertRows(rows: DataStoreRows) { - return await dataStoreService.insertRows(dataStoreId, projectId, rows); + return await dataStoreService.insertRows(dataStoreId, projectId, rows, true); }, async upsertRows(options: UpsertDataStoreRowsOptions) { - return await dataStoreService.upsertRows(dataStoreId, projectId, options); + return await dataStoreService.upsertRows(dataStoreId, projectId, options, true); + }, + + async deleteRows(ids: number[]) { + return await dataStoreService.deleteRows(dataStoreId, projectId, ids); }, }; } diff --git a/packages/cli/src/modules/data-table/data-store-rows.repository.ts b/packages/cli/src/modules/data-table/data-store-rows.repository.ts index 255736b71e..f5ec6ba6d5 100644 --- a/packages/cli/src/modules/data-table/data-store-rows.repository.ts +++ b/packages/cli/src/modules/data-table/data-store-rows.repository.ts @@ -6,8 +6,10 @@ import { DataSource, DataSourceOptions, QueryRunner, SelectQueryBuilder, In } fr import { DataStoreColumnJsType, DataStoreRows, - DataStoreRowWithId, + DataStoreRowReturn, UnexpectedError, + DataStoreRowsReturn, + DATA_TABLE_SYSTEM_COLUMNS, } from 'n8n-workflow'; import { DataStoreColumn } from './data-store-column.entity'; @@ -116,20 +118,29 @@ export class DataStoreRowsRepository { return `${tablePrefix}data_store_user_${dataStoreId}`; } + async insertRows( + dataStoreId: string, + rows: DataStoreRows, + columns: DataStoreColumn[], + returnData?: T, + ): Promise>>; async insertRows( dataStoreId: string, rows: DataStoreRows, columns: DataStoreColumn[], - returnData: boolean = false, - ) { - const inserted: DataStoreRowWithId[] = []; + returnData?: boolean, + ): Promise>> { + const inserted: Array> = []; const dbType = this.dataSource.options.type; const useReturning = dbType === 'postgres' || dbType === 'mariadb'; const table = this.toTableName(dataStoreId); const columnNames = columns.map((c) => c.name); const escapedColumns = columns.map((c) => this.dataSource.driver.escape(c.name)); - const selectColumns = ['id', ...escapedColumns]; + const escapedSystemColumns = DATA_TABLE_SYSTEM_COLUMNS.map((x) => + this.dataSource.driver.escape(x), + ); + const selectColumns = [...escapedSystemColumns, ...escapedColumns]; // We insert one by one as the default behavior of returning the last inserted ID // is consistent, whereas getting all inserted IDs when inserting multiple values is @@ -153,7 +164,9 @@ export class DataStoreRowsRepository { const result = await query.execute(); if (useReturning) { - const returned = normalizeRows(extractReturningData(result.raw), columns); + const returned = returnData + ? normalizeRows(extractReturningData(result.raw), columns) + : extractInsertedIds(result.raw, dbType).map((id) => ({ id })); inserted.push.apply(inserted, returned); continue; } @@ -169,11 +182,7 @@ export class DataStoreRowsRepository { continue; } - const insertedRows = (await this.getManyByIds( - dataStoreId, - ids, - columns, - )) as DataStoreRowWithId[]; + const insertedRows = await this.getManyByIds(dataStoreId, ids, columns); inserted.push(...insertedRows); } @@ -193,7 +202,10 @@ export class DataStoreRowsRepository { const table = this.toTableName(dataStoreId); const escapedColumns = columns.map((c) => this.dataSource.driver.escape(c.name)); - const selectColumns = ['id', ...escapedColumns]; + const escapedSystemColumns = DATA_TABLE_SYSTEM_COLUMNS.map((x) => + this.dataSource.driver.escape(x), + ); + const selectColumns = [...escapedSystemColumns, ...escapedColumns]; for (const column of columns) { if (column.name in setData) { @@ -204,7 +216,7 @@ export class DataStoreRowsRepository { } } - let affectedRows: DataStoreRowWithId[] = []; + let affectedRows: Array> = []; if (!useReturning && returnData) { // Only Postgres supports RETURNING statement on updates (with our typeorm), // on other engines we must query the list of updates rows later by ID @@ -239,20 +251,28 @@ export class DataStoreRowsRepository { } // TypeORM cannot infer the columns for a dynamic table name, so we use a raw query + async upsertRows( + dataStoreId: string, + matchFields: string[], + rows: DataStoreRows, + columns: DataStoreColumn[], + returnData?: T, + ): Promise; async upsertRows( dataStoreId: string, matchFields: string[], rows: DataStoreRows, columns: DataStoreColumn[], - returnData = false, + returnData?: boolean, ) { + returnData = returnData ?? false; const { rowsToInsert, rowsToUpdate } = await this.fetchAndSplitRowsByExistence( dataStoreId, matchFields, rows, ); - const output: DataStoreRowWithId[] = []; + const output: DataStoreRowReturn[] = []; if (rowsToInsert.length > 0) { const result = await this.insertRows(dataStoreId, rowsToInsert, columns, returnData); @@ -335,7 +355,7 @@ export class DataStoreRowsRepository { async getManyAndCount(dataStoreId: string, dto: ListDataStoreContentQueryDto) { const [countQuery, query] = this.getManyQuery(dataStoreId, dto); - const data: DataStoreRows = await query.select('*').getRawMany(); + const data: DataStoreRowsReturn = await query.select('*').getRawMany(); const countResult = await countQuery.select('COUNT(*) as count').getRawOne<{ count: number | string | null; }>(); @@ -347,7 +367,10 @@ export class DataStoreRowsRepository { async getManyByIds(dataStoreId: string, ids: number[], columns: DataStoreColumn[]) { const table = this.toTableName(dataStoreId); const escapedColumns = columns.map((c) => this.dataSource.driver.escape(c.name)); - const selectColumns = ['id', ...escapedColumns]; + const escapedSystemColumns = DATA_TABLE_SYSTEM_COLUMNS.map((x) => + this.dataSource.driver.escape(x), + ); + const selectColumns = [...escapedSystemColumns, ...escapedColumns]; if (ids.length === 0) { return []; @@ -358,7 +381,7 @@ export class DataStoreRowsRepository { .select(selectColumns) .from(table, 'dataStore') .where({ id: In(ids) }) - .getRawMany(); + .getRawMany(); return normalizeRows(updatedRows, columns); } diff --git a/packages/cli/src/modules/data-table/data-store.controller.ts b/packages/cli/src/modules/data-table/data-store.controller.ts index bb46de749b..11a4d98db6 100644 --- a/packages/cli/src/modules/data-table/data-store.controller.ts +++ b/packages/cli/src/modules/data-table/data-store.controller.ts @@ -34,6 +34,7 @@ import { DataStoreColumnNotFoundError } from './errors/data-store-column-not-fou import { DataStoreNameConflictError } from './errors/data-store-name-conflict.error'; import { DataStoreNotFoundError } from './errors/data-store-not-found.error'; import { DataStoreValidationError } from './errors/data-store-validation.error'; +import { DataStoreRowReturn } from 'n8n-workflow'; @RestController('/projects/:projectId/data-stores') export class DataStoreController { @@ -237,6 +238,12 @@ export class DataStoreController { /** * @returns the IDs of the inserted rows */ + async appendDataStoreRows( + req: AuthenticatedRequest<{ projectId: string }>, + _res: Response, + dataStoreId: string, + dto: AddDataStoreRowsDto & { returnData?: T }, + ): Promise>>; @Post('/:dataStoreId/insert') @ProjectScope('dataStore:writeRow') async appendDataStoreRows( diff --git a/packages/cli/src/modules/data-table/data-store.service.ts b/packages/cli/src/modules/data-table/data-store.service.ts index ff141df595..7f7458002c 100644 --- a/packages/cli/src/modules/data-table/data-store.service.ts +++ b/packages/cli/src/modules/data-table/data-store.service.ts @@ -11,7 +11,7 @@ import type { } from '@n8n/api-types'; import { Logger } from '@n8n/backend-common'; import { Service } from '@n8n/di'; -import type { DataStoreRow, DataStoreRows } from 'n8n-workflow'; +import type { DataStoreRow, DataStoreRowReturn, DataStoreRows } from 'n8n-workflow'; import { DataStoreColumnRepository } from './data-store-column.repository'; import { DataStoreRowsRepository } from './data-store-rows.repository'; @@ -126,11 +126,17 @@ export class DataStoreService { return await this.dataStoreColumnRepository.getColumns(dataStoreId); } + async insertRows( + dataStoreId: string, + projectId: string, + rows: DataStoreRows, + returnData?: T, + ): Promise>>; async insertRows( dataStoreId: string, projectId: string, rows: DataStoreRows, - returnData: boolean = false, + returnData?: boolean, ) { await this.validateDataStoreExists(dataStoreId, projectId); await this.validateRows(dataStoreId, rows); @@ -139,6 +145,12 @@ export class DataStoreService { return await this.dataStoreRowsRepository.insertRows(dataStoreId, rows, columns, returnData); } + async upsertRows( + dataStoreId: string, + projectId: string, + dto: Omit, + returnData?: T, + ): Promise; async upsertRows( dataStoreId: string, projectId: string, diff --git a/packages/cli/src/modules/data-table/utils/sql-utils.ts b/packages/cli/src/modules/data-table/utils/sql-utils.ts index 4eb64ad263..936b68878b 100644 --- a/packages/cli/src/modules/data-table/utils/sql-utils.ts +++ b/packages/cli/src/modules/data-table/utils/sql-utils.ts @@ -5,13 +5,18 @@ import { } from '@n8n/api-types'; import { DslColumn } from '@n8n/db'; import type { DataSourceOptions } from '@n8n/typeorm'; -import type { DataStoreColumnJsType, DataStoreRows, DataStoreRowWithId } from 'n8n-workflow'; +import type { + DataStoreColumnJsType, + DataStoreRows, + DataStoreRowReturn, + DataStoreRowsReturn, +} from 'n8n-workflow'; import { UnexpectedError } from 'n8n-workflow'; -import { NotFoundError } from '@/errors/response-errors/not-found.error'; - import type { DataStoreUserTableName } from '../data-store.types'; +import { NotFoundError } from '@/errors/response-errors/not-found.error'; + export function toDslColumns(columns: DataStoreCreateColumnSchema[]): DslColumn[] { return columns.map((col) => { const name = new DslColumn(col.name.trim()); @@ -138,7 +143,6 @@ export function quoteIdentifier(name: string, dbType: DataSourceOptions['type']) } type WithInsertId = { insertId: number }; -type WithRowId = { id: number }; const isArrayOf = (data: unknown, itemGuard: (x: unknown) => x is T): data is T[] => Array.isArray(data) && data.every(itemGuard); @@ -147,18 +151,35 @@ const isNumber = (value: unknown): value is number => { return typeof value === 'number' && Number.isFinite(value); }; +const isDate = (value: unknown): value is Date => { + return value instanceof Date; +}; + function hasInsertId(data: unknown): data is WithInsertId { return typeof data === 'object' && data !== null && 'insertId' in data && isNumber(data.insertId); } -function hasRowId(data: unknown): data is WithRowId { +function hasRowReturnData(data: unknown): data is DataStoreRowReturn { + return ( + typeof data === 'object' && + data !== null && + 'id' in data && + isNumber(data.id) && + 'createdAt' in data && + isDate(data.createdAt) && + 'updatedAt' in data && + isDate(data.updatedAt) + ); +} + +function hasRowId(data: unknown): data is Pick { return typeof data === 'object' && data !== null && 'id' in data && isNumber(data.id); } -export function extractReturningData(raw: unknown): DataStoreRowWithId[] { - if (!isArrayOf(raw, hasRowId)) { +export function extractReturningData(raw: unknown): DataStoreRowReturn[] { + if (!isArrayOf(raw, hasRowReturnData)) { throw new UnexpectedError( - 'Expected INSERT INTO raw to be { id: number }[] on Postgres or MariaDB', + `Expected INSERT INTO raw to be { id: number; createdAt: string; updatedAt: string }[] on Postgres or MariaDB. Is '${JSON.stringify(raw)}'`, ); } @@ -171,7 +192,7 @@ export function extractInsertedIds(raw: unknown, dbType: DataSourceOptions['type case 'mariadb': { if (!isArrayOf(raw, hasRowId)) { throw new UnexpectedError( - 'Expected INSERT INTO raw to be { id: number }[] on Postgres or MariaDB', + `Expected INSERT INTO raw to be { id: number }[] on Postgres or MariaDB. Is '${JSON.stringify(raw)}'`, ); } return raw.map((r) => r.id); @@ -192,7 +213,7 @@ export function extractInsertedIds(raw: unknown, dbType: DataSourceOptions['type } } -export function normalizeRows(rows: DataStoreRows, columns: DataStoreColumn[]) { +export function normalizeRows(rows: DataStoreRowsReturn, columns: DataStoreColumn[]) { // we need to normalize system dates as well const systemColumns = [ { name: 'createdAt', type: 'date' }, diff --git a/packages/workflow/src/data-store.types.ts b/packages/workflow/src/data-store.types.ts index 7fb49e5d25..467d60f095 100644 --- a/packages/workflow/src/data-store.types.ts +++ b/packages/workflow/src/data-store.types.ts @@ -72,9 +72,17 @@ export type AddDataStoreColumnOptions = Pick & export type DataStoreColumnJsType = string | number | boolean | Date | null; +export const DATA_TABLE_SYSTEM_COLUMNS = ['id', 'createdAt', 'updatedAt'] as const; + +export type DataStoreRowReturnBase = { + id: number; + createdAt: Date; + updatedAt: Date; +}; export type DataStoreRow = Record; export type DataStoreRows = DataStoreRow[]; -export type DataStoreRowWithId = DataStoreRow & { id: number }; +export type DataStoreRowReturn = DataStoreRow & DataStoreRowReturnBase; +export type DataStoreRowsReturn = DataStoreRowReturn[]; // APIs for a data store service operating on a specific projectId export interface IDataStoreProjectAggregateService { @@ -100,9 +108,11 @@ export interface IDataStoreProjectService { getManyRowsAndCount( dto: Partial, - ): Promise<{ count: number; data: DataStoreRows }>; + ): Promise<{ count: number; data: DataStoreRowsReturn }>; - insertRows(rows: DataStoreRows): Promise>; + insertRows(rows: DataStoreRows): Promise; - upsertRows(options: UpsertDataStoreRowsOptions): Promise; + upsertRows(options: UpsertDataStoreRowsOptions): Promise; + + deleteRows(ids: number[]): Promise; }