diff --git a/packages/@n8n/api-types/src/dto/data-store/add-data-store-rows.dto.ts b/packages/@n8n/api-types/src/dto/data-store/add-data-store-rows.dto.ts index 5fa3909820..faf3aa03b1 100644 --- a/packages/@n8n/api-types/src/dto/data-store/add-data-store-rows.dto.ts +++ b/packages/@n8n/api-types/src/dto/data-store/add-data-store-rows.dto.ts @@ -7,5 +7,6 @@ import { } from '../../schemas/data-store.schema'; export class AddDataStoreRowsDto extends Z.class({ + returnData: z.boolean().default(false), data: z.array(z.record(dataStoreColumnNameSchema, dataStoreColumnValueSchema)), }) {} diff --git a/packages/cli/src/modules/data-store/__tests__/data-store.controller.test.ts b/packages/cli/src/modules/data-store/__tests__/data-store.controller.test.ts index 8befd42d68..82bdbd099a 100644 --- a/packages/cli/src/modules/data-store/__tests__/data-store.controller.test.ts +++ b/packages/cli/src/modules/data-store/__tests__/data-store.controller.test.ts @@ -1835,7 +1835,7 @@ describe('POST /projects/:projectId/data-stores/:dataStoreId/insert', () => { .expect(200); expect(response.body).toEqual({ - data: [1], + data: [{ id: 1 }], }); const rowsInDb = await dataStoreRowsRepository.getManyAndCount(dataStore.id, {}); @@ -1875,7 +1875,7 @@ describe('POST /projects/:projectId/data-stores/:dataStoreId/insert', () => { .expect(200); expect(response.body).toEqual({ - data: [1], + data: [{ id: 1 }], }); const rowsInDb = await dataStoreRowsRepository.getManyAndCount(dataStore.id, {}); @@ -1912,7 +1912,7 @@ describe('POST /projects/:projectId/data-stores/:dataStoreId/insert', () => { .expect(200); expect(response.body).toEqual({ - data: [1], + data: [{ id: 1 }], }); const rowsInDb = await dataStoreRowsRepository.getManyAndCount(dataStore.id, {}); @@ -1920,6 +1920,59 @@ describe('POST /projects/:projectId/data-stores/:dataStoreId/insert', () => { expect(rowsInDb.data[0]).toMatchObject(payload.data[0]); }); + test('should return inserted data if returnData is set', async () => { + const dataStore = await createDataStore(memberProject, { + columns: [ + { + name: 'first', + type: 'string', + }, + { + name: 'second', + type: 'string', + }, + ], + }); + + const payload = { + returnData: true, + data: [ + { + first: 'first row', + second: 'some value', + }, + { + first: 'another row', + second: 'another value', + }, + ], + }; + + const response = await authMemberAgent + .post(`/projects/${memberProject.id}/data-stores/${dataStore.id}/insert`) + .send(payload) + .expect(200); + + expect(response.body).toEqual({ + data: [ + { + id: 1, + first: 'first row', + second: 'some value', + }, + { + id: 2, + first: 'another row', + second: 'another value', + }, + ], + }); + + const rowsInDb = await dataStoreRowsRepository.getManyAndCount(dataStore.id, {}); + expect(rowsInDb.count).toBe(2); + expect(rowsInDb.data[0]).toMatchObject(payload.data[0]); + }); + test('should not insert rows when column does not exist', async () => { const dataStore = await createDataStore(memberProject, { columns: [ @@ -1982,7 +2035,7 @@ describe('POST /projects/:projectId/data-stores/:dataStoreId/insert', () => { .expect(200); expect(response.body).toEqual({ - data: [1], + data: [{ id: 1 }], }); const readResponse = await authMemberAgent @@ -2030,7 +2083,7 @@ describe('POST /projects/:projectId/data-stores/:dataStoreId/insert', () => { .expect(200); expect(response.body).toEqual({ - data: [1], + data: [{ id: 1 }], }); const readResponse = await authMemberAgent @@ -2070,7 +2123,7 @@ describe('POST /projects/:projectId/data-stores/:dataStoreId/insert', () => { .expect(200); expect(response.body).toEqual({ - data: [1], + data: [{ id: 1 }], }); const readResponse = await authMemberAgent @@ -2125,7 +2178,7 @@ describe('POST /projects/:projectId/data-stores/:dataStoreId/insert', () => { .expect(200); expect(response.body).toEqual({ - data: [1], + data: [{ id: 1 }], }); const readResponse = await authMemberAgent @@ -2175,7 +2228,7 @@ describe('POST /projects/:projectId/data-stores/:dataStoreId/insert', () => { .expect(200); expect(response.body).toEqual({ - data: [1], + data: [{ id: 1 }], }); const readResponse = await authMemberAgent @@ -2223,7 +2276,7 @@ describe('POST /projects/:projectId/data-stores/:dataStoreId/insert', () => { .expect(200); expect(first.body).toEqual({ - data: [1, 2, 3], + data: [{ id: 1 }, { id: 2 }, { id: 3 }], }); const second = await authMemberAgent @@ -2232,7 +2285,7 @@ describe('POST /projects/:projectId/data-stores/:dataStoreId/insert', () => { .expect(200); expect(second.body).toEqual({ - data: [4, 5, 6], + data: [{ id: 4 }, { id: 5 }, { id: 6 }], }); const readResponse = await authMemberAgent diff --git a/packages/cli/src/modules/data-store/__tests__/data-store.service.test.ts b/packages/cli/src/modules/data-store/__tests__/data-store.service.test.ts index aeb9341a43..c08bd1f680 100644 --- a/packages/cli/src/modules/data-store/__tests__/data-store.service.test.ts +++ b/packages/cli/src/modules/data-store/__tests__/data-store.service.test.ts @@ -462,7 +462,17 @@ describe('dataStore', () => { { name: 'Charlie', age: 35 }, ]); - expect(results).toEqual([1, 2, 3]); + expect(results).toEqual([ + { + id: 1, + }, + { + id: 2, + }, + { + id: 3, + }, + ]); // ACT const newColumn = await dataStoreService.addColumn(dataStoreId, project1.id, { @@ -491,7 +501,11 @@ describe('dataStore', () => { const newRow = await dataStoreService.insertRows(dataStoreId, project1.id, [ { name: 'David', age: 28, email: 'david@example.com' }, ]); - expect(newRow).toEqual([4]); + expect(newRow).toEqual([ + { + id: 4, + }, + ]); const finalData = await dataStoreService.getManyRowsAndCount(dataStoreId, project1.id, {}); expect(finalData.count).toBe(4); @@ -975,7 +989,7 @@ describe('dataStore', () => { const result = await dataStoreService.insertRows(dataStoreId, project1.id, rows); // ASSERT - expect(result).toEqual([1, 2, 3, 4]); + expect(result).toEqual([{ id: 1 }, { id: 2 }, { id: 3 }, { id: 4 }]); const { count, data } = await dataStoreService.getManyRowsAndCount( dataStoreId, @@ -1009,7 +1023,7 @@ describe('dataStore', () => { const initial = await dataStoreService.insertRows(dataStoreId, project1.id, [ { c1: 1, c2: 'foo' }, ]); - expect(initial).toEqual([1]); + expect(initial).toEqual([{ id: 1 }]); // Attempt to insert a row with the same primary key const result = await dataStoreService.insertRows(dataStoreId, project1.id, [ @@ -1017,7 +1031,7 @@ describe('dataStore', () => { ]); // ASSERT - expect(result).toEqual([2]); + expect(result).toEqual([{ id: 2 }]); const { count, data } = await dataStoreRowsRepository.getManyAndCount(dataStoreId, {}); @@ -1043,9 +1057,9 @@ describe('dataStore', () => { { c1: 1, c2: 'foo' }, { c1: 2, c2: 'bar' }, ]); - expect(ids).toEqual([1, 2]); + expect(ids).toEqual([{ id: 1 }, { id: 2 }]); - await dataStoreService.deleteRows(dataStoreId, project1.id, [ids[0]]); + await dataStoreService.deleteRows(dataStoreId, project1.id, [ids[0].id]); // Insert a new row const result = await dataStoreService.insertRows(dataStoreId, project1.id, [ @@ -1054,7 +1068,7 @@ describe('dataStore', () => { ]); // ASSERT - expect(result).toEqual([3, 4]); + expect(result).toEqual([{ id: 3 }, { id: 4 }]); const { count, data } = await dataStoreRowsRepository.getManyAndCount(dataStoreId, {}); @@ -1066,6 +1080,50 @@ describe('dataStore', () => { ]); }); + it('return inserted data if requested', async () => { + // ARRANGE + const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, { + name: 'myDataStore', + columns: [ + { name: 'c1', type: 'number' }, + { name: 'c2', type: 'string' }, + ], + }); + + // Insert initial row + const ids = await dataStoreService.insertRows( + dataStoreId, + project1.id, + [ + { c1: 1, c2: 'foo' }, + { c1: 2, c2: 'bar' }, + ], + true, + ); + expect(ids).toEqual([ + { id: 1, c1: 1, c2: 'foo' }, + { id: 2, c1: 2, c2: 'bar' }, + ]); + + await dataStoreService.deleteRows(dataStoreId, project1.id, [ids[0].id]); + + const result = await dataStoreService.insertRows( + dataStoreId, + project1.id, + [ + { c1: 1, c2: 'baz' }, + { c1: 2, c2: 'faz' }, + ], + true, + ); + + // ASSERT + expect(result).toEqual([ + { id: 3, c1: 1, c2: 'baz' }, + { id: 4, c1: 2, c2: 'faz' }, + ]); + }); + it('rejects a mismatched row with extra column', async () => { // ARRANGE const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, { @@ -1205,7 +1263,7 @@ describe('dataStore', () => { const result = await dataStoreService.insertRows(dataStoreId, project1.id, rows); // ASSERT - expect(result).toEqual([1, 2, 3]); + expect(result).toEqual([{ id: 1 }, { id: 2 }, { id: 3 }]); const { count, data } = await dataStoreService.getManyRowsAndCount( dataStoreId, @@ -1229,12 +1287,14 @@ describe('dataStore', () => { ], }); - await dataStoreService.insertRows(dataStoreId, project1.id, [ + const ids = await dataStoreService.insertRows(dataStoreId, project1.id, [ { pid: '1995-111a', name: 'Alice', age: 30 }, { pid: '1994-222a', name: 'John', age: 31 }, { pid: '1993-333a', name: 'Paul', age: 32 }, ]); + expect(ids).toEqual([{ id: 1 }, { id: 2 }, { id: 3 }]); + // ACT const result = await dataStoreService.upsertRows(dataStoreId, project1.id, { rows: [ @@ -1323,7 +1383,7 @@ describe('dataStore', () => { const ids = await dataStoreService.insertRows(dataStoreId, project1.id, [ { pid: '1995-111a', name: 'Alice', age: 30 }, ]); - expect(ids).toEqual([1]); + expect(ids).toEqual([{ id: 1 }]); // ACT const result = await dataStoreService.upsertRows(dataStoreId, project1.id, { @@ -1366,7 +1426,7 @@ describe('dataStore', () => { { name: 'Bob', age: 25 }, { name: 'Charlie', age: 35 }, ]); - expect(ids).toEqual([1, 2, 3]); + expect(ids).toEqual([{ id: 1 }, { id: 2 }, { id: 3 }]); // Get initial data to find row IDs const initialData = await dataStoreService.getManyRowsAndCount(dataStoreId, project1.id, {}); @@ -1415,7 +1475,7 @@ describe('dataStore', () => { // Insert one row const ids = await dataStoreService.insertRows(dataStoreId, project1.id, [{ name: 'Alice' }]); - expect(ids).toEqual([1]); + expect(ids).toEqual([{ id: 1 }]); // ACT - Try to delete existing and non-existing IDs const result = await dataStoreService.deleteRows(dataStoreId, project1.id, [1, 999, 1000]); @@ -1757,7 +1817,7 @@ describe('dataStore', () => { ]; const ids = await dataStoreService.insertRows(dataStoreId, project1.id, rows); - expect(ids).toEqual([1, 2, 3]); + expect(ids).toEqual([{ id: 1 }, { id: 2 }, { id: 3 }]); // ACT const result = await dataStoreService.getManyRowsAndCount(dataStoreId, project1.id, {}); diff --git a/packages/cli/src/modules/data-store/data-store-proxy.service.ts b/packages/cli/src/modules/data-store/data-store-proxy.service.ts index c8a284f3f8..e5d53f9191 100644 --- a/packages/cli/src/modules/data-store/data-store-proxy.service.ts +++ b/packages/cli/src/modules/data-store/data-store-proxy.service.ts @@ -19,10 +19,10 @@ import { Workflow, } from 'n8n-workflow'; -import { DataStoreService } from './data-store.service'; - import { OwnershipService } from '@/services/ownership.service'; +import { DataStoreService } from './data-store.service'; + @Service() export class DataStoreProxyService implements DataStoreProxyProvider { constructor( diff --git a/packages/cli/src/modules/data-store/data-store-rows.repository.ts b/packages/cli/src/modules/data-store/data-store-rows.repository.ts index db5b48782b..4fa7b297a0 100644 --- a/packages/cli/src/modules/data-store/data-store-rows.repository.ts +++ b/packages/cli/src/modules/data-store/data-store-rows.repository.ts @@ -6,8 +6,13 @@ import type { import { GlobalConfig } from '@n8n/config'; import { CreateTable, DslColumn } from '@n8n/db'; import { Service } from '@n8n/di'; -import { DataSource, DataSourceOptions, QueryRunner, SelectQueryBuilder } from '@n8n/typeorm'; -import { DataStoreColumnJsType, DataStoreRows } from 'n8n-workflow'; +import { DataSource, DataSourceOptions, QueryRunner, SelectQueryBuilder, In } from '@n8n/typeorm'; +import { + DataStoreColumnJsType, + DataStoreRows, + DataStoreRowWithId, + UnexpectedError, +} from 'n8n-workflow'; import { DataStoreColumn } from './data-store-column.entity'; import { DataStoreUserTableName } from './data-store.types'; @@ -16,6 +21,7 @@ import { buildColumnTypeMap, deleteColumnQuery, extractInsertedIds, + extractReturningData, getPlaceholder, normalizeValue, quoteIdentifier, @@ -54,16 +60,26 @@ export class DataStoreRowsRepository { return `${tablePrefix}data_store_user_${dataStoreId}`; } - async insertRows(dataStoreId: string, rows: DataStoreRows, columns: DataStoreColumn[]) { - const insertedIds: number[] = []; + async insertRows( + dataStoreId: string, + rows: DataStoreRows, + columns: DataStoreColumn[], + returnData: boolean = false, + ) { + const inserted: DataStoreRowWithId[] = []; + const dbType = this.dataSource.options.type; + const useReturning = dbType === 'postgres' || dbType === 'mariadb'; + + const table = this.toTableName(dataStoreId); + const columnNames = columns.map((c) => c.name); + const escapedColumns = columns.map((c) => this.dataSource.driver.escape(c.name)); + const selectColumns = ['id', ...escapedColumns]; // We insert one by one as the default behavior of returning the last inserted ID // is consistent, whereas getting all inserted IDs when inserting multiple values is // surprisingly awkward without Entities, e.g. `RETURNING id` explicitly does not aggregate // and the `identifiers` array output of `execute()` is empty for (const row of rows) { - const dbType = this.dataSource.options.type; - for (const column of columns) { row[column.name] = normalizeValue(row[column.name], column.type, dbType); } @@ -71,22 +87,47 @@ export class DataStoreRowsRepository { const query = this.dataSource .createQueryBuilder() .insert() - .into( - this.toTableName(dataStoreId), - columns.map((c) => c.name), - ) + .into(table, columnNames) .values(row); - if (dbType === 'postgres' || dbType === 'mariadb') { - query.returning('id'); + if (useReturning) { + query.returning(returnData ? selectColumns.join(',') : 'id'); } const result = await query.execute(); - insertedIds.push(...extractInsertedIds(result.raw, dbType)); + if (useReturning) { + const returned = extractReturningData(result.raw); + inserted.push.apply(inserted, returned); + continue; + } + + // Engines without RETURNING support + const rowIds = extractInsertedIds(result.raw, dbType); + if (rowIds.length === 0) { + throw new UnexpectedError("Couldn't find the inserted row ID"); + } + + if (!returnData) { + inserted.push(...rowIds.map((id) => ({ id }))); + continue; + } + + const insertedRow = await this.dataSource + .createQueryBuilder() + .select(selectColumns) + .from(table, 'dataStore') + .where({ id: In(rowIds) }) + .getRawOne(); + + if (!insertedRow) { + throw new UnexpectedError("Couldn't find the inserted row"); + } + + inserted.push(insertedRow); } - return insertedIds; + return inserted; } // TypeORM cannot infer the columns for a dynamic table name, so we use a raw query diff --git a/packages/cli/src/modules/data-store/data-store.controller.ts b/packages/cli/src/modules/data-store/data-store.controller.ts index 4b3f593c2d..c827e5326d 100644 --- a/packages/cli/src/modules/data-store/data-store.controller.ts +++ b/packages/cli/src/modules/data-store/data-store.controller.ts @@ -246,7 +246,12 @@ export class DataStoreController { @Body dto: AddDataStoreRowsDto, ) { try { - return await this.dataStoreService.insertRows(dataStoreId, req.params.projectId, dto.data); + return await this.dataStoreService.insertRows( + dataStoreId, + req.params.projectId, + dto.data, + dto.returnData, + ); } catch (e: unknown) { if (e instanceof DataStoreNotFoundError) { throw new NotFoundError(e.message); diff --git a/packages/cli/src/modules/data-store/data-store.service.ts b/packages/cli/src/modules/data-store/data-store.service.ts index 8628e30370..ca9ad4f905 100644 --- a/packages/cli/src/modules/data-store/data-store.service.ts +++ b/packages/cli/src/modules/data-store/data-store.service.ts @@ -125,12 +125,17 @@ export class DataStoreService { return await this.dataStoreColumnRepository.getColumns(dataStoreId); } - async insertRows(dataStoreId: string, projectId: string, rows: DataStoreRows) { + async insertRows( + dataStoreId: string, + projectId: string, + rows: DataStoreRows, + returnData: boolean = false, + ) { await this.validateDataStoreExists(dataStoreId, projectId); await this.validateRows(dataStoreId, rows); const columns = await this.dataStoreColumnRepository.getColumns(dataStoreId); - return await this.dataStoreRowsRepository.insertRows(dataStoreId, rows, columns); + return await this.dataStoreRowsRepository.insertRows(dataStoreId, rows, columns, returnData); } async upsertRows(dataStoreId: string, projectId: string, dto: UpsertDataStoreRowsDto) { diff --git a/packages/cli/src/modules/data-store/utils/sql-utils.ts b/packages/cli/src/modules/data-store/utils/sql-utils.ts index d8dbb6113c..0157010892 100644 --- a/packages/cli/src/modules/data-store/utils/sql-utils.ts +++ b/packages/cli/src/modules/data-store/utils/sql-utils.ts @@ -5,13 +5,13 @@ import { } from '@n8n/api-types'; import { DslColumn } from '@n8n/db'; import type { DataSourceOptions } from '@n8n/typeorm'; -import type { DataStoreColumnJsType, DataStoreRows } from 'n8n-workflow'; +import type { DataStoreColumnJsType, DataStoreRows, DataStoreRowWithId } from 'n8n-workflow'; import { UnexpectedError } from 'n8n-workflow'; -import type { DataStoreUserTableName } from '../data-store.types'; - import { NotFoundError } from '@/errors/response-errors/not-found.error'; +import type { DataStoreUserTableName } from '../data-store.types'; + export function toDslColumns(columns: DataStoreCreateColumnSchema[]): DslColumn[] { return columns.map((col) => { const name = new DslColumn(col.name.trim()); @@ -155,6 +155,16 @@ function hasRowId(data: unknown): data is WithRowId { return typeof data === 'object' && data !== null && 'id' in data && isNumber(data.id); } +export function extractReturningData(raw: unknown): DataStoreRowWithId[] { + if (!isArrayOf(raw, hasRowId)) { + throw new UnexpectedError( + 'Expected INSERT INTO raw to be { id: number }[] on Postgres or MariaDB', + ); + } + + return raw; +} + export function extractInsertedIds(raw: unknown, dbType: DataSourceOptions['type']): number[] { switch (dbType) { case 'postgres': diff --git a/packages/workflow/src/data-store.types.ts b/packages/workflow/src/data-store.types.ts index 3ddfde62ed..3da84e14e2 100644 --- a/packages/workflow/src/data-store.types.ts +++ b/packages/workflow/src/data-store.types.ts @@ -74,6 +74,7 @@ export type DataStoreColumnJsType = string | number | boolean | Date; export type DataStoreRow = Record; export type DataStoreRows = DataStoreRow[]; +export type DataStoreRowWithId = DataStoreRow & { id: number }; // APIs for a data store service operating on a specific projectId export interface IDataStoreProjectAggregateService { @@ -101,7 +102,7 @@ export interface IDataStoreProjectService { dto: Partial, ): Promise<{ count: number; data: DataStoreRows }>; - insertRows(rows: DataStoreRows): Promise; + insertRows(rows: DataStoreRows): Promise>; upsertRows(options: UpsertDataStoreRowsOptions): Promise; }