From 897c69c70dbdc5576f92785058ecb587cb96bf20 Mon Sep 17 00:00:00 2001 From: Charlie Kolb Date: Tue, 9 Sep 2025 14:01:40 +0200 Subject: [PATCH] feat(Data Table Node): Add bulk insert mode (no-changelog) (#19294) --- .../dto/data-store/add-data-store-rows.dto.ts | 3 +- .../src/schemas/data-store.schema.ts | 2 + .../data-store-proxy.service.test.ts | 9 +- .../__tests__/data-store.controller.test.ts | 16 +- .../__tests__/data-store.service.test.ts | 280 +++++++++++++----- .../data-table/data-store-proxy.service.ts | 8 +- .../data-table/data-store-rows.repository.ts | 76 ++++- .../data-table/data-store.controller.ts | 6 +- .../modules/data-table/data-store.service.ts | 19 +- .../test/integration/shared/db/data-stores.ts | 2 +- .../src/features/dataStore/dataStore.api.ts | 2 +- .../nodes/DataTable/actions/router.ts | 7 +- .../DataTable/actions/row/insert.operation.ts | 47 ++- packages/workflow/src/data-store.types.ts | 15 +- 14 files changed, 383 insertions(+), 109 deletions(-) diff --git a/packages/@n8n/api-types/src/dto/data-store/add-data-store-rows.dto.ts b/packages/@n8n/api-types/src/dto/data-store/add-data-store-rows.dto.ts index 0797f0823e..094564b05c 100644 --- a/packages/@n8n/api-types/src/dto/data-store/add-data-store-rows.dto.ts +++ b/packages/@n8n/api-types/src/dto/data-store/add-data-store-rows.dto.ts @@ -4,9 +4,10 @@ import { Z } from 'zod-class'; import { dataStoreColumnNameSchema, dataStoreColumnValueSchema, + insertRowReturnType, } from '../../schemas/data-store.schema'; export class AddDataStoreRowsDto extends Z.class({ - returnData: z.boolean().optional().default(false), data: z.array(z.record(dataStoreColumnNameSchema, dataStoreColumnValueSchema)), + returnType: insertRowReturnType, }) {} diff --git a/packages/@n8n/api-types/src/schemas/data-store.schema.ts b/packages/@n8n/api-types/src/schemas/data-store.schema.ts index 3e3588307c..edf41538cd 100644 --- a/packages/@n8n/api-types/src/schemas/data-store.schema.ts +++ b/packages/@n8n/api-types/src/schemas/data-store.schema.ts @@ -2,6 +2,8 @@ import { z } from 'zod'; import type { ListDataStoreQueryDto } from '../dto'; +export const insertRowReturnType = z.union([z.literal('all'), z.literal('count'), z.literal('id')]); + export const dataStoreNameSchema = z.string().trim().min(1).max(128); export const dataStoreIdSchema = z.string().max(36); diff --git a/packages/cli/src/modules/data-table/__tests__/data-store-proxy.service.test.ts b/packages/cli/src/modules/data-table/__tests__/data-store-proxy.service.test.ts index 6d67a81613..40f00d1524 100644 --- a/packages/cli/src/modules/data-table/__tests__/data-store-proxy.service.test.ts +++ b/packages/cli/src/modules/data-table/__tests__/data-store-proxy.service.test.ts @@ -207,9 +207,14 @@ describe('DataStoreProxyService', () => { node, 'dataStore-id', ); - await dataStoreOperations.insertRows(rows); + await dataStoreOperations.insertRows(rows, 'count'); - expect(dataStoreServiceMock.insertRows).toBeCalledWith('dataStore-id', PROJECT_ID, rows, true); + expect(dataStoreServiceMock.insertRows).toBeCalledWith( + 'dataStore-id', + PROJECT_ID, + rows, + 'count', + ); }); it('should call upsertRow with correct parameters', async () => { diff --git a/packages/cli/src/modules/data-table/__tests__/data-store.controller.test.ts b/packages/cli/src/modules/data-table/__tests__/data-store.controller.test.ts index f7ff5d6a85..b76ca001de 100644 --- a/packages/cli/src/modules/data-table/__tests__/data-store.controller.test.ts +++ b/packages/cli/src/modules/data-table/__tests__/data-store.controller.test.ts @@ -1904,6 +1904,7 @@ describe('POST /projects/:projectId/data-tables/:dataStoreId/insert', () => { second: 'another value', }, ], + returnType: 'id', }; await authOwnerAgent @@ -1921,6 +1922,7 @@ describe('POST /projects/:projectId/data-tables/:dataStoreId/insert', () => { second: 'another value', }, ], + returnType: 'id', }; await authOwnerAgent @@ -1950,6 +1952,7 @@ describe('POST /projects/:projectId/data-tables/:dataStoreId/insert', () => { second: 'another value', }, ], + returnType: 'id', }; await authMemberAgent @@ -1981,6 +1984,7 @@ describe('POST /projects/:projectId/data-tables/:dataStoreId/insert', () => { second: 'another value', }, ], + returnType: 'id', }; await authMemberAgent @@ -2013,6 +2017,7 @@ describe('POST /projects/:projectId/data-tables/:dataStoreId/insert', () => { second: 'another value', }, ], + returnType: 'id', }; const response = await authMemberAgent @@ -2053,6 +2058,7 @@ describe('POST /projects/:projectId/data-tables/:dataStoreId/insert', () => { second: 'another value', }, ], + returnType: 'id', }; const response = await authAdminAgent @@ -2090,6 +2096,7 @@ describe('POST /projects/:projectId/data-tables/:dataStoreId/insert', () => { second: 'another value', }, ], + returnType: 'id', }; const response = await authMemberAgent @@ -2121,7 +2128,7 @@ describe('POST /projects/:projectId/data-tables/:dataStoreId/insert', () => { }); const payload = { - returnData: true, + returnType: 'all', data: [ { first: 'first row', @@ -2184,6 +2191,7 @@ describe('POST /projects/:projectId/data-tables/:dataStoreId/insert', () => { nonexisting: 'this does not exist', }, ], + returnType: 'id', }; const response = await authMemberAgent @@ -2217,6 +2225,7 @@ describe('POST /projects/:projectId/data-tables/:dataStoreId/insert', () => { b: '2025-08-15T12:34:56+02:00', }, ], + returnType: 'id', }; const response = await authMemberAgent @@ -2265,6 +2274,7 @@ describe('POST /projects/:projectId/data-tables/:dataStoreId/insert', () => { c: '2025-08-15T09:48:14.259Z', }, ], + returnType: 'id', }; const response = await authMemberAgent @@ -2305,6 +2315,7 @@ describe('POST /projects/:projectId/data-tables/:dataStoreId/insert', () => { b: false, }, ], + returnType: 'id', }; const response = await authMemberAgent @@ -2360,6 +2371,7 @@ describe('POST /projects/:projectId/data-tables/:dataStoreId/insert', () => { e: 2340439341231259, }, ], + returnType: 'id', }; const response = await authMemberAgent @@ -2410,6 +2422,7 @@ describe('POST /projects/:projectId/data-tables/:dataStoreId/insert', () => { d: null, }, ], + returnType: 'id', }; const response = await authMemberAgent @@ -2458,6 +2471,7 @@ describe('POST /projects/:projectId/data-tables/:dataStoreId/insert', () => { b: 3, }, ], + returnType: 'id', }; const first = await authMemberAgent diff --git a/packages/cli/src/modules/data-table/__tests__/data-store.service.test.ts b/packages/cli/src/modules/data-table/__tests__/data-store.service.test.ts index b715f3ba0f..8e911404b6 100644 --- a/packages/cli/src/modules/data-table/__tests__/data-store.service.test.ts +++ b/packages/cli/src/modules/data-table/__tests__/data-store.service.test.ts @@ -417,11 +417,16 @@ describe('dataStore', () => { ], }); - const results = await dataStoreService.insertRows(dataStoreId, project1.id, [ - { name: 'Alice', age: 30 }, - { name: 'Bob', age: 25 }, - { name: 'Charlie', age: 35 }, - ]); + const results = await dataStoreService.insertRows( + dataStoreId, + project1.id, + [ + { name: 'Alice', age: 30 }, + { name: 'Bob', age: 25 }, + { name: 'Charlie', age: 35 }, + ], + 'id', + ); expect(results).toEqual([ { @@ -474,9 +479,12 @@ describe('dataStore', () => { ]); // Verify we can insert new rows with the new column - const newRow = await dataStoreService.insertRows(dataStoreId, project1.id, [ - { name: 'David', age: 28, email: 'david@example.com' }, - ]); + const newRow = await dataStoreService.insertRows( + dataStoreId, + project1.id, + [{ name: 'David', age: 28, email: 'david@example.com' }], + 'id', + ); expect(newRow).toEqual([ { id: 4, @@ -832,7 +840,7 @@ describe('dataStore', () => { c4: 'iso 8601 date strings are okay too', }, ]; - const result = await dataStoreService.insertRows(dataStoreId, project1.id, rows); + const result = await dataStoreService.insertRows(dataStoreId, project1.id, rows, 'id'); // ASSERT expect(result).toEqual([{ id: 1 }, { id: 2 }, { id: 3 }, { id: 4 }]); @@ -867,15 +875,21 @@ describe('dataStore', () => { }); // Insert initial row - const initial = await dataStoreService.insertRows(dataStoreId, project1.id, [ - { c1: 1, c2: 'foo' }, - ]); + const initial = await dataStoreService.insertRows( + dataStoreId, + project1.id, + [{ c1: 1, c2: 'foo' }], + 'id', + ); expect(initial).toEqual([{ id: 1 }]); // Attempt to insert a row with the same primary key - const result = await dataStoreService.insertRows(dataStoreId, project1.id, [ - { c1: 1, c2: 'foo' }, - ]); + const result = await dataStoreService.insertRows( + dataStoreId, + project1.id, + [{ c1: 1, c2: 'foo' }], + 'id', + ); // ASSERT expect(result).toEqual([{ id: 2 }]); @@ -908,19 +922,29 @@ describe('dataStore', () => { }); // Insert initial row - const ids = await dataStoreService.insertRows(dataStoreId, project1.id, [ - { c1: 1, c2: 'foo' }, - { c1: 2, c2: 'bar' }, - ]); + const ids = await dataStoreService.insertRows( + dataStoreId, + project1.id, + [ + { c1: 1, c2: 'foo' }, + { c1: 2, c2: 'bar' }, + ], + 'id', + ); expect(ids).toEqual([{ id: 1 }, { id: 2 }]); await dataStoreService.deleteRows(dataStoreId, project1.id, [ids[0].id]); // Insert a new row - const result = await dataStoreService.insertRows(dataStoreId, project1.id, [ - { c1: 1, c2: 'baz' }, - { c1: 2, c2: 'faz' }, - ]); + const result = await dataStoreService.insertRows( + dataStoreId, + project1.id, + [ + { c1: 1, c2: 'baz' }, + { c1: 2, c2: 'faz' }, + ], + 'id', + ); // ASSERT expect(result).toEqual([{ id: 3 }, { id: 4 }]); @@ -970,7 +994,7 @@ describe('dataStore', () => { { c1: 2, c2: 'bar', c3: false, c4: now }, { c1: null, c2: null, c3: null, c4: null }, ], - true, + 'all', ); expect(ids).toEqual([ { @@ -1026,7 +1050,7 @@ describe('dataStore', () => { { c2: 'bar', c1: 2, c3: false, c4: now }, { c1: null, c2: null, c3: null, c4: null }, ], - true, + 'all', ); expect(ids).toEqual([ { @@ -1072,10 +1096,15 @@ describe('dataStore', () => { }); // ACT - const result = dataStoreService.insertRows(dataStoreId, project1.id, [ - { c1: 3, c2: true, c3: new Date(), c4: 'hello?' }, - { cWrong: 3, c1: 4, c2: true, c3: new Date(), c4: 'hello?' }, - ]); + const result = dataStoreService.insertRows( + dataStoreId, + project1.id, + [ + { c1: 3, c2: true, c3: new Date(), c4: 'hello?' }, + { cWrong: 3, c1: 4, c2: true, c3: new Date(), c4: 'hello?' }, + ], + 'id', + ); // ASSERT await expect(result).rejects.toThrow( @@ -1096,12 +1125,17 @@ describe('dataStore', () => { }); // ACT - await dataStoreService.insertRows(dataStoreId, project1.id, [ - { name: 'Mary', age: 20, email: 'mary@example.com', active: true }, // full row - { name: 'Alice', age: 30 }, // missing email and active - { name: 'Bob' }, // missing age, email and active - {}, // missing all columns - ]); + await dataStoreService.insertRows( + dataStoreId, + project1.id, + [ + { name: 'Mary', age: 20, email: 'mary@example.com', active: true }, // full row + { name: 'Alice', age: 30 }, // missing email and active + { name: 'Bob' }, // missing age, email and active + {}, // missing all columns + ], + 'id', + ); const { count, data } = await dataStoreService.getManyRowsAndCount( dataStoreId, @@ -1150,10 +1184,15 @@ describe('dataStore', () => { }); // ACT - const result = dataStoreService.insertRows(dataStoreId, project1.id, [ - { c1: 3, c2: true, c3: new Date(), c4: 'hello?' }, - { cWrong: 3, c2: true, c3: new Date(), c4: 'hello?' }, - ]); + const result = dataStoreService.insertRows( + dataStoreId, + project1.id, + [ + { c1: 3, c2: true, c3: new Date(), c4: 'hello?' }, + { cWrong: 3, c2: true, c3: new Date(), c4: 'hello?' }, + ], + 'id', + ); // ASSERT await expect(result).rejects.toThrow( @@ -1169,9 +1208,12 @@ describe('dataStore', () => { }); // ACT - const result = dataStoreService.insertRows(dataStoreId, project1.id, [ - { c1: '2025-99-15T09:48:14.259Z' }, - ]); + const result = dataStoreService.insertRows( + dataStoreId, + project1.id, + [{ c1: '2025-99-15T09:48:14.259Z' }], + 'id', + ); // ASSERT await expect(result).rejects.toThrow(DataStoreValidationError); @@ -1193,10 +1235,15 @@ describe('dataStore', () => { }); // ACT - const result = dataStoreService.insertRows('this is not an id', project1.id, [ - { c1: 3, c2: true, c3: new Date(), c4: 'hello?' }, - { cWrong: 3, c2: true, c3: new Date(), c4: 'hello?' }, - ]); + const result = dataStoreService.insertRows( + 'this is not an id', + project1.id, + [ + { c1: 3, c2: true, c3: new Date(), c4: 'hello?' }, + { cWrong: 3, c2: true, c3: new Date(), c4: 'hello?' }, + ], + 'id', + ); // ASSERT await expect(result).rejects.toThrow(DataStoreNotFoundError); @@ -1211,10 +1258,12 @@ describe('dataStore', () => { // ACT const wrongValue = new Date().toISOString(); - const result = dataStoreService.insertRows(dataStoreId, project1.id, [ - { c1: 3 }, - { c1: wrongValue }, - ]); + const result = dataStoreService.insertRows( + dataStoreId, + project1.id, + [{ c1: 3 }, { c1: wrongValue }], + 'id', + ); // ASSERT await expect(result).rejects.toThrow(DataStoreValidationError); @@ -1232,7 +1281,7 @@ describe('dataStore', () => { // ACT const rows = [{}, {}, {}]; - const result = await dataStoreService.insertRows(dataStoreId, project1.id, rows); + const result = await dataStoreService.insertRows(dataStoreId, project1.id, rows, 'id'); // ASSERT expect(result).toEqual([{ id: 1 }, { id: 2 }, { id: 3 }]); @@ -1261,6 +1310,69 @@ describe('dataStore', () => { }, ]); }); + describe('bulk', () => { + it('handles single empty row correctly in bulk mode', async () => { + // ARRANGE + const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, { + name: 'dataStore', + columns: [], + }); + + // ACT + const rows = [{}]; + const result = await dataStoreService.insertRows(dataStoreId, project1.id, rows); + + // ASSERT + expect(result).toEqual({ success: true, insertedRows: 1 }); + + const { count, data } = await dataStoreService.getManyRowsAndCount( + dataStoreId, + project1.id, + {}, + ); + expect(count).toEqual(1); + + expect(data).toEqual([{ id: 1, createdAt: expect.any(Date), updatedAt: expect.any(Date) }]); + }); + it('handles multi-batch bulk correctly in bulk mode', async () => { + // ARRANGE + const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, { + name: 'dataStore', + columns: [ + { name: 'c1', type: 'number' }, + { name: 'c2', type: 'boolean' }, + { name: 'c3', type: 'string' }, + ], + }); + + // ACT + const rows = Array.from({ length: 3000 }, (_, index) => ({ + c1: index, + c2: index % 2 === 0, + c3: `index ${index}`, + })); + const result = await dataStoreService.insertRows(dataStoreId, project1.id, rows); + + // ASSERT + expect(result).toEqual({ success: true, insertedRows: rows.length }); + + const { count, data } = await dataStoreService.getManyRowsAndCount( + dataStoreId, + project1.id, + {}, + ); + expect(count).toEqual(rows.length); + + const expected = rows.map( + (row, i) => + expect.objectContaining({ + ...row, + id: i + 1, + }) as jest.AsymmetricMatcher, + ); + expect(data).toEqual(expected); + }); + }); }); describe('upsertRow', () => { @@ -1275,11 +1387,16 @@ describe('dataStore', () => { ], }); - const ids = await dataStoreService.insertRows(dataStoreId, project1.id, [ - { pid: '1995-111a', name: 'Alice', age: 30 }, - { pid: '1994-222a', name: 'John', age: 31 }, - { pid: '1993-333a', name: 'Paul', age: 32 }, - ]); + const ids = await dataStoreService.insertRows( + dataStoreId, + project1.id, + [ + { pid: '1995-111a', name: 'Alice', age: 30 }, + { pid: '1994-222a', name: 'John', age: 31 }, + { pid: '1993-333a', name: 'Paul', age: 32 }, + ], + 'id', + ); expect(ids).toEqual([{ id: 1 }, { id: 2 }, { id: 3 }]); @@ -1335,9 +1452,12 @@ describe('dataStore', () => { }); // Insert initial row - const ids = await dataStoreService.insertRows(dataStoreId, project1.id, [ - { pid: '1995-111a', name: 'Alice', age: 30 }, - ]); + const ids = await dataStoreService.insertRows( + dataStoreId, + project1.id, + [{ pid: '1995-111a', name: 'Alice', age: 30 }], + 'id', + ); expect(ids).toEqual([{ id: 1 }]); // ACT @@ -1387,9 +1507,12 @@ describe('dataStore', () => { }); // Insert initial row - const ids = await dataStoreService.insertRows(dataStoreId, project1.id, [ - { fullName: 'Alice Cooper', age: 30, birthday: new Date('1995-01-01') }, - ]); + const ids = await dataStoreService.insertRows( + dataStoreId, + project1.id, + [{ fullName: 'Alice Cooper', age: 30, birthday: new Date('1995-01-01') }], + 'id', + ); expect(ids).toEqual([{ id: 1 }]); // ACT @@ -1431,9 +1554,12 @@ describe('dataStore', () => { }); // Insert initial row - const ids = await dataStoreService.insertRows(dataStoreId, project1.id, [ - { fullName: 'Alice Cooper', age: 30, birthday: new Date('1995-01-01') }, - ]); + const ids = await dataStoreService.insertRows( + dataStoreId, + project1.id, + [{ fullName: 'Alice Cooper', age: 30, birthday: new Date('1995-01-01') }], + 'id', + ); expect(ids).toEqual([{ id: 1 }]); // ACT @@ -1477,11 +1603,16 @@ describe('dataStore', () => { const { id: dataStoreId } = dataStore; // Insert test rows - const ids = await dataStoreService.insertRows(dataStoreId, project1.id, [ - { name: 'Alice', age: 30 }, - { name: 'Bob', age: 25 }, - { name: 'Charlie', age: 35 }, - ]); + const ids = await dataStoreService.insertRows( + dataStoreId, + project1.id, + [ + { name: 'Alice', age: 30 }, + { name: 'Bob', age: 25 }, + { name: 'Charlie', age: 35 }, + ], + 'id', + ); expect(ids).toEqual([{ id: 1 }, { id: 2 }, { id: 3 }]); // ACT - Delete first and third rows @@ -1531,7 +1662,12 @@ describe('dataStore', () => { const { id: dataStoreId } = dataStore; // Insert one row - const ids = await dataStoreService.insertRows(dataStoreId, project1.id, [{ name: 'Alice' }]); + const ids = await dataStoreService.insertRows( + dataStoreId, + project1.id, + [{ name: 'Alice' }], + 'id', + ); expect(ids).toEqual([{ id: 1 }]); // ACT - Try to delete existing and non-existing IDs @@ -2299,7 +2435,7 @@ describe('dataStore', () => { }, ]; - const ids = await dataStoreService.insertRows(dataStoreId, project1.id, rows); + const ids = await dataStoreService.insertRows(dataStoreId, project1.id, rows, 'id'); expect(ids).toEqual([{ id: 1 }, { id: 2 }, { id: 3 }]); // ACT diff --git a/packages/cli/src/modules/data-table/data-store-proxy.service.ts b/packages/cli/src/modules/data-table/data-store-proxy.service.ts index 068e038c78..24ac2fffc3 100644 --- a/packages/cli/src/modules/data-table/data-store-proxy.service.ts +++ b/packages/cli/src/modules/data-table/data-store-proxy.service.ts @@ -10,6 +10,7 @@ import { DataStoreRows, IDataStoreProjectAggregateService, IDataStoreProjectService, + DataTableInsertRowsReturnType, INode, ListDataStoreOptions, ListDataStoreRowsOptions, @@ -131,8 +132,11 @@ export class DataStoreProxyService implements DataStoreProxyProvider { return await dataStoreService.getManyRowsAndCount(dataStoreId, projectId, options); }, - async insertRows(rows: DataStoreRows) { - return await dataStoreService.insertRows(dataStoreId, projectId, rows, true); + async insertRows( + rows: DataStoreRows, + returnType: T, + ) { + return await dataStoreService.insertRows(dataStoreId, projectId, rows, returnType); }, async updateRow(options: UpdateDataStoreRowOptions) { diff --git a/packages/cli/src/modules/data-table/data-store-rows.repository.ts b/packages/cli/src/modules/data-table/data-store-rows.repository.ts index 49008d0c98..fb543fb36f 100644 --- a/packages/cli/src/modules/data-table/data-store-rows.repository.ts +++ b/packages/cli/src/modules/data-table/data-store-rows.repository.ts @@ -18,6 +18,8 @@ import { UnexpectedError, DataStoreRowsReturn, DATA_TABLE_SYSTEM_COLUMNS, + DataTableInsertRowsReturnType, + DataTableInsertRowsResult, } from 'n8n-workflow'; import { DataStoreUserTableName } from './data-store.types'; @@ -156,18 +158,65 @@ export class DataStoreRowsRepository { return `${tablePrefix}data_table_user_${dataStoreId}`; } - async insertRows( + async insertRowsBulk( + table: DataStoreUserTableName, + rows: DataStoreRows, + columns: DataTableColumn[], + ) { + // DB systems have different maximum parameters per query + // with old sqlite versions having the lowest in 999 parameters + // In practice 20000 works here, but performance didn't meaningfully change + // so this should be a safe limit + const batchSize = 800; + const batches = Math.max(1, Math.ceil((columns.length * rows.length) / batchSize)); + const rowsPerBatch = Math.ceil(rows.length / batches); + + const columnNames = columns.map((x) => x.name); + const dbType = this.dataSource.options.type; + + let insertedRows = 0; + for (let i = 0; i < batches; ++i) { + const start = i * rowsPerBatch; + const endExclusive = Math.min(rows.length, (i + 1) * rowsPerBatch); + + if (endExclusive <= start) break; + + const completeRows = new Array(endExclusive - start); + for (let j = start; j < endExclusive; ++j) { + const insertArray: DataStoreColumnJsType[] = []; + + for (let h = 0; h < columnNames.length; ++h) { + const column = columns[h]; + // Fill missing columns with null values to support partial data insertion + const value = rows[j][column.name] ?? null; + insertArray[h] = normalizeValue(value, column.type, dbType); + } + completeRows[j - start] = insertArray; + } + + const query = this.dataSource + .createQueryBuilder() + .insert() + .into(table, columnNames) + .values(completeRows); + await query.execute(); + insertedRows += completeRows.length; + } + return { success: true, insertedRows } as const; + } + + async insertRows( dataStoreId: string, rows: DataStoreRows, columns: DataTableColumn[], - returnData?: T, - ): Promise>>; - async insertRows( + returnType: T, + ): Promise>; + async insertRows( dataStoreId: string, rows: DataStoreRows, columns: DataTableColumn[], - returnData?: boolean, - ): Promise>> { + returnType: T, + ): Promise { const inserted: Array> = []; const dbType = this.dataSource.options.type; const useReturning = dbType === 'postgres' || dbType === 'mariadb'; @@ -179,6 +228,10 @@ export class DataStoreRowsRepository { ); const selectColumns = [...escapedSystemColumns, ...escapedColumns]; + if (returnType === 'count') { + return await this.insertRowsBulk(table, rows, columns); + } + // We insert one by one as the default behavior of returning the last inserted ID // is consistent, whereas getting all inserted IDs when inserting multiple values is // surprisingly awkward without Entities, e.g. `RETURNING id` explicitly does not aggregate @@ -196,15 +249,16 @@ export class DataStoreRowsRepository { const query = this.dataSource.createQueryBuilder().insert().into(table).values(completeRow); if (useReturning) { - query.returning(returnData ? selectColumns.join(',') : 'id'); + query.returning(returnType === 'all' ? selectColumns.join(',') : 'id'); } const result = await query.execute(); if (useReturning) { - const returned = returnData - ? normalizeRows(extractReturningData(result.raw), columns) - : extractInsertedIds(result.raw, dbType).map((id) => ({ id })); + const returned = + returnType === 'all' + ? normalizeRows(extractReturningData(result.raw), columns) + : extractInsertedIds(result.raw, dbType).map((id) => ({ id })); inserted.push.apply(inserted, returned); continue; } @@ -215,7 +269,7 @@ export class DataStoreRowsRepository { throw new UnexpectedError("Couldn't find the inserted row ID"); } - if (!returnData) { + if (returnType === 'id') { inserted.push(...ids.map((id) => ({ id }))); continue; } diff --git a/packages/cli/src/modules/data-table/data-store.controller.ts b/packages/cli/src/modules/data-table/data-store.controller.ts index 5eb5b5d7e9..3ee9e6b8dd 100644 --- a/packages/cli/src/modules/data-table/data-store.controller.ts +++ b/packages/cli/src/modules/data-table/data-store.controller.ts @@ -238,11 +238,11 @@ export class DataStoreController { /** * @returns the IDs of the inserted rows */ - async appendDataStoreRows( + async appendDataStoreRows( req: AuthenticatedRequest<{ projectId: string }>, _res: Response, dataStoreId: string, - dto: AddDataStoreRowsDto & { returnData?: T }, + dto: AddDataStoreRowsDto & { returnType?: T }, ): Promise>>; @Post('/:dataStoreId/insert') @ProjectScope('dataStore:writeRow') @@ -257,7 +257,7 @@ export class DataStoreController { dataStoreId, req.params.projectId, dto.data, - dto.returnData, + dto.returnType, ); } catch (e: unknown) { if (e instanceof DataStoreNotFoundError) { diff --git a/packages/cli/src/modules/data-table/data-store.service.ts b/packages/cli/src/modules/data-table/data-store.service.ts index 6d1a8b9ba3..4e036aea14 100644 --- a/packages/cli/src/modules/data-table/data-store.service.ts +++ b/packages/cli/src/modules/data-table/data-store.service.ts @@ -17,6 +17,8 @@ import type { DataStoreRow, DataStoreRowReturn, DataStoreRows, + DataTableInsertRowsReturnType, + DataTableInsertRowsResult, } from 'n8n-workflow'; import { validateFieldType } from 'n8n-workflow'; @@ -134,23 +136,23 @@ export class DataStoreService { return await this.dataStoreColumnRepository.getColumns(dataStoreId); } - async insertRows( + async insertRows( dataStoreId: string, projectId: string, rows: DataStoreRows, - returnData?: T, - ): Promise>>; + returnType?: T, + ): Promise>; async insertRows( dataStoreId: string, projectId: string, rows: DataStoreRows, - returnData?: boolean, + returnType: DataTableInsertRowsReturnType = 'count', ) { await this.validateDataStoreExists(dataStoreId, projectId); await this.validateRows(dataStoreId, rows); const columns = await this.dataStoreColumnRepository.getColumns(dataStoreId); - return await this.dataStoreRowsRepository.insertRows(dataStoreId, rows, columns, returnData); + return await this.dataStoreRowsRepository.insertRows(dataStoreId, rows, columns, returnType); } async upsertRow( @@ -172,7 +174,12 @@ export class DataStoreService { } // No rows were updated, so insert a new one - const inserted = await this.insertRows(dataStoreId, projectId, [dto.data], returnData); + const inserted = await this.insertRows( + dataStoreId, + projectId, + [dto.data], + returnData ? 'all' : 'count', + ); return returnData ? inserted : true; } diff --git a/packages/cli/test/integration/shared/db/data-stores.ts b/packages/cli/test/integration/shared/db/data-stores.ts index 1e56da78fe..c917bcd9d2 100644 --- a/packages/cli/test/integration/shared/db/data-stores.ts +++ b/packages/cli/test/integration/shared/db/data-stores.ts @@ -36,7 +36,7 @@ export const createDataStore = async ( const columns = await dataStoreColumnRepository.getColumns(dataStore.id); const dataStoreRowsRepository = Container.get(DataStoreRowsRepository); - await dataStoreRowsRepository.insertRows(dataStore.id, options.data, columns); + await dataStoreRowsRepository.insertRows(dataStore.id, options.data, columns, 'count'); } return dataStore; diff --git a/packages/frontend/editor-ui/src/features/dataStore/dataStore.api.ts b/packages/frontend/editor-ui/src/features/dataStore/dataStore.api.ts index 29f36a8beb..1ccf9860fb 100644 --- a/packages/frontend/editor-ui/src/features/dataStore/dataStore.api.ts +++ b/packages/frontend/editor-ui/src/features/dataStore/dataStore.api.ts @@ -157,7 +157,7 @@ export const insertDataStoreRowApi = async ( 'POST', `/projects/${projectId}/data-tables/${dataStoreId}/insert`, { - returnData: true, + returnType: 'all', data: [row], }, ); diff --git a/packages/nodes-base/nodes/DataTable/actions/router.ts b/packages/nodes-base/nodes/DataTable/actions/router.ts index 0a0149996a..dc3755eb11 100644 --- a/packages/nodes-base/nodes/DataTable/actions/router.ts +++ b/packages/nodes-base/nodes/DataTable/actions/router.ts @@ -15,7 +15,7 @@ type DataTableNodeType = AllEntities<{ const BULK_OPERATIONS = ['insert'] as const; -function canBulk(operation: string): operation is (typeof BULK_OPERATIONS)[number] { +function hasBulkExecute(operation: string): operation is (typeof BULK_OPERATIONS)[number] { return (BULK_OPERATIONS as readonly string[]).includes(operation); } @@ -41,7 +41,7 @@ export async function router(this: IExecuteFunctions): Promise { + const optimizeBulkEnabled = this.getNodeParameter('options.optimizeBulk', index, false); const dataStoreProxy = await getDataTableProxyExecute(this, index); const row = getAddRow(this, index); - const insertedRows = await dataStoreProxy.insertRows([row]); - return insertedRows.map((json) => ({ json })); + if (optimizeBulkEnabled) { + // This function is always called by index, so we inherently cannot operate in bulk + this.addExecutionHints({ + message: 'Unable to optimize bulk insert due to expression in Data Table ID ', + location: 'outputPane', + }); + const json = await dataStoreProxy.insertRows([row], 'count'); + return [{ json }]; + } else { + const insertedRows = await dataStoreProxy.insertRows([row], 'all'); + return insertedRows.map((json, item) => ({ json, pairedItem: { item } })); + } } export async function executeBulk( this: IExecuteFunctions, proxy: IDataStoreProjectService, ): Promise { + const optimizeBulkEnabled = this.getNodeParameter('options.optimizeBulk', 0, false); const rows = this.getInputData().flatMap((_, i) => [getAddRow(this, i)]); - const insertedRows = await proxy.insertRows(rows); - return insertedRows.map((json, item) => ({ json, pairedItem: { item } })); + if (optimizeBulkEnabled) { + const json = await proxy.insertRows(rows, 'count'); + return [{ json }]; + } else { + const insertedRows = await proxy.insertRows(rows, 'all'); + return insertedRows.map((json, item) => ({ json, pairedItem: { item } })); + } } diff --git a/packages/workflow/src/data-store.types.ts b/packages/workflow/src/data-store.types.ts index 0305510651..29053234d7 100644 --- a/packages/workflow/src/data-store.types.ts +++ b/packages/workflow/src/data-store.types.ts @@ -88,6 +88,16 @@ export type DataStoreRows = DataStoreRow[]; export type DataStoreRowReturn = DataStoreRow & DataStoreRowReturnBase; export type DataStoreRowsReturn = DataStoreRowReturn[]; +export type DataTableInsertRowsReturnType = 'all' | 'id' | 'count'; +export type DataTableInsertRowsBulkResult = { success: true; insertedRows: number }; +export type DataTableInsertRowsResult< + T extends DataTableInsertRowsReturnType = DataTableInsertRowsReturnType, +> = T extends 'all' + ? DataStoreRowReturn[] + : T extends 'id' + ? Array> + : DataTableInsertRowsBulkResult; + // APIs for a data store service operating on a specific projectId export interface IDataStoreProjectAggregateService { getProjectId(): string; @@ -116,7 +126,10 @@ export interface IDataStoreProjectService { dto: Partial, ): Promise<{ count: number; data: DataStoreRowsReturn }>; - insertRows(rows: DataStoreRows): Promise; + insertRows( + rows: DataStoreRows, + returnType: T, + ): Promise>; updateRow(options: UpdateDataStoreRowOptions): Promise;