feat(core): Optionally return full rows from Data Table inserts (no-changelog) (#18657)

packages/@n8n/api-types/src/dto/data-store/add-data-store-rows.dto.ts

@@ -7,5 +7,6 @@ import {
 } from '../../schemas/data-store.schema';
 
 export class AddDataStoreRowsDto extends Z.class({
+	returnData: z.boolean().default(false),
 	data: z.array(z.record(dataStoreColumnNameSchema, dataStoreColumnValueSchema)),
 }) {}

packages/cli/src/modules/data-store/__tests__/data-store.controller.test.ts

@@ -1835,7 +1835,7 @@ describe('POST /projects/:projectId/data-stores/:dataStoreId/insert', () => {
 			.expect(200);
 
 		expect(response.body).toEqual({
-			data: [1],
+			data: [{ id: 1 }],
 		});
 
 		const rowsInDb = await dataStoreRowsRepository.getManyAndCount(dataStore.id, {});
@@ -1875,7 +1875,7 @@ describe('POST /projects/:projectId/data-stores/:dataStoreId/insert', () => {
 			.expect(200);
 
 		expect(response.body).toEqual({
-			data: [1],
+			data: [{ id: 1 }],
 		});
 
 		const rowsInDb = await dataStoreRowsRepository.getManyAndCount(dataStore.id, {});
@@ -1912,7 +1912,7 @@ describe('POST /projects/:projectId/data-stores/:dataStoreId/insert', () => {
 			.expect(200);
 
 		expect(response.body).toEqual({
-			data: [1],
+			data: [{ id: 1 }],
 		});
 
 		const rowsInDb = await dataStoreRowsRepository.getManyAndCount(dataStore.id, {});
@@ -1920,6 +1920,59 @@ describe('POST /projects/:projectId/data-stores/:dataStoreId/insert', () => {
 		expect(rowsInDb.data[0]).toMatchObject(payload.data[0]);
 	});
 
+	test('should return inserted data if returnData is set', async () => {
+		const dataStore = await createDataStore(memberProject, {
+			columns: [
+				{
+					name: 'first',
+					type: 'string',
+				},
+				{
+					name: 'second',
+					type: 'string',
+				},
+			],
+		});
+
+		const payload = {
+			returnData: true,
+			data: [
+				{
+					first: 'first row',
+					second: 'some value',
+				},
+				{
+					first: 'another row',
+					second: 'another value',
+				},
+			],
+		};
+
+		const response = await authMemberAgent
+			.post(`/projects/${memberProject.id}/data-stores/${dataStore.id}/insert`)
+			.send(payload)
+			.expect(200);
+
+		expect(response.body).toEqual({
+			data: [
+				{
+					id: 1,
+					first: 'first row',
+					second: 'some value',
+				},
+				{
+					id: 2,
+					first: 'another row',
+					second: 'another value',
+				},
+			],
+		});
+
+		const rowsInDb = await dataStoreRowsRepository.getManyAndCount(dataStore.id, {});
+		expect(rowsInDb.count).toBe(2);
+		expect(rowsInDb.data[0]).toMatchObject(payload.data[0]);
+	});
+
 	test('should not insert rows when column does not exist', async () => {
 		const dataStore = await createDataStore(memberProject, {
 			columns: [
@@ -1982,7 +2035,7 @@ describe('POST /projects/:projectId/data-stores/:dataStoreId/insert', () => {
 			.expect(200);
 
 		expect(response.body).toEqual({
-			data: [1],
+			data: [{ id: 1 }],
 		});
 
 		const readResponse = await authMemberAgent
@@ -2030,7 +2083,7 @@ describe('POST /projects/:projectId/data-stores/:dataStoreId/insert', () => {
 			.expect(200);
 
 		expect(response.body).toEqual({
-			data: [1],
+			data: [{ id: 1 }],
 		});
 
 		const readResponse = await authMemberAgent
@@ -2070,7 +2123,7 @@ describe('POST /projects/:projectId/data-stores/:dataStoreId/insert', () => {
 			.expect(200);
 
 		expect(response.body).toEqual({
-			data: [1],
+			data: [{ id: 1 }],
 		});
 
 		const readResponse = await authMemberAgent
@@ -2125,7 +2178,7 @@ describe('POST /projects/:projectId/data-stores/:dataStoreId/insert', () => {
 			.expect(200);
 
 		expect(response.body).toEqual({
-			data: [1],
+			data: [{ id: 1 }],
 		});
 
 		const readResponse = await authMemberAgent
@@ -2175,7 +2228,7 @@ describe('POST /projects/:projectId/data-stores/:dataStoreId/insert', () => {
 			.expect(200);
 
 		expect(response.body).toEqual({
-			data: [1],
+			data: [{ id: 1 }],
 		});
 
 		const readResponse = await authMemberAgent
@@ -2223,7 +2276,7 @@ describe('POST /projects/:projectId/data-stores/:dataStoreId/insert', () => {
 			.expect(200);
 
 		expect(first.body).toEqual({
-			data: [1, 2, 3],
+			data: [{ id: 1 }, { id: 2 }, { id: 3 }],
 		});
 
 		const second = await authMemberAgent
@@ -2232,7 +2285,7 @@ describe('POST /projects/:projectId/data-stores/:dataStoreId/insert', () => {
 			.expect(200);
 
 		expect(second.body).toEqual({
-			data: [4, 5, 6],
+			data: [{ id: 4 }, { id: 5 }, { id: 6 }],
 		});
 
 		const readResponse = await authMemberAgent

packages/cli/src/modules/data-store/__tests__/data-store.service.test.ts

@@ -462,7 +462,17 @@ describe('dataStore', () => {
 				{ name: 'Charlie', age: 35 },
 			]);
 
-			expect(results).toEqual([1, 2, 3]);
+			expect(results).toEqual([
+				{
+					id: 1,
+				},
+				{
+					id: 2,
+				},
+				{
+					id: 3,
+				},
+			]);
 
 			// ACT
 			const newColumn = await dataStoreService.addColumn(dataStoreId, project1.id, {
@@ -491,7 +501,11 @@ describe('dataStore', () => {
 			const newRow = await dataStoreService.insertRows(dataStoreId, project1.id, [
 				{ name: 'David', age: 28, email: 'david@example.com' },
 			]);
-			expect(newRow).toEqual([4]);
+			expect(newRow).toEqual([
+				{
+					id: 4,
+				},
+			]);
 
 			const finalData = await dataStoreService.getManyRowsAndCount(dataStoreId, project1.id, {});
 			expect(finalData.count).toBe(4);
@@ -975,7 +989,7 @@ describe('dataStore', () => {
 			const result = await dataStoreService.insertRows(dataStoreId, project1.id, rows);
 
 			// ASSERT
-			expect(result).toEqual([1, 2, 3, 4]);
+			expect(result).toEqual([{ id: 1 }, { id: 2 }, { id: 3 }, { id: 4 }]);
 
 			const { count, data } = await dataStoreService.getManyRowsAndCount(
 				dataStoreId,
@@ -1009,7 +1023,7 @@ describe('dataStore', () => {
 			const initial = await dataStoreService.insertRows(dataStoreId, project1.id, [
 				{ c1: 1, c2: 'foo' },
 			]);
-			expect(initial).toEqual([1]);
+			expect(initial).toEqual([{ id: 1 }]);
 
 			// Attempt to insert a row with the same primary key
 			const result = await dataStoreService.insertRows(dataStoreId, project1.id, [
@@ -1017,7 +1031,7 @@ describe('dataStore', () => {
 			]);
 
 			// ASSERT
-			expect(result).toEqual([2]);
+			expect(result).toEqual([{ id: 2 }]);
 
 			const { count, data } = await dataStoreRowsRepository.getManyAndCount(dataStoreId, {});
 
@@ -1043,9 +1057,9 @@ describe('dataStore', () => {
 				{ c1: 1, c2: 'foo' },
 				{ c1: 2, c2: 'bar' },
 			]);
-			expect(ids).toEqual([1, 2]);
+			expect(ids).toEqual([{ id: 1 }, { id: 2 }]);
 
-			await dataStoreService.deleteRows(dataStoreId, project1.id, [ids[0]]);
+			await dataStoreService.deleteRows(dataStoreId, project1.id, [ids[0].id]);
 
 			// Insert a new row
 			const result = await dataStoreService.insertRows(dataStoreId, project1.id, [
@@ -1054,7 +1068,7 @@ describe('dataStore', () => {
 			]);
 
 			// ASSERT
-			expect(result).toEqual([3, 4]);
+			expect(result).toEqual([{ id: 3 }, { id: 4 }]);
 
 			const { count, data } = await dataStoreRowsRepository.getManyAndCount(dataStoreId, {});
 
@@ -1066,6 +1080,50 @@ describe('dataStore', () => {
 			]);
 		});
 
+		it('return inserted data if requested', async () => {
+			// ARRANGE
+			const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, {
+				name: 'myDataStore',
+				columns: [
+					{ name: 'c1', type: 'number' },
+					{ name: 'c2', type: 'string' },
+				],
+			});
+
+			// Insert initial row
+			const ids = await dataStoreService.insertRows(
+				dataStoreId,
+				project1.id,
+				[
+					{ c1: 1, c2: 'foo' },
+					{ c1: 2, c2: 'bar' },
+				],
+				true,
+			);
+			expect(ids).toEqual([
+				{ id: 1, c1: 1, c2: 'foo' },
+				{ id: 2, c1: 2, c2: 'bar' },
+			]);
+
+			await dataStoreService.deleteRows(dataStoreId, project1.id, [ids[0].id]);
+
+			const result = await dataStoreService.insertRows(
+				dataStoreId,
+				project1.id,
+				[
+					{ c1: 1, c2: 'baz' },
+					{ c1: 2, c2: 'faz' },
+				],
+				true,
+			);
+
+			// ASSERT
+			expect(result).toEqual([
+				{ id: 3, c1: 1, c2: 'baz' },
+				{ id: 4, c1: 2, c2: 'faz' },
+			]);
+		});
+
 		it('rejects a mismatched row with extra column', async () => {
 			// ARRANGE
 			const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, {
@@ -1205,7 +1263,7 @@ describe('dataStore', () => {
 			const result = await dataStoreService.insertRows(dataStoreId, project1.id, rows);
 
 			// ASSERT
-			expect(result).toEqual([1, 2, 3]);
+			expect(result).toEqual([{ id: 1 }, { id: 2 }, { id: 3 }]);
 
 			const { count, data } = await dataStoreService.getManyRowsAndCount(
 				dataStoreId,
@@ -1229,12 +1287,14 @@ describe('dataStore', () => {
 				],
 			});
 
-			await dataStoreService.insertRows(dataStoreId, project1.id, [
+			const ids = await dataStoreService.insertRows(dataStoreId, project1.id, [
 				{ pid: '1995-111a', name: 'Alice', age: 30 },
 				{ pid: '1994-222a', name: 'John', age: 31 },
 				{ pid: '1993-333a', name: 'Paul', age: 32 },
 			]);
 
+			expect(ids).toEqual([{ id: 1 }, { id: 2 }, { id: 3 }]);
+
 			// ACT
 			const result = await dataStoreService.upsertRows(dataStoreId, project1.id, {
 				rows: [
@@ -1323,7 +1383,7 @@ describe('dataStore', () => {
 			const ids = await dataStoreService.insertRows(dataStoreId, project1.id, [
 				{ pid: '1995-111a', name: 'Alice', age: 30 },
 			]);
-			expect(ids).toEqual([1]);
+			expect(ids).toEqual([{ id: 1 }]);
 
 			// ACT
 			const result = await dataStoreService.upsertRows(dataStoreId, project1.id, {
@@ -1366,7 +1426,7 @@ describe('dataStore', () => {
 				{ name: 'Bob', age: 25 },
 				{ name: 'Charlie', age: 35 },
 			]);
-			expect(ids).toEqual([1, 2, 3]);
+			expect(ids).toEqual([{ id: 1 }, { id: 2 }, { id: 3 }]);
 
 			// Get initial data to find row IDs
 			const initialData = await dataStoreService.getManyRowsAndCount(dataStoreId, project1.id, {});
@@ -1415,7 +1475,7 @@ describe('dataStore', () => {
 
 			// Insert one row
 			const ids = await dataStoreService.insertRows(dataStoreId, project1.id, [{ name: 'Alice' }]);
-			expect(ids).toEqual([1]);
+			expect(ids).toEqual([{ id: 1 }]);
 
 			// ACT - Try to delete existing and non-existing IDs
 			const result = await dataStoreService.deleteRows(dataStoreId, project1.id, [1, 999, 1000]);
@@ -1757,7 +1817,7 @@ describe('dataStore', () => {
 			];
 
 			const ids = await dataStoreService.insertRows(dataStoreId, project1.id, rows);
-			expect(ids).toEqual([1, 2, 3]);
+			expect(ids).toEqual([{ id: 1 }, { id: 2 }, { id: 3 }]);
 
 			// ACT
 			const result = await dataStoreService.getManyRowsAndCount(dataStoreId, project1.id, {});

packages/cli/src/modules/data-store/data-store-proxy.service.ts

@@ -19,10 +19,10 @@ import {
 	Workflow,
 } from 'n8n-workflow';
 
-import { DataStoreService } from './data-store.service';
-
 import { OwnershipService } from '@/services/ownership.service';
 
+import { DataStoreService } from './data-store.service';
+
 @Service()
 export class DataStoreProxyService implements DataStoreProxyProvider {
 	constructor(

packages/cli/src/modules/data-store/data-store-rows.repository.ts

@@ -6,8 +6,13 @@ import type {
 import { GlobalConfig } from '@n8n/config';
 import { CreateTable, DslColumn } from '@n8n/db';
 import { Service } from '@n8n/di';
-import { DataSource, DataSourceOptions, QueryRunner, SelectQueryBuilder } from '@n8n/typeorm';
-import { DataStoreColumnJsType, DataStoreRows } from 'n8n-workflow';
+import { DataSource, DataSourceOptions, QueryRunner, SelectQueryBuilder, In } from '@n8n/typeorm';
+import {
+	DataStoreColumnJsType,
+	DataStoreRows,
+	DataStoreRowWithId,
+	UnexpectedError,
+} from 'n8n-workflow';
 
 import { DataStoreColumn } from './data-store-column.entity';
 import { DataStoreUserTableName } from './data-store.types';
@@ -16,6 +21,7 @@ import {
 	buildColumnTypeMap,
 	deleteColumnQuery,
 	extractInsertedIds,
+	extractReturningData,
 	getPlaceholder,
 	normalizeValue,
 	quoteIdentifier,
@@ -54,16 +60,26 @@ export class DataStoreRowsRepository {
 		return `${tablePrefix}data_store_user_${dataStoreId}`;
 	}
 
-	async insertRows(dataStoreId: string, rows: DataStoreRows, columns: DataStoreColumn[]) {
-		const insertedIds: number[] = [];
+	async insertRows(
+		dataStoreId: string,
+		rows: DataStoreRows,
+		columns: DataStoreColumn[],
+		returnData: boolean = false,
+	) {
+		const inserted: DataStoreRowWithId[] = [];
+		const dbType = this.dataSource.options.type;
+		const useReturning = dbType === 'postgres' || dbType === 'mariadb';
+
+		const table = this.toTableName(dataStoreId);
+		const columnNames = columns.map((c) => c.name);
+		const escapedColumns = columns.map((c) => this.dataSource.driver.escape(c.name));
+		const selectColumns = ['id', ...escapedColumns];
 
 		// We insert one by one as the default behavior of returning the last inserted ID
 		// is consistent, whereas getting all inserted IDs when inserting multiple values is
 		// surprisingly awkward without Entities, e.g. `RETURNING id` explicitly does not aggregate
 		// and the `identifiers` array output of `execute()` is empty
 		for (const row of rows) {
-			const dbType = this.dataSource.options.type;
-
 			for (const column of columns) {
 				row[column.name] = normalizeValue(row[column.name], column.type, dbType);
 			}
@@ -71,22 +87,47 @@ export class DataStoreRowsRepository {
 			const query = this.dataSource
 				.createQueryBuilder()
 				.insert()
-				.into(
-					this.toTableName(dataStoreId),
-					columns.map((c) => c.name),
-				)
+				.into(table, columnNames)
 				.values(row);
 
-			if (dbType === 'postgres' || dbType === 'mariadb') {
-				query.returning('id');
+			if (useReturning) {
+				query.returning(returnData ? selectColumns.join(',') : 'id');
 			}
 
 			const result = await query.execute();
 
-			insertedIds.push(...extractInsertedIds(result.raw, dbType));
+			if (useReturning) {
+				const returned = extractReturningData(result.raw);
+				inserted.push.apply(inserted, returned);
+				continue;
+			}
+
+			// Engines without RETURNING support
+			const rowIds = extractInsertedIds(result.raw, dbType);
+			if (rowIds.length === 0) {
+				throw new UnexpectedError("Couldn't find the inserted row ID");
+			}
+
+			if (!returnData) {
+				inserted.push(...rowIds.map((id) => ({ id })));
+				continue;
+			}
+
+			const insertedRow = await this.dataSource
+				.createQueryBuilder()
+				.select(selectColumns)
+				.from(table, 'dataStore')
+				.where({ id: In(rowIds) })
+				.getRawOne<DataStoreRowWithId>();
+
+			if (!insertedRow) {
+				throw new UnexpectedError("Couldn't find the inserted row");
+			}
+
+			inserted.push(insertedRow);
 		}
 
-		return insertedIds;
+		return inserted;
 	}
 
 	// TypeORM cannot infer the columns for a dynamic table name, so we use a raw query

packages/cli/src/modules/data-store/data-store.controller.ts

@@ -246,7 +246,12 @@ export class DataStoreController {
 		@Body dto: AddDataStoreRowsDto,
 	) {
 		try {
-			return await this.dataStoreService.insertRows(dataStoreId, req.params.projectId, dto.data);
+			return await this.dataStoreService.insertRows(
+				dataStoreId,
+				req.params.projectId,
+				dto.data,
+				dto.returnData,
+			);
 		} catch (e: unknown) {
 			if (e instanceof DataStoreNotFoundError) {
 				throw new NotFoundError(e.message);

packages/cli/src/modules/data-store/data-store.service.ts

@@ -125,12 +125,17 @@ export class DataStoreService {
 		return await this.dataStoreColumnRepository.getColumns(dataStoreId);
 	}
 
-	async insertRows(dataStoreId: string, projectId: string, rows: DataStoreRows) {
+	async insertRows(
+		dataStoreId: string,
+		projectId: string,
+		rows: DataStoreRows,
+		returnData: boolean = false,
+	) {
 		await this.validateDataStoreExists(dataStoreId, projectId);
 		await this.validateRows(dataStoreId, rows);
 
 		const columns = await this.dataStoreColumnRepository.getColumns(dataStoreId);
-		return await this.dataStoreRowsRepository.insertRows(dataStoreId, rows, columns);
+		return await this.dataStoreRowsRepository.insertRows(dataStoreId, rows, columns, returnData);
 	}
 
 	async upsertRows(dataStoreId: string, projectId: string, dto: UpsertDataStoreRowsDto) {

packages/cli/src/modules/data-store/utils/sql-utils.ts

@@ -5,13 +5,13 @@ import {
 } from '@n8n/api-types';
 import { DslColumn } from '@n8n/db';
 import type { DataSourceOptions } from '@n8n/typeorm';
-import type { DataStoreColumnJsType, DataStoreRows } from 'n8n-workflow';
+import type { DataStoreColumnJsType, DataStoreRows, DataStoreRowWithId } from 'n8n-workflow';
 import { UnexpectedError } from 'n8n-workflow';
 
-import type { DataStoreUserTableName } from '../data-store.types';
-
 import { NotFoundError } from '@/errors/response-errors/not-found.error';
 
+import type { DataStoreUserTableName } from '../data-store.types';
+
 export function toDslColumns(columns: DataStoreCreateColumnSchema[]): DslColumn[] {
 	return columns.map((col) => {
 		const name = new DslColumn(col.name.trim());
@@ -155,6 +155,16 @@ function hasRowId(data: unknown): data is WithRowId {
 	return typeof data === 'object' && data !== null && 'id' in data && isNumber(data.id);
 }
 
+export function extractReturningData(raw: unknown): DataStoreRowWithId[] {
+	if (!isArrayOf(raw, hasRowId)) {
+		throw new UnexpectedError(
+			'Expected INSERT INTO raw to be { id: number }[] on Postgres or MariaDB',
+		);
+	}
+
+	return raw;
+}
+
 export function extractInsertedIds(raw: unknown, dbType: DataSourceOptions['type']): number[] {
 	switch (dbType) {
 		case 'postgres':

packages/workflow/src/data-store.types.ts

@@ -74,6 +74,7 @@ export type DataStoreColumnJsType = string | number | boolean | Date;
 
 export type DataStoreRow = Record<string, DataStoreColumnJsType | null>;
 export type DataStoreRows = DataStoreRow[];
+export type DataStoreRowWithId = DataStoreRow & { id: number };
 
 // APIs for a data store service operating on a specific projectId
 export interface IDataStoreProjectAggregateService {
@@ -101,7 +102,7 @@ export interface IDataStoreProjectService {
 		dto: Partial<ListDataStoreRowsOptions>,
 	): Promise<{ count: number; data: DataStoreRows }>;
 
-	insertRows(rows: DataStoreRows): Promise<number[]>;
+	insertRows(rows: DataStoreRows): Promise<Array<{ id: number }>>;
 
 	upsertRows(options: UpsertDataStoreRowsOptions): Promise<boolean>;
 }