feat(core): Allow partial data table row inserts (no-changelog) (#18914)

packages/cli/src/modules/data-table/__tests__/data-store.service.test.ts

@@ -1003,7 +1003,7 @@ describe('dataStore', () => {
 			]);
 		});
 
-		it('rejects a mismatched row with extra column', async () => {
+		it('rejects a mismatched row with unknown column', async () => {
 			// ARRANGE
 			const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, {
 				name: 'dataStore',
@@ -1022,29 +1022,63 @@ describe('dataStore', () => {
 			]);
 
 			// ASSERT
-			await expect(result).rejects.toThrow(new DataStoreValidationError('mismatched key count'));
+			await expect(result).rejects.toThrow(
+				new DataStoreValidationError("unknown column name 'cWrong'"),
+			);
 		});
 
-		it('rejects a mismatched row with missing column', async () => {
+		it('inserts rows with partial data (some columns missing)', async () => {
 			// ARRANGE
 			const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, {
 				name: 'dataStore',
 				columns: [
-					{ name: 'c1', type: 'number' },
-					{ name: 'c2', type: 'boolean' },
-					{ name: 'c3', type: 'date' },
-					{ name: 'c4', type: 'string' },
+					{ name: 'name', type: 'string' },
+					{ name: 'age', type: 'number' },
+					{ name: 'email', type: 'string' },
+					{ name: 'active', type: 'boolean' },
 				],
 			});
 
 			// ACT
-			const result = dataStoreService.insertRows(dataStoreId, project1.id, [
-				{ c1: 3, c2: true, c3: new Date(), c4: 'hello?' },
-				{ c2: true, c3: new Date(), c4: 'hello?' },
+			await dataStoreService.insertRows(dataStoreId, project1.id, [
+				{ name: 'Mary', age: 20, email: 'mary@example.com', active: true }, // full row
+				{ name: 'Alice', age: 30 }, // missing email and active
+				{ name: 'Bob' }, // missing age, email and active
+				{}, // missing all columns
 			]);
 
-			// ASSERT
-			await expect(result).rejects.toThrow(new DataStoreValidationError('mismatched key count'));
+			const { count, data } = await dataStoreService.getManyRowsAndCount(
+				dataStoreId,
+				project1.id,
+				{},
+			);
+			expect(count).toEqual(4);
+			expect(data).toEqual([
+				expect.objectContaining({
+					name: 'Mary',
+					age: 20,
+					email: 'mary@example.com',
+					active: true,
+				}),
+				expect.objectContaining({
+					name: 'Alice',
+					age: 30,
+					email: null,
+					active: null,
+				}),
+				expect.objectContaining({
+					name: 'Bob',
+					age: null,
+					email: null,
+					active: null,
+				}),
+				expect.objectContaining({
+					name: null,
+					age: null,
+					email: null,
+					active: null,
+				}),
+			]);
 		});
 
 		it('rejects a mismatched row with replaced column', async () => {
@@ -1336,6 +1370,57 @@ describe('dataStore', () => {
 			]);
 		});
 
+		it('should allow adding partial data', async () => {
+			// ARRANGE
+			const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, {
+				name: 'dataStore',
+				columns: [
+					{ name: 'pid', type: 'string' },
+					{ name: 'name', type: 'string' },
+					{ name: 'age', type: 'number' },
+				],
+			});
+
+			const ids = await dataStoreService.insertRows(dataStoreId, project1.id, [
+				{ pid: '1995-111a', name: 'Alice', age: 30 },
+			]);
+			expect(ids).toEqual([{ id: 1 }]);
+
+			// ACT
+			const result = await dataStoreService.upsertRows(dataStoreId, project1.id, {
+				rows: [
+					{ pid: '1992-222b', name: 'Alice' }, // age is missing
+					{ pid: '1995-111a', age: 35 }, // name is missing
+				],
+				matchFields: ['pid'],
+			});
+
+			// ASSERT
+			expect(result).toBe(true);
+
+			const { count, data } = await dataStoreService.getManyRowsAndCount(
+				dataStoreId,
+				project1.id,
+				{},
+			);
+
+			expect(count).toEqual(2);
+			expect(data).toEqual(
+				expect.arrayContaining([
+					expect.objectContaining({
+						name: 'Alice',
+						age: 35, // updated age
+						pid: '1995-111a',
+					}),
+					expect.objectContaining({
+						name: 'Alice',
+						age: null, // missing age
+						pid: '1992-222b',
+					}),
+				]),
+			);
+		});
+
 		it('should return full upserted rows if returnData is set', async () => {
 			// ARRANGE
 			const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, {

packages/cli/src/modules/data-table/data-store-rows.repository.ts

@@ -170,15 +170,20 @@ export class DataStoreRowsRepository {
 		// surprisingly awkward without Entities, e.g. `RETURNING id` explicitly does not aggregate
 		// and the `identifiers` array output of `execute()` is empty
 		for (const row of rows) {
+			// Fill missing columns with null values to support partial data insertion
+			const completeRow = { ...row };
 			for (const column of columns) {
-				row[column.name] = normalizeValue(row[column.name], column.type, dbType);
+				if (!(column.name in completeRow)) {
+					completeRow[column.name] = null;
+				}
+				completeRow[column.name] = normalizeValue(completeRow[column.name], column.type, dbType);
 			}
 
 			const query = this.dataSource
 				.createQueryBuilder()
 				.insert()
 				.into(table, columnNames)
-				.values(row);
+				.values(completeRow);
 
 			if (useReturning) {
 				query.returning(returnData ? selectColumns.join(',') : 'id');

packages/cli/src/modules/data-table/data-store.service.ts

@@ -158,7 +158,7 @@ export class DataStoreService {
 		returnData: boolean = false,
 	) {
 		await this.validateDataStoreExists(dataStoreId, projectId);
-		await this.validateRows(dataStoreId, dto.rows);
+		await this.validateRows(dataStoreId, dto.rows, true);
 
 		if (dto.rows.length === 0) {
 			throw new DataStoreValidationError('No rows provided for upsertRows');
@@ -205,8 +205,8 @@ export class DataStoreService {
 			throw new DataStoreValidationError('Data columns must not be empty for updateRow');
 		}
 
-		this.validateRowsWithColumns([filter], columns, true, true);
-		this.validateRowsWithColumns([data], columns, true, false);
+		this.validateRowsWithColumns([filter], columns, true);
+		this.validateRowsWithColumns([data], columns, false);
 
 		return await this.dataStoreRowsRepository.updateRow(
 			dataStoreId,
@@ -226,7 +226,6 @@ export class DataStoreService {
 	private validateRowsWithColumns(
 		rows: DataStoreRows,
 		columns: Array<{ name: string; type: string }>,
-		allowPartial = false,
 		includeSystemColumns = false,
 	): void {
 		// Include system columns like 'id' if requested
@@ -242,9 +241,6 @@ export class DataStoreService {
 		const columnTypeMap = new Map(allColumns.map((x) => [x.name, x.type]));
 		for (const row of rows) {
 			const keys = Object.keys(row);
-			if (!allowPartial && columnNames.size !== keys.length) {
-				throw new DataStoreValidationError('mismatched key count');
-			}
 			for (const key of keys) {
 				if (!columnNames.has(key)) {
 					throw new DataStoreValidationError(`unknown column name '${key}'`);
@@ -257,11 +253,10 @@ export class DataStoreService {
 	private async validateRows(
 		dataStoreId: string,
 		rows: DataStoreRows,
-		allowPartial = false,
 		includeSystemColumns = false,
 	): Promise<void> {
 		const columns = await this.dataStoreColumnRepository.getColumns(dataStoreId);
-		this.validateRowsWithColumns(rows, columns, allowPartial, includeSystemColumns);
+		this.validateRowsWithColumns(rows, columns, includeSystemColumns);
 	}
 
 	private validateCell(row: DataStoreRow, key: string, columnTypeMap: Map<string, string>) {