From 84936848c956ad7478ca366dbf06d9d268a2bb65 Mon Sep 17 00:00:00 2001 From: Daria Date: Thu, 28 Aug 2025 16:07:24 +0300 Subject: [PATCH] feat(core): Allow partial data table row inserts (no-changelog) (#18914) --- .../__tests__/data-store.service.test.ts | 109 ++++++++++++++++-- .../data-table/data-store-rows.repository.ts | 9 +- .../modules/data-table/data-store.service.ts | 13 +-- 3 files changed, 108 insertions(+), 23 deletions(-) diff --git a/packages/cli/src/modules/data-table/__tests__/data-store.service.test.ts b/packages/cli/src/modules/data-table/__tests__/data-store.service.test.ts index 65f2f5b154..799d2ff9d5 100644 --- a/packages/cli/src/modules/data-table/__tests__/data-store.service.test.ts +++ b/packages/cli/src/modules/data-table/__tests__/data-store.service.test.ts @@ -1003,7 +1003,7 @@ describe('dataStore', () => { ]); }); - it('rejects a mismatched row with extra column', async () => { + it('rejects a mismatched row with unknown column', async () => { // ARRANGE const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, { name: 'dataStore', @@ -1022,29 +1022,63 @@ describe('dataStore', () => { ]); // ASSERT - await expect(result).rejects.toThrow(new DataStoreValidationError('mismatched key count')); + await expect(result).rejects.toThrow( + new DataStoreValidationError("unknown column name 'cWrong'"), + ); }); - it('rejects a mismatched row with missing column', async () => { + it('inserts rows with partial data (some columns missing)', async () => { // ARRANGE const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, { name: 'dataStore', columns: [ - { name: 'c1', type: 'number' }, - { name: 'c2', type: 'boolean' }, - { name: 'c3', type: 'date' }, - { name: 'c4', type: 'string' }, + { name: 'name', type: 'string' }, + { name: 'age', type: 'number' }, + { name: 'email', type: 'string' }, + { name: 'active', type: 'boolean' }, ], }); // ACT - const result = dataStoreService.insertRows(dataStoreId, project1.id, [ - { c1: 3, c2: true, c3: new Date(), c4: 'hello?' }, - { c2: true, c3: new Date(), c4: 'hello?' }, + await dataStoreService.insertRows(dataStoreId, project1.id, [ + { name: 'Mary', age: 20, email: 'mary@example.com', active: true }, // full row + { name: 'Alice', age: 30 }, // missing email and active + { name: 'Bob' }, // missing age, email and active + {}, // missing all columns ]); - // ASSERT - await expect(result).rejects.toThrow(new DataStoreValidationError('mismatched key count')); + const { count, data } = await dataStoreService.getManyRowsAndCount( + dataStoreId, + project1.id, + {}, + ); + expect(count).toEqual(4); + expect(data).toEqual([ + expect.objectContaining({ + name: 'Mary', + age: 20, + email: 'mary@example.com', + active: true, + }), + expect.objectContaining({ + name: 'Alice', + age: 30, + email: null, + active: null, + }), + expect.objectContaining({ + name: 'Bob', + age: null, + email: null, + active: null, + }), + expect.objectContaining({ + name: null, + age: null, + email: null, + active: null, + }), + ]); }); it('rejects a mismatched row with replaced column', async () => { @@ -1336,6 +1370,57 @@ describe('dataStore', () => { ]); }); + it('should allow adding partial data', async () => { + // ARRANGE + const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, { + name: 'dataStore', + columns: [ + { name: 'pid', type: 'string' }, + { name: 'name', type: 'string' }, + { name: 'age', type: 'number' }, + ], + }); + + const ids = await dataStoreService.insertRows(dataStoreId, project1.id, [ + { pid: '1995-111a', name: 'Alice', age: 30 }, + ]); + expect(ids).toEqual([{ id: 1 }]); + + // ACT + const result = await dataStoreService.upsertRows(dataStoreId, project1.id, { + rows: [ + { pid: '1992-222b', name: 'Alice' }, // age is missing + { pid: '1995-111a', age: 35 }, // name is missing + ], + matchFields: ['pid'], + }); + + // ASSERT + expect(result).toBe(true); + + const { count, data } = await dataStoreService.getManyRowsAndCount( + dataStoreId, + project1.id, + {}, + ); + + expect(count).toEqual(2); + expect(data).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + name: 'Alice', + age: 35, // updated age + pid: '1995-111a', + }), + expect.objectContaining({ + name: 'Alice', + age: null, // missing age + pid: '1992-222b', + }), + ]), + ); + }); + it('should return full upserted rows if returnData is set', async () => { // ARRANGE const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, { diff --git a/packages/cli/src/modules/data-table/data-store-rows.repository.ts b/packages/cli/src/modules/data-table/data-store-rows.repository.ts index 11d7466c77..f9ab249b52 100644 --- a/packages/cli/src/modules/data-table/data-store-rows.repository.ts +++ b/packages/cli/src/modules/data-table/data-store-rows.repository.ts @@ -170,15 +170,20 @@ export class DataStoreRowsRepository { // surprisingly awkward without Entities, e.g. `RETURNING id` explicitly does not aggregate // and the `identifiers` array output of `execute()` is empty for (const row of rows) { + // Fill missing columns with null values to support partial data insertion + const completeRow = { ...row }; for (const column of columns) { - row[column.name] = normalizeValue(row[column.name], column.type, dbType); + if (!(column.name in completeRow)) { + completeRow[column.name] = null; + } + completeRow[column.name] = normalizeValue(completeRow[column.name], column.type, dbType); } const query = this.dataSource .createQueryBuilder() .insert() .into(table, columnNames) - .values(row); + .values(completeRow); if (useReturning) { query.returning(returnData ? selectColumns.join(',') : 'id'); diff --git a/packages/cli/src/modules/data-table/data-store.service.ts b/packages/cli/src/modules/data-table/data-store.service.ts index c9f3b1f31f..01127be5a7 100644 --- a/packages/cli/src/modules/data-table/data-store.service.ts +++ b/packages/cli/src/modules/data-table/data-store.service.ts @@ -158,7 +158,7 @@ export class DataStoreService { returnData: boolean = false, ) { await this.validateDataStoreExists(dataStoreId, projectId); - await this.validateRows(dataStoreId, dto.rows); + await this.validateRows(dataStoreId, dto.rows, true); if (dto.rows.length === 0) { throw new DataStoreValidationError('No rows provided for upsertRows'); @@ -205,8 +205,8 @@ export class DataStoreService { throw new DataStoreValidationError('Data columns must not be empty for updateRow'); } - this.validateRowsWithColumns([filter], columns, true, true); - this.validateRowsWithColumns([data], columns, true, false); + this.validateRowsWithColumns([filter], columns, true); + this.validateRowsWithColumns([data], columns, false); return await this.dataStoreRowsRepository.updateRow( dataStoreId, @@ -226,7 +226,6 @@ export class DataStoreService { private validateRowsWithColumns( rows: DataStoreRows, columns: Array<{ name: string; type: string }>, - allowPartial = false, includeSystemColumns = false, ): void { // Include system columns like 'id' if requested @@ -242,9 +241,6 @@ export class DataStoreService { const columnTypeMap = new Map(allColumns.map((x) => [x.name, x.type])); for (const row of rows) { const keys = Object.keys(row); - if (!allowPartial && columnNames.size !== keys.length) { - throw new DataStoreValidationError('mismatched key count'); - } for (const key of keys) { if (!columnNames.has(key)) { throw new DataStoreValidationError(`unknown column name '${key}'`); @@ -257,11 +253,10 @@ export class DataStoreService { private async validateRows( dataStoreId: string, rows: DataStoreRows, - allowPartial = false, includeSystemColumns = false, ): Promise { const columns = await this.dataStoreColumnRepository.getColumns(dataStoreId); - this.validateRowsWithColumns(rows, columns, allowPartial, includeSystemColumns); + this.validateRowsWithColumns(rows, columns, includeSystemColumns); } private validateCell(row: DataStoreRow, key: string, columnTypeMap: Map) {