diff --git a/packages/@n8n/api-types/src/dto/data-store/update-data-store-row.dto.ts b/packages/@n8n/api-types/src/dto/data-store/update-data-store-row.dto.ts index 981e94b8cb..87c3997a3f 100644 --- a/packages/@n8n/api-types/src/dto/data-store/update-data-store-row.dto.ts +++ b/packages/@n8n/api-types/src/dto/data-store/update-data-store-row.dto.ts @@ -17,6 +17,7 @@ const updateDataStoreRowShape = { .refine((obj) => Object.keys(obj).length > 0, { message: 'data must not be empty', }), + returnData: z.boolean().default(false), }; export class UpdateDataStoreRowDto extends Z.class(updateDataStoreRowShape) {} diff --git a/packages/@n8n/api-types/src/dto/data-store/upsert-data-store-rows.dto.ts b/packages/@n8n/api-types/src/dto/data-store/upsert-data-store-rows.dto.ts index 385ce39539..2205707ddb 100644 --- a/packages/@n8n/api-types/src/dto/data-store/upsert-data-store-rows.dto.ts +++ b/packages/@n8n/api-types/src/dto/data-store/upsert-data-store-rows.dto.ts @@ -9,6 +9,7 @@ import { const upsertDataStoreRowsShape = { rows: z.array(z.record(dataStoreColumnNameSchema, dataStoreColumnValueSchema)), matchFields: z.array(dataStoreColumnNameSchema).min(1), + returnData: z.boolean().optional().default(false), }; export class UpsertDataStoreRowsDto extends Z.class(upsertDataStoreRowsShape) {} diff --git a/packages/cli/src/modules/data-table/__tests__/data-store.controller.test.ts b/packages/cli/src/modules/data-table/__tests__/data-store.controller.test.ts index fa0ce36625..b30d5f09e0 100644 --- a/packages/cli/src/modules/data-table/__tests__/data-store.controller.test.ts +++ b/packages/cli/src/modules/data-table/__tests__/data-store.controller.test.ts @@ -2889,11 +2889,13 @@ describe('POST /projects/:projectId/data-stores/:dataStoreId/upsert', () => { matchFields: ['first'], }; - await authMemberAgent + const result = await authMemberAgent .post(`/projects/${memberProject.id}/data-stores/${dataStore.id}/upsert`) .send(payload) .expect(200); + expect(result.body.data).toBe(true); + const rowsInDb = await dataStoreRowsRepository.getManyAndCount(dataStore.id, { sortBy: ['id', 'ASC'], }); @@ -2902,6 +2904,59 @@ describe('POST /projects/:projectId/data-stores/:dataStoreId/upsert', () => { expect(rowsInDb.data[1]).toMatchObject(payload.rows[0]); expect(rowsInDb.data[2]).toMatchObject(payload.rows[1]); }); + + test('should return affected rows if returnData is set', async () => { + const dataStore = await createDataStore(memberProject, { + columns: [ + { + name: 'first', + type: 'string', + }, + { + name: 'second', + type: 'string', + }, + ], + data: [ + { + first: 'test row', + second: 'test value', + }, + { + first: 'test row', + second: 'another row with same first column', + }, + ], + }); + + const payload = { + rows: [ + { + first: 'test row', + second: 'updated value', + }, + { + first: 'new row', + second: 'new value', + }, + ], + matchFields: ['first'], + returnData: true, + }; + + const result = await authMemberAgent + .post(`/projects/${memberProject.id}/data-stores/${dataStore.id}/upsert`) + .send(payload) + .expect(200); + + expect(result.body.data).toEqual( + expect.arrayContaining([ + { id: 1, first: 'test row', second: 'updated value' }, + { id: 2, first: 'test row', second: 'updated value' }, + { id: 3, first: 'new row', second: 'new value' }, + ]), + ); + }); }); describe('PATCH /projects/:projectId/data-stores/:dataStoreId/rows', () => { @@ -2979,26 +3034,36 @@ describe('PATCH /projects/:projectId/data-stores/:dataStoreId/rows', () => { columns: [ { name: 'name', type: 'string' }, { name: 'age', type: 'number' }, + { name: 'active', type: 'boolean' }, + { name: 'birthday', type: 'date' }, ], - data: [{ name: 'Alice', age: 30 }], + data: [{ name: 'Alice', age: 30, active: true, birthday: new Date('1990-01-01') }], }); const payload = { filter: { name: 'Alice' }, - data: { age: 31 }, + data: { name: 'Alicia', age: 31, active: false, birthday: new Date('1990-01-02') }, }; - await authMemberAgent + const result = await authMemberAgent .patch(`/projects/${project.id}/data-stores/${dataStore.id}/rows`) .send(payload) .expect(200); + expect(result.body.data).toBe(true); + const readResponse = await authMemberAgent .get(`/projects/${project.id}/data-stores/${dataStore.id}/rows`) .expect(200); expect(readResponse.body.data.count).toBe(1); - expect(readResponse.body.data.data[0]).toMatchObject({ id: 1, name: 'Alice', age: 31 }); + expect(readResponse.body.data.data[0]).toMatchObject({ + id: 1, + name: 'Alicia', + age: 31, + active: false, + birthday: new Date('1990-01-02').toISOString(), + }); }); test('should update row if user has project:admin role in team project', async () => { @@ -3208,7 +3273,7 @@ describe('PATCH /projects/:projectId/data-stores/:dataStoreId/rows', () => { .send(payload) .expect(200); - expect(response.body.data).toBe(true); + expect(response.body.data).toEqual(true); const readResponse = await authMemberAgent .get(`/projects/${memberProject.id}/data-stores/${dataStore.id}/rows`) @@ -3400,4 +3465,47 @@ describe('PATCH /projects/:projectId/data-stores/:dataStoreId/rows', () => { birthdate: '1995-05-15T12:30:00.000Z', }); }); + + test('should return updated data if returnData is set', async () => { + const dataStore = await createDataStore(memberProject, { + columns: [ + { name: 'name', type: 'string' }, + { name: 'age', type: 'number' }, + { name: 'active', type: 'boolean' }, + { name: 'birthday', type: 'date' }, + ], + data: [ + { name: 'Alice', age: 30, active: true, birthday: new Date('1990-01-01T00:00:00.000Z') }, + { name: 'Bob', age: 25, active: true, birthday: new Date('1995-05-15T00:00:00.000Z') }, + ], + }); + + const payload = { + filter: { active: true }, + data: { active: false }, + returnData: true, + }; + + const result = await authMemberAgent + .patch(`/projects/${memberProject.id}/data-stores/${dataStore.id}/rows`) + .send(payload) + .expect(200); + + expect(result.body.data).toMatchObject([ + { + id: 1, + name: 'Alice', + age: 30, + active: false, + birthday: '1990-01-01T00:00:00.000Z', + }, + { + id: 2, + name: 'Bob', + age: 25, + active: false, + birthday: '1995-05-15T00:00:00.000Z', + }, + ]); + }); }); diff --git a/packages/cli/src/modules/data-table/__tests__/data-store.service.test.ts b/packages/cli/src/modules/data-table/__tests__/data-store.service.test.ts index f8aad282e5..f7649faef7 100644 --- a/packages/cli/src/modules/data-table/__tests__/data-store.service.test.ts +++ b/packages/cli/src/modules/data-table/__tests__/data-store.service.test.ts @@ -977,15 +977,15 @@ describe('dataStore', () => { ); expect(count).toEqual(4); expect(data).toEqual( - rows.map( - (row) => - expect.objectContaining({ - ...row, - c1: row.c1, - c2: row.c2, - c3: row.c3 instanceof Date ? row.c3.toISOString() : row.c3, - c4: row.c4, - }) as Record, + rows.map((row, i) => + expect.objectContaining({ + ...row, + id: i + 1, + c1: row.c1, + c2: row.c2, + c3: typeof row.c3 === 'string' ? new Date(row.c3) : row.c3, + c4: row.c4, + }), ), ); }); @@ -1081,47 +1081,35 @@ describe('dataStore', () => { ]); }); - it('return inserted data if requested', async () => { + it('return full inserted data if returnData is set', async () => { // ARRANGE const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, { name: 'myDataStore', columns: [ { name: 'c1', type: 'number' }, { name: 'c2', type: 'string' }, + { name: 'c3', type: 'boolean' }, + { name: 'c4', type: 'date' }, ], }); + const now = new Date(); + // Insert initial row const ids = await dataStoreService.insertRows( dataStoreId, project1.id, [ - { c1: 1, c2: 'foo' }, - { c1: 2, c2: 'bar' }, + { c1: 1, c2: 'foo', c3: true, c4: now }, + { c1: 2, c2: 'bar', c3: false, c4: now }, + { c1: null, c2: null, c3: null, c4: null }, ], true, ); expect(ids).toEqual([ - { id: 1, c1: 1, c2: 'foo' }, - { id: 2, c1: 2, c2: 'bar' }, - ]); - - await dataStoreService.deleteRows(dataStoreId, project1.id, [ids[0].id]); - - const result = await dataStoreService.insertRows( - dataStoreId, - project1.id, - [ - { c1: 1, c2: 'baz' }, - { c1: 2, c2: 'faz' }, - ], - true, - ); - - // ASSERT - expect(result).toEqual([ - { id: 3, c1: 1, c2: 'baz' }, - { id: 4, c1: 2, c2: 'faz' }, + { id: 1, c1: 1, c2: 'foo', c3: true, c4: now }, + { id: 2, c1: 2, c2: 'bar', c3: false, c4: now }, + { id: 3, c1: null, c2: null, c3: null, c4: null }, ]); }); @@ -1275,25 +1263,25 @@ describe('dataStore', () => { expect(data).toEqual([ { id: 1, - createdAt: expect.any(String), - updatedAt: expect.any(String), + createdAt: expect.any(Date), + updatedAt: expect.any(Date), }, { id: 2, - createdAt: expect.any(String), - updatedAt: expect.any(String), + createdAt: expect.any(Date), + updatedAt: expect.any(Date), }, { id: 3, - createdAt: expect.any(String), - updatedAt: expect.any(String), + createdAt: expect.any(Date), + updatedAt: expect.any(Date), }, ]); }); }); describe('upsertRows', () => { - it('updates rows if filter matches', async () => { + it('should update a row if filter matches', async () => { // ARRANGE const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, { name: 'dataStore', @@ -1352,7 +1340,7 @@ describe('dataStore', () => { ); }); - it('works correctly with multiple filters', async () => { + it('should work correctly with multiple filters', async () => { // ARRANGE const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, { name: 'dataStore', @@ -1409,7 +1397,7 @@ describe('dataStore', () => { ); }); - it('inserts a row if filter does not match', async () => { + it('should insert a row if filter does not match', async () => { // ARRANGE const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, { name: 'dataStore', @@ -1455,6 +1443,59 @@ describe('dataStore', () => { }), ]); }); + + it('should return full upserted rows if returnData is set', async () => { + // ARRANGE + const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, { + name: 'dataStore', + columns: [ + { name: 'pid', type: 'string' }, + { name: 'fullName', type: 'string' }, + { name: 'age', type: 'number' }, + { name: 'birthday', type: 'date' }, + ], + }); + + // Insert initial row + const ids = await dataStoreService.insertRows(dataStoreId, project1.id, [ + { pid: '1995-111a', fullName: 'Alice', age: 30, birthday: new Date('1995-01-01') }, + ]); + expect(ids).toEqual([{ id: 1 }]); + + // ACT + const result = await dataStoreService.upsertRows( + dataStoreId, + project1.id, + { + rows: [ + { pid: '1995-111a', fullName: 'Alicia', age: 31, birthday: new Date('1995-01-01') }, + { pid: '1992-222b', fullName: 'Bob', age: 30, birthday: new Date('1992-01-01') }, + ], + matchFields: ['pid'], + }, + true, + ); + + // ASSERT + expect(result).toEqual( + expect.arrayContaining([ + { + id: 1, + fullName: 'Alicia', + age: 31, + pid: '1995-111a', + birthday: new Date('1995-01-01'), + }, + { + id: 2, + fullName: 'Bob', + age: 30, + pid: '1992-222b', + birthday: new Date('1992-01-01'), + }, + ]), + ); + }); }); describe('deleteRows', () => { @@ -1547,37 +1588,40 @@ describe('dataStore', () => { { name: 'name', type: 'string' }, { name: 'age', type: 'number' }, { name: 'active', type: 'boolean' }, + { name: 'birthday', type: 'date' }, ], }); await dataStoreService.insertRows(dataStoreId, project1.id, [ - { name: 'Alice', age: 30, active: true }, - { name: 'Bob', age: 25, active: false }, + { name: 'Alice', age: 30, active: true, birthday: new Date('1990-01-01') }, + { name: 'Bob', age: 25, active: false, birthday: new Date('1995-01-01') }, ]); // ACT const result = await dataStoreService.updateRow(dataStoreId, project1.id, { filter: { name: 'Alice' }, - data: { age: 31, active: false }, + data: { name: 'Alicia', age: 31, active: false, birthday: new Date('1990-01-02') }, }); // ASSERT - expect(result).toBe(true); + expect(result).toEqual(true); const { data } = await dataStoreService.getManyRowsAndCount(dataStoreId, project1.id, {}); expect(data).toEqual( expect.arrayContaining([ expect.objectContaining({ id: 1, - name: 'Alice', + name: 'Alicia', age: 31, active: false, + birthday: new Date('1990-01-02'), }), expect.objectContaining({ id: 2, name: 'Bob', age: 25, active: false, + birthday: new Date('1995-01-01'), }), ]), ); @@ -1614,7 +1658,7 @@ describe('dataStore', () => { }); // ASSERT - expect(result).toBe(true); + expect(result).toEqual(true); const { data } = await dataStoreService.getManyRowsAndCount(dataStoreId, project1.id, {}); expect(data).toEqual( @@ -1685,6 +1729,194 @@ describe('dataStore', () => { ); }); + it('should be able to update by string column', async () => { + // ARRANGE + const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, { + name: 'dataStore', + columns: [ + { name: 'name', type: 'string' }, + { name: 'age', type: 'number' }, + { name: 'active', type: 'boolean' }, + { name: 'birthday', type: 'date' }, + ], + }); + + await dataStoreService.insertRows(dataStoreId, project1.id, [ + { name: 'Alice', age: 30, active: true, birthday: new Date('1990-01-01') }, + { name: 'Bob', age: 25, active: false, birthday: new Date('1995-01-01') }, + ]); + + // ACT + const result = await dataStoreService.updateRow(dataStoreId, project1.id, { + filter: { name: 'Alice' }, + data: { name: 'Alicia' }, + }); + + // ASSERT + expect(result).toEqual(true); + + const { data } = await dataStoreService.getManyRowsAndCount(dataStoreId, project1.id, {}); + expect(data).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: 1, + name: 'Alicia', + age: 30, + active: true, + birthday: new Date('1990-01-01'), + }), + expect.objectContaining({ + id: 2, + name: 'Bob', + age: 25, + active: false, + birthday: new Date('1995-01-01'), + }), + ]), + ); + }); + + it('should be able to update by number column', async () => { + // ARRANGE + const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, { + name: 'dataStore', + columns: [ + { name: 'name', type: 'string' }, + { name: 'age', type: 'number' }, + { name: 'active', type: 'boolean' }, + { name: 'birthday', type: 'date' }, + ], + }); + + await dataStoreService.insertRows(dataStoreId, project1.id, [ + { name: 'Alice', age: 30, active: true, birthday: new Date('1990-01-01') }, + { name: 'Bob', age: 25, active: false, birthday: new Date('1995-01-01') }, + ]); + + // ACT + const result = await dataStoreService.updateRow(dataStoreId, project1.id, { + filter: { age: 30 }, + data: { age: 31 }, + }); + + // ASSERT + expect(result).toEqual(true); + + const { data } = await dataStoreService.getManyRowsAndCount(dataStoreId, project1.id, {}); + expect(data).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: 1, + name: 'Alice', + age: 31, + active: true, + birthday: new Date('1990-01-01'), + }), + expect.objectContaining({ + id: 2, + name: 'Bob', + age: 25, + active: false, + birthday: new Date('1995-01-01'), + }), + ]), + ); + }); + + it('should be able to update by boolean column', async () => { + // ARRANGE + const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, { + name: 'dataStore', + columns: [ + { name: 'name', type: 'string' }, + { name: 'age', type: 'number' }, + { name: 'active', type: 'boolean' }, + { name: 'birthday', type: 'date' }, + ], + }); + + await dataStoreService.insertRows(dataStoreId, project1.id, [ + { name: 'Alice', age: 30, active: true, birthday: new Date('1990-01-01') }, + { name: 'Bob', age: 25, active: false, birthday: new Date('1995-01-01') }, + ]); + + // ACT + const result = await dataStoreService.updateRow(dataStoreId, project1.id, { + filter: { active: true }, + data: { active: false }, + }); + + // ASSERT + expect(result).toEqual(true); + + const { data } = await dataStoreService.getManyRowsAndCount(dataStoreId, project1.id, {}); + expect(data).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: 1, + name: 'Alice', + age: 30, + active: false, + birthday: new Date('1990-01-01'), + }), + expect.objectContaining({ + id: 2, + name: 'Bob', + age: 25, + active: false, + birthday: new Date('1995-01-01'), + }), + ]), + ); + }); + + it('should be able to update by date column', async () => { + // ARRANGE + const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, { + name: 'dataStore', + columns: [ + { name: 'name', type: 'string' }, + { name: 'age', type: 'number' }, + { name: 'active', type: 'boolean' }, + { name: 'birthday', type: 'date' }, + ], + }); + + await dataStoreService.insertRows(dataStoreId, project1.id, [ + { name: 'Alice', age: 30, active: true, birthday: new Date('1990-01-01') }, + { name: 'Bob', age: 25, active: false, birthday: new Date('1995-01-01') }, + ]); + + // ACT + const result = await dataStoreService.updateRow(dataStoreId, project1.id, { + filter: { birthday: new Date('1990-01-01') }, + data: { birthday: new Date('1990-01-02') }, + }); + + // ASSERT + expect(result).toEqual(true); + + const { data } = await dataStoreService.getManyRowsAndCount(dataStoreId, project1.id, {}); + expect(data).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: 1, + name: 'Alice', + age: 30, + active: true, + birthday: new Date('1990-01-02'), + }), + expect.objectContaining({ + id: 2, + name: 'Bob', + age: 25, + active: false, + birthday: new Date('1995-01-01'), + }), + ]), + ); + }); + it('should update row with multiple filter conditions', async () => { // ARRANGE const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, { @@ -1709,7 +1941,7 @@ describe('dataStore', () => { }); // ASSERT - expect(result).toBe(true); + expect(result).toEqual(true); const { data } = await dataStoreService.getManyRowsAndCount(dataStoreId, project1.id, {}); expect(data).toEqual( @@ -1755,7 +1987,7 @@ describe('dataStore', () => { }); // ASSERT - expect(result).toBe(true); + expect(result).toEqual(true); const { data } = await dataStoreService.getManyRowsAndCount(dataStoreId, project1.id, {}); expect(data).toEqual([ @@ -1919,7 +2151,7 @@ describe('dataStore', () => { }); // ASSERT - expect(result).toBe(true); + expect(result).toEqual(true); const { data } = await dataStoreService.getManyRowsAndCount(dataStoreId, project1.id, {}); expect(data).toEqual([ @@ -1954,14 +2186,48 @@ describe('dataStore', () => { }); // ASSERT - expect(result).toBe(true); + expect(result).toEqual(true); const { data } = await dataStoreService.getManyRowsAndCount(dataStoreId, project1.id, {}); - expect(data).toEqual([ - expect.objectContaining({ - name: 'Alice', - birthDate: newDate.toISOString(), - }), + + expect(data).toEqual([expect.objectContaining({ id: 1, name: 'Alice', birthDate: newDate })]); + }); + + it('should return full updated rows if returnData is set', async () => { + // ARRANGE + const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, { + name: 'dataStore', + columns: [ + { name: 'name', type: 'string' }, + { name: 'age', type: 'number' }, + { name: 'active', type: 'boolean' }, + { name: 'timestamp', type: 'date' }, + ], + }); + + const now = new Date(); + await dataStoreService.insertRows(dataStoreId, project1.id, [ + { name: 'Alice', age: 30, active: true, timestamp: now }, + { name: 'Bob', age: 25, active: false, timestamp: now }, + ]); + + const soon = new Date(); + soon.setDate(now.getDate() + 1); + + // ACT + const result = await dataStoreService.updateRow( + dataStoreId, + project1.id, + { + filter: { name: 'Alice' }, + data: { age: 31, active: false, timestamp: soon }, + }, + true, + ); + + // ASSERT + expect(result).toEqual([ + expect.objectContaining({ id: 1, name: 'Alice', age: 31, active: false, timestamp: soon }), ]); }); }); @@ -2013,29 +2279,29 @@ describe('dataStore', () => { { c1: rows[0].c1, c2: rows[0].c2, - c3: '1970-01-01T00:00:00.000Z', + c3: new Date(0), c4: rows[0].c4, id: 1, - createdAt: expect.any(String), - updatedAt: expect.any(String), + createdAt: expect.any(Date), + updatedAt: expect.any(Date), }, { c1: rows[1].c1, c2: rows[1].c2, - c3: '1970-01-01T00:00:00.001Z', + c3: new Date(1), c4: rows[1].c4, id: 2, - createdAt: expect.any(String), - updatedAt: expect.any(String), + createdAt: expect.any(Date), + updatedAt: expect.any(Date), }, { c1: rows[2].c1, c2: rows[2].c2, - c3: '1970-01-01T00:00:00.002Z', + c3: new Date(2), c4: rows[2].c4, id: 3, - createdAt: expect.any(String), - updatedAt: expect.any(String), + createdAt: expect.any(Date), + updatedAt: expect.any(Date), }, ]); }); diff --git a/packages/cli/src/modules/data-table/data-store-rows.repository.ts b/packages/cli/src/modules/data-table/data-store-rows.repository.ts index fee965b2da..c3370bdd58 100644 --- a/packages/cli/src/modules/data-table/data-store-rows.repository.ts +++ b/packages/cli/src/modules/data-table/data-store-rows.repository.ts @@ -1,8 +1,4 @@ -import type { - ListDataStoreContentQueryDto, - ListDataStoreContentFilter, - UpsertDataStoreRowsDto, -} from '@n8n/api-types'; +import type { ListDataStoreContentQueryDto, ListDataStoreContentFilter } from '@n8n/api-types'; import { GlobalConfig } from '@n8n/config'; import { CreateTable, DslColumn } from '@n8n/db'; import { Service } from '@n8n/di'; @@ -18,11 +14,11 @@ import { DataStoreColumn } from './data-store-column.entity'; import { DataStoreUserTableName } from './data-store.types'; import { addColumnQuery, - buildColumnTypeMap, deleteColumnQuery, extractInsertedIds, extractReturningData, getPlaceholder, + normalizeRows, normalizeValue, quoteIdentifier, splitRowsByExistence, @@ -97,51 +93,112 @@ export class DataStoreRowsRepository { const result = await query.execute(); if (useReturning) { - const returned = extractReturningData(result.raw); + const returned = normalizeRows(extractReturningData(result.raw), columns); inserted.push.apply(inserted, returned); continue; } // Engines without RETURNING support - const rowIds = extractInsertedIds(result.raw, dbType); - if (rowIds.length === 0) { + const ids = extractInsertedIds(result.raw, dbType); + if (ids.length === 0) { throw new UnexpectedError("Couldn't find the inserted row ID"); } if (!returnData) { - inserted.push(...rowIds.map((id) => ({ id }))); + inserted.push(...ids.map((id) => ({ id }))); continue; } - const insertedRow = await this.dataSource - .createQueryBuilder() - .select(selectColumns) - .from(table, 'dataStore') - .where({ id: In(rowIds) }) - .getRawOne(); + const insertedRows = (await this.getManyByIds( + dataStoreId, + ids, + columns, + )) as DataStoreRowWithId[]; - if (!insertedRow) { - throw new UnexpectedError("Couldn't find the inserted row"); - } - - inserted.push(insertedRow); + inserted.push(...insertedRows); } return inserted; } - // TypeORM cannot infer the columns for a dynamic table name, so we use a raw query - async upsertRows(dataStoreId: string, dto: UpsertDataStoreRowsDto, columns: DataStoreColumn[]) { - const { rows, matchFields } = dto; + async updateRow( + dataStoreId: string, + setData: Record, + whereData: Record, + columns: DataStoreColumn[], + returnData: boolean = false, + ) { + const dbType = this.dataSource.options.type; + const useReturning = dbType === 'postgres'; + const table = this.toTableName(dataStoreId); + const escapedColumns = columns.map((c) => this.dataSource.driver.escape(c.name)); + const selectColumns = ['id', ...escapedColumns]; + + for (const column of columns) { + if (column.name in setData) { + setData[column.name] = normalizeValue(setData[column.name], column.type, dbType); + } + if (column.name in whereData) { + whereData[column.name] = normalizeValue(whereData[column.name], column.type, dbType); + } + } + + let affectedRows: DataStoreRowWithId[] = []; + if (!useReturning && returnData) { + // Only Postgres supports RETURNING statement on updates (with our typeorm), + // on other engines we must query the list of updates rows later by ID + affectedRows = await this.dataSource + .createQueryBuilder() + .select('id') + .from(table, 'dataStore') + .where(whereData) + .getRawMany<{ id: number }>(); + } + + setData.updatedAt = normalizeValue(new Date(), 'date', dbType); + + const query = this.dataSource.createQueryBuilder().update(table).set(setData).where(whereData); + + if (useReturning && returnData) { + query.returning(selectColumns.join(',')); + } + + const result = await query.execute(); + + if (!returnData) { + return true; + } + + if (useReturning) { + return extractReturningData(result.raw); + } + + const ids = affectedRows.map((row) => row.id); + return await this.getManyByIds(dataStoreId, ids, columns); + } + + // TypeORM cannot infer the columns for a dynamic table name, so we use a raw query + async upsertRows( + dataStoreId: string, + matchFields: string[], + rows: DataStoreRows, + columns: DataStoreColumn[], + returnData = false, + ) { const { rowsToInsert, rowsToUpdate } = await this.fetchAndSplitRowsByExistence( dataStoreId, matchFields, rows, ); + const output: DataStoreRowWithId[] = []; + if (rowsToInsert.length > 0) { - await this.insertRows(dataStoreId, rowsToInsert, columns); + const result = await this.insertRows(dataStoreId, rowsToInsert, columns, returnData); + if (returnData) { + output.push.apply(output, result); + } } if (rowsToUpdate.length > 0) { @@ -154,41 +211,14 @@ export class DataStoreRowsRepository { const setData = Object.fromEntries(updateKeys.map((key) => [key, row[key]])); const whereData = Object.fromEntries(matchFields.map((key) => [key, row[key]])); - await this.updateRow(dataStoreId, setData, whereData, columns); + const result = await this.updateRow(dataStoreId, setData, whereData, columns, returnData); + if (returnData) { + output.push.apply(output, result); + } } } - return true; - } - - async updateRow( - dataStoreId: string, - setData: Record, - whereData: Record, - columns: DataStoreColumn[], - ) { - const dbType = this.dataSource.options.type; - const columnTypeMap = buildColumnTypeMap(columns); - - const queryBuilder = this.dataSource.createQueryBuilder().update(this.toTableName(dataStoreId)); - - const setValues: Record = {}; - for (const [key, value] of Object.entries(setData)) { - setValues[key] = normalizeValue(value, columnTypeMap[key], dbType); - } - - // Always update the updatedAt timestamp - setValues.updatedAt = normalizeValue(new Date(), 'date', dbType); - - queryBuilder.set(setValues); - - const normalizedWhereData: Record = {}; - for (const [field, value] of Object.entries(whereData)) { - normalizedWhereData[field] = normalizeValue(value, columnTypeMap[field], dbType); - } - queryBuilder.where(normalizedWhereData); - - await queryBuilder.execute(); + return returnData ? output : true; } async deleteRows(dataStoreId: string, ids: number[]) { @@ -251,6 +281,25 @@ export class DataStoreRowsRepository { return { count: count ?? -1, data }; } + async getManyByIds(dataStoreId: string, ids: number[], columns: DataStoreColumn[]) { + const table = this.toTableName(dataStoreId); + const escapedColumns = columns.map((c) => this.dataSource.driver.escape(c.name)); + const selectColumns = ['id', ...escapedColumns]; + + if (ids.length === 0) { + return []; + } + + const updatedRows = await this.dataSource + .createQueryBuilder() + .select(selectColumns) + .from(table, 'dataStore') + .where({ id: In(ids) }) + .getRawMany(); + + return normalizeRows(updatedRows, columns); + } + async getRowIds(dataStoreId: string, dto: ListDataStoreContentQueryDto) { const [_, query] = this.getManyQuery(dataStoreId, dto); const result = await query.select('dataStore.id').getRawMany(); diff --git a/packages/cli/src/modules/data-table/data-store.controller.ts b/packages/cli/src/modules/data-table/data-store.controller.ts index c827e5326d..bb46de749b 100644 --- a/packages/cli/src/modules/data-table/data-store.controller.ts +++ b/packages/cli/src/modules/data-table/data-store.controller.ts @@ -274,7 +274,12 @@ export class DataStoreController { @Body dto: UpsertDataStoreRowsDto, ) { try { - return await this.dataStoreService.upsertRows(dataStoreId, req.params.projectId, dto); + return await this.dataStoreService.upsertRows( + dataStoreId, + req.params.projectId, + dto, + dto.returnData, + ); } catch (e: unknown) { if (e instanceof DataStoreNotFoundError) { throw new NotFoundError(e.message); @@ -297,7 +302,12 @@ export class DataStoreController { @Body dto: UpdateDataStoreRowDto, ) { try { - return await this.dataStoreService.updateRow(dataStoreId, req.params.projectId, dto); + return await this.dataStoreService.updateRow( + dataStoreId, + req.params.projectId, + dto, + dto.returnData, + ); } catch (e: unknown) { if (e instanceof DataStoreNotFoundError) { throw new NotFoundError(e.message); diff --git a/packages/cli/src/modules/data-table/data-store.service.ts b/packages/cli/src/modules/data-table/data-store.service.ts index 5eb2080eea..49564e8a3a 100644 --- a/packages/cli/src/modules/data-table/data-store.service.ts +++ b/packages/cli/src/modules/data-table/data-store.service.ts @@ -138,7 +138,12 @@ export class DataStoreService { return await this.dataStoreRowsRepository.insertRows(dataStoreId, rows, columns, returnData); } - async upsertRows(dataStoreId: string, projectId: string, dto: UpsertDataStoreRowsDto) { + async upsertRows( + dataStoreId: string, + projectId: string, + dto: Omit, + returnData: boolean = false, + ) { await this.validateDataStoreExists(dataStoreId, projectId); await this.validateRows(dataStoreId, dto.rows); @@ -146,12 +151,24 @@ export class DataStoreService { throw new DataStoreValidationError('No rows provided for upsertRows'); } + const { matchFields, rows } = dto; const columns = await this.dataStoreColumnRepository.getColumns(dataStoreId); - return await this.dataStoreRowsRepository.upsertRows(dataStoreId, dto, columns); + return await this.dataStoreRowsRepository.upsertRows( + dataStoreId, + matchFields, + rows, + columns, + returnData, + ); } - async updateRow(dataStoreId: string, projectId: string, dto: UpdateDataStoreRowDto) { + async updateRow( + dataStoreId: string, + projectId: string, + dto: Omit, + returnData = false, + ) { await this.validateDataStoreExists(dataStoreId, projectId); const columns = await this.dataStoreColumnRepository.getColumns(dataStoreId); @@ -172,8 +189,13 @@ export class DataStoreService { this.validateRowsWithColumns([filter], columns, true, true); this.validateRowsWithColumns([data], columns, true, false); - await this.dataStoreRowsRepository.updateRow(dataStoreId, data, filter, columns); - return true; + return await this.dataStoreRowsRepository.updateRow( + dataStoreId, + data, + filter, + columns, + returnData, + ); } async deleteRows(dataStoreId: string, projectId: string, ids: number[]) { diff --git a/packages/cli/src/modules/data-table/utils/sql-utils.ts b/packages/cli/src/modules/data-table/utils/sql-utils.ts index 0f70a0dc55..af96e651c0 100644 --- a/packages/cli/src/modules/data-table/utils/sql-utils.ts +++ b/packages/cli/src/modules/data-table/utils/sql-utils.ts @@ -216,19 +216,25 @@ export function normalizeRows(rows: DataStoreRows, columns: DataStoreColumn[]) { } } if (type === 'date' && value !== null && value !== undefined) { - // Convert date objects or strings to ISO string + // Convert date objects or strings to dates in UTC let dateObj: Date | null = null; if (value instanceof Date) { dateObj = value; - } else if (typeof value === 'string' || typeof value === 'number') { + } else if (typeof value === 'string') { + // sqlite returns date strings without timezone information, but we store them as UTC + const parsed = new Date(value.endsWith('Z') ? value : value + 'Z'); + if (!isNaN(parsed.getTime())) { + dateObj = parsed; + } + } else if (typeof value === 'number') { const parsed = new Date(value); if (!isNaN(parsed.getTime())) { dateObj = parsed; } } - normalized[key] = dateObj ? dateObj.toISOString() : value; + normalized[key] = dateObj ?? value; } } return normalized; @@ -259,9 +265,3 @@ export function normalizeValue( export function getPlaceholder(index: number, dbType: DataSourceOptions['type']): string { return dbType.includes('postgres') ? `$${index}` : '?'; } - -export function buildColumnTypeMap( - columns: Array<{ name: string; type: string }>, -): Record { - return Object.fromEntries(columns.map((col) => [col.name, col.type])); -} diff --git a/packages/workflow/src/data-store.types.ts b/packages/workflow/src/data-store.types.ts index f52f23803e..7fb49e5d25 100644 --- a/packages/workflow/src/data-store.types.ts +++ b/packages/workflow/src/data-store.types.ts @@ -104,5 +104,5 @@ export interface IDataStoreProjectService { insertRows(rows: DataStoreRows): Promise>; - upsertRows(options: UpsertDataStoreRowsOptions): Promise; + upsertRows(options: UpsertDataStoreRowsOptions): Promise; }