feat(core): Optionally return updated/upserted Data Table rows (no-changelog) (#18735)

This commit is contained in:
Jaakko Husso
2025-08-26 11:50:13 +03:00
committed by GitHub
parent 1e58e24400
commit 8defb2b17c
9 changed files with 602 additions and 145 deletions

View File

@@ -17,6 +17,7 @@ const updateDataStoreRowShape = {
.refine((obj) => Object.keys(obj).length > 0, {
message: 'data must not be empty',
}),
returnData: z.boolean().default(false),
};
export class UpdateDataStoreRowDto extends Z.class(updateDataStoreRowShape) {}

View File

@@ -9,6 +9,7 @@ import {
const upsertDataStoreRowsShape = {
rows: z.array(z.record(dataStoreColumnNameSchema, dataStoreColumnValueSchema)),
matchFields: z.array(dataStoreColumnNameSchema).min(1),
returnData: z.boolean().optional().default(false),
};
export class UpsertDataStoreRowsDto extends Z.class(upsertDataStoreRowsShape) {}

View File

@@ -2889,11 +2889,13 @@ describe('POST /projects/:projectId/data-stores/:dataStoreId/upsert', () => {
matchFields: ['first'],
};
await authMemberAgent
const result = await authMemberAgent
.post(`/projects/${memberProject.id}/data-stores/${dataStore.id}/upsert`)
.send(payload)
.expect(200);
expect(result.body.data).toBe(true);
const rowsInDb = await dataStoreRowsRepository.getManyAndCount(dataStore.id, {
sortBy: ['id', 'ASC'],
});
@@ -2902,6 +2904,59 @@ describe('POST /projects/:projectId/data-stores/:dataStoreId/upsert', () => {
expect(rowsInDb.data[1]).toMatchObject(payload.rows[0]);
expect(rowsInDb.data[2]).toMatchObject(payload.rows[1]);
});
test('should return affected rows if returnData is set', async () => {
const dataStore = await createDataStore(memberProject, {
columns: [
{
name: 'first',
type: 'string',
},
{
name: 'second',
type: 'string',
},
],
data: [
{
first: 'test row',
second: 'test value',
},
{
first: 'test row',
second: 'another row with same first column',
},
],
});
const payload = {
rows: [
{
first: 'test row',
second: 'updated value',
},
{
first: 'new row',
second: 'new value',
},
],
matchFields: ['first'],
returnData: true,
};
const result = await authMemberAgent
.post(`/projects/${memberProject.id}/data-stores/${dataStore.id}/upsert`)
.send(payload)
.expect(200);
expect(result.body.data).toEqual(
expect.arrayContaining([
{ id: 1, first: 'test row', second: 'updated value' },
{ id: 2, first: 'test row', second: 'updated value' },
{ id: 3, first: 'new row', second: 'new value' },
]),
);
});
});
describe('PATCH /projects/:projectId/data-stores/:dataStoreId/rows', () => {
@@ -2979,26 +3034,36 @@ describe('PATCH /projects/:projectId/data-stores/:dataStoreId/rows', () => {
columns: [
{ name: 'name', type: 'string' },
{ name: 'age', type: 'number' },
{ name: 'active', type: 'boolean' },
{ name: 'birthday', type: 'date' },
],
data: [{ name: 'Alice', age: 30 }],
data: [{ name: 'Alice', age: 30, active: true, birthday: new Date('1990-01-01') }],
});
const payload = {
filter: { name: 'Alice' },
data: { age: 31 },
data: { name: 'Alicia', age: 31, active: false, birthday: new Date('1990-01-02') },
};
await authMemberAgent
const result = await authMemberAgent
.patch(`/projects/${project.id}/data-stores/${dataStore.id}/rows`)
.send(payload)
.expect(200);
expect(result.body.data).toBe(true);
const readResponse = await authMemberAgent
.get(`/projects/${project.id}/data-stores/${dataStore.id}/rows`)
.expect(200);
expect(readResponse.body.data.count).toBe(1);
expect(readResponse.body.data.data[0]).toMatchObject({ id: 1, name: 'Alice', age: 31 });
expect(readResponse.body.data.data[0]).toMatchObject({
id: 1,
name: 'Alicia',
age: 31,
active: false,
birthday: new Date('1990-01-02').toISOString(),
});
});
test('should update row if user has project:admin role in team project', async () => {
@@ -3208,7 +3273,7 @@ describe('PATCH /projects/:projectId/data-stores/:dataStoreId/rows', () => {
.send(payload)
.expect(200);
expect(response.body.data).toBe(true);
expect(response.body.data).toEqual(true);
const readResponse = await authMemberAgent
.get(`/projects/${memberProject.id}/data-stores/${dataStore.id}/rows`)
@@ -3400,4 +3465,47 @@ describe('PATCH /projects/:projectId/data-stores/:dataStoreId/rows', () => {
birthdate: '1995-05-15T12:30:00.000Z',
});
});
test('should return updated data if returnData is set', async () => {
const dataStore = await createDataStore(memberProject, {
columns: [
{ name: 'name', type: 'string' },
{ name: 'age', type: 'number' },
{ name: 'active', type: 'boolean' },
{ name: 'birthday', type: 'date' },
],
data: [
{ name: 'Alice', age: 30, active: true, birthday: new Date('1990-01-01T00:00:00.000Z') },
{ name: 'Bob', age: 25, active: true, birthday: new Date('1995-05-15T00:00:00.000Z') },
],
});
const payload = {
filter: { active: true },
data: { active: false },
returnData: true,
};
const result = await authMemberAgent
.patch(`/projects/${memberProject.id}/data-stores/${dataStore.id}/rows`)
.send(payload)
.expect(200);
expect(result.body.data).toMatchObject([
{
id: 1,
name: 'Alice',
age: 30,
active: false,
birthday: '1990-01-01T00:00:00.000Z',
},
{
id: 2,
name: 'Bob',
age: 25,
active: false,
birthday: '1995-05-15T00:00:00.000Z',
},
]);
});
});

View File

@@ -977,15 +977,15 @@ describe('dataStore', () => {
);
expect(count).toEqual(4);
expect(data).toEqual(
rows.map(
(row) =>
expect.objectContaining({
...row,
c1: row.c1,
c2: row.c2,
c3: row.c3 instanceof Date ? row.c3.toISOString() : row.c3,
c4: row.c4,
}) as Record<string, unknown>,
rows.map((row, i) =>
expect.objectContaining({
...row,
id: i + 1,
c1: row.c1,
c2: row.c2,
c3: typeof row.c3 === 'string' ? new Date(row.c3) : row.c3,
c4: row.c4,
}),
),
);
});
@@ -1081,47 +1081,35 @@ describe('dataStore', () => {
]);
});
it('return inserted data if requested', async () => {
it('return full inserted data if returnData is set', async () => {
// ARRANGE
const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, {
name: 'myDataStore',
columns: [
{ name: 'c1', type: 'number' },
{ name: 'c2', type: 'string' },
{ name: 'c3', type: 'boolean' },
{ name: 'c4', type: 'date' },
],
});
const now = new Date();
// Insert initial row
const ids = await dataStoreService.insertRows(
dataStoreId,
project1.id,
[
{ c1: 1, c2: 'foo' },
{ c1: 2, c2: 'bar' },
{ c1: 1, c2: 'foo', c3: true, c4: now },
{ c1: 2, c2: 'bar', c3: false, c4: now },
{ c1: null, c2: null, c3: null, c4: null },
],
true,
);
expect(ids).toEqual([
{ id: 1, c1: 1, c2: 'foo' },
{ id: 2, c1: 2, c2: 'bar' },
]);
await dataStoreService.deleteRows(dataStoreId, project1.id, [ids[0].id]);
const result = await dataStoreService.insertRows(
dataStoreId,
project1.id,
[
{ c1: 1, c2: 'baz' },
{ c1: 2, c2: 'faz' },
],
true,
);
// ASSERT
expect(result).toEqual([
{ id: 3, c1: 1, c2: 'baz' },
{ id: 4, c1: 2, c2: 'faz' },
{ id: 1, c1: 1, c2: 'foo', c3: true, c4: now },
{ id: 2, c1: 2, c2: 'bar', c3: false, c4: now },
{ id: 3, c1: null, c2: null, c3: null, c4: null },
]);
});
@@ -1275,25 +1263,25 @@ describe('dataStore', () => {
expect(data).toEqual([
{
id: 1,
createdAt: expect.any(String),
updatedAt: expect.any(String),
createdAt: expect.any(Date),
updatedAt: expect.any(Date),
},
{
id: 2,
createdAt: expect.any(String),
updatedAt: expect.any(String),
createdAt: expect.any(Date),
updatedAt: expect.any(Date),
},
{
id: 3,
createdAt: expect.any(String),
updatedAt: expect.any(String),
createdAt: expect.any(Date),
updatedAt: expect.any(Date),
},
]);
});
});
describe('upsertRows', () => {
it('updates rows if filter matches', async () => {
it('should update a row if filter matches', async () => {
// ARRANGE
const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, {
name: 'dataStore',
@@ -1352,7 +1340,7 @@ describe('dataStore', () => {
);
});
it('works correctly with multiple filters', async () => {
it('should work correctly with multiple filters', async () => {
// ARRANGE
const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, {
name: 'dataStore',
@@ -1409,7 +1397,7 @@ describe('dataStore', () => {
);
});
it('inserts a row if filter does not match', async () => {
it('should insert a row if filter does not match', async () => {
// ARRANGE
const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, {
name: 'dataStore',
@@ -1455,6 +1443,59 @@ describe('dataStore', () => {
}),
]);
});
it('should return full upserted rows if returnData is set', async () => {
// ARRANGE
const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, {
name: 'dataStore',
columns: [
{ name: 'pid', type: 'string' },
{ name: 'fullName', type: 'string' },
{ name: 'age', type: 'number' },
{ name: 'birthday', type: 'date' },
],
});
// Insert initial row
const ids = await dataStoreService.insertRows(dataStoreId, project1.id, [
{ pid: '1995-111a', fullName: 'Alice', age: 30, birthday: new Date('1995-01-01') },
]);
expect(ids).toEqual([{ id: 1 }]);
// ACT
const result = await dataStoreService.upsertRows(
dataStoreId,
project1.id,
{
rows: [
{ pid: '1995-111a', fullName: 'Alicia', age: 31, birthday: new Date('1995-01-01') },
{ pid: '1992-222b', fullName: 'Bob', age: 30, birthday: new Date('1992-01-01') },
],
matchFields: ['pid'],
},
true,
);
// ASSERT
expect(result).toEqual(
expect.arrayContaining([
{
id: 1,
fullName: 'Alicia',
age: 31,
pid: '1995-111a',
birthday: new Date('1995-01-01'),
},
{
id: 2,
fullName: 'Bob',
age: 30,
pid: '1992-222b',
birthday: new Date('1992-01-01'),
},
]),
);
});
});
describe('deleteRows', () => {
@@ -1547,37 +1588,40 @@ describe('dataStore', () => {
{ name: 'name', type: 'string' },
{ name: 'age', type: 'number' },
{ name: 'active', type: 'boolean' },
{ name: 'birthday', type: 'date' },
],
});
await dataStoreService.insertRows(dataStoreId, project1.id, [
{ name: 'Alice', age: 30, active: true },
{ name: 'Bob', age: 25, active: false },
{ name: 'Alice', age: 30, active: true, birthday: new Date('1990-01-01') },
{ name: 'Bob', age: 25, active: false, birthday: new Date('1995-01-01') },
]);
// ACT
const result = await dataStoreService.updateRow(dataStoreId, project1.id, {
filter: { name: 'Alice' },
data: { age: 31, active: false },
data: { name: 'Alicia', age: 31, active: false, birthday: new Date('1990-01-02') },
});
// ASSERT
expect(result).toBe(true);
expect(result).toEqual(true);
const { data } = await dataStoreService.getManyRowsAndCount(dataStoreId, project1.id, {});
expect(data).toEqual(
expect.arrayContaining([
expect.objectContaining({
id: 1,
name: 'Alice',
name: 'Alicia',
age: 31,
active: false,
birthday: new Date('1990-01-02'),
}),
expect.objectContaining({
id: 2,
name: 'Bob',
age: 25,
active: false,
birthday: new Date('1995-01-01'),
}),
]),
);
@@ -1614,7 +1658,7 @@ describe('dataStore', () => {
});
// ASSERT
expect(result).toBe(true);
expect(result).toEqual(true);
const { data } = await dataStoreService.getManyRowsAndCount(dataStoreId, project1.id, {});
expect(data).toEqual(
@@ -1685,6 +1729,194 @@ describe('dataStore', () => {
);
});
it('should be able to update by string column', async () => {
// ARRANGE
const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, {
name: 'dataStore',
columns: [
{ name: 'name', type: 'string' },
{ name: 'age', type: 'number' },
{ name: 'active', type: 'boolean' },
{ name: 'birthday', type: 'date' },
],
});
await dataStoreService.insertRows(dataStoreId, project1.id, [
{ name: 'Alice', age: 30, active: true, birthday: new Date('1990-01-01') },
{ name: 'Bob', age: 25, active: false, birthday: new Date('1995-01-01') },
]);
// ACT
const result = await dataStoreService.updateRow(dataStoreId, project1.id, {
filter: { name: 'Alice' },
data: { name: 'Alicia' },
});
// ASSERT
expect(result).toEqual(true);
const { data } = await dataStoreService.getManyRowsAndCount(dataStoreId, project1.id, {});
expect(data).toEqual(
expect.arrayContaining([
expect.objectContaining({
id: 1,
name: 'Alicia',
age: 30,
active: true,
birthday: new Date('1990-01-01'),
}),
expect.objectContaining({
id: 2,
name: 'Bob',
age: 25,
active: false,
birthday: new Date('1995-01-01'),
}),
]),
);
});
it('should be able to update by number column', async () => {
// ARRANGE
const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, {
name: 'dataStore',
columns: [
{ name: 'name', type: 'string' },
{ name: 'age', type: 'number' },
{ name: 'active', type: 'boolean' },
{ name: 'birthday', type: 'date' },
],
});
await dataStoreService.insertRows(dataStoreId, project1.id, [
{ name: 'Alice', age: 30, active: true, birthday: new Date('1990-01-01') },
{ name: 'Bob', age: 25, active: false, birthday: new Date('1995-01-01') },
]);
// ACT
const result = await dataStoreService.updateRow(dataStoreId, project1.id, {
filter: { age: 30 },
data: { age: 31 },
});
// ASSERT
expect(result).toEqual(true);
const { data } = await dataStoreService.getManyRowsAndCount(dataStoreId, project1.id, {});
expect(data).toEqual(
expect.arrayContaining([
expect.objectContaining({
id: 1,
name: 'Alice',
age: 31,
active: true,
birthday: new Date('1990-01-01'),
}),
expect.objectContaining({
id: 2,
name: 'Bob',
age: 25,
active: false,
birthday: new Date('1995-01-01'),
}),
]),
);
});
it('should be able to update by boolean column', async () => {
// ARRANGE
const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, {
name: 'dataStore',
columns: [
{ name: 'name', type: 'string' },
{ name: 'age', type: 'number' },
{ name: 'active', type: 'boolean' },
{ name: 'birthday', type: 'date' },
],
});
await dataStoreService.insertRows(dataStoreId, project1.id, [
{ name: 'Alice', age: 30, active: true, birthday: new Date('1990-01-01') },
{ name: 'Bob', age: 25, active: false, birthday: new Date('1995-01-01') },
]);
// ACT
const result = await dataStoreService.updateRow(dataStoreId, project1.id, {
filter: { active: true },
data: { active: false },
});
// ASSERT
expect(result).toEqual(true);
const { data } = await dataStoreService.getManyRowsAndCount(dataStoreId, project1.id, {});
expect(data).toEqual(
expect.arrayContaining([
expect.objectContaining({
id: 1,
name: 'Alice',
age: 30,
active: false,
birthday: new Date('1990-01-01'),
}),
expect.objectContaining({
id: 2,
name: 'Bob',
age: 25,
active: false,
birthday: new Date('1995-01-01'),
}),
]),
);
});
it('should be able to update by date column', async () => {
// ARRANGE
const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, {
name: 'dataStore',
columns: [
{ name: 'name', type: 'string' },
{ name: 'age', type: 'number' },
{ name: 'active', type: 'boolean' },
{ name: 'birthday', type: 'date' },
],
});
await dataStoreService.insertRows(dataStoreId, project1.id, [
{ name: 'Alice', age: 30, active: true, birthday: new Date('1990-01-01') },
{ name: 'Bob', age: 25, active: false, birthday: new Date('1995-01-01') },
]);
// ACT
const result = await dataStoreService.updateRow(dataStoreId, project1.id, {
filter: { birthday: new Date('1990-01-01') },
data: { birthday: new Date('1990-01-02') },
});
// ASSERT
expect(result).toEqual(true);
const { data } = await dataStoreService.getManyRowsAndCount(dataStoreId, project1.id, {});
expect(data).toEqual(
expect.arrayContaining([
expect.objectContaining({
id: 1,
name: 'Alice',
age: 30,
active: true,
birthday: new Date('1990-01-02'),
}),
expect.objectContaining({
id: 2,
name: 'Bob',
age: 25,
active: false,
birthday: new Date('1995-01-01'),
}),
]),
);
});
it('should update row with multiple filter conditions', async () => {
// ARRANGE
const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, {
@@ -1709,7 +1941,7 @@ describe('dataStore', () => {
});
// ASSERT
expect(result).toBe(true);
expect(result).toEqual(true);
const { data } = await dataStoreService.getManyRowsAndCount(dataStoreId, project1.id, {});
expect(data).toEqual(
@@ -1755,7 +1987,7 @@ describe('dataStore', () => {
});
// ASSERT
expect(result).toBe(true);
expect(result).toEqual(true);
const { data } = await dataStoreService.getManyRowsAndCount(dataStoreId, project1.id, {});
expect(data).toEqual([
@@ -1919,7 +2151,7 @@ describe('dataStore', () => {
});
// ASSERT
expect(result).toBe(true);
expect(result).toEqual(true);
const { data } = await dataStoreService.getManyRowsAndCount(dataStoreId, project1.id, {});
expect(data).toEqual([
@@ -1954,14 +2186,48 @@ describe('dataStore', () => {
});
// ASSERT
expect(result).toBe(true);
expect(result).toEqual(true);
const { data } = await dataStoreService.getManyRowsAndCount(dataStoreId, project1.id, {});
expect(data).toEqual([
expect.objectContaining({
name: 'Alice',
birthDate: newDate.toISOString(),
}),
expect(data).toEqual([expect.objectContaining({ id: 1, name: 'Alice', birthDate: newDate })]);
});
it('should return full updated rows if returnData is set', async () => {
// ARRANGE
const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, {
name: 'dataStore',
columns: [
{ name: 'name', type: 'string' },
{ name: 'age', type: 'number' },
{ name: 'active', type: 'boolean' },
{ name: 'timestamp', type: 'date' },
],
});
const now = new Date();
await dataStoreService.insertRows(dataStoreId, project1.id, [
{ name: 'Alice', age: 30, active: true, timestamp: now },
{ name: 'Bob', age: 25, active: false, timestamp: now },
]);
const soon = new Date();
soon.setDate(now.getDate() + 1);
// ACT
const result = await dataStoreService.updateRow(
dataStoreId,
project1.id,
{
filter: { name: 'Alice' },
data: { age: 31, active: false, timestamp: soon },
},
true,
);
// ASSERT
expect(result).toEqual([
expect.objectContaining({ id: 1, name: 'Alice', age: 31, active: false, timestamp: soon }),
]);
});
});
@@ -2013,29 +2279,29 @@ describe('dataStore', () => {
{
c1: rows[0].c1,
c2: rows[0].c2,
c3: '1970-01-01T00:00:00.000Z',
c3: new Date(0),
c4: rows[0].c4,
id: 1,
createdAt: expect.any(String),
updatedAt: expect.any(String),
createdAt: expect.any(Date),
updatedAt: expect.any(Date),
},
{
c1: rows[1].c1,
c2: rows[1].c2,
c3: '1970-01-01T00:00:00.001Z',
c3: new Date(1),
c4: rows[1].c4,
id: 2,
createdAt: expect.any(String),
updatedAt: expect.any(String),
createdAt: expect.any(Date),
updatedAt: expect.any(Date),
},
{
c1: rows[2].c1,
c2: rows[2].c2,
c3: '1970-01-01T00:00:00.002Z',
c3: new Date(2),
c4: rows[2].c4,
id: 3,
createdAt: expect.any(String),
updatedAt: expect.any(String),
createdAt: expect.any(Date),
updatedAt: expect.any(Date),
},
]);
});

View File

@@ -1,8 +1,4 @@
import type {
ListDataStoreContentQueryDto,
ListDataStoreContentFilter,
UpsertDataStoreRowsDto,
} from '@n8n/api-types';
import type { ListDataStoreContentQueryDto, ListDataStoreContentFilter } from '@n8n/api-types';
import { GlobalConfig } from '@n8n/config';
import { CreateTable, DslColumn } from '@n8n/db';
import { Service } from '@n8n/di';
@@ -18,11 +14,11 @@ import { DataStoreColumn } from './data-store-column.entity';
import { DataStoreUserTableName } from './data-store.types';
import {
addColumnQuery,
buildColumnTypeMap,
deleteColumnQuery,
extractInsertedIds,
extractReturningData,
getPlaceholder,
normalizeRows,
normalizeValue,
quoteIdentifier,
splitRowsByExistence,
@@ -97,51 +93,112 @@ export class DataStoreRowsRepository {
const result = await query.execute();
if (useReturning) {
const returned = extractReturningData(result.raw);
const returned = normalizeRows(extractReturningData(result.raw), columns);
inserted.push.apply(inserted, returned);
continue;
}
// Engines without RETURNING support
const rowIds = extractInsertedIds(result.raw, dbType);
if (rowIds.length === 0) {
const ids = extractInsertedIds(result.raw, dbType);
if (ids.length === 0) {
throw new UnexpectedError("Couldn't find the inserted row ID");
}
if (!returnData) {
inserted.push(...rowIds.map((id) => ({ id })));
inserted.push(...ids.map((id) => ({ id })));
continue;
}
const insertedRow = await this.dataSource
.createQueryBuilder()
.select(selectColumns)
.from(table, 'dataStore')
.where({ id: In(rowIds) })
.getRawOne<DataStoreRowWithId>();
const insertedRows = (await this.getManyByIds(
dataStoreId,
ids,
columns,
)) as DataStoreRowWithId[];
if (!insertedRow) {
throw new UnexpectedError("Couldn't find the inserted row");
}
inserted.push(insertedRow);
inserted.push(...insertedRows);
}
return inserted;
}
// TypeORM cannot infer the columns for a dynamic table name, so we use a raw query
async upsertRows(dataStoreId: string, dto: UpsertDataStoreRowsDto, columns: DataStoreColumn[]) {
const { rows, matchFields } = dto;
async updateRow(
dataStoreId: string,
setData: Record<string, DataStoreColumnJsType | null>,
whereData: Record<string, DataStoreColumnJsType | null>,
columns: DataStoreColumn[],
returnData: boolean = false,
) {
const dbType = this.dataSource.options.type;
const useReturning = dbType === 'postgres';
const table = this.toTableName(dataStoreId);
const escapedColumns = columns.map((c) => this.dataSource.driver.escape(c.name));
const selectColumns = ['id', ...escapedColumns];
for (const column of columns) {
if (column.name in setData) {
setData[column.name] = normalizeValue(setData[column.name], column.type, dbType);
}
if (column.name in whereData) {
whereData[column.name] = normalizeValue(whereData[column.name], column.type, dbType);
}
}
let affectedRows: DataStoreRowWithId[] = [];
if (!useReturning && returnData) {
// Only Postgres supports RETURNING statement on updates (with our typeorm),
// on other engines we must query the list of updates rows later by ID
affectedRows = await this.dataSource
.createQueryBuilder()
.select('id')
.from(table, 'dataStore')
.where(whereData)
.getRawMany<{ id: number }>();
}
setData.updatedAt = normalizeValue(new Date(), 'date', dbType);
const query = this.dataSource.createQueryBuilder().update(table).set(setData).where(whereData);
if (useReturning && returnData) {
query.returning(selectColumns.join(','));
}
const result = await query.execute();
if (!returnData) {
return true;
}
if (useReturning) {
return extractReturningData(result.raw);
}
const ids = affectedRows.map((row) => row.id);
return await this.getManyByIds(dataStoreId, ids, columns);
}
// TypeORM cannot infer the columns for a dynamic table name, so we use a raw query
async upsertRows(
dataStoreId: string,
matchFields: string[],
rows: DataStoreRows,
columns: DataStoreColumn[],
returnData = false,
) {
const { rowsToInsert, rowsToUpdate } = await this.fetchAndSplitRowsByExistence(
dataStoreId,
matchFields,
rows,
);
const output: DataStoreRowWithId[] = [];
if (rowsToInsert.length > 0) {
await this.insertRows(dataStoreId, rowsToInsert, columns);
const result = await this.insertRows(dataStoreId, rowsToInsert, columns, returnData);
if (returnData) {
output.push.apply(output, result);
}
}
if (rowsToUpdate.length > 0) {
@@ -154,41 +211,14 @@ export class DataStoreRowsRepository {
const setData = Object.fromEntries(updateKeys.map((key) => [key, row[key]]));
const whereData = Object.fromEntries(matchFields.map((key) => [key, row[key]]));
await this.updateRow(dataStoreId, setData, whereData, columns);
const result = await this.updateRow(dataStoreId, setData, whereData, columns, returnData);
if (returnData) {
output.push.apply(output, result);
}
}
}
return true;
}
async updateRow(
dataStoreId: string,
setData: Record<string, DataStoreColumnJsType>,
whereData: Record<string, DataStoreColumnJsType>,
columns: DataStoreColumn[],
) {
const dbType = this.dataSource.options.type;
const columnTypeMap = buildColumnTypeMap(columns);
const queryBuilder = this.dataSource.createQueryBuilder().update(this.toTableName(dataStoreId));
const setValues: Record<string, DataStoreColumnJsType> = {};
for (const [key, value] of Object.entries(setData)) {
setValues[key] = normalizeValue(value, columnTypeMap[key], dbType);
}
// Always update the updatedAt timestamp
setValues.updatedAt = normalizeValue(new Date(), 'date', dbType);
queryBuilder.set(setValues);
const normalizedWhereData: Record<string, DataStoreColumnJsType | null> = {};
for (const [field, value] of Object.entries(whereData)) {
normalizedWhereData[field] = normalizeValue(value, columnTypeMap[field], dbType);
}
queryBuilder.where(normalizedWhereData);
await queryBuilder.execute();
return returnData ? output : true;
}
async deleteRows(dataStoreId: string, ids: number[]) {
@@ -251,6 +281,25 @@ export class DataStoreRowsRepository {
return { count: count ?? -1, data };
}
async getManyByIds(dataStoreId: string, ids: number[], columns: DataStoreColumn[]) {
const table = this.toTableName(dataStoreId);
const escapedColumns = columns.map((c) => this.dataSource.driver.escape(c.name));
const selectColumns = ['id', ...escapedColumns];
if (ids.length === 0) {
return [];
}
const updatedRows = await this.dataSource
.createQueryBuilder()
.select(selectColumns)
.from(table, 'dataStore')
.where({ id: In(ids) })
.getRawMany<DataStoreRowWithId>();
return normalizeRows(updatedRows, columns);
}
async getRowIds(dataStoreId: string, dto: ListDataStoreContentQueryDto) {
const [_, query] = this.getManyQuery(dataStoreId, dto);
const result = await query.select('dataStore.id').getRawMany<number>();

View File

@@ -274,7 +274,12 @@ export class DataStoreController {
@Body dto: UpsertDataStoreRowsDto,
) {
try {
return await this.dataStoreService.upsertRows(dataStoreId, req.params.projectId, dto);
return await this.dataStoreService.upsertRows(
dataStoreId,
req.params.projectId,
dto,
dto.returnData,
);
} catch (e: unknown) {
if (e instanceof DataStoreNotFoundError) {
throw new NotFoundError(e.message);
@@ -297,7 +302,12 @@ export class DataStoreController {
@Body dto: UpdateDataStoreRowDto,
) {
try {
return await this.dataStoreService.updateRow(dataStoreId, req.params.projectId, dto);
return await this.dataStoreService.updateRow(
dataStoreId,
req.params.projectId,
dto,
dto.returnData,
);
} catch (e: unknown) {
if (e instanceof DataStoreNotFoundError) {
throw new NotFoundError(e.message);

View File

@@ -138,7 +138,12 @@ export class DataStoreService {
return await this.dataStoreRowsRepository.insertRows(dataStoreId, rows, columns, returnData);
}
async upsertRows(dataStoreId: string, projectId: string, dto: UpsertDataStoreRowsDto) {
async upsertRows(
dataStoreId: string,
projectId: string,
dto: Omit<UpsertDataStoreRowsDto, 'returnData'>,
returnData: boolean = false,
) {
await this.validateDataStoreExists(dataStoreId, projectId);
await this.validateRows(dataStoreId, dto.rows);
@@ -146,12 +151,24 @@ export class DataStoreService {
throw new DataStoreValidationError('No rows provided for upsertRows');
}
const { matchFields, rows } = dto;
const columns = await this.dataStoreColumnRepository.getColumns(dataStoreId);
return await this.dataStoreRowsRepository.upsertRows(dataStoreId, dto, columns);
return await this.dataStoreRowsRepository.upsertRows(
dataStoreId,
matchFields,
rows,
columns,
returnData,
);
}
async updateRow(dataStoreId: string, projectId: string, dto: UpdateDataStoreRowDto) {
async updateRow(
dataStoreId: string,
projectId: string,
dto: Omit<UpdateDataStoreRowDto, 'returnData'>,
returnData = false,
) {
await this.validateDataStoreExists(dataStoreId, projectId);
const columns = await this.dataStoreColumnRepository.getColumns(dataStoreId);
@@ -172,8 +189,13 @@ export class DataStoreService {
this.validateRowsWithColumns([filter], columns, true, true);
this.validateRowsWithColumns([data], columns, true, false);
await this.dataStoreRowsRepository.updateRow(dataStoreId, data, filter, columns);
return true;
return await this.dataStoreRowsRepository.updateRow(
dataStoreId,
data,
filter,
columns,
returnData,
);
}
async deleteRows(dataStoreId: string, projectId: string, ids: number[]) {

View File

@@ -216,19 +216,25 @@ export function normalizeRows(rows: DataStoreRows, columns: DataStoreColumn[]) {
}
}
if (type === 'date' && value !== null && value !== undefined) {
// Convert date objects or strings to ISO string
// Convert date objects or strings to dates in UTC
let dateObj: Date | null = null;
if (value instanceof Date) {
dateObj = value;
} else if (typeof value === 'string' || typeof value === 'number') {
} else if (typeof value === 'string') {
// sqlite returns date strings without timezone information, but we store them as UTC
const parsed = new Date(value.endsWith('Z') ? value : value + 'Z');
if (!isNaN(parsed.getTime())) {
dateObj = parsed;
}
} else if (typeof value === 'number') {
const parsed = new Date(value);
if (!isNaN(parsed.getTime())) {
dateObj = parsed;
}
}
normalized[key] = dateObj ? dateObj.toISOString() : value;
normalized[key] = dateObj ?? value;
}
}
return normalized;
@@ -259,9 +265,3 @@ export function normalizeValue(
export function getPlaceholder(index: number, dbType: DataSourceOptions['type']): string {
return dbType.includes('postgres') ? `$${index}` : '?';
}
export function buildColumnTypeMap(
columns: Array<{ name: string; type: string }>,
): Record<string, string> {
return Object.fromEntries(columns.map((col) => [col.name, col.type]));
}

View File

@@ -104,5 +104,5 @@ export interface IDataStoreProjectService {
insertRows(rows: DataStoreRows): Promise<Array<{ id: number }>>;
upsertRows(options: UpsertDataStoreRowsOptions): Promise<boolean>;
upsertRows(options: UpsertDataStoreRowsOptions): Promise<boolean | DataStoreRowWithId[]>;
}