feat(core): Refactor data table update row to use filters (no-changelog) (#19092)

This commit is contained in:
Daria
2025-09-04 18:25:29 +03:00
committed by GitHub
parent 2f2672c551
commit ed6f60f52e
18 changed files with 1421 additions and 233 deletions

View File

@@ -3,40 +3,9 @@ import { z } from 'zod';
import { Z } from 'zod-class';
import { dataStoreColumnNameSchema } from '../../schemas/data-store.schema';
import { dataTableFilterSchema } from '../../schemas/data-table-filter.schema';
import { paginationSchema } from '../pagination/pagination.dto';
const FilterConditionSchema = z.union([
z.literal('eq'),
z.literal('neq'),
z.literal('like'),
z.literal('ilike'),
z.literal('gt'),
z.literal('gte'),
z.literal('lt'),
z.literal('lte'),
]);
export type ListDataStoreContentFilterConditionType = z.infer<typeof FilterConditionSchema>;
const filterRecord = z.object({
columnName: dataStoreColumnNameSchema,
condition: FilterConditionSchema.default('eq'),
value: z.union([z.string(), z.number(), z.boolean(), z.date(), z.null()]),
});
const chainedFilterSchema = z.union([z.literal('and'), z.literal('or')]);
export type ListDataStoreContentFilter = z.infer<typeof filterSchema>;
// ---------------------
// Parameter Validators
// ---------------------
const filterSchema = z.object({
type: chainedFilterSchema.default('and'),
filters: z.array(filterRecord).default([]),
});
// Filter parameter validation
const filterValidator = z
.string()
.optional()
@@ -45,7 +14,7 @@ const filterValidator = z
try {
const parsed: unknown = jsonParse(val);
try {
return filterSchema.parse(parsed);
return dataTableFilterSchema.parse(parsed);
} catch (e) {
ctx.addIssue({
code: z.ZodIssueCode.custom,
@@ -64,7 +33,6 @@ const filterValidator = z
}
});
// SortBy parameter validation
const sortByValidator = z
.string()
.optional()

View File

@@ -5,13 +5,14 @@ import {
dataStoreColumnNameSchema,
dataStoreColumnValueSchema,
} from '../../schemas/data-store.schema';
import { dataTableFilterSchema } from '../../schemas/data-table-filter.schema';
const updateDataStoreRowShape = {
filter: z
.record(dataStoreColumnNameSchema, dataStoreColumnValueSchema)
.refine((obj) => Object.keys(obj).length > 0, {
message: 'filter must not be empty',
}),
const updateFilterSchema = dataTableFilterSchema.refine((filter) => filter.filters.length > 0, {
message: 'filter must not be empty',
});
const updateDataTableRowShape = {
filter: updateFilterSchema,
data: z
.record(dataStoreColumnNameSchema, dataStoreColumnValueSchema)
.refine((obj) => Object.keys(obj).length > 0, {
@@ -20,4 +21,4 @@ const updateDataStoreRowShape = {
returnData: z.boolean().default(false),
};
export class UpdateDataStoreRowDto extends Z.class(updateDataStoreRowShape) {}
export class UpdateDataTableRowDto extends Z.class(updateDataTableRowShape) {}

View File

@@ -85,14 +85,10 @@ export { OidcConfigDto } from './oidc/config.dto';
export { CreateDataStoreDto } from './data-store/create-data-store.dto';
export { UpdateDataStoreDto } from './data-store/update-data-store.dto';
export { UpdateDataStoreRowDto } from './data-store/update-data-store-row.dto';
export { UpdateDataTableRowDto } from './data-store/update-data-store-row.dto';
export { UpsertDataStoreRowsDto } from './data-store/upsert-data-store-rows.dto';
export { ListDataStoreQueryDto } from './data-store/list-data-store-query.dto';
export { ListDataStoreContentQueryDto } from './data-store/list-data-store-content-query.dto';
export type {
ListDataStoreContentFilter,
ListDataStoreContentFilterConditionType,
} from './data-store/list-data-store-content-query.dto';
export { CreateDataStoreColumnDto } from './data-store/create-data-store-column.dto';
export { AddDataStoreRowsDto } from './data-store/add-data-store-rows.dto';
export { AddDataStoreColumnDto } from './data-store/add-data-store-column.dto';

View File

@@ -56,3 +56,8 @@ export {
type DataStoreListOptions,
dateTimeSchema,
} from './schemas/data-store.schema';
export type {
DataTableFilter,
DataTableFilterConditionType,
} from './schemas/data-table-filter.schema';

View File

@@ -0,0 +1,31 @@
import { z } from 'zod';
import { dataStoreColumnNameSchema } from './data-store.schema';
export const FilterConditionSchema = z.union([
z.literal('eq'),
z.literal('neq'),
z.literal('like'),
z.literal('ilike'),
z.literal('gt'),
z.literal('gte'),
z.literal('lt'),
z.literal('lte'),
]);
export type DataTableFilterConditionType = z.infer<typeof FilterConditionSchema>;
export const dataTableFilterRecordSchema = z.object({
columnName: dataStoreColumnNameSchema,
condition: FilterConditionSchema.default('eq'),
value: z.union([z.string(), z.number(), z.boolean(), z.date(), z.null()]),
});
export const dataTableFilterTypeSchema = z.union([z.literal('and'), z.literal('or')]);
export const dataTableFilterSchema = z.object({
type: dataTableFilterTypeSchema.default('and'),
filters: z.array(dataTableFilterRecordSchema).default([]),
});
export type DataTableFilter = z.infer<typeof dataTableFilterSchema>;

View File

@@ -3181,7 +3181,7 @@ describe('PATCH /projects/:projectId/data-tables/:dataStoreId/rows', () => {
test('should not update row when data store does not exist', async () => {
const project = await createTeamProject('test project', owner);
const payload = {
filter: { name: 'Alice' },
filter: { type: 'and', filters: [{ columnName: 'name', condition: 'eq', value: 'Alice' }] },
data: { age: 31 },
};
@@ -3201,7 +3201,7 @@ describe('PATCH /projects/:projectId/data-tables/:dataStoreId/rows', () => {
});
const payload = {
filter: { name: 'Alice' },
filter: { type: 'and', filters: [{ columnName: 'name', condition: 'eq', value: 'Alice' }] },
data: { age: 31 },
};
@@ -3247,7 +3247,7 @@ describe('PATCH /projects/:projectId/data-tables/:dataStoreId/rows', () => {
});
const payload = {
filter: { name: 'Alice' },
filter: { type: 'and', filters: [{ columnName: 'name', condition: 'eq', value: 'Alice' }] },
data: { name: 'Alicia', age: 31, active: false, birthday: new Date('1990-01-02') },
};
@@ -3284,7 +3284,7 @@ describe('PATCH /projects/:projectId/data-tables/:dataStoreId/rows', () => {
});
const payload = {
filter: { name: 'Alice' },
filter: { type: 'and', filters: [{ columnName: 'name', condition: 'eq', value: 'Alice' }] },
data: { age: 31 },
};
@@ -3312,7 +3312,7 @@ describe('PATCH /projects/:projectId/data-tables/:dataStoreId/rows', () => {
});
const payload = {
filter: { name: 'Alice' },
filter: { type: 'and', filters: [{ columnName: 'name', condition: 'eq', value: 'Alice' }] },
data: { age: 31 },
};
@@ -3343,7 +3343,7 @@ describe('PATCH /projects/:projectId/data-tables/:dataStoreId/rows', () => {
});
const payload = {
filter: { name: 'Alice' },
filter: { type: 'and', filters: [{ columnName: 'name', condition: 'eq', value: 'Alice' }] },
data: { age: 31 },
};
@@ -3377,7 +3377,7 @@ describe('PATCH /projects/:projectId/data-tables/:dataStoreId/rows', () => {
});
const payload = {
filter: { id: 1 },
filter: { type: 'and', filters: [{ columnName: 'id', condition: 'eq', value: 1 }] },
data: { age: 31 },
};
@@ -3422,7 +3422,13 @@ describe('PATCH /projects/:projectId/data-tables/:dataStoreId/rows', () => {
});
const payload = {
filter: { name: 'Alice', age: 30 },
filter: {
type: 'and',
filters: [
{ columnName: 'name', condition: 'eq', value: 'Alice' },
{ columnName: 'age', condition: 'eq', value: 30 },
],
},
data: { department: 'Management' },
};
@@ -3470,7 +3476,7 @@ describe('PATCH /projects/:projectId/data-tables/:dataStoreId/rows', () => {
});
const payload = {
filter: { name: 'Charlie' },
filter: { type: 'and', filters: [{ columnName: 'name', condition: 'eq', value: 'Charlie' }] },
data: { age: 25 },
};
@@ -3516,7 +3522,7 @@ describe('PATCH /projects/:projectId/data-tables/:dataStoreId/rows', () => {
});
const payload = {
filter: { name: 'Alice' },
filter: { type: 'and', filters: [{ columnName: 'name', condition: 'eq', value: 'Alice' }] },
data: {},
};
@@ -3535,7 +3541,7 @@ describe('PATCH /projects/:projectId/data-tables/:dataStoreId/rows', () => {
});
const payload = {
filter: { name: 'Alice' },
filter: { type: 'and', filters: [{ columnName: 'name', condition: 'eq', value: 'Alice' }] },
data: { invalidColumn: 'value' },
};
@@ -3554,7 +3560,10 @@ describe('PATCH /projects/:projectId/data-tables/:dataStoreId/rows', () => {
});
const payload = {
filter: { invalidColumn: 'Alice' },
filter: {
type: 'and',
filters: [{ columnName: 'invalidColumn', condition: 'eq', value: 'Alice' }],
},
data: { name: 'Updated' },
};
@@ -3576,7 +3585,10 @@ describe('PATCH /projects/:projectId/data-tables/:dataStoreId/rows', () => {
});
const payload = {
filter: { age: 'invalid_number' },
filter: {
type: 'and',
filters: [{ columnName: 'age', condition: 'eq', value: 'invalid_number' }],
},
data: { name: 'Updated' },
};
@@ -3598,7 +3610,7 @@ describe('PATCH /projects/:projectId/data-tables/:dataStoreId/rows', () => {
});
const payload = {
filter: { name: 'Alice' },
filter: { type: 'and', filters: [{ columnName: 'name', condition: 'eq', value: 'Alice' }] },
data: { age: 'invalid_number' },
};
@@ -3621,7 +3633,7 @@ describe('PATCH /projects/:projectId/data-tables/:dataStoreId/rows', () => {
});
const payload = {
filter: { name: 'Alice' },
filter: { type: 'and', filters: [{ columnName: 'name', condition: 'eq', value: 'Alice' }] },
data: { age: 31 }, // Only updating age, not name or active
};
@@ -3652,7 +3664,7 @@ describe('PATCH /projects/:projectId/data-tables/:dataStoreId/rows', () => {
});
const payload = {
filter: { name: 'Alice' },
filter: { type: 'and', filters: [{ columnName: 'name', condition: 'eq', value: 'Alice' }] },
data: { birthdate: '1995-05-15T12:30:00.000Z' },
};
@@ -3687,7 +3699,7 @@ describe('PATCH /projects/:projectId/data-tables/:dataStoreId/rows', () => {
});
const payload = {
filter: { active: true },
filter: { type: 'and', filters: [{ columnName: 'active', condition: 'eq', value: true }] },
data: { active: false },
returnData: true,
};
@@ -3714,4 +3726,32 @@ describe('PATCH /projects/:projectId/data-tables/:dataStoreId/rows', () => {
},
]);
});
test.each(['like', 'ilike'])(
'should auto-wrap %s filters if no wildcard is present',
async (condition) => {
const dataStore = await createDataStore(memberProject, {
columns: [
{
name: 'name',
type: 'string',
},
],
data: [{ name: 'Alice Smith' }, { name: 'Bob Jones' }],
});
const payload = {
filter: { type: 'and', filters: [{ columnName: 'name', value: 'Alice', condition }] },
data: { name: 'Alice Johnson' },
returnData: true,
};
const result = await authMemberAgent
.patch(`/projects/${memberProject.id}/data-tables/${dataStore.id}/rows`)
.send(payload)
.expect(200);
expect(result.body.data).toEqual([expect.objectContaining({ name: 'Alice Johnson' })]);
},
);
});

View File

@@ -3,6 +3,7 @@ import type { AddDataStoreColumnDto, CreateDataStoreColumnDto } from '@n8n/api-t
import { createTeamProject, testDb, testModules } from '@n8n/backend-test-utils';
import type { Project } from '@n8n/db';
import { Container } from '@n8n/di';
import type { DataStoreRow } from 'n8n-workflow';
import { DataStoreRowsRepository } from '../data-store-rows.repository';
import { DataStoreRepository } from '../data-store.repository';
@@ -842,18 +843,17 @@ describe('dataStore', () => {
{},
);
expect(count).toEqual(4);
expect(data).toEqual(
rows.map((row, i) =>
expect.objectContaining({
const expected = rows.map(
(row, i) =>
expect.objectContaining<DataStoreRow>({
...row,
id: i + 1,
c1: row.c1,
c2: row.c2,
c3: typeof row.c3 === 'string' ? new Date(row.c3) : row.c3,
c4: row.c4,
}),
),
}) as jest.AsymmetricMatcher,
);
expect(data).toEqual(expected);
});
it('inserts a row even if it matches with the existing one', async () => {
@@ -1161,7 +1161,7 @@ describe('dataStore', () => {
);
});
it('rejects a invalid date string to date column', async () => {
it('rejects an invalid date string to date column', async () => {
// ARRANGE
const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, {
name: 'dataStore',
@@ -1174,10 +1174,9 @@ describe('dataStore', () => {
]);
// ASSERT
await expect(result).rejects.toThrow(DataStoreValidationError);
await expect(result).rejects.toThrow(
new DataStoreValidationError(
"value '2025-99-15T09:48:14.259Z' does not match column type 'date'",
),
"value '2025-99-15T09:48:14.259Z' does not match column type 'date'",
);
});
@@ -1211,14 +1210,16 @@ describe('dataStore', () => {
});
// ACT
const wrongValue = new Date().toISOString();
const result = dataStoreService.insertRows(dataStoreId, project1.id, [
{ c1: 3 },
{ c1: true },
{ c1: wrongValue },
]);
// ASSERT
await expect(result).rejects.toThrow(DataStoreValidationError);
await expect(result).rejects.toThrow(
new DataStoreValidationError("value 'true' does not match column type 'number'"),
`value '${wrongValue}' does not match column type 'number'`,
);
});
@@ -1636,7 +1637,7 @@ describe('dataStore', () => {
// ACT
const result = await dataStoreService.updateRow(dataStoreId, project1.id, {
filter: { name: 'Alice' },
filter: { type: 'and', filters: [{ columnName: 'name', condition: 'eq', value: 'Alice' }] },
data: { name: 'Alicia', age: 31, active: false, birthday: new Date('1990-01-02') },
});
@@ -1690,7 +1691,7 @@ describe('dataStore', () => {
// ACT
const result = await dataStoreService.updateRow(dataStoreId, project1.id, {
filter: { id: 1 },
filter: { type: 'and', filters: [{ columnName: 'id', condition: 'eq', value: 1 }] },
data: { name: 'Alicia', age: 31, active: false },
});
@@ -1742,7 +1743,7 @@ describe('dataStore', () => {
// ACT
const result = await dataStoreService.updateRow(dataStoreId, project1.id, {
filter: { name: 'Alice' },
filter: { type: 'and', filters: [{ columnName: 'name', condition: 'eq', value: 'Alice' }] },
data: { age: 31, active: false },
});
@@ -1785,7 +1786,7 @@ describe('dataStore', () => {
// ACT
const result = await dataStoreService.updateRow(dataStoreId, project1.id, {
filter: { name: 'Alice' },
filter: { type: 'and', filters: [{ columnName: 'name', condition: 'eq', value: 'Alice' }] },
data: { name: 'Alicia' },
});
@@ -1832,7 +1833,7 @@ describe('dataStore', () => {
// ACT
const result = await dataStoreService.updateRow(dataStoreId, project1.id, {
filter: { age: 30 },
filter: { type: 'and', filters: [{ columnName: 'age', condition: 'eq', value: 30 }] },
data: { age: 31 },
});
@@ -1860,6 +1861,51 @@ describe('dataStore', () => {
);
});
it('should be able to update by numeric string', async () => {
// ARRANGE
const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, {
name: 'dataStore',
columns: [{ name: 'age', type: 'number' }],
});
await dataStoreService.insertRows(dataStoreId, project1.id, [{ age: 30 }]);
// ACT
const result = await dataStoreService.updateRow(dataStoreId, project1.id, {
filter: { type: 'and', filters: [{ columnName: 'age', condition: 'eq', value: '30' }] },
data: { age: '31' },
});
// ASSERT
expect(result).toEqual(true);
const { data } = await dataStoreService.getManyRowsAndCount(dataStoreId, project1.id, {});
expect(data).toEqual([expect.objectContaining({ age: 31 })]);
});
it('should throw on invalid numeric string', async () => {
// ARRANGE
const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, {
name: 'dataStore',
columns: [{ name: 'age', type: 'number' }],
});
await dataStoreService.insertRows(dataStoreId, project1.id, [{ age: 30 }]);
// ACT
const result = dataStoreService.updateRow(dataStoreId, project1.id, {
filter: {
type: 'and',
filters: [{ columnName: 'age', condition: 'eq', value: '30dfddf' }],
},
data: { age: '31' },
});
// ASSERT
await expect(result).rejects.toThrow(DataStoreValidationError);
await expect(result).rejects.toThrow("value '30dfddf' does not match column type 'number'");
});
it('should be able to update by boolean column', async () => {
// ARRANGE
const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, {
@@ -1879,7 +1925,7 @@ describe('dataStore', () => {
// ACT
const result = await dataStoreService.updateRow(dataStoreId, project1.id, {
filter: { active: true },
filter: { type: 'and', filters: [{ columnName: 'active', condition: 'eq', value: true }] },
data: { active: false },
});
@@ -1909,6 +1955,9 @@ describe('dataStore', () => {
it('should be able to update by date column', async () => {
// ARRANGE
const aliceBirthday = new Date('1990-01-02');
const bobBirthday = new Date('1995-01-01');
const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, {
name: 'dataStore',
columns: [
@@ -1920,14 +1969,18 @@ describe('dataStore', () => {
});
await dataStoreService.insertRows(dataStoreId, project1.id, [
{ name: 'Alice', age: 30, active: true, birthday: new Date('1990-01-01') },
{ name: 'Bob', age: 25, active: false, birthday: new Date('1995-01-01') },
{ name: 'Alice', age: 30, active: true, birthday: aliceBirthday },
{ name: 'Bob', age: 25, active: false, birthday: bobBirthday },
]);
// ACT
const newBirthday = new Date('1990-01-03');
const result = await dataStoreService.updateRow(dataStoreId, project1.id, {
filter: { birthday: new Date('1990-01-01') },
data: { birthday: new Date('1990-01-02') },
filter: {
type: 'and',
filters: [{ columnName: 'birthday', condition: 'eq', value: aliceBirthday }],
},
data: { birthday: newBirthday },
});
// ASSERT
@@ -1941,14 +1994,14 @@ describe('dataStore', () => {
name: 'Alice',
age: 30,
active: true,
birthday: new Date('1990-01-02'),
birthday: newBirthday,
}),
expect.objectContaining({
id: 2,
name: 'Bob',
age: 25,
active: false,
birthday: new Date('1995-01-01'),
birthday: bobBirthday,
}),
]),
);
@@ -1973,7 +2026,13 @@ describe('dataStore', () => {
// ACT
const result = await dataStoreService.updateRow(dataStoreId, project1.id, {
filter: { name: 'Alice', age: 30 },
filter: {
type: 'and',
filters: [
{ columnName: 'name', condition: 'eq', value: 'Alice' },
{ columnName: 'age', condition: 'eq', value: 30 },
],
},
data: { department: 'Management' },
});
@@ -2019,7 +2078,10 @@ describe('dataStore', () => {
// ACT
const result = await dataStoreService.updateRow(dataStoreId, project1.id, {
filter: { name: 'Charlie' },
filter: {
type: 'and',
filters: [{ columnName: 'name', condition: 'eq', value: 'Charlie' }],
},
data: { age: 25 },
});
@@ -2049,13 +2111,13 @@ describe('dataStore', () => {
// ACT
const result = dataStoreService.updateRow(dataStoreId, project1.id, {
filter: {},
filter: { type: 'and', filters: [] },
data: { name: 'Alice', age: 31 },
});
// ASSERT
await expect(result).rejects.toThrow(
new DataStoreValidationError('Filter columns must not be empty for updateRow'),
new DataStoreValidationError('Filter must not be empty for updateRow'),
);
const { data } = await dataStoreService.getManyRowsAndCount(dataStoreId, project1.id, {});
@@ -2081,7 +2143,7 @@ describe('dataStore', () => {
// ACT
const result = dataStoreService.updateRow(dataStoreId, project1.id, {
filter: { name: 'Alice' },
filter: { type: 'and', filters: [{ columnName: 'name', condition: 'eq', value: 'Alice' }] },
data: {},
});
@@ -2102,7 +2164,7 @@ describe('dataStore', () => {
it('should fail when data store does not exist', async () => {
// ACT & ASSERT
const result = dataStoreService.updateRow('non-existent-id', project1.id, {
filter: { name: 'Alice' },
filter: { type: 'and', filters: [{ columnName: 'name', condition: 'eq', value: 'Alice' }] },
data: { age: 25 },
});
@@ -2120,7 +2182,7 @@ describe('dataStore', () => {
// ACT & ASSERT
const result = dataStoreService.updateRow(dataStoreId, project1.id, {
filter: { name: 'Alice' },
filter: { type: 'and', filters: [{ columnName: 'name', condition: 'eq', value: 'Alice' }] },
data: { invalidColumn: 'value' },
});
@@ -2138,7 +2200,10 @@ describe('dataStore', () => {
// ACT & ASSERT
const result = dataStoreService.updateRow(dataStoreId, project1.id, {
filter: { invalidColumn: 'Alice' },
filter: {
type: 'and',
filters: [{ columnName: 'invalidColumn', condition: 'eq', value: 'Alice' }],
},
data: { name: 'Bob' },
});
@@ -2159,7 +2224,7 @@ describe('dataStore', () => {
// ACT & ASSERT
const result = dataStoreService.updateRow(dataStoreId, project1.id, {
filter: { name: 'Alice' },
filter: { type: 'and', filters: [{ columnName: 'name', condition: 'eq', value: 'Alice' }] },
data: { age: 'not-a-number' },
});
@@ -2183,7 +2248,7 @@ describe('dataStore', () => {
// ACT - only update age, leaving name and active unchanged
const result = await dataStoreService.updateRow(dataStoreId, project1.id, {
filter: { name: 'Alice' },
filter: { type: 'and', filters: [{ columnName: 'name', condition: 'eq', value: 'Alice' }] },
data: { age: 31 },
});
@@ -2218,7 +2283,7 @@ describe('dataStore', () => {
// ACT
const newDate = new Date('1991-02-02');
const result = await dataStoreService.updateRow(dataStoreId, project1.id, {
filter: { name: 'Alice' },
filter: { type: 'and', filters: [{ columnName: 'name', condition: 'eq', value: 'Alice' }] },
data: { birthDate: newDate.toISOString() },
});
@@ -2256,7 +2321,10 @@ describe('dataStore', () => {
dataStoreId,
project1.id,
{
filter: { name: 'Alice' },
filter: {
type: 'and',
filters: [{ columnName: 'name', condition: 'eq', value: 'Alice' }],
},
data: { age: 31, active: false, timestamp: soon },
},
true,
@@ -2342,5 +2410,27 @@ describe('dataStore', () => {
},
]);
});
it('should fail when filter contains invalid column names', async () => {
// ARRANGE
const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, {
name: 'dataStore',
columns: [{ name: 'name', type: 'string' }],
});
await dataStoreService.insertRows(dataStoreId, project1.id, [{ name: 'Alice' }]);
// ACT
const result = dataStoreService.updateRow(dataStoreId, project1.id, {
filter: {
type: 'and',
filters: [{ columnName: 'invalidColumn', condition: 'eq', value: 'Alice' }],
},
data: { name: 'Bob' },
});
// ASSERT
await expect(result).rejects.toThrow(DataStoreValidationError);
});
});
});

View File

@@ -1,8 +1,16 @@
import type { ListDataStoreContentQueryDto, ListDataStoreContentFilter } from '@n8n/api-types';
import type { ListDataStoreContentQueryDto, DataTableFilter } from '@n8n/api-types';
import { GlobalConfig } from '@n8n/config';
import { CreateTable, DslColumn } from '@n8n/db';
import { Service } from '@n8n/di';
import { DataSource, DataSourceOptions, QueryRunner, SelectQueryBuilder, In } from '@n8n/typeorm';
import {
DataSource,
DataSourceOptions,
QueryRunner,
SelectQueryBuilder,
UpdateQueryBuilder,
In,
ObjectLiteral,
} from '@n8n/typeorm';
import {
DataStoreColumnJsType,
DataStoreRows,
@@ -44,20 +52,23 @@ type QueryBuilder = SelectQueryBuilder<any>;
* - MySQL/MariaDB: the SQL literal itself requires two backslashes (`'\\'`) to mean one.
*/
function getConditionAndParams(
filter: ListDataStoreContentFilter['filters'][number],
filter: DataTableFilter['filters'][number],
index: number,
dbType: DataSourceOptions['type'],
tableReference?: string,
columns?: DataTableColumn[],
): [string, Record<string, unknown>] {
const paramName = `filter_${index}`;
const column = `${quoteIdentifier('dataStore', dbType)}.${quoteIdentifier(filter.columnName, dbType)}`;
const columnRef = tableReference
? `${quoteIdentifier(tableReference, dbType)}.${quoteIdentifier(filter.columnName, dbType)}`
: quoteIdentifier(filter.columnName, dbType);
if (filter.value === null) {
switch (filter.condition) {
case 'eq':
return [`${column} IS NULL`, {}];
return [`${columnRef} IS NULL`, {}];
case 'neq':
return [`${column} IS NOT NULL`, {}];
return [`${columnRef} IS NOT NULL`, {}];
}
}
@@ -76,7 +87,7 @@ function getConditionAndParams(
};
if (operators[filter.condition]) {
return [`${column} ${operators[filter.condition]} :${paramName}`, { [paramName]: value }];
return [`${columnRef} ${operators[filter.condition]} :${paramName}`, { [paramName]: value }];
}
switch (filter.condition) {
@@ -84,29 +95,32 @@ function getConditionAndParams(
case 'like':
if (['sqlite', 'sqlite-pooled'].includes(dbType)) {
const globValue = toSqliteGlobFromPercent(value as string);
return [`${column} GLOB :${paramName}`, { [paramName]: globValue }];
return [`${columnRef} GLOB :${paramName}`, { [paramName]: globValue }];
}
if (['mysql', 'mariadb'].includes(dbType)) {
const escapedValue = escapeLikeSpecials(value as string);
return [`${column} LIKE BINARY :${paramName} ESCAPE '\\\\'`, { [paramName]: escapedValue }];
return [
`${columnRef} LIKE BINARY :${paramName} ESCAPE '\\\\'`,
{ [paramName]: escapedValue },
];
}
// PostgreSQL: LIKE is case-sensitive
if (dbType === 'postgres') {
const escapedValue = escapeLikeSpecials(value as string);
return [`${column} LIKE :${paramName} ESCAPE '\\'`, { [paramName]: escapedValue }];
return [`${columnRef} LIKE :${paramName} ESCAPE '\\'`, { [paramName]: escapedValue }];
}
// Generic fallback
return [`${column} LIKE :${paramName}`, { [paramName]: value }];
return [`${columnRef} LIKE :${paramName}`, { [paramName]: value }];
// case-insensitive
case 'ilike':
if (['sqlite', 'sqlite-pooled'].includes(dbType)) {
const escapedValue = escapeLikeSpecials(value as string);
return [
`UPPER(${column}) LIKE UPPER(:${paramName}) ESCAPE '\\'`,
`UPPER(${columnRef}) LIKE UPPER(:${paramName}) ESCAPE '\\'`,
{ [paramName]: escapedValue },
];
}
@@ -114,17 +128,17 @@ function getConditionAndParams(
if (['mysql', 'mariadb'].includes(dbType)) {
const escapedValue = escapeLikeSpecials(value as string);
return [
`UPPER(${column}) LIKE UPPER(:${paramName}) ESCAPE '\\\\'`,
`UPPER(${columnRef}) LIKE UPPER(:${paramName}) ESCAPE '\\\\'`,
{ [paramName]: escapedValue },
];
}
if (dbType === 'postgres') {
const escapedValue = escapeLikeSpecials(value as string);
return [`${column} ILIKE :${paramName} ESCAPE '\\'`, { [paramName]: escapedValue }];
return [`${columnRef} ILIKE :${paramName} ESCAPE '\\'`, { [paramName]: escapedValue }];
}
return [`UPPER(${column}) LIKE UPPER(:${paramName})`, { [paramName]: value }];
return [`UPPER(${columnRef}) LIKE UPPER(:${paramName})`, { [paramName]: value }];
}
// This should never happen as all valid conditions are handled above
@@ -218,7 +232,7 @@ export class DataStoreRowsRepository {
async updateRow(
dataStoreId: string,
setData: Record<string, DataStoreColumnJsType | null>,
whereData: Record<string, DataStoreColumnJsType | null>,
filter: DataTableFilter,
columns: DataTableColumn[],
returnData: boolean = false,
) {
@@ -236,26 +250,26 @@ export class DataStoreRowsRepository {
if (column.name in setData) {
setData[column.name] = normalizeValue(setData[column.name], column.type, dbType);
}
if (column.name in whereData) {
whereData[column.name] = normalizeValue(whereData[column.name], column.type, dbType);
}
}
let affectedRows: Array<Pick<DataStoreRowReturn, 'id'>> = [];
if (!useReturning && returnData) {
// Only Postgres supports RETURNING statement on updates (with our typeorm),
// on other engines we must query the list of updates rows later by ID
affectedRows = await this.dataSource
const selectQuery = this.dataSource
.createQueryBuilder()
.select('id')
.from(table, 'dataStore')
.where(whereData)
.getRawMany<{ id: number }>();
.from(table, 'dataTable');
this.applyFilters(selectQuery, filter, 'dataTable', columns);
affectedRows = await selectQuery.getRawMany<{ id: number }>();
}
setData.updatedAt = normalizeValue(new Date(), 'date', dbType);
const query = this.dataSource.createQueryBuilder().update(table).set(setData).where(whereData);
const query = this.dataSource.createQueryBuilder().update(table);
// Some DBs (like SQLite) don't allow using table aliases as column prefixes in UPDATE statements
this.applyFilters(query, filter, undefined, columns);
query.set(setData);
if (useReturning && returnData) {
query.returning(selectColumns.join(','));
@@ -316,7 +330,16 @@ export class DataStoreRowsRepository {
const setData = Object.fromEntries(updateKeys.map((key) => [key, row[key]]));
const whereData = Object.fromEntries(matchFields.map((key) => [key, row[key]]));
const result = await this.updateRow(dataStoreId, setData, whereData, columns, returnData);
// Convert whereData object to DataTableFilter format
const filter: DataTableFilter = {
type: 'and',
filters: Object.entries(whereData).map(([columnName, value]) => ({
columnName,
condition: 'eq' as const,
value,
})),
};
const result = await this.updateRow(dataStoreId, setData, filter, columns, returnData);
if (returnData) {
output.push.apply(output, result);
}
@@ -336,7 +359,7 @@ export class DataStoreRowsRepository {
await this.dataSource
.createQueryBuilder()
.delete()
.from(table, 'dataStore')
.from(table, 'dataTable')
.where({ id: In(ids) })
.execute();
@@ -408,7 +431,7 @@ export class DataStoreRowsRepository {
const updatedRows = await this.dataSource
.createQueryBuilder()
.select(selectColumns)
.from(table, 'dataStore')
.from(table, 'dataTable')
.where({ id: In(ids) })
.getRawMany<DataStoreRowReturn>();
@@ -428,8 +451,11 @@ export class DataStoreRowsRepository {
): [QueryBuilder, QueryBuilder] {
const query = this.dataSource.createQueryBuilder();
query.from(this.toTableName(dataStoreId), 'dataStore');
this.applyFilters(query, dto, columns);
const tableReference = 'dataTable';
query.from(this.toTableName(dataStoreId), tableReference);
if (dto.filter) {
this.applyFilters(query, dto.filter, tableReference, columns);
}
const countQuery = query.clone().select('COUNT(*)');
this.applySorting(query, dto);
this.applyPagination(query, dto);
@@ -437,17 +463,18 @@ export class DataStoreRowsRepository {
return [countQuery, query];
}
private applyFilters(
query: QueryBuilder,
dto: ListDataStoreContentQueryDto,
private applyFilters<T extends ObjectLiteral>(
query: SelectQueryBuilder<T> | UpdateQueryBuilder<T>,
filter: DataTableFilter,
tableReference?: string,
columns?: DataTableColumn[],
): void {
const filters = dto.filter?.filters ?? [];
const filterType = dto.filter?.type ?? 'and';
const filters = filter.filters ?? [];
const filterType = filter.type ?? 'and';
const dbType = this.dataSource.options.type;
const conditionsAndParams = filters.map((filter, i) =>
getConditionAndParams(filter, i, dbType, columns),
getConditionAndParams(filter, i, dbType, tableReference, columns),
);
for (const [condition, params] of conditionsAndParams) {
@@ -470,7 +497,7 @@ export class DataStoreRowsRepository {
private applySortingByField(query: QueryBuilder, field: string, direction: 'DESC' | 'ASC'): void {
const dbType = this.dataSource.options.type;
const quotedField = `${quoteIdentifier('dataStore', dbType)}.${quoteIdentifier(field, dbType)}`;
const quotedField = `${quoteIdentifier('dataTable', dbType)}.${quoteIdentifier(field, dbType)}`;
query.orderBy(quotedField, direction);
}
@@ -487,7 +514,7 @@ export class DataStoreRowsRepository {
const queryBuilder = this.dataSource
.createQueryBuilder()
.select(matchFields)
.from(this.toTableName(dataStoreId), 'datastore');
.from(this.toTableName(dataStoreId), 'datatable');
rows.forEach((row, index) => {
const matchData = Object.fromEntries(matchFields.map((field) => [field, row[field]]));

View File

@@ -7,7 +7,7 @@ import {
ListDataStoreQueryDto,
MoveDataStoreColumnDto,
UpdateDataStoreDto,
UpdateDataStoreRowDto,
UpdateDataTableRowDto,
UpsertDataStoreRowsDto,
} from '@n8n/api-types';
import { AuthenticatedRequest } from '@n8n/db';
@@ -306,7 +306,7 @@ export class DataStoreController {
req: AuthenticatedRequest<{ projectId: string }>,
_res: Response,
@Param('dataStoreId') dataStoreId: string,
@Body dto: UpdateDataStoreRowDto,
@Body dto: UpdateDataTableRowDto,
) {
try {
return await this.dataStoreService.updateRow(

View File

@@ -1,4 +1,3 @@
import { dateTimeSchema } from '@n8n/api-types';
import type {
AddDataStoreColumnDto,
CreateDataStoreDto,
@@ -7,15 +6,25 @@ import type {
DataStoreListOptions,
UpsertDataStoreRowsDto,
UpdateDataStoreDto,
UpdateDataStoreRowDto,
UpdateDataTableRowDto,
} from '@n8n/api-types';
import { Logger } from '@n8n/backend-common';
import { Service } from '@n8n/di';
import type { DataStoreRow, DataStoreRowReturn, DataStoreRows } from 'n8n-workflow';
import { DateTime } from 'luxon';
import type {
DataStoreColumnJsType,
DataTableFilter,
DataStoreRow,
DataStoreRowReturn,
DataStoreRows,
} from 'n8n-workflow';
import { validateFieldType } from 'n8n-workflow';
import { DataStoreColumnRepository } from './data-store-column.repository';
import { DataStoreRowsRepository } from './data-store-rows.repository';
import { DataStoreRepository } from './data-store.repository';
import { columnTypeToFieldType } from './data-store.types';
import { DataTableColumn } from './data-table-column.entity';
import { DataStoreColumnNotFoundError } from './errors/data-store-column-not-found.error';
import { DataStoreNameConflictError } from './errors/data-store-name-conflict.error';
import { DataStoreNotFoundError } from './errors/data-store-not-found.error';
@@ -107,12 +116,11 @@ export class DataStoreService {
dto: ListDataStoreContentQueryDto,
) {
await this.validateDataStoreExists(dataStoreId, projectId);
this.validateAndTransformFilters(dto);
// unclear if we should validate here, only use case would be to reduce the chance of
// a renamed/removed column appearing here (or added column missing) if the store was
// modified between when the frontend sent the request and we received it
const columns = await this.dataStoreColumnRepository.getColumns(dataStoreId);
if (dto.filter) {
this.validateAndTransformFilters(dto.filter, columns);
}
const result = await this.dataStoreRowsRepository.getManyAndCount(dataStoreId, dto, columns);
return {
count: result.count,
@@ -177,39 +185,39 @@ export class DataStoreService {
}
async updateRow<T extends boolean | undefined>(
dataStoreId: string,
dataTableId: string,
projectId: string,
dto: Omit<UpdateDataStoreRowDto, 'returnData'>,
dto: Omit<UpdateDataTableRowDto, 'returnData'>,
returnData?: T,
): Promise<T extends true ? DataStoreRowReturn[] : true>;
async updateRow(
dataStoreId: string,
dataTableId: string,
projectId: string,
dto: Omit<UpdateDataStoreRowDto, 'returnData'>,
dto: Omit<UpdateDataTableRowDto, 'returnData'>,
returnData = false,
) {
await this.validateDataStoreExists(dataStoreId, projectId);
await this.validateDataStoreExists(dataTableId, projectId);
const columns = await this.dataStoreColumnRepository.getColumns(dataStoreId);
const columns = await this.dataStoreColumnRepository.getColumns(dataTableId);
if (columns.length === 0) {
throw new DataStoreValidationError(
'No columns found for this data store or data store not found',
'No columns found for this data table or data table not found',
);
}
const { data, filter } = dto;
if (!filter || Object.keys(filter).length === 0) {
throw new DataStoreValidationError('Filter columns must not be empty for updateRow');
if (!filter?.filters || filter.filters.length === 0) {
throw new DataStoreValidationError('Filter must not be empty for updateRow');
}
if (!data || Object.keys(data).length === 0) {
throw new DataStoreValidationError('Data columns must not be empty for updateRow');
}
this.validateRowsWithColumns([filter], columns, true);
this.validateRowsWithColumns([data], columns, false);
this.validateAndTransformFilters(filter, columns);
return await this.dataStoreRowsRepository.updateRow(
dataStoreId,
dataTableId,
data,
filter,
columns,
@@ -264,42 +272,36 @@ export class DataStoreService {
if (cell === null) return;
const columnType = columnTypeMap.get(key);
switch (columnType) {
case 'boolean':
if (typeof cell !== 'boolean') {
throw new DataStoreValidationError(
`value '${String(cell)}' does not match column type 'boolean'`,
);
}
break;
case 'date':
if (typeof cell === 'string') {
const validated = dateTimeSchema.safeParse(cell);
if (validated.success) {
row[key] = validated.data.toISOString();
break;
}
} else if (cell instanceof Date) {
row[key] = cell.toISOString();
break;
}
if (!columnType) return;
throw new DataStoreValidationError(`value '${cell}' does not match column type 'date'`);
case 'string':
if (typeof cell !== 'string') {
throw new DataStoreValidationError(
`value '${String(cell)}' does not match column type 'string'`,
);
}
break;
case 'number':
if (typeof cell !== 'number') {
throw new DataStoreValidationError(
`value '${String(cell)}' does not match column type 'number'`,
);
}
break;
const fieldType = columnTypeToFieldType[columnType];
if (!fieldType) return;
const validationResult = validateFieldType(key, cell, fieldType, {
strict: false, // Allow type coercion (e.g., string numbers to numbers)
parseStrings: false,
});
if (!validationResult.valid) {
throw new DataStoreValidationError(
`value '${String(cell)}' does not match column type '${columnType}': ${validationResult.errorMessage}`,
);
}
// Special handling for date type to convert from luxon DateTime to ISO string
if (columnType === 'date') {
try {
const dateInISO = (validationResult.newValue as DateTime).toISO();
row[key] = dateInISO;
return;
} catch {
throw new DataStoreValidationError(
`value '${String(cell)}' does not match column type 'date'`,
);
}
}
row[key] = validationResult.newValue as DataStoreColumnJsType;
}
private async validateDataStoreExists(dataStoreId: string, projectId: string) {
@@ -341,12 +343,21 @@ export class DataStoreService {
}
}
private validateAndTransformFilters(dto: ListDataStoreContentQueryDto): void {
if (!dto.filter?.filters) {
return;
}
private validateAndTransformFilters(
filterObject: DataTableFilter,
columns: DataTableColumn[],
): void {
this.validateRowsWithColumns(
filterObject.filters.map((f) => {
return {
[f.columnName]: f.value,
};
}),
columns,
true,
);
for (const filter of dto.filter.filters) {
for (const filter of filterObject.filters) {
if (['like', 'ilike'].includes(filter.condition)) {
if (filter.value === null || filter.value === undefined) {
throw new DataStoreValidationError(

View File

@@ -1 +1,13 @@
import type { FieldTypeMap } from 'n8n-workflow';
export type DataStoreUserTableName = `${string}data_table_user_${string}`;
export const columnTypeToFieldType: Record<string, keyof FieldTypeMap> = {
// eslint-disable-next-line id-denylist
number: 'number',
// eslint-disable-next-line id-denylist
string: 'string',
// eslint-disable-next-line id-denylist
boolean: 'boolean',
date: 'dateTime',
};

View File

@@ -175,7 +175,10 @@ export const updateDataStoreRowsApi = async (
'PATCH',
`/projects/${projectId}/data-tables/${dataStoreId}/rows`,
{
filter: { id: rowId },
filter: {
type: 'and',
filters: [{ columnName: 'id', condition: 'eq', value: rowId }],
},
data: rowData,
},
);

View File

@@ -7,7 +7,7 @@ import {
} from 'n8n-workflow';
import { makeAddRow, getAddRow } from '../../common/addRow';
import { executeSelectMany, getSelectFields } from '../../common/selectMany';
import { getSelectFields, getSelectFilter } from '../../common/selectMany';
import { getDataTableProxyExecute } from '../../common/utils';
export const FIELD: string = 'update';
@@ -31,26 +31,16 @@ export async function execute(
const dataStoreProxy = await getDataTableProxyExecute(this, index);
const row = getAddRow(this, index);
const filter = getSelectFilter(this, index);
const matches = await executeSelectMany(this, index, dataStoreProxy, true);
const result = [];
for (const x of matches) {
const updatedRows = await dataStoreProxy.updateRows({
data: row,
filter: { id: x.json.id },
});
if (updatedRows.length !== 1) {
throw new NodeOperationError(this.getNode(), 'invariant broken');
}
// The input object gets updated in the api call, somehow
// And providing this column to the backend causes an unexpected column error
// So let's just re-delete the field until we have a more aligned API
delete row['updatedAt'];
result.push(updatedRows[0]);
if (filter.filters.length === 0) {
throw new NodeOperationError(this.getNode(), 'At least one condition is required');
}
return result.map((json) => ({ json }));
const updatedRows = await dataStoreProxy.updateRows({
data: row,
filter,
});
return updatedRows.map((json) => ({ json }));
}

View File

@@ -45,7 +45,10 @@ export async function execute(
for (const match of matches) {
const updatedRows = await dataStoreProxy.updateRows({
data: row,
filter: { id: match.json.id },
filter: {
type: 'and',
filters: [{ columnName: 'id', condition: 'eq', value: match.json.id }],
},
});
if (updatedRows.length !== 1) {
throw new NodeOperationError(this.getNode(), 'invariant broken');

View File

@@ -1,5 +1,6 @@
import { NodeOperationError } from 'n8n-workflow';
import type {
DataTableFilter,
DataStoreRowReturn,
IDataStoreProjectService,
IDisplayOptions,
@@ -94,7 +95,7 @@ export function getSelectFields(
];
}
export function getSelectFilter(ctx: IExecuteFunctions, index: number) {
export function getSelectFilter(ctx: IExecuteFunctions, index: number): DataTableFilter {
const fields = ctx.getNodeParameter('filters.conditions', index, []);
const matchType = ctx.getNodeParameter('matchType', index, ANY_CONDITION);
const node = ctx.getNode();

View File

@@ -2,7 +2,7 @@ import { DateTime } from 'luxon';
import type {
IDataObject,
INode,
ListDataStoreContentFilter,
DataTableFilter,
IDataStoreProjectAggregateService,
IDataStoreProjectService,
IExecuteFunctions,
@@ -82,7 +82,7 @@ export function isMatchType(obj: unknown): obj is FilterType {
export function buildGetManyFilter(
fieldEntries: FieldEntry[],
matchType: FilterType,
): ListDataStoreContentFilter {
): DataTableFilter {
const filters = fieldEntries.map((x) => {
switch (x.condition) {
case 'isEmpty':

View File

@@ -41,7 +41,7 @@ export type ListDataStoreOptions = {
skip?: number;
};
export type ListDataStoreContentFilter = {
export type DataTableFilter = {
type: 'and' | 'or';
filters: Array<{
columnName: string;
@@ -51,14 +51,14 @@ export type ListDataStoreContentFilter = {
};
export type ListDataStoreRowsOptions = {
filter?: ListDataStoreContentFilter;
filter?: DataTableFilter;
sortBy?: [string, 'ASC' | 'DESC'];
take?: number;
skip?: number;
};
export type UpdateDataStoreRowsOptions = {
filter: Record<string, DataStoreColumnJsType>;
filter: DataTableFilter;
data: DataStoreRow;
};