mirror of
https://github.com/Abdulazizzn/n8n-enterprise-unlocked.git
synced 2025-12-16 17:46:45 +00:00
feat(core): Add support for like match filters (no-changelog) (#18755)
This commit is contained in:
@@ -5,7 +5,12 @@ import { Z } from 'zod-class';
|
||||
import { dataStoreColumnNameSchema } from '../../schemas/data-store.schema';
|
||||
import { paginationSchema } from '../pagination/pagination.dto';
|
||||
|
||||
const FilterConditionSchema = z.union([z.literal('eq'), z.literal('neq')]);
|
||||
const FilterConditionSchema = z.union([
|
||||
z.literal('eq'),
|
||||
z.literal('neq'),
|
||||
z.literal('like'),
|
||||
z.literal('ilike'),
|
||||
]);
|
||||
export type ListDataStoreContentFilterConditionType = z.infer<typeof FilterConditionSchema>;
|
||||
|
||||
const filterRecord = z.object({
|
||||
|
||||
@@ -86,6 +86,7 @@ export { UpdateDataStoreRowDto } from './data-store/update-data-store-row.dto';
|
||||
export { UpsertDataStoreRowsDto } from './data-store/upsert-data-store-rows.dto';
|
||||
export { ListDataStoreQueryDto } from './data-store/list-data-store-query.dto';
|
||||
export { ListDataStoreContentQueryDto } from './data-store/list-data-store-content-query.dto';
|
||||
export { ListDataStoreContentFilterConditionType } from './data-store/list-data-store-content-query.dto';
|
||||
export type { ListDataStoreContentFilter } from './data-store/list-data-store-content-query.dto';
|
||||
export { CreateDataStoreColumnDto } from './data-store/create-data-store-column.dto';
|
||||
export { AddDataStoreRowsDto } from './data-store/add-data-store-rows.dto';
|
||||
|
||||
@@ -1709,6 +1709,93 @@ describe('GET /projects/:projectId/data-stores/:dataStoreId/rows', () => {
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
test("should parse 'eq' filters correctly", async () => {
|
||||
const dataStore = await createDataStore(memberProject, {
|
||||
columns: [
|
||||
{
|
||||
name: 'name',
|
||||
type: 'string',
|
||||
},
|
||||
],
|
||||
data: [
|
||||
{
|
||||
name: 'John',
|
||||
},
|
||||
{
|
||||
name: 'Jane',
|
||||
},
|
||||
{
|
||||
name: 'Tom',
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const filterParam = encodeURIComponent(
|
||||
JSON.stringify({
|
||||
type: 'and',
|
||||
filters: [{ columnName: 'name', value: 'John', condition: 'eq' }],
|
||||
}),
|
||||
);
|
||||
|
||||
const response = await authMemberAgent
|
||||
.get(`/projects/${memberProject.id}/data-stores/${dataStore.id}/rows?filter=${filterParam}`)
|
||||
.expect(200);
|
||||
|
||||
expect(response.body.data).toEqual({
|
||||
count: 1,
|
||||
data: [
|
||||
expect.objectContaining({
|
||||
name: 'John',
|
||||
}),
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
test("should parse 'like' filters correctly", async () => {
|
||||
const dataStore = await createDataStore(memberProject, {
|
||||
columns: [
|
||||
{
|
||||
name: 'name',
|
||||
type: 'string',
|
||||
},
|
||||
],
|
||||
data: [
|
||||
{
|
||||
name: 'John',
|
||||
},
|
||||
{
|
||||
name: 'Jane',
|
||||
},
|
||||
{
|
||||
name: 'Tom',
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const filterParam = encodeURIComponent(
|
||||
JSON.stringify({
|
||||
type: 'and',
|
||||
filters: [{ columnName: 'name', value: '%j%', condition: 'ilike' }],
|
||||
}),
|
||||
);
|
||||
|
||||
const response = await authMemberAgent
|
||||
.get(`/projects/${memberProject.id}/data-stores/${dataStore.id}/rows?filter=${filterParam}`)
|
||||
.expect(200);
|
||||
|
||||
expect(response.body.data).toEqual({
|
||||
count: 2,
|
||||
data: [
|
||||
expect.objectContaining({
|
||||
name: 'John',
|
||||
}),
|
||||
expect.objectContaining({
|
||||
name: 'Jane',
|
||||
}),
|
||||
],
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('POST /projects/:projectId/data-stores/:dataStoreId/insert', () => {
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
/* eslint-disable @typescript-eslint/no-unsafe-assignment */
|
||||
import type { AddDataStoreColumnDto, CreateDataStoreColumnDto } from '@n8n/api-types';
|
||||
import type {
|
||||
AddDataStoreColumnDto,
|
||||
CreateDataStoreColumnDto,
|
||||
ListDataStoreContentFilterConditionType,
|
||||
} from '@n8n/api-types';
|
||||
import { createTeamProject, testDb, testModules } from '@n8n/backend-test-utils';
|
||||
import { Project } from '@n8n/db';
|
||||
import { Container } from '@n8n/di';
|
||||
@@ -2305,5 +2309,605 @@ describe('dataStore', () => {
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("retrieves rows with 'equals' filter correctly", async () => {
|
||||
// ARRANGE
|
||||
const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, {
|
||||
name: 'dataStore',
|
||||
columns: [
|
||||
{ name: 'name', type: 'string' },
|
||||
{ name: 'age', type: 'number' },
|
||||
],
|
||||
});
|
||||
|
||||
const rows = [
|
||||
{ name: 'John', age: 30 },
|
||||
{ name: 'Mary', age: 25 },
|
||||
{ name: 'Jack', age: 35 },
|
||||
];
|
||||
|
||||
await dataStoreService.insertRows(dataStoreId, project1.id, rows);
|
||||
|
||||
// ACT
|
||||
const result = await dataStoreService.getManyRowsAndCount(dataStoreId, project1.id, {
|
||||
filter: { type: 'and', filters: [{ columnName: 'name', value: 'Mary', condition: 'eq' }] },
|
||||
});
|
||||
|
||||
// ASSERT
|
||||
expect(result.count).toEqual(1);
|
||||
expect(result.data).toEqual([expect.objectContaining({ name: 'Mary', age: 25 })]);
|
||||
});
|
||||
|
||||
it("retrieves rows with 'not equals' filter correctly", async () => {
|
||||
// ARRANGE
|
||||
const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, {
|
||||
name: 'dataStore',
|
||||
columns: [
|
||||
{ name: 'name', type: 'string' },
|
||||
{ name: 'age', type: 'number' },
|
||||
],
|
||||
});
|
||||
|
||||
const rows = [
|
||||
{ name: 'John', age: 30 },
|
||||
{ name: 'Mary', age: 25 },
|
||||
{ name: 'Jack', age: 35 },
|
||||
];
|
||||
|
||||
await dataStoreService.insertRows(dataStoreId, project1.id, rows);
|
||||
|
||||
// ACT
|
||||
const result = await dataStoreService.getManyRowsAndCount(dataStoreId, project1.id, {
|
||||
filter: { type: 'and', filters: [{ columnName: 'name', value: 'Mary', condition: 'neq' }] },
|
||||
});
|
||||
|
||||
// ASSERT
|
||||
expect(result.count).toEqual(2);
|
||||
expect(result.data).toEqual([
|
||||
expect.objectContaining({ name: 'John', age: 30 }),
|
||||
expect.objectContaining({ name: 'Jack', age: 35 }),
|
||||
]);
|
||||
});
|
||||
|
||||
it("retrieves rows with 'contains sensitive' filter correctly", async () => {
|
||||
// ARRANGE
|
||||
const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, {
|
||||
name: 'dataStore',
|
||||
columns: [
|
||||
{ name: 'name', type: 'string' },
|
||||
{ name: 'age', type: 'number' },
|
||||
],
|
||||
});
|
||||
|
||||
const rows = [
|
||||
{ name: 'Arnold', age: 30 },
|
||||
{ name: 'Mary', age: 25 },
|
||||
{ name: 'Charlie', age: 35 },
|
||||
];
|
||||
|
||||
await dataStoreService.insertRows(dataStoreId, project1.id, rows);
|
||||
|
||||
// ACT
|
||||
const result = await dataStoreService.getManyRowsAndCount(dataStoreId, project1.id, {
|
||||
filter: {
|
||||
type: 'and',
|
||||
filters: [{ columnName: 'name', value: '%ar%', condition: 'like' }],
|
||||
},
|
||||
});
|
||||
|
||||
// ASSERT
|
||||
expect(result.count).toEqual(2);
|
||||
expect(result.data).toEqual([
|
||||
expect.objectContaining({ name: 'Mary', age: 25 }),
|
||||
expect.objectContaining({ name: 'Charlie', age: 35 }),
|
||||
]);
|
||||
});
|
||||
|
||||
it("retrieves rows with 'contains insensitive' filter correctly", async () => {
|
||||
// ARRANGE
|
||||
const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, {
|
||||
name: 'dataStore',
|
||||
columns: [
|
||||
{ name: 'name', type: 'string' },
|
||||
{ name: 'age', type: 'number' },
|
||||
],
|
||||
});
|
||||
|
||||
const rows = [
|
||||
{ name: 'John', age: 30 },
|
||||
{ name: 'Mary', age: 20 },
|
||||
{ name: 'Benjamin', age: 25 },
|
||||
{ name: 'Taj', age: 35 },
|
||||
];
|
||||
|
||||
await dataStoreService.insertRows(dataStoreId, project1.id, rows);
|
||||
|
||||
// ACT
|
||||
const result = await dataStoreService.getManyRowsAndCount(dataStoreId, project1.id, {
|
||||
filter: {
|
||||
type: 'and',
|
||||
filters: [{ columnName: 'name', value: '%J%', condition: 'ilike' }],
|
||||
},
|
||||
});
|
||||
|
||||
// ASSERT
|
||||
expect(result.count).toEqual(3);
|
||||
expect(result.data).toEqual([
|
||||
expect.objectContaining({ name: 'John', age: 30 }),
|
||||
expect.objectContaining({ name: 'Benjamin', age: 25 }),
|
||||
expect.objectContaining({ name: 'Taj', age: 35 }),
|
||||
]);
|
||||
});
|
||||
|
||||
it("retrieves rows with 'starts with' filter correctly", async () => {
|
||||
// ARRANGE
|
||||
const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, {
|
||||
name: 'dataStore',
|
||||
columns: [
|
||||
{ name: 'name', type: 'string' },
|
||||
{ name: 'age', type: 'number' },
|
||||
],
|
||||
});
|
||||
|
||||
const rows = [
|
||||
{ name: 'Arnold', age: 30 },
|
||||
{ name: 'Mary', age: 25 },
|
||||
{ name: 'Charlie', age: 35 },
|
||||
];
|
||||
|
||||
await dataStoreService.insertRows(dataStoreId, project1.id, rows);
|
||||
|
||||
// ACT
|
||||
const result = await dataStoreService.getManyRowsAndCount(dataStoreId, project1.id, {
|
||||
filter: {
|
||||
type: 'and',
|
||||
filters: [{ columnName: 'name', value: 'Ar%', condition: 'ilike' }],
|
||||
},
|
||||
});
|
||||
|
||||
// ASSERT
|
||||
expect(result.count).toEqual(1);
|
||||
expect(result.data).toEqual([expect.objectContaining({ name: 'Arnold', age: 30 })]);
|
||||
});
|
||||
|
||||
it("retrieves rows with 'ends with' filter correctly", async () => {
|
||||
// ARRANGE
|
||||
const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, {
|
||||
name: 'dataStore',
|
||||
columns: [
|
||||
{ name: 'name', type: 'string' },
|
||||
{ name: 'age', type: 'number' },
|
||||
],
|
||||
});
|
||||
|
||||
const rows = [
|
||||
{ name: 'Arnold', age: 30 },
|
||||
{ name: 'Mary', age: 25 },
|
||||
{ name: 'Charlie', age: 35 },
|
||||
{ name: 'Harold', age: 40 },
|
||||
];
|
||||
|
||||
await dataStoreService.insertRows(dataStoreId, project1.id, rows);
|
||||
|
||||
// ACT
|
||||
const result = await dataStoreService.getManyRowsAndCount(dataStoreId, project1.id, {
|
||||
filter: {
|
||||
type: 'and',
|
||||
filters: [{ columnName: 'name', value: '%old', condition: 'ilike' }],
|
||||
},
|
||||
});
|
||||
|
||||
// ASSERT
|
||||
expect(result.count).toEqual(2);
|
||||
expect(result.data).toEqual([
|
||||
expect.objectContaining({ name: 'Arnold', age: 30 }),
|
||||
expect.objectContaining({ name: 'Harold', age: 40 }),
|
||||
]);
|
||||
});
|
||||
|
||||
describe.each(['like', 'ilike'] as ListDataStoreContentFilterConditionType[])(
|
||||
'%s filter validation',
|
||||
(condition) => {
|
||||
it(`throws error when '${condition}' filter value is null`, async () => {
|
||||
// ARRANGE
|
||||
const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, {
|
||||
name: 'dataStore',
|
||||
columns: [
|
||||
{ name: 'name', type: 'string' },
|
||||
{ name: 'age', type: 'number' },
|
||||
],
|
||||
});
|
||||
|
||||
const rows = [
|
||||
{ name: 'John', age: 30 },
|
||||
{ name: 'Mary', age: 25 },
|
||||
];
|
||||
|
||||
await dataStoreService.insertRows(dataStoreId, project1.id, rows);
|
||||
|
||||
// ACT
|
||||
const result = dataStoreService.getManyRowsAndCount(dataStoreId, project1.id, {
|
||||
filter: {
|
||||
type: 'and',
|
||||
filters: [{ columnName: 'name', value: null, condition }],
|
||||
},
|
||||
});
|
||||
|
||||
// ASSERT
|
||||
await expect(result).rejects.toThrow(
|
||||
new DataStoreValidationError(
|
||||
`${condition.toUpperCase()} filter value cannot be null or undefined`,
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
it(`throws error when '${condition}' filter value is not a string`, async () => {
|
||||
// ARRANGE
|
||||
const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, {
|
||||
name: 'dataStore',
|
||||
columns: [
|
||||
{ name: 'name', type: 'string' },
|
||||
{ name: 'age', type: 'number' },
|
||||
],
|
||||
});
|
||||
|
||||
const rows = [
|
||||
{ name: 'John', age: 30 },
|
||||
{ name: 'Mary', age: 25 },
|
||||
];
|
||||
|
||||
await dataStoreService.insertRows(dataStoreId, project1.id, rows);
|
||||
|
||||
// ACT
|
||||
const result = dataStoreService.getManyRowsAndCount(dataStoreId, project1.id, {
|
||||
filter: {
|
||||
type: 'and',
|
||||
filters: [{ columnName: 'age', value: 123, condition }],
|
||||
},
|
||||
});
|
||||
|
||||
// ASSERT
|
||||
await expect(result).rejects.toThrow(
|
||||
new DataStoreValidationError(
|
||||
`${condition.toUpperCase()} filter value must be a string`,
|
||||
),
|
||||
);
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
describe('like filter with special characters', () => {
|
||||
let dataStoreId: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
const { id } = await dataStoreService.createDataStore(project1.id, {
|
||||
name: 'dataStore',
|
||||
columns: [{ name: 'text', type: 'string' }],
|
||||
});
|
||||
dataStoreId = id;
|
||||
});
|
||||
|
||||
it('should treat square brackets literally in like patterns', async () => {
|
||||
// ARRANGE
|
||||
await dataStoreService.insertRows(dataStoreId, project1.id, [
|
||||
{ text: 'test[abc]data' },
|
||||
{ text: 'Test[abc]Data' },
|
||||
{ text: 'testAdata' },
|
||||
{ text: 'testBdata' },
|
||||
]);
|
||||
|
||||
// ACT
|
||||
const result = await dataStoreService.getManyRowsAndCount(dataStoreId, project1.id, {
|
||||
filter: {
|
||||
type: 'and',
|
||||
filters: [{ columnName: 'text', value: 'test%[abc]%', condition: 'like' }],
|
||||
},
|
||||
});
|
||||
|
||||
// ASSERT
|
||||
expect(result.count).toEqual(1);
|
||||
expect(result.data).toEqual([expect.objectContaining({ text: 'test[abc]data' })]);
|
||||
});
|
||||
|
||||
it('should treat asterisk literally in like patterns', async () => {
|
||||
// ARRANGE
|
||||
await dataStoreService.insertRows(dataStoreId, project1.id, [
|
||||
{ text: 'test*data' },
|
||||
{ text: 'Test*Data' },
|
||||
{ text: 'testAdata' },
|
||||
{ text: 'testABCdata' },
|
||||
]);
|
||||
|
||||
// ACT
|
||||
const result = await dataStoreService.getManyRowsAndCount(dataStoreId, project1.id, {
|
||||
filter: {
|
||||
type: 'and',
|
||||
filters: [{ columnName: 'text', value: 'test%*%', condition: 'like' }],
|
||||
},
|
||||
});
|
||||
|
||||
// ASSERT
|
||||
expect(result.count).toEqual(1);
|
||||
expect(result.data).toEqual([expect.objectContaining({ text: 'test*data' })]);
|
||||
});
|
||||
|
||||
it('should treat question mark literally in like patterns', async () => {
|
||||
// ARRANGE
|
||||
await dataStoreService.insertRows(dataStoreId, project1.id, [
|
||||
{ text: 'test?data' },
|
||||
{ text: 'Test?Data' },
|
||||
{ text: 'testAdata' },
|
||||
{ text: 'testXdata' },
|
||||
]);
|
||||
|
||||
// ACT
|
||||
const result = await dataStoreService.getManyRowsAndCount(dataStoreId, project1.id, {
|
||||
filter: {
|
||||
type: 'and',
|
||||
filters: [{ columnName: 'text', value: 'test%?%', condition: 'like' }],
|
||||
},
|
||||
});
|
||||
|
||||
// ASSERT
|
||||
expect(result.count).toEqual(1);
|
||||
expect(result.data).toEqual([expect.objectContaining({ text: 'test?data' })]);
|
||||
});
|
||||
|
||||
it('should convert LIKE % wildcard to match zero or more characters', async () => {
|
||||
// ARRANGE
|
||||
await dataStoreService.insertRows(dataStoreId, project1.id, [
|
||||
{ text: 'data%more' },
|
||||
{ text: 'Data%More' },
|
||||
{ text: 'datamore' },
|
||||
{ text: 'dataABCmore' },
|
||||
{ text: 'different' },
|
||||
]);
|
||||
|
||||
// ACT
|
||||
const result = await dataStoreService.getManyRowsAndCount(dataStoreId, project1.id, {
|
||||
filter: {
|
||||
type: 'and',
|
||||
filters: [{ columnName: 'text', value: 'data%more', condition: 'like' }],
|
||||
},
|
||||
});
|
||||
|
||||
// ASSERT
|
||||
expect(result.count).toEqual(3);
|
||||
expect(result.data).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({ text: 'data%more' }),
|
||||
expect.objectContaining({ text: 'datamore' }),
|
||||
expect.objectContaining({ text: 'dataABCmore' }),
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
it('should treat underscore literally in like patterns', async () => {
|
||||
// ARRANGE
|
||||
await dataStoreService.insertRows(dataStoreId, project1.id, [
|
||||
{ text: 'prefix_suffix' },
|
||||
{ text: 'Prefix_Suffix' },
|
||||
{ text: 'prefix\\_suffix' },
|
||||
{ text: 'prefixAsuffix' },
|
||||
{ text: 'prefixsuffix' },
|
||||
]);
|
||||
|
||||
// ACT
|
||||
const result = await dataStoreService.getManyRowsAndCount(dataStoreId, project1.id, {
|
||||
filter: {
|
||||
type: 'and',
|
||||
filters: [{ columnName: 'text', value: 'prefix_suffix', condition: 'like' }],
|
||||
},
|
||||
});
|
||||
|
||||
// ASSERT
|
||||
expect(result.count).toEqual(1);
|
||||
expect(result.data).toEqual([expect.objectContaining({ text: 'prefix_suffix' })]);
|
||||
});
|
||||
|
||||
it('should handle multiple special characters', async () => {
|
||||
// ARRANGE
|
||||
await dataStoreService.insertRows(dataStoreId, project1.id, [
|
||||
{ text: 'test[*?]data' },
|
||||
{ text: 'Test[*?]Data' },
|
||||
{ text: 'testOtherData' },
|
||||
{ text: 'test123data' },
|
||||
]);
|
||||
|
||||
// ACT
|
||||
const result = await dataStoreService.getManyRowsAndCount(dataStoreId, project1.id, {
|
||||
filter: {
|
||||
type: 'and',
|
||||
filters: [{ columnName: 'text', value: 'test%[*?]%', condition: 'like' }],
|
||||
},
|
||||
});
|
||||
|
||||
// ASSERT
|
||||
expect(result.count).toEqual(1);
|
||||
expect(result.data).toEqual([expect.objectContaining({ text: 'test[*?]data' })]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('ilike filter with special characters (case-insensitive)', () => {
|
||||
let dataStoreId: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
const { id } = await dataStoreService.createDataStore(project1.id, {
|
||||
name: 'dataStore',
|
||||
columns: [{ name: 'text', type: 'string' }],
|
||||
});
|
||||
dataStoreId = id;
|
||||
});
|
||||
|
||||
it('should treat square brackets literally', async () => {
|
||||
// ARRANGE
|
||||
await dataStoreService.insertRows(dataStoreId, project1.id, [
|
||||
{ text: 'test[abc]data' },
|
||||
{ text: 'Test[ABC]Data' },
|
||||
{ text: 'testAdata' },
|
||||
{ text: 'testBdata' },
|
||||
]);
|
||||
|
||||
// ACT
|
||||
const result = await dataStoreService.getManyRowsAndCount(dataStoreId, project1.id, {
|
||||
filter: {
|
||||
type: 'and',
|
||||
filters: [{ columnName: 'text', value: '%[abc]%', condition: 'ilike' }],
|
||||
},
|
||||
});
|
||||
|
||||
// ASSERT
|
||||
expect(result.count).toEqual(2);
|
||||
expect(result.data).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({ text: 'test[abc]data' }),
|
||||
expect.objectContaining({ text: 'Test[ABC]Data' }),
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
it('should treat asterisk literally', async () => {
|
||||
// ARRANGE
|
||||
await dataStoreService.insertRows(dataStoreId, project1.id, [
|
||||
{ text: 'test*data' },
|
||||
{ text: 'Test*Data' },
|
||||
{ text: 'testOtherData' },
|
||||
{ text: 'testABCdata' },
|
||||
]);
|
||||
|
||||
// ACT
|
||||
const result = await dataStoreService.getManyRowsAndCount(dataStoreId, project1.id, {
|
||||
filter: {
|
||||
type: 'and',
|
||||
filters: [{ columnName: 'text', value: '%*%', condition: 'ilike' }],
|
||||
},
|
||||
});
|
||||
|
||||
// ASSERT
|
||||
expect(result.count).toEqual(2);
|
||||
expect(result.data).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({ text: 'test*data' }),
|
||||
expect.objectContaining({ text: 'Test*Data' }),
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
it('should treat question mark literally', async () => {
|
||||
// ARRANGE
|
||||
await dataStoreService.insertRows(dataStoreId, project1.id, [
|
||||
{ text: 'test?data' },
|
||||
{ text: 'Test?Data' },
|
||||
{ text: 'testSingleChar' },
|
||||
{ text: 'testMultiChar' },
|
||||
]);
|
||||
|
||||
// ACT
|
||||
const result = await dataStoreService.getManyRowsAndCount(dataStoreId, project1.id, {
|
||||
filter: {
|
||||
type: 'and',
|
||||
filters: [{ columnName: 'text', value: '%?%', condition: 'ilike' }],
|
||||
},
|
||||
});
|
||||
|
||||
// ASSERT
|
||||
expect(result.count).toEqual(2);
|
||||
expect(result.data).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({ text: 'test?data' }),
|
||||
expect.objectContaining({ text: 'Test?Data' }),
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
it('should convert % wildcard to match zero or more characters', async () => {
|
||||
// ARRANGE
|
||||
await dataStoreService.insertRows(dataStoreId, project1.id, [
|
||||
{ text: 'data%more' },
|
||||
{ text: 'Data%More' },
|
||||
{ text: 'datamore' },
|
||||
{ text: 'DataMore' },
|
||||
{ text: 'dataABCmore' },
|
||||
{ text: 'DataABCMore' },
|
||||
{ text: 'different' },
|
||||
]);
|
||||
|
||||
// ACT
|
||||
const result = await dataStoreService.getManyRowsAndCount(dataStoreId, project1.id, {
|
||||
filter: {
|
||||
type: 'and',
|
||||
filters: [{ columnName: 'text', value: 'data%more', condition: 'ilike' }],
|
||||
},
|
||||
});
|
||||
|
||||
// ASSERT
|
||||
expect(result.count).toEqual(6);
|
||||
expect(result.data).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({ text: 'data%more' }),
|
||||
expect.objectContaining({ text: 'Data%More' }),
|
||||
expect.objectContaining({ text: 'datamore' }),
|
||||
expect.objectContaining({ text: 'DataMore' }),
|
||||
expect.objectContaining({ text: 'dataABCmore' }),
|
||||
expect.objectContaining({ text: 'DataABCMore' }),
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
it('should treat underscore literally', async () => {
|
||||
// ARRANGE
|
||||
await dataStoreService.insertRows(dataStoreId, project1.id, [
|
||||
{ text: 'prefix_suffix' },
|
||||
{ text: 'Prefix_Suffix' },
|
||||
{ text: 'Prefix\\_Suffix' },
|
||||
{ text: 'prefixASuffix' },
|
||||
{ text: 'prefixsuffix' },
|
||||
]);
|
||||
|
||||
// ACT
|
||||
const result = await dataStoreService.getManyRowsAndCount(dataStoreId, project1.id, {
|
||||
filter: {
|
||||
type: 'and',
|
||||
filters: [{ columnName: 'text', value: 'prefix_suffix', condition: 'ilike' }],
|
||||
},
|
||||
});
|
||||
|
||||
// ASSERT
|
||||
expect(result.count).toEqual(2);
|
||||
expect(result.data).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({ text: 'prefix_suffix' }),
|
||||
expect.objectContaining({ text: 'Prefix_Suffix' }),
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
it('should handle multiple special characters', async () => {
|
||||
// ARRANGE
|
||||
await dataStoreService.insertRows(dataStoreId, project1.id, [
|
||||
{ text: 'test[*?]data' },
|
||||
{ text: 'Test[*?]Data' },
|
||||
{ text: 'testOtherData' },
|
||||
{ text: 'test123data' },
|
||||
]);
|
||||
|
||||
// ACT
|
||||
const result = await dataStoreService.getManyRowsAndCount(dataStoreId, project1.id, {
|
||||
filter: {
|
||||
type: 'and',
|
||||
filters: [{ columnName: 'text', value: '%[*?]%', condition: 'ilike' }],
|
||||
},
|
||||
});
|
||||
|
||||
// ASSERT
|
||||
expect(result.count).toEqual(2);
|
||||
expect(result.data).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({ text: 'test[*?]data' }),
|
||||
expect.objectContaining({ text: 'Test[*?]Data' }),
|
||||
]),
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -15,6 +15,7 @@ import { DataStoreUserTableName } from './data-store.types';
|
||||
import {
|
||||
addColumnQuery,
|
||||
deleteColumnQuery,
|
||||
escapeLikeSpecials,
|
||||
extractInsertedIds,
|
||||
extractReturningData,
|
||||
normalizeRows,
|
||||
@@ -22,11 +23,24 @@ import {
|
||||
quoteIdentifier,
|
||||
splitRowsByExistence,
|
||||
toDslColumns,
|
||||
toSqliteGlobFromPercent,
|
||||
} from './utils/sql-utils';
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
type QueryBuilder = SelectQueryBuilder<any>;
|
||||
|
||||
/**
|
||||
* Converts filter conditions to SQL WHERE clauses with parameters.
|
||||
*
|
||||
* LIKE / ILIKE rules:
|
||||
* - Only '%' is a wildcard (zero or more chars).
|
||||
* - All other special chars ('_', '*', '?', '[', ']') are treated literally.
|
||||
* - '_' and '\' are escaped in the value; SQL uses `ESCAPE '\'` so `\_` means literal underscore.
|
||||
*
|
||||
* Why the crazy backslashes:
|
||||
* - Postgres/SQLite/Oracle/SQL Server: `ESCAPE '\'` is written as-is.
|
||||
* - MySQL/MariaDB: the SQL literal itself requires two backslashes (`'\\'`) to mean one.
|
||||
*/
|
||||
function getConditionAndParams(
|
||||
filter: ListDataStoreContentFilter['filters'][number],
|
||||
index: number,
|
||||
@@ -38,8 +52,55 @@ function getConditionAndParams(
|
||||
switch (filter.condition) {
|
||||
case 'eq':
|
||||
return [`${column} = :${paramName}`, { [paramName]: filter.value }];
|
||||
|
||||
case 'neq':
|
||||
return [`${column} != :${paramName}`, { [paramName]: filter.value }];
|
||||
|
||||
// case-sensitive
|
||||
case 'like':
|
||||
if (['sqlite', 'sqlite-pooled'].includes(dbType)) {
|
||||
const globValue = toSqliteGlobFromPercent(filter.value as string);
|
||||
return [`${column} GLOB :${paramName}`, { [paramName]: globValue }];
|
||||
}
|
||||
|
||||
if (['mysql', 'mariadb'].includes(dbType)) {
|
||||
const escapedValue = escapeLikeSpecials(filter.value as string);
|
||||
return [`${column} LIKE BINARY :${paramName} ESCAPE '\\\\'`, { [paramName]: escapedValue }];
|
||||
}
|
||||
|
||||
// PostgreSQL: LIKE is case-sensitive
|
||||
if (dbType === 'postgres') {
|
||||
const escapedValue = escapeLikeSpecials(filter.value as string);
|
||||
return [`${column} LIKE :${paramName} ESCAPE '\\'`, { [paramName]: escapedValue }];
|
||||
}
|
||||
|
||||
// Generic fallback
|
||||
return [`${column} LIKE :${paramName}`, { [paramName]: filter.value }];
|
||||
|
||||
// case-insensitive
|
||||
case 'ilike':
|
||||
if (['sqlite', 'sqlite-pooled'].includes(dbType)) {
|
||||
const escapedValue = escapeLikeSpecials(filter.value as string);
|
||||
return [
|
||||
`UPPER(${column}) LIKE UPPER(:${paramName}) ESCAPE '\\'`,
|
||||
{ [paramName]: escapedValue },
|
||||
];
|
||||
}
|
||||
|
||||
if (['mysql', 'mariadb'].includes(dbType)) {
|
||||
const escapedValue = escapeLikeSpecials(filter.value as string);
|
||||
return [
|
||||
`UPPER(${column}) LIKE UPPER(:${paramName}) ESCAPE '\\\\'`,
|
||||
{ [paramName]: escapedValue },
|
||||
];
|
||||
}
|
||||
|
||||
if (dbType === 'postgres') {
|
||||
const escapedValue = escapeLikeSpecials(filter.value as string);
|
||||
return [`${column} ILIKE :${paramName} ESCAPE '\\'`, { [paramName]: escapedValue }];
|
||||
}
|
||||
|
||||
return [`UPPER(${column}) LIKE UPPER(:${paramName})`, { [paramName]: filter.value }];
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -107,6 +107,7 @@ export class DataStoreService {
|
||||
dto: ListDataStoreContentQueryDto,
|
||||
) {
|
||||
await this.validateDataStoreExists(dataStoreId, projectId);
|
||||
this.validateFilters(dto);
|
||||
|
||||
// unclear if we should validate here, only use case would be to reduce the chance of
|
||||
// a renamed/removed column appearing here (or added column missing) if the store was
|
||||
@@ -321,4 +322,25 @@ export class DataStoreService {
|
||||
throw new DataStoreNameConflictError(name);
|
||||
}
|
||||
}
|
||||
|
||||
private validateFilters(dto: ListDataStoreContentQueryDto): void {
|
||||
if (!dto.filter?.filters) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (const filter of dto.filter.filters) {
|
||||
if (['like', 'ilike'].includes(filter.condition)) {
|
||||
if (filter.value === null || filter.value === undefined) {
|
||||
throw new DataStoreValidationError(
|
||||
`${filter.condition.toUpperCase()} filter value cannot be null or undefined`,
|
||||
);
|
||||
}
|
||||
if (typeof filter.value !== 'string') {
|
||||
throw new DataStoreValidationError(
|
||||
`${filter.condition.toUpperCase()} filter value must be a string`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -261,3 +261,29 @@ export function normalizeValue(
|
||||
|
||||
return value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a LIKE-style pattern (only % is wildcard) into a SQLite GLOB pattern.
|
||||
*/
|
||||
export function toSqliteGlobFromPercent(input: string): string {
|
||||
const out: string[] = [];
|
||||
for (const ch of String(input ?? '')) {
|
||||
if (ch === '%') out.push('*');
|
||||
else if (ch === '[') out.push('[[]');
|
||||
else if (ch === ']') out.push('[]]');
|
||||
else if (ch === '*') out.push('[*]');
|
||||
else if (ch === '?') out.push('[?]');
|
||||
else out.push(ch);
|
||||
}
|
||||
return out.join('');
|
||||
}
|
||||
|
||||
/**
|
||||
* LIKE escaper for DBs where we use ESCAPE '\'.
|
||||
* Keep '%' as wildcard; make '_' literal; escape the escape char itself.
|
||||
*/
|
||||
export function escapeLikeSpecials(input: string): string {
|
||||
return input
|
||||
.replace(/\\/g, '\\\\') // escape the escape char itself
|
||||
.replace(/_/g, '\\_'); // make '_' literal ('%' stays a wildcard)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user