admin/n8n-enterprise-unlocked
1
0
Fork 0
mirror of https://github.com/Abdulazizzn/n8n-enterprise-unlocked.git synced 2025-12-17 18:12:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(core): Restrict data store access to authorized projects (no-changelog) (#18342)

Browse Source
This commit is contained in:
Jaakko Husso
2025-08-14 15:56:44 +03:00
committed by GitHub
parent 8442382471
commit 7fefd54edf
9 changed files with 746 additions and 257 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
packages/@n8n/permissions/src/roles/scopes/global-scopes.ee.ts
View File

@@ -79,13 +79,6 @@ export const GLOBAL_OWNER_SCOPES: Scope[] = [
'insights:list',
'folder:move',
'oidc:manage',
'dataStore:create',
'dataStore:delete',
'dataStore:read',
'dataStore:update',
'dataStore:list',
'dataStore:readRow',
'dataStore:writeRow',
];
export const GLOBAL_ADMIN_SCOPES = GLOBAL_OWNER_SCOPES.concat();
@@ -105,7 +98,4 @@ export const GLOBAL_MEMBER_SCOPES: Scope[] = [
'user:list',
'variable:list',
'variable:read',
'dataStore:read',
'dataStore:list',
'dataStore:readRow',
];

485
packages/cli/src/modules/data-store/__tests__/data-store.controller.test.ts Normal file
View File

@@ -0,0 +1,485 @@
import type { DataStore } from '@n8n/api-types';
import {
createTeamProject,
getPersonalProject,
linkUserToProject,
testDb,
} from '@n8n/backend-test-utils';
import type { Project, User } from '@n8n/db';
import { ProjectRepository } from '@n8n/db';
import { Container } from '@n8n/di';
import { createDataStore } from '@test-integration/db/data-stores';
import { createOwner, createMember, createAdmin } from '@test-integration/db/users';
import type { SuperAgentTest } from '@test-integration/types';
import * as utils from '@test-integration/utils';
import { DateTime } from 'luxon';
import { DataStoreRepository } from '../data-store.repository';
let owner: User;
let member: User;
let admin: User;
let authOwnerAgent: SuperAgentTest;
let authMemberAgent: SuperAgentTest;
let authAdminAgent: SuperAgentTest;
let ownerProject: Project;
let memberProject: Project;
const testServer = utils.setupTestServer({
endpointGroups: ['data-store'],
modules: ['data-store'],
});
let projectRepository: ProjectRepository;
let dataStoreRepository: DataStoreRepository;
beforeAll(async () => {
await testDb.init();
});
beforeEach(async () => {
await testDb.truncate(['DataStore', 'DataStoreColumn', 'Project', 'ProjectRelation']);
projectRepository = Container.get(ProjectRepository);
dataStoreRepository = Container.get(DataStoreRepository);
owner = await createOwner();
member = await createMember();
admin = await createAdmin();
authOwnerAgent = testServer.authAgentFor(owner);
authMemberAgent = testServer.authAgentFor(member);
authAdminAgent = testServer.authAgentFor(admin);
ownerProject = await getPersonalProject(owner);
memberProject = await getPersonalProject(member);
});
afterAll(async () => {
await testDb.terminate();
});
describe('POST /projects/:projectId/data-stores', () => {
test('should not create data store when project does not exist', async () => {
const payload = {
name: 'Test Data Store',
columns: [
{
name: 'test-ccolumn',
type: 'string',
},
],
};
await authMemberAgent.post('/projects/non-existing-id/data-stores').send(payload).expect(403);
await authAdminAgent.post('/projects/non-existing-id/data-stores').send(payload).expect(403);
await authOwnerAgent.post('/projects/non-existing-id/data-stores').send(payload).expect(403);
});
test('should not create data store when name is empty', async () => {
const project = await createTeamProject(undefined, owner);
const payload = {
name: '',
columns: [
{
name: 'test-ccolumn',
type: 'string',
},
],
};
await authOwnerAgent.post(`/projects/${project.id}/data-stores`).send(payload).expect(400);
});
test('should not create data store if user has project:viewer role in team project', async () => {
const project = await createTeamProject(undefined, owner);
await linkUserToProject(member, project, 'project:viewer');
const payload = {
name: 'Test Data Store',
columns: [
{
name: 'test-ccolumn',
type: 'string',
},
],
};
await authMemberAgent.post(`/projects/${project.id}/data-stores`).send(payload).expect(403);
const dataStoresInDb = await dataStoreRepository.find();
expect(dataStoresInDb).toHaveLength(0);
});
test("should not allow creating data store in another user's personal project", async () => {
const ownerPersonalProject = await projectRepository.getPersonalProjectForUserOrFail(owner.id);
const payload = {
name: 'Test Data Store',
columns: [
{
name: 'test-ccolumn',
type: 'string',
},
],
};
await authMemberAgent
.post(`/projects/${ownerPersonalProject.id}/data-stores`)
.send(payload)
.expect(403);
});
test('should create data store if user has project:editor role in team project', async () => {
const project = await createTeamProject(undefined, owner);
await linkUserToProject(member, project, 'project:editor');
const payload = {
name: 'Test Data Store',
columns: [
{
name: 'test-ccolumn',
type: 'string',
},
],
};
await authMemberAgent.post(`/projects/${project.id}/data-stores`).send(payload).expect(200);
const dataStoresInDb = await dataStoreRepository.find();
expect(dataStoresInDb).toHaveLength(1);
});
test('should create data store if user has project:admin role in team project', async () => {
const project = await createTeamProject(undefined, owner);
const payload = {
name: 'Test Data Store',
columns: [
{
name: 'test-ccolumn',
type: 'string',
},
],
};
await authOwnerAgent.post(`/projects/${project.id}/data-stores`).send(payload).expect(200);
const dataStoresInDb = await dataStoreRepository.find();
expect(dataStoresInDb).toHaveLength(1);
});
test('should create data store in personal project', async () => {
const personalProject = await projectRepository.getPersonalProjectForUserOrFail(owner.id);
const payload = {
name: 'Test Data Store',
columns: [
{
name: 'test-ccolumn',
type: 'string',
},
],
};
const response = await authOwnerAgent
.post(`/projects/${personalProject.id}/data-stores`)
.send(payload)
.expect(200);
expect(response.body.data).toEqual(
expect.objectContaining({
id: expect.any(String),
name: payload.name,
projectId: personalProject.id,
createdAt: expect.any(String),
updatedAt: expect.any(String),
}),
);
const dataStoreInDb = await dataStoreRepository.findOneBy({ id: response.body.data.id });
expect(dataStoreInDb).toBeDefined();
expect(dataStoreInDb?.name).toBe(payload.name);
});
});
describe('GET /projects/:projectId/data-stores', () => {
test('should not list data stores when project does not exist', async () => {
await authMemberAgent.get('/projects/non-existing-id/data-stores').expect(403);
await authAdminAgent.get('/projects/non-existing-id/data-stores').expect(403);
await authOwnerAgent.get('/projects/non-existing-id/data-stores').expect(403);
});
test('should not list data stores if user has no access to project', async () => {
const project = await createTeamProject('test project', owner);
await authMemberAgent.get(`/projects/${project.id}/data-stores`).expect(403);
});
test('should not list data stores if admin has no access to project', async () => {
const project = await createTeamProject('test project', owner);
await authAdminAgent.get(`/projects/${project.id}/data-stores`).expect(403);
});
test("should not allow listing data stores from another user's personal project", async () => {
await authMemberAgent.get(`/projects/${ownerProject.id}/data-stores`).expect(403);
});
test('should list data stores if user has project:viewer role in team project', async () => {
const project = await createTeamProject('test project', owner);
await linkUserToProject(member, project, 'project:viewer');
await createDataStore(project, { name: 'Test Data Store' });
const response = await authMemberAgent.get(`/projects/${project.id}/data-stores`).expect(200);
expect(response.body.data.count).toBe(1);
expect(response.body.data.data).toHaveLength(1);
expect(response.body.data.data[0].name).toBe('Test Data Store');
});
test('should list data stores from personal project', async () => {
await createDataStore(ownerProject, { name: 'Personal Data Store 1' });
await createDataStore(ownerProject, { name: 'Personal Data Store 2' });
const response = await authOwnerAgent
.get(`/projects/${ownerProject.id}/data-stores`)
.expect(200);
expect(response.body.data.count).toBe(2);
expect(response.body.data.data).toHaveLength(2);
expect(response.body.data.data.map((f: any) => f.name).sort()).toEqual(
['Personal Data Store 1', 'Personal Data Store 2'].sort(),
);
});
test('should filter data stores by projectId', async () => {
await createDataStore(ownerProject, { name: 'Test Data Store 1' });
await createDataStore(ownerProject, { name: 'Test Data Store 2' });
await createDataStore(memberProject, { name: 'Another Data Store' });
const response = await authOwnerAgent
.get(`/projects/${ownerProject.id}/data-stores`)
.query({ filter: JSON.stringify({ name: 'test' }) })
.expect(200);
expect(response.body.data.count).toBe(2);
expect(response.body.data.data).toHaveLength(2);
expect(response.body.data.data.map((f: any) => f.name).sort()).toEqual(
['Test Data Store 1', 'Test Data Store 2'].sort(),
);
});
test('should filter data stores by name', async () => {
await createDataStore(ownerProject, { name: 'Test Data Store' });
await createDataStore(ownerProject, { name: 'Another Data Store' });
await createDataStore(ownerProject, { name: 'Test Something Else' });
const response = await authOwnerAgent
.get(`/projects/${ownerProject.id}/data-stores`)
.query({ filter: JSON.stringify({ name: 'test' }) })
.expect(200);
expect(response.body.data.count).toBe(2);
expect(response.body.data.data).toHaveLength(2);
expect(response.body.data.data.map((f: any) => f.name).sort()).toEqual(
['Test Data Store', 'Test Something Else'].sort(),
);
});
test('should filter data stores by id', async () => {
const dataStore1 = await createDataStore(ownerProject, { name: 'Data Store 1' });
await createDataStore(ownerProject, { name: 'Data Store 2' });
await createDataStore(ownerProject, { name: 'Data Store 3' });
const response = await authOwnerAgent
.get(`/projects/${ownerProject.id}/data-stores`)
.query({ filter: JSON.stringify({ id: dataStore1.id }) })
.expect(200);
expect(response.body.data.count).toBe(1);
expect(response.body.data.data).toHaveLength(1);
expect(response.body.data.data[0].name).toBe('Data Store 1');
});
test('should filter data stores by multiple names (AND operator)', async () => {
await createDataStore(ownerProject, { name: 'Data Store' });
await createDataStore(ownerProject, { name: 'Test Store' });
await createDataStore(ownerProject, { name: 'Another Store' });
const response = await authOwnerAgent
.get(`/projects/${ownerProject.id}/data-stores?filter={ "name": ["Store", "Test"]}`)
.expect(200);
expect(response.body.data.count).toBe(1);
expect(response.body.data.data).toHaveLength(1);
expect(response.body.data.data[0].name).toBe('Test Store');
});
test('should apply pagination with take parameter', async () => {
for (let i = 1; i <= 5; i++) {
await createDataStore(ownerProject, {
name: `Data Store ${i}`,
updatedAt: DateTime.now()
.minus({ minutes: 6 - i })
.toJSDate(),
});
}
const response = await authOwnerAgent
.get(`/projects/${ownerProject.id}/data-stores`)
.query({ take: 3 })
.expect(200);
expect(response.body.data.count).toBe(5); // Total count should be 5
expect(response.body.data.data).toHaveLength(3); // But only 3 returned
expect(response.body.data.data.map((store: DataStore) => store.name)).toEqual([
'Data Store 5',
'Data Store 4',
'Data Store 3',
]);
});
test('should apply pagination with skip parameter', async () => {
for (let i = 1; i <= 5; i++) {
await createDataStore(ownerProject, {
name: `Data Store ${i}`,
updatedAt: DateTime.now()
.minus({ minutes: 6 - i })
.toJSDate(),
});
}
const response = await authOwnerAgent
.get(`/projects/${ownerProject.id}/data-stores`)
.query({ skip: 2 })
.expect(200);
expect(response.body.data.count).toBe(5);
expect(response.body.data.data).toHaveLength(3);
expect(response.body.data.data.map((store: DataStore) => store.name)).toEqual([
'Data Store 3',
'Data Store 2',
'Data Store 1',
]);
});
test('should apply combined skip and take parameters', async () => {
for (let i = 1; i <= 5; i++) {
await createDataStore(ownerProject, {
name: `Data Store ${i}`,
updatedAt: DateTime.now()
.minus({ minutes: 6 - i })
.toJSDate(),
});
}
const response = await authOwnerAgent
.get(`/projects/${ownerProject.id}/data-stores`)
.query({ skip: 1, take: 2 })
.expect(200);
expect(response.body.data.count).toBe(5);
expect(response.body.data.data).toHaveLength(2);
expect(response.body.data.data.map((store: DataStore) => store.name)).toEqual([
'Data Store 4',
'Data Store 3',
]);
});
test('should sort data stores by name ascending', async () => {
await createDataStore(ownerProject, { name: 'Z Data Store' });
await createDataStore(ownerProject, { name: 'A Data Store' });
await createDataStore(ownerProject, { name: 'M Data Store' });
const response = await authOwnerAgent
.get(`/projects/${ownerProject.id}/data-stores`)
.query({ sortBy: 'name:asc' })
.expect(200);
expect(response.body.data.data.map((store: DataStore) => store.name)).toEqual([
'A Data Store',
'M Data Store',
'Z Data Store',
]);
});
test('should sort data stores by name descending', async () => {
await createDataStore(ownerProject, { name: 'Z Data Store' });
await createDataStore(ownerProject, { name: 'A Data Store' });
await createDataStore(ownerProject, { name: 'M Data Store' });
const response = await authOwnerAgent
.get(`/projects/${ownerProject.id}/data-stores`)
.query({ sortBy: 'name:desc' })
.expect(200);
expect(response.body.data.data.map((f: DataStore) => f.name)).toEqual([
'Z Data Store',
'M Data Store',
'A Data Store',
]);
});
test('should sort data stores by updatedAt', async () => {
await createDataStore(ownerProject, {
name: 'Older Data Store',
updatedAt: DateTime.now().minus({ days: 2 }).toJSDate(),
});
await createDataStore(ownerProject, {
name: 'Newest Data Store',
updatedAt: DateTime.now().toJSDate(),
});
await createDataStore(ownerProject, {
name: 'Middle Data Store',
updatedAt: DateTime.now().minus({ days: 1 }).toJSDate(),
});
const response = await authOwnerAgent
.get(`/projects/${ownerProject.id}/data-stores`)
.query({ sortBy: 'updatedAt:desc' })
.expect(200);
expect(response.body.data.data.map((f: DataStore) => f.name)).toEqual([
'Newest Data Store',
'Middle Data Store',
'Older Data Store',
]);
});
test('should combine multiple query parameters correctly', async () => {
const dataStore1 = await createDataStore(ownerProject, { name: 'Test Data Store' });
await createDataStore(ownerProject, { name: 'Another Data Store' });
const response = await authOwnerAgent
.get(`/projects/${ownerProject.id}/data-stores`)
.query({ filter: JSON.stringify({ name: 'data', id: dataStore1.id }), sortBy: 'name:asc' })
.expect(200);
expect(response.body.data.count).toBe(1);
expect(response.body.data.data).toHaveLength(1);
expect(response.body.data.data[0].name).toBe('Test Data Store');
});
test('should include columns', async () => {
await createDataStore(ownerProject, {
name: 'Test Data Store',
columns: [
{
name: 'test-column-1',
type: 'string',
},
{
name: 'test-column-2',
type: 'boolean',
},
],
});
const response = await authOwnerAgent
.get(`/projects/${ownerProject.id}/data-stores`)
.query({ filter: JSON.stringify({ name: 'test' }) })
.expect(200);
expect(response.body.data.count).toBe(1);
expect(response.body.data.data).toHaveLength(1);
expect(response.body.data.data[0].columns).toHaveLength(2);
});
});

222
packages/cli/src/modules/data-store/__tests__/data-store.service.test.ts
View File

@@ -100,24 +100,22 @@ describe('dataStore', () => {
});
// ACT
const result = await dataStoreService.createDataStore(project1.id, {
const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, {
name,
columns: [],
});
const { id: dataStoreId } = result;
const created = await dataStoreRepository.findOneBy({ name, projectId: project1.id });
expect(created?.id).toBe(dataStoreId);
});
it('should create the user table and columns entity immediately if columns are provided', async () => {
const dataStore = await dataStoreService.createDataStore(project1.id, {
const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, {
name: 'dataStoreWithColumns',
columns: [{ name: 'foo', type: 'string' }],
});
const { id: dataStoreId } = dataStore;
await expect(dataStoreService.getColumns(dataStoreId)).resolves.toEqual([
await expect(dataStoreService.getColumns(dataStoreId, project1.id)).resolves.toEqual([
{
name: 'foo',
type: 'string',
@@ -148,13 +146,12 @@ describe('dataStore', () => {
const name = 'dataStore';
// ACT
const result = await dataStoreService.createDataStore(project1.id, {
const { project } = await dataStoreService.createDataStore(project1.id, {
name,
columns: [],
});
// ASSERT
const { project } = result;
expect(project.id).toBe(project1.id);
expect(project.name).toBe(project1.name);
});
@@ -184,17 +181,18 @@ describe('dataStore', () => {
describe('updateDataStore', () => {
it('should succeed when renaming to an available name', async () => {
// ARRANGE
const dataStore = await dataStoreService.createDataStore(project1.id, {
const { id: dataStoreId, updatedAt } = await dataStoreService.createDataStore(project1.id, {
name: 'myDataStore1',
columns: [],
});
const { id: dataStoreId, updatedAt } = dataStore;
// ACT
// Wait to get second difference
await new Promise((resolve) => setTimeout(resolve, 1001));
const result = await dataStoreService.updateDataStore(dataStoreId, { name: 'aNewName' });
const result = await dataStoreService.updateDataStore(dataStoreId, project1.id, {
name: 'aNewName',
});
// ASSERT
expect(result).toEqual(true);
@@ -206,47 +204,12 @@ describe('dataStore', () => {
it('should fail when renaming a non-existent data store', async () => {
// ACT
const result = dataStoreService.updateDataStore('this is not an id', {
const result = dataStoreService.updateDataStore('this is not an id', project1.id, {
name: 'aNewName',
});
// ASSERT
await expect(result).rejects.toThrow(
"Tried to rename non-existent data store 'this is not an id'",
);
});
it('should fail when renaming to an empty name', async () => {
// ARRANGE
const dataStore = await dataStoreService.createDataStore(project1.id, {
name: 'myDataStore',
columns: [],
});
const { id: dataStoreId } = dataStore;
// ACT
const result = dataStoreService.updateDataStore(dataStoreId, { name: '' });
// ASSERT
await expect(result).rejects.toThrow('Data store name must not be empty');
});
it('should trim the name', async () => {
// ARRANGE
const dataStore = await dataStoreService.createDataStore(project1.id, {
name: 'myDataStore1',
columns: [],
});
const { id: dataStoreId } = dataStore;
// ACT
const result = dataStoreService.updateDataStore(dataStoreId, { name: ' aNewName ' });
// ASSERT
await expect(result).resolves.toEqual(true);
const updated = await dataStoreRepository.findOneBy({ id: dataStoreId });
expect(updated?.name).toBe('aNewName');
await expect(result).rejects.toThrow("Data Store 'this is not an id' does not exist.");
});
it('should fail when renaming to a taken name', async () => {
@@ -257,18 +220,17 @@ describe('dataStore', () => {
columns: [],
});
const dataStoreNew = await dataStoreService.createDataStore(project1.id, {
const { id: dataStoreNewId } = await dataStoreService.createDataStore(project1.id, {
name: 'myDataStoreNew',
columns: [],
});
const { id: dataStoreNewId } = dataStoreNew;
// ACT
const result = dataStoreService.updateDataStore(dataStoreNewId, { name });
const result = dataStoreService.updateDataStore(dataStoreNewId, project1.id, { name });
// ASSERT
await expect(result).rejects.toThrow(
`The name '${name}' is already taken within this project`,
`Data store with name '${name}' already exists in this project`,
);
});
});
@@ -276,14 +238,13 @@ describe('dataStore', () => {
describe('deleteDataStore', () => {
it('should succeed with deleting a store', async () => {
// ARRANGE
const dataStore = await dataStoreService.createDataStore(project1.id, {
const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, {
name: 'myDataStore1',
columns: [],
});
const { id: dataStoreId } = dataStore;
// ACT
const result = await dataStoreService.deleteDataStore(dataStoreId);
const result = await dataStoreService.deleteDataStore(dataStoreId, project1.id);
const userTableName = toTableName(dataStoreId);
// ASSERT
@@ -302,7 +263,7 @@ describe('dataStore', () => {
it('should fail when deleting a non-existent id', async () => {
// ACT
const result = dataStoreService.deleteDataStore('this is not an id');
const result = dataStoreService.deleteDataStore('this is not an id', project1.id);
// ASSERT
await expect(result).rejects.toThrow(
@@ -315,11 +276,10 @@ describe('dataStore', () => {
it('should succeed with adding columns to a non-empty table', async () => {
const existingColumns: CreateDataStoreColumnDto[] = [{ name: 'myColumn0', type: 'string' }];
const dataStore = await dataStoreService.createDataStore(project1.id, {
const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, {
name: 'dataStoreWithColumns',
columns: existingColumns,
});
const { id: dataStoreId } = dataStore;
const columns: AddDataStoreColumnDto[] = [
{ name: 'myColumn1', type: 'string' },
@@ -329,11 +289,11 @@ describe('dataStore', () => {
];
for (const column of columns) {
// ACT
const result = await dataStoreService.addColumn(dataStoreId, column);
const result = await dataStoreService.addColumn(dataStoreId, project1.id, column);
// ASSERT
expect(result).toMatchObject(column);
}
const columnResult = await dataStoreService.getColumns(dataStoreId);
const columnResult = await dataStoreService.getColumns(dataStoreId, project1.id);
expect(columnResult).toEqual([
{
index: 0,
@@ -400,14 +360,13 @@ describe('dataStore', () => {
it('should create the user table on first addColumn if it does not exist', async () => {
// ARRANGE
const dataStore = await dataStoreService.createDataStore(project1.id, {
const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, {
name: 'dataStore',
columns: [],
});
const { id: dataStoreId } = dataStore;
// ACT
const result = await dataStoreService.addColumn(dataStoreId, {
const result = await dataStoreService.addColumn(dataStoreId, project1.id, {
name: 'foo',
type: 'string',
});
@@ -428,7 +387,7 @@ describe('dataStore', () => {
it('should fail with adding two columns of the same name', async () => {
// ARRANGE
const dataStore = await dataStoreService.createDataStore(project1.id, {
const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, {
name: 'myDataStore',
columns: [
{
@@ -437,10 +396,9 @@ describe('dataStore', () => {
},
],
});
const { id: dataStoreId } = dataStore;
// ACT
const result = dataStoreService.addColumn(dataStoreId, {
const result = dataStoreService.addColumn(dataStoreId, project1.id, {
name: 'myColumn1',
type: 'number',
});
@@ -453,7 +411,7 @@ describe('dataStore', () => {
it('should fail with adding column of non-existent table', async () => {
// ACT
const result = dataStoreService.addColumn('this is not an id', {
const result = dataStoreService.addColumn('this is not an id', project1.id, {
name: 'myColumn1',
type: 'number',
});
@@ -468,28 +426,27 @@ describe('dataStore', () => {
describe('deleteColumn', () => {
it('should succeed with deleting a column', async () => {
// ARRANGE
const dataStore = await dataStoreService.createDataStore(project1.id, {
const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, {
name: 'dataStore',
columns: [],
});
const { id: dataStoreId } = dataStore;
const c1 = await dataStoreService.addColumn(dataStoreId, {
const c1 = await dataStoreService.addColumn(dataStoreId, project1.id, {
name: 'myColumn1',
type: 'string',
});
const c2 = await dataStoreService.addColumn(dataStoreId, {
const c2 = await dataStoreService.addColumn(dataStoreId, project1.id, {
name: 'myColumn2',
type: 'number',
});
// ACT
const result = await dataStoreService.deleteColumn(dataStoreId, c1.id);
const result = await dataStoreService.deleteColumn(dataStoreId, project1.id, c1.id);
// ASSERT
expect(result).toEqual(true);
const columns = await dataStoreService.getColumns(dataStoreId);
const columns = await dataStoreService.getColumns(dataStoreId, project1.id);
expect(columns).toEqual([
{
index: 0,
@@ -506,7 +463,7 @@ describe('dataStore', () => {
it('should fail when deleting unknown column', async () => {
// ARRANGE
const dataStore = await dataStoreService.createDataStore(project1.id, {
const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, {
name: 'dataStore',
columns: [
{
@@ -515,10 +472,9 @@ describe('dataStore', () => {
},
],
});
const { id: dataStoreId } = dataStore;
// ACT
const result = dataStoreService.deleteColumn(dataStoreId, 'thisIsNotAnId');
const result = dataStoreService.deleteColumn(dataStoreId, project1.id, 'thisIsNotAnId');
// ASSERT
await expect(result).rejects.toThrow(
@@ -528,17 +484,17 @@ describe('dataStore', () => {
it('should fail when deleting column from unknown table', async () => {
// ARRANGE
const dataStore = await dataStoreService.createDataStore(project1.id, {
const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, {
name: 'dataStore',
columns: [],
});
const c1 = await dataStoreService.addColumn(dataStore.id, {
const c1 = await dataStoreService.addColumn(dataStoreId, project1.id, {
name: 'myColumn1',
type: 'string',
});
// ACT
const result = dataStoreService.deleteColumn('this is not an id', c1.id);
const result = dataStoreService.deleteColumn('this is not an id', project1.id, c1.id);
// ASSERT
await expect(result).rejects.toThrow(
@@ -550,30 +506,29 @@ describe('dataStore', () => {
describe('moveColumn', () => {
it('should succeed with moving a column', async () => {
// ARRANGE
const dataStore = await dataStoreService.createDataStore(project1.id, {
const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, {
name: 'dataStore',
columns: [],
});
const { id: dataStoreId } = dataStore;
const c1 = await dataStoreService.addColumn(dataStoreId, {
const c1 = await dataStoreService.addColumn(dataStoreId, project1.id, {
name: 'myColumn1',
type: 'string',
});
const c2 = await dataStoreService.addColumn(dataStoreId, {
const c2 = await dataStoreService.addColumn(dataStoreId, project1.id, {
name: 'myColumn2',
type: 'number',
});
// ACT
const result = await dataStoreService.moveColumn(dataStoreId, c2.id, {
const result = await dataStoreService.moveColumn(dataStoreId, project1.id, c2.id, {
targetIndex: 0,
});
// ASSERT
expect(result).toEqual(true);
const columns = await dataStoreService.getColumns(dataStoreId);
const columns = await dataStoreService.getColumns(dataStoreId, project1.id);
expect(columns).toMatchObject([
{
index: 0,
@@ -600,11 +555,10 @@ describe('dataStore', () => {
name: 'dataStore',
columns: [],
});
const { name } = dataStore;
// ACT
const result = await dataStoreService.getManyAndCount({
filter: { projectId: project1.id, name },
filter: { projectId: project1.id, name: dataStore.name },
});
// ASSERT
@@ -629,17 +583,15 @@ describe('dataStore', () => {
name: 'myDataStore1',
columns: [],
});
const { id: dataStoreId1 } = dataStore1;
const dataStore2 = await dataStoreService.createDataStore(project1.id, {
name: 'myDataStore2',
columns: [],
});
const { id: dataStoreId2 } = dataStore2;
// ACT
const result = await dataStoreService.getManyAndCount({
filter: { projectId: project1.id, id: [dataStoreId1, dataStoreId2] },
filter: { projectId: project1.id, id: [dataStore1.id, dataStore2.id] },
});
// ASSERT
@@ -663,8 +615,7 @@ describe('dataStore', () => {
name: 'myDataStore',
columns: [],
});
const { name } = dataStore;
const names = [name];
const names = [dataStore.name];
for (let i = 0; i < 10; ++i) {
const ds = await dataStoreService.createDataStore(project1.id, {
name: `anotherDataStore${i}`,
@@ -689,8 +640,7 @@ describe('dataStore', () => {
name: 'myDataStore',
columns: [],
});
const { name } = dataStore;
const names = [name];
const names = [dataStore.name];
for (let i = 0; i < 10; ++i) {
const ds = await dataStoreService.createDataStore(project1.id, {
@@ -744,11 +694,10 @@ describe('dataStore', () => {
{ name: 'myColumn3', type: 'number' },
{ name: 'myColumn4', type: 'date' },
];
const dataStore = await dataStoreService.createDataStore(project1.id, {
const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, {
name: 'myDataStore',
columns,
});
const { id: dataStoreId } = dataStore;
// ACT
const result = await dataStoreService.getManyAndCount({
@@ -883,11 +832,10 @@ describe('dataStore', () => {
it('sorts by updatedAt', async () => {
// ARRANGE
const ds1 = await dataStoreService.createDataStore(project1.id, {
const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, {
name: 'ds1',
columns: [],
});
const { id: ds1Id } = ds1;
// Wait to get seconds difference
await new Promise((resolve) => setTimeout(resolve, 1001));
@@ -898,7 +846,7 @@ describe('dataStore', () => {
// Wait to get seconds difference
await new Promise((resolve) => setTimeout(resolve, 1001));
await dataStoreService.updateDataStore(ds1Id, { name: 'ds1Updated' });
await dataStoreService.updateDataStore(dataStoreId, project1.id, { name: 'ds1Updated' });
// ACT
const updatedAsc = await dataStoreService.getManyAndCount({
@@ -921,7 +869,7 @@ describe('dataStore', () => {
describe('insertRows', () => {
it('inserts rows into an existing table', async () => {
// ARRANGE
const dataStore = await dataStoreService.createDataStore(project1.id, {
const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, {
name: 'dataStore',
columns: [
{ name: 'c1', type: 'number' },
@@ -930,7 +878,6 @@ describe('dataStore', () => {
{ name: 'c4', type: 'string' },
],
});
const { id: dataStoreId } = dataStore;
// ACT
const rows = [
@@ -938,12 +885,16 @@ describe('dataStore', () => {
{ c1: 4, c2: false, c3: new Date(), c4: 'hello!' },
{ c1: 5, c2: true, c3: new Date(), c4: 'hello.' },
];
const result = await dataStoreService.insertRows(dataStoreId, rows);
const result = await dataStoreService.insertRows(dataStoreId, project1.id, rows);
// ASSERT
expect(result).toBe(true);
const { count, data } = await dataStoreService.getManyRowsAndCount(dataStoreId, {});
const { count, data } = await dataStoreService.getManyRowsAndCount(
dataStoreId,
project1.id,
{},
);
expect(count).toEqual(3);
expect(data).toEqual(
rows.map((row, i) => ({
@@ -959,20 +910,21 @@ describe('dataStore', () => {
it('inserts a row even if it matches with the existing one', async () => {
// ARRANGE
const dataStore = await dataStoreService.createDataStore(project1.id, {
const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, {
name: 'myDataStore',
columns: [
{ name: 'c1', type: 'number' },
{ name: 'c2', type: 'string' },
],
});
const { id: dataStoreId } = dataStore;
// Insert initial row
await dataStoreService.insertRows(dataStoreId, [{ c1: 1, c2: 'foo' }]);
await dataStoreService.insertRows(dataStoreId, project1.id, [{ c1: 1, c2: 'foo' }]);
// Attempt to insert a row with the same primary key
const result = await dataStoreService.insertRows(dataStoreId, [{ c1: 1, c2: 'foo' }]);
const result = await dataStoreService.insertRows(dataStoreId, project1.id, [
{ c1: 1, c2: 'foo' },
]);
// ASSERT
expect(result).toBe(true);
@@ -991,7 +943,7 @@ describe('dataStore', () => {
it('rejects a mismatched row with extra column', async () => {
// ARRANGE
const dataStore = await dataStoreService.createDataStore(project1.id, {
const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, {
name: 'dataStore',
columns: [
{ name: 'c1', type: 'number' },
@@ -1000,10 +952,9 @@ describe('dataStore', () => {
{ name: 'c4', type: 'string' },
],
});
const { id: dataStoreId } = dataStore;
// ACT
const result = dataStoreService.insertRows(dataStoreId, [
const result = dataStoreService.insertRows(dataStoreId, project1.id, [
{ c1: 3, c2: true, c3: new Date(), c4: 'hello?' },
{ cWrong: 3, c1: 4, c2: true, c3: new Date(), c4: 'hello?' },
]);
@@ -1014,7 +965,7 @@ describe('dataStore', () => {
it('rejects a mismatched row with missing column', async () => {
// ARRANGE
const dataStore = await dataStoreService.createDataStore(project1.id, {
const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, {
name: 'dataStore',
columns: [
{ name: 'c1', type: 'number' },
@@ -1023,10 +974,9 @@ describe('dataStore', () => {
{ name: 'c4', type: 'string' },
],
});
const { id: dataStoreId } = dataStore;
// ACT
const result = dataStoreService.insertRows(dataStoreId, [
const result = dataStoreService.insertRows(dataStoreId, project1.id, [
{ c1: 3, c2: true, c3: new Date(), c4: 'hello?' },
{ c2: true, c3: new Date(), c4: 'hello?' },
]);
@@ -1037,7 +987,7 @@ describe('dataStore', () => {
it('rejects a mismatched row with replaced column', async () => {
// ARRANGE
const dataStore = await dataStoreService.createDataStore(project1.id, {
const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, {
name: 'dataStore',
columns: [
{ name: 'c1', type: 'number' },
@@ -1046,10 +996,9 @@ describe('dataStore', () => {
{ name: 'c4', type: 'string' },
],
});
const { id: dataStoreId } = dataStore;
// ACT
const result = dataStoreService.insertRows(dataStoreId, [
const result = dataStoreService.insertRows(dataStoreId, project1.id, [
{ c1: 3, c2: true, c3: new Date(), c4: 'hello?' },
{ cWrong: 3, c2: true, c3: new Date(), c4: 'hello?' },
]);
@@ -1071,20 +1020,24 @@ describe('dataStore', () => {
});
// ACT
const result = dataStoreService.insertRows('this is not an id', [
const result = dataStoreService.insertRows('this is not an id', project1.id, [
{ c1: 3, c2: true, c3: new Date(), c4: 'hello?' },
{ cWrong: 3, c2: true, c3: new Date(), c4: 'hello?' },
]);
// ASSERT
await expect(result).rejects.toThrow(
'No columns found for this data store or data store not found',
);
await expect(result).rejects.toThrow("Data Store 'this is not an id' does not exist.");
});
it('rejects on empty column list', async () => {
// ARRANGE
const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, {
name: 'dataStore',
columns: [],
});
// ACT
const result = dataStoreService.insertRows('this is not an id', [{}, {}]);
const result = dataStoreService.insertRows(dataStoreId, project1.id, [{}, {}]);
// ASSERT
await expect(result).rejects.toThrow(
@@ -1094,14 +1047,16 @@ describe('dataStore', () => {
it('fails on type mismatch', async () => {
// ARRANGE
const dataStore = await dataStoreService.createDataStore(project1.id, {
const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, {
name: 'dataStore',
columns: [{ name: 'c1', type: 'number' }],
});
const { id: dataStoreId } = dataStore;
// ACT
const result = dataStoreService.insertRows(dataStoreId, [{ c1: 3 }, { c1: true }]);
const result = dataStoreService.insertRows(dataStoreId, project1.id, [
{ c1: 3 },
{ c1: true },
]);
// ASSERT
await expect(result).rejects.toThrow("value 'true' does not match column type 'number'");
@@ -1111,7 +1066,7 @@ describe('dataStore', () => {
describe('upsertRows', () => {
it('updates a row if filter matches', async () => {
// ARRANGE
const dataStore = await dataStoreService.createDataStore(project1.id, {
const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, {
name: 'dataStore',
columns: [
{ name: 'pid', type: 'string' },
@@ -1119,15 +1074,14 @@ describe('dataStore', () => {
{ name: 'age', type: 'number' },
],
});
const { id: dataStoreId } = dataStore;
// Insert initial row
await dataStoreService.insertRows(dataStoreId, [
await dataStoreService.insertRows(dataStoreId, project1.id, [
{ pid: '1995-111a', fullName: 'Alice', age: 30 },
]);
// ACT
const result = await dataStoreService.upsertRows(dataStoreId, {
const result = await dataStoreService.upsertRows(dataStoreId, project1.id, {
rows: [{ pid: '1995-111a', fullName: 'Alicia', age: 31 }],
matchFields: ['pid'],
});
@@ -1146,7 +1100,7 @@ describe('dataStore', () => {
it('inserts a row if filter does not match', async () => {
// ARRANGE
const dataStore = await dataStoreService.createDataStore(project1.id, {
const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, {
name: 'dataStore',
columns: [
{ name: 'pid', type: 'string' },
@@ -1154,15 +1108,14 @@ describe('dataStore', () => {
{ name: 'age', type: 'number' },
],
});
const { id: dataStoreId } = dataStore;
// Insert initial row
await dataStoreService.insertRows(dataStoreId, [
await dataStoreService.insertRows(dataStoreId, project1.id, [
{ pid: '1995-111a', fullName: 'Alice', age: 30 },
]);
// ACT
const result = await dataStoreService.upsertRows(dataStoreId, {
const result = await dataStoreService.upsertRows(dataStoreId, project1.id, {
rows: [{ pid: '1992-222b', fullName: 'Alice', age: 30 }],
matchFields: ['pid'],
});
@@ -1186,7 +1139,7 @@ describe('dataStore', () => {
describe('getManyRowsAndCount', () => {
it('retrieves rows correctly', async () => {
// ARRANGE
const dataStore = await dataStoreService.createDataStore(project1.id, {
const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, {
name: 'dataStore',
columns: [
{ name: 'c1', type: 'number' },
@@ -1195,7 +1148,6 @@ describe('dataStore', () => {
{ name: 'c4', type: 'string' },
],
});
const { id: dataStoreId } = dataStore;
const rows = [
{ c1: 3, c2: true, c3: new Date(0), c4: 'hello?' },
@@ -1203,10 +1155,10 @@ describe('dataStore', () => {
{ c1: 5, c2: true, c3: new Date(2), c4: 'hello.' },
];
await dataStoreService.insertRows(dataStoreId, rows);
await dataStoreService.insertRows(dataStoreId, project1.id, rows);
// ACT
const result = await dataStoreService.getManyRowsAndCount(dataStoreId, {});
const result = await dataStoreService.getManyRowsAndCount(dataStoreId, project1.id, {});
// ASSERT
expect(result.count).toEqual(3);

36
packages/cli/src/modules/data-store/data-store.controller.ts
View File

@@ -54,98 +54,98 @@ export class DataStoreController {
@Patch('/:dataStoreId')
@ProjectScope('dataStore:update')
async updateDataStore(
_req: AuthenticatedRequest<{ projectId: string }>,
req: AuthenticatedRequest<{ projectId: string }>,
_res: Response,
@Param('dataStoreId') dataStoreId: string,
@Body dto: UpdateDataStoreDto,
) {
return await this.dataStoreService.updateDataStore(dataStoreId, dto);
return await this.dataStoreService.updateDataStore(dataStoreId, req.params.projectId, dto);
}
@Delete('/:dataStoreId')
@ProjectScope('dataStore:delete')
async deleteDataStore(
_req: AuthenticatedRequest<{ projectId: string }>,
req: AuthenticatedRequest<{ projectId: string }>,
_res: Response,
@Param('dataStoreId') dataStoreId: string,
) {
return await this.dataStoreService.deleteDataStore(dataStoreId);
return await this.dataStoreService.deleteDataStore(dataStoreId, req.params.projectId);
}
@Get('/:dataStoreId/columns')
@ProjectScope('dataStore:read')
async getColumns(
_req: AuthenticatedRequest<{ projectId: string }>,
req: AuthenticatedRequest<{ projectId: string }>,
_res: Response,
@Param('dataStoreId') dataStoreId: string,
) {
return await this.dataStoreService.getColumns(dataStoreId);
return await this.dataStoreService.getColumns(dataStoreId, req.params.projectId);
}
@Post('/:dataStoreId/columns')
@ProjectScope('dataStore:update')
async addColumn(
_req: AuthenticatedRequest<{ projectId: string }>,
req: AuthenticatedRequest<{ projectId: string }>,
_res: Response,
@Param('dataStoreId') dataStoreId: string,
@Body dto: AddDataStoreColumnDto,
) {
return await this.dataStoreService.addColumn(dataStoreId, dto);
return await this.dataStoreService.addColumn(dataStoreId, req.params.projectId, dto);
}
@Delete('/:dataStoreId/columns/:columnId')
@ProjectScope('dataStore:update')
async deleteColumn(
_req: AuthenticatedRequest<{ projectId: string }>,
req: AuthenticatedRequest<{ projectId: string }>,
_res: Response,
@Param('dataStoreId') dataStoreId: string,
@Param('columnId') columnId: string,
) {
return await this.dataStoreService.deleteColumn(dataStoreId, columnId);
return await this.dataStoreService.deleteColumn(dataStoreId, req.params.projectId, columnId);
}
@Patch('/:dataStoreId/columns/:columnId/move')
@ProjectScope('dataStore:update')
async moveColumn(
_req: AuthenticatedRequest<{ projectId: string }>,
req: AuthenticatedRequest<{ projectId: string }>,
_res: Response,
@Param('dataStoreId') dataStoreId: string,
@Param('columnId') columnId: string,
@Body dto: MoveDataStoreColumnDto,
) {
return await this.dataStoreService.moveColumn(dataStoreId, columnId, dto);
return await this.dataStoreService.moveColumn(dataStoreId, req.params.projectId, columnId, dto);
}
@Get('/:dataStoreId/rows')
@ProjectScope('dataStore:readRow')
async getDataStoreRows(
_req: AuthenticatedRequest<{ projectId: string }>,
req: AuthenticatedRequest<{ projectId: string }>,
_res: Response,
@Param('dataStoreId') dataStoreId: string,
@Query dto: ListDataStoreContentQueryDto,
) {
return await this.dataStoreService.getManyRowsAndCount(dataStoreId, dto);
return await this.dataStoreService.getManyRowsAndCount(dataStoreId, req.params.projectId, dto);
}
@Post('/:dataStoreId/insert')
@ProjectScope('dataStore:writeRow')
async appendDataStoreRows(
_req: AuthenticatedRequest<{ projectId: string }>,
req: AuthenticatedRequest<{ projectId: string }>,
_res: Response,
@Param('dataStoreId') dataStoreId: string,
@Body dto: AddDataStoreRowsDto,
) {
return await this.dataStoreService.insertRows(dataStoreId, dto.data);
return await this.dataStoreService.insertRows(dataStoreId, req.params.projectId, dto.data);
}
@Post('/:dataStoreId/upsert')
@ProjectScope('dataStore:writeRow')
async upsertDataStoreRows(
_req: AuthenticatedRequest<{ projectId: string }>,
req: AuthenticatedRequest<{ projectId: string }>,
_res: Response,
@Param('dataStoreId') dataStoreId: string,
@Body dto: UpsertDataStoreRowsDto,
) {
return await this.dataStoreService.upsertRows(dataStoreId, dto);
return await this.dataStoreService.upsertRows(dataStoreId, req.params.projectId, dto);
}
}

11
packages/cli/src/modules/data-store/data-store.repository.ts
View File

@@ -165,12 +165,15 @@ export class DataStoreRepository extends Repository<DataStore> {
});
}
if (filter?.name && typeof filter.name === 'string') {
if (filter?.name) {
const nameFilters = typeof filter.name === 'string' ? [filter.name] : filter.name;
for (const name of nameFilters) {
query.andWhere('LOWER(dataStore.name) LIKE LOWER(:name)', {
name: `%${filter.name}%`,
name: `%${name}%`,
});
}
}
}
private applySorting(query: SelectQueryBuilder<DataStore>, sortBy?: string): void {
if (!sortBy) {
@@ -193,7 +196,9 @@ export class DataStoreRepository extends Repository<DataStore> {
direction: 'DESC' | 'ASC',
): void {
if (field === 'name') {
query.orderBy('LOWER(dataStore.name)', direction);
query
.addSelect('LOWER(dataStore.name)', 'datastore_name_lower')
.orderBy('datastore_name_lower', direction);
} else if (['createdAt', 'updatedAt'].includes(field)) {
query.orderBy(`dataStore.${field}`, direction);
}

142
packages/cli/src/modules/data-store/data-store.service.ts
View File

@@ -12,11 +12,13 @@ import { Logger } from '@n8n/backend-common';
import { Service } from '@n8n/di';
import { UserError } from 'n8n-workflow';
import { DataStoreColumn } from './data-store-column.entity';
import { ConflictError } from '@/errors/response-errors/conflict.error';
import { NotFoundError } from '@/errors/response-errors/not-found.error';
import { DataStoreColumnRepository } from './data-store-column.repository';
import { DataStoreRowsRepository } from './data-store-rows.repository';
import { DataStoreRepository } from './data-store.repository';
import { toTableName } from './utils/sql-utils';
import { toTableName, normalizeRows } from './utils/sql-utils';
@Service()
export class DataStoreService {
@@ -33,42 +35,17 @@ export class DataStoreService {
async shutdown() {}
async createDataStore(projectId: string, dto: CreateDataStoreDto) {
const existingTable = await this.dataStoreRepository.findOneBy({
name: dto.name,
projectId,
});
if (existingTable !== null) {
throw new UserError(`Data store with name '${dto.name}' already exists in this project`);
}
await this.validateUniqueName(dto.name, projectId);
return await this.dataStoreRepository.createDataStore(projectId, dto.name, dto.columns);
}
// Currently only renames data stores
async updateDataStore(dataStoreId: string, dto: UpdateDataStoreDto) {
const name = dto.name.trim();
async updateDataStore(dataStoreId: string, projectId: string, dto: UpdateDataStoreDto) {
await this.validateDataStoreExists(dataStoreId, projectId);
await this.validateUniqueName(dto.name, projectId);
if (!name) {
throw new UserError('Data store name must not be empty');
}
const existingTable = await this.dataStoreRepository.findOneBy({
id: dataStoreId,
});
if (existingTable === null) {
throw new UserError(`Tried to rename non-existent data store '${dataStoreId}'`);
}
const hasNameClash = await this.dataStoreRepository.existsBy({
name,
projectId: existingTable.projectId,
});
if (hasNameClash) {
throw new UserError(`The name '${name}' is already taken within this project`);
}
await this.dataStoreRepository.update({ id: dataStoreId }, { name });
await this.dataStoreRepository.update({ id: dataStoreId }, { name: dto.name });
return true;
}
@@ -81,9 +58,10 @@ export class DataStoreService {
return await this.dataStoreRepository.deleteDataStoreAll();
}
async deleteDataStore(dataStoreId: string) {
async deleteDataStore(dataStoreId: string, projectId: string) {
await this.validateDataStoreExists(
dataStoreId,
projectId,
`Tried to delete non-existent data store '${dataStoreId}'`,
);
@@ -92,18 +70,25 @@ export class DataStoreService {
return true;
}
async addColumn(dataStoreId: string, dto: AddDataStoreColumnDto) {
async addColumn(dataStoreId: string, projectId: string, dto: AddDataStoreColumnDto) {
await this.validateDataStoreExists(
dataStoreId,
projectId,
`Tried to add column to non-existent data store '${dataStoreId}'`,
);
return await this.dataStoreColumnRepository.addColumn(dataStoreId, dto);
}
async moveColumn(dataStoreId: string, columnId: string, dto: MoveDataStoreColumnDto) {
async moveColumn(
dataStoreId: string,
projectId: string,
columnId: string,
dto: MoveDataStoreColumnDto,
) {
await this.validateDataStoreExists(
dataStoreId,
projectId,
`Tried to move column from non-existent data store '${dataStoreId}'`,
);
@@ -112,9 +97,10 @@ export class DataStoreService {
return true;
}
async deleteColumn(dataStoreId: string, columnId: string) {
async deleteColumn(dataStoreId: string, projectId: string, columnId: string) {
await this.validateDataStoreExists(
dataStoreId,
projectId,
`Tried to delete column from non-existent data store '${dataStoreId}'`,
);
@@ -138,7 +124,13 @@ export class DataStoreService {
return await this.dataStoreRepository.getManyAndCount(options);
}
async getManyRowsAndCount(dataStoreId: string, dto: ListDataStoreContentQueryDto) {
async getManyRowsAndCount(
dataStoreId: string,
projectId: string,
dto: ListDataStoreContentQueryDto,
) {
await this.validateDataStoreExists(dataStoreId, projectId);
// unclear if we should validate here, only use case would be to reduce the chance of
// a renamed/removed column appearing here (or added column missing) if the store was
// modified between when the frontend sent the request and we received it
@@ -149,66 +141,34 @@ export class DataStoreService {
);
return {
count: result.count,
data: this.normalizeRows(result.data, columns),
data: normalizeRows(result.data, columns),
};
}
async getColumns(dataStoreId: string) {
async getColumns(dataStoreId: string, projectId: string) {
await this.validateDataStoreExists(dataStoreId, projectId);
return await this.dataStoreColumnRepository.getColumns(dataStoreId);
}
async insertRows(dataStoreId: string, rows: DataStoreRows) {
async insertRows(dataStoreId: string, projectId: string, rows: DataStoreRows) {
await this.validateDataStoreExists(dataStoreId, projectId);
await this.validateRows(dataStoreId, rows);
const columns = await this.dataStoreColumnRepository.getColumns(dataStoreId);
return await this.dataStoreRowsRepository.insertRows(toTableName(dataStoreId), rows, columns);
}
async upsertRows(dataStoreId: string, dto: UpsertDataStoreRowsDto) {
async upsertRows(dataStoreId: string, projectId: string, dto: UpsertDataStoreRowsDto) {
await this.validateDataStoreExists(dataStoreId, projectId);
await this.validateRows(dataStoreId, dto.rows);
const columns = await this.dataStoreColumnRepository.getColumns(dataStoreId);
return await this.dataStoreRowsRepository.upsertRows(toTableName(dataStoreId), dto, columns);
}
// TODO: move to utils and test
private normalizeRows(rows: DataStoreRows, columns: DataStoreColumn[]): DataStoreRows {
const typeMap = new Map(columns.map((col) => [col.name, col.type]));
return rows.map((row) => {
const normalized = { ...row };
for (const [key, value] of Object.entries(row)) {
const type = typeMap.get(key);
if (type === 'boolean') {
// Convert boolean values to true/false
if (typeof value === 'boolean') {
normalized[key] = value;
} else if (value === 1 || value === '1') {
normalized[key] = true;
} else if (value === 0 || value === '0') {
normalized[key] = false;
}
}
if (type === 'date' && value !== null && value !== undefined) {
// Convert date objects or strings to ISO string
let dateObj: Date | null = null;
if (value instanceof Date) {
dateObj = value;
} else if (typeof value === 'string' || typeof value === 'number') {
const parsed = new Date(value);
if (!isNaN(parsed.getTime())) {
dateObj = parsed;
}
}
normalized[key] = dateObj ? dateObj.toISOString() : value;
}
}
return normalized;
});
}
private async validateRows(dataStoreId: string, rows: DataStoreRows): Promise<void> {
const columns = await this.dataStoreColumnRepository.getColumns(dataStoreId);
if (columns.length === 0) {
@@ -253,13 +213,31 @@ export class DataStoreService {
}
}
private async validateDataStoreExists(dataStoreId: string, msg?: string) {
private async validateDataStoreExists(dataStoreId: string, projectId: string, msg?: string) {
const existingTable = await this.dataStoreRepository.findOneBy({
id: dataStoreId,
project: {
id: projectId,
},
});
if (!existingTable) {
throw new UserError(msg ?? `Data Store '${dataStoreId}' does not exist.`);
throw new NotFoundError(msg ?? `Data Store '${dataStoreId}' does not exist.`);
}
return existingTable;
}
private async validateUniqueName(name: string, projectId: string, msg?: string) {
const hasNameClash = await this.dataStoreRepository.existsBy({
name,
projectId,
});
if (hasNameClash) {
throw new ConflictError(
msg ?? `Data store with name '${name}' already exists in this project`,
);
}
}
}

42
packages/cli/src/modules/data-store/utils/sql-utils.ts
View File

@@ -2,15 +2,16 @@ import {
DATA_STORE_COLUMN_REGEX,
type DataStoreRows,
type DataStoreCreateColumnSchema,
type DataStoreColumn,
} from '@n8n/api-types';
import { DslColumn } from '@n8n/db';
import type { DataSourceOptions } from '@n8n/typeorm';
import { UnexpectedError } from 'n8n-workflow';
import type { DataStoreUserTableName } from '../data-store.types';
import { NotFoundError } from '@/errors/response-errors/not-found.error';
import type { DataStoreUserTableName } from '../data-store.types';
export function toDslColumns(columns: DataStoreCreateColumnSchema[]): DslColumn[] {
return columns.map((col) => {
const name = new DslColumn(col.name.trim());
@@ -204,6 +205,43 @@ export function toTableName(dataStoreId: string): DataStoreUserTableName {
return `data_store_user_${dataStoreId}`;
}
export function normalizeRows(rows: DataStoreRows, columns: DataStoreColumn[]) {
const typeMap = new Map(columns.map((col) => [col.name, col.type]));
return rows.map((row) => {
const normalized = { ...row };
for (const [key, value] of Object.entries(row)) {
const type = typeMap.get(key);
if (type === 'boolean') {
// Convert boolean values to true/false
if (typeof value === 'boolean') {
normalized[key] = value;
} else if (value === 1 || value === '1') {
normalized[key] = true;
} else if (value === 0 || value === '0') {
normalized[key] = false;
}
}
if (type === 'date' && value !== null && value !== undefined) {
// Convert date objects or strings to ISO string
let dateObj: Date | null = null;
if (value instanceof Date) {
dateObj = value;
} else if (typeof value === 'string' || typeof value === 'number') {
const parsed = new Date(value);
if (!isNaN(parsed.getTime())) {
dateObj = parsed;
}
}
normalized[key] = dateObj ? dateObj.toISOString() : value;
}
}
return normalized;
});
}
function normalizeValue(
value: unknown,
columnType: string | undefined,

31
packages/cli/test/integration/shared/db/data-stores.ts Normal file
View File

@@ -0,0 +1,31 @@
import type { CreateDataStoreColumnDto } from '@n8n/api-types';
import { randomName } from '@n8n/backend-test-utils';
import type { Project } from '@n8n/db';
import { Container } from '@n8n/di';
import { DataStoreRepository } from '@/modules/data-store/data-store.repository';
export const createDataStore = async (
project: Project,
options: {
name?: string;
columns?: CreateDataStoreColumnDto[];
updatedAt?: Date;
} = {},
) => {
const dataStoreRepository = Container.get(DataStoreRepository);
const dataStore = await dataStoreRepository.createDataStore(
project.id,
options.name ?? randomName(),
options.columns ?? [],
);
if (options.updatedAt) {
await dataStoreRepository.update(dataStore.id, {
updatedAt: options.updatedAt,
});
dataStore.updatedAt = options.updatedAt;
}
return dataStore;
};

20
packages/cli/test/integration/shared/utils/test-server.ts
View File

@@ -1,5 +1,6 @@
import { LicenseState, ModuleRegistry } from '@n8n/backend-common';
import { mockInstance, mockLogger, testModules, testDb } from '@n8n/backend-test-utils';
import { GlobalConfig } from '@n8n/config';
import type { APIRequest, User } from '@n8n/db';
import { Container } from '@n8n/di';
import cookieParser from 'cookie-parser';
@@ -21,7 +22,6 @@ import { LicenseMocker } from '@test-integration/license';
import { PUBLIC_API_REST_PATH_SEGMENT, REST_PATH_SEGMENT } from '../constants';
import type { SetupProps, TestServer } from '../types';
import { GlobalConfig } from '@n8n/config';
/**
* Plugin to prefix a path segment into a request URL pathname.
@@ -186,12 +186,13 @@ export const setupTestServer = ({
await import('@/license/license.controller');
break;
case 'metrics':
case 'metrics': {
const { PrometheusMetricsService } = await import(
'@/metrics/prometheus-metrics.service'
);
await Container.get(PrometheusMetricsService).init(app);
break;
}
case 'eventBus':
await import('@/eventbus/event-bus.controller');
@@ -209,20 +210,22 @@ export const setupTestServer = ({
await import('@/controllers/mfa.controller');
break;
case 'ldap':
case 'ldap': {
const { LdapService } = await import('@/ldap.ee/ldap.service.ee');
await import('@/ldap.ee/ldap.controller.ee');
testServer.license.enable('feat:ldap');
await Container.get(LdapService).init();
break;
}
case 'saml':
case 'saml': {
const { SamlService } = await import('@/sso.ee/saml/saml.service.ee');
await Container.get(SamlService).init();
await import('@/sso.ee/saml/routes/saml.controller.ee');
const { setSamlLoginEnabled } = await import('@/sso.ee/saml/saml-helpers');
await setSamlLoginEnabled(true);
break;
}
case 'sourceControl':
await import('@/environments.ee/source-control/source-control.controller.ee');
@@ -290,15 +293,22 @@ export const setupTestServer = ({
case 'ai':
await import('@/controllers/ai.controller');
break;
case 'folder':
await import('@/controllers/folder.controller');
break;
case 'externalSecrets':
await import('@/modules/external-secrets.ee/external-secrets.module');
break;
case 'insights':
await import('@/modules/insights/insights.module');
break;
case 'data-store':
await import('@/modules/data-store/data-store.module');
break;
}
}