diff --git a/packages/@n8n/permissions/src/roles/scopes/global-scopes.ee.ts b/packages/@n8n/permissions/src/roles/scopes/global-scopes.ee.ts index e7cfd059e5..6574339187 100644 --- a/packages/@n8n/permissions/src/roles/scopes/global-scopes.ee.ts +++ b/packages/@n8n/permissions/src/roles/scopes/global-scopes.ee.ts @@ -79,13 +79,6 @@ export const GLOBAL_OWNER_SCOPES: Scope[] = [ 'insights:list', 'folder:move', 'oidc:manage', - 'dataStore:create', - 'dataStore:delete', - 'dataStore:read', - 'dataStore:update', - 'dataStore:list', - 'dataStore:readRow', - 'dataStore:writeRow', ]; export const GLOBAL_ADMIN_SCOPES = GLOBAL_OWNER_SCOPES.concat(); @@ -105,7 +98,4 @@ export const GLOBAL_MEMBER_SCOPES: Scope[] = [ 'user:list', 'variable:list', 'variable:read', - 'dataStore:read', - 'dataStore:list', - 'dataStore:readRow', ]; diff --git a/packages/cli/src/modules/data-store/__tests__/data-store.controller.test.ts b/packages/cli/src/modules/data-store/__tests__/data-store.controller.test.ts new file mode 100644 index 0000000000..782ca92e0f --- /dev/null +++ b/packages/cli/src/modules/data-store/__tests__/data-store.controller.test.ts @@ -0,0 +1,485 @@ +import type { DataStore } from '@n8n/api-types'; +import { + createTeamProject, + getPersonalProject, + linkUserToProject, + testDb, +} from '@n8n/backend-test-utils'; +import type { Project, User } from '@n8n/db'; +import { ProjectRepository } from '@n8n/db'; +import { Container } from '@n8n/di'; +import { createDataStore } from '@test-integration/db/data-stores'; +import { createOwner, createMember, createAdmin } from '@test-integration/db/users'; +import type { SuperAgentTest } from '@test-integration/types'; +import * as utils from '@test-integration/utils'; +import { DateTime } from 'luxon'; + +import { DataStoreRepository } from '../data-store.repository'; + +let owner: User; +let member: User; +let admin: User; +let authOwnerAgent: SuperAgentTest; +let authMemberAgent: SuperAgentTest; +let authAdminAgent: SuperAgentTest; +let ownerProject: Project; +let memberProject: Project; + +const testServer = utils.setupTestServer({ + endpointGroups: ['data-store'], + modules: ['data-store'], +}); +let projectRepository: ProjectRepository; +let dataStoreRepository: DataStoreRepository; + +beforeAll(async () => { + await testDb.init(); +}); + +beforeEach(async () => { + await testDb.truncate(['DataStore', 'DataStoreColumn', 'Project', 'ProjectRelation']); + + projectRepository = Container.get(ProjectRepository); + dataStoreRepository = Container.get(DataStoreRepository); + + owner = await createOwner(); + member = await createMember(); + admin = await createAdmin(); + + authOwnerAgent = testServer.authAgentFor(owner); + authMemberAgent = testServer.authAgentFor(member); + authAdminAgent = testServer.authAgentFor(admin); + + ownerProject = await getPersonalProject(owner); + memberProject = await getPersonalProject(member); +}); + +afterAll(async () => { + await testDb.terminate(); +}); + +describe('POST /projects/:projectId/data-stores', () => { + test('should not create data store when project does not exist', async () => { + const payload = { + name: 'Test Data Store', + columns: [ + { + name: 'test-ccolumn', + type: 'string', + }, + ], + }; + + await authMemberAgent.post('/projects/non-existing-id/data-stores').send(payload).expect(403); + await authAdminAgent.post('/projects/non-existing-id/data-stores').send(payload).expect(403); + await authOwnerAgent.post('/projects/non-existing-id/data-stores').send(payload).expect(403); + }); + + test('should not create data store when name is empty', async () => { + const project = await createTeamProject(undefined, owner); + const payload = { + name: '', + columns: [ + { + name: 'test-ccolumn', + type: 'string', + }, + ], + }; + + await authOwnerAgent.post(`/projects/${project.id}/data-stores`).send(payload).expect(400); + }); + + test('should not create data store if user has project:viewer role in team project', async () => { + const project = await createTeamProject(undefined, owner); + await linkUserToProject(member, project, 'project:viewer'); + + const payload = { + name: 'Test Data Store', + columns: [ + { + name: 'test-ccolumn', + type: 'string', + }, + ], + }; + + await authMemberAgent.post(`/projects/${project.id}/data-stores`).send(payload).expect(403); + + const dataStoresInDb = await dataStoreRepository.find(); + expect(dataStoresInDb).toHaveLength(0); + }); + + test("should not allow creating data store in another user's personal project", async () => { + const ownerPersonalProject = await projectRepository.getPersonalProjectForUserOrFail(owner.id); + const payload = { + name: 'Test Data Store', + columns: [ + { + name: 'test-ccolumn', + type: 'string', + }, + ], + }; + + await authMemberAgent + .post(`/projects/${ownerPersonalProject.id}/data-stores`) + .send(payload) + .expect(403); + }); + + test('should create data store if user has project:editor role in team project', async () => { + const project = await createTeamProject(undefined, owner); + await linkUserToProject(member, project, 'project:editor'); + + const payload = { + name: 'Test Data Store', + columns: [ + { + name: 'test-ccolumn', + type: 'string', + }, + ], + }; + + await authMemberAgent.post(`/projects/${project.id}/data-stores`).send(payload).expect(200); + + const dataStoresInDb = await dataStoreRepository.find(); + expect(dataStoresInDb).toHaveLength(1); + }); + + test('should create data store if user has project:admin role in team project', async () => { + const project = await createTeamProject(undefined, owner); + + const payload = { + name: 'Test Data Store', + columns: [ + { + name: 'test-ccolumn', + type: 'string', + }, + ], + }; + + await authOwnerAgent.post(`/projects/${project.id}/data-stores`).send(payload).expect(200); + + const dataStoresInDb = await dataStoreRepository.find(); + expect(dataStoresInDb).toHaveLength(1); + }); + + test('should create data store in personal project', async () => { + const personalProject = await projectRepository.getPersonalProjectForUserOrFail(owner.id); + const payload = { + name: 'Test Data Store', + columns: [ + { + name: 'test-ccolumn', + type: 'string', + }, + ], + }; + + const response = await authOwnerAgent + .post(`/projects/${personalProject.id}/data-stores`) + .send(payload) + .expect(200); + + expect(response.body.data).toEqual( + expect.objectContaining({ + id: expect.any(String), + name: payload.name, + projectId: personalProject.id, + createdAt: expect.any(String), + updatedAt: expect.any(String), + }), + ); + + const dataStoreInDb = await dataStoreRepository.findOneBy({ id: response.body.data.id }); + expect(dataStoreInDb).toBeDefined(); + expect(dataStoreInDb?.name).toBe(payload.name); + }); +}); + +describe('GET /projects/:projectId/data-stores', () => { + test('should not list data stores when project does not exist', async () => { + await authMemberAgent.get('/projects/non-existing-id/data-stores').expect(403); + await authAdminAgent.get('/projects/non-existing-id/data-stores').expect(403); + await authOwnerAgent.get('/projects/non-existing-id/data-stores').expect(403); + }); + + test('should not list data stores if user has no access to project', async () => { + const project = await createTeamProject('test project', owner); + + await authMemberAgent.get(`/projects/${project.id}/data-stores`).expect(403); + }); + + test('should not list data stores if admin has no access to project', async () => { + const project = await createTeamProject('test project', owner); + + await authAdminAgent.get(`/projects/${project.id}/data-stores`).expect(403); + }); + + test("should not allow listing data stores from another user's personal project", async () => { + await authMemberAgent.get(`/projects/${ownerProject.id}/data-stores`).expect(403); + }); + + test('should list data stores if user has project:viewer role in team project', async () => { + const project = await createTeamProject('test project', owner); + await linkUserToProject(member, project, 'project:viewer'); + await createDataStore(project, { name: 'Test Data Store' }); + + const response = await authMemberAgent.get(`/projects/${project.id}/data-stores`).expect(200); + + expect(response.body.data.count).toBe(1); + expect(response.body.data.data).toHaveLength(1); + expect(response.body.data.data[0].name).toBe('Test Data Store'); + }); + + test('should list data stores from personal project', async () => { + await createDataStore(ownerProject, { name: 'Personal Data Store 1' }); + await createDataStore(ownerProject, { name: 'Personal Data Store 2' }); + + const response = await authOwnerAgent + .get(`/projects/${ownerProject.id}/data-stores`) + .expect(200); + + expect(response.body.data.count).toBe(2); + expect(response.body.data.data).toHaveLength(2); + expect(response.body.data.data.map((f: any) => f.name).sort()).toEqual( + ['Personal Data Store 1', 'Personal Data Store 2'].sort(), + ); + }); + + test('should filter data stores by projectId', async () => { + await createDataStore(ownerProject, { name: 'Test Data Store 1' }); + await createDataStore(ownerProject, { name: 'Test Data Store 2' }); + await createDataStore(memberProject, { name: 'Another Data Store' }); + + const response = await authOwnerAgent + .get(`/projects/${ownerProject.id}/data-stores`) + .query({ filter: JSON.stringify({ name: 'test' }) }) + .expect(200); + + expect(response.body.data.count).toBe(2); + expect(response.body.data.data).toHaveLength(2); + expect(response.body.data.data.map((f: any) => f.name).sort()).toEqual( + ['Test Data Store 1', 'Test Data Store 2'].sort(), + ); + }); + + test('should filter data stores by name', async () => { + await createDataStore(ownerProject, { name: 'Test Data Store' }); + await createDataStore(ownerProject, { name: 'Another Data Store' }); + await createDataStore(ownerProject, { name: 'Test Something Else' }); + + const response = await authOwnerAgent + .get(`/projects/${ownerProject.id}/data-stores`) + .query({ filter: JSON.stringify({ name: 'test' }) }) + .expect(200); + + expect(response.body.data.count).toBe(2); + expect(response.body.data.data).toHaveLength(2); + expect(response.body.data.data.map((f: any) => f.name).sort()).toEqual( + ['Test Data Store', 'Test Something Else'].sort(), + ); + }); + + test('should filter data stores by id', async () => { + const dataStore1 = await createDataStore(ownerProject, { name: 'Data Store 1' }); + await createDataStore(ownerProject, { name: 'Data Store 2' }); + await createDataStore(ownerProject, { name: 'Data Store 3' }); + + const response = await authOwnerAgent + .get(`/projects/${ownerProject.id}/data-stores`) + .query({ filter: JSON.stringify({ id: dataStore1.id }) }) + .expect(200); + + expect(response.body.data.count).toBe(1); + expect(response.body.data.data).toHaveLength(1); + expect(response.body.data.data[0].name).toBe('Data Store 1'); + }); + + test('should filter data stores by multiple names (AND operator)', async () => { + await createDataStore(ownerProject, { name: 'Data Store' }); + await createDataStore(ownerProject, { name: 'Test Store' }); + await createDataStore(ownerProject, { name: 'Another Store' }); + + const response = await authOwnerAgent + .get(`/projects/${ownerProject.id}/data-stores?filter={ "name": ["Store", "Test"]}`) + .expect(200); + + expect(response.body.data.count).toBe(1); + expect(response.body.data.data).toHaveLength(1); + expect(response.body.data.data[0].name).toBe('Test Store'); + }); + + test('should apply pagination with take parameter', async () => { + for (let i = 1; i <= 5; i++) { + await createDataStore(ownerProject, { + name: `Data Store ${i}`, + updatedAt: DateTime.now() + .minus({ minutes: 6 - i }) + .toJSDate(), + }); + } + + const response = await authOwnerAgent + .get(`/projects/${ownerProject.id}/data-stores`) + .query({ take: 3 }) + .expect(200); + + expect(response.body.data.count).toBe(5); // Total count should be 5 + expect(response.body.data.data).toHaveLength(3); // But only 3 returned + expect(response.body.data.data.map((store: DataStore) => store.name)).toEqual([ + 'Data Store 5', + 'Data Store 4', + 'Data Store 3', + ]); + }); + + test('should apply pagination with skip parameter', async () => { + for (let i = 1; i <= 5; i++) { + await createDataStore(ownerProject, { + name: `Data Store ${i}`, + updatedAt: DateTime.now() + .minus({ minutes: 6 - i }) + .toJSDate(), + }); + } + + const response = await authOwnerAgent + .get(`/projects/${ownerProject.id}/data-stores`) + .query({ skip: 2 }) + .expect(200); + + expect(response.body.data.count).toBe(5); + expect(response.body.data.data).toHaveLength(3); + expect(response.body.data.data.map((store: DataStore) => store.name)).toEqual([ + 'Data Store 3', + 'Data Store 2', + 'Data Store 1', + ]); + }); + + test('should apply combined skip and take parameters', async () => { + for (let i = 1; i <= 5; i++) { + await createDataStore(ownerProject, { + name: `Data Store ${i}`, + updatedAt: DateTime.now() + .minus({ minutes: 6 - i }) + .toJSDate(), + }); + } + + const response = await authOwnerAgent + .get(`/projects/${ownerProject.id}/data-stores`) + .query({ skip: 1, take: 2 }) + .expect(200); + + expect(response.body.data.count).toBe(5); + expect(response.body.data.data).toHaveLength(2); + expect(response.body.data.data.map((store: DataStore) => store.name)).toEqual([ + 'Data Store 4', + 'Data Store 3', + ]); + }); + + test('should sort data stores by name ascending', async () => { + await createDataStore(ownerProject, { name: 'Z Data Store' }); + await createDataStore(ownerProject, { name: 'A Data Store' }); + await createDataStore(ownerProject, { name: 'M Data Store' }); + + const response = await authOwnerAgent + .get(`/projects/${ownerProject.id}/data-stores`) + .query({ sortBy: 'name:asc' }) + .expect(200); + + expect(response.body.data.data.map((store: DataStore) => store.name)).toEqual([ + 'A Data Store', + 'M Data Store', + 'Z Data Store', + ]); + }); + + test('should sort data stores by name descending', async () => { + await createDataStore(ownerProject, { name: 'Z Data Store' }); + await createDataStore(ownerProject, { name: 'A Data Store' }); + await createDataStore(ownerProject, { name: 'M Data Store' }); + + const response = await authOwnerAgent + .get(`/projects/${ownerProject.id}/data-stores`) + .query({ sortBy: 'name:desc' }) + .expect(200); + + expect(response.body.data.data.map((f: DataStore) => f.name)).toEqual([ + 'Z Data Store', + 'M Data Store', + 'A Data Store', + ]); + }); + + test('should sort data stores by updatedAt', async () => { + await createDataStore(ownerProject, { + name: 'Older Data Store', + updatedAt: DateTime.now().minus({ days: 2 }).toJSDate(), + }); + await createDataStore(ownerProject, { + name: 'Newest Data Store', + updatedAt: DateTime.now().toJSDate(), + }); + await createDataStore(ownerProject, { + name: 'Middle Data Store', + updatedAt: DateTime.now().minus({ days: 1 }).toJSDate(), + }); + + const response = await authOwnerAgent + .get(`/projects/${ownerProject.id}/data-stores`) + .query({ sortBy: 'updatedAt:desc' }) + .expect(200); + + expect(response.body.data.data.map((f: DataStore) => f.name)).toEqual([ + 'Newest Data Store', + 'Middle Data Store', + 'Older Data Store', + ]); + }); + + test('should combine multiple query parameters correctly', async () => { + const dataStore1 = await createDataStore(ownerProject, { name: 'Test Data Store' }); + await createDataStore(ownerProject, { name: 'Another Data Store' }); + + const response = await authOwnerAgent + .get(`/projects/${ownerProject.id}/data-stores`) + .query({ filter: JSON.stringify({ name: 'data', id: dataStore1.id }), sortBy: 'name:asc' }) + .expect(200); + + expect(response.body.data.count).toBe(1); + expect(response.body.data.data).toHaveLength(1); + expect(response.body.data.data[0].name).toBe('Test Data Store'); + }); + + test('should include columns', async () => { + await createDataStore(ownerProject, { + name: 'Test Data Store', + columns: [ + { + name: 'test-column-1', + type: 'string', + }, + { + name: 'test-column-2', + type: 'boolean', + }, + ], + }); + + const response = await authOwnerAgent + .get(`/projects/${ownerProject.id}/data-stores`) + .query({ filter: JSON.stringify({ name: 'test' }) }) + .expect(200); + + expect(response.body.data.count).toBe(1); + expect(response.body.data.data).toHaveLength(1); + expect(response.body.data.data[0].columns).toHaveLength(2); + }); +}); diff --git a/packages/cli/src/modules/data-store/__tests__/data-store.service.test.ts b/packages/cli/src/modules/data-store/__tests__/data-store.service.test.ts index 9f17e60999..b5c050910d 100644 --- a/packages/cli/src/modules/data-store/__tests__/data-store.service.test.ts +++ b/packages/cli/src/modules/data-store/__tests__/data-store.service.test.ts @@ -100,24 +100,22 @@ describe('dataStore', () => { }); // ACT - const result = await dataStoreService.createDataStore(project1.id, { + const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, { name, columns: [], }); - const { id: dataStoreId } = result; const created = await dataStoreRepository.findOneBy({ name, projectId: project1.id }); expect(created?.id).toBe(dataStoreId); }); it('should create the user table and columns entity immediately if columns are provided', async () => { - const dataStore = await dataStoreService.createDataStore(project1.id, { + const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, { name: 'dataStoreWithColumns', columns: [{ name: 'foo', type: 'string' }], }); - const { id: dataStoreId } = dataStore; - await expect(dataStoreService.getColumns(dataStoreId)).resolves.toEqual([ + await expect(dataStoreService.getColumns(dataStoreId, project1.id)).resolves.toEqual([ { name: 'foo', type: 'string', @@ -148,13 +146,12 @@ describe('dataStore', () => { const name = 'dataStore'; // ACT - const result = await dataStoreService.createDataStore(project1.id, { + const { project } = await dataStoreService.createDataStore(project1.id, { name, columns: [], }); // ASSERT - const { project } = result; expect(project.id).toBe(project1.id); expect(project.name).toBe(project1.name); }); @@ -184,17 +181,18 @@ describe('dataStore', () => { describe('updateDataStore', () => { it('should succeed when renaming to an available name', async () => { // ARRANGE - const dataStore = await dataStoreService.createDataStore(project1.id, { + const { id: dataStoreId, updatedAt } = await dataStoreService.createDataStore(project1.id, { name: 'myDataStore1', columns: [], }); - const { id: dataStoreId, updatedAt } = dataStore; // ACT // Wait to get second difference await new Promise((resolve) => setTimeout(resolve, 1001)); - const result = await dataStoreService.updateDataStore(dataStoreId, { name: 'aNewName' }); + const result = await dataStoreService.updateDataStore(dataStoreId, project1.id, { + name: 'aNewName', + }); // ASSERT expect(result).toEqual(true); @@ -206,47 +204,12 @@ describe('dataStore', () => { it('should fail when renaming a non-existent data store', async () => { // ACT - const result = dataStoreService.updateDataStore('this is not an id', { + const result = dataStoreService.updateDataStore('this is not an id', project1.id, { name: 'aNewName', }); // ASSERT - await expect(result).rejects.toThrow( - "Tried to rename non-existent data store 'this is not an id'", - ); - }); - - it('should fail when renaming to an empty name', async () => { - // ARRANGE - const dataStore = await dataStoreService.createDataStore(project1.id, { - name: 'myDataStore', - columns: [], - }); - const { id: dataStoreId } = dataStore; - - // ACT - const result = dataStoreService.updateDataStore(dataStoreId, { name: '' }); - - // ASSERT - await expect(result).rejects.toThrow('Data store name must not be empty'); - }); - - it('should trim the name', async () => { - // ARRANGE - const dataStore = await dataStoreService.createDataStore(project1.id, { - name: 'myDataStore1', - columns: [], - }); - const { id: dataStoreId } = dataStore; - - // ACT - const result = dataStoreService.updateDataStore(dataStoreId, { name: ' aNewName ' }); - - // ASSERT - await expect(result).resolves.toEqual(true); - - const updated = await dataStoreRepository.findOneBy({ id: dataStoreId }); - expect(updated?.name).toBe('aNewName'); + await expect(result).rejects.toThrow("Data Store 'this is not an id' does not exist."); }); it('should fail when renaming to a taken name', async () => { @@ -257,18 +220,17 @@ describe('dataStore', () => { columns: [], }); - const dataStoreNew = await dataStoreService.createDataStore(project1.id, { + const { id: dataStoreNewId } = await dataStoreService.createDataStore(project1.id, { name: 'myDataStoreNew', columns: [], }); - const { id: dataStoreNewId } = dataStoreNew; // ACT - const result = dataStoreService.updateDataStore(dataStoreNewId, { name }); + const result = dataStoreService.updateDataStore(dataStoreNewId, project1.id, { name }); // ASSERT await expect(result).rejects.toThrow( - `The name '${name}' is already taken within this project`, + `Data store with name '${name}' already exists in this project`, ); }); }); @@ -276,14 +238,13 @@ describe('dataStore', () => { describe('deleteDataStore', () => { it('should succeed with deleting a store', async () => { // ARRANGE - const dataStore = await dataStoreService.createDataStore(project1.id, { + const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, { name: 'myDataStore1', columns: [], }); - const { id: dataStoreId } = dataStore; // ACT - const result = await dataStoreService.deleteDataStore(dataStoreId); + const result = await dataStoreService.deleteDataStore(dataStoreId, project1.id); const userTableName = toTableName(dataStoreId); // ASSERT @@ -302,7 +263,7 @@ describe('dataStore', () => { it('should fail when deleting a non-existent id', async () => { // ACT - const result = dataStoreService.deleteDataStore('this is not an id'); + const result = dataStoreService.deleteDataStore('this is not an id', project1.id); // ASSERT await expect(result).rejects.toThrow( @@ -315,11 +276,10 @@ describe('dataStore', () => { it('should succeed with adding columns to a non-empty table', async () => { const existingColumns: CreateDataStoreColumnDto[] = [{ name: 'myColumn0', type: 'string' }]; - const dataStore = await dataStoreService.createDataStore(project1.id, { + const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, { name: 'dataStoreWithColumns', columns: existingColumns, }); - const { id: dataStoreId } = dataStore; const columns: AddDataStoreColumnDto[] = [ { name: 'myColumn1', type: 'string' }, @@ -329,11 +289,11 @@ describe('dataStore', () => { ]; for (const column of columns) { // ACT - const result = await dataStoreService.addColumn(dataStoreId, column); + const result = await dataStoreService.addColumn(dataStoreId, project1.id, column); // ASSERT expect(result).toMatchObject(column); } - const columnResult = await dataStoreService.getColumns(dataStoreId); + const columnResult = await dataStoreService.getColumns(dataStoreId, project1.id); expect(columnResult).toEqual([ { index: 0, @@ -400,14 +360,13 @@ describe('dataStore', () => { it('should create the user table on first addColumn if it does not exist', async () => { // ARRANGE - const dataStore = await dataStoreService.createDataStore(project1.id, { + const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, { name: 'dataStore', columns: [], }); - const { id: dataStoreId } = dataStore; // ACT - const result = await dataStoreService.addColumn(dataStoreId, { + const result = await dataStoreService.addColumn(dataStoreId, project1.id, { name: 'foo', type: 'string', }); @@ -428,7 +387,7 @@ describe('dataStore', () => { it('should fail with adding two columns of the same name', async () => { // ARRANGE - const dataStore = await dataStoreService.createDataStore(project1.id, { + const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, { name: 'myDataStore', columns: [ { @@ -437,10 +396,9 @@ describe('dataStore', () => { }, ], }); - const { id: dataStoreId } = dataStore; // ACT - const result = dataStoreService.addColumn(dataStoreId, { + const result = dataStoreService.addColumn(dataStoreId, project1.id, { name: 'myColumn1', type: 'number', }); @@ -453,7 +411,7 @@ describe('dataStore', () => { it('should fail with adding column of non-existent table', async () => { // ACT - const result = dataStoreService.addColumn('this is not an id', { + const result = dataStoreService.addColumn('this is not an id', project1.id, { name: 'myColumn1', type: 'number', }); @@ -468,28 +426,27 @@ describe('dataStore', () => { describe('deleteColumn', () => { it('should succeed with deleting a column', async () => { // ARRANGE - const dataStore = await dataStoreService.createDataStore(project1.id, { + const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, { name: 'dataStore', columns: [], }); - const { id: dataStoreId } = dataStore; - const c1 = await dataStoreService.addColumn(dataStoreId, { + const c1 = await dataStoreService.addColumn(dataStoreId, project1.id, { name: 'myColumn1', type: 'string', }); - const c2 = await dataStoreService.addColumn(dataStoreId, { + const c2 = await dataStoreService.addColumn(dataStoreId, project1.id, { name: 'myColumn2', type: 'number', }); // ACT - const result = await dataStoreService.deleteColumn(dataStoreId, c1.id); + const result = await dataStoreService.deleteColumn(dataStoreId, project1.id, c1.id); // ASSERT expect(result).toEqual(true); - const columns = await dataStoreService.getColumns(dataStoreId); + const columns = await dataStoreService.getColumns(dataStoreId, project1.id); expect(columns).toEqual([ { index: 0, @@ -506,7 +463,7 @@ describe('dataStore', () => { it('should fail when deleting unknown column', async () => { // ARRANGE - const dataStore = await dataStoreService.createDataStore(project1.id, { + const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, { name: 'dataStore', columns: [ { @@ -515,10 +472,9 @@ describe('dataStore', () => { }, ], }); - const { id: dataStoreId } = dataStore; // ACT - const result = dataStoreService.deleteColumn(dataStoreId, 'thisIsNotAnId'); + const result = dataStoreService.deleteColumn(dataStoreId, project1.id, 'thisIsNotAnId'); // ASSERT await expect(result).rejects.toThrow( @@ -528,17 +484,17 @@ describe('dataStore', () => { it('should fail when deleting column from unknown table', async () => { // ARRANGE - const dataStore = await dataStoreService.createDataStore(project1.id, { + const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, { name: 'dataStore', columns: [], }); - const c1 = await dataStoreService.addColumn(dataStore.id, { + const c1 = await dataStoreService.addColumn(dataStoreId, project1.id, { name: 'myColumn1', type: 'string', }); // ACT - const result = dataStoreService.deleteColumn('this is not an id', c1.id); + const result = dataStoreService.deleteColumn('this is not an id', project1.id, c1.id); // ASSERT await expect(result).rejects.toThrow( @@ -550,30 +506,29 @@ describe('dataStore', () => { describe('moveColumn', () => { it('should succeed with moving a column', async () => { // ARRANGE - const dataStore = await dataStoreService.createDataStore(project1.id, { + const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, { name: 'dataStore', columns: [], }); - const { id: dataStoreId } = dataStore; - const c1 = await dataStoreService.addColumn(dataStoreId, { + const c1 = await dataStoreService.addColumn(dataStoreId, project1.id, { name: 'myColumn1', type: 'string', }); - const c2 = await dataStoreService.addColumn(dataStoreId, { + const c2 = await dataStoreService.addColumn(dataStoreId, project1.id, { name: 'myColumn2', type: 'number', }); // ACT - const result = await dataStoreService.moveColumn(dataStoreId, c2.id, { + const result = await dataStoreService.moveColumn(dataStoreId, project1.id, c2.id, { targetIndex: 0, }); // ASSERT expect(result).toEqual(true); - const columns = await dataStoreService.getColumns(dataStoreId); + const columns = await dataStoreService.getColumns(dataStoreId, project1.id); expect(columns).toMatchObject([ { index: 0, @@ -600,11 +555,10 @@ describe('dataStore', () => { name: 'dataStore', columns: [], }); - const { name } = dataStore; // ACT const result = await dataStoreService.getManyAndCount({ - filter: { projectId: project1.id, name }, + filter: { projectId: project1.id, name: dataStore.name }, }); // ASSERT @@ -629,17 +583,15 @@ describe('dataStore', () => { name: 'myDataStore1', columns: [], }); - const { id: dataStoreId1 } = dataStore1; const dataStore2 = await dataStoreService.createDataStore(project1.id, { name: 'myDataStore2', columns: [], }); - const { id: dataStoreId2 } = dataStore2; // ACT const result = await dataStoreService.getManyAndCount({ - filter: { projectId: project1.id, id: [dataStoreId1, dataStoreId2] }, + filter: { projectId: project1.id, id: [dataStore1.id, dataStore2.id] }, }); // ASSERT @@ -663,8 +615,7 @@ describe('dataStore', () => { name: 'myDataStore', columns: [], }); - const { name } = dataStore; - const names = [name]; + const names = [dataStore.name]; for (let i = 0; i < 10; ++i) { const ds = await dataStoreService.createDataStore(project1.id, { name: `anotherDataStore${i}`, @@ -689,8 +640,7 @@ describe('dataStore', () => { name: 'myDataStore', columns: [], }); - const { name } = dataStore; - const names = [name]; + const names = [dataStore.name]; for (let i = 0; i < 10; ++i) { const ds = await dataStoreService.createDataStore(project1.id, { @@ -744,11 +694,10 @@ describe('dataStore', () => { { name: 'myColumn3', type: 'number' }, { name: 'myColumn4', type: 'date' }, ]; - const dataStore = await dataStoreService.createDataStore(project1.id, { + const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, { name: 'myDataStore', columns, }); - const { id: dataStoreId } = dataStore; // ACT const result = await dataStoreService.getManyAndCount({ @@ -883,11 +832,10 @@ describe('dataStore', () => { it('sorts by updatedAt', async () => { // ARRANGE - const ds1 = await dataStoreService.createDataStore(project1.id, { + const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, { name: 'ds1', columns: [], }); - const { id: ds1Id } = ds1; // Wait to get seconds difference await new Promise((resolve) => setTimeout(resolve, 1001)); @@ -898,7 +846,7 @@ describe('dataStore', () => { // Wait to get seconds difference await new Promise((resolve) => setTimeout(resolve, 1001)); - await dataStoreService.updateDataStore(ds1Id, { name: 'ds1Updated' }); + await dataStoreService.updateDataStore(dataStoreId, project1.id, { name: 'ds1Updated' }); // ACT const updatedAsc = await dataStoreService.getManyAndCount({ @@ -921,7 +869,7 @@ describe('dataStore', () => { describe('insertRows', () => { it('inserts rows into an existing table', async () => { // ARRANGE - const dataStore = await dataStoreService.createDataStore(project1.id, { + const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, { name: 'dataStore', columns: [ { name: 'c1', type: 'number' }, @@ -930,7 +878,6 @@ describe('dataStore', () => { { name: 'c4', type: 'string' }, ], }); - const { id: dataStoreId } = dataStore; // ACT const rows = [ @@ -938,12 +885,16 @@ describe('dataStore', () => { { c1: 4, c2: false, c3: new Date(), c4: 'hello!' }, { c1: 5, c2: true, c3: new Date(), c4: 'hello.' }, ]; - const result = await dataStoreService.insertRows(dataStoreId, rows); + const result = await dataStoreService.insertRows(dataStoreId, project1.id, rows); // ASSERT expect(result).toBe(true); - const { count, data } = await dataStoreService.getManyRowsAndCount(dataStoreId, {}); + const { count, data } = await dataStoreService.getManyRowsAndCount( + dataStoreId, + project1.id, + {}, + ); expect(count).toEqual(3); expect(data).toEqual( rows.map((row, i) => ({ @@ -959,20 +910,21 @@ describe('dataStore', () => { it('inserts a row even if it matches with the existing one', async () => { // ARRANGE - const dataStore = await dataStoreService.createDataStore(project1.id, { + const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, { name: 'myDataStore', columns: [ { name: 'c1', type: 'number' }, { name: 'c2', type: 'string' }, ], }); - const { id: dataStoreId } = dataStore; // Insert initial row - await dataStoreService.insertRows(dataStoreId, [{ c1: 1, c2: 'foo' }]); + await dataStoreService.insertRows(dataStoreId, project1.id, [{ c1: 1, c2: 'foo' }]); // Attempt to insert a row with the same primary key - const result = await dataStoreService.insertRows(dataStoreId, [{ c1: 1, c2: 'foo' }]); + const result = await dataStoreService.insertRows(dataStoreId, project1.id, [ + { c1: 1, c2: 'foo' }, + ]); // ASSERT expect(result).toBe(true); @@ -991,7 +943,7 @@ describe('dataStore', () => { it('rejects a mismatched row with extra column', async () => { // ARRANGE - const dataStore = await dataStoreService.createDataStore(project1.id, { + const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, { name: 'dataStore', columns: [ { name: 'c1', type: 'number' }, @@ -1000,10 +952,9 @@ describe('dataStore', () => { { name: 'c4', type: 'string' }, ], }); - const { id: dataStoreId } = dataStore; // ACT - const result = dataStoreService.insertRows(dataStoreId, [ + const result = dataStoreService.insertRows(dataStoreId, project1.id, [ { c1: 3, c2: true, c3: new Date(), c4: 'hello?' }, { cWrong: 3, c1: 4, c2: true, c3: new Date(), c4: 'hello?' }, ]); @@ -1014,7 +965,7 @@ describe('dataStore', () => { it('rejects a mismatched row with missing column', async () => { // ARRANGE - const dataStore = await dataStoreService.createDataStore(project1.id, { + const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, { name: 'dataStore', columns: [ { name: 'c1', type: 'number' }, @@ -1023,10 +974,9 @@ describe('dataStore', () => { { name: 'c4', type: 'string' }, ], }); - const { id: dataStoreId } = dataStore; // ACT - const result = dataStoreService.insertRows(dataStoreId, [ + const result = dataStoreService.insertRows(dataStoreId, project1.id, [ { c1: 3, c2: true, c3: new Date(), c4: 'hello?' }, { c2: true, c3: new Date(), c4: 'hello?' }, ]); @@ -1037,7 +987,7 @@ describe('dataStore', () => { it('rejects a mismatched row with replaced column', async () => { // ARRANGE - const dataStore = await dataStoreService.createDataStore(project1.id, { + const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, { name: 'dataStore', columns: [ { name: 'c1', type: 'number' }, @@ -1046,10 +996,9 @@ describe('dataStore', () => { { name: 'c4', type: 'string' }, ], }); - const { id: dataStoreId } = dataStore; // ACT - const result = dataStoreService.insertRows(dataStoreId, [ + const result = dataStoreService.insertRows(dataStoreId, project1.id, [ { c1: 3, c2: true, c3: new Date(), c4: 'hello?' }, { cWrong: 3, c2: true, c3: new Date(), c4: 'hello?' }, ]); @@ -1071,20 +1020,24 @@ describe('dataStore', () => { }); // ACT - const result = dataStoreService.insertRows('this is not an id', [ + const result = dataStoreService.insertRows('this is not an id', project1.id, [ { c1: 3, c2: true, c3: new Date(), c4: 'hello?' }, { cWrong: 3, c2: true, c3: new Date(), c4: 'hello?' }, ]); // ASSERT - await expect(result).rejects.toThrow( - 'No columns found for this data store or data store not found', - ); + await expect(result).rejects.toThrow("Data Store 'this is not an id' does not exist."); }); it('rejects on empty column list', async () => { + // ARRANGE + const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, { + name: 'dataStore', + columns: [], + }); + // ACT - const result = dataStoreService.insertRows('this is not an id', [{}, {}]); + const result = dataStoreService.insertRows(dataStoreId, project1.id, [{}, {}]); // ASSERT await expect(result).rejects.toThrow( @@ -1094,14 +1047,16 @@ describe('dataStore', () => { it('fails on type mismatch', async () => { // ARRANGE - const dataStore = await dataStoreService.createDataStore(project1.id, { + const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, { name: 'dataStore', columns: [{ name: 'c1', type: 'number' }], }); - const { id: dataStoreId } = dataStore; // ACT - const result = dataStoreService.insertRows(dataStoreId, [{ c1: 3 }, { c1: true }]); + const result = dataStoreService.insertRows(dataStoreId, project1.id, [ + { c1: 3 }, + { c1: true }, + ]); // ASSERT await expect(result).rejects.toThrow("value 'true' does not match column type 'number'"); @@ -1111,7 +1066,7 @@ describe('dataStore', () => { describe('upsertRows', () => { it('updates a row if filter matches', async () => { // ARRANGE - const dataStore = await dataStoreService.createDataStore(project1.id, { + const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, { name: 'dataStore', columns: [ { name: 'pid', type: 'string' }, @@ -1119,15 +1074,14 @@ describe('dataStore', () => { { name: 'age', type: 'number' }, ], }); - const { id: dataStoreId } = dataStore; // Insert initial row - await dataStoreService.insertRows(dataStoreId, [ + await dataStoreService.insertRows(dataStoreId, project1.id, [ { pid: '1995-111a', fullName: 'Alice', age: 30 }, ]); // ACT - const result = await dataStoreService.upsertRows(dataStoreId, { + const result = await dataStoreService.upsertRows(dataStoreId, project1.id, { rows: [{ pid: '1995-111a', fullName: 'Alicia', age: 31 }], matchFields: ['pid'], }); @@ -1146,7 +1100,7 @@ describe('dataStore', () => { it('inserts a row if filter does not match', async () => { // ARRANGE - const dataStore = await dataStoreService.createDataStore(project1.id, { + const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, { name: 'dataStore', columns: [ { name: 'pid', type: 'string' }, @@ -1154,15 +1108,14 @@ describe('dataStore', () => { { name: 'age', type: 'number' }, ], }); - const { id: dataStoreId } = dataStore; // Insert initial row - await dataStoreService.insertRows(dataStoreId, [ + await dataStoreService.insertRows(dataStoreId, project1.id, [ { pid: '1995-111a', fullName: 'Alice', age: 30 }, ]); // ACT - const result = await dataStoreService.upsertRows(dataStoreId, { + const result = await dataStoreService.upsertRows(dataStoreId, project1.id, { rows: [{ pid: '1992-222b', fullName: 'Alice', age: 30 }], matchFields: ['pid'], }); @@ -1186,7 +1139,7 @@ describe('dataStore', () => { describe('getManyRowsAndCount', () => { it('retrieves rows correctly', async () => { // ARRANGE - const dataStore = await dataStoreService.createDataStore(project1.id, { + const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, { name: 'dataStore', columns: [ { name: 'c1', type: 'number' }, @@ -1195,7 +1148,6 @@ describe('dataStore', () => { { name: 'c4', type: 'string' }, ], }); - const { id: dataStoreId } = dataStore; const rows = [ { c1: 3, c2: true, c3: new Date(0), c4: 'hello?' }, @@ -1203,10 +1155,10 @@ describe('dataStore', () => { { c1: 5, c2: true, c3: new Date(2), c4: 'hello.' }, ]; - await dataStoreService.insertRows(dataStoreId, rows); + await dataStoreService.insertRows(dataStoreId, project1.id, rows); // ACT - const result = await dataStoreService.getManyRowsAndCount(dataStoreId, {}); + const result = await dataStoreService.getManyRowsAndCount(dataStoreId, project1.id, {}); // ASSERT expect(result.count).toEqual(3); diff --git a/packages/cli/src/modules/data-store/data-store.controller.ts b/packages/cli/src/modules/data-store/data-store.controller.ts index 0901638dcc..4ba9e3f90a 100644 --- a/packages/cli/src/modules/data-store/data-store.controller.ts +++ b/packages/cli/src/modules/data-store/data-store.controller.ts @@ -54,98 +54,98 @@ export class DataStoreController { @Patch('/:dataStoreId') @ProjectScope('dataStore:update') async updateDataStore( - _req: AuthenticatedRequest<{ projectId: string }>, + req: AuthenticatedRequest<{ projectId: string }>, _res: Response, @Param('dataStoreId') dataStoreId: string, @Body dto: UpdateDataStoreDto, ) { - return await this.dataStoreService.updateDataStore(dataStoreId, dto); + return await this.dataStoreService.updateDataStore(dataStoreId, req.params.projectId, dto); } @Delete('/:dataStoreId') @ProjectScope('dataStore:delete') async deleteDataStore( - _req: AuthenticatedRequest<{ projectId: string }>, + req: AuthenticatedRequest<{ projectId: string }>, _res: Response, @Param('dataStoreId') dataStoreId: string, ) { - return await this.dataStoreService.deleteDataStore(dataStoreId); + return await this.dataStoreService.deleteDataStore(dataStoreId, req.params.projectId); } @Get('/:dataStoreId/columns') @ProjectScope('dataStore:read') async getColumns( - _req: AuthenticatedRequest<{ projectId: string }>, + req: AuthenticatedRequest<{ projectId: string }>, _res: Response, @Param('dataStoreId') dataStoreId: string, ) { - return await this.dataStoreService.getColumns(dataStoreId); + return await this.dataStoreService.getColumns(dataStoreId, req.params.projectId); } @Post('/:dataStoreId/columns') @ProjectScope('dataStore:update') async addColumn( - _req: AuthenticatedRequest<{ projectId: string }>, + req: AuthenticatedRequest<{ projectId: string }>, _res: Response, @Param('dataStoreId') dataStoreId: string, @Body dto: AddDataStoreColumnDto, ) { - return await this.dataStoreService.addColumn(dataStoreId, dto); + return await this.dataStoreService.addColumn(dataStoreId, req.params.projectId, dto); } @Delete('/:dataStoreId/columns/:columnId') @ProjectScope('dataStore:update') async deleteColumn( - _req: AuthenticatedRequest<{ projectId: string }>, + req: AuthenticatedRequest<{ projectId: string }>, _res: Response, @Param('dataStoreId') dataStoreId: string, @Param('columnId') columnId: string, ) { - return await this.dataStoreService.deleteColumn(dataStoreId, columnId); + return await this.dataStoreService.deleteColumn(dataStoreId, req.params.projectId, columnId); } @Patch('/:dataStoreId/columns/:columnId/move') @ProjectScope('dataStore:update') async moveColumn( - _req: AuthenticatedRequest<{ projectId: string }>, + req: AuthenticatedRequest<{ projectId: string }>, _res: Response, @Param('dataStoreId') dataStoreId: string, @Param('columnId') columnId: string, @Body dto: MoveDataStoreColumnDto, ) { - return await this.dataStoreService.moveColumn(dataStoreId, columnId, dto); + return await this.dataStoreService.moveColumn(dataStoreId, req.params.projectId, columnId, dto); } @Get('/:dataStoreId/rows') @ProjectScope('dataStore:readRow') async getDataStoreRows( - _req: AuthenticatedRequest<{ projectId: string }>, + req: AuthenticatedRequest<{ projectId: string }>, _res: Response, @Param('dataStoreId') dataStoreId: string, @Query dto: ListDataStoreContentQueryDto, ) { - return await this.dataStoreService.getManyRowsAndCount(dataStoreId, dto); + return await this.dataStoreService.getManyRowsAndCount(dataStoreId, req.params.projectId, dto); } @Post('/:dataStoreId/insert') @ProjectScope('dataStore:writeRow') async appendDataStoreRows( - _req: AuthenticatedRequest<{ projectId: string }>, + req: AuthenticatedRequest<{ projectId: string }>, _res: Response, @Param('dataStoreId') dataStoreId: string, @Body dto: AddDataStoreRowsDto, ) { - return await this.dataStoreService.insertRows(dataStoreId, dto.data); + return await this.dataStoreService.insertRows(dataStoreId, req.params.projectId, dto.data); } @Post('/:dataStoreId/upsert') @ProjectScope('dataStore:writeRow') async upsertDataStoreRows( - _req: AuthenticatedRequest<{ projectId: string }>, + req: AuthenticatedRequest<{ projectId: string }>, _res: Response, @Param('dataStoreId') dataStoreId: string, @Body dto: UpsertDataStoreRowsDto, ) { - return await this.dataStoreService.upsertRows(dataStoreId, dto); + return await this.dataStoreService.upsertRows(dataStoreId, req.params.projectId, dto); } } diff --git a/packages/cli/src/modules/data-store/data-store.repository.ts b/packages/cli/src/modules/data-store/data-store.repository.ts index caf4f3f648..b94530b921 100644 --- a/packages/cli/src/modules/data-store/data-store.repository.ts +++ b/packages/cli/src/modules/data-store/data-store.repository.ts @@ -165,10 +165,13 @@ export class DataStoreRepository extends Repository { }); } - if (filter?.name && typeof filter.name === 'string') { - query.andWhere('LOWER(dataStore.name) LIKE LOWER(:name)', { - name: `%${filter.name}%`, - }); + if (filter?.name) { + const nameFilters = typeof filter.name === 'string' ? [filter.name] : filter.name; + for (const name of nameFilters) { + query.andWhere('LOWER(dataStore.name) LIKE LOWER(:name)', { + name: `%${name}%`, + }); + } } } @@ -193,7 +196,9 @@ export class DataStoreRepository extends Repository { direction: 'DESC' | 'ASC', ): void { if (field === 'name') { - query.orderBy('LOWER(dataStore.name)', direction); + query + .addSelect('LOWER(dataStore.name)', 'datastore_name_lower') + .orderBy('datastore_name_lower', direction); } else if (['createdAt', 'updatedAt'].includes(field)) { query.orderBy(`dataStore.${field}`, direction); } diff --git a/packages/cli/src/modules/data-store/data-store.service.ts b/packages/cli/src/modules/data-store/data-store.service.ts index cc2a86f413..b0169f2449 100644 --- a/packages/cli/src/modules/data-store/data-store.service.ts +++ b/packages/cli/src/modules/data-store/data-store.service.ts @@ -12,11 +12,13 @@ import { Logger } from '@n8n/backend-common'; import { Service } from '@n8n/di'; import { UserError } from 'n8n-workflow'; -import { DataStoreColumn } from './data-store-column.entity'; +import { ConflictError } from '@/errors/response-errors/conflict.error'; +import { NotFoundError } from '@/errors/response-errors/not-found.error'; + import { DataStoreColumnRepository } from './data-store-column.repository'; import { DataStoreRowsRepository } from './data-store-rows.repository'; import { DataStoreRepository } from './data-store.repository'; -import { toTableName } from './utils/sql-utils'; +import { toTableName, normalizeRows } from './utils/sql-utils'; @Service() export class DataStoreService { @@ -33,42 +35,17 @@ export class DataStoreService { async shutdown() {} async createDataStore(projectId: string, dto: CreateDataStoreDto) { - const existingTable = await this.dataStoreRepository.findOneBy({ - name: dto.name, - projectId, - }); - if (existingTable !== null) { - throw new UserError(`Data store with name '${dto.name}' already exists in this project`); - } + await this.validateUniqueName(dto.name, projectId); + return await this.dataStoreRepository.createDataStore(projectId, dto.name, dto.columns); } // Currently only renames data stores - async updateDataStore(dataStoreId: string, dto: UpdateDataStoreDto) { - const name = dto.name.trim(); + async updateDataStore(dataStoreId: string, projectId: string, dto: UpdateDataStoreDto) { + await this.validateDataStoreExists(dataStoreId, projectId); + await this.validateUniqueName(dto.name, projectId); - if (!name) { - throw new UserError('Data store name must not be empty'); - } - - const existingTable = await this.dataStoreRepository.findOneBy({ - id: dataStoreId, - }); - - if (existingTable === null) { - throw new UserError(`Tried to rename non-existent data store '${dataStoreId}'`); - } - - const hasNameClash = await this.dataStoreRepository.existsBy({ - name, - projectId: existingTable.projectId, - }); - - if (hasNameClash) { - throw new UserError(`The name '${name}' is already taken within this project`); - } - - await this.dataStoreRepository.update({ id: dataStoreId }, { name }); + await this.dataStoreRepository.update({ id: dataStoreId }, { name: dto.name }); return true; } @@ -81,9 +58,10 @@ export class DataStoreService { return await this.dataStoreRepository.deleteDataStoreAll(); } - async deleteDataStore(dataStoreId: string) { + async deleteDataStore(dataStoreId: string, projectId: string) { await this.validateDataStoreExists( dataStoreId, + projectId, `Tried to delete non-existent data store '${dataStoreId}'`, ); @@ -92,18 +70,25 @@ export class DataStoreService { return true; } - async addColumn(dataStoreId: string, dto: AddDataStoreColumnDto) { + async addColumn(dataStoreId: string, projectId: string, dto: AddDataStoreColumnDto) { await this.validateDataStoreExists( dataStoreId, + projectId, `Tried to add column to non-existent data store '${dataStoreId}'`, ); return await this.dataStoreColumnRepository.addColumn(dataStoreId, dto); } - async moveColumn(dataStoreId: string, columnId: string, dto: MoveDataStoreColumnDto) { + async moveColumn( + dataStoreId: string, + projectId: string, + columnId: string, + dto: MoveDataStoreColumnDto, + ) { await this.validateDataStoreExists( dataStoreId, + projectId, `Tried to move column from non-existent data store '${dataStoreId}'`, ); @@ -112,9 +97,10 @@ export class DataStoreService { return true; } - async deleteColumn(dataStoreId: string, columnId: string) { + async deleteColumn(dataStoreId: string, projectId: string, columnId: string) { await this.validateDataStoreExists( dataStoreId, + projectId, `Tried to delete column from non-existent data store '${dataStoreId}'`, ); @@ -138,7 +124,13 @@ export class DataStoreService { return await this.dataStoreRepository.getManyAndCount(options); } - async getManyRowsAndCount(dataStoreId: string, dto: ListDataStoreContentQueryDto) { + async getManyRowsAndCount( + dataStoreId: string, + projectId: string, + dto: ListDataStoreContentQueryDto, + ) { + await this.validateDataStoreExists(dataStoreId, projectId); + // unclear if we should validate here, only use case would be to reduce the chance of // a renamed/removed column appearing here (or added column missing) if the store was // modified between when the frontend sent the request and we received it @@ -149,66 +141,34 @@ export class DataStoreService { ); return { count: result.count, - data: this.normalizeRows(result.data, columns), + data: normalizeRows(result.data, columns), }; } - async getColumns(dataStoreId: string) { + async getColumns(dataStoreId: string, projectId: string) { + await this.validateDataStoreExists(dataStoreId, projectId); + return await this.dataStoreColumnRepository.getColumns(dataStoreId); } - async insertRows(dataStoreId: string, rows: DataStoreRows) { + async insertRows(dataStoreId: string, projectId: string, rows: DataStoreRows) { + await this.validateDataStoreExists(dataStoreId, projectId); await this.validateRows(dataStoreId, rows); + const columns = await this.dataStoreColumnRepository.getColumns(dataStoreId); return await this.dataStoreRowsRepository.insertRows(toTableName(dataStoreId), rows, columns); } - async upsertRows(dataStoreId: string, dto: UpsertDataStoreRowsDto) { + async upsertRows(dataStoreId: string, projectId: string, dto: UpsertDataStoreRowsDto) { + await this.validateDataStoreExists(dataStoreId, projectId); await this.validateRows(dataStoreId, dto.rows); + const columns = await this.dataStoreColumnRepository.getColumns(dataStoreId); return await this.dataStoreRowsRepository.upsertRows(toTableName(dataStoreId), dto, columns); } - // TODO: move to utils and test - private normalizeRows(rows: DataStoreRows, columns: DataStoreColumn[]): DataStoreRows { - const typeMap = new Map(columns.map((col) => [col.name, col.type])); - return rows.map((row) => { - const normalized = { ...row }; - for (const [key, value] of Object.entries(row)) { - const type = typeMap.get(key); - - if (type === 'boolean') { - // Convert boolean values to true/false - if (typeof value === 'boolean') { - normalized[key] = value; - } else if (value === 1 || value === '1') { - normalized[key] = true; - } else if (value === 0 || value === '0') { - normalized[key] = false; - } - } - if (type === 'date' && value !== null && value !== undefined) { - // Convert date objects or strings to ISO string - let dateObj: Date | null = null; - - if (value instanceof Date) { - dateObj = value; - } else if (typeof value === 'string' || typeof value === 'number') { - const parsed = new Date(value); - if (!isNaN(parsed.getTime())) { - dateObj = parsed; - } - } - - normalized[key] = dateObj ? dateObj.toISOString() : value; - } - } - return normalized; - }); - } - private async validateRows(dataStoreId: string, rows: DataStoreRows): Promise { const columns = await this.dataStoreColumnRepository.getColumns(dataStoreId); if (columns.length === 0) { @@ -253,13 +213,31 @@ export class DataStoreService { } } - private async validateDataStoreExists(dataStoreId: string, msg?: string) { + private async validateDataStoreExists(dataStoreId: string, projectId: string, msg?: string) { const existingTable = await this.dataStoreRepository.findOneBy({ id: dataStoreId, + project: { + id: projectId, + }, }); if (!existingTable) { - throw new UserError(msg ?? `Data Store '${dataStoreId}' does not exist.`); + throw new NotFoundError(msg ?? `Data Store '${dataStoreId}' does not exist.`); + } + + return existingTable; + } + + private async validateUniqueName(name: string, projectId: string, msg?: string) { + const hasNameClash = await this.dataStoreRepository.existsBy({ + name, + projectId, + }); + + if (hasNameClash) { + throw new ConflictError( + msg ?? `Data store with name '${name}' already exists in this project`, + ); } } } diff --git a/packages/cli/src/modules/data-store/utils/sql-utils.ts b/packages/cli/src/modules/data-store/utils/sql-utils.ts index 6e91ecba48..6becf3c5e6 100644 --- a/packages/cli/src/modules/data-store/utils/sql-utils.ts +++ b/packages/cli/src/modules/data-store/utils/sql-utils.ts @@ -2,15 +2,16 @@ import { DATA_STORE_COLUMN_REGEX, type DataStoreRows, type DataStoreCreateColumnSchema, + type DataStoreColumn, } from '@n8n/api-types'; import { DslColumn } from '@n8n/db'; import type { DataSourceOptions } from '@n8n/typeorm'; import { UnexpectedError } from 'n8n-workflow'; -import type { DataStoreUserTableName } from '../data-store.types'; - import { NotFoundError } from '@/errors/response-errors/not-found.error'; +import type { DataStoreUserTableName } from '../data-store.types'; + export function toDslColumns(columns: DataStoreCreateColumnSchema[]): DslColumn[] { return columns.map((col) => { const name = new DslColumn(col.name.trim()); @@ -204,6 +205,43 @@ export function toTableName(dataStoreId: string): DataStoreUserTableName { return `data_store_user_${dataStoreId}`; } +export function normalizeRows(rows: DataStoreRows, columns: DataStoreColumn[]) { + const typeMap = new Map(columns.map((col) => [col.name, col.type])); + return rows.map((row) => { + const normalized = { ...row }; + for (const [key, value] of Object.entries(row)) { + const type = typeMap.get(key); + + if (type === 'boolean') { + // Convert boolean values to true/false + if (typeof value === 'boolean') { + normalized[key] = value; + } else if (value === 1 || value === '1') { + normalized[key] = true; + } else if (value === 0 || value === '0') { + normalized[key] = false; + } + } + if (type === 'date' && value !== null && value !== undefined) { + // Convert date objects or strings to ISO string + let dateObj: Date | null = null; + + if (value instanceof Date) { + dateObj = value; + } else if (typeof value === 'string' || typeof value === 'number') { + const parsed = new Date(value); + if (!isNaN(parsed.getTime())) { + dateObj = parsed; + } + } + + normalized[key] = dateObj ? dateObj.toISOString() : value; + } + } + return normalized; + }); +} + function normalizeValue( value: unknown, columnType: string | undefined, diff --git a/packages/cli/test/integration/shared/db/data-stores.ts b/packages/cli/test/integration/shared/db/data-stores.ts new file mode 100644 index 0000000000..e0bb6902bc --- /dev/null +++ b/packages/cli/test/integration/shared/db/data-stores.ts @@ -0,0 +1,31 @@ +import type { CreateDataStoreColumnDto } from '@n8n/api-types'; +import { randomName } from '@n8n/backend-test-utils'; +import type { Project } from '@n8n/db'; +import { Container } from '@n8n/di'; + +import { DataStoreRepository } from '@/modules/data-store/data-store.repository'; + +export const createDataStore = async ( + project: Project, + options: { + name?: string; + columns?: CreateDataStoreColumnDto[]; + updatedAt?: Date; + } = {}, +) => { + const dataStoreRepository = Container.get(DataStoreRepository); + const dataStore = await dataStoreRepository.createDataStore( + project.id, + options.name ?? randomName(), + options.columns ?? [], + ); + + if (options.updatedAt) { + await dataStoreRepository.update(dataStore.id, { + updatedAt: options.updatedAt, + }); + dataStore.updatedAt = options.updatedAt; + } + + return dataStore; +}; diff --git a/packages/cli/test/integration/shared/utils/test-server.ts b/packages/cli/test/integration/shared/utils/test-server.ts index 4e48570160..7a25ae6557 100644 --- a/packages/cli/test/integration/shared/utils/test-server.ts +++ b/packages/cli/test/integration/shared/utils/test-server.ts @@ -1,5 +1,6 @@ import { LicenseState, ModuleRegistry } from '@n8n/backend-common'; import { mockInstance, mockLogger, testModules, testDb } from '@n8n/backend-test-utils'; +import { GlobalConfig } from '@n8n/config'; import type { APIRequest, User } from '@n8n/db'; import { Container } from '@n8n/di'; import cookieParser from 'cookie-parser'; @@ -21,7 +22,6 @@ import { LicenseMocker } from '@test-integration/license'; import { PUBLIC_API_REST_PATH_SEGMENT, REST_PATH_SEGMENT } from '../constants'; import type { SetupProps, TestServer } from '../types'; -import { GlobalConfig } from '@n8n/config'; /** * Plugin to prefix a path segment into a request URL pathname. @@ -186,12 +186,13 @@ export const setupTestServer = ({ await import('@/license/license.controller'); break; - case 'metrics': + case 'metrics': { const { PrometheusMetricsService } = await import( '@/metrics/prometheus-metrics.service' ); await Container.get(PrometheusMetricsService).init(app); break; + } case 'eventBus': await import('@/eventbus/event-bus.controller'); @@ -209,20 +210,22 @@ export const setupTestServer = ({ await import('@/controllers/mfa.controller'); break; - case 'ldap': + case 'ldap': { const { LdapService } = await import('@/ldap.ee/ldap.service.ee'); await import('@/ldap.ee/ldap.controller.ee'); testServer.license.enable('feat:ldap'); await Container.get(LdapService).init(); break; + } - case 'saml': + case 'saml': { const { SamlService } = await import('@/sso.ee/saml/saml.service.ee'); await Container.get(SamlService).init(); await import('@/sso.ee/saml/routes/saml.controller.ee'); const { setSamlLoginEnabled } = await import('@/sso.ee/saml/saml-helpers'); await setSamlLoginEnabled(true); break; + } case 'sourceControl': await import('@/environments.ee/source-control/source-control.controller.ee'); @@ -290,15 +293,22 @@ export const setupTestServer = ({ case 'ai': await import('@/controllers/ai.controller'); - + break; case 'folder': await import('@/controllers/folder.controller'); + break; case 'externalSecrets': await import('@/modules/external-secrets.ee/external-secrets.module'); + break; case 'insights': await import('@/modules/insights/insights.module'); + break; + + case 'data-store': + await import('@/modules/data-store/data-store.module'); + break; } }