mirror of
https://github.com/Abdulazizzn/n8n-enterprise-unlocked.git
synced 2025-12-16 17:46:45 +00:00
feat(core): Show data table per table storage usage (no-changelog) (#19480)
This commit is contained in:
@@ -4,12 +4,14 @@ import { createTeamProject, testDb, testModules } from '@n8n/backend-test-utils'
|
|||||||
import { Project } from '@n8n/db';
|
import { Project } from '@n8n/db';
|
||||||
import { Container } from '@n8n/di';
|
import { Container } from '@n8n/di';
|
||||||
|
|
||||||
|
import { mockDataStoreSizeValidator } from './test-helpers';
|
||||||
import { DataStoreService } from '../data-store.service';
|
import { DataStoreService } from '../data-store.service';
|
||||||
import { DataStoreValidationError } from '../errors/data-store-validation.error';
|
import { DataStoreValidationError } from '../errors/data-store-validation.error';
|
||||||
|
|
||||||
beforeAll(async () => {
|
beforeAll(async () => {
|
||||||
await testModules.loadModules(['data-table']);
|
await testModules.loadModules(['data-table']);
|
||||||
await testDb.init();
|
await testDb.init();
|
||||||
|
mockDataStoreSizeValidator();
|
||||||
});
|
});
|
||||||
|
|
||||||
beforeEach(async () => {
|
beforeEach(async () => {
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ describe('DataStoreSizeValidator', () => {
|
|||||||
|
|
||||||
describe('basic functionality', () => {
|
describe('basic functionality', () => {
|
||||||
it('should fetch size on first call', async () => {
|
it('should fetch size on first call', async () => {
|
||||||
fetchSizeFn.mockResolvedValue(50 * 1024 * 1024); // 50MB
|
fetchSizeFn.mockResolvedValue({ totalBytes: 50 * 1024 * 1024, dataTables: {} }); // 50MB
|
||||||
|
|
||||||
await validator.validateSize(fetchSizeFn, new Date('2024-01-01T00:00:00Z'));
|
await validator.validateSize(fetchSizeFn, new Date('2024-01-01T00:00:00Z'));
|
||||||
|
|
||||||
@@ -28,7 +28,7 @@ describe('DataStoreSizeValidator', () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it('should pass validation when size is under limit', async () => {
|
it('should pass validation when size is under limit', async () => {
|
||||||
fetchSizeFn.mockResolvedValue(50 * 1024 * 1024);
|
fetchSizeFn.mockResolvedValue({ totalBytes: 50 * 1024 * 1024, dataTables: {} });
|
||||||
|
|
||||||
await expect(
|
await expect(
|
||||||
validator.validateSize(fetchSizeFn, new Date('2024-01-01T00:00:00Z')),
|
validator.validateSize(fetchSizeFn, new Date('2024-01-01T00:00:00Z')),
|
||||||
@@ -36,7 +36,7 @@ describe('DataStoreSizeValidator', () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it('should throw error when size exceeds limit', async () => {
|
it('should throw error when size exceeds limit', async () => {
|
||||||
fetchSizeFn.mockResolvedValue(150 * 1024 * 1024);
|
fetchSizeFn.mockResolvedValue({ totalBytes: 150 * 1024 * 1024, dataTables: {} });
|
||||||
|
|
||||||
await expect(
|
await expect(
|
||||||
validator.validateSize(fetchSizeFn, new Date('2024-01-01T00:00:00Z')),
|
validator.validateSize(fetchSizeFn, new Date('2024-01-01T00:00:00Z')),
|
||||||
@@ -44,7 +44,7 @@ describe('DataStoreSizeValidator', () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it('should throw error when size equals limit', async () => {
|
it('should throw error when size equals limit', async () => {
|
||||||
fetchSizeFn.mockResolvedValue(100 * 1024 * 1024);
|
fetchSizeFn.mockResolvedValue({ totalBytes: 100 * 1024 * 1024, dataTables: {} });
|
||||||
|
|
||||||
await expect(
|
await expect(
|
||||||
validator.validateSize(fetchSizeFn, new Date('2024-01-01T00:00:00Z')),
|
validator.validateSize(fetchSizeFn, new Date('2024-01-01T00:00:00Z')),
|
||||||
@@ -54,7 +54,7 @@ describe('DataStoreSizeValidator', () => {
|
|||||||
|
|
||||||
describe('caching behavior', () => {
|
describe('caching behavior', () => {
|
||||||
it('should use cached value within cache duration', async () => {
|
it('should use cached value within cache duration', async () => {
|
||||||
fetchSizeFn.mockResolvedValue(50);
|
fetchSizeFn.mockResolvedValue({ totalBytes: 50 * 1024 * 1024, dataTables: {} });
|
||||||
const time1 = new Date('2024-01-01T00:00:00Z');
|
const time1 = new Date('2024-01-01T00:00:00Z');
|
||||||
const time2 = new Date('2024-01-01T00:00:00.500Z'); // 500ms later
|
const time2 = new Date('2024-01-01T00:00:00.500Z'); // 500ms later
|
||||||
|
|
||||||
@@ -65,7 +65,7 @@ describe('DataStoreSizeValidator', () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it('should refresh cache after cache duration expires', async () => {
|
it('should refresh cache after cache duration expires', async () => {
|
||||||
fetchSizeFn.mockResolvedValue(50 * 1024 * 1024);
|
fetchSizeFn.mockResolvedValue({ totalBytes: 50 * 1024 * 1024, dataTables: {} });
|
||||||
const time1 = new Date('2024-01-01T00:00:00Z');
|
const time1 = new Date('2024-01-01T00:00:00Z');
|
||||||
const time2 = new Date('2024-01-01T00:00:01.001Z'); // 1001ms later
|
const time2 = new Date('2024-01-01T00:00:01.001Z'); // 1001ms later
|
||||||
|
|
||||||
@@ -77,7 +77,7 @@ describe('DataStoreSizeValidator', () => {
|
|||||||
|
|
||||||
it('should always validate against cached value even without refresh', async () => {
|
it('should always validate against cached value even without refresh', async () => {
|
||||||
// First call: DB at 50MB
|
// First call: DB at 50MB
|
||||||
fetchSizeFn.mockResolvedValue(50 * 1024 * 1024);
|
fetchSizeFn.mockResolvedValue({ totalBytes: 50 * 1024 * 1024, dataTables: {} });
|
||||||
const time1 = new Date('2024-01-01T00:00:00Z');
|
const time1 = new Date('2024-01-01T00:00:00Z');
|
||||||
await validator.validateSize(fetchSizeFn, time1);
|
await validator.validateSize(fetchSizeFn, time1);
|
||||||
|
|
||||||
@@ -91,7 +91,7 @@ describe('DataStoreSizeValidator', () => {
|
|||||||
|
|
||||||
it('should fail validation once cached value shows full DB', async () => {
|
it('should fail validation once cached value shows full DB', async () => {
|
||||||
// First call: DB becomes full (100MB)
|
// First call: DB becomes full (100MB)
|
||||||
fetchSizeFn.mockResolvedValue(100 * 1024 * 1024);
|
fetchSizeFn.mockResolvedValue({ totalBytes: 100 * 1024 * 1024, dataTables: {} });
|
||||||
const time1 = new Date('2024-01-01T00:00:00Z');
|
const time1 = new Date('2024-01-01T00:00:00Z');
|
||||||
|
|
||||||
await expect(validator.validateSize(fetchSizeFn, time1)).rejects.toThrow(
|
await expect(validator.validateSize(fetchSizeFn, time1)).rejects.toThrow(
|
||||||
@@ -111,10 +111,12 @@ describe('DataStoreSizeValidator', () => {
|
|||||||
|
|
||||||
describe('concurrent calls', () => {
|
describe('concurrent calls', () => {
|
||||||
it('should handle concurrent calls correctly', async () => {
|
it('should handle concurrent calls correctly', async () => {
|
||||||
let resolveCheck: (value: number) => void;
|
let resolveCheck: (value: { totalBytes: number; dataTables: Record<string, number> }) => void;
|
||||||
const checkPromise = new Promise<number>((resolve) => {
|
const checkPromise = new Promise<{ totalBytes: number; dataTables: Record<string, number> }>(
|
||||||
resolveCheck = resolve;
|
(resolve) => {
|
||||||
});
|
resolveCheck = resolve;
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
fetchSizeFn.mockImplementation(async () => await checkPromise);
|
fetchSizeFn.mockImplementation(async () => await checkPromise);
|
||||||
|
|
||||||
@@ -129,7 +131,7 @@ describe('DataStoreSizeValidator', () => {
|
|||||||
await new Promise((resolve) => setImmediate(resolve));
|
await new Promise((resolve) => setImmediate(resolve));
|
||||||
|
|
||||||
// Resolve the check with a value under the limit
|
// Resolve the check with a value under the limit
|
||||||
resolveCheck!(50 * 1024 * 1024);
|
resolveCheck!({ totalBytes: 50 * 1024 * 1024, dataTables: {} });
|
||||||
|
|
||||||
await Promise.all([promise1, promise2, promise3]);
|
await Promise.all([promise1, promise2, promise3]);
|
||||||
|
|
||||||
@@ -138,10 +140,12 @@ describe('DataStoreSizeValidator', () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it('should share failure state among concurrent calls', async () => {
|
it('should share failure state among concurrent calls', async () => {
|
||||||
let resolveCheck: (value: number) => void;
|
let resolveCheck: (value: { totalBytes: number; dataTables: Record<string, number> }) => void;
|
||||||
const checkPromise = new Promise<number>((resolve) => {
|
const checkPromise = new Promise<{ totalBytes: number; dataTables: Record<string, number> }>(
|
||||||
resolveCheck = resolve;
|
(resolve) => {
|
||||||
});
|
resolveCheck = resolve;
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
fetchSizeFn.mockImplementation(async () => await checkPromise);
|
fetchSizeFn.mockImplementation(async () => await checkPromise);
|
||||||
|
|
||||||
@@ -153,7 +157,7 @@ describe('DataStoreSizeValidator', () => {
|
|||||||
const promise3 = validator.validateSize(fetchSizeFn, time);
|
const promise3 = validator.validateSize(fetchSizeFn, time);
|
||||||
|
|
||||||
// Resolve with size over limit
|
// Resolve with size over limit
|
||||||
resolveCheck!(150 * 1024 * 1024);
|
resolveCheck!({ totalBytes: 150 * 1024 * 1024, dataTables: {} });
|
||||||
|
|
||||||
// All should fail with the same error
|
// All should fail with the same error
|
||||||
await expect(promise1).rejects.toThrow(
|
await expect(promise1).rejects.toThrow(
|
||||||
@@ -173,7 +177,7 @@ describe('DataStoreSizeValidator', () => {
|
|||||||
|
|
||||||
describe('reset functionality', () => {
|
describe('reset functionality', () => {
|
||||||
it('should clear cache when reset is called', async () => {
|
it('should clear cache when reset is called', async () => {
|
||||||
fetchSizeFn.mockResolvedValue(50 * 1024 * 1024);
|
fetchSizeFn.mockResolvedValue({ totalBytes: 50 * 1024 * 1024, dataTables: {} });
|
||||||
const time1 = new Date('2024-01-01T00:00:00Z');
|
const time1 = new Date('2024-01-01T00:00:00Z');
|
||||||
|
|
||||||
// First call
|
// First call
|
||||||
@@ -202,7 +206,7 @@ describe('DataStoreSizeValidator', () => {
|
|||||||
// t=1500ms: Still within new cache window, uses cached 100MB, continues to FAIL
|
// t=1500ms: Still within new cache window, uses cached 100MB, continues to FAIL
|
||||||
|
|
||||||
// First check: DB at 99MB (under limit)
|
// First check: DB at 99MB (under limit)
|
||||||
fetchSizeFn.mockResolvedValueOnce(99 * 1024 * 1024);
|
fetchSizeFn.mockResolvedValueOnce({ totalBytes: 99 * 1024 * 1024, dataTables: {} });
|
||||||
const time1 = new Date('2024-01-01T00:00:00Z');
|
const time1 = new Date('2024-01-01T00:00:00Z');
|
||||||
await expect(validator.validateSize(fetchSizeFn, time1)).resolves.toBeUndefined();
|
await expect(validator.validateSize(fetchSizeFn, time1)).resolves.toBeUndefined();
|
||||||
|
|
||||||
@@ -212,7 +216,7 @@ describe('DataStoreSizeValidator', () => {
|
|||||||
await expect(validator.validateSize(fetchSizeFn, time2)).resolves.toBeUndefined();
|
await expect(validator.validateSize(fetchSizeFn, time2)).resolves.toBeUndefined();
|
||||||
|
|
||||||
// After cache expires: new check fetches current state showing DB is now full
|
// After cache expires: new check fetches current state showing DB is now full
|
||||||
fetchSizeFn.mockResolvedValueOnce(100 * 1024 * 1024);
|
fetchSizeFn.mockResolvedValueOnce({ totalBytes: 100 * 1024 * 1024, dataTables: {} });
|
||||||
const time3 = new Date('2024-01-01T00:00:01.001Z');
|
const time3 = new Date('2024-01-01T00:00:01.001Z');
|
||||||
await expect(validator.validateSize(fetchSizeFn, time3)).rejects.toThrow(
|
await expect(validator.validateSize(fetchSizeFn, time3)).rejects.toThrow(
|
||||||
'Data store size limit exceeded: 100MB used, limit is 100MB',
|
'Data store size limit exceeded: 100MB used, limit is 100MB',
|
||||||
|
|||||||
@@ -0,0 +1,198 @@
|
|||||||
|
import { createTeamProject, testDb, testModules } from '@n8n/backend-test-utils';
|
||||||
|
import { GlobalConfig } from '@n8n/config';
|
||||||
|
import type { Project } from '@n8n/db';
|
||||||
|
import { Container } from '@n8n/di';
|
||||||
|
|
||||||
|
import { DataStoreSizeValidator } from '../data-store-size-validator.service';
|
||||||
|
import { DataStoreRepository } from '../data-store.repository';
|
||||||
|
import { DataStoreService } from '../data-store.service';
|
||||||
|
import { DataStoreValidationError } from '../errors/data-store-validation.error';
|
||||||
|
|
||||||
|
beforeAll(async () => {
|
||||||
|
await testModules.loadModules(['data-table']);
|
||||||
|
await testDb.init();
|
||||||
|
});
|
||||||
|
|
||||||
|
beforeEach(async () => {
|
||||||
|
const dataStoreService = Container.get(DataStoreService);
|
||||||
|
await dataStoreService.deleteDataStoreAll();
|
||||||
|
await testDb.truncate(['DataTable', 'DataTableColumn']);
|
||||||
|
|
||||||
|
const dataStoreSizeValidator = Container.get(DataStoreSizeValidator);
|
||||||
|
dataStoreSizeValidator.reset();
|
||||||
|
});
|
||||||
|
|
||||||
|
afterAll(async () => {
|
||||||
|
await testDb.terminate();
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('Data Store Size Tests', () => {
|
||||||
|
let dataStoreService: DataStoreService;
|
||||||
|
let dataStoreRepository: DataStoreRepository;
|
||||||
|
|
||||||
|
beforeAll(() => {
|
||||||
|
dataStoreService = Container.get(DataStoreService);
|
||||||
|
dataStoreRepository = Container.get(DataStoreRepository);
|
||||||
|
});
|
||||||
|
|
||||||
|
let project1: Project;
|
||||||
|
|
||||||
|
beforeEach(async () => {
|
||||||
|
project1 = await createTeamProject();
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('size validation', () => {
|
||||||
|
it('should prevent insertRows when size limit exceeded', async () => {
|
||||||
|
// ARRANGE
|
||||||
|
const dataStoreSizeValidator = Container.get(DataStoreSizeValidator);
|
||||||
|
dataStoreSizeValidator.reset();
|
||||||
|
|
||||||
|
const maxSize = Container.get(GlobalConfig).dataTable.maxSize;
|
||||||
|
|
||||||
|
const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, {
|
||||||
|
name: 'dataStore',
|
||||||
|
columns: [{ name: 'data', type: 'string' }],
|
||||||
|
});
|
||||||
|
|
||||||
|
const mockFindDataTablesSize = jest
|
||||||
|
.spyOn(dataStoreRepository, 'findDataTablesSize')
|
||||||
|
.mockResolvedValue({ totalBytes: maxSize + 1, dataTables: {} });
|
||||||
|
|
||||||
|
// ACT & ASSERT
|
||||||
|
await expect(
|
||||||
|
dataStoreService.insertRows(dataStoreId, project1.id, [{ data: 'test' }]),
|
||||||
|
).rejects.toThrow(DataStoreValidationError);
|
||||||
|
|
||||||
|
expect(mockFindDataTablesSize).toHaveBeenCalled();
|
||||||
|
mockFindDataTablesSize.mockRestore();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should prevent updateRow when size limit exceeded', async () => {
|
||||||
|
// ARRANGE
|
||||||
|
const dataStoreSizeValidator = Container.get(DataStoreSizeValidator);
|
||||||
|
dataStoreSizeValidator.reset();
|
||||||
|
|
||||||
|
const maxSize = Container.get(GlobalConfig).dataTable.maxSize;
|
||||||
|
|
||||||
|
const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, {
|
||||||
|
name: 'dataStore',
|
||||||
|
columns: [{ name: 'data', type: 'string' }],
|
||||||
|
});
|
||||||
|
|
||||||
|
// Now mock the size check to be over limit
|
||||||
|
const mockFindDataTablesSize = jest
|
||||||
|
.spyOn(dataStoreRepository, 'findDataTablesSize')
|
||||||
|
.mockResolvedValue({ totalBytes: maxSize + 1, dataTables: {} });
|
||||||
|
|
||||||
|
// ACT & ASSERT
|
||||||
|
await expect(
|
||||||
|
dataStoreService.updateRow(dataStoreId, project1.id, {
|
||||||
|
filter: {
|
||||||
|
type: 'and',
|
||||||
|
filters: [{ columnName: 'id', condition: 'eq', value: 1 }],
|
||||||
|
},
|
||||||
|
data: { data: 'updated' },
|
||||||
|
}),
|
||||||
|
).rejects.toThrow(DataStoreValidationError);
|
||||||
|
|
||||||
|
expect(mockFindDataTablesSize).toHaveBeenCalled();
|
||||||
|
mockFindDataTablesSize.mockRestore();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should prevent upsertRow when size limit exceeded (insert case)', async () => {
|
||||||
|
// ARRANGE
|
||||||
|
|
||||||
|
const maxSize = Container.get(GlobalConfig).dataTable.maxSize;
|
||||||
|
|
||||||
|
const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, {
|
||||||
|
name: 'dataStore',
|
||||||
|
columns: [{ name: 'data', type: 'string' }],
|
||||||
|
});
|
||||||
|
|
||||||
|
const mockFindDataTablesSize = jest
|
||||||
|
.spyOn(dataStoreRepository, 'findDataTablesSize')
|
||||||
|
.mockResolvedValue({ totalBytes: maxSize + 1, dataTables: {} });
|
||||||
|
|
||||||
|
// ACT & ASSERT
|
||||||
|
await expect(
|
||||||
|
dataStoreService.upsertRow(dataStoreId, project1.id, {
|
||||||
|
filter: {
|
||||||
|
type: 'and',
|
||||||
|
filters: [{ columnName: 'data', condition: 'eq', value: 'nonexistent' }],
|
||||||
|
},
|
||||||
|
data: { data: 'new' },
|
||||||
|
}),
|
||||||
|
).rejects.toThrow(DataStoreValidationError);
|
||||||
|
|
||||||
|
expect(mockFindDataTablesSize).toHaveBeenCalled();
|
||||||
|
mockFindDataTablesSize.mockRestore();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('findDataTablesSize', () => {
|
||||||
|
it('should return size information for data tables', async () => {
|
||||||
|
// ARRANGE
|
||||||
|
const dataStore1 = await dataStoreService.createDataStore(project1.id, {
|
||||||
|
name: 'dataStore1',
|
||||||
|
columns: [{ name: 'data', type: 'string' }],
|
||||||
|
});
|
||||||
|
|
||||||
|
const dataStore2 = await dataStoreService.createDataStore(project1.id, {
|
||||||
|
name: 'dataStore2',
|
||||||
|
columns: [{ name: 'data', type: 'string' }],
|
||||||
|
});
|
||||||
|
|
||||||
|
const data = new Array(1000).fill(0).map((_, i) => ({ data: `test_data_${i}` }));
|
||||||
|
|
||||||
|
await dataStoreService.insertRows(dataStore1.id, project1.id, data);
|
||||||
|
|
||||||
|
await dataStoreService.insertRows(dataStore2.id, project1.id, [{ data: 'test' }]);
|
||||||
|
|
||||||
|
// ACT
|
||||||
|
const result = await dataStoreRepository.findDataTablesSize();
|
||||||
|
|
||||||
|
// ASSERT
|
||||||
|
expect(result).toBeDefined();
|
||||||
|
expect(result.totalBytes).toBeGreaterThan(0);
|
||||||
|
expect(result.dataTables).toBeDefined();
|
||||||
|
expect(Object.keys(result.dataTables)).toHaveLength(2);
|
||||||
|
|
||||||
|
expect(result.dataTables[dataStore1.id]).toBeGreaterThan(0);
|
||||||
|
expect(result.dataTables[dataStore2.id]).toBeGreaterThan(0);
|
||||||
|
|
||||||
|
expect(result.dataTables[dataStore1.id]).toBeGreaterThan(result.dataTables[dataStore2.id]);
|
||||||
|
|
||||||
|
// Total should be sum of individual tables
|
||||||
|
const expectedTotal = result.dataTables[dataStore1.id] + result.dataTables[dataStore2.id];
|
||||||
|
expect(result.totalBytes).toBe(expectedTotal);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should return empty result when no data tables exist', async () => {
|
||||||
|
// ACT
|
||||||
|
const result = await dataStoreRepository.findDataTablesSize();
|
||||||
|
|
||||||
|
// ASSERT
|
||||||
|
expect(result).toBeDefined();
|
||||||
|
expect(result.totalBytes).toBe(0);
|
||||||
|
expect(result.dataTables).toBeDefined();
|
||||||
|
expect(Object.keys(result.dataTables)).toHaveLength(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should handle data tables with no rows', async () => {
|
||||||
|
// ARRANGE
|
||||||
|
const dataStore = await dataStoreService.createDataStore(project1.id, {
|
||||||
|
name: 'emptyDataStore',
|
||||||
|
columns: [{ name: 'data', type: 'string' }],
|
||||||
|
});
|
||||||
|
|
||||||
|
// ACT
|
||||||
|
const result = await dataStoreRepository.findDataTablesSize();
|
||||||
|
|
||||||
|
// ASSERT
|
||||||
|
expect(result).toBeDefined();
|
||||||
|
expect(result.totalBytes).toBeGreaterThanOrEqual(0);
|
||||||
|
expect(result.dataTables).toBeDefined();
|
||||||
|
expect(result.dataTables[dataStore.id]).toBeGreaterThanOrEqual(0);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -21,6 +21,7 @@ import * as utils from '@test-integration/utils';
|
|||||||
import { DataStoreColumnRepository } from '../data-store-column.repository';
|
import { DataStoreColumnRepository } from '../data-store-column.repository';
|
||||||
import { DataStoreRowsRepository } from '../data-store-rows.repository';
|
import { DataStoreRowsRepository } from '../data-store-rows.repository';
|
||||||
import { DataStoreRepository } from '../data-store.repository';
|
import { DataStoreRepository } from '../data-store.repository';
|
||||||
|
import { mockDataStoreSizeValidator } from './test-helpers';
|
||||||
|
|
||||||
let owner: User;
|
let owner: User;
|
||||||
let member: User;
|
let member: User;
|
||||||
@@ -42,6 +43,7 @@ let dataStoreRowsRepository: DataStoreRowsRepository;
|
|||||||
|
|
||||||
beforeAll(async () => {
|
beforeAll(async () => {
|
||||||
await testDb.init();
|
await testDb.init();
|
||||||
|
mockDataStoreSizeValidator();
|
||||||
});
|
});
|
||||||
|
|
||||||
beforeEach(async () => {
|
beforeEach(async () => {
|
||||||
|
|||||||
@@ -1,15 +1,14 @@
|
|||||||
/* eslint-disable @typescript-eslint/no-unsafe-assignment */
|
/* eslint-disable @typescript-eslint/no-unsafe-assignment */
|
||||||
import type { AddDataStoreColumnDto, CreateDataStoreColumnDto } from '@n8n/api-types';
|
import type { AddDataStoreColumnDto, CreateDataStoreColumnDto } from '@n8n/api-types';
|
||||||
import { createTeamProject, testDb, testModules } from '@n8n/backend-test-utils';
|
import { createTeamProject, testDb, testModules } from '@n8n/backend-test-utils';
|
||||||
import { GlobalConfig } from '@n8n/config';
|
|
||||||
import type { Project } from '@n8n/db';
|
import type { Project } from '@n8n/db';
|
||||||
import { Container } from '@n8n/di';
|
import { Container } from '@n8n/di';
|
||||||
import type { DataStoreRow } from 'n8n-workflow';
|
import type { DataStoreRow } from 'n8n-workflow';
|
||||||
|
|
||||||
import { DataStoreRowsRepository } from '../data-store-rows.repository';
|
import { DataStoreRowsRepository } from '../data-store-rows.repository';
|
||||||
import { DataStoreSizeValidator } from '../data-store-size-validator.service';
|
|
||||||
import { DataStoreRepository } from '../data-store.repository';
|
import { DataStoreRepository } from '../data-store.repository';
|
||||||
import { DataStoreService } from '../data-store.service';
|
import { DataStoreService } from '../data-store.service';
|
||||||
|
import { mockDataStoreSizeValidator } from './test-helpers';
|
||||||
import { DataStoreColumnNameConflictError } from '../errors/data-store-column-name-conflict.error';
|
import { DataStoreColumnNameConflictError } from '../errors/data-store-column-name-conflict.error';
|
||||||
import { DataStoreColumnNotFoundError } from '../errors/data-store-column-not-found.error';
|
import { DataStoreColumnNotFoundError } from '../errors/data-store-column-not-found.error';
|
||||||
import { DataStoreNameConflictError } from '../errors/data-store-name-conflict.error';
|
import { DataStoreNameConflictError } from '../errors/data-store-name-conflict.error';
|
||||||
@@ -20,12 +19,11 @@ import { toTableName } from '../utils/sql-utils';
|
|||||||
beforeAll(async () => {
|
beforeAll(async () => {
|
||||||
await testModules.loadModules(['data-table']);
|
await testModules.loadModules(['data-table']);
|
||||||
await testDb.init();
|
await testDb.init();
|
||||||
|
mockDataStoreSizeValidator();
|
||||||
});
|
});
|
||||||
|
|
||||||
beforeEach(async () => {
|
beforeEach(async () => {
|
||||||
await testDb.truncate(['DataTable', 'DataTableColumn']);
|
await testDb.truncate(['DataTable', 'DataTableColumn']);
|
||||||
const dataStoreSizeValidator = Container.get(DataStoreSizeValidator);
|
|
||||||
dataStoreSizeValidator.reset();
|
|
||||||
});
|
});
|
||||||
|
|
||||||
afterAll(async () => {
|
afterAll(async () => {
|
||||||
@@ -2657,92 +2655,4 @@ describe('dataStore', () => {
|
|||||||
await expect(result).rejects.toThrow(DataStoreValidationError);
|
await expect(result).rejects.toThrow(DataStoreValidationError);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe('size validation', () => {
|
|
||||||
it('should prevent insertRows when size limit exceeded', async () => {
|
|
||||||
// ARRANGE
|
|
||||||
const dataStoreSizeValidator = Container.get(DataStoreSizeValidator);
|
|
||||||
dataStoreSizeValidator.reset();
|
|
||||||
|
|
||||||
const maxSize = Container.get(GlobalConfig).dataTable.maxSize;
|
|
||||||
|
|
||||||
const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, {
|
|
||||||
name: 'dataStore',
|
|
||||||
columns: [{ name: 'data', type: 'string' }],
|
|
||||||
});
|
|
||||||
|
|
||||||
const mockFindDataTablesSize = jest
|
|
||||||
.spyOn(dataStoreRepository, 'findDataTablesSize')
|
|
||||||
.mockResolvedValue(maxSize + 1);
|
|
||||||
|
|
||||||
// ACT & ASSERT
|
|
||||||
await expect(
|
|
||||||
dataStoreService.insertRows(dataStoreId, project1.id, [{ data: 'test' }]),
|
|
||||||
).rejects.toThrow(DataStoreValidationError);
|
|
||||||
|
|
||||||
expect(mockFindDataTablesSize).toHaveBeenCalled();
|
|
||||||
mockFindDataTablesSize.mockRestore();
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should prevent updateRow when size limit exceeded', async () => {
|
|
||||||
// ARRANGE
|
|
||||||
const dataStoreSizeValidator = Container.get(DataStoreSizeValidator);
|
|
||||||
dataStoreSizeValidator.reset();
|
|
||||||
|
|
||||||
const maxSize = Container.get(GlobalConfig).dataTable.maxSize;
|
|
||||||
|
|
||||||
const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, {
|
|
||||||
name: 'dataStore',
|
|
||||||
columns: [{ name: 'data', type: 'string' }],
|
|
||||||
});
|
|
||||||
|
|
||||||
// Now mock the size check to be over limit
|
|
||||||
const mockFindDataTablesSize = jest
|
|
||||||
.spyOn(dataStoreRepository, 'findDataTablesSize')
|
|
||||||
.mockResolvedValue(maxSize + 1);
|
|
||||||
|
|
||||||
// ACT & ASSERT
|
|
||||||
await expect(
|
|
||||||
dataStoreService.updateRow(dataStoreId, project1.id, {
|
|
||||||
filter: {
|
|
||||||
type: 'and',
|
|
||||||
filters: [{ columnName: 'id', condition: 'eq', value: 1 }],
|
|
||||||
},
|
|
||||||
data: { data: 'updated' },
|
|
||||||
}),
|
|
||||||
).rejects.toThrow(DataStoreValidationError);
|
|
||||||
|
|
||||||
expect(mockFindDataTablesSize).toHaveBeenCalled();
|
|
||||||
mockFindDataTablesSize.mockRestore();
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should prevent upsertRow when size limit exceeded (insert case)', async () => {
|
|
||||||
// ARRANGE
|
|
||||||
|
|
||||||
const maxSize = Container.get(GlobalConfig).dataTable.maxSize;
|
|
||||||
|
|
||||||
const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, {
|
|
||||||
name: 'dataStore',
|
|
||||||
columns: [{ name: 'data', type: 'string' }],
|
|
||||||
});
|
|
||||||
|
|
||||||
const mockFindDataTablesSize = jest
|
|
||||||
.spyOn(dataStoreRepository, 'findDataTablesSize')
|
|
||||||
.mockResolvedValue(maxSize + 1);
|
|
||||||
|
|
||||||
// ACT & ASSERT
|
|
||||||
await expect(
|
|
||||||
dataStoreService.upsertRow(dataStoreId, project1.id, {
|
|
||||||
filter: {
|
|
||||||
type: 'and',
|
|
||||||
filters: [{ columnName: 'data', condition: 'eq', value: 'nonexistent' }],
|
|
||||||
},
|
|
||||||
data: { data: 'new' },
|
|
||||||
}),
|
|
||||||
).rejects.toThrow(DataStoreValidationError);
|
|
||||||
|
|
||||||
expect(mockFindDataTablesSize).toHaveBeenCalled();
|
|
||||||
mockFindDataTablesSize.mockRestore();
|
|
||||||
});
|
|
||||||
});
|
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -0,0 +1,13 @@
|
|||||||
|
import { Container } from '@n8n/di';
|
||||||
|
|
||||||
|
import { DataStoreSizeValidator } from '../data-store-size-validator.service';
|
||||||
|
|
||||||
|
export function mockDataStoreSizeValidator() {
|
||||||
|
const sizeValidator = Container.get(DataStoreSizeValidator);
|
||||||
|
jest.spyOn(sizeValidator, 'validateSize').mockResolvedValue();
|
||||||
|
jest.spyOn(sizeValidator, 'getCachedSizeData').mockResolvedValue({
|
||||||
|
totalBytes: 50 * 1024 * 1024, // 50MB - under the default limit
|
||||||
|
dataTables: {},
|
||||||
|
});
|
||||||
|
return sizeValidator;
|
||||||
|
}
|
||||||
@@ -1,40 +1,46 @@
|
|||||||
import { GlobalConfig } from '@n8n/config';
|
import { GlobalConfig } from '@n8n/config';
|
||||||
import { Service } from '@n8n/di';
|
import { Service } from '@n8n/di';
|
||||||
|
|
||||||
import { DataStoreValidationError } from './errors/data-store-validation.error';
|
|
||||||
import { DataTableSizeStatus } from 'n8n-workflow';
|
import { DataTableSizeStatus } from 'n8n-workflow';
|
||||||
|
|
||||||
|
import { DataTablesSizeData } from './data-store.types';
|
||||||
|
import { DataStoreValidationError } from './errors/data-store-validation.error';
|
||||||
|
|
||||||
@Service()
|
@Service()
|
||||||
export class DataStoreSizeValidator {
|
export class DataStoreSizeValidator {
|
||||||
private lastCheck: Date | undefined;
|
private lastCheck: Date | undefined;
|
||||||
private cachedSizeInBytes: number | undefined;
|
private cachedSizeData: DataTablesSizeData | undefined;
|
||||||
private pendingCheck: Promise<number> | null = null;
|
private pendingCheck: Promise<DataTablesSizeData> | null = null;
|
||||||
|
|
||||||
constructor(private readonly globalConfig: GlobalConfig) {}
|
constructor(private readonly globalConfig: GlobalConfig) {}
|
||||||
|
|
||||||
private shouldRefresh(sizeInBytes: number | undefined, now: Date): sizeInBytes is undefined {
|
private shouldRefresh(
|
||||||
|
cachedData: DataTablesSizeData | undefined,
|
||||||
|
now: Date,
|
||||||
|
): cachedData is undefined {
|
||||||
if (
|
if (
|
||||||
!this.lastCheck ||
|
!this.lastCheck ||
|
||||||
|
!cachedData ||
|
||||||
now.getTime() - this.lastCheck.getTime() >= this.globalConfig.dataTable.sizeCheckCacheDuration
|
now.getTime() - this.lastCheck.getTime() >= this.globalConfig.dataTable.sizeCheckCacheDuration
|
||||||
) {
|
) {
|
||||||
sizeInBytes = undefined;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
return sizeInBytes === undefined;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
async getCachedSize(fetchSizeFn: () => Promise<number>, now = new Date()): Promise<number> {
|
async getCachedSizeData(
|
||||||
|
fetchSizeDataFn: () => Promise<DataTablesSizeData>,
|
||||||
|
now = new Date(),
|
||||||
|
): Promise<DataTablesSizeData> {
|
||||||
// If there's a pending check, wait for it to complete
|
// If there's a pending check, wait for it to complete
|
||||||
|
|
||||||
if (this.pendingCheck) {
|
if (this.pendingCheck) {
|
||||||
this.cachedSizeInBytes = await this.pendingCheck;
|
this.cachedSizeData = await this.pendingCheck;
|
||||||
} else {
|
} else {
|
||||||
// Check if we need to refresh the db size
|
// Check if we need to refresh the size data
|
||||||
|
if (this.shouldRefresh(this.cachedSizeData, now)) {
|
||||||
if (this.shouldRefresh(this.cachedSizeInBytes, now)) {
|
this.pendingCheck = fetchSizeDataFn();
|
||||||
this.pendingCheck = fetchSizeFn();
|
|
||||||
try {
|
try {
|
||||||
this.cachedSizeInBytes = await this.pendingCheck;
|
this.cachedSizeData = await this.pendingCheck;
|
||||||
this.lastCheck = now;
|
this.lastCheck = now;
|
||||||
} finally {
|
} finally {
|
||||||
this.pendingCheck = null;
|
this.pendingCheck = null;
|
||||||
@@ -42,14 +48,17 @@ export class DataStoreSizeValidator {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return this.cachedSizeInBytes;
|
return this.cachedSizeData;
|
||||||
}
|
}
|
||||||
|
|
||||||
async validateSize(fetchSizeFn: () => Promise<number>, now = new Date()): Promise<void> {
|
async validateSize(
|
||||||
const size = await this.getCachedSize(fetchSizeFn, now);
|
fetchSizeFn: () => Promise<DataTablesSizeData>,
|
||||||
if (size >= this.globalConfig.dataTable.maxSize) {
|
now = new Date(),
|
||||||
|
): Promise<void> {
|
||||||
|
const size = await this.getCachedSizeData(fetchSizeFn, now);
|
||||||
|
if (size.totalBytes >= this.globalConfig.dataTable.maxSize) {
|
||||||
throw new DataStoreValidationError(
|
throw new DataStoreValidationError(
|
||||||
`Data store size limit exceeded: ${this.toMb(size)}MB used, limit is ${this.toMb(this.globalConfig.dataTable.maxSize)}MB`,
|
`Data store size limit exceeded: ${this.toMb(size.totalBytes)}MB used, limit is ${this.toMb(this.globalConfig.dataTable.maxSize)}MB`,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -63,9 +72,9 @@ export class DataStoreSizeValidator {
|
|||||||
return 'ok';
|
return 'ok';
|
||||||
}
|
}
|
||||||
|
|
||||||
async getSizeStatus(fetchSizeFn: () => Promise<number>, now = new Date()) {
|
async getSizeStatus(fetchSizeFn: () => Promise<DataTablesSizeData>, now = new Date()) {
|
||||||
const size = await this.getCachedSize(fetchSizeFn, now);
|
const size = await this.getCachedSizeData(fetchSizeFn, now);
|
||||||
return this.sizeToState(size);
|
return this.sizeToState(size.totalBytes);
|
||||||
}
|
}
|
||||||
|
|
||||||
private toMb(sizeInBytes: number): number {
|
private toMb(sizeInBytes: number): number {
|
||||||
@@ -74,7 +83,7 @@ export class DataStoreSizeValidator {
|
|||||||
|
|
||||||
reset() {
|
reset() {
|
||||||
this.lastCheck = undefined;
|
this.lastCheck = undefined;
|
||||||
this.cachedSizeInBytes = undefined;
|
this.cachedSizeData = undefined;
|
||||||
this.pendingCheck = null;
|
this.pendingCheck = null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -9,9 +9,10 @@ import { DataSource, EntityManager, Repository, SelectQueryBuilder } from '@n8n/
|
|||||||
import { UnexpectedError } from 'n8n-workflow';
|
import { UnexpectedError } from 'n8n-workflow';
|
||||||
|
|
||||||
import { DataStoreRowsRepository } from './data-store-rows.repository';
|
import { DataStoreRowsRepository } from './data-store-rows.repository';
|
||||||
|
import { DataStoreUserTableName, DataTablesSizeData } from './data-store.types';
|
||||||
import { DataTableColumn } from './data-table-column.entity';
|
import { DataTableColumn } from './data-table-column.entity';
|
||||||
import { DataTable } from './data-table.entity';
|
import { DataTable } from './data-table.entity';
|
||||||
import { toTableName } from './utils/sql-utils';
|
import { toTableId, toTableName } from './utils/sql-utils';
|
||||||
|
|
||||||
@Service()
|
@Service()
|
||||||
export class DataStoreRepository extends Repository<DataTable> {
|
export class DataStoreRepository extends Repository<DataTable> {
|
||||||
@@ -240,49 +241,83 @@ export class DataStoreRepository extends Repository<DataTable> {
|
|||||||
return [`${alias}.id`, `${alias}.name`, `${alias}.type`, `${alias}.icon`];
|
return [`${alias}.id`, `${alias}.name`, `${alias}.type`, `${alias}.icon`];
|
||||||
}
|
}
|
||||||
|
|
||||||
async findDataTablesSize(): Promise<number> {
|
private parseSize = (bytes: number | string | null): number =>
|
||||||
|
bytes === null ? 0 : typeof bytes === 'string' ? parseInt(bytes, 10) : bytes;
|
||||||
|
|
||||||
|
async findDataTablesSize(): Promise<DataTablesSizeData> {
|
||||||
const dbType = this.globalConfig.database.type;
|
const dbType = this.globalConfig.database.type;
|
||||||
const schemaName = this.globalConfig.database.postgresdb.schema;
|
const tablePattern = toTableName('%');
|
||||||
|
|
||||||
let sql = '';
|
let sql = '';
|
||||||
|
|
||||||
switch (dbType) {
|
switch (dbType) {
|
||||||
case 'sqlite':
|
case 'sqlite':
|
||||||
sql = `
|
sql = `
|
||||||
SELECT SUM(pgsize) AS total_bytes
|
SELECT name AS table_name, SUM(pgsize) AS table_bytes
|
||||||
FROM dbstat
|
FROM dbstat
|
||||||
WHERE name LIKE '${toTableName('%')}'
|
WHERE name LIKE '${tablePattern}'
|
||||||
`;
|
GROUP BY name
|
||||||
|
`;
|
||||||
break;
|
break;
|
||||||
|
|
||||||
case 'postgresdb':
|
case 'postgresdb': {
|
||||||
|
const schemaName = this.globalConfig.database.postgresdb?.schema;
|
||||||
sql = `
|
sql = `
|
||||||
SELECT SUM(pg_relation_size(c.oid)) AS total_bytes
|
SELECT c.relname AS table_name, pg_relation_size(c.oid) AS table_bytes
|
||||||
FROM pg_class c
|
FROM pg_class c
|
||||||
JOIN pg_namespace n ON n.oid = c.relnamespace
|
JOIN pg_namespace n ON n.oid = c.relnamespace
|
||||||
WHERE n.nspname = '${schemaName}'
|
WHERE n.nspname = '${schemaName}'
|
||||||
AND c.relname LIKE '${toTableName('%')}'
|
AND c.relname LIKE '${tablePattern}'
|
||||||
AND c.relkind IN ('r', 'm', 'p')
|
AND c.relkind IN ('r', 'm', 'p')
|
||||||
`;
|
`;
|
||||||
break;
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
case 'mysqldb':
|
case 'mysqldb':
|
||||||
case 'mariadb': {
|
case 'mariadb': {
|
||||||
const databaseName = this.globalConfig.database.mysqldb.database;
|
const databaseName = this.globalConfig.database.mysqldb.database;
|
||||||
|
const isMariaDb = dbType === 'mariadb';
|
||||||
|
const innodbTables = isMariaDb ? 'INNODB_SYS_TABLES' : 'INNODB_TABLES';
|
||||||
|
const innodbTablespaces = isMariaDb ? 'INNODB_SYS_TABLESPACES' : 'INNODB_TABLESPACES';
|
||||||
sql = `
|
sql = `
|
||||||
SELECT SUM((DATA_LENGTH + INDEX_LENGTH)) AS total_bytes
|
SELECT t.TABLE_NAME AS table_name,
|
||||||
FROM information_schema.tables
|
COALESCE(
|
||||||
WHERE table_schema = '${databaseName}'
|
(
|
||||||
AND table_name LIKE '${toTableName('%')}'
|
SELECT SUM(ists.ALLOCATED_SIZE)
|
||||||
`;
|
FROM information_schema.${innodbTables} ist
|
||||||
|
JOIN information_schema.${innodbTablespaces} ists
|
||||||
|
ON ists.SPACE = ist.SPACE
|
||||||
|
WHERE ist.NAME = CONCAT(t.TABLE_SCHEMA, '/', t.TABLE_NAME)
|
||||||
|
),
|
||||||
|
(t.DATA_LENGTH + t.INDEX_LENGTH)
|
||||||
|
) AS table_bytes
|
||||||
|
FROM information_schema.TABLES t
|
||||||
|
WHERE t.TABLE_SCHEMA = '${databaseName}'
|
||||||
|
AND t.TABLE_NAME LIKE '${tablePattern}'
|
||||||
|
`;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
default:
|
default:
|
||||||
return 0;
|
return { totalBytes: 0, dataTables: {} };
|
||||||
}
|
}
|
||||||
|
|
||||||
const result = (await this.query(sql)) as Array<{ total_bytes: number | null }>;
|
const result = (await this.query(sql)) as Array<{
|
||||||
return result[0]?.total_bytes ?? 0;
|
table_name: string;
|
||||||
|
table_bytes: number | string | null;
|
||||||
|
}>;
|
||||||
|
|
||||||
|
return result
|
||||||
|
.filter((row) => row.table_bytes !== null && row.table_name)
|
||||||
|
.reduce(
|
||||||
|
(acc, row) => {
|
||||||
|
const dataStoreId = toTableId(row.table_name as DataStoreUserTableName);
|
||||||
|
const sizeBytes = this.parseSize(row.table_bytes);
|
||||||
|
acc.dataTables[dataStoreId] = (acc.dataTables[dataStoreId] ?? 0) + sizeBytes;
|
||||||
|
acc.totalBytes += sizeBytes;
|
||||||
|
return acc;
|
||||||
|
},
|
||||||
|
{ dataTables: {} as Record<string, number>, totalBytes: 0 },
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -453,12 +453,13 @@ export class DataStoreService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async getDataTablesSize() {
|
async getDataTablesSize() {
|
||||||
const sizeBytes = await this.dataStoreSizeValidator.getCachedSize(
|
const sizeData = await this.dataStoreSizeValidator.getCachedSizeData(
|
||||||
async () => await this.dataStoreRepository.findDataTablesSize(),
|
async () => await this.dataStoreRepository.findDataTablesSize(),
|
||||||
);
|
);
|
||||||
return {
|
return {
|
||||||
sizeBytes,
|
sizeBytes: sizeData.totalBytes,
|
||||||
sizeState: this.dataStoreSizeValidator.sizeToState(sizeBytes),
|
sizeState: this.dataStoreSizeValidator.sizeToState(sizeData.totalBytes),
|
||||||
|
dataTables: sizeData.dataTables,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,6 +2,11 @@ import type { FieldTypeMap } from 'n8n-workflow';
|
|||||||
|
|
||||||
export type DataStoreUserTableName = `${string}data_table_user_${string}`;
|
export type DataStoreUserTableName = `${string}data_table_user_${string}`;
|
||||||
|
|
||||||
|
export type DataTablesSizeData = {
|
||||||
|
totalBytes: number;
|
||||||
|
dataTables: Record<string, number>;
|
||||||
|
};
|
||||||
|
|
||||||
export const columnTypeToFieldType: Record<string, keyof FieldTypeMap> = {
|
export const columnTypeToFieldType: Record<string, keyof FieldTypeMap> = {
|
||||||
// eslint-disable-next-line id-denylist
|
// eslint-disable-next-line id-denylist
|
||||||
number: 'number',
|
number: 'number',
|
||||||
|
|||||||
@@ -293,3 +293,7 @@ export function toTableName(dataStoreId: string): DataStoreUserTableName {
|
|||||||
const { tablePrefix } = Container.get(GlobalConfig).database;
|
const { tablePrefix } = Container.get(GlobalConfig).database;
|
||||||
return `${tablePrefix}data_table_user_${dataStoreId}`;
|
return `${tablePrefix}data_table_user_${dataStoreId}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function toTableId(tableName: DataStoreUserTableName) {
|
||||||
|
return tableName.replace(/.*data_table_user_/, '');
|
||||||
|
}
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import { DATA_STORE_DETAILS } from '@/features/dataStore/constants';
|
|||||||
import { useI18n } from '@n8n/i18n';
|
import { useI18n } from '@n8n/i18n';
|
||||||
import { computed } from 'vue';
|
import { computed } from 'vue';
|
||||||
import DataStoreActions from '@/features/dataStore/components/DataStoreActions.vue';
|
import DataStoreActions from '@/features/dataStore/components/DataStoreActions.vue';
|
||||||
|
import { useDataStoreStore } from '@/features/dataStore/dataStore.store';
|
||||||
|
|
||||||
type Props = {
|
type Props = {
|
||||||
dataStore: DataStoreResource;
|
dataStore: DataStoreResource;
|
||||||
@@ -12,6 +13,7 @@ type Props = {
|
|||||||
};
|
};
|
||||||
|
|
||||||
const i18n = useI18n();
|
const i18n = useI18n();
|
||||||
|
const dataStoreStore = useDataStoreStore();
|
||||||
|
|
||||||
const props = withDefaults(defineProps<Props>(), {
|
const props = withDefaults(defineProps<Props>(), {
|
||||||
actions: () => [],
|
actions: () => [],
|
||||||
@@ -28,6 +30,11 @@ const dataStoreRoute = computed(() => {
|
|||||||
},
|
},
|
||||||
};
|
};
|
||||||
});
|
});
|
||||||
|
|
||||||
|
const getDataStoreSize = computed(() => {
|
||||||
|
const size = dataStoreStore.dataStoreSizes[props.dataStore.id] ?? 0;
|
||||||
|
return size;
|
||||||
|
});
|
||||||
</script>
|
</script>
|
||||||
<template>
|
<template>
|
||||||
<div data-test-id="data-store-card">
|
<div data-test-id="data-store-card">
|
||||||
@@ -54,6 +61,18 @@ const dataStoreRoute = computed(() => {
|
|||||||
</template>
|
</template>
|
||||||
<template #footer>
|
<template #footer>
|
||||||
<div :class="$style['card-footer']">
|
<div :class="$style['card-footer']">
|
||||||
|
<N8nText
|
||||||
|
size="small"
|
||||||
|
color="text-light"
|
||||||
|
:class="[$style['info-cell'], $style['info-cell--size']]"
|
||||||
|
data-test-id="data-store-card-size"
|
||||||
|
>
|
||||||
|
{{
|
||||||
|
i18n.baseText('dataStore.card.size', {
|
||||||
|
interpolate: { size: getDataStoreSize },
|
||||||
|
})
|
||||||
|
}}
|
||||||
|
</N8nText>
|
||||||
<N8nText
|
<N8nText
|
||||||
size="small"
|
size="small"
|
||||||
color="text-light"
|
color="text-light"
|
||||||
@@ -143,7 +162,8 @@ const dataStoreRoute = computed(() => {
|
|||||||
flex-wrap: wrap;
|
flex-wrap: wrap;
|
||||||
}
|
}
|
||||||
.info-cell--created,
|
.info-cell--created,
|
||||||
.info-cell--column-count {
|
.info-cell--column-count,
|
||||||
|
.info-cell--size {
|
||||||
display: none;
|
display: none;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -35,11 +35,24 @@ export const useDataStoreStore = defineStore(DATA_STORE_STORE, () => {
|
|||||||
const totalCount = ref(0);
|
const totalCount = ref(0);
|
||||||
const dataStoreSize = ref(0);
|
const dataStoreSize = ref(0);
|
||||||
const dataStoreSizeLimitState = ref<DataTableSizeStatus>('ok');
|
const dataStoreSizeLimitState = ref<DataTableSizeStatus>('ok');
|
||||||
|
const dataStoreTableSizes = ref<Record<string, number>>({});
|
||||||
|
|
||||||
|
const formatSize = (sizeBytes: number) => {
|
||||||
|
return Number((sizeBytes / 1024 / 1024).toFixed(2));
|
||||||
|
};
|
||||||
|
|
||||||
const maxSizeMB = computed(() =>
|
const maxSizeMB = computed(() =>
|
||||||
Math.floor(settingsStore.settings?.dataTables?.maxSize / 1024 / 1024),
|
Math.floor(settingsStore.settings?.dataTables?.maxSize / 1024 / 1024),
|
||||||
);
|
);
|
||||||
|
|
||||||
|
const dataStoreSizes = computed(() => {
|
||||||
|
const formattedSizes: Record<string, number> = {};
|
||||||
|
for (const [dataStoreId, sizeBytes] of Object.entries(dataStoreTableSizes.value)) {
|
||||||
|
formattedSizes[dataStoreId] = formatSize(sizeBytes);
|
||||||
|
}
|
||||||
|
return formattedSizes;
|
||||||
|
});
|
||||||
|
|
||||||
const fetchDataStores = async (projectId: string, page: number, pageSize: number) => {
|
const fetchDataStores = async (projectId: string, page: number, pageSize: number) => {
|
||||||
const response = await fetchDataStoresApi(rootStore.restApiContext, projectId, {
|
const response = await fetchDataStoresApi(rootStore.restApiContext, projectId, {
|
||||||
skip: (page - 1) * pageSize,
|
skip: (page - 1) * pageSize,
|
||||||
@@ -219,8 +232,9 @@ export const useDataStoreStore = defineStore(DATA_STORE_STORE, () => {
|
|||||||
|
|
||||||
const fetchDataStoreSize = async () => {
|
const fetchDataStoreSize = async () => {
|
||||||
const result = await fetchDataStoreGlobalLimitInBytes(rootStore.restApiContext);
|
const result = await fetchDataStoreGlobalLimitInBytes(rootStore.restApiContext);
|
||||||
dataStoreSize.value = Number((result.sizeBytes / 1024 / 1024).toFixed(2));
|
dataStoreSize.value = formatSize(result.sizeBytes);
|
||||||
dataStoreSizeLimitState.value = result.sizeState;
|
dataStoreSizeLimitState.value = result.sizeState;
|
||||||
|
dataStoreTableSizes.value = result.dataTables;
|
||||||
return result;
|
return result;
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -231,6 +245,7 @@ export const useDataStoreStore = defineStore(DATA_STORE_STORE, () => {
|
|||||||
fetchDataStoreSize,
|
fetchDataStoreSize,
|
||||||
dataStoreSize: computed(() => dataStoreSize.value),
|
dataStoreSize: computed(() => dataStoreSize.value),
|
||||||
dataStoreSizeLimitState: computed(() => dataStoreSizeLimitState.value),
|
dataStoreSizeLimitState: computed(() => dataStoreSizeLimitState.value),
|
||||||
|
dataStoreSizes,
|
||||||
maxSizeMB,
|
maxSizeMB,
|
||||||
createDataStore,
|
createDataStore,
|
||||||
deleteDataStore,
|
deleteDataStore,
|
||||||
|
|||||||
@@ -106,6 +106,7 @@ export type DataTableSizeStatus = 'ok' | 'warn' | 'error';
|
|||||||
export type DataTablesSizeResult = {
|
export type DataTablesSizeResult = {
|
||||||
sizeBytes: number;
|
sizeBytes: number;
|
||||||
sizeState: DataTableSizeStatus;
|
sizeState: DataTableSizeStatus;
|
||||||
|
dataTables: Record<string, number>;
|
||||||
};
|
};
|
||||||
|
|
||||||
// APIs for a data store service operating on a specific projectId
|
// APIs for a data store service operating on a specific projectId
|
||||||
|
|||||||
Reference in New Issue
Block a user