chore(core): Rename Data Store DB entries to Data Table (no-changelog) (#18670)

This commit is contained in:
Charlie Kolb
2025-08-27 15:36:00 +02:00
committed by GitHub
parent 2ea3d034e3
commit 820fd12f08
20 changed files with 244 additions and 166 deletions

View File

@@ -73,8 +73,8 @@ type EntityName =
| 'InsightsRaw'
| 'InsightsByPeriod'
| 'InsightsMetadata'
| 'DataStore'
| 'DataStoreColumn';
| 'DataTable'
| 'DataTableColumn';
/**
* Truncate specific DB tables in a test DB.

View File

@@ -0,0 +1,84 @@
import type { MigrationContext, ReversibleMigration } from '../migration-types';
const TABLE_TABLE_NAME_BEFORE = 'data_store';
const COLUMN_TABLE_NAME_BEFORE = 'data_store_column';
const TABLE_TABLE_NAME_AFTER = 'data_table';
const COLUMN_TABLE_NAME_AFTER = 'data_table_column';
export class ReplaceDataStoreTablesWithDataTables1754475614602 implements ReversibleMigration {
async up({ schemaBuilder: { createTable, column, dropTable } }: MigrationContext) {
await dropTable(COLUMN_TABLE_NAME_BEFORE);
await dropTable(TABLE_TABLE_NAME_BEFORE);
await createTable(TABLE_TABLE_NAME_AFTER)
.withColumns(
column('id').varchar(36).primary,
column('name').varchar(128).notNull,
column('projectId').varchar(36).notNull,
)
.withForeignKey('projectId', {
tableName: 'project',
columnName: 'id',
onDelete: 'CASCADE',
})
.withUniqueConstraintOn(['projectId', 'name']).withTimestamps;
await createTable(COLUMN_TABLE_NAME_AFTER)
.withColumns(
column('id').varchar(36).primary.notNull,
column('name').varchar(128).notNull,
column('type')
.varchar(32)
.notNull.comment(
'Expected: string, number, boolean, or date (not enforced as a constraint)',
),
column('index').int.notNull.comment('Column order, starting from 0 (0 = first column)'),
column('dataTableId').varchar(36).notNull,
)
.withForeignKey('dataTableId', {
tableName: TABLE_TABLE_NAME_AFTER,
columnName: 'id',
onDelete: 'CASCADE',
})
.withUniqueConstraintOn(['dataTableId', 'name']).withTimestamps;
}
async down({ schemaBuilder: { createTable, column, dropTable } }: MigrationContext) {
await dropTable(COLUMN_TABLE_NAME_AFTER);
await dropTable(TABLE_TABLE_NAME_AFTER);
await createTable(TABLE_TABLE_NAME_BEFORE)
.withColumns(
column('id').varchar(36).primary,
column('name').varchar(128).notNull,
column('projectId').varchar(36).notNull,
column('sizeBytes').int.default(0).notNull,
)
.withForeignKey('projectId', {
tableName: 'project',
columnName: 'id',
onDelete: 'CASCADE',
})
.withUniqueConstraintOn(['projectId', 'name']).withTimestamps;
await createTable(COLUMN_TABLE_NAME_BEFORE)
.withColumns(
column('id').varchar(36).primary.notNull,
column('name').varchar(128).notNull,
column('type')
.varchar(32)
.notNull.comment(
'Expected: string, number, boolean, or date (not enforced as a constraint)',
),
column('index').int.notNull.comment('Column order, starting from 0 (0 = first column)'),
column('dataStoreId').varchar(36).notNull,
)
.withForeignKey('dataStoreId', {
tableName: TABLE_TABLE_NAME_BEFORE,
columnName: 'id',
onDelete: 'CASCADE',
})
.withUniqueConstraintOn(['dataStoreId', 'name']).withTimestamps;
}
}

View File

@@ -94,6 +94,7 @@ import { LinkRoleToUserTable1750252139168 } from '../common/1750252139168-LinkRo
import { RemoveOldRoleColumn1750252139170 } from '../common/1750252139170-RemoveOldRoleColumn';
import { AddInputsOutputsToTestCaseExecution1752669793000 } from '../common/1752669793000-AddInputsOutputsToTestCaseExecution';
import { CreateDataStoreTables1754475614601 } from '../common/1754475614601-CreateDataStoreTables';
import { ReplaceDataStoreTablesWithDataTables1754475614602 } from '../common/1754475614602-ReplaceDataStoreTablesWithDataTables';
import type { Migration } from '../migration-types';
import { UpdateParentFolderIdColumn1740445074052 } from '../mysqldb/1740445074052-UpdateParentFolderIdColumn';
@@ -195,4 +196,5 @@ export const mysqlMigrations: Migration[] = [
AddInputsOutputsToTestCaseExecution1752669793000,
CreateDataStoreTables1754475614601,
RemoveOldRoleColumn1750252139170,
ReplaceDataStoreTablesWithDataTables1754475614602,
];

View File

@@ -94,6 +94,7 @@ import { AddRolesTables1750252139167 } from '../common/1750252139167-AddRolesTab
import { LinkRoleToUserTable1750252139168 } from '../common/1750252139168-LinkRoleToUserTable';
import { RemoveOldRoleColumn1750252139170 } from '../common/1750252139170-RemoveOldRoleColumn';
import { CreateDataStoreTables1754475614601 } from '../common/1754475614601-CreateDataStoreTables';
import { ReplaceDataStoreTablesWithDataTables1754475614602 } from '../common/1754475614602-ReplaceDataStoreTablesWithDataTables';
import type { Migration } from '../migration-types';
export const postgresMigrations: Migration[] = [
@@ -193,4 +194,5 @@ export const postgresMigrations: Migration[] = [
AddInputsOutputsToTestCaseExecution1752669793000,
CreateDataStoreTables1754475614601,
RemoveOldRoleColumn1750252139170,
ReplaceDataStoreTablesWithDataTables1754475614602,
];

View File

@@ -91,6 +91,7 @@ import { LinkRoleToUserTable1750252139168 } from '../common/1750252139168-LinkRo
import { RemoveOldRoleColumn1750252139170 } from '../common/1750252139170-RemoveOldRoleColumn';
import { AddInputsOutputsToTestCaseExecution1752669793000 } from '../common/1752669793000-AddInputsOutputsToTestCaseExecution';
import { CreateDataStoreTables1754475614601 } from '../common/1754475614601-CreateDataStoreTables';
import { ReplaceDataStoreTablesWithDataTables1754475614602 } from '../common/1754475614602-ReplaceDataStoreTablesWithDataTables';
import type { Migration } from '../migration-types';
const sqliteMigrations: Migration[] = [
@@ -187,6 +188,7 @@ const sqliteMigrations: Migration[] = [
AddInputsOutputsToTestCaseExecution1752669793000,
CreateDataStoreTables1754475614601,
RemoveOldRoleColumn1750252139170,
ReplaceDataStoreTablesWithDataTables1754475614602,
];
export { sqliteMigrations };

View File

@@ -32,7 +32,7 @@ beforeAll(async () => {
});
beforeEach(async () => {
await testDb.truncate(['DataStore', 'DataStoreColumn', 'Project', 'ProjectRelation']);
await testDb.truncate(['DataTable', 'DataTableColumn', 'Project', 'ProjectRelation']);
owner = await createOwner();
member = await createMember();

View File

@@ -21,7 +21,7 @@ beforeAll(async () => {
});
beforeEach(async () => {
await testDb.truncate(['DataStore', 'DataStoreColumn']);
await testDb.truncate(['DataTable', 'DataTableColumn']);
});
afterAll(async () => {

View File

@@ -45,7 +45,7 @@ beforeAll(async () => {
});
beforeEach(async () => {
await testDb.truncate(['DataStore', 'DataStoreColumn', 'Project', 'ProjectRelation']);
await testDb.truncate(['DataTable', 'DataTableColumn', 'Project', 'ProjectRelation']);
projectRepository = Container.get(ProjectRepository);
dataStoreRepository = Container.get(DataStoreRepository);
@@ -758,7 +758,7 @@ describe('DELETE /projects/:projectId/data-stores/:dataStoreId', () => {
expect(dataStoreInDb).toBeNull();
});
test("should delete data from 'data_store', 'data_store_column' tables and drop 'data_store_user_<id>' table", async () => {
test("should delete data from 'data_table', 'data_table_column' tables and drop 'data_table_user_<id>' table", async () => {
const personalProject = await projectRepository.getPersonalProjectForUserOrFail(owner.id);
const dataStore = await createDataStore(personalProject, {
name: 'Test Data Store',
@@ -779,7 +779,7 @@ describe('DELETE /projects/:projectId/data-stores/:dataStoreId', () => {
expect(dataStoreInDb).toBeNull();
const dataStoreColumnInDb = await dataStoreColumnRepository.findOneBy({
dataStoreId: dataStore.id,
dataTableId: dataStore.id,
});
expect(dataStoreColumnInDb).toBeNull();
@@ -924,7 +924,7 @@ describe('POST /projects/:projectId/data-stores/:dataStoreId/columns', () => {
.send(payload)
.expect(400);
const columnsInDb = await dataStoreColumnRepository.findBy({ dataStoreId: dataStore.id });
const columnsInDb = await dataStoreColumnRepository.findBy({ dataTableId: dataStore.id });
expect(columnsInDb).toHaveLength(0);
});
@@ -942,7 +942,7 @@ describe('POST /projects/:projectId/data-stores/:dataStoreId/columns', () => {
.send(payload)
.expect(400);
const columnsInDb = await dataStoreColumnRepository.findBy({ dataStoreId: dataStore.id });
const columnsInDb = await dataStoreColumnRepository.findBy({ dataTableId: dataStore.id });
expect(columnsInDb).toHaveLength(0);
});
@@ -964,7 +964,7 @@ describe('POST /projects/:projectId/data-stores/:dataStoreId/columns', () => {
})
.expect(403);
const columnsInDb = await dataStoreColumnRepository.findBy({ dataStoreId: dataStore.id });
const columnsInDb = await dataStoreColumnRepository.findBy({ dataTableId: dataStore.id });
expect(columnsInDb).toHaveLength(1);
expect(columnsInDb[0].name).toBe('test-column');
});
@@ -984,7 +984,7 @@ describe('POST /projects/:projectId/data-stores/:dataStoreId/columns', () => {
.send(payload)
.expect(403);
const columnsInDb = await dataStoreColumnRepository.findBy({ dataStoreId: dataStore.id });
const columnsInDb = await dataStoreColumnRepository.findBy({ dataTableId: dataStore.id });
expect(columnsInDb).toHaveLength(0);
});
@@ -1011,7 +1011,7 @@ describe('POST /projects/:projectId/data-stores/:dataStoreId/columns', () => {
.send(payload)
.expect(200);
const columnsInDb = await dataStoreColumnRepository.findBy({ dataStoreId: dataStore.id });
const columnsInDb = await dataStoreColumnRepository.findBy({ dataTableId: dataStore.id });
expect(columnsInDb).toHaveLength(2);
expect(columnsInDb[0].name).toBe('new-column');
expect(columnsInDb[0].type).toBe('string');
@@ -1040,7 +1040,7 @@ describe('POST /projects/:projectId/data-stores/:dataStoreId/columns', () => {
.send(payload)
.expect(200);
const columnsInDb = await dataStoreColumnRepository.findBy({ dataStoreId: dataStore.id });
const columnsInDb = await dataStoreColumnRepository.findBy({ dataTableId: dataStore.id });
expect(columnsInDb).toHaveLength(2);
expect(columnsInDb[0].name).toBe('new-column');
expect(columnsInDb[0].type).toBe('boolean');
@@ -1070,7 +1070,7 @@ describe('POST /projects/:projectId/data-stores/:dataStoreId/columns', () => {
.send(payload)
.expect(200);
const columnsInDb = await dataStoreColumnRepository.findBy({ dataStoreId: dataStore.id });
const columnsInDb = await dataStoreColumnRepository.findBy({ dataTableId: dataStore.id });
expect(columnsInDb).toHaveLength(2);
expect(columnsInDb[0].name).toBe('new-column');
expect(columnsInDb[0].type).toBe('boolean');
@@ -1163,7 +1163,7 @@ describe('DELETE /projects/:projectId/data-stores/:dataStoreId/columns/:columnId
.expect(403);
const columnInDb = await dataStoreColumnRepository.findOneBy({
dataStoreId: dataStore.id,
dataTableId: dataStore.id,
name: 'test-column',
});
expect(columnInDb).toBeDefined();
@@ -1187,7 +1187,7 @@ describe('DELETE /projects/:projectId/data-stores/:dataStoreId/columns/:columnId
.expect(403);
const columnInDb = await dataStoreColumnRepository.findOneBy({
dataStoreId: dataStore.id,
dataTableId: dataStore.id,
name: 'test-column',
});
expect(columnInDb).toBeDefined();
@@ -1214,7 +1214,7 @@ describe('DELETE /projects/:projectId/data-stores/:dataStoreId/columns/:columnId
.expect(200);
const columnInDb = await dataStoreColumnRepository.findOneBy({
dataStoreId: dataStore.id,
dataTableId: dataStore.id,
name: 'test-column',
});
expect(columnInDb).toBeNull();
@@ -1240,7 +1240,7 @@ describe('DELETE /projects/:projectId/data-stores/:dataStoreId/columns/:columnId
.expect(200);
const columnInDb = await dataStoreColumnRepository.findOneBy({
dataStoreId: dataStore.id,
dataTableId: dataStore.id,
name: 'test-column',
});
expect(columnInDb).toBeNull();
@@ -1265,7 +1265,7 @@ describe('DELETE /projects/:projectId/data-stores/:dataStoreId/columns/:columnId
.expect(200);
const columnInDb = await dataStoreColumnRepository.findOneBy({
dataStoreId: dataStore.id,
dataTableId: dataStore.id,
name: 'test-column',
});
expect(columnInDb).toBeNull();
@@ -1289,7 +1289,7 @@ describe('DELETE /projects/:projectId/data-stores/:dataStoreId/columns/:columnId
.expect(200);
const columnInDb = await dataStoreColumnRepository.findOneBy({
dataStoreId: dataStore.id,
dataTableId: dataStore.id,
name: 'test-column',
});
expect(columnInDb).toBeNull();
@@ -1364,7 +1364,7 @@ describe('PATCH /projects/:projectId/data-stores/:dataStoreId/columns/:columnId/
.expect(403);
const columnInDb = await dataStoreColumnRepository.findOneBy({
dataStoreId: dataStore.id,
dataTableId: dataStore.id,
name: 'test-column',
index: 0,
});
@@ -1395,7 +1395,7 @@ describe('PATCH /projects/:projectId/data-stores/:dataStoreId/columns/:columnId/
.expect(403);
const columnInDb = await dataStoreColumnRepository.findOneBy({
dataStoreId: dataStore.id,
dataTableId: dataStore.id,
name: 'test-column',
index: 0,
});
@@ -1426,7 +1426,7 @@ describe('PATCH /projects/:projectId/data-stores/:dataStoreId/columns/:columnId/
.expect(200);
const columnInDb = await dataStoreColumnRepository.findOneBy({
dataStoreId: dataStore.id,
dataTableId: dataStore.id,
name: 'test-column',
index: 1,
});
@@ -1457,7 +1457,7 @@ describe('PATCH /projects/:projectId/data-stores/:dataStoreId/columns/:columnId/
.expect(200);
const columnInDb = await dataStoreColumnRepository.findOneBy({
dataStoreId: dataStore.id,
dataTableId: dataStore.id,
name: 'test-column',
index: 1,
});
@@ -1488,7 +1488,7 @@ describe('PATCH /projects/:projectId/data-stores/:dataStoreId/columns/:columnId/
.expect(200);
const columnInDb = await dataStoreColumnRepository.findOneBy({
dataStoreId: dataStore.id,
dataTableId: dataStore.id,
name: 'test-column',
index: 1,
});
@@ -1517,7 +1517,7 @@ describe('PATCH /projects/:projectId/data-stores/:dataStoreId/columns/:columnId/
.expect(200);
const columnInDb = await dataStoreColumnRepository.findOneBy({
dataStoreId: dataStore.id,
dataTableId: dataStore.id,
name: 'test-column',
index: 1,
});

View File

@@ -19,7 +19,7 @@ beforeAll(async () => {
});
beforeEach(async () => {
await testDb.truncate(['DataStore', 'DataStoreColumn']);
await testDb.truncate(['DataTable', 'DataTableColumn']);
});
afterAll(async () => {
@@ -52,17 +52,17 @@ describe('dataStore', () => {
describe('createDataStore', () => {
it('should create a columns table and a user table if columns are provided', async () => {
const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, {
const { id: dataTableId } = await dataStoreService.createDataStore(project1.id, {
name: 'dataStoreWithColumns',
columns: [{ name: 'foo', type: 'string' }],
});
await expect(dataStoreService.getColumns(dataStoreId, project1.id)).resolves.toEqual([
await expect(dataStoreService.getColumns(dataTableId, project1.id)).resolves.toEqual([
{
name: 'foo',
type: 'string',
index: 0,
dataStoreId,
dataTableId,
id: expect.any(String),
createdAt: expect.any(Date),
updatedAt: expect.any(Date),
@@ -70,7 +70,7 @@ describe('dataStore', () => {
]);
// Select the column from user table to check for its existence
const userTableName = dataStoreRowsRepository.toTableName(dataStoreId);
const userTableName = dataStoreRowsRepository.toTableName(dataTableId);
const rows = await dataStoreRepository.manager
.createQueryBuilder()
.select('foo')
@@ -251,7 +251,7 @@ describe('dataStore', () => {
it('should succeed with adding columns to a non-empty table as well as to a user table', async () => {
const existingColumns: CreateDataStoreColumnDto[] = [{ name: 'myColumn0', type: 'string' }];
const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, {
const { id: dataTableId } = await dataStoreService.createDataStore(project1.id, {
name: 'dataStoreWithColumns',
columns: existingColumns,
});
@@ -264,45 +264,45 @@ describe('dataStore', () => {
];
for (const column of columns) {
// ACT
const result = await dataStoreService.addColumn(dataStoreId, project1.id, column);
const result = await dataStoreService.addColumn(dataTableId, project1.id, column);
// ASSERT
expect(result).toMatchObject(column);
}
const columnResult = await dataStoreService.getColumns(dataStoreId, project1.id);
const columnResult = await dataStoreService.getColumns(dataTableId, project1.id);
expect(columnResult).toEqual([
expect.objectContaining({
index: 0,
dataStoreId,
dataTableId,
name: 'myColumn0',
type: 'string',
}),
expect.objectContaining({
index: 1,
dataStoreId,
dataTableId,
name: 'myColumn1',
type: 'string',
}),
expect.objectContaining({
index: 2,
dataStoreId,
dataTableId,
name: 'myColumn2',
type: 'number',
}),
expect.objectContaining({
index: 3,
dataStoreId,
dataTableId,
name: 'myColumn3',
type: 'number',
}),
expect.objectContaining({
index: 4,
dataStoreId,
dataTableId,
name: 'myColumn4',
type: 'date',
}),
]);
const userTableName = dataStoreRowsRepository.toTableName(dataStoreId);
const userTableName = dataStoreRowsRepository.toTableName(dataTableId);
const queryRunner = dataStoreRepository.manager.connection.createQueryRunner();
try {
const table = await queryRunner.getTable(userTableName);
@@ -326,7 +326,7 @@ describe('dataStore', () => {
});
it('should succeed with adding columns to an empty table', async () => {
const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, {
const { id: dataTableId } = await dataStoreService.createDataStore(project1.id, {
name: 'dataStore',
columns: [],
});
@@ -337,29 +337,29 @@ describe('dataStore', () => {
];
for (const column of columns) {
// ACT
const result = await dataStoreService.addColumn(dataStoreId, project1.id, column);
const result = await dataStoreService.addColumn(dataTableId, project1.id, column);
// ASSERT
expect(result).toMatchObject(column);
}
const columnResult = await dataStoreService.getColumns(dataStoreId, project1.id);
const columnResult = await dataStoreService.getColumns(dataTableId, project1.id);
expect(columnResult.length).toBe(2);
expect(columnResult).toEqual([
expect.objectContaining({
index: 0,
dataStoreId,
dataTableId,
name: 'myColumn0',
type: 'string',
}),
expect.objectContaining({
index: 1,
dataStoreId,
dataTableId,
name: 'myColumn1',
type: 'number',
}),
]);
const userTableName = dataStoreRowsRepository.toTableName(dataStoreId);
const userTableName = dataStoreRowsRepository.toTableName(dataTableId);
const queryRunner = dataStoreRepository.manager.connection.createQueryRunner();
try {
const table = await queryRunner.getTable(userTableName);
@@ -516,31 +516,31 @@ describe('dataStore', () => {
describe('deleteColumn', () => {
it('should succeed with deleting a column', async () => {
// ARRANGE
const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, {
const { id: dataTableId } = await dataStoreService.createDataStore(project1.id, {
name: 'dataStore',
columns: [],
});
const c1 = await dataStoreService.addColumn(dataStoreId, project1.id, {
const c1 = await dataStoreService.addColumn(dataTableId, project1.id, {
name: 'myColumn1',
type: 'string',
});
const c2 = await dataStoreService.addColumn(dataStoreId, project1.id, {
const c2 = await dataStoreService.addColumn(dataTableId, project1.id, {
name: 'myColumn2',
type: 'number',
});
// ACT
const result = await dataStoreService.deleteColumn(dataStoreId, project1.id, c1.id);
const result = await dataStoreService.deleteColumn(dataTableId, project1.id, c1.id);
// ASSERT
expect(result).toEqual(true);
const columns = await dataStoreService.getColumns(dataStoreId, project1.id);
const columns = await dataStoreService.getColumns(dataTableId, project1.id);
expect(columns).toEqual([
{
index: 0,
dataStoreId,
dataTableId,
id: c2.id,
name: 'myColumn2',
type: 'number',
@@ -591,40 +591,40 @@ describe('dataStore', () => {
describe('moveColumn', () => {
it('should succeed with moving a column', async () => {
// ARRANGE
const { id: dataStoreId } = await dataStoreService.createDataStore(project1.id, {
const { id: dataTableId } = await dataStoreService.createDataStore(project1.id, {
name: 'dataStore',
columns: [],
});
const c1 = await dataStoreService.addColumn(dataStoreId, project1.id, {
const c1 = await dataStoreService.addColumn(dataTableId, project1.id, {
name: 'myColumn1',
type: 'string',
});
const c2 = await dataStoreService.addColumn(dataStoreId, project1.id, {
const c2 = await dataStoreService.addColumn(dataTableId, project1.id, {
name: 'myColumn2',
type: 'number',
});
// ACT
const result = await dataStoreService.moveColumn(dataStoreId, project1.id, c2.id, {
const result = await dataStoreService.moveColumn(dataTableId, project1.id, c2.id, {
targetIndex: 0,
});
// ASSERT
expect(result).toEqual(true);
const columns = await dataStoreService.getColumns(dataStoreId, project1.id);
const columns = await dataStoreService.getColumns(dataTableId, project1.id);
expect(columns).toMatchObject([
{
index: 0,
dataStoreId,
dataTableId,
id: c2.id,
name: 'myColumn2',
type: 'number',
},
{
index: 1,
dataStoreId,
dataTableId,
id: c1.id,
name: 'myColumn1',
type: 'string',

View File

@@ -5,50 +5,50 @@ import { addColumnQuery, deleteColumnQuery, splitRowsByExistence } from '../util
describe('sql-utils', () => {
describe('addColumnQuery', () => {
it('should generate a valid SQL query for adding columns to a table, sqlite', () => {
const tableName = 'data_store_user_abc';
const tableName = 'data_table_user_abc';
const column = { name: 'email', type: 'number' as const };
const query = addColumnQuery(tableName, column, 'sqlite');
expect(query).toBe('ALTER TABLE "data_store_user_abc" ADD "email" REAL');
expect(query).toBe('ALTER TABLE "data_table_user_abc" ADD "email" REAL');
});
it('should generate a valid SQL query for adding columns to a table, postgres', () => {
const tableName = 'data_store_user_abc';
const tableName = 'data_table_user_abc';
const column = { name: 'email', type: 'number' as const };
const query = addColumnQuery(tableName, column, 'postgres');
expect(query).toBe('ALTER TABLE "data_store_user_abc" ADD "email" DOUBLE PRECISION');
expect(query).toBe('ALTER TABLE "data_table_user_abc" ADD "email" DOUBLE PRECISION');
});
it('should generate a valid SQL query for adding columns to a table, mysql', () => {
const tableName = 'data_store_user_abc';
const tableName = 'data_table_user_abc';
const column = { name: 'email', type: 'number' as const };
const query = addColumnQuery(tableName, column, 'mysql');
expect(query).toBe('ALTER TABLE `data_store_user_abc` ADD `email` DOUBLE');
expect(query).toBe('ALTER TABLE `data_table_user_abc` ADD `email` DOUBLE');
});
it('should generate a valid SQL query for adding columns to a table, mariadb', () => {
const tableName = 'data_store_user_abc';
const tableName = 'data_table_user_abc';
const column = { name: 'email', type: 'number' as const };
const query = addColumnQuery(tableName, column, 'mariadb');
expect(query).toBe('ALTER TABLE `data_store_user_abc` ADD `email` DOUBLE');
expect(query).toBe('ALTER TABLE `data_table_user_abc` ADD `email` DOUBLE');
});
});
describe('deleteColumnQuery', () => {
it('should generate a valid SQL query for deleting columns from a table', () => {
const tableName = 'data_store_user_abc';
const tableName = 'data_table_user_abc';
const column = 'email';
const query = deleteColumnQuery(tableName, column, 'sqlite');
expect(query).toBe('ALTER TABLE "data_store_user_abc" DROP COLUMN "email"');
expect(query).toBe('ALTER TABLE "data_table_user_abc" DROP COLUMN "email"');
});
});

View File

@@ -3,25 +3,25 @@ import { Service } from '@n8n/di';
import { DataSource, EntityManager, Repository } from '@n8n/typeorm';
import { UnexpectedError } from 'n8n-workflow';
import { DataStoreColumn } from './data-store-column.entity';
import { DataStoreRowsRepository } from './data-store-rows.repository';
import { DataTableColumn } from './data-table-column.entity';
import { DataStoreColumnNameConflictError } from './errors/data-store-column-name-conflict.error';
import { DataStoreValidationError } from './errors/data-store-validation.error';
@Service()
export class DataStoreColumnRepository extends Repository<DataStoreColumn> {
export class DataStoreColumnRepository extends Repository<DataTableColumn> {
constructor(
dataSource: DataSource,
private dataStoreRowsRepository: DataStoreRowsRepository,
) {
super(DataStoreColumn, dataSource.manager);
super(DataTableColumn, dataSource.manager);
}
async getColumns(rawDataStoreId: string, em?: EntityManager) {
async getColumns(dataTableId: string, em?: EntityManager) {
const executor = em ?? this.manager;
const columns = await executor
.createQueryBuilder(DataStoreColumn, 'dsc')
.where('dsc.dataStoreId = :dataStoreId', { dataStoreId: rawDataStoreId })
.createQueryBuilder(DataTableColumn, 'dsc')
.where('dsc.dataTableId = :dataTableId', { dataTableId })
.getMany();
// Ensure columns are always returned in the correct order by index,
@@ -32,30 +32,30 @@ export class DataStoreColumnRepository extends Repository<DataStoreColumn> {
return columns;
}
async addColumn(dataStoreId: string, schema: DataStoreCreateColumnSchema) {
async addColumn(dataTableId: string, schema: DataStoreCreateColumnSchema) {
return await this.manager.transaction(async (em) => {
const existingColumnMatch = await em.existsBy(DataStoreColumn, {
const existingColumnMatch = await em.existsBy(DataTableColumn, {
name: schema.name,
dataStoreId,
dataTableId,
});
if (existingColumnMatch) {
throw new DataStoreColumnNameConflictError(schema.name, dataStoreId);
throw new DataStoreColumnNameConflictError(schema.name, dataTableId);
}
if (schema.index === undefined) {
const columns = await this.getColumns(dataStoreId, em);
const columns = await this.getColumns(dataTableId, em);
schema.index = columns.length;
} else {
await this.shiftColumns(dataStoreId, schema.index, 1, em);
await this.shiftColumns(dataTableId, schema.index, 1, em);
}
const column = em.create(DataStoreColumn, {
const column = em.create(DataTableColumn, {
...schema,
dataStoreId,
dataTableId,
});
await em.insert(DataStoreColumn, column);
await em.insert(DataTableColumn, column);
const queryRunner = em.queryRunner;
if (!queryRunner) {
@@ -63,7 +63,7 @@ export class DataStoreColumnRepository extends Repository<DataStoreColumn> {
}
await this.dataStoreRowsRepository.addColumn(
dataStoreId,
dataTableId,
column,
queryRunner,
em.connection.options.type,
@@ -73,9 +73,9 @@ export class DataStoreColumnRepository extends Repository<DataStoreColumn> {
});
}
async deleteColumn(dataStoreId: string, column: DataStoreColumn) {
async deleteColumn(dataStoreId: string, column: DataTableColumn) {
await this.manager.transaction(async (em) => {
await em.remove(DataStoreColumn, column);
await em.remove(DataTableColumn, column);
const queryRunner = em.queryRunner;
if (!queryRunner) {
@@ -92,9 +92,9 @@ export class DataStoreColumnRepository extends Repository<DataStoreColumn> {
});
}
async moveColumn(dataStoreId: string, column: DataStoreColumn, targetIndex: number) {
async moveColumn(dataTableId: string, column: DataTableColumn, targetIndex: number) {
await this.manager.transaction(async (em) => {
const columnCount = await em.countBy(DataStoreColumn, { dataStoreId });
const columnCount = await em.countBy(DataTableColumn, { dataTableId });
if (targetIndex < 0) {
throw new DataStoreValidationError('tried to move column to negative index');
@@ -106,27 +106,22 @@ export class DataStoreColumnRepository extends Repository<DataStoreColumn> {
);
}
await this.shiftColumns(dataStoreId, column.index, -1, em);
await this.shiftColumns(dataStoreId, targetIndex, 1, em);
await em.update(DataStoreColumn, { id: column.id }, { index: targetIndex });
await this.shiftColumns(dataTableId, column.index, -1, em);
await this.shiftColumns(dataTableId, targetIndex, 1, em);
await em.update(DataTableColumn, { id: column.id }, { index: targetIndex });
});
}
async shiftColumns(
rawDataStoreId: string,
lowestIndex: number,
delta: -1 | 1,
em?: EntityManager,
) {
async shiftColumns(dataTableId: string, lowestIndex: number, delta: -1 | 1, em?: EntityManager) {
const executor = em ?? this.manager;
await executor
.createQueryBuilder()
.update(DataStoreColumn)
.update(DataTableColumn)
.set({
index: () => `index + ${delta}`,
})
.where('dataStoreId = :dataStoreId AND index >= :thresholdValue', {
dataStoreId: rawDataStoreId,
.where('dataTableId = :dataTableId AND index >= :thresholdValue', {
dataTableId,
thresholdValue: lowestIndex,
})
.execute();

View File

@@ -12,8 +12,8 @@ import {
DATA_TABLE_SYSTEM_COLUMNS,
} from 'n8n-workflow';
import { DataStoreColumn } from './data-store-column.entity';
import { DataStoreUserTableName } from './data-store.types';
import { DataTableColumn } from './data-table-column.entity';
import {
addColumnQuery,
deleteColumnQuery,
@@ -138,19 +138,19 @@ export class DataStoreRowsRepository {
toTableName(dataStoreId: string): DataStoreUserTableName {
const { tablePrefix } = this.globalConfig.database;
return `${tablePrefix}data_store_user_${dataStoreId}`;
return `${tablePrefix}data_table_user_${dataStoreId}`;
}
async insertRows<T extends boolean | undefined>(
dataStoreId: string,
rows: DataStoreRows,
columns: DataStoreColumn[],
columns: DataTableColumn[],
returnData?: T,
): Promise<Array<T extends true ? DataStoreRowReturn : Pick<DataStoreRowReturn, 'id'>>>;
async insertRows(
dataStoreId: string,
rows: DataStoreRows,
columns: DataStoreColumn[],
columns: DataTableColumn[],
returnData?: boolean,
): Promise<Array<DataStoreRowReturn | Pick<DataStoreRowReturn, 'id'>>> {
const inserted: Array<Pick<DataStoreRowReturn, 'id'>> = [];
@@ -217,7 +217,7 @@ export class DataStoreRowsRepository {
dataStoreId: string,
setData: Record<string, DataStoreColumnJsType | null>,
whereData: Record<string, DataStoreColumnJsType | null>,
columns: DataStoreColumn[],
columns: DataTableColumn[],
returnData: boolean = false,
) {
const dbType = this.dataSource.options.type;
@@ -278,14 +278,14 @@ export class DataStoreRowsRepository {
dataStoreId: string,
matchFields: string[],
rows: DataStoreRows,
columns: DataStoreColumn[],
columns: DataTableColumn[],
returnData?: T,
): Promise<T extends true ? DataStoreRowReturn[] : true>;
async upsertRows(
dataStoreId: string,
matchFields: string[],
rows: DataStoreRows,
columns: DataStoreColumn[],
columns: DataTableColumn[],
returnData?: boolean,
) {
returnData = returnData ?? false;
@@ -343,7 +343,7 @@ export class DataStoreRowsRepository {
async createTableWithColumns(
dataStoreId: string,
columns: DataStoreColumn[],
columns: DataTableColumn[],
queryRunner: QueryRunner,
) {
const dslColumns = [new DslColumn('id').int.autoGenerate2.primary, ...toDslColumns(columns)];
@@ -360,7 +360,7 @@ export class DataStoreRowsRepository {
async addColumn(
dataStoreId: string,
column: DataStoreColumn,
column: DataTableColumn,
queryRunner: QueryRunner,
dbType: DataSourceOptions['type'],
) {
@@ -387,7 +387,7 @@ export class DataStoreRowsRepository {
return { count: count ?? -1, data };
}
async getManyByIds(dataStoreId: string, ids: number[], columns: DataStoreColumn[]) {
async getManyByIds(dataStoreId: string, ids: number[], columns: DataTableColumn[]) {
const table = this.toTableName(dataStoreId);
const escapedColumns = columns.map((c) => this.dataSource.driver.escape(c.name));
const escapedSystemColumns = DATA_TABLE_SYSTEM_COLUMNS.map((x) =>

View File

@@ -7,17 +7,17 @@ import { Service } from '@n8n/di';
import { DataSource, EntityManager, Repository, SelectQueryBuilder } from '@n8n/typeorm';
import { UnexpectedError } from 'n8n-workflow';
import { DataStoreColumn } from './data-store-column.entity';
import { DataStoreRowsRepository } from './data-store-rows.repository';
import { DataStore } from './data-store.entity';
import { DataTableColumn } from './data-table-column.entity';
import { DataTable } from './data-table.entity';
@Service()
export class DataStoreRepository extends Repository<DataStore> {
export class DataStoreRepository extends Repository<DataTable> {
constructor(
dataSource: DataSource,
private dataStoreRowsRepository: DataStoreRowsRepository,
) {
super(DataStore, dataSource.manager);
super(DataTable, dataSource.manager);
}
async createDataStore(projectId: string, name: string, columns: DataStoreCreateColumnSchema[]) {
@@ -25,11 +25,11 @@ export class DataStoreRepository extends Repository<DataStore> {
throw new UnexpectedError('bad column name');
}
let dataStoreId: string | undefined;
let dataTableId: string | undefined;
await this.manager.transaction(async (em) => {
const dataStore = em.create(DataStore, { name, columns, projectId });
await em.insert(DataStore, dataStore);
dataStoreId = dataStore.id;
const dataStore = em.create(DataTable, { name, columns, projectId });
await em.insert(DataTable, dataStore);
dataTableId = dataStore.id;
const queryRunner = em.queryRunner;
if (!queryRunner) {
@@ -38,8 +38,8 @@ export class DataStoreRepository extends Repository<DataStore> {
// insert columns
const columnEntities = columns.map((col, index) =>
em.create(DataStoreColumn, {
dataStoreId,
em.create(DataTableColumn, {
dataTableId,
name: col.name,
type: col.type,
index: col.index ?? index,
@@ -47,23 +47,23 @@ export class DataStoreRepository extends Repository<DataStore> {
);
if (columnEntities.length > 0) {
await em.insert(DataStoreColumn, columnEntities);
await em.insert(DataTableColumn, columnEntities);
}
// create user table (will create empty table with just id column if no columns)
await this.dataStoreRowsRepository.createTableWithColumns(
dataStoreId,
dataTableId,
columnEntities,
queryRunner,
);
});
if (!dataStoreId) {
if (!dataTableId) {
throw new UnexpectedError('Data store creation failed');
}
const createdDataStore = await this.findOneOrFail({
where: { id: dataStoreId },
where: { id: dataTableId },
relations: ['project', 'columns'],
});
@@ -78,7 +78,7 @@ export class DataStoreRepository extends Repository<DataStore> {
throw new UnexpectedError('QueryRunner is not available');
}
await em.delete(DataStore, { id: dataStoreId });
await em.delete(DataTable, { id: dataStoreId });
await this.dataStoreRowsRepository.dropTable(dataStoreId, queryRunner);
return true;
@@ -87,7 +87,7 @@ export class DataStoreRepository extends Repository<DataStore> {
async deleteDataStoreByProjectId(projectId: string) {
return await this.manager.transaction(async (em) => {
const existingTables = await em.findBy(DataStore, { projectId });
const existingTables = await em.findBy(DataTable, { projectId });
let changed = false;
for (const match of existingTables) {
@@ -106,11 +106,11 @@ export class DataStoreRepository extends Repository<DataStore> {
throw new UnexpectedError('QueryRunner is not available');
}
const existingTables = await em.findBy(DataStore, {});
const existingTables = await em.findBy(DataTable, {});
let changed = false;
for (const match of existingTables) {
const result = await em.delete(DataStore, { id: match.id });
const result = await em.delete(DataTable, { id: match.id });
await this.dataStoreRowsRepository.dropTable(match.id, queryRunner);
changed = changed || (result.affected ?? 0) > 0;
}
@@ -130,7 +130,7 @@ export class DataStoreRepository extends Repository<DataStore> {
return await query.getMany();
}
private getManyQuery(options: Partial<ListDataStoreQueryDto>): SelectQueryBuilder<DataStore> {
private getManyQuery(options: Partial<ListDataStoreQueryDto>): SelectQueryBuilder<DataTable> {
const query = this.createQueryBuilder('dataStore');
this.applySelections(query);
@@ -141,12 +141,12 @@ export class DataStoreRepository extends Repository<DataStore> {
return query;
}
private applySelections(query: SelectQueryBuilder<DataStore>): void {
private applySelections(query: SelectQueryBuilder<DataTable>): void {
this.applyDefaultSelect(query);
}
private applyFilters(
query: SelectQueryBuilder<DataStore>,
query: SelectQueryBuilder<DataTable>,
filter: Partial<ListDataStoreQueryDto>['filter'],
): void {
for (const x of ['id', 'projectId'] as const) {
@@ -172,7 +172,7 @@ export class DataStoreRepository extends Repository<DataStore> {
}
}
private applySorting(query: SelectQueryBuilder<DataStore>, sortBy?: string): void {
private applySorting(query: SelectQueryBuilder<DataTable>, sortBy?: string): void {
if (!sortBy) {
query.orderBy('dataStore.updatedAt', 'DESC');
return;
@@ -188,7 +188,7 @@ export class DataStoreRepository extends Repository<DataStore> {
}
private applySortingByField(
query: SelectQueryBuilder<DataStore>,
query: SelectQueryBuilder<DataTable>,
field: string,
direction: 'DESC' | 'ASC',
): void {
@@ -202,7 +202,7 @@ export class DataStoreRepository extends Repository<DataStore> {
}
private applyPagination(
query: SelectQueryBuilder<DataStore>,
query: SelectQueryBuilder<DataTable>,
options: Partial<ListDataStoreQueryDto>,
): void {
query.skip(options.skip ?? 0);
@@ -211,7 +211,7 @@ export class DataStoreRepository extends Repository<DataStore> {
}
}
private applyDefaultSelect(query: SelectQueryBuilder<DataStore>): void {
private applyDefaultSelect(query: SelectQueryBuilder<DataTable>): void {
query
.leftJoinAndSelect('dataStore.project', 'project')
.leftJoinAndSelect('dataStore.columns', 'data_store_column')

View File

@@ -311,14 +311,14 @@ export class DataStoreService {
return existingTable;
}
private async validateColumnExists(dataStoreId: string, columnId: string) {
private async validateColumnExists(dataTableId: string, columnId: string) {
const existingColumn = await this.dataStoreColumnRepository.findOneBy({
id: columnId,
dataStoreId,
dataTableId,
});
if (existingColumn === null) {
throw new DataStoreColumnNotFoundError(dataStoreId, columnId);
throw new DataStoreColumnNotFoundError(dataTableId, columnId);
}
return existingColumn;

View File

@@ -1 +1 @@
export type DataStoreUserTableName = `${string}data_store_user_${string}`;
export type DataStoreUserTableName = `${string}data_table_user_${string}`;

View File

@@ -1,13 +1,13 @@
import { WithTimestampsAndStringId } from '@n8n/db';
import { Column, Entity, Index, JoinColumn, ManyToOne } from '@n8n/typeorm';
import { type DataStore } from './data-store.entity';
import { type DataTable } from './data-table.entity';
@Entity()
@Index(['dataStoreId', 'name'], { unique: true })
export class DataStoreColumn extends WithTimestampsAndStringId {
@Index(['dataTableId', 'name'], { unique: true })
export class DataTableColumn extends WithTimestampsAndStringId {
@Column()
dataStoreId: string;
dataTableId: string;
@Column()
name: string;
@@ -18,7 +18,7 @@ export class DataStoreColumn extends WithTimestampsAndStringId {
@Column({ type: 'int' })
index: number;
@ManyToOne('DataStore', 'columns')
@JoinColumn({ name: 'dataStoreId' })
dataStore: DataStore;
@ManyToOne('DataTable', 'columns')
@JoinColumn({ name: 'dataTableId' })
dataTable: DataTable;
}

View File

@@ -1,11 +1,11 @@
import { Project, WithTimestampsAndStringId } from '@n8n/db';
import { Column, Entity, Index, JoinColumn, ManyToOne, OneToMany } from '@n8n/typeorm';
import { DataStoreColumn } from './data-store-column.entity';
import { DataTableColumn } from './data-table-column.entity';
@Entity()
@Index(['name', 'projectId'], { unique: true })
export class DataStore extends WithTimestampsAndStringId {
export class DataTable extends WithTimestampsAndStringId {
constructor() {
super();
}
@@ -14,13 +14,13 @@ export class DataStore extends WithTimestampsAndStringId {
name: string;
@OneToMany(
() => DataStoreColumn,
(dataStoreColumn) => dataStoreColumn.dataStore,
() => DataTableColumn,
(dataTableColumn) => dataTableColumn.dataTable,
{
cascade: true,
},
)
columns: DataStoreColumn[];
columns: DataTableColumn[];
@ManyToOne(() => Project)
@JoinColumn({ name: 'projectId' })
@@ -28,7 +28,4 @@ export class DataStore extends WithTimestampsAndStringId {
@Column()
projectId: string;
@Column({ type: 'int', default: 0 })
sizeBytes: number;
}

View File

@@ -35,9 +35,9 @@ export class DataStoreModule implements ModuleInterface {
}
async entities() {
const { DataStore } = await import('./data-store.entity');
const { DataStoreColumn } = await import('./data-store-column.entity');
const { DataTable } = await import('./data-table.entity');
const { DataTableColumn } = await import('./data-table-column.entity');
return [DataStore, DataStoreColumn] as unknown as Array<new () => BaseEntity>;
return [DataTable, DataTableColumn] as unknown as Array<new () => BaseEntity>;
}
}

View File

@@ -1,8 +1,4 @@
import {
DATA_STORE_COLUMN_REGEX,
type DataStoreCreateColumnSchema,
type DataStoreColumn,
} from '@n8n/api-types';
import { DATA_STORE_COLUMN_REGEX, type DataStoreCreateColumnSchema } from '@n8n/api-types';
import { DslColumn } from '@n8n/db';
import type { DataSourceOptions } from '@n8n/typeorm';
import type {
@@ -14,6 +10,7 @@ import type {
import { UnexpectedError } from 'n8n-workflow';
import type { DataStoreUserTableName } from '../data-store.types';
import type { DataTableColumn } from '../data-table-column.entity';
import { NotFoundError } from '@/errors/response-errors/not-found.error';
@@ -213,7 +210,7 @@ export function extractInsertedIds(raw: unknown, dbType: DataSourceOptions['type
}
}
export function normalizeRows(rows: DataStoreRowsReturn, columns: DataStoreColumn[]) {
export function normalizeRows(rows: DataStoreRowsReturn, columns: DataTableColumn[]) {
// we need to normalize system dates as well
const systemColumns = [
{ name: 'createdAt', type: 'date' },

View File

@@ -5,7 +5,7 @@ export type DataStoreColumn = {
name: string;
type: DataStoreColumnType;
index: number;
dataStoreId: string;
dataTableId: string;
};
export type DataStore = {
@@ -15,7 +15,6 @@ export type DataStore = {
createdAt: Date;
updatedAt: Date;
projectId: string;
sizeBytes: number;
};
export type CreateDataStoreColumnOptions = Pick<DataStoreColumn, 'name' | 'type'> &