diff --git a/.github/docker-compose.yml b/.github/docker-compose.yml index d4c8dc2ba8..6eb0c3d81a 100644 --- a/.github/docker-compose.yml +++ b/.github/docker-compose.yml @@ -1,6 +1,6 @@ services: mariadb: - image: mariadb:10.9 + image: mariadb:10.5 environment: - MARIADB_DATABASE=n8n - MARIADB_ROOT_PASSWORD=password @@ -10,6 +10,26 @@ services: tmpfs: - /var/lib/mysql + mysql-8.0.13: + image: mysql:8.0.13 + environment: + - MYSQL_DATABASE=n8n + - MYSQL_ROOT_PASSWORD=password + ports: + - 3306:3306 + tmpfs: + - /var/lib/mysql + + mysql-8.4: + image: mysql:8.4 + environment: + - MYSQL_DATABASE=n8n + - MYSQL_ROOT_PASSWORD=password + ports: + - 3306:3306 + tmpfs: + - /var/lib/mysql + postgres: image: postgres:16 restart: always diff --git a/.github/workflows/ci-postgres-mysql.yml b/.github/workflows/ci-postgres-mysql.yml index 19fa1cad4f..77f4c285d6 100644 --- a/.github/workflows/ci-postgres-mysql.yml +++ b/.github/workflows/ci-postgres-mysql.yml @@ -123,6 +123,50 @@ jobs: working-directory: packages/cli run: pnpm test:mariadb --testTimeout 30000 + mysql: + name: MySQL (${{ matrix.service-name }}) + runs-on: ubuntu-latest + needs: build + timeout-minutes: 20 + strategy: + matrix: + service-name: [ 'mysql-8.0.13', 'mysql-8.4' ] + env: + DB_MYSQLDB_PASSWORD: password + steps: + - uses: actions/checkout@v4.1.1 + + - uses: actions/setup-node@v4.2.0 + with: + node-version: 20.x + + - name: Setup corepack and pnpm + run: | + npm i -g corepack@0.31 + corepack enable + + - run: pnpm install --frozen-lockfile + + - name: Setup build cache + uses: rharkor/caching-for-turbo@v1.5 + + - name: Restore cached build artifacts + uses: actions/cache/restore@v4.2.0 + with: + path: ./packages/**/dist + key: ${{ github.sha }}:db-tests + + - name: Start MySQL + uses: isbang/compose-action@v2.0.0 + with: + compose-file: ./.github/docker-compose.yml + services: | + ${{ matrix.service-name }} + + - name: Test MySQL + working-directory: packages/cli + run: pnpm test:mysql --testTimeout 30000 + postgres: name: Postgres runs-on: ubuntu-latest @@ -168,7 +212,7 @@ jobs: notify-on-failure: name: Notify Slack on failure runs-on: ubuntu-latest - needs: [mariadb, postgres] + needs: [mariadb, postgres, mysql] steps: - name: Notify Slack on failure uses: act10ns/slack@v2.0.0 @@ -177,4 +221,4 @@ jobs: status: ${{ job.status }} channel: '#alerts-build' webhook-url: ${{ secrets.SLACK_WEBHOOK_URL }} - message: Postgres or MariaDB tests failed (${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) + message: Postgres, MariaDB or MySQL tests failed (${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) diff --git a/packages/cli/package.json b/packages/cli/package.json index 3e67d4eae4..cdc34293fc 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -29,6 +29,7 @@ "test:sqlite": "N8N_LOG_LEVEL=silent DB_TYPE=sqlite jest", "test:postgres": "N8N_LOG_LEVEL=silent DB_TYPE=postgresdb DB_POSTGRESDB_SCHEMA=alt_schema DB_TABLE_PREFIX=test_ jest --no-coverage", "test:mariadb": "N8N_LOG_LEVEL=silent DB_TYPE=mariadb DB_TABLE_PREFIX=test_ jest --no-coverage", + "test:mysql": "N8N_LOG_LEVEL=silent DB_TYPE=mysqldb DB_TABLE_PREFIX=test_ jest --no-coverage", "watch": "tsc-watch -p tsconfig.build.json --onCompilationComplete \"tsc-alias -p tsconfig.build.json\"" }, "bin": { diff --git a/packages/cli/src/databases/migrations/mysqldb/1731582748663-MigrateTestDefinitionKeyToString.ts b/packages/cli/src/databases/migrations/mysqldb/1731582748663-MigrateTestDefinitionKeyToString.ts index 793aa3a262..44e03ddbcc 100644 --- a/packages/cli/src/databases/migrations/mysqldb/1731582748663-MigrateTestDefinitionKeyToString.ts +++ b/packages/cli/src/databases/migrations/mysqldb/1731582748663-MigrateTestDefinitionKeyToString.ts @@ -14,5 +14,19 @@ export class MigrateTestDefinitionKeyToString1731582748663 implements Irreversib await queryRunner.query( `CREATE INDEX \`TMP_idx_${tablePrefix}test_definition_id\` ON ${tablePrefix}test_definition (\`id\`);`, ); + + // Note: this part was missing in initial release and was added after. Without it the migration run successfully, + // but left the table in inconsistent state, because it didn't finish changing the primary key and deleting the old one. + // This prevented the next migration from running on MySQL 8.4.4 + await queryRunner.query( + `ALTER TABLE ${tablePrefix}test_definition MODIFY COLUMN tmp_id INT NOT NULL;`, + ); + await queryRunner.query( + `ALTER TABLE ${tablePrefix}test_definition DROP PRIMARY KEY, ADD PRIMARY KEY (\`id\`);`, + ); + await queryRunner.query( + `DROP INDEX \`TMP_idx_${tablePrefix}test_definition_id\` ON ${tablePrefix}test_definition;`, + ); + await queryRunner.query(`ALTER TABLE ${tablePrefix}test_definition DROP COLUMN tmp_id;`); } } diff --git a/packages/cli/src/databases/migrations/mysqldb/1732271325258-CreateTestMetricTable.ts b/packages/cli/src/databases/migrations/mysqldb/1732271325258-CreateTestMetricTable.ts new file mode 100644 index 0000000000..329822f886 --- /dev/null +++ b/packages/cli/src/databases/migrations/mysqldb/1732271325258-CreateTestMetricTable.ts @@ -0,0 +1,49 @@ +import assert from 'node:assert'; + +import type { MigrationContext, ReversibleMigration } from '@/databases/types'; + +const testMetricEntityTableName = 'test_metric'; + +export class CreateTestMetricTable1732271325258 implements ReversibleMigration { + async up({ schemaBuilder: { createTable, column }, queryRunner, tablePrefix }: MigrationContext) { + // Check if the previous migration MigrateTestDefinitionKeyToString1731582748663 properly updated the primary key + const table = await queryRunner.getTable(`${tablePrefix}test_definition`); + assert(table, 'test_definition table not found'); + + const brokenPrimaryColumn = table.primaryColumns.some( + (c) => c.name === 'tmp_id' && c.isPrimary, + ); + + if (brokenPrimaryColumn) { + // The migration was completed, but left the table in inconsistent state, let's finish the primary key change + await queryRunner.query( + `ALTER TABLE ${tablePrefix}test_definition MODIFY COLUMN tmp_id INT NOT NULL;`, + ); + await queryRunner.query( + `ALTER TABLE ${tablePrefix}test_definition DROP PRIMARY KEY, ADD PRIMARY KEY (\`id\`);`, + ); + await queryRunner.query( + `DROP INDEX \`TMP_idx_${tablePrefix}test_definition_id\` ON ${tablePrefix}test_definition;`, + ); + await queryRunner.query(`ALTER TABLE ${tablePrefix}test_definition DROP COLUMN tmp_id;`); + } + // End of test_definition PK check + + await createTable(testMetricEntityTableName) + .withColumns( + column('id').varchar(36).primary.notNull, + column('name').varchar(255).notNull, + column('testDefinitionId').varchar(36).notNull, + ) + .withIndexOn('testDefinitionId') + .withForeignKey('testDefinitionId', { + tableName: 'test_definition', + columnName: 'id', + onDelete: 'CASCADE', + }).withTimestamps; + } + + async down({ schemaBuilder: { dropTable } }: MigrationContext) { + await dropTable(testMetricEntityTableName); + } +} diff --git a/packages/cli/src/databases/migrations/mysqldb/1736172058779-AddStatsColumnsToTestRun.ts b/packages/cli/src/databases/migrations/mysqldb/1736172058779-AddStatsColumnsToTestRun.ts new file mode 100644 index 0000000000..9f0b6ef3f5 --- /dev/null +++ b/packages/cli/src/databases/migrations/mysqldb/1736172058779-AddStatsColumnsToTestRun.ts @@ -0,0 +1,28 @@ +import type { MigrationContext, ReversibleMigration } from '@/databases/types'; + +const columns = ['totalCases', 'passedCases', 'failedCases'] as const; + +// Note: This migration was separated from common after release to remove column check constraints +// because they were causing issues with MySQL + +export class AddStatsColumnsToTestRun1736172058779 implements ReversibleMigration { + async up({ escape, runQuery }: MigrationContext) { + const tableName = escape.tableName('test_run'); + const columnNames = columns.map((name) => escape.columnName(name)); + + // Values can be NULL only if the test run is new, otherwise they must be non-negative integers. + // Test run might be cancelled or interrupted by unexpected error at any moment, so values can be either NULL or non-negative integers. + for (const name of columnNames) { + await runQuery(`ALTER TABLE ${tableName} ADD COLUMN ${name} INT;`); + } + } + + async down({ escape, runQuery }: MigrationContext) { + const tableName = escape.tableName('test_run'); + const columnNames = columns.map((name) => escape.columnName(name)); + + for (const name of columnNames) { + await runQuery(`ALTER TABLE ${tableName} DROP COLUMN ${name}`); + } + } +} diff --git a/packages/cli/src/databases/migrations/mysqldb/1739873751194-FixTestDefinitionPrimaryKey.ts b/packages/cli/src/databases/migrations/mysqldb/1739873751194-FixTestDefinitionPrimaryKey.ts new file mode 100644 index 0000000000..ee70fcc26e --- /dev/null +++ b/packages/cli/src/databases/migrations/mysqldb/1739873751194-FixTestDefinitionPrimaryKey.ts @@ -0,0 +1,44 @@ +import assert from 'node:assert'; + +import type { MigrationContext, IrreversibleMigration } from '@/databases/types'; + +export class FixTestDefinitionPrimaryKey1739873751194 implements IrreversibleMigration { + async up({ queryRunner, tablePrefix }: MigrationContext) { + /** + * MigrateTestDefinitionKeyToString migration for MySQL/MariaDB had missing part, + * and didn't complete primary key type change and deletion of the temporary column. + * + * This migration checks if table is in inconsistent state and finishes the primary key type change when needed. + * + * The MigrateTestDefinitionKeyToString migration has been patched to properly change the primary key. + * + * As the primary key issue might prevent the CreateTestMetricTable migration from running successfully on MySQL 8.4.4, + * the CreateTestMetricTable also contains the patch. + * + * For users who already ran the MigrateTestDefinitionKeyToString and CreateTestMetricTable, this migration should fix the primary key. + * For users who run these migrations in the same batch, this migration would be no-op, as the test_definition table should be already fixed + * by either of the previous patched migrations. + */ + + const table = await queryRunner.getTable(`${tablePrefix}test_definition`); + assert(table, 'test_definition table not found'); + + const brokenPrimaryColumn = table.primaryColumns.some( + (c) => c.name === 'tmp_id' && c.isPrimary, + ); + + if (brokenPrimaryColumn) { + // The migration was completed, but left the table in inconsistent state, let's finish the primary key change + await queryRunner.query( + `ALTER TABLE ${tablePrefix}test_definition MODIFY COLUMN tmp_id INT NOT NULL;`, + ); + await queryRunner.query( + `ALTER TABLE ${tablePrefix}test_definition DROP PRIMARY KEY, ADD PRIMARY KEY (\`id\`);`, + ); + await queryRunner.query( + `DROP INDEX \`TMP_idx_${tablePrefix}test_definition_id\` ON ${tablePrefix}test_definition;`, + ); + await queryRunner.query(`ALTER TABLE ${tablePrefix}test_definition DROP COLUMN tmp_id;`); + } + } +} diff --git a/packages/cli/src/databases/migrations/mysqldb/index.ts b/packages/cli/src/databases/migrations/mysqldb/index.ts index c339420a04..e16c387167 100644 --- a/packages/cli/src/databases/migrations/mysqldb/index.ts +++ b/packages/cli/src/databases/migrations/mysqldb/index.ts @@ -44,6 +44,9 @@ import { SeparateExecutionData1690000000030 } from './1690000000030-SeparateExec import { FixExecutionDataType1690000000031 } from './1690000000031-FixExecutionDataType'; import { AddActivatedAtUserSetting1717498465931 } from './1717498465931-AddActivatedAtUserSetting'; import { MigrateTestDefinitionKeyToString1731582748663 } from './1731582748663-MigrateTestDefinitionKeyToString'; +import { CreateTestMetricTable1732271325258 } from './1732271325258-CreateTestMetricTable'; +import { AddStatsColumnsToTestRun1736172058779 } from './1736172058779-AddStatsColumnsToTestRun'; +import { FixTestDefinitionPrimaryKey1739873751194 } from './1739873751194-FixTestDefinitionPrimaryKey'; import { CreateLdapEntities1674509946020 } from '../common/1674509946020-CreateLdapEntities'; import { PurgeInvalidWorkflowConnections1675940580449 } from '../common/1675940580449-PurgeInvalidWorkflowConnections'; import { RemoveResetPasswordColumns1690000000030 } from '../common/1690000000030-RemoveResetPasswordColumns'; @@ -72,11 +75,9 @@ import { UpdateProcessedDataValueColumnToText1729607673464 } from '../common/172 import { AddProjectIcons1729607673469 } from '../common/1729607673469-AddProjectIcons'; import { CreateTestDefinitionTable1730386903556 } from '../common/1730386903556-CreateTestDefinitionTable'; import { AddDescriptionToTestDefinition1731404028106 } from '../common/1731404028106-AddDescriptionToTestDefinition'; -import { CreateTestMetricTable1732271325258 } from '../common/1732271325258-CreateTestMetricTable'; import { CreateTestRun1732549866705 } from '../common/1732549866705-CreateTestRunTable'; import { AddMockedNodesColumnToTestDefinition1733133775640 } from '../common/1733133775640-AddMockedNodesColumnToTestDefinition'; import { AddManagedColumnToCredentialsTable1734479635324 } from '../common/1734479635324-AddManagedColumnToCredentialsTable'; -import { AddStatsColumnsToTestRun1736172058779 } from '../common/1736172058779-AddStatsColumnsToTestRun'; import { CreateTestCaseExecutionTable1736947513045 } from '../common/1736947513045-CreateTestCaseExecutionTable'; import { AddErrorColumnsToTestRuns1737715421462 } from '../common/1737715421462-AddErrorColumnsToTestRuns'; import { CreateFolderTable1738709609940 } from '../common/1738709609940-CreateFolderTable'; @@ -162,4 +163,5 @@ export const mysqlMigrations: Migration[] = [ CreateTestCaseExecutionTable1736947513045, AddErrorColumnsToTestRuns1737715421462, CreateFolderTable1738709609940, + FixTestDefinitionPrimaryKey1739873751194, ];