fix(core): Fix DB migrations for MySQL (#13261)

.github/docker-compose.yml

@@ -1,6 +1,6 @@
 services:
   mariadb:
-    image: mariadb:10.9
+    image: mariadb:10.5
     environment:
       - MARIADB_DATABASE=n8n
       - MARIADB_ROOT_PASSWORD=password
@@ -10,6 +10,26 @@ services:
     tmpfs:
       - /var/lib/mysql
 
+  mysql-8.0.13:
+    image: mysql:8.0.13
+    environment:
+      - MYSQL_DATABASE=n8n
+      - MYSQL_ROOT_PASSWORD=password
+    ports:
+      - 3306:3306
+    tmpfs:
+      - /var/lib/mysql
+
+  mysql-8.4:
+    image: mysql:8.4
+    environment:
+      - MYSQL_DATABASE=n8n
+      - MYSQL_ROOT_PASSWORD=password
+    ports:
+      - 3306:3306
+    tmpfs:
+      - /var/lib/mysql
+
   postgres:
     image: postgres:16
     restart: always

.github/workflows/ci-postgres-mysql.yml

@@ -123,6 +123,50 @@ jobs:
         working-directory: packages/cli
         run: pnpm test:mariadb --testTimeout 30000
 
+  mysql:
+    name: MySQL (${{ matrix.service-name }})
+    runs-on: ubuntu-latest
+    needs: build
+    timeout-minutes: 20
+    strategy:
+      matrix:
+        service-name: [ 'mysql-8.0.13', 'mysql-8.4' ]
+    env:
+      DB_MYSQLDB_PASSWORD: password
+    steps:
+      - uses: actions/checkout@v4.1.1
+
+      - uses: actions/setup-node@v4.2.0
+        with:
+          node-version: 20.x
+
+      - name: Setup corepack and pnpm
+        run: |
+          npm i -g corepack@0.31
+          corepack enable
+
+      - run: pnpm install --frozen-lockfile
+
+      - name: Setup build cache
+        uses: rharkor/caching-for-turbo@v1.5
+
+      - name: Restore cached build artifacts
+        uses: actions/cache/restore@v4.2.0
+        with:
+          path: ./packages/**/dist
+          key: ${{ github.sha }}:db-tests
+
+      - name: Start MySQL
+        uses: isbang/compose-action@v2.0.0
+        with:
+          compose-file: ./.github/docker-compose.yml
+          services: |
+            ${{ matrix.service-name }}
+
+      - name: Test MySQL
+        working-directory: packages/cli
+        run: pnpm test:mysql --testTimeout 30000
+
   postgres:
     name: Postgres
     runs-on: ubuntu-latest
@@ -168,7 +212,7 @@ jobs:
   notify-on-failure:
     name: Notify Slack on failure
     runs-on: ubuntu-latest
-    needs: [mariadb, postgres]
+    needs: [mariadb, postgres, mysql]
     steps:
       - name: Notify Slack on failure
         uses: act10ns/slack@v2.0.0
@@ -177,4 +221,4 @@ jobs:
           status: ${{ job.status }}
           channel: '#alerts-build'
           webhook-url: ${{ secrets.SLACK_WEBHOOK_URL }}
-          message: Postgres or MariaDB tests failed (${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})
+          message: Postgres, MariaDB or MySQL tests failed (${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})

packages/cli/package.json

@@ -29,6 +29,7 @@
     "test:sqlite": "N8N_LOG_LEVEL=silent DB_TYPE=sqlite jest",
     "test:postgres": "N8N_LOG_LEVEL=silent DB_TYPE=postgresdb DB_POSTGRESDB_SCHEMA=alt_schema DB_TABLE_PREFIX=test_ jest --no-coverage",
     "test:mariadb": "N8N_LOG_LEVEL=silent DB_TYPE=mariadb DB_TABLE_PREFIX=test_ jest --no-coverage",
+    "test:mysql": "N8N_LOG_LEVEL=silent DB_TYPE=mysqldb DB_TABLE_PREFIX=test_ jest --no-coverage",
     "watch": "tsc-watch -p tsconfig.build.json --onCompilationComplete \"tsc-alias -p tsconfig.build.json\""
   },
   "bin": {

packages/cli/src/databases/migrations/mysqldb/1731582748663-MigrateTestDefinitionKeyToString.ts

@@ -14,5 +14,19 @@ export class MigrateTestDefinitionKeyToString1731582748663 implements Irreversib
 		await queryRunner.query(
 			`CREATE INDEX \`TMP_idx_${tablePrefix}test_definition_id\` ON ${tablePrefix}test_definition (\`id\`);`,
 		);
+
+		// Note: this part was missing in initial release and was added after. Without it the migration run successfully,
+		// but left the table in inconsistent state, because it didn't finish changing the primary key and deleting the old one.
+		// This prevented the next migration from running on MySQL 8.4.4
+		await queryRunner.query(
+			`ALTER TABLE ${tablePrefix}test_definition MODIFY COLUMN tmp_id INT NOT NULL;`,
+		);
+		await queryRunner.query(
+			`ALTER TABLE ${tablePrefix}test_definition DROP PRIMARY KEY, ADD PRIMARY KEY (\`id\`);`,
+		);
+		await queryRunner.query(
+			`DROP INDEX \`TMP_idx_${tablePrefix}test_definition_id\` ON ${tablePrefix}test_definition;`,
+		);
+		await queryRunner.query(`ALTER TABLE ${tablePrefix}test_definition DROP COLUMN tmp_id;`);
 	}
 }

packages/cli/src/databases/migrations/mysqldb/1732271325258-CreateTestMetricTable.ts

@@ -0,0 +1,49 @@
+import assert from 'node:assert';
+
+import type { MigrationContext, ReversibleMigration } from '@/databases/types';
+
+const testMetricEntityTableName = 'test_metric';
+
+export class CreateTestMetricTable1732271325258 implements ReversibleMigration {
+	async up({ schemaBuilder: { createTable, column }, queryRunner, tablePrefix }: MigrationContext) {
+		// Check if the previous migration MigrateTestDefinitionKeyToString1731582748663 properly updated the primary key
+		const table = await queryRunner.getTable(`${tablePrefix}test_definition`);
+		assert(table, 'test_definition table not found');
+
+		const brokenPrimaryColumn = table.primaryColumns.some(
+			(c) => c.name === 'tmp_id' && c.isPrimary,
+		);
+
+		if (brokenPrimaryColumn) {
+			// The migration was completed, but left the table in inconsistent state, let's finish the primary key change
+			await queryRunner.query(
+				`ALTER TABLE ${tablePrefix}test_definition MODIFY COLUMN tmp_id INT NOT NULL;`,
+			);
+			await queryRunner.query(
+				`ALTER TABLE ${tablePrefix}test_definition DROP PRIMARY KEY, ADD PRIMARY KEY (\`id\`);`,
+			);
+			await queryRunner.query(
+				`DROP INDEX \`TMP_idx_${tablePrefix}test_definition_id\` ON ${tablePrefix}test_definition;`,
+			);
+			await queryRunner.query(`ALTER TABLE ${tablePrefix}test_definition DROP COLUMN tmp_id;`);
+		}
+		// End of test_definition PK check
+
+		await createTable(testMetricEntityTableName)
+			.withColumns(
+				column('id').varchar(36).primary.notNull,
+				column('name').varchar(255).notNull,
+				column('testDefinitionId').varchar(36).notNull,
+			)
+			.withIndexOn('testDefinitionId')
+			.withForeignKey('testDefinitionId', {
+				tableName: 'test_definition',
+				columnName: 'id',
+				onDelete: 'CASCADE',
+			}).withTimestamps;
+	}
+
+	async down({ schemaBuilder: { dropTable } }: MigrationContext) {
+		await dropTable(testMetricEntityTableName);
+	}
+}

packages/cli/src/databases/migrations/mysqldb/1736172058779-AddStatsColumnsToTestRun.ts

@@ -0,0 +1,28 @@
+import type { MigrationContext, ReversibleMigration } from '@/databases/types';
+
+const columns = ['totalCases', 'passedCases', 'failedCases'] as const;
+
+// Note: This migration was separated from common after release to remove column check constraints
+// because they were causing issues with MySQL
+
+export class AddStatsColumnsToTestRun1736172058779 implements ReversibleMigration {
+	async up({ escape, runQuery }: MigrationContext) {
+		const tableName = escape.tableName('test_run');
+		const columnNames = columns.map((name) => escape.columnName(name));
+
+		// Values can be NULL only if the test run is new, otherwise they must be non-negative integers.
+		// Test run might be cancelled or interrupted by unexpected error at any moment, so values can be either NULL or non-negative integers.
+		for (const name of columnNames) {
+			await runQuery(`ALTER TABLE ${tableName} ADD COLUMN ${name} INT;`);
+		}
+	}
+
+	async down({ escape, runQuery }: MigrationContext) {
+		const tableName = escape.tableName('test_run');
+		const columnNames = columns.map((name) => escape.columnName(name));
+
+		for (const name of columnNames) {
+			await runQuery(`ALTER TABLE ${tableName} DROP COLUMN ${name}`);
+		}
+	}
+}

packages/cli/src/databases/migrations/mysqldb/1739873751194-FixTestDefinitionPrimaryKey.ts

@@ -0,0 +1,44 @@
+import assert from 'node:assert';
+
+import type { MigrationContext, IrreversibleMigration } from '@/databases/types';
+
+export class FixTestDefinitionPrimaryKey1739873751194 implements IrreversibleMigration {
+	async up({ queryRunner, tablePrefix }: MigrationContext) {
+		/**
+		 * MigrateTestDefinitionKeyToString migration for MySQL/MariaDB had missing part,
+		 * and didn't complete primary key type change and deletion of the temporary column.
+		 *
+		 * This migration checks if table is in inconsistent state and finishes the primary key type change when needed.
+		 *
+		 * The MigrateTestDefinitionKeyToString migration has been patched to properly change the primary key.
+		 *
+		 * As the primary key issue might prevent the CreateTestMetricTable migration from running successfully on MySQL 8.4.4,
+		 * the CreateTestMetricTable also contains the patch.
+		 *
+		 * For users who already ran the MigrateTestDefinitionKeyToString and CreateTestMetricTable, this migration should fix the primary key.
+		 * For users who run these migrations in the same batch, this migration would be no-op, as the test_definition table should be already fixed
+		 * by either of the previous patched migrations.
+		 */
+
+		const table = await queryRunner.getTable(`${tablePrefix}test_definition`);
+		assert(table, 'test_definition table not found');
+
+		const brokenPrimaryColumn = table.primaryColumns.some(
+			(c) => c.name === 'tmp_id' && c.isPrimary,
+		);
+
+		if (brokenPrimaryColumn) {
+			// The migration was completed, but left the table in inconsistent state, let's finish the primary key change
+			await queryRunner.query(
+				`ALTER TABLE ${tablePrefix}test_definition MODIFY COLUMN tmp_id INT NOT NULL;`,
+			);
+			await queryRunner.query(
+				`ALTER TABLE ${tablePrefix}test_definition DROP PRIMARY KEY, ADD PRIMARY KEY (\`id\`);`,
+			);
+			await queryRunner.query(
+				`DROP INDEX \`TMP_idx_${tablePrefix}test_definition_id\` ON ${tablePrefix}test_definition;`,
+			);
+			await queryRunner.query(`ALTER TABLE ${tablePrefix}test_definition DROP COLUMN tmp_id;`);
+		}
+	}
+}

packages/cli/src/databases/migrations/mysqldb/index.ts

@@ -44,6 +44,9 @@ import { SeparateExecutionData1690000000030 } from './1690000000030-SeparateExec
 import { FixExecutionDataType1690000000031 } from './1690000000031-FixExecutionDataType';
 import { AddActivatedAtUserSetting1717498465931 } from './1717498465931-AddActivatedAtUserSetting';
 import { MigrateTestDefinitionKeyToString1731582748663 } from './1731582748663-MigrateTestDefinitionKeyToString';
+import { CreateTestMetricTable1732271325258 } from './1732271325258-CreateTestMetricTable';
+import { AddStatsColumnsToTestRun1736172058779 } from './1736172058779-AddStatsColumnsToTestRun';
+import { FixTestDefinitionPrimaryKey1739873751194 } from './1739873751194-FixTestDefinitionPrimaryKey';
 import { CreateLdapEntities1674509946020 } from '../common/1674509946020-CreateLdapEntities';
 import { PurgeInvalidWorkflowConnections1675940580449 } from '../common/1675940580449-PurgeInvalidWorkflowConnections';
 import { RemoveResetPasswordColumns1690000000030 } from '../common/1690000000030-RemoveResetPasswordColumns';
@@ -72,11 +75,9 @@ import { UpdateProcessedDataValueColumnToText1729607673464 } from '../common/172
 import { AddProjectIcons1729607673469 } from '../common/1729607673469-AddProjectIcons';
 import { CreateTestDefinitionTable1730386903556 } from '../common/1730386903556-CreateTestDefinitionTable';
 import { AddDescriptionToTestDefinition1731404028106 } from '../common/1731404028106-AddDescriptionToTestDefinition';
-import { CreateTestMetricTable1732271325258 } from '../common/1732271325258-CreateTestMetricTable';
 import { CreateTestRun1732549866705 } from '../common/1732549866705-CreateTestRunTable';
 import { AddMockedNodesColumnToTestDefinition1733133775640 } from '../common/1733133775640-AddMockedNodesColumnToTestDefinition';
 import { AddManagedColumnToCredentialsTable1734479635324 } from '../common/1734479635324-AddManagedColumnToCredentialsTable';
-import { AddStatsColumnsToTestRun1736172058779 } from '../common/1736172058779-AddStatsColumnsToTestRun';
 import { CreateTestCaseExecutionTable1736947513045 } from '../common/1736947513045-CreateTestCaseExecutionTable';
 import { AddErrorColumnsToTestRuns1737715421462 } from '../common/1737715421462-AddErrorColumnsToTestRuns';
 import { CreateFolderTable1738709609940 } from '../common/1738709609940-CreateFolderTable';
@@ -162,4 +163,5 @@ export const mysqlMigrations: Migration[] = [
 	CreateTestCaseExecutionTable1736947513045,
 	AddErrorColumnsToTestRuns1737715421462,
 	CreateFolderTable1738709609940,
+	FixTestDefinitionPrimaryKey1739873751194,
 ];