ci: Improve Flaky Test Debugging and CAT-726 Fix (no-changelog) (#14298)

cypress/.eslintrc.js

@@ -26,9 +26,15 @@ module.exports = {
 		'@typescript-eslint/no-use-before-define': 'off',
 		'@typescript-eslint/promise-function-async': 'off',
 		'n8n-local-rules/no-uncaught-json-parse': 'off',
-
 		'cypress/no-assigning-return-values': 'warn',
 		'cypress/no-unnecessary-waiting': 'warn',
 		'cypress/unsafe-to-chain-command': 'warn',
+		'import/no-extraneous-dependencies': [
+			'error',
+			{
+				devDependencies: ['**/cypress/**'],
+				optionalDependencies: false,
+			},
+		],
 	},
 };

cypress/README.md

@@ -0,0 +1,32 @@
+## Debugging Flaky End-to-End Tests - Usage
+
+To debug flaky end-to-end (E2E) tests, use the following command:
+
+```bash
+pnpm run debug:flaky:e2e -- <grep_filter> <burn_count>
+```
+
+**Parameters:**
+
+* `<grep_filter>`: (Optional) A string to filter tests by their `it()` or `describe()` block titles, or by tags if using the `@cypress/grep` plugin. If omitted, all tests will be run.
+* `<burn_count>`: (Optional) The number of times to run the filtered tests. Defaults to 5 if not provided.
+
+**Examples:**
+
+1.  **Run all tests tagged with `@CAT-726` ten times:**
+
+    ```bash
+    pnpm run debug:flaky:e2e -- @CAT-726 10
+    ```
+
+2.  **Run all tests containing "login" five times (default burn count):**
+
+    ```bash
+    pnpm run debug:flaky:e2e -- login
+    ```
+
+3.  **Run all tests five times (default grep and burn count):**
+
+    ```bash
+    pnpm run debug:flaky:e2e
+    ```

cypress/composables/workflow.ts

@@ -25,6 +25,12 @@ export type EndpointType =
  * Getters
  */
 
+export function executeWorkflowAndWait() {
+	cy.get('[data-test-id="execute-workflow-button"]').click();
+	cy.contains('Workflow executed successfully', { timeout: 4000 }).should('be.visible');
+	cy.contains('Workflow executed successfully', { timeout: 10000 }).should('not.exist');
+}
+
 export function getCanvas() {
 	return cy.getByTestId('canvas');
 }

cypress/cypress.config.js

@@ -26,5 +26,9 @@ module.exports = defineConfig({
 		downloadsFolder: 'downloads',
 		screenshotsFolder: 'screenshots',
 		videosFolder: 'videos',
+		setupNodeEvents(on, config) {
+			require('@cypress/grep/src/plugin')(config);
+			return config;
+		},
 	},
 });

cypress/e2e/726-CAT-canvas-node-connectors-not-rendered-when-nodes-inserted.cy.ts

@@ -1,8 +1,8 @@
+import * as workflow from '../composables/workflow';
 import { EDIT_FIELDS_SET_NODE_NAME, LOOP_OVER_ITEMS_NODE_NAME } from '../constants';
 import { NodeCreator } from '../pages/features/node-creator';
 import { NDV } from '../pages/ndv';
 import { WorkflowPage as WorkflowPageClass } from '../pages/workflow';
-
 const nodeCreatorFeature = new NodeCreator();
 const WorkflowPage = new WorkflowPageClass();
 const NDVModal = new NDV();
@@ -18,7 +18,7 @@ describe('CAT-726 Node connectors not rendered when nodes inserted on the canvas
 		nodeCreatorFeature.getters.getCreatorItem(EDIT_FIELDS_SET_NODE_NAME).click();
 		NDVModal.actions.close();
 
-		WorkflowPage.actions.executeWorkflow();
+		workflow.executeWorkflowAndWait();
 
 		cy.getByTestId('edge-label').realHover();
 		cy.getByTestId('add-connection-button').realClick();

cypress/package.json

@@ -8,6 +8,7 @@
     "test:e2e:dev": "scripts/run-e2e.js dev",
     "test:e2e:dev:v1": "scripts/run-e2e.js dev:v1",
     "test:e2e:all": "scripts/run-e2e.js all",
+    "test:flaky": "scripts/run-e2e.js debugFlaky",
     "format": "biome format --write .",
     "format:check": "biome ci .",
     "lint": "eslint . --quiet",
@@ -16,6 +17,7 @@
     "start": "cd ..; pnpm start"
   },
   "devDependencies": {
+    "@cypress/grep": "^4.1.0",
     "@n8n/api-types": "workspace:*",
     "@types/lodash": "catalog:",
     "eslint-plugin-cypress": "^3.5.0",

cypress/scripts/run-e2e.js

@@ -85,6 +85,33 @@ switch (scenario) {
 			},
 		});
 		break;
+	case 'debugFlaky': {
+		const filter = process.argv[3];
+		const burnCount = process.argv[4] || 5;
+
+		const envArgs = [`burn=${burnCount}`];
+
+		if (filter) {
+			envArgs.push(`grep=${filter}`);
+			envArgs.push(`grepFilterSpecs=true`);
+		}
+
+		const envString = envArgs.join(',');
+		const testCommand = `cypress run --headless --env "${envString}"`;
+
+		console.log(`Executing test command: ${testCommand}`);
+
+		runTests({
+			startCommand: 'start',
+			url: 'http://localhost:5678/favicon.ico',
+			testCommand: testCommand,
+			customEnv: {
+				CYPRESS_NODE_VIEW_VERSION: 2,
+			},
+			failFast: true,
+		});
+		break;
+	}
 	default:
 		console.error('Unknown scenario');
 		process.exit(1);

cypress/support/e2e.ts

@@ -1,8 +1,11 @@
+import registerCypressGrep from '@cypress/grep/src/support';
 import cloneDeep from 'lodash/cloneDeep';
 import merge from 'lodash/merge';
 
 import { settings } from './commands';
 
+registerCypressGrep();
+
 before(() => {
 	cy.resetDatabase();
 

cypress/support/types/cypress-grep.d.ts

@@ -0,0 +1 @@
+declare module '@cypress/grep/src/support';

jest.config.js

@@ -1,4 +1,4 @@
-const { pathsToModuleNameMapper } = require('ts-jest')
+const { pathsToModuleNameMapper } = require('ts-jest');
 const { compilerOptions } = require('get-tsconfig').getTsconfig().config;
 
 /** @type {import('ts-jest').TsJestGlobalOptions} */
@@ -11,7 +11,6 @@ const tsJestOptions = {
 	},
 };
 
-
 const isCoverageEnabled = process.env.COVERAGE_ENABLED === 'true';
 
 /** @type {import('jest').Config} */
@@ -24,7 +23,11 @@ const config = {
 		'^.+\\.ts$': ['ts-jest', tsJestOptions],
 	},
 	// This resolve the path mappings from the tsconfig relative to each jest.config.js
-	moduleNameMapper: compilerOptions?.paths ? pathsToModuleNameMapper(compilerOptions.paths, { prefix: `<rootDir>${compilerOptions.baseUrl ? `/${compilerOptions.baseUrl.replace(/^\.\//, '')}` : ''}` }) : {},
+	moduleNameMapper: compilerOptions?.paths
+		? pathsToModuleNameMapper(compilerOptions.paths, {
+				prefix: `<rootDir>${compilerOptions.baseUrl ? `/${compilerOptions.baseUrl.replace(/^\.\//, '')}` : ''}`,
+			})
+		: {},
 	setupFilesAfterEnv: ['jest-expect-message'],
 	collectCoverage: isCoverageEnabled,
 	coverageReporters: ['text-summary', 'lcov', 'html-spa'],

package.json

@@ -22,6 +22,7 @@
     "dev:fe:editor": "turbo run dev --parallel --env-mode=loose --filter=n8n-editor-ui",
     "dev:e2e": "cd cypress && pnpm run test:e2e:dev",
     "dev:e2e:v1": "cd cypress && pnpm run test:e2e:dev:v1",
+    "debug:flaky:e2e": "cd cypress && pnpm run test:flaky",
     "dev:e2e:server": "run-p start dev:fe:editor",
     "clean": "turbo run clean --parallel",
     "reset": "node scripts/ensure-zx.mjs && zx scripts/reset.mjs",

pnpm-lock.yaml

@@ -284,6 +284,9 @@ importers:
         specifier: ^2.0.10
         version: 2.0.10
     devDependencies:
+      '@cypress/grep':
+        specifier: ^4.1.0
+        version: 4.1.0(@babel/core@7.26.10)(cypress@13.14.2)
       '@n8n/api-types':
         specifier: workspace:*
         version: link:../packages/@n8n/api-types
@@ -2842,6 +2845,12 @@ packages:
     peerDependencies:
       '@babel/core': ^7.0.0-0
 
+  '@babel/plugin-syntax-jsx@7.25.9':
+    resolution: {integrity: sha512-ld6oezHQMZsZfp6pWtbjaNDF2tiiCYYDqQszHt5VV437lewP9aSi2Of99CK0D0XB21k7FLgnLcmQKyKzynfeAA==}
+    engines: {node: '>=6.9.0'}
+    peerDependencies:
+      '@babel/core': ^7.0.0-0
+
   '@babel/plugin-syntax-logical-assignment-operators@7.10.4':
     resolution: {integrity: sha512-d8waShlpFDinQ5MtvGU9xDAOzKH47+FFoney2baFIoMr952hKOLp1HR7VszoZvOsV/4+RRszNY7D17ba0te0ig==}
     peerDependencies:
@@ -3365,6 +3374,11 @@ packages:
     resolution: {integrity: sha512-/Z3l6pXthq0JvMYdUFyX9j0MaCltlIn6mfh9jLyQwg5aPKxkyNa0PTHtU1AlFXLNk55ZuAeJRcpvq+tmLfKmaQ==}
     engines: {node: '>=10'}
 
+  '@cypress/grep@4.1.0':
+    resolution: {integrity: sha512-yUscMiUgM28VDPrNxL19/BhgHZOVrAPrzVsuEcy6mqPqDYt8H8fIaHeeGQPW4CbMu/ry9sehjH561WDDBIXOIg==}
+    peerDependencies:
+      cypress: '>=10'
+
   '@cypress/request@3.0.1':
     resolution: {integrity: sha512-TWivJlJi8ZDx2wGOw1dbLuHJKUYX7bWySw377nlnGOW3hP9/MUKIsEdXT/YngWxVdgNCHRBmFlBipE+5/2ZZlQ==}
     engines: {node: '>= 6'}
@@ -3570,6 +3584,7 @@ packages:
   '@faker-js/faker@8.0.2':
     resolution: {integrity: sha512-Uo3pGspElQW91PCvKSIAXoEgAUlRnH29sX2/p89kg7sP1m2PzCufHINd0FhTXQf6DYGiUlVncdSPa2F9wxed2A==}
     engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0, npm: '>=6.14.13'}
+    deprecated: Please update to a newer version
 
   '@fastify/busboy@2.1.1':
     resolution: {integrity: sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA==}
@@ -8631,6 +8646,10 @@ packages:
   find-package-json@1.2.0:
     resolution: {integrity: sha512-+SOGcLGYDJHtyqHd87ysBhmaeQ95oWspDKnMXBrnQ9Eq4OkLNqejgoaD8xVWu6GPa0B6roa6KinCMEMcVeqONw==}
 
+  find-test-names@1.29.7:
+    resolution: {integrity: sha512-Ps/+M9+rvYqR/gzvfjsfrdeypfSViGZ7Cn7clOGllTlwBcKVGqwfgllGBJ4XwzGp+PaEZZ1MbG4qT1qp4AD9DQ==}
+    hasBin: true
+
   find-up@4.1.0:
     resolution: {integrity: sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==}
     engines: {node: '>=8'}
@@ -12168,6 +12187,10 @@ packages:
     resolution: {integrity: sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==}
     engines: {node: '>=14'}
 
+  simple-bin-help@1.8.0:
+    resolution: {integrity: sha512-0LxHn+P1lF5r2WwVB/za3hLRIsYoLaNq1CXqjbrs3ZvLuvlWnRKrUjEWzV7umZL7hpQ7xULiQMV+0iXdRa5iFg==}
+    engines: {node: '>=14.16'}
+
   simple-concat@1.0.1:
     resolution: {integrity: sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==}
 
@@ -15075,6 +15098,11 @@ snapshots:
       '@babel/core': 7.26.10
       '@babel/helper-plugin-utils': 7.26.5
 
+  '@babel/plugin-syntax-jsx@7.25.9(@babel/core@7.26.10)':
+    dependencies:
+      '@babel/core': 7.26.10
+      '@babel/helper-plugin-utils': 7.26.5
+
   '@babel/plugin-syntax-logical-assignment-operators@7.10.4(@babel/core@7.26.10)':
     dependencies:
       '@babel/core': 7.26.10
@@ -15750,6 +15778,16 @@ snapshots:
 
   '@ctrl/tinycolor@3.6.0': {}
 
+  '@cypress/grep@4.1.0(@babel/core@7.26.10)(cypress@13.14.2)':
+    dependencies:
+      cypress: 13.14.2
+      debug: 4.4.0(supports-color@8.1.1)
+      find-test-names: 1.29.7(@babel/core@7.26.10)
+      globby: 11.1.0
+    transitivePeerDependencies:
+      - '@babel/core'
+      - supports-color
+
   '@cypress/request@3.0.1':
     dependencies:
       aws-sign2: 0.7.0
@@ -21616,7 +21654,7 @@ snapshots:
       ajv: 6.12.6
       chalk: 4.1.2
       cross-spawn: 7.0.6
-      debug: 4.3.4
+      debug: 4.4.0(supports-color@8.1.1)
       doctrine: 3.0.0
       escape-string-regexp: 4.0.0
       eslint-scope: 7.2.2
@@ -21971,6 +22009,18 @@ snapshots:
 
   find-package-json@1.2.0: {}
 
+  find-test-names@1.29.7(@babel/core@7.26.10):
+    dependencies:
+      '@babel/parser': 7.26.10
+      '@babel/plugin-syntax-jsx': 7.25.9(@babel/core@7.26.10)
+      acorn-walk: 8.3.4
+      debug: 4.4.0(supports-color@8.1.1)
+      globby: 11.1.0
+      simple-bin-help: 1.8.0
+    transitivePeerDependencies:
+      - '@babel/core'
+      - supports-color
+
   find-up@4.1.0:
     dependencies:
       locate-path: 5.0.0
@@ -25952,6 +26002,8 @@ snapshots:
       onetime: 5.1.2
       signal-exit: 3.0.7
 
+  ret@0.1.15: {}
+
   retry-axios@2.6.0(axios@1.8.2):
     dependencies:
       axios: 1.8.2
@@ -26298,6 +26350,8 @@ snapshots:
 
   signal-exit@4.1.0: {}
 
+  simple-bin-help@1.8.0: {}
+
   simple-concat@1.0.1: {}
 
   simple-get@4.0.1: