diff --git a/cypress/.eslintrc.js b/cypress/.eslintrc.js index 4d42f23154..59f4e11ea7 100644 --- a/cypress/.eslintrc.js +++ b/cypress/.eslintrc.js @@ -26,9 +26,15 @@ module.exports = { '@typescript-eslint/no-use-before-define': 'off', '@typescript-eslint/promise-function-async': 'off', 'n8n-local-rules/no-uncaught-json-parse': 'off', - 'cypress/no-assigning-return-values': 'warn', 'cypress/no-unnecessary-waiting': 'warn', 'cypress/unsafe-to-chain-command': 'warn', + 'import/no-extraneous-dependencies': [ + 'error', + { + devDependencies: ['**/cypress/**'], + optionalDependencies: false, + }, + ], }, }; diff --git a/cypress/README.md b/cypress/README.md new file mode 100644 index 0000000000..0c5835ea10 --- /dev/null +++ b/cypress/README.md @@ -0,0 +1,32 @@ +## Debugging Flaky End-to-End Tests - Usage + +To debug flaky end-to-end (E2E) tests, use the following command: + +```bash +pnpm run debug:flaky:e2e -- +``` + +**Parameters:** + +* ``: (Optional) A string to filter tests by their `it()` or `describe()` block titles, or by tags if using the `@cypress/grep` plugin. If omitted, all tests will be run. +* ``: (Optional) The number of times to run the filtered tests. Defaults to 5 if not provided. + +**Examples:** + +1. **Run all tests tagged with `@CAT-726` ten times:** + + ```bash + pnpm run debug:flaky:e2e -- @CAT-726 10 + ``` + +2. **Run all tests containing "login" five times (default burn count):** + + ```bash + pnpm run debug:flaky:e2e -- login + ``` + +3. **Run all tests five times (default grep and burn count):** + + ```bash + pnpm run debug:flaky:e2e + ``` diff --git a/cypress/composables/workflow.ts b/cypress/composables/workflow.ts index ff9788349b..e2474ae948 100644 --- a/cypress/composables/workflow.ts +++ b/cypress/composables/workflow.ts @@ -25,6 +25,12 @@ export type EndpointType = * Getters */ +export function executeWorkflowAndWait() { + cy.get('[data-test-id="execute-workflow-button"]').click(); + cy.contains('Workflow executed successfully', { timeout: 4000 }).should('be.visible'); + cy.contains('Workflow executed successfully', { timeout: 10000 }).should('not.exist'); +} + export function getCanvas() { return cy.getByTestId('canvas'); } diff --git a/cypress/cypress.config.js b/cypress/cypress.config.js index e7b953d6ca..aac10c0577 100644 --- a/cypress/cypress.config.js +++ b/cypress/cypress.config.js @@ -26,5 +26,9 @@ module.exports = defineConfig({ downloadsFolder: 'downloads', screenshotsFolder: 'screenshots', videosFolder: 'videos', + setupNodeEvents(on, config) { + require('@cypress/grep/src/plugin')(config); + return config; + }, }, }); diff --git a/cypress/e2e/726-CAT-canvas-node-connectors-not-rendered-when-nodes-inserted.cy.ts b/cypress/e2e/726-CAT-canvas-node-connectors-not-rendered-when-nodes-inserted.cy.ts index f0147187b7..25ce64e1d2 100644 --- a/cypress/e2e/726-CAT-canvas-node-connectors-not-rendered-when-nodes-inserted.cy.ts +++ b/cypress/e2e/726-CAT-canvas-node-connectors-not-rendered-when-nodes-inserted.cy.ts @@ -1,8 +1,8 @@ +import * as workflow from '../composables/workflow'; import { EDIT_FIELDS_SET_NODE_NAME, LOOP_OVER_ITEMS_NODE_NAME } from '../constants'; import { NodeCreator } from '../pages/features/node-creator'; import { NDV } from '../pages/ndv'; import { WorkflowPage as WorkflowPageClass } from '../pages/workflow'; - const nodeCreatorFeature = new NodeCreator(); const WorkflowPage = new WorkflowPageClass(); const NDVModal = new NDV(); @@ -18,7 +18,7 @@ describe('CAT-726 Node connectors not rendered when nodes inserted on the canvas nodeCreatorFeature.getters.getCreatorItem(EDIT_FIELDS_SET_NODE_NAME).click(); NDVModal.actions.close(); - WorkflowPage.actions.executeWorkflow(); + workflow.executeWorkflowAndWait(); cy.getByTestId('edge-label').realHover(); cy.getByTestId('add-connection-button').realClick(); diff --git a/cypress/package.json b/cypress/package.json index 97419b07ab..5ac39dd747 100644 --- a/cypress/package.json +++ b/cypress/package.json @@ -8,6 +8,7 @@ "test:e2e:dev": "scripts/run-e2e.js dev", "test:e2e:dev:v1": "scripts/run-e2e.js dev:v1", "test:e2e:all": "scripts/run-e2e.js all", + "test:flaky": "scripts/run-e2e.js debugFlaky", "format": "biome format --write .", "format:check": "biome ci .", "lint": "eslint . --quiet", @@ -16,6 +17,7 @@ "start": "cd ..; pnpm start" }, "devDependencies": { + "@cypress/grep": "^4.1.0", "@n8n/api-types": "workspace:*", "@types/lodash": "catalog:", "eslint-plugin-cypress": "^3.5.0", diff --git a/cypress/scripts/run-e2e.js b/cypress/scripts/run-e2e.js index c7f9ccf749..009085a1e8 100755 --- a/cypress/scripts/run-e2e.js +++ b/cypress/scripts/run-e2e.js @@ -85,6 +85,33 @@ switch (scenario) { }, }); break; + case 'debugFlaky': { + const filter = process.argv[3]; + const burnCount = process.argv[4] || 5; + + const envArgs = [`burn=${burnCount}`]; + + if (filter) { + envArgs.push(`grep=${filter}`); + envArgs.push(`grepFilterSpecs=true`); + } + + const envString = envArgs.join(','); + const testCommand = `cypress run --headless --env "${envString}"`; + + console.log(`Executing test command: ${testCommand}`); + + runTests({ + startCommand: 'start', + url: 'http://localhost:5678/favicon.ico', + testCommand: testCommand, + customEnv: { + CYPRESS_NODE_VIEW_VERSION: 2, + }, + failFast: true, + }); + break; + } default: console.error('Unknown scenario'); process.exit(1); diff --git a/cypress/support/e2e.ts b/cypress/support/e2e.ts index 297fcfa9b6..91fee5abff 100644 --- a/cypress/support/e2e.ts +++ b/cypress/support/e2e.ts @@ -1,8 +1,11 @@ +import registerCypressGrep from '@cypress/grep/src/support'; import cloneDeep from 'lodash/cloneDeep'; import merge from 'lodash/merge'; import { settings } from './commands'; +registerCypressGrep(); + before(() => { cy.resetDatabase(); diff --git a/cypress/support/types/cypress-grep.d.ts b/cypress/support/types/cypress-grep.d.ts new file mode 100644 index 0000000000..b27ed35246 --- /dev/null +++ b/cypress/support/types/cypress-grep.d.ts @@ -0,0 +1 @@ +declare module '@cypress/grep/src/support'; diff --git a/jest.config.js b/jest.config.js index d85107ac83..e49df10422 100644 --- a/jest.config.js +++ b/jest.config.js @@ -1,4 +1,4 @@ -const { pathsToModuleNameMapper } = require('ts-jest') +const { pathsToModuleNameMapper } = require('ts-jest'); const { compilerOptions } = require('get-tsconfig').getTsconfig().config; /** @type {import('ts-jest').TsJestGlobalOptions} */ @@ -11,7 +11,6 @@ const tsJestOptions = { }, }; - const isCoverageEnabled = process.env.COVERAGE_ENABLED === 'true'; /** @type {import('jest').Config} */ @@ -24,7 +23,11 @@ const config = { '^.+\\.ts$': ['ts-jest', tsJestOptions], }, // This resolve the path mappings from the tsconfig relative to each jest.config.js - moduleNameMapper: compilerOptions?.paths ? pathsToModuleNameMapper(compilerOptions.paths, { prefix: `${compilerOptions.baseUrl ? `/${compilerOptions.baseUrl.replace(/^\.\//, '')}` : ''}` }) : {}, + moduleNameMapper: compilerOptions?.paths + ? pathsToModuleNameMapper(compilerOptions.paths, { + prefix: `${compilerOptions.baseUrl ? `/${compilerOptions.baseUrl.replace(/^\.\//, '')}` : ''}`, + }) + : {}, setupFilesAfterEnv: ['jest-expect-message'], collectCoverage: isCoverageEnabled, coverageReporters: ['text-summary', 'lcov', 'html-spa'], diff --git a/package.json b/package.json index f574702bc8..768425a4c8 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,7 @@ "dev:fe:editor": "turbo run dev --parallel --env-mode=loose --filter=n8n-editor-ui", "dev:e2e": "cd cypress && pnpm run test:e2e:dev", "dev:e2e:v1": "cd cypress && pnpm run test:e2e:dev:v1", + "debug:flaky:e2e": "cd cypress && pnpm run test:flaky", "dev:e2e:server": "run-p start dev:fe:editor", "clean": "turbo run clean --parallel", "reset": "node scripts/ensure-zx.mjs && zx scripts/reset.mjs", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index bbb6a0e3ca..64c42c80e0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -284,6 +284,9 @@ importers: specifier: ^2.0.10 version: 2.0.10 devDependencies: + '@cypress/grep': + specifier: ^4.1.0 + version: 4.1.0(@babel/core@7.26.10)(cypress@13.14.2) '@n8n/api-types': specifier: workspace:* version: link:../packages/@n8n/api-types @@ -2842,6 +2845,12 @@ packages: peerDependencies: '@babel/core': ^7.0.0-0 + '@babel/plugin-syntax-jsx@7.25.9': + resolution: {integrity: sha512-ld6oezHQMZsZfp6pWtbjaNDF2tiiCYYDqQszHt5VV437lewP9aSi2Of99CK0D0XB21k7FLgnLcmQKyKzynfeAA==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + '@babel/plugin-syntax-logical-assignment-operators@7.10.4': resolution: {integrity: sha512-d8waShlpFDinQ5MtvGU9xDAOzKH47+FFoney2baFIoMr952hKOLp1HR7VszoZvOsV/4+RRszNY7D17ba0te0ig==} peerDependencies: @@ -3365,6 +3374,11 @@ packages: resolution: {integrity: sha512-/Z3l6pXthq0JvMYdUFyX9j0MaCltlIn6mfh9jLyQwg5aPKxkyNa0PTHtU1AlFXLNk55ZuAeJRcpvq+tmLfKmaQ==} engines: {node: '>=10'} + '@cypress/grep@4.1.0': + resolution: {integrity: sha512-yUscMiUgM28VDPrNxL19/BhgHZOVrAPrzVsuEcy6mqPqDYt8H8fIaHeeGQPW4CbMu/ry9sehjH561WDDBIXOIg==} + peerDependencies: + cypress: '>=10' + '@cypress/request@3.0.1': resolution: {integrity: sha512-TWivJlJi8ZDx2wGOw1dbLuHJKUYX7bWySw377nlnGOW3hP9/MUKIsEdXT/YngWxVdgNCHRBmFlBipE+5/2ZZlQ==} engines: {node: '>= 6'} @@ -3570,6 +3584,7 @@ packages: '@faker-js/faker@8.0.2': resolution: {integrity: sha512-Uo3pGspElQW91PCvKSIAXoEgAUlRnH29sX2/p89kg7sP1m2PzCufHINd0FhTXQf6DYGiUlVncdSPa2F9wxed2A==} engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0, npm: '>=6.14.13'} + deprecated: Please update to a newer version '@fastify/busboy@2.1.1': resolution: {integrity: sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA==} @@ -8631,6 +8646,10 @@ packages: find-package-json@1.2.0: resolution: {integrity: sha512-+SOGcLGYDJHtyqHd87ysBhmaeQ95oWspDKnMXBrnQ9Eq4OkLNqejgoaD8xVWu6GPa0B6roa6KinCMEMcVeqONw==} + find-test-names@1.29.7: + resolution: {integrity: sha512-Ps/+M9+rvYqR/gzvfjsfrdeypfSViGZ7Cn7clOGllTlwBcKVGqwfgllGBJ4XwzGp+PaEZZ1MbG4qT1qp4AD9DQ==} + hasBin: true + find-up@4.1.0: resolution: {integrity: sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==} engines: {node: '>=8'} @@ -12168,6 +12187,10 @@ packages: resolution: {integrity: sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==} engines: {node: '>=14'} + simple-bin-help@1.8.0: + resolution: {integrity: sha512-0LxHn+P1lF5r2WwVB/za3hLRIsYoLaNq1CXqjbrs3ZvLuvlWnRKrUjEWzV7umZL7hpQ7xULiQMV+0iXdRa5iFg==} + engines: {node: '>=14.16'} + simple-concat@1.0.1: resolution: {integrity: sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==} @@ -15075,6 +15098,11 @@ snapshots: '@babel/core': 7.26.10 '@babel/helper-plugin-utils': 7.26.5 + '@babel/plugin-syntax-jsx@7.25.9(@babel/core@7.26.10)': + dependencies: + '@babel/core': 7.26.10 + '@babel/helper-plugin-utils': 7.26.5 + '@babel/plugin-syntax-logical-assignment-operators@7.10.4(@babel/core@7.26.10)': dependencies: '@babel/core': 7.26.10 @@ -15750,6 +15778,16 @@ snapshots: '@ctrl/tinycolor@3.6.0': {} + '@cypress/grep@4.1.0(@babel/core@7.26.10)(cypress@13.14.2)': + dependencies: + cypress: 13.14.2 + debug: 4.4.0(supports-color@8.1.1) + find-test-names: 1.29.7(@babel/core@7.26.10) + globby: 11.1.0 + transitivePeerDependencies: + - '@babel/core' + - supports-color + '@cypress/request@3.0.1': dependencies: aws-sign2: 0.7.0 @@ -21616,7 +21654,7 @@ snapshots: ajv: 6.12.6 chalk: 4.1.2 cross-spawn: 7.0.6 - debug: 4.3.4 + debug: 4.4.0(supports-color@8.1.1) doctrine: 3.0.0 escape-string-regexp: 4.0.0 eslint-scope: 7.2.2 @@ -21971,6 +22009,18 @@ snapshots: find-package-json@1.2.0: {} + find-test-names@1.29.7(@babel/core@7.26.10): + dependencies: + '@babel/parser': 7.26.10 + '@babel/plugin-syntax-jsx': 7.25.9(@babel/core@7.26.10) + acorn-walk: 8.3.4 + debug: 4.4.0(supports-color@8.1.1) + globby: 11.1.0 + simple-bin-help: 1.8.0 + transitivePeerDependencies: + - '@babel/core' + - supports-color + find-up@4.1.0: dependencies: locate-path: 5.0.0 @@ -25952,6 +26002,8 @@ snapshots: onetime: 5.1.2 signal-exit: 3.0.7 + ret@0.1.15: {} + retry-axios@2.6.0(axios@1.8.2): dependencies: axios: 1.8.2 @@ -26298,6 +26350,8 @@ snapshots: signal-exit@4.1.0: {} + simple-bin-help@1.8.0: {} + simple-concat@1.0.1: {} simple-get@4.0.1: diff --git a/scripts/run-e2e.js b/scripts/run-e2e.js new file mode 100644 index 0000000000..e69de29bb2