ci: Playwright project organization (#17905)

.github/workflows/playwright-test-reusable.yml

@@ -54,29 +54,19 @@ jobs:
         if: inputs.test-mode == 'docker-build'
         run: pnpm turbo install-browsers:ci
 
-      - name: Start Local Server
-        if: inputs.test-mode == 'local'
-        env:
-          E2E_TESTS: true
-        run: |
-          pnpm start &
-          npx wait-on http://localhost:5678 --timeout 15000
-
       - name: Run Tests (Local)
         if: inputs.test-mode == 'local'
         run: |
-          pnpm --filter=n8n-playwright test \
+          pnpm --filter=n8n-playwright test:local \
             --shard=${{ matrix.shard }}/${{ strategy.job-total }} \
             --workers=2
         env:
-          N8N_BASE_URL: http://localhost:5678
-          RESET_E2E_DB: true
           CURRENTS_RECORD_KEY: ${{ secrets.CURRENTS_RECORD_KEY }}
 
       - name: Run Tests (Docker)
         if: inputs.test-mode != 'local'
         run: |
-          pnpm --filter=n8n-playwright run test:standard \
+          pnpm --filter=n8n-playwright test:container:standard \
             --shard=${{ matrix.shard }}/${{ strategy.job-total }} \
             --workers=2
         env:

package.json

@@ -15,7 +15,7 @@
     "build:deploy": "node scripts/build-n8n.mjs",
     "build:docker": "node scripts/build-n8n.mjs && node scripts/dockerize-n8n.mjs",
     "build:docker:scan": "node scripts/build-n8n.mjs && node scripts/dockerize-n8n.mjs && node scripts/scan-n8n-image.mjs",
-    "build:docker:test": "node scripts/build-n8n.mjs && node scripts/dockerize-n8n.mjs && turbo run test:standard --filter=n8n-playwright",
+    "build:docker:test": "node scripts/build-n8n.mjs && node scripts/dockerize-n8n.mjs && turbo run test:container:standard --filter=n8n-playwright",
     "typecheck": "turbo typecheck",
     "dev": "turbo run dev --parallel --env-mode=loose --filter=!@n8n/design-system --filter=!@n8n/chat --filter=!@n8n/task-runner",
     "dev:be": "turbo run dev --parallel --env-mode=loose --filter=!@n8n/design-system --filter=!@n8n/chat --filter=!@n8n/task-runner --filter=!n8n-editor-ui",
@@ -43,7 +43,7 @@
     "test": "JEST_JUNIT_CLASSNAME={filepath} turbo run test",
     "test:ci": "turbo run test --continue --concurrency=1",
     "test:affected": "turbo run test --affected --concurrency=1",
-    "test:with:docker": "pnpm --filter=n8n-playwright run test:standard",
+    "test:with:docker": "pnpm --filter=n8n-playwright test:container:standard",
     "test:show:report": "pnpm --filter=n8n-playwright exec playwright show-report",
     "watch": "turbo run watch",
     "webhook": "./packages/cli/bin/n8n webhook",

packages/testing/playwright/README.md

@@ -2,20 +2,25 @@
 
 ## Quick Start
 ```bash
-pnpm test                 # Run all tests (fresh containers, pnpm build:local from root first to ensure local containers)
+pnpm test:all                 									# Run all tests (fresh containers, pnpm build:local from root first to ensure local containers)
-pnpm test:local           # Creates isolated n8n instance on port 5679 and runs the tests against it
+pnpm test:local           											# Starts a local server and runs the UI tests
+N8N_BASE_URL=localhost:5068 pnpm test:local			# Runs the UI tests against the instance running
 ```
 
 ## Test Commands
 ```bash
 # By Mode
-pnpm run test:standard    # Basic n8n
+pnpm test:container:standard    # Sqlite
-pnpm run test:postgres    # PostgreSQL
+pnpm test:container:postgres    # PostgreSQL
-pnpm run test:queue       # Queue mode
+pnpm test:container:queue       # Queue mode
-pnpm run test:multi-main  # HA setup
+pnpm test:container:multi-main  # HA setup
+
+pnpm test:performance						# Runs the performance tests against Sqlite container
+pnpm test:chaos									# Runs the chaos tests
+
 
 # Development
-pnpm test --grep "workflow"           # Pattern match
+pnpm test:all --grep "workflow"           # Pattern match, can run across all test types UI/cli-workflow/performance
 ```
 
 ## Test Tags

packages/testing/playwright/package.json

@@ -3,29 +3,31 @@
   "private": true,
   "scripts": {
     "test:all": "playwright test",
-    "start:isolated": "cd ..; N8N_PORT=5679 N8N_USER_FOLDER=/tmp/n8n-test-$(date +%s) E2E_TESTS=true pnpm start",
+    "test:local": "N8N_BASE_URL=http://localhost:5680 RESET_E2E_DB=true playwright test --project=*ui*",
-    "test:local": "RESET_E2E_DB=true N8N_BASE_URL=http://localhost:5679 start-server-and-test 'pnpm start:isolated' http://localhost:5679/favicon.ico 'sleep 1 && pnpm test:standard --workers 4'",
+    "test:ui": "playwright test --project=*ui*",
-    "test:standard": "playwright test --project=mode:standard*",
+    "test:performance": "playwright test --project=performance",
-    "test:postgres": "playwright test --project=mode:postgres*",
+    "test:chaos": "playwright test --project='*:chaos'",
-    "test:queue": "playwright test --project=mode:queue*",
+    "test:container:standard": "playwright test --project='standard:*'",
-    "test:multi-main": "playwright test --project=mode:multi-main*",
+    "test:container:postgres": "playwright test --project='postgres:*'",
-    "test:clean": "docker rm -f $(docker ps -aq --filter 'name=n8n-*') 2>/dev/null || true && docker network prune -f",
+    "test:container:queue": "playwright test --project='queue:*'",
-    "test:workflows:setup": "tsx test-workflows/setup-workflow-tests.ts",
+    "test:container:multi-main": "playwright test --project='multi-main:*'",
-    "test:workflows": "playwright test --project=mode:workflows",
+    "test:workflows:setup": "tsx ./tests/cli-workflows/setup-workflow-tests.ts",
-    "test:workflows:schema": "SCHEMA=true playwright test --project=mode:workflows",
+    "test:workflows": "playwright test --project=cli-workflows",
-    "test:workflows:update": "playwright test --project=mode:workflows --update-snapshots",
+    "test:workflows:schema": "SCHEMA=true playwright test --project=cli-workflows",
+    "test:workflows:update": "playwright test --project=cli-workflows --update-snapshots",
+    "install-browsers:local": "playwright install chromium --with-deps",
+    "install-browsers:ci": "PLAYWRIGHT_BROWSERS_PATH=./ms-playwright-cache playwright install chromium --with-deps",
+    "browsers:uninstall": "playwright uninstall --all",
     "lint": "eslint .",
-    "lint:fix": "eslint . --fix",
+    "lint:fix": "eslint . --fix"
-    "install-browsers:ci": "PLAYWRIGHT_BROWSERS_PATH=./ms-playwright-cache playwright install chromium --with-deps --no-shell",
-    "install-browsers:local": "playwright install chromium --with-deps --no-shell"
   },
   "devDependencies": {
-    "@currents/playwright": "1.14.1",
+    "@currents/playwright": "^1.15.3",
-    "@playwright/test": "1.53.0",
+    "@playwright/test": "1.54.2",
     "@types/lodash": "catalog:",
-    "eslint-plugin-playwright": "2.2.0",
+    "eslint-plugin-playwright": "2.2.2",
     "generate-schema": "2.6.0",
-    "json-diff": "1.0.6",
+    "n8n-containers": "workspace:*",
-    "n8n-containers": "workspace:*"
+    "tsx": "catalog:"
   }
 }

packages/testing/playwright/pages/CanvasPage.ts

@@ -139,7 +139,6 @@ export class CanvasPage extends BasePage {
 			this.clickByText('Import from File...'),
 		]);
 		await fileChooser.setFiles(resolveFromRoot('workflows', fixtureKey));
-		await this.page.waitForTimeout(250);
 
 		await this.clickByTestId('inline-edit-preview');
 		await this.fillByTestId('inline-edit-input', workflowName);

packages/testing/playwright/playwright-projects.ts

@@ -0,0 +1,90 @@
+import type { Project } from '@playwright/test';
+import type { N8NConfig } from 'n8n-containers/n8n-test-container-creation';
+
+// Tags that require test containers environment
+// These tests won't be run against local
+const CONTAINER_ONLY_TAGS = ['proxy', 'multi-node', 'postgres', 'queue', 'multi-main'];
+const CONTAINER_ONLY = new RegExp(`@capability:(${CONTAINER_ONLY_TAGS.join('|')})`);
+
+// Tags that need serial execution
+// These tests will be run AFTER the first run of the UI tests
+// In local run they are a "dependency" which means they will be skipped if earlier tests fail, not ideal but needed for isolation
+const SERIAL_EXECUTION = /@db:reset/;
+
+const CONTAINER_CONFIGS: Array<{ name: string; config: N8NConfig }> = [
+	{ name: 'standard', config: {} },
+	{ name: 'postgres', config: { postgres: true } },
+	{ name: 'queue', config: { queueMode: true } },
+	{ name: 'multi-main', config: { queueMode: { mains: 2, workers: 1 } } }, // Multi main is having timing issues on startup, needs to be resolved
+];
+
+export function getProjects(): Project[] {
+	const isLocal = !!process.env.N8N_BASE_URL;
+	const projects: Project[] = [];
+
+	if (isLocal) {
+		projects.push(
+			{
+				name: 'ui',
+				testDir: './tests/ui',
+				grepInvert: new RegExp([CONTAINER_ONLY.source, SERIAL_EXECUTION.source].join('|')),
+				fullyParallel: true,
+				use: { baseURL: process.env.N8N_BASE_URL },
+			},
+			{
+				name: 'ui:isolated',
+				testDir: './tests/ui',
+				grep: SERIAL_EXECUTION,
+				workers: 1,
+				dependencies: ['ui'],
+				use: { baseURL: process.env.N8N_BASE_URL },
+			},
+		);
+	} else {
+		for (const { name, config } of CONTAINER_CONFIGS) {
+			projects.push(
+				{
+					name: `${name}:ui`,
+					testDir: './tests/ui',
+					grepInvert: SERIAL_EXECUTION,
+					timeout: name === 'standard' ? 60000 : 180000, // 60 seconds for standard container test, 180 for containers to allow startup etc
+					fullyParallel: true,
+					use: { containerConfig: config },
+				},
+				{
+					name: `${name}:ui:isolated`,
+					testDir: './tests/ui',
+					grep: SERIAL_EXECUTION,
+					workers: 1,
+					use: { containerConfig: config },
+				},
+				{
+					name: `${name}:chaos`,
+					testDir: './tests/chaos',
+					grep: new RegExp(`@mode:${name}`),
+					workers: 1,
+					timeout: 180000,
+					use: { containerConfig: config },
+				},
+			);
+		}
+	}
+
+	projects.push({
+		name: 'cli-workflows',
+		testDir: './tests/cli-workflows',
+		fullyParallel: true,
+		timeout: 60000,
+	});
+
+	projects.push({
+		name: 'performance',
+		testDir: './tests/performance',
+		workers: 1,
+		timeout: 300000,
+		retries: 0,
+		use: { containerConfig: {} },
+	});
+
+	return projects;
+}

packages/testing/playwright/playwright.config.ts

@@ -1,143 +1,42 @@
 /* eslint-disable import-x/no-default-export */
 import { currentsReporter } from '@currents/playwright';
-import type { Project } from '@playwright/test';
 import { defineConfig } from '@playwright/test';
+import os from 'os';
 
 import currentsConfig from './currents.config';
+import { getProjects } from './playwright-projects';
+import { getPortFromUrl } from './utils/url-helper';
 
-// Type definitions for container configurations
+const IS_CI = !!process.env.CI;
-interface ContainerConfig {
-	postgres?: boolean;
-	queueMode?: {
-		mains: number;
-		workers: number;
-	};
-	env?: Record<string, string>;
-}
 
-interface ContainerConfigEntry {
+const MACBOOK_WINDOW_SIZE = { width: 1536, height: 960 };
-	name: string;
-	config: ContainerConfig;
-}
 
-/*
+// Calculate workers based on environment
- * Mode-based Test Configuration
+// The amount of workers to run, limited to 6 as higher causes instability in the local server
- *
+// Use half the CPUs in local, full in CI (CI has no other processes so we can use more)
- * Usage examples:
+const CPU_COUNT = os.cpus().length;
- *
+const LOCAL_WORKERS = Math.min(6, Math.floor(CPU_COUNT / 2));
- * 1. Run only mode:standard tests:
+const CI_WORKERS = CPU_COUNT;
- *    npx playwright test --project="mode:standard*"
+const WORKERS = IS_CI ? CI_WORKERS : LOCAL_WORKERS;
- *
- * 2. Run only parallel tests for all modes:
- *    npx playwright test --project="*Parallel"
- *
- * 3. Run a specific mode's sequential tests:
- *    npx playwright test --project="mode:multi-main - Sequential"
- *
- * Test tagging examples:
- *
- * // Runs on all modes
- * test('basic functionality', async ({ page }) => { ... });
- *
- * // Only runs on multi-main mode
- * test('multi-main specific @mode:multi-main', async ({ page }) => { ... });
- *
- * // Only runs on postgres mode, and in sequential execution
- * test('database reset test @mode:postgres @db:reset', async ({ page }) => { ... });
- *
- * // Runs on all modes, but in sequential execution
- * test('another reset test @db:reset', async ({ page }) => { ... });
- */
-
-// Container configurations
-const containerConfigs: ContainerConfigEntry[] = [
-	{ name: 'mode:standard', config: {} },
-	{ name: 'mode:postgres', config: { postgres: true } },
-	{ name: 'mode:queue', config: { queueMode: { mains: 1, workers: 1 } } },
-	{ name: 'mode:multi-main', config: { queueMode: { mains: 2, workers: 1 } } },
-];
-
-// Workflow tests are run in a separate project, since they are not run in parallel with the other tests
-const workflowProject: Project = {
-	name: 'mode:workflows',
-	testDir: './test-workflows',
-	testMatch: 'workflow-tests.spec.ts',
-	retries: process.env.CI ? 2 : 0,
-	fullyParallel: true,
-};
-
-// Parallel tests can run fully parallel on a worker
-// Sequential tests can run on a single worker, since the need a DB reset
-// Chaos tests can run on a single worker, since they can destroy containers etc, these need to be isolate from DB tests since they are destructive
-function createProjectTrio(name: string, containerConfig: ContainerConfig): Project[] {
-	const modeTag = `@${name}`;
-
-	// Parse custom env vars from command line
-	const customEnv = process.env.N8N_TEST_ENV ? JSON.parse(process.env.N8N_TEST_ENV) : {};
-
-	// Merge custom env vars into container config
-	const mergedConfig = {
-		...containerConfig,
-		env: {
-			...containerConfig.env,
-			...customEnv,
-		},
-	};
-
-	// Only add dependencies when using external URL (i.e., using containers)
-	// This is to stop DB reset tests from running in parallel with other tests when more than 1 worker is used
-	const shouldAddDependencies = process.env.N8N_BASE_URL;
-
-	return [
-		{
-			name: `${name} - Parallel`,
-			grep: new RegExp(
-				`${modeTag}(?!.*(@db:reset|@chaostest))|^(?!.*(@mode:|@db:reset|@chaostest))`,
-			),
-			testIgnore: '*examples*',
-			fullyParallel: true,
-			use: { containerConfig: mergedConfig },
-		},
-		{
-			name: `${name} - Sequential`,
-			grep: new RegExp(`${modeTag}.*@db:reset|@db:reset(?!.*@mode:)`),
-			fullyParallel: false,
-			testIgnore: '*examples*',
-			workers: 1,
-			...(shouldAddDependencies && { dependencies: [`${name} - Parallel`] }),
-			use: { containerConfig: mergedConfig },
-		},
-		{
-			name: `${name} - Chaos`,
-			grep: new RegExp(`${modeTag}.*@chaostest`),
-			testIgnore: '*examples*',
-			fullyParallel: false,
-			workers: 1,
-			use: { containerConfig: mergedConfig },
-			timeout: 120000,
-		},
-	];
-}
 
 export default defineConfig({
 	globalSetup: './global-setup.ts',
-	testDir: './tests',
+	forbidOnly: IS_CI,
-	forbidOnly: !!process.env.CI,
+	retries: IS_CI ? 2 : 0,
-	retries: process.env.CI ? 2 : 0,
+	workers: WORKERS,
-	workers: process.env.CI ? 2 : 8,
 	timeout: 60000,
 
-	reporter: process.env.CI
+	projects: getProjects(),
-		? [
+
-				['list'],
+	// We use this if an n8n url is passed in. If the server is already running, we reuse it.
-				['github'],
+	webServer: process.env.N8N_BASE_URL
-				['junit', { outputFile: process.env.PLAYWRIGHT_JUNIT_OUTPUT_NAME ?? 'results.xml' }],
+		? {
-				['html', { open: 'never' }],
+				command: `cd .. && N8N_PORT=${getPortFromUrl(process.env.N8N_BASE_URL)} N8N_USER_FOLDER=/${os.tmpdir()}/n8n-main-$(date +%s) E2E_TESTS=true pnpm start`,
-				['json', { outputFile: 'test-results.json' }],
+				url: `${process.env.N8N_BASE_URL}/favicon.ico`,
-				['blob'],
+				timeout: 20000,
-				currentsReporter(currentsConfig),
+				reuseExistingServer: true,
-			]
+			}
-		: [['html']],
+		: undefined,
 
 	use: {
 		trace: 'on',
@@ -145,18 +44,19 @@ export default defineConfig({
 		screenshot: 'on',
 		testIdAttribute: 'data-test-id',
 		headless: process.env.SHOW_BROWSER !== 'true',
-		viewport: { width: 1536, height: 960 },
+		viewport: MACBOOK_WINDOW_SIZE,
-		actionTimeout: 30000,
+		actionTimeout: 20000, // TODO: We might need to make this dynamic for container tests if we have low resource containers etc
 		navigationTimeout: 10000,
-		channel: 'chromium',
 	},
 
-	projects: process.env.N8N_BASE_URL
+	reporter: IS_CI
-		? containerConfigs
+		? [
-				.filter(({ name }) => name === 'mode:standard')
+				['list'],
-				.flatMap(({ name, config }) => createProjectTrio(name, config))
+				['github'],
-				.concat([workflowProject])
+				['junit', { outputFile: process.env.PLAYWRIGHT_JUNIT_OUTPUT_NAME ?? 'results.xml' }],
-		: containerConfigs
+				['html', { open: 'never' }],
-				.flatMap(({ name, config }) => createProjectTrio(name, config))
+				['json', { outputFile: 'test-results.json' }],
-				.concat([workflowProject]),
+				currentsReporter(currentsConfig),
+			]
+		: [['html']],
 });

packages/testing/playwright/tests/chaos/multimain.spec.ts

@@ -1,4 +1,4 @@
-import { test, expect } from '../fixtures/base';
+import { test, expect } from '../../fixtures/base';
 
 test('Leader election @mode:multi-main @chaostest', async ({ chaos }) => {
 	// First get the container (try main 1 first)

packages/testing/playwright/tests/cli-workflows/setup-workflow-tests.ts

@@ -3,13 +3,15 @@ import { promises as fsPromises } from 'fs';
 import path from 'path';
 import { promisify } from 'util';
 
+import { findPackagesRoot } from '../../utils/path-helper';
+
 // Only run the file once, so we don't run it multiple times
 const execFileAsync = promisify(execFile);
 
 const CREDENTIALS_FILE_NAME = 'credentials.json';
 const WORKFLOWS_DIR_NAME = 'workflows';
 
-const ASSETS_SOURCE_PATH = path.join(__dirname, '../../../../assets');
+const ASSETS_SOURCE_PATH = path.join(__dirname, '../../../../../assets');
 const PDF_SOURCE_DIR = path.join(__dirname, 'testData', 'pdfs');
 
 const BASE_TMP_DIR = '/tmp';
@@ -23,7 +25,8 @@ const TMP_PDF_DEST_DIR = path.join(BASE_TMP_DIR, 'testData', 'pdfs');
  * @returns A promise that resolves with the stdout of the command, or rejects on error.
  */
 async function runN8nCliCommand(command: string, args: string[], options: { cwd: string }) {
-	const n8nExecutablePath = '../../../cli/bin/n8n';
+	const packagesRoot = findPackagesRoot('cli');
+	const n8nExecutablePath = path.join(packagesRoot, 'cli/bin/n8n');
 	console.log(`Executing n8n command: n8n ${command} ${args.join(' ')}`);
 	await execFileAsync(n8nExecutablePath, [command, ...args], options);
 }

packages/testing/playwright/tests/cli-workflows/workflow-tests.spec.ts

@@ -7,10 +7,12 @@ import * as fs from 'fs';
 import GenerateSchema from 'generate-schema';
 import * as path from 'path';
 
+import { findPackagesRoot } from '../../utils/path-helper';
+
 // --- Configuration ---
 const IGNORE_SKIPLIST = process.env.IGNORE_SKIPLIST === 'true';
 const SCHEMA_MODE = process.env.SCHEMA === 'true';
-const WORKFLOWS_DIR = path.join(__dirname, '../test-workflows/workflows');
+const WORKFLOWS_DIR = path.join(__dirname, '../cli-workflows/workflows');
 const WORKFLOW_CONFIG_PATH = path.join(__dirname, 'workflowConfig.json');
 
 interface Workflow {
@@ -72,7 +74,9 @@ function loadWorkflows(): Workflow[] {
  * @returns An object containing the execution status, data, and any errors.
  */
 function executeWorkflow(workflowId: string): ExecutionResult {
-	const command = `../../cli/bin/n8n execute --id="${workflowId}"`;
+	const packagesRoot = findPackagesRoot('cli');
+	const n8nExecutablePath = path.join(packagesRoot, 'cli/bin/n8n');
+	const command = `"${n8nExecutablePath}" execute --id="${workflowId}"`;
 	const options = {
 		encoding: 'utf-8' as const,
 		maxBuffer: 10 * 1024 * 1024,