From d29e583c45da89821e679ea53f59aaecd973d07c Mon Sep 17 00:00:00 2001 From: shortstacked Date: Fri, 25 Jul 2025 15:45:40 +0100 Subject: [PATCH] test: Add performance tests to Playwright (#17574) --- .../composables/WorkflowComposer.ts | 8 +- packages/testing/playwright/package.json | 2 +- .../testing/playwright/playwright.config.ts | 5 +- .../playwright/tests/performance/README.md | 118 ++++++++++++++ .../tests/performance/perf-examples.spec.ts | 127 +++++++++++++++ .../playwright/utils/performance-helper.ts | 30 ++++ .../testing/playwright/workflows/large.json | 149 ++++++++++++++++++ 7 files changed, 435 insertions(+), 4 deletions(-) create mode 100644 packages/testing/playwright/tests/performance/README.md create mode 100644 packages/testing/playwright/tests/performance/perf-examples.spec.ts create mode 100644 packages/testing/playwright/utils/performance-helper.ts create mode 100644 packages/testing/playwright/workflows/large.json diff --git a/packages/testing/playwright/composables/WorkflowComposer.ts b/packages/testing/playwright/composables/WorkflowComposer.ts index 7014ac3bfa..263884244f 100644 --- a/packages/testing/playwright/composables/WorkflowComposer.ts +++ b/packages/testing/playwright/composables/WorkflowComposer.ts @@ -10,7 +10,11 @@ export class WorkflowComposer { * Executes a successful workflow and waits for the notification to be closed. * This waits for http calls and also closes the notification. */ - async executeWorkflowAndWaitForNotification(notificationMessage: string) { + async executeWorkflowAndWaitForNotification( + notificationMessage: string, + options: { timeout?: number } = {}, + ) { + const { timeout = 3000 } = options; const responsePromise = this.n8n.page.waitForResponse( (response) => response.url().includes('/rest/workflows/') && @@ -20,6 +24,6 @@ export class WorkflowComposer { await this.n8n.canvas.clickExecuteWorkflowButton(); await responsePromise; - await this.n8n.notifications.waitForNotificationAndClose(notificationMessage); + await this.n8n.notifications.waitForNotificationAndClose(notificationMessage, { timeout }); } } diff --git a/packages/testing/playwright/package.json b/packages/testing/playwright/package.json index 210f711e39..f060beed50 100644 --- a/packages/testing/playwright/package.json +++ b/packages/testing/playwright/package.json @@ -10,7 +10,7 @@ "test:queue": "playwright test --project=mode:queue*", "test:multi-main": "playwright test --project=mode:multi-main*", "test:clean": "docker rm -f $(docker ps -aq --filter 'name=n8n-*') 2>/dev/null || true && docker network prune -f", - "lint": "eslint . --quiet", + "lint": "eslint .", "lintfix": "eslint . --fix", "install-browsers:ci": "PLAYWRIGHT_BROWSERS_PATH=./ms-playwright-cache playwright install chromium --with-deps --no-shell", "install-browsers:local": "playwright install chromium --with-deps --no-shell" diff --git a/packages/testing/playwright/playwright.config.ts b/packages/testing/playwright/playwright.config.ts index 958bb456b8..064efafc0d 100644 --- a/packages/testing/playwright/playwright.config.ts +++ b/packages/testing/playwright/playwright.config.ts @@ -70,6 +70,7 @@ function createProjectTrio(name: string, containerConfig: any): Project[] { grep: new RegExp( `${modeTag}(?!.*(@db:reset|@chaostest))|^(?!.*(@mode:|@db:reset|@chaostest))`, ), + testIgnore: '*examples*', fullyParallel: true, use: { containerConfig: mergedConfig } as any, }, @@ -77,6 +78,7 @@ function createProjectTrio(name: string, containerConfig: any): Project[] { name: `${name} - Sequential`, grep: new RegExp(`${modeTag}.*@db:reset|@db:reset(?!.*@mode:)`), fullyParallel: false, + testIgnore: '*examples*', workers: 1, ...(shouldAddDependencies && { dependencies: [`${name} - Parallel`] }), use: { containerConfig: mergedConfig } as any, @@ -84,6 +86,7 @@ function createProjectTrio(name: string, containerConfig: any): Project[] { { name: `${name} - Chaos`, grep: new RegExp(`${modeTag}.*@chaostest`), + testIgnore: '*examples*', fullyParallel: false, workers: 1, use: { containerConfig: mergedConfig } as any, @@ -119,7 +122,7 @@ export default defineConfig({ testIdAttribute: 'data-test-id', headless: true, viewport: { width: 1536, height: 960 }, - actionTimeout: 10000, + actionTimeout: 30000, navigationTimeout: 10000, channel: 'chromium', }, diff --git a/packages/testing/playwright/tests/performance/README.md b/packages/testing/playwright/tests/performance/README.md new file mode 100644 index 0000000000..9b0b427780 --- /dev/null +++ b/packages/testing/playwright/tests/performance/README.md @@ -0,0 +1,118 @@ +# Performance Testing Helper + +A simple toolkit for measuring and asserting performance in Playwright tests. + +## Quick Start + +### "I just want to measure how long something takes" +```typescript +const duration = await measurePerformance(page, 'open-node', async () => { + await n8n.canvas.openNode('Code'); +}); +console.log(`Opening node took ${duration.toFixed(1)}ms`); +``` + +### "I want to ensure an action completes within a time limit" +```typescript +const openNodeDuration = await measurePerformance(page, 'open-node', async () => { + await n8n.canvas.openNode('Code'); +}); +expect(openNodeDuration).toBeLessThan(2000); // Must complete in under 2 seconds +``` + +### "I want to measure the same action multiple times" +```typescript +const stats = []; +for (let i = 0; i < 20; i++) { + const duration = await measurePerformance(page, `open-node-${i}`, async () => { + await n8n.canvas.openNode('Code'); + }); + await n8n.ndv.clickBackToCanvasButton(); + stats.push(duration); +} +const average = stats.reduce((a, b) => a + b, 0) / stats.length; +console.log(`Average: ${average.toFixed(1)}ms`); +expect(average).toBeLessThan(2000); +``` + +### "I want to set performance budgets for different actions" +```typescript +const budgets = { + triggerWorkflow: 8000, // 8 seconds + openLargeNode: 2500, // 2.5 seconds +}; + +// Measure workflow execution +const triggerDuration = await measurePerformance(page, 'trigger-workflow', async () => { + await n8n.workflowComposer.executeWorkflowAndWaitForNotification('Successful'); +}); +expect(triggerDuration).toBeLessThan(budgets.triggerWorkflow); + +// Measure node opening +const openDuration = await measurePerformance(page, 'open-large-node', async () => { + await n8n.canvas.openNode('Code'); +}); +expect(openDuration).toBeLessThan(budgets.openLargeNode); +``` + +### "I want to test performance with different data sizes" +```typescript +const testData = [ + { size: 30000, budgets: { triggerWorkflow: 8000, openLargeNode: 2500 } }, + { size: 60000, budgets: { triggerWorkflow: 15000, openLargeNode: 6000 } }, +]; + +testData.forEach(({ size, budgets }) => { + test(`performance - ${size.toLocaleString()} items`, async ({ page }) => { + // Setup test with specific data size + await setupTest(size); + + // Measure against size-specific budgets + const duration = await measurePerformance(page, 'trigger-workflow', async () => { + await n8n.workflowComposer.executeWorkflowAndWaitForNotification('Successful') + }); + expect(duration).toBeLessThan(budgets.triggerWorkflow); + }); +}); +``` + +### "I want to see all performance metrics from my test" +```typescript +// After running various performance measurements... +const allMetrics = await getAllPerformanceMetrics(page); +console.log('All performance metrics:', allMetrics); +// Output: { 'open-node': 1234.5, 'save-workflow': 567.8, ... } +``` + +### "I want to attach performance results to my test report" +```typescript +const allMetrics = await getAllPerformanceMetrics(page); +await test.info().attach('performance-metrics', { + body: JSON.stringify({ + dataSize: 30000, + metrics: allMetrics, + budgets: { triggerWorkflow: 8000, openLargeNode: 2500 }, + passed: { + triggerWorkflow: allMetrics['trigger-workflow'] < 8000, + openNode: allMetrics['open-large-node'] < 2500, + } + }, null, 2), + contentType: 'application/json', +}); +``` + +## API Reference + +### `measurePerformance(page, actionName, actionFn)` +Measures the duration of an async action using the Performance API. +- **Returns:** `Promise` - Duration in milliseconds + +### `getAllPerformanceMetrics(page)` +Retrieves all performance measurements from the current page. +- **Returns:** `Promise>` - Map of action names to durations + +## Tips + +- Use unique names for measurements in loops (e.g., `open-node-${i}`) to avoid conflicts +- Set realistic budgets - add some buffer to account for variance +- Consider different budgets for different data sizes or environments diff --git a/packages/testing/playwright/tests/performance/perf-examples.spec.ts b/packages/testing/playwright/tests/performance/perf-examples.spec.ts new file mode 100644 index 0000000000..6989d451cd --- /dev/null +++ b/packages/testing/playwright/tests/performance/perf-examples.spec.ts @@ -0,0 +1,127 @@ +import { test, expect } from '../../fixtures/base'; +import type { n8nPage } from '../../pages/n8nPage'; +import { getAllPerformanceMetrics, measurePerformance } from '../../utils/performance-helper'; + +async function setupPerformanceTest(n8n: n8nPage, size: number) { + await n8n.goHome(); + await n8n.workflows.clickNewWorkflowCard(); + await n8n.workflows.importWorkflow('large.json', 'Large Workflow'); + await n8n.notifications.closeNotificationByText('Successful'); + + // Configure data size + await n8n.canvas.openNode('Edit Fields'); + await n8n.page + .getByTestId('parameter-input-value') + .getByTestId('parameter-input-field') + .fill(size.toString()); + await n8n.ndv.clickBackToCanvasButton(); +} + +test.describe('Performance Example: Multiple sets}', () => { + const testData = [ + { + size: 30000, + timeout: 40000, + budgets: { + triggerWorkflow: 8000, // 8s budget (actual: 6.4s) + openLargeNode: 2500, // 2.5s budget (actual: 1.6s) + }, + }, + { + size: 60000, + timeout: 60000, + budgets: { + triggerWorkflow: 15000, // 15s budget (actual: 12.4s) + openLargeNode: 6000, // 6s budget (actual: 4.9s) + }, + }, + ]; + + testData.forEach(({ size, timeout, budgets }) => { + test(`workflow performance - ${size.toLocaleString()} items @db:reset`, async ({ n8n }) => { + test.setTimeout(timeout); + + // Setup workflow + await setupPerformanceTest(n8n, size); + + // Measure workflow execution + const triggerDuration = await measurePerformance(n8n.page, 'trigger-workflow', async () => { + await n8n.workflowComposer.executeWorkflowAndWaitForNotification('Successful', { + timeout: budgets.triggerWorkflow + 5000, // Add buffer + }); + }); + + // Assert trigger performance + expect(triggerDuration).toBeLessThan(budgets.triggerWorkflow); + console.log( + `✓ Trigger workflow (${size} items): ${triggerDuration.toFixed(1)}ms < ${budgets.triggerWorkflow}ms`, + ); + + // Measure node opening + const openNodeDuration = await measurePerformance(n8n.page, 'open-large-node', async () => { + await n8n.canvas.openNode('Code'); + }); + + // Assert node opening performance + expect(openNodeDuration).toBeLessThan(budgets.openLargeNode); + console.log( + `✓ Open node (${size} items): ${openNodeDuration.toFixed(1)}ms < ${budgets.openLargeNode}ms`, + ); + + // Get all metrics and attach to test report + const allMetrics = await getAllPerformanceMetrics(n8n.page); + console.log(`\nAll performance metrics for ${size.toLocaleString()} items:`, allMetrics); + + // Attach metrics to test report + await test.info().attach('performance-metrics', { + body: JSON.stringify( + { + dataSize: size, + metrics: allMetrics, + budgets, + passed: { + triggerWorkflow: triggerDuration < budgets.triggerWorkflow, + openNode: openNodeDuration < budgets.openLargeNode, + }, + }, + null, + 2, + ), + contentType: 'application/json', + }); + }); + }); +}); + +test('Performance Example: Multiple Loops in a single test @db:reset', async ({ n8n }) => { + await setupPerformanceTest(n8n, 30000); + const loopSize = 20; + const stats = []; + + await n8n.workflowComposer.executeWorkflowAndWaitForNotification('Successful'); + + for (let i = 0; i < loopSize; i++) { + // Measure node opening + const openNodeDuration = await measurePerformance(n8n.page, `open-node-${i}`, async () => { + await n8n.canvas.openNode('Code'); + }); + + stats.push(openNodeDuration); + await n8n.ndv.clickBackToCanvasButton(); + + console.log(`✓ Open node (${i + 1} of ${loopSize}): ${openNodeDuration.toFixed(1)}ms`); + } + // Get the average of the stats + const average = stats.reduce((a, b) => a + b, 0) / stats.length; + console.log(`Average open node duration: ${average.toFixed(1)}ms`); + expect(average).toBeLessThan(2000); +}); + +test('Performance Example: Aserting on a performance metric @db:reset', async ({ n8n }) => { + await setupPerformanceTest(n8n, 30000); + await n8n.workflowComposer.executeWorkflowAndWaitForNotification('Successful'); + const openNodeDuration = await measurePerformance(n8n.page, 'open-node', async () => { + await n8n.canvas.openNode('Code'); + }); + expect(openNodeDuration).toBeLessThan(2000); +}); diff --git a/packages/testing/playwright/utils/performance-helper.ts b/packages/testing/playwright/utils/performance-helper.ts new file mode 100644 index 0000000000..789dc24834 --- /dev/null +++ b/packages/testing/playwright/utils/performance-helper.ts @@ -0,0 +1,30 @@ +import type { Page } from '@playwright/test'; + +export async function measurePerformance( + page: Page, + actionName: string, + actionFn: () => Promise, +): Promise { + // Mark start + await page.evaluate((name) => performance.mark(`${name}-start`), actionName); + + // Execute action + await actionFn(); + + // Mark end and get duration + return await page.evaluate((name) => { + performance.mark(`${name}-end`); + performance.measure(name, `${name}-start`, `${name}-end`); + const measure = performance.getEntriesByName(name)[0] as PerformanceMeasure; + return measure.duration; + }, actionName); +} + +export async function getAllPerformanceMetrics(page: Page) { + return await page.evaluate(() => { + const metrics: Record = {}; + const measures = performance.getEntriesByType('measure') as PerformanceMeasure[]; + measures.forEach((m) => (metrics[m.name] = m.duration)); + return metrics; + }); +} diff --git a/packages/testing/playwright/workflows/large.json b/packages/testing/playwright/workflows/large.json new file mode 100644 index 0000000000..ab7c6d298c --- /dev/null +++ b/packages/testing/playwright/workflows/large.json @@ -0,0 +1,149 @@ +{ + "nodes": [ + { + "parameters": {}, + "type": "n8n-nodes-base.manualTrigger", + "typeVersion": 1, + "position": [-496, 192], + "id": "eb667a4b-8428-455e-9a09-2bc4b9d43ee6", + "name": "When clicking ‘Execute workflow’" + }, + { + "parameters": { + "jsCode": "// The array to hold our user data\nconst items = [];\nconst numberOfItems = Number($input.first().json.itemCount)\n\n// Loop 500 times to create 500 user objects\nfor (let i = 1; i <= numberOfItems; i++) {\n const user = {\n id: i,\n firstName: 'User',\n lastName: `${i}`,\n email: `user.${i}@example.com`,\n };\n items.push(user);\n}\n\n// Return the data in the format n8n expects.\n// Each object in the array becomes a separate item in the n8n workflow.\nreturn items.map(item => ({ json: item }));" + }, + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [-80, 192], + "id": "abb04bd3-f0c7-490c-a86f-76e473b9ece3", + "name": "Code" + }, + { + "parameters": { + "conditions": { + "options": { + "caseSensitive": true, + "leftValue": "", + "typeValidation": "strict", + "version": 2 + }, + "conditions": [ + { + "id": "90606208-ab4b-4b4e-a73e-d01f8c53a142", + "leftValue": "={{ $json.id }}", + "rightValue": 3000, + "operator": { + "type": "number", + "operation": "lt" + } + } + ], + "combinator": "and" + }, + "options": {} + }, + "type": "n8n-nodes-base.if", + "typeVersion": 2.2, + "position": [208, 0], + "id": "3a218f47-a535-4b6a-b8c8-5b21c48e578c", + "name": "If" + }, + { + "parameters": { + "jsCode": "// Loop over input items and add a new field called 'myNewField' to the JSON of each one\nfor (const item of $input.all()) {\n item.json.myNewField = 1;\n}\n\nreturn $input.all();" + }, + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [480, 272], + "id": "7df7bf46-4a1b-4cb5-a5ff-140eff995129", + "name": "Code1" + }, + { + "parameters": { + "jsCode": "// Loop over input items and add a new field called 'myNewField' to the JSON of each one\nfor (const item of $input.all()) {\n item.json.myNewField = 1;\n}\n\nreturn $input.all();" + }, + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [480, 80], + "id": "ab5a4ad6-bbf5-4b6b-a5ba-f6d46e14f598", + "name": "Code2" + }, + { + "parameters": { + "assignments": { + "assignments": [ + { + "id": "99ed2132-dca6-4e9f-b7e3-5612ad22ee29", + "name": "itemCount", + "value": "10000", + "type": "string" + } + ] + }, + "options": {} + }, + "type": "n8n-nodes-base.set", + "typeVersion": 3.4, + "position": [-288, 192], + "id": "ff650b7e-9b79-4403-b66f-0b11755d7999", + "name": "Edit Fields" + } + ], + "connections": { + "When clicking ‘Execute workflow’": { + "main": [ + [ + { + "node": "Edit Fields", + "type": "main", + "index": 0 + } + ] + ] + }, + "Code": { + "main": [ + [ + { + "node": "If", + "type": "main", + "index": 0 + } + ] + ] + }, + "If": { + "main": [ + [ + { + "node": "Code2", + "type": "main", + "index": 0 + } + ], + [ + { + "node": "Code1", + "type": "main", + "index": 0 + } + ] + ] + }, + "Edit Fields": { + "main": [ + [ + { + "node": "Code", + "type": "main", + "index": 0 + } + ] + ] + } + }, + "pinData": {}, + "meta": { + "instanceId": "9864951ea4472b9d0ea716c66bb2527efb446aa6309ab9f74077ab1db432402b" + } +}