ci: Increase Playwright test parallelism (#18484)

This commit is contained in:
shortstacked
2025-08-19 09:13:05 +01:00
committed by GitHub
parent 3386047321
commit e87395304d
15 changed files with 251 additions and 96 deletions

View File

@@ -30,7 +30,7 @@ on:
containers:
description: 'Number of containers to run tests in.'
required: false
default: '[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]'
default: '[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]'
type: string
pr_number:
description: 'PR number to run tests for.'

View File

@@ -11,7 +11,7 @@ on:
shards:
description: 'Shards for parallel execution'
required: false
default: '[1]'
default: '[1, 2]'
type: string
docker-image:
description: 'Docker image to use (for docker-pull mode)'
@@ -28,6 +28,7 @@ env:
NODE_OPTIONS: --max-old-space-size=3072
# Disable Ryuk to avoid issues with Docker since it needs privileged access, containers are cleaned on teardown anyway
TESTCONTAINERS_RYUK_DISABLED: true
PLAYWRIGHT_WORKERS: 3 # We have 2 CPUs on this runner but we can use more workers since it's low CPU intensive
jobs:
test:
@@ -35,7 +36,7 @@ jobs:
strategy:
fail-fast: false
matrix:
shard: ${{ fromJSON(inputs.shards || '[1]') }}
shard: ${{ fromJSON(inputs.shards || '[1, 2]') }}
name: Test (Shard ${{ matrix.shard }}/${{ strategy.job-total }})
steps:
@@ -61,7 +62,7 @@ jobs:
run: |
pnpm --filter=n8n-playwright test:local \
--shard=${{ matrix.shard }}/${{ strategy.job-total }} \
--workers=2
--workers=${{ env.PLAYWRIGHT_WORKERS }}
env:
CURRENTS_RECORD_KEY: ${{ secrets.CURRENTS_RECORD_KEY }}
@@ -70,7 +71,7 @@ jobs:
run: |
pnpm --filter=n8n-playwright test:container:standard \
--shard=${{ matrix.shard }}/${{ strategy.job-total }} \
--workers=2
--workers=${{ env.PLAYWRIGHT_WORKERS }}
env:
N8N_DOCKER_IMAGE: ${{ inputs.test-mode == 'docker-build' && 'n8nio/n8n:local' || inputs.docker-image }}
CURRENTS_RECORD_KEY: ${{ secrets.CURRENTS_RECORD_KEY }}

View File

@@ -1,3 +1,5 @@
import { nanoid } from 'nanoid';
import type { n8nPage } from '../pages/n8nPage';
export class ProjectComposer {
@@ -10,13 +12,10 @@ export class ProjectComposer {
*/
async createProject(projectName?: string) {
await this.n8n.page.getByTestId('universal-add').click();
await Promise.all([
this.n8n.page.waitForResponse('**/rest/projects/*'),
this.n8n.page.getByTestId('navigation-menu-item').filter({ hasText: 'Project' }).click(),
]);
await this.n8n.page.getByTestId('navigation-menu-item').filter({ hasText: 'Project' }).click();
await this.n8n.notifications.waitForNotificationAndClose('saved successfully');
await this.n8n.page.waitForLoadState();
const projectNameUnique = projectName ?? `Project ${Date.now()}`;
const projectNameUnique = projectName ?? `Project ${nanoid(8)}`;
await this.n8n.projectSettings.fillProjectName(projectNameUnique);
await this.n8n.projectSettings.clickSaveButton();
const projectId = this.extractProjectIdFromPage('projects', 'settings');

View File

@@ -36,7 +36,14 @@ export class WorkflowComposer {
async createWorkflow(workflowName = 'My New Workflow') {
await this.n8n.workflows.clickAddWorkflowButton();
await this.n8n.canvas.setWorkflowName(workflowName);
const responsePromise = this.n8n.page.waitForResponse(
(response) =>
response.url().includes('/rest/workflows') && response.request().method() === 'POST',
);
await this.n8n.canvas.saveWorkflow();
await responsePromise;
}
/**

View File

@@ -170,8 +170,11 @@ export class CanvasPage extends BasePage {
(response) =>
response.url().includes('/rest/workflows/') && response.request().method() === 'PATCH',
);
await this.page.getByTestId('workflow-activate-switch').click();
await responsePromise;
await this.page.waitForTimeout(200);
}
async clickZoomToFitButton(): Promise<void> {
@@ -269,10 +272,6 @@ export class CanvasPage extends BasePage {
await this.page.getByTestId('context-menu').getByText('Duplicate').click();
}
getCanvasNodes(): Locator {
return this.page.getByTestId('canvas-node');
}
nodeConnections(): Locator {
return this.page.locator('[data-test-id="edge"]');
}

View File

@@ -79,7 +79,7 @@ export class WorkflowsPage extends BasePage {
async shareWorkflow(workflowName: string) {
const workflow = this.getWorkflowByName(workflowName);
await workflow.getByTestId('workflow-card-actions').click();
await this.page.getByRole('menuitem', { name: 'Share' }).click();
await this.page.getByRole('menuitem', { name: 'Share...' }).click();
}
getArchiveMenuItem() {

View File

@@ -36,7 +36,6 @@ export function getProjects(): Project[] {
testDir: './tests/ui',
grep: SERIAL_EXECUTION,
workers: 1,
dependencies: ['ui'],
use: { baseURL: process.env.N8N_BASE_URL },
},
);

View File

@@ -2,6 +2,7 @@
import { currentsReporter } from '@currents/playwright';
import { defineConfig } from '@playwright/test';
import os from 'os';
import path from 'path';
import currentsConfig from './currents.config';
import { getProjects } from './playwright-projects';
@@ -11,6 +12,7 @@ const IS_CI = !!process.env.CI;
const MACBOOK_WINDOW_SIZE = { width: 1536, height: 960 };
const USER_FOLDER = path.join(os.tmpdir(), `n8n-main-${Date.now()}`);
// Calculate workers based on environment
// The amount of workers to run, limited to 6 as higher causes instability in the local server
// Use half the CPUs in local, full in CI (CI has no other processes so we can use more)
@@ -31,10 +33,18 @@ export default defineConfig({
// We use this if an n8n url is passed in. If the server is already running, we reuse it.
webServer: process.env.N8N_BASE_URL
? {
command: `cd .. && N8N_PORT=${getPortFromUrl(process.env.N8N_BASE_URL)} N8N_USER_FOLDER=/${os.tmpdir()}/n8n-main-$(date +%s) E2E_TESTS=true pnpm start`,
command: 'cd .. && pnpm start',
url: `${process.env.N8N_BASE_URL}/favicon.ico`,
timeout: 20000,
reuseExistingServer: true,
env: {
DB_SQLITE_POOL_SIZE: '40',
E2E_TESTS: 'true',
N8N_PORT: getPortFromUrl(process.env.N8N_BASE_URL),
N8N_USER_FOLDER: USER_FOLDER,
N8N_LOG_LEVEL: 'debug',
N8N_METRICS: 'true',
},
}
: undefined,

View File

@@ -0,0 +1,160 @@
import { readFileSync } from 'fs';
import { nanoid } from 'nanoid';
import { setTimeout } from 'timers/promises';
import type { ApiHelpers } from './api-helper';
import { TestError } from '../Types';
import { resolveFromRoot } from '../utils/path-helper';
type WorkflowDefinition = {
name?: string;
active?: boolean;
nodes: Array<{
id?: string;
name?: string;
type: string;
typeVersion?: number;
position?: [number, number];
webhookId?: string;
parameters: { [key: string]: unknown } & { path?: string };
}>;
connections?: Record<string, unknown>;
};
/**
* Generate and assign a unique webhook id and path to the first Webhook node in a workflow.
*
* - Uniqueness: Uses nanoid to ensure both the internal `webhookId` and external `parameters.path`
* are unique per call, avoiding collisions across parallel tests and instances.
* - Path format: `${prefix}-${nanoid}` (default prefix: `test-webhook`).
* - Mutation: Updates the passed-in `workflow` object in-place.
*/
export function applyUniqueWebhookIds(
workflow: WorkflowDefinition,
options?: { prefix?: string; idLength?: number },
) {
const idLength = options?.idLength ?? 12;
const prefix = options?.prefix ?? 'test-webhook';
const generatedId = nanoid(idLength);
const generatedPath = `${prefix}-${generatedId}`;
for (const node of workflow.nodes) {
if (node.type === 'n8n-nodes-base.webhook') {
node.webhookId = generatedId;
node.parameters.path = generatedPath;
}
}
return { webhookId: generatedId, webhookPath: generatedPath, workflow };
}
/**
* Create a webhook workflow from an in-memory definition after assigning unique webhook id/path.
*
* Returns the externally callable `webhookPath` (what to pass to triggerWebhook)
* and the `workflowId` created by the API.
*/
export async function createWebhookWorkflow(
api: ApiHelpers,
workflow: WorkflowDefinition,
options?: { prefix?: string; idLength?: number },
) {
const { webhookPath } = applyUniqueWebhookIds(workflow, options);
const createdWorkflow = await api.workflowApi.createWorkflow(workflow as object);
const workflowId = createdWorkflow.id as string;
return { webhookPath, workflowId, createdWorkflow };
}
/**
* Import a webhook workflow from `packages/testing/playwright/workflows/{fileName}` and create it
* with a unique webhook id/path.
*/
export async function importWebhookWorkflow(
api: ApiHelpers,
fileName: string,
options?: { prefix?: string; idLength?: number },
) {
const workflowDefinition = JSON.parse(
readFileSync(resolveFromRoot('workflows', fileName), 'utf8'),
) as WorkflowDefinition;
return await createWebhookWorkflow(api, workflowDefinition, options);
}
/**
* Convenience: import a webhook workflow from file, ensure unique webhook id/path, and activate it.
*
* Returns the `webhookPath` to call and the `workflowId` for follow-up assertions.
*/
export async function importAndActivateWebhookWorkflow(
api: ApiHelpers,
fileName: string,
options?: { prefix?: string; idLength?: number },
) {
const { webhookPath, workflowId, createdWorkflow } = await importWebhookWorkflow(
api,
fileName,
options,
);
await setTimeout(500);
await api.workflowApi.setActive(workflowId, true);
return { webhookPath, workflowId, createdWorkflow };
}
/**
* Convenience: create a webhook workflow from an in-memory definition and activate it.
*/
export async function createAndActivateWebhookWorkflow(
api: ApiHelpers,
workflow: WorkflowDefinition,
options?: { prefix?: string; idLength?: number },
) {
const { webhookPath, workflowId, createdWorkflow } = await createWebhookWorkflow(
api,
workflow,
options,
);
// Timing issue between workflow creation and activation
await setTimeout(500);
await api.workflowApi.setActive(workflowId, true);
return { webhookPath, workflowId, createdWorkflow };
}
/**
* Trigger a webhook endpoint with optional data and parameters.
*
* @param api - The API helpers instance
* @param path - The webhook path (without /webhook/ prefix)
* @param options - Configuration for the webhook request
*/
export async function triggerWebhook(
api: ApiHelpers,
path: string,
options: { method?: 'GET' | 'POST'; data?: object; params?: Record<string, string> } = {},
) {
const { method = 'POST', data, params } = options;
let url = `/webhook/${path}`;
if (params && Object.keys(params).length > 0) {
const searchParams = new URLSearchParams(params);
url += `?${searchParams.toString()}`;
}
const requestOptions: Record<string, unknown> = {
headers: { 'Content-Type': 'application/json' },
};
if (data && method === 'POST') {
requestOptions.data = data;
}
const response =
method === 'GET' ? await api.request.get(url) : await api.request.post(url, requestOptions);
if (!response.ok()) {
throw new TestError(`Webhook trigger failed: ${await response.text()}`);
}
return response;
}

View File

@@ -16,7 +16,7 @@ export class WorkflowApiHelper {
}
async setActive(workflowId: string, active: boolean) {
const response = await this.api.request.patch(`/rest/workflows/${workflowId}`, {
const response = await this.api.request.patch(`/rest/workflows/${workflowId}?forceSave=true`, {
data: { active },
});
@@ -89,36 +89,4 @@ export class WorkflowApiHelper {
throw new TestError(`Execution did not complete within ${timeoutMs}ms`);
}
async triggerWebhook(
path: string,
options: { method?: 'GET' | 'POST'; data?: object; params?: Record<string, string> } = {},
) {
const { method = 'POST', data, params } = options;
let url = `/webhook/${path}`;
if (params && Object.keys(params).length > 0) {
const searchParams = new URLSearchParams(params);
url += `?${searchParams.toString()}`;
}
const requestOptions: Record<string, unknown> = {
headers: { 'Content-Type': 'application/json' },
};
if (data && method === 'POST') {
requestOptions.data = data;
}
const response =
method === 'GET'
? await this.api.request.get(url)
: await this.api.request.post(url, requestOptions);
if (!response.ok()) {
throw new TestError(`Webhook trigger failed: ${await response.text()}`);
}
return response;
}
}

View File

@@ -38,7 +38,7 @@ test.describe('Performance Example: Multiple sets}', () => {
];
testData.forEach(({ size, timeout, budgets }) => {
test(`workflow performance - ${size.toLocaleString()} items @db:reset`, async ({ n8n }) => {
test(`workflow performance - ${size.toLocaleString()} items`, async ({ n8n }) => {
test.setTimeout(timeout);
// Setup workflow
@@ -93,7 +93,7 @@ test.describe('Performance Example: Multiple sets}', () => {
});
});
test('Performance Example: Multiple Loops in a single test @db:reset', async ({ n8n }) => {
test('Performance Example: Multiple Loops in a single test', async ({ n8n }) => {
await setupPerformanceTest(n8n, 30000);
const loopSize = 20;
const stats = [];
@@ -117,7 +117,7 @@ test('Performance Example: Multiple Loops in a single test @db:reset', async ({
expect(average).toBeLessThan(2000);
});
test('Performance Example: Aserting on a performance metric @db:reset', async ({ n8n }) => {
test('Performance Example: Aserting on a performance metric', async ({ n8n }) => {
await setupPerformanceTest(n8n, 30000);
await n8n.workflowComposer.executeWorkflowAndWaitForNotification('Successful');
const openNodeDuration = await measurePerformance(n8n.page, 'open-node', async () => {

View File

@@ -10,14 +10,21 @@ const NOTIFICATIONS = {
};
test.describe('Workflows', () => {
test.beforeEach(async ({ n8n }) => {
test.beforeEach(async ({ n8n, api }) => {
await api.enableFeature('sharing');
await api.enableFeature('folders');
await api.enableFeature('advancedPermissions');
await api.enableFeature('projectRole:admin');
await api.enableFeature('projectRole:editor');
await api.setMaxTeamProjectsQuota(-1);
await n8n.goHome();
});
test('should create a new workflow using empty state card @db:reset', async ({ n8n }) => {
test('should create a new workflow using empty state card', async ({ n8n }) => {
const { projectId } = await n8n.projectComposer.createProject();
await n8n.page.goto(`projects/${projectId}/workflows`);
await n8n.workflows.clickNewWorkflowCard();
await n8n.canvas.importWorkflow('Test_workflow_1.json', 'Empty State Card Workflow');
await expect(n8n.canvas.getWorkflowTags()).toHaveText(['some-tag-1', 'some-tag-2']);
await expect(n8n.page).toHaveURL(/workflow\/new/);
});
test('should create a new workflow using add workflow button and save successfully', async ({
@@ -25,7 +32,8 @@ test.describe('Workflows', () => {
}) => {
await n8n.workflows.clickAddWorkflowButton();
const workflowName = `Test Workflow ${Date.now()}`;
const uniqueIdForCreate = nanoid(8);
const workflowName = `Test Workflow ${uniqueIdForCreate}`;
await n8n.canvas.setWorkflowName(workflowName);
await n8n.canvas.clickSaveWorkflowButton();
@@ -60,7 +68,8 @@ test.describe('Workflows', () => {
});
test('should archive and unarchive a workflow', async ({ n8n }) => {
const workflowName = `Archive Test ${Date.now()}`;
const uniqueIdForArchive = nanoid(8);
const workflowName = `Archive Test ${uniqueIdForArchive}`;
await n8n.workflowComposer.createWorkflow(workflowName);
await n8n.goHome();
@@ -81,7 +90,8 @@ test.describe('Workflows', () => {
});
test('should delete an archived workflow', async ({ n8n }) => {
const workflowName = `Delete Test ${Date.now()}`;
const uniqueIdForDelete = nanoid(8);
const workflowName = `Delete Test ${uniqueIdForDelete}`;
await n8n.workflowComposer.createWorkflow(workflowName);
await n8n.goHome();
await n8n.workflowComposer.createWorkflow();
@@ -99,43 +109,49 @@ test.describe('Workflows', () => {
await expect(workflow).toBeHidden();
});
test('should filter workflows by tag @db:reset', async ({ n8n }) => {
const taggedWorkflow =
await n8n.workflowComposer.createWorkflowFromJsonFile('Test_workflow_1.json');
await n8n.workflowComposer.createWorkflowFromJsonFile('Test_workflow_2.json');
test('should filter workflows by tag', async ({ n8n }) => {
const { projectId } = await n8n.projectComposer.createProject();
await n8n.page.goto(`projects/${projectId}/workflows`);
// Create tagged workflow
const uniqueIdForTagged = nanoid(8);
await n8n.workflowComposer.createWorkflow(uniqueIdForTagged);
await expect(n8n.canvas.getWorkflowSaveButton()).toContainText('Saved');
const tags = await n8n.canvas.addTags();
await n8n.goHome();
await n8n.workflows.filterByTag('some-tag-1');
// Create untagged workflow
await n8n.workflowComposer.createWorkflow();
await n8n.goHome();
await n8n.workflows.filterByTag(tags[0]);
await expect(n8n.workflows.getWorkflowByName(taggedWorkflow.workflowName)).toBeVisible();
await expect(n8n.workflows.getWorkflowByName(uniqueIdForTagged)).toBeVisible();
});
test('should preserve search and filters in URL @db:reset', async ({ n8n }) => {
const date = Date.now();
await n8n.workflowComposer.createWorkflowFromJsonFile(
'Test_workflow_2.json',
`My Tagged Workflow ${date}`,
);
test('should preserve search and filters in URL', async ({ n8n }) => {
const { projectId } = await n8n.projectComposer.createProject();
await n8n.page.goto(`projects/${projectId}/workflows`);
const uniqueIdForTagged = nanoid(8);
await n8n.workflowComposer.createWorkflow(`My Tagged Workflow ${uniqueIdForTagged}`);
await expect(n8n.canvas.getWorkflowSaveButton()).toContainText('Saved');
const tags = await n8n.canvas.addTags(2);
await n8n.goHome();
// Apply search
await n8n.workflows.searchWorkflows('Tagged');
await n8n.workflows.filterByTag(tags[0]);
// Apply tag filter
await n8n.workflows.filterByTag('other-tag-1');
// Verify URL contains filters
await expect(n8n.page).toHaveURL(/search=Tagged/);
// Reload and verify filters persist
await n8n.page.reload();
await expect(n8n.workflows.getSearchBar()).toHaveValue('Tagged');
await expect(n8n.workflows.getWorkflowByName(`My Tagged Workflow ${date}`)).toBeVisible();
await expect(
n8n.workflows.getWorkflowByName(`My Tagged Workflow ${uniqueIdForTagged}`),
).toBeVisible();
});
test('should share a workflow', async ({ n8n }) => {
const workflowName = `Share Test ${Date.now()}`;
const uniqueIdForShare = nanoid(8);
const workflowName = `Share Test ${uniqueIdForShare}`;
await n8n.workflowComposer.createWorkflow(workflowName);
await n8n.goHome();

View File

@@ -211,11 +211,14 @@ test.describe('Data pinning', () => {
setupRequirements,
}) => {
await setupRequirements(webhookTestRequirements);
await expect(n8n.canvas.getWorkflowSaveButton()).toContainText('Saved');
await n8n.page.waitForTimeout(500);
await n8n.canvas.activateWorkflow();
await n8n.page.waitForTimeout(500);
const webhookUrl = '/webhook/b0d79ddb-df2d-49b1-8555-9fa2b482608f';
const response = await n8n.ndv.makeWebhookRequest(webhookUrl);
expect(response.status()).toBe(200);
expect(response.status(), 'Webhook response is: ' + (await response.text())).toBe(200);
const responseBody = await response.json();
expect(responseBody).toEqual({ nodeData: 'pin' });

View File

@@ -17,7 +17,7 @@ async function getCredentialsForProject(api: ApiHelpers, projectId?: string) {
return await api.get('/rest/credentials', params);
}
test.describe('Projects @db:reset', () => {
test.describe('Projects', () => {
test.beforeEach(async ({ api, n8n }) => {
await api.enableFeature('sharing');
await api.enableFeature('folders');

View File

@@ -1,25 +1,18 @@
import { readFileSync } from 'fs';
import { test, expect } from '../../fixtures/base';
import { resolveFromRoot } from '../../utils/path-helper';
import { importAndActivateWebhookWorkflow, triggerWebhook } from '../../services/webhook-helper';
test.describe('External Webhook Triggering @auth:owner', () => {
test('should create workflow via API, activate it, trigger webhook externally, and verify execution', async ({
api,
}) => {
const workflowDefinition = JSON.parse(
readFileSync(resolveFromRoot('workflows', 'simple-webhook-test.json'), 'utf8'),
const { webhookPath, workflowId } = await importAndActivateWebhookWorkflow(
api,
'simple-webhook-test.json',
);
const createdWorkflow = await api.workflowApi.createWorkflow(workflowDefinition);
expect(createdWorkflow.id).toBeDefined();
const workflowId = createdWorkflow.id;
await api.workflowApi.setActive(workflowId, true);
const testPayload = { message: 'Hello from Playwright test' };
const webhookResponse = await api.workflowApi.triggerWebhook('test-webhook', {
const webhookResponse = await triggerWebhook(api, webhookPath, {
data: testPayload,
});
expect(webhookResponse.ok()).toBe(true);