ci: Enable Playwright tests in PRs (#17238)

.github/actions/setup-nodejs-blacksmith/action.yml

@@ -0,0 +1,42 @@
+name: 'Blacksmith Node.js Build Setup'
+description: 'Configures Node.js with pnpm, installs dependencies, enables Turborepo caching, (optional) sets up Docker layer caching, and builds the project or an optional command.'
+
+inputs:
+  node-version:
+    description: 'Node.js version to use. Uses latest 22.x by default.'
+    required: false
+    default: '22.x'
+  enable-docker-cache:
+    description: 'Whether to set up Blacksmith Buildx for Docker layer caching.'
+    required: false
+    default: 'false'
+    type: boolean
+  build-command:
+    description: 'Command to execute for building the project or an optional command. Leave empty to skip build step.'
+    required: false
+    default: 'pnpm build'
+    type: string
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Setup Node.js
+      uses: useblacksmith/setup-node@65c6ca86fdeb0ab3d85e78f57e4f6a7e4780b391 # v5.0.4
+      with:
+        node-version: ${{ inputs.node-version }}
+
+    - name: Setup pnpm and Install Dependencies
+      uses: pnpm/action-setup@a7487c7e89a18df4991f7f222e4898a00d66ddda # v4.0.0
+      with:
+        run_install: true
+
+    - name: Configure Turborepo Cache
+      uses: useblacksmith/caching-for-turbo@bafb57e7ebdbf1185762286ec94d24648cd3938a # v1
+
+    - name: Setup Blacksmith Buildx for Docker Cache
+      if: ${{ inputs.enable-docker-cache == 'true' }}
+      uses: useblacksmith/build-push-action@574eb0ee0b59c6a687ace24192f0727dfb65d6d7 # v1.2.0
+
+    - name: Build Project
+      run: ${{ inputs.build-command }}
+      shell: bash

.github/workflows/e2e-reusable.yml

@@ -40,81 +40,18 @@ on:
       CYPRESS_RECORD_KEY:
         description: 'Cypress record key.'
         required: true
-    outputs:
-      tests_passed:
-        description: 'True if all E2E tests passed, otherwise false'
-        value: ${{ jobs.check_testing_matrix.outputs.all_tests_passed }}
+      CURRENTS_RECORD_KEY:
+        description: 'Currents record key.'
+        required: true
+
+env:
+  NODE_OPTIONS: --max-old-space-size=4096
 
 jobs:
-  # single job that generates and outputs a common id
-  prepare:
-    runs-on: ubuntu-latest
-    outputs:
-      uuid: ${{ steps.uuid.outputs.value }}
-    steps:
-      - name: Generate unique ID 💎
-        id: uuid
-        # take the current commit + timestamp together
-        # the typical value would be something like
-        # "sha-5d3fe...35d3-time-1620841214"
-        run: echo "value=sha-$GITHUB_SHA-time-$(date +"%s")" >> $GITHUB_OUTPUT
-
-      - name: Calculate Git Ref 🤔
-        id: calculate_ref
-        run: |
-          if [ -n "${{ inputs.pr_number }}" ]; then
-            echo "value=refs/pull/${{ inputs.pr_number }}/head" >> $GITHUB_OUTPUT
-          else
-            echo "value=${{ inputs.branch }}" >> $GITHUB_OUTPUT
-          fi
-  install:
-    runs-on: blacksmith-4vcpu-ubuntu-2204
-    needs: ['prepare']
-
-    steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-        with:
-          ref: ${{ steps.calculate_ref.outputs.value }}
-
-      - uses: pnpm/action-setup@a7487c7e89a18df4991f7f222e4898a00d66ddda # v4.0.0
-
-      - name: Setup Node.js
-        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
-        with:
-          node-version: 22.x
-          cache: 'pnpm'
-
-      - name: Cache build artifacts
-        id: cache-build-artifacts
-        uses: useblacksmith/cache@c5fe29eb0efdf1cf4186b9f7fcbbcbc0cf025662 # v5
-        with:
-          path: |
-            /home/runner/.cache/Cypress
-            /github/home/.pnpm-store
-            ./packages/**/dist
-          key: ${{ github.sha }}-ui
-
-      - name: Install dependencies
-        if: steps.cache-build-artifacts.outputs.cache-hit != 'true'
-        run: pnpm install --frozen-lockfile
-
-      - name: Cypress build
-        if: steps.cache-build-artifacts.outputs.cache-hit != 'true'
-        uses: cypress-io/github-action@be1bab96b388bbd9ce3887e397d373c8557e15af # v6.9.2
-        with:
-          # Disable running of tests within install job
-          runTests: false
-          install: false
-          build: pnpm build
-
-      - name: Cypress install
-        if: steps.cache-build-artifacts.outputs.cache-hit != 'true'
-        working-directory: cypress
-        run: pnpm cypress:install
-
   testing:
     runs-on: blacksmith-2vcpu-ubuntu-2204
-    needs: ['prepare', 'install']
+    outputs:
+      dashboardUrl: ${{ steps.cypress.outputs.dashboardUrl }}
     strategy:
       fail-fast: false
       matrix:
@@ -123,31 +60,15 @@ jobs:
         containers: ${{ fromJSON( inputs.spec == 'e2e/*' && inputs.containers || '[1]' ) }}
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-        with:
-          ref: ${{ steps.calculate_ref.outputs.value }}
+      - name: Set up and build
+        uses: ./.github/actions/setup-nodejs-blacksmith
 
-      - uses: pnpm/action-setup@a7487c7e89a18df4991f7f222e4898a00d66ddda # v4.0.0
-
-      - name: Setup Node.js
-        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
-        with:
-          node-version: 22.x
-          cache: 'pnpm'
-
-      - name: Restore cached pnpm modules
-        id: cache-build-artifacts
-        uses: useblacksmith/cache@c5fe29eb0efdf1cf4186b9f7fcbbcbc0cf025662 # v5
-        with:
-          path: |
-            /home/runner/.cache/Cypress
-            /github/home/.pnpm-store
-            ./packages/**/dist
-          key: ${{ github.sha }}-ui
-
-      - name: Install dependencies
-        run: pnpm install --frozen-lockfile
+      - name: Install Cypress
+        working-directory: cypress
+        run: pnpm cypress:install
 
       - name: Cypress run
+        id: cypress
         uses: cypress-io/github-action@be1bab96b388bbd9ce3887e397d373c8557e15af # v6.9.2
         with:
           working-directory: cypress
@@ -159,7 +80,7 @@ jobs:
           parallel: ${{ fromJSON( inputs.spec == 'e2e/*' && inputs.parallel || false ) }}
           # We have to provide custom ci-build-id key to make sure that this workflow could be run multiple times
           # in the same parent workflow
-          ci-build-id: ${{ needs.prepare.outputs.uuid }}
+          ci-build-id: ${{ github.run_id }}-${{ github.run_attempt }}
           spec: '${{ inputs.spec }}'
         env:
           NODE_OPTIONS: --dns-result-order=ipv4first
@@ -169,21 +90,49 @@ jobs:
           COMMIT_INFO_MESSAGE: 🌳 ${{ inputs.branch }} 🤖 ${{ inputs.user }} 🗃️ ${{ inputs.spec }}
           SHELL: /bin/sh
 
-  # Check if all tests passed and set the output variable
-  check_testing_matrix:
+      - name: Upload test results artifact
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: test-results-${{ matrix.containers }}
+          path: cypress/test-results-*.xml
+
+  upload-to-currents:
+    needs: testing
+    if: always()
     runs-on: ubuntu-latest
-    needs: [testing]
-    outputs:
-      all_tests_passed: ${{ steps.all_tests_passed.outputs.result }}
     steps:
-      - name: Check all tests passed
-        id: all_tests_passed
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
+      - name: Download all test results
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+        with:
+          path: test-results
+
+      - name: Merge and upload to Currents
         run: |
-          success=true
-          for status in ${{ needs.testing.result }}; do
-            if [ $status != "success" ]; then
-              success=false
-              break
-            fi
-          done
-          echo "::set-output name=result::$success"
+          npm install -g @currents/cmd junit-report-merger
+          # Merge all XML files, so Currents can show a single view for Cypress
+          jrm combined-results.xml "test-results/**/test-results-*.xml"
+
+      - name: Upload merged XML as artifact
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: merged-junit-results
+          path: combined-results.xml
+
+      - name: Convert and upload to Currents
+        run: |
+          currents convert \
+            --input-format=junit \
+            --input-file=combined-results.xml \
+            --output-dir=.currents \
+            --framework=node \
+            --framework-version=cypress-14.4.0
+
+          currents upload \
+            --project-id=I0yzoc \
+            --key=${{ secrets.CURRENTS_RECORD_KEY }} \
+            --ci-build-id=n8n-io/n8n-${{ github.run_id }}-${{ github.run_attempt }} \
+            --report-dir=.currents \
+            --tag=cypress

.github/workflows/e2e-tests-pr.yml

@@ -24,56 +24,21 @@ jobs:
     with:
       pr_number: ${{ github.event.pull_request.number }}
       user: ${{ github.event.pull_request.user.login || 'PR User' }}
-    secrets:
-      CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
+    secrets: inherit
+
+  run-playwright-tests:
+    name: Playwright
+    uses: ./.github/workflows/playwright-test-reusable.yml
+    needs: [eligibility_check]
+    if: needs.eligibility_check.outputs.should_run == 'true'
+    secrets: inherit
 
   post-e2e-tests:
     name: E2E - Checks
     runs-on: ubuntu-latest
-    needs: [eligibility_check, run-e2e-tests]
+    needs: [eligibility_check, run-e2e-tests, run-playwright-tests]
     if: always() && needs.eligibility_check.result != 'skipped'
     steps:
-      - name: Determine Outcome and Comment Message
-        id: determine_outcome
-        run: |
-          JOB_OUTCOME="success"
-          COMMENT_BODY=""
-          SHOULD_POST_COMMENT="false"
-
-          if [[ "${{ needs.eligibility_check.outputs.should_run }}" == "false" ]]; then
-            COMMENT_BODY="ℹ️ E2E tests were not run for this PR based on the eligibility criteria."
-            SHOULD_POST_COMMENT="true"
-            JOB_OUTCOME="success"
-          elif [[ "${{ needs.run-e2e-tests.result }}" == "success" ]]; then
-            COMMENT_BODY=":white_check_mark: All Cypress E2E specs passed"
-            SHOULD_POST_COMMENT="true"
-            JOB_OUTCOME="success"
-          elif [[ "${{ needs.run-e2e-tests.result }}" == "failure" ]]; then
-            COMMENT_BODY=":warning: Some Cypress E2E specs are failing, please fix them before merging"
-            SHOULD_POST_COMMENT="true"
-            JOB_OUTCOME="failure"
-          else
-            COMMENT_BODY="ℹ️ E2E tests were scheduled but did not complete as expected (Result: ${{ needs.run-e2e-tests.result }})."
-            SHOULD_POST_COMMENT="true"
-            JOB_OUTCOME="failure"
-          fi
-
-          echo "comment_body=$COMMENT_BODY" >> $GITHUB_OUTPUT
-          echo "should_post_comment=$SHOULD_POST_COMMENT" >> $GITHUB_OUTPUT
-          echo "job_outcome=$JOB_OUTCOME" >> $GITHUB_OUTPUT
-
-      - name: Create or Update PR Comment
-        if: steps.determine_outcome.outputs.should_post_comment == 'true' && needs.eligibility_check.outputs.should_run == 'true'
-        uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043
-        with:
-          issue-number: ${{ github.event.pull_request.number }}
-          body: ${{ steps.determine_outcome.outputs.comment_body }}
-          token: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Finalize Job Status
-        run: |
-          if [[ "${{ steps.determine_outcome.outputs.job_outcome }}" == "failure" ]]; then
-            exit 1
-          else
-            exit 0
-          fi
+      - name: Fail if tests failed
+        if: needs.run-e2e-tests.result == 'failure' || needs.run-playwright-tests.result == 'failure'
+        run: exit 1

.github/workflows/e2e-tests.yml

@@ -48,13 +48,17 @@ jobs:
       branch: ${{ github.event.inputs.branch || 'master' }}
       user: ${{ github.event.inputs.user || 'PR User' }}
       spec: ${{ github.event.inputs.spec || 'e2e/*' }}
-    secrets:
-      CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
+    secrets: inherit
+
+  run-playwright-tests:
+    name: Playwright
+    uses: ./.github/workflows/playwright-test-reusable.yml
+    secrets: inherit
 
   calls-success-url-notify:
     name: Calls success URL and notifies
     runs-on: ubuntu-latest
-    needs: [run-e2e-tests]
+    needs: [run-e2e-tests, run-playwright-tests]
     if: ${{ github.event.inputs.success-url != '' }}
     steps:
       - name: Notify Slack on failure

.github/workflows/playwright-test-docker-build.yml

@@ -0,0 +1,13 @@
+name: Run Playwright Tests (Docker Build)
+# This workflow is used to run Playwright tests in a Docker container built from the current branch
+
+on:
+  workflow_call:
+  workflow_dispatch:
+
+jobs:
+  build-and-test:
+    uses: ./.github/workflows/playwright-test-reusable.yml
+    with:
+      test-mode: docker-build
+    secrets: inherit

.github/workflows/playwright-test-docker-pull.yml

@@ -0,0 +1,37 @@
+name: Run Playwright Tests (Docker Pull)
+# This workflow is used to run Playwright tests in a Docker container pulled from the registry
+
+on:
+  workflow_call:
+    inputs:
+      shards:
+        description: 'Shards for parallel execution'
+        required: false
+        default: '[1]'
+        type: string
+      image:
+        description: 'Image to use'
+        required: false
+        default: 'n8nio/n8n:nightly'
+        type: string
+  workflow_dispatch:
+    inputs:
+      shards:
+        description: 'Shards for parallel execution'
+        required: false
+        default: '[1]'
+        type: string
+      image:
+        description: 'Image to use'
+        required: false
+        default: 'n8nio/n8n:nightly'
+        type: string
+
+jobs:
+  build-and-test:
+    uses: ./.github/workflows/playwright-test-reusable.yml
+    with:
+      test-mode: docker-pull
+      shards: ${{ inputs.shards }}
+      docker-image: ${{ inputs.image }}
+    secrets: inherit

.github/workflows/playwright-test-reusable.yml

@@ -0,0 +1,84 @@
+name: Playwright Tests - Reusable
+
+on:
+  workflow_call:
+    inputs:
+      test-mode:
+        description: 'Test mode: local (pnpm start from local), docker-build, or docker-pull'
+        required: false
+        default: 'local'
+        type: string
+      shards:
+        description: 'Shards for parallel execution'
+        required: false
+        default: '[1]'
+        type: string
+      docker-image:
+        description: 'Docker image to use (for docker-pull mode)'
+        required: false
+        default: 'n8nio/n8n:nightly'
+        type: string
+
+    secrets:
+      CURRENTS_RECORD_KEY:
+        required: true
+
+env:
+  PLAYWRIGHT_BROWSERS_PATH: packages/testing/playwright/ms-playwright-cache
+  NODE_OPTIONS: --max-old-space-size=4096
+  # Disable Ryuk to avoid issues with Docker since it needs privileged access, containers are cleaned on teardown anyway
+  TESTCONTAINERS_RYUK_DISABLED: true
+
+jobs:
+  test:
+    runs-on: blacksmith-2vcpu-ubuntu-2204
+    strategy:
+      fail-fast: false
+      matrix:
+        shard: ${{ fromJSON(inputs.shards || '[1]') }}
+    name: Test (Shard ${{ matrix.shard }}/${{ strategy.job-total }})
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          fetch-depth: 1
+
+      - name: Setup Environment
+        uses: ./.github/actions/setup-nodejs-blacksmith
+        with:
+          build-command: ${{ inputs.test-mode == 'docker-build' && 'pnpm build:docker' || 'pnpm turbo build:playwright' }}
+          enable-docker-cache: ${{ inputs.test-mode != 'local' }}
+
+      - name: Install Browsers (Docker Build)
+        if: inputs.test-mode == 'docker-build'
+        run: pnpm turbo install-browsers:ci
+
+      - name: Start Local Server
+        if: inputs.test-mode == 'local'
+        env:
+          E2E_TESTS: true
+        run: |
+          pnpm start &
+          npx wait-on http://localhost:5678 --timeout 15000
+
+      - name: Run Tests (Local)
+        if: inputs.test-mode == 'local'
+        run: |
+          pnpm --filter=n8n-playwright test \
+            --shard=${{ matrix.shard }}/${{ strategy.job-total }} \
+            --workers=2
+        env:
+          N8N_BASE_URL: http://localhost:5678
+          RESET_E2E_DB: true
+          CURRENTS_RECORD_KEY: ${{ secrets.CURRENTS_RECORD_KEY }}
+
+      - name: Run Tests (Docker)
+        if: inputs.test-mode != 'local'
+        run: |
+          pnpm --filter=n8n-playwright run test:standard \
+            --shard=${{ matrix.shard }}/${{ strategy.job-total }} \
+            --workers=2
+        env:
+          N8N_DOCKER_IMAGE: ${{ inputs.test-mode == 'docker-build' && 'n8nio/n8n:local' || inputs.docker-image }}
+          CURRENTS_RECORD_KEY: ${{ secrets.CURRENTS_RECORD_KEY }}