feat: Migrate Test Workflows to Main Repo (#15504)

.github/actions/setup-and-build/action.yml

@@ -0,0 +1,49 @@
+name: 'Setup Environment and Build Project'
+description: 'Sets up Node.js with pnpm, installs dependencies, enables Turborepo caching, and builds the project.'
+
+inputs:
+  node-version:
+    description: 'Node.js version to use.'
+    required: false
+    default: '22.x'
+  enable-caching:
+    description: Flag to enable/disable all caching (pnpm store, Turborepo, and dist folders).'
+    required: false
+    default: 'true'
+  cache-suffix:
+    description: 'Suffix to add to the dist folder cache key.'
+    required: false
+    default: 'build'
+
+runs:
+  using: "composite"
+  steps:
+    - name: Setup pnpm CLI
+      uses: pnpm/action-setup@a7487c7e89a18df4991f7f222e4898a00d66ddda # v4.1.0
+      with:
+        run_install: false
+
+    - name: Setup Node.js
+      uses: useblacksmith/setup-node@v5
+      with:
+        node-version: ${{ inputs.node-version }}
+        cache: pnpm
+
+    - name: Install dependencies
+      run: pnpm install --frozen-lockfile
+      shell: bash
+
+    - name: Configure Turborepo Cache
+      if: inputs.enable-caching == 'true'
+      uses: useblacksmith/caching-for-turbo@v1
+
+    - name: Build packages
+      run: pnpm build
+      shell: bash
+
+    - name: Cache 'dist' folders
+      if: inputs.enable-caching == 'true'
+      uses: useblacksmith/cache@v5
+      with:
+        path: ./packages/**/dist
+        key: ${{ github.sha }}:${{ inputs.cache-suffix }}

.github/workflows/test-workflows-callable.yml

@@ -0,0 +1,217 @@
+name: Callable Test Workflows
+
+on:
+  workflow_call:
+    inputs:
+      git_ref:
+        description: 'The Git ref (branch, tag, or SHA) to checkout and test.'
+        required: true
+        type: string
+      send_webhook_report:
+        description: 'Set to true to send test results to the webhook.'
+        required: false
+        type: boolean
+        default: false
+      pr_number:
+        description: 'The PR number, if applicable (for context in webhook).'
+        required: false
+        type: string
+        default: ''
+    secrets:
+      N8N_ENCRYPTION_KEY:
+        description: 'Encryption key for n8n operations.'
+        required: true
+      CI_SENTRY_DSN:
+        description: 'Sentry DSN for CI test runs.'
+        required: false
+      RESULTS_WEBHOOK_URL:
+        description: 'Webhook URL to send test results to (if enabled).'
+        required: false
+
+jobs:
+  build_and_test:
+    name: Install, Build, and Test Workflows
+    runs-on: blacksmith-2vcpu-ubuntu-2204
+    timeout-minutes: 10
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        with:
+          ref: ${{ inputs.git_ref }}
+
+      - name: Setup Environment and Build Project
+        uses: ./.github/actions/setup-and-build
+        with:
+          node-version: '22.x'
+          cache-suffix: 'workflow-test'
+
+      - name: Install OS dependencies
+        run: |
+          sudo apt update -y
+          echo 'tzdata tzdata/Areas select Europe' | sudo debconf-set-selections
+          echo 'tzdata tzdata/Zones/Europe select Paris' | sudo debconf-set-selections
+          sudo DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends graphicsmagick
+          sudo apt-get clean
+          sudo rm -rf /var/lib/apt/lists/*
+
+      - name: Import credentials
+        run: ./packages/cli/bin/n8n import:credentials --input=test-workflows/credentials.json
+        env:
+          N8N_ENCRYPTION_KEY: ${{ secrets.N8N_ENCRYPTION_KEY }}
+
+      - name: Import workflows
+        run: ./packages/cli/bin/n8n import:workflow --separate --input=test-workflows/workflows
+        env:
+          N8N_ENCRYPTION_KEY: ${{ secrets.N8N_ENCRYPTION_KEY }}
+
+      - name: Copy static assets
+        run: |
+          mkdir -p /tmp/testData/pdfs
+          cp assets/n8n-logo.png /tmp/n8n-logo.png
+          cp assets/n8n-screenshot.png /tmp/n8n-screenshot.png
+          cp test-workflows/testData/pdfs/*.pdf /tmp/testData/pdfs/
+
+      - name: Run tests
+        id: tests
+        run: ./packages/cli/bin/n8n executeBatch --shallow --skipList=test-workflows/skipList.json --githubWorkflow --shortOutput --output=test-results.json --concurrency=16 --compare=test-workflows/snapshots
+        continue-on-error: true
+        env:
+          N8N_ENCRYPTION_KEY: ${{ secrets.N8N_ENCRYPTION_KEY }}
+          SKIP_STATISTICS_EVENTS: "true"
+          DB_SQLITE_POOL_SIZE: "4"
+          N8N_SENTRY_DSN: ${{ secrets.CI_SENTRY_DSN }}
+
+      - name: Report test outcome
+        if: always()
+        run: |
+          echo "Test step outcome was: ${{ steps.tests.outcome }}"
+          if [[ "${{ steps.tests.outcome }}" == "failure" ]]; then
+            echo "Workflow tests failed but the workflow will continue."
+          elif [[ "${{ steps.tests.outcome }}" == "success" ]]; then
+            echo "Workflow tests passed."
+          else
+            echo "Workflow tests outcome: ${{ steps.tests.outcome }}"
+          fi
+
+      - name: Prepare and Send Test Results to Webhook
+        if: inputs.send_webhook_report == true
+        shell: bash
+        env:
+          WEBHOOK_URL: ${{ secrets.RESULTS_WEBHOOK_URL }}
+          TEST_RESULTS_FILE: ./test-results.json
+          GH_REPOSITORY: ${{ github.repository }}
+          GH_RUN_ID: ${{ github.run_id }}
+          GH_RUN_ATTEMPT: ${{ github.run_attempt }}
+          GH_REF_TESTED: ${{ inputs.git_ref }}
+          GH_EVENT_NAME: ${{ github.event_name }}
+          GH_PR_NUMBER_INPUT: ${{ inputs.pr_number }}
+          GH_WORKFLOW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          GH_ACTOR: ${{ github.actor }}
+        run: |
+          echo "Attempting to send test results to webhook..."
+          echo "Test results file expected at: $TEST_RESULTS_FILE"
+
+          if [ ! -f "$TEST_RESULTS_FILE" ]; then
+            echo "::warning::Test results file ($TEST_RESULTS_FILE) not found. Skipping webhook."
+            exit 0
+          fi
+
+          if ! command -v jq &> /dev/null; then
+            echo "jq not found. Installing jq..."
+            sudo apt-get update -qq && sudo apt-get install -y -qq jq
+            if ! command -v jq &> /dev/null; then
+              echo "::error::Failed to install jq. Cannot process JSON."
+              exit 1
+            fi
+          fi
+
+          pr_number_to_send="$GH_PR_NUMBER_INPUT"
+
+          echo "Preparing JSON payload..."
+          if [ ! -s "$TEST_RESULTS_FILE" ]; then
+            echo "::warning::Test results file ($TEST_RESULTS_FILE) is empty. Sending only GitHub context."
+            enriched_payload=$(jq -n \
+              --arg repository "$GH_REPOSITORY" \
+              --arg run_id "$GH_RUN_ID" \
+              --arg run_attempt "$GH_RUN_ATTEMPT" \
+              --arg ref_tested "$GH_REF_TESTED" \
+              --arg event_name "$GH_EVENT_NAME" \
+              --arg pr_num "$pr_number_to_send" \
+              --arg workflow_run_url "$GH_WORKFLOW_RUN_URL" \
+              --arg actor "$GH_ACTOR" \
+              '{
+                githubWorkflowContext: {
+                  repository: $repository,
+                  runId: $run_id,
+                  runAttempt: $run_attempt,
+                  gitRefTested: $ref_tested,
+                  triggeringEventName: $event_name,
+                  prNumber: (if $pr_num == "" then null else $pr_num | tonumber? // $pr_num end),
+                  workflowRunUrl: $workflow_run_url,
+                  triggeredBy: $actor
+                }
+              }')
+          else
+            enriched_payload=$(jq \
+              --arg repository "$GH_REPOSITORY" \
+              --arg run_id "$GH_RUN_ID" \
+              --arg run_attempt "$GH_RUN_ATTEMPT" \
+              --arg ref_tested "$GH_REF_TESTED" \
+              --arg event_name "$GH_EVENT_NAME" \
+              --arg pr_num "$pr_number_to_send" \
+              --arg workflow_run_url "$GH_WORKFLOW_RUN_URL" \
+              --arg actor "$GH_ACTOR" \
+              '. + {
+                githubWorkflowContext: {
+                  repository: $repository,
+                  runId: $run_id,
+                  runAttempt: $run_attempt,
+                  gitRefTested: $ref_tested,
+                  triggeringEventName: $event_name,
+                  prNumber: (if $pr_num == "" then null else $pr_num | tonumber? // $pr_num end),
+                  workflowRunUrl: $workflow_run_url,
+                  triggeredBy: $actor
+                }
+              }' "$TEST_RESULTS_FILE")
+          fi
+
+          jq_exit_code=$?
+          if [ $jq_exit_code -ne 0 ] || [ -z "$enriched_payload" ]; then
+            echo "::error::Failed to process JSON with jq (exit code: $jq_exit_code). Input file: $TEST_RESULTS_FILE"
+            if [ -s "$TEST_RESULTS_FILE" ]; then
+              echo "Contents of $TEST_RESULTS_FILE that may have caused an error:"
+              head -c 1000 "$TEST_RESULTS_FILE" # Print first 1000 chars
+              echo "" # Newline after head
+            elif [ -f "$TEST_RESULTS_FILE" ]; then
+              echo "$TEST_RESULTS_FILE exists but is empty."
+            fi
+            exit 1
+          fi
+
+          echo "Enriched payload to send (first 500 chars):"
+          echo "$enriched_payload" | head -c 500
+          echo ""
+
+          echo "Sending data to webhook: $WEBHOOK_URL"
+          http_response_code=$(curl -s -w "%{http_code}" \
+            -X POST \
+            -H "Content-Type: application/json" \
+            -H "X-GitHub-Event: $GH_EVENT_NAME" \
+            -H "X-GitHub-Run-Id: $GH_RUN_ID" \
+            --data "$enriched_payload" \
+            "$WEBHOOK_URL" \
+            -o curl_response_body.txt 2>curl_stderr.txt)
+
+          curl_stderr_content=$(cat curl_stderr.txt)
+          if [ -n "$curl_stderr_content" ]; then
+            echo "::warning::curl stderr: $curl_stderr_content"
+          fi
+          echo "Webhook response code: $http_response_code"
+          echo "Webhook response body:"
+          cat curl_response_body.txt
+          if [[ "$http_response_code" -ge 200 && "$http_response_code" -lt 300 ]]; then
+            echo "Successfully sent data to webhook."
+          else
+            echo "::error::Webhook call failed with status code $http_response_code."
+          fi

.github/workflows/test-workflows-nightly.yml

@@ -0,0 +1,45 @@
+name: Test Workflows Nightly and Manual
+
+on:
+  schedule:
+    - cron: '0 2 * * *'
+  workflow_dispatch:
+    inputs:
+      git_ref_to_test:
+        description: 'The Git ref (branch, tag, or SHA) to run tests against.'
+        required: true
+        type: string
+        default: 'master'
+
+permissions:
+  contents: read
+
+jobs:
+  run_tests:
+    name: Run Workflow Tests
+    runs-on: blacksmith-2vcpu-ubuntu-2204
+    timeout-minutes: 10
+
+    steps:
+      - name: Determine Git Ref for Testing
+        id: determine_ref
+        shell: bash
+        run: |
+          if [[ "${{ github.event_name }}" == "schedule" ]]; then
+            echo "EFFECTIVE_GIT_REF=master" >> $GITHUB_OUTPUT
+            echo "Scheduled run: Using 'master' branch."
+          elif [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
+            echo "EFFECTIVE_GIT_REF=${{ github.event.inputs.git_ref_to_test }}" >> $GITHUB_OUTPUT
+            echo "Manual dispatch: Using ref '${{ github.event.inputs.git_ref_to_test }}'."
+          else
+            echo "EFFECTIVE_GIT_REF=master" >> $GITHUB_OUTPUT
+            echo "Warning: Unknown event type '${{ github.event_name }}', defaulting to 'master'."
+          fi
+
+      - name: Call Reusable Test Workflow
+        uses: ./.github/workflows/run-test-workflows.yml
+        with:
+          git_ref: ${{ steps.determine_ref.outputs.EFFECTIVE_GIT_REF }}
+          send_webhook_report: false
+          pr_number: ''
+        secrets: inherit

.github/workflows/test-workflows-pr-approved.yml

@@ -0,0 +1,25 @@
+name: Test Workflows on PR Approval
+
+on:
+  pull_request_review:
+    types: [submitted]
+
+permissions:
+  contents: read
+  pull-requests: read
+
+jobs:
+  run_tests_after_approval:
+    name: Run Tests on Approved PR
+    if: github.event.review.state == 'approved'
+    runs-on: blacksmith-2vcpu-ubuntu-2204
+    timeout-minutes: 10
+
+    steps:
+      - name: Call Reusable Test Workflow on Approved PR
+        uses: ./.github/workflows/test-workflows-callable.yml
+        with:
+          git_ref: ${{ github.event.pull_request.head.sha }}
+          send_webhook_report: true
+          pr_number: ${{ github.event.pull_request.number }}
+        secrets: inherit

.github/workflows/test-workflows-pr-comment.yml

@@ -0,0 +1,78 @@
+name: Test Workflows on PR Comment
+
+on:
+  issue_comment:
+    types: [created]
+
+permissions:
+  pull-requests: read
+  contents: read
+
+jobs:
+  trigger_tests_on_comment:
+    name: Handle /test-workflows command
+    if: github.event.issue.pull_request && startsWith(github.event.comment.body, '/test-workflows')
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check User Permission and Get PR Details
+        id: pr_check
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          result-encoding: json
+          script: |
+            const commenter = context.actor;
+            const issue = context.issue;
+            let hasPermission = false;
+            let prDetails = null;
+
+            try {
+              const { data: permissions } = await github.rest.repos.getCollaboratorPermissionLevel({
+                owner: issue.owner,
+                repo: issue.repo,
+                username: commenter
+              });
+
+              const allowedPermissions = ['admin', 'write', 'maintain'];
+              if (allowedPermissions.includes(permissions.permission)) {
+                console.log(`User @${commenter} has '${permissions.permission}' permission.`);
+                hasPermission = true;
+              } else {
+                core.setFailed(`User @${commenter} does not have sufficient permissions (admin/write/maintain) to trigger workflows.`);
+              }
+            } catch (error) {
+              core.setFailed(`Could not verify permissions for @${commenter}: ${error.message}`);
+            }
+
+            if (!hasPermission) {
+              return { permission_granted: false };
+            }
+
+            const prNumber = issue.number;
+            try {
+              const { data: pr } = await github.rest.pulls.get({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                pull_number: prNumber,
+              });
+              prDetails = {
+                head_sha: pr.head.sha,
+                pr_number_string: prNumber.toString()
+              };
+              console.log(`Workspaceed PR details: SHA - ${prDetails.head_sha}, PR Number - ${prDetails.pr_number_string}`);
+            } catch (error) {
+              core.setFailed(`Failed to fetch PR details for PR #${prNumber}: ${error.message}`);
+              return { permission_granted: true, pr_fetch_error: true };
+            }
+
+            return { permission_granted: true, ...prDetails };
+
+      - name: Call Reusable Test Workflow
+        if: steps.pr_check.outcome == 'success' && fromJson(steps.pr_check.outputs.result).permission_granted == true && fromJson(steps.pr_check.outputs.result).head_sha
+        uses: ./.github/workflows/test-workflows-callable.yml
+        with:
+          git_ref: ${{ fromJson(steps.pr_check.outputs.result).head_sha }}
+          send_webhook_report: true
+          pr_number: ${{ fromJson(steps.pr_check.outputs.result).pr_number_string }}
+        secrets: inherit