diff --git a/.github/actions/setup-and-build/action.yml b/.github/actions/setup-and-build/action.yml index b866a89947..3eca39472e 100644 --- a/.github/actions/setup-and-build/action.yml +++ b/.github/actions/setup-and-build/action.yml @@ -1,22 +1,30 @@ name: 'Setup Environment and Build Project' -description: 'Sets up Node.js with pnpm, installs dependencies, enables Turborepo caching, and builds the project.' +description: 'Sets up Node.js with pnpm, installs dependencies, (optional) enables Turborepo caching, and (optional) builds the project.' inputs: node-version: description: 'Node.js version to use.' required: false - default: '22.x' + default: '20.x' enable-caching: - description: Flag to enable/disable all caching (pnpm store, Turborepo, and dist folders).' + description: 'Flag to enable/disable all caching.' required: false default: 'true' cache-suffix: - description: 'Suffix to add to the dist folder cache key.' + description: 'Suffix to add to the cache key.' required: false - default: 'build' + default: 'base:build' + skip-build: + description: 'Skip the build step, useful when restoring cached artifacts.' + required: false + default: 'false' + cache-paths: + description: 'Paths to cache. Defaults to dist folders.' + required: false + default: './packages/**/dist' runs: - using: "composite" + using: 'composite' steps: - name: Setup pnpm CLI uses: pnpm/action-setup@a7487c7e89a18df4991f7f222e4898a00d66ddda # v4.1.0 @@ -38,12 +46,13 @@ runs: uses: useblacksmith/caching-for-turbo@v1 - name: Build packages + if: inputs.skip-build == 'false' run: pnpm build shell: bash - - name: Cache 'dist' folders - if: inputs.enable-caching == 'true' + - name: Cache artifacts + if: inputs.enable-caching == 'true' && inputs.skip-build == 'false' uses: useblacksmith/cache@v5 with: - path: ./packages/**/dist - key: ${{ github.sha }}:${{ inputs.cache-suffix }} \ No newline at end of file + path: ${{ inputs.cache-paths }} + key: ${{ github.sha }}-${{ inputs.cache-suffix }} diff --git a/.github/workflows/ci-pull-requests.yml b/.github/workflows/ci-pull-requests.yml index c7f6fcedcf..63ee693ac0 100644 --- a/.github/workflows/ci-pull-requests.yml +++ b/.github/workflows/ci-pull-requests.yml @@ -17,23 +17,11 @@ jobs: with: ref: refs/pull/${{ github.event.pull_request.number }}/merge - - uses: useblacksmith/setup-node@65c6ca86fdeb0ab3d85e78f57e4f6a7e4780b391 # v5 + - name: Setup Environment and Build Project + uses: ./.github/actions/setup-and-build with: - node-version: 20.x - - - name: Setup corepack and pnpm - run: | - npm i -g corepack@0.31 - corepack enable - - - name: Install dependencies - run: pnpm install --frozen-lockfile - - - name: Setup build cache - uses: useblacksmith/caching-for-turbo@bafb57e7ebdbf1185762286ec94d24648cd3938a # v1 - - - name: Build - run: pnpm build + node-version: '20.x' + enable-caching: true - name: Run formatcheck run: pnpm format:check @@ -41,12 +29,6 @@ jobs: - name: Run typecheck run: pnpm typecheck - - name: Cache build artifacts - uses: useblacksmith/cache/save@c5fe29eb0efdf1cf4186b9f7fcbbcbc0cf025662 # v5 - with: - path: ./packages/**/dist - key: ${{ github.sha }}-base:build - unit-test: name: Unit tests uses: ./.github/workflows/units-tests-reusable.yml diff --git a/.github/workflows/linting-reusable.yml b/.github/workflows/linting-reusable.yml index 32db6d1e48..30dc0ff73b 100644 --- a/.github/workflows/linting-reusable.yml +++ b/.github/workflows/linting-reusable.yml @@ -8,11 +8,16 @@ on: required: false type: string default: master + nodeVersion: + description: Version of node to use. + required: false + type: string + default: 20.x cacheKey: description: Cache key for modules and build artifacts. required: false - default: '' type: string + default: '' jobs: lint: @@ -23,32 +28,12 @@ jobs: with: ref: ${{ inputs.ref }} - - uses: useblacksmith/setup-node@65c6ca86fdeb0ab3d85e78f57e4f6a7e4780b391 # v5 + - name: Setup Environment + uses: ./.github/actions/setup-and-build with: - node-version: 20.x - - - name: Setup corepack and pnpm - run: | - npm i -g corepack@0.31 - corepack enable - - - name: Install dependencies - run: pnpm install --frozen-lockfile - - - name: Setup build cache - uses: useblacksmith/caching-for-turbo@bafb57e7ebdbf1185762286ec94d24648cd3938a # v1 - - - name: Build - if: ${{ inputs.cacheKey == '' }} - run: pnpm build - - - name: Restore cached build artifacts - if: ${{ inputs.cacheKey != '' }} - uses: useblacksmith/cache/restore@c5fe29eb0efdf1cf4186b9f7fcbbcbc0cf025662 # v5 - with: - path: ./packages/**/dist - key: ${{ inputs.cacheKey }} - fail-on-cache-miss: true + node-version: ${{ inputs.nodeVersion }} + enable-caching: true + skip-build: ${{ inputs.cacheKey != '' }} - name: Lint Backend run: pnpm lint:backend diff --git a/.github/workflows/units-tests-reusable.yml b/.github/workflows/units-tests-reusable.yml index 091511ce17..a2f7991fea 100644 --- a/.github/workflows/units-tests-reusable.yml +++ b/.github/workflows/units-tests-reusable.yml @@ -47,27 +47,13 @@ jobs: with: ref: ${{ inputs.ref }} - - name: Use Node.js ${{ inputs.nodeVersion }} - uses: useblacksmith/setup-node@65c6ca86fdeb0ab3d85e78f57e4f6a7e4780b391 # v5.0.4 + - name: Setup Environment and Build Project + uses: ./.github/actions/setup-and-build with: node-version: ${{ inputs.nodeVersion }} + skip-build: ${{ inputs.cacheKey != '' }} - - name: Setup corepack and pnpm - run: | - npm i -g corepack@0.31 - corepack enable - - - name: Install dependencies - run: pnpm install --frozen-lockfile - - - name: Setup build cache - uses: useblacksmith/caching-for-turbo@bafb57e7ebdbf1185762286ec94d24648cd3938a # v1 - - - name: Build - if: ${{ inputs.cacheKey == '' }} - run: pnpm build - - - name: Restore cached build artifacts + - name: Restore cached build artifacts only if: ${{ inputs.cacheKey != '' }} uses: useblacksmith/cache/restore@c5fe29eb0efdf1cf4186b9f7fcbbcbc0cf025662 # v5.0.2 with: @@ -86,13 +72,12 @@ jobs: run: pnpm test:frontend - name: Upload test results to Codecov - if: ${{ !cancelled() }} # Run even if tests fail + if: ${{ !cancelled() }} uses: codecov/test-results-action@47f89e9acb64b76debcd5ea40642d25a4adced9f # v1.1.1 with: token: ${{ secrets.CODECOV_TOKEN }} - name: Upload coverage to Codecov - if: inputs.collectCoverage uses: codecov/codecov-action@18283e04ce6e62d37312384ff67231eb8fd56d24 # v5.4.3 with: token: ${{ secrets.CODECOV_TOKEN }} diff --git a/package.json b/package.json index 82e4d94dce..3391624fda 100644 --- a/package.json +++ b/package.json @@ -69,7 +69,7 @@ "ts-jest": "^29.1.1", "tsc-alias": "^1.8.10", "tsc-watch": "^6.2.0", - "turbo": "2.3.3", + "turbo": "2.5.3", "typescript": "*", "zx": "^8.1.4" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 76507bf4c2..8f66a508cc 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -279,8 +279,8 @@ importers: specifier: ^6.2.0 version: 6.2.0(typescript@5.8.2) turbo: - specifier: 2.3.3 - version: 2.3.3 + specifier: 2.5.3 + version: 2.5.3 typescript: specifier: ^5.8.2 version: 5.8.2 @@ -750,7 +750,7 @@ importers: version: 4.3.0 '@getzep/zep-cloud': specifier: 1.0.12 - version: 1.0.12(@langchain/core@0.3.39(openai@4.78.1(encoding@0.1.13)(zod@3.24.1)))(encoding@0.1.13)(langchain@0.3.11(d71d1df33a22803bba4e47303d410a51)) + version: 1.0.12(@langchain/core@0.3.39(openai@4.78.1(encoding@0.1.13)(zod@3.24.1)))(encoding@0.1.13)(langchain@0.3.11(a9c8f655d0ec4bd70e0f938ef73f85da)) '@getzep/zep-js': specifier: 0.9.0 version: 0.9.0 @@ -777,7 +777,7 @@ importers: version: 0.3.2(@langchain/core@0.3.39(openai@4.78.1(encoding@0.1.13)(zod@3.24.1)))(encoding@0.1.13) '@langchain/community': specifier: 'catalog:' - version: 0.3.24(9b62d544268bf8b21f905847a18f95f1) + version: 0.3.24(90a94ff6ee073f963f6e65dd73dfbb27) '@langchain/core': specifier: 'catalog:' version: 0.3.39(openai@4.78.1(encoding@0.1.13)(zod@3.24.1)) @@ -882,7 +882,7 @@ importers: version: 23.0.1 langchain: specifier: 0.3.11 - version: 0.3.11(d71d1df33a22803bba4e47303d410a51) + version: 0.3.11(a9c8f655d0ec4bd70e0f938ef73f85da) lodash: specifier: 'catalog:' version: 4.17.21 @@ -13449,38 +13449,38 @@ packages: tunnel-agent@0.6.0: resolution: {integrity: sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==} - turbo-darwin-64@2.3.3: - resolution: {integrity: sha512-bxX82xe6du/3rPmm4aCC5RdEilIN99VUld4HkFQuw+mvFg6darNBuQxyWSHZTtc25XgYjQrjsV05888w1grpaA==} + turbo-darwin-64@2.5.3: + resolution: {integrity: sha512-YSItEVBUIvAGPUDpAB9etEmSqZI3T6BHrkBkeSErvICXn3dfqXUfeLx35LfptLDEbrzFUdwYFNmt8QXOwe9yaw==} cpu: [x64] os: [darwin] - turbo-darwin-arm64@2.3.3: - resolution: {integrity: sha512-DYbQwa3NsAuWkCUYVzfOUBbSUBVQzH5HWUFy2Kgi3fGjIWVZOFk86ss+xsWu//rlEAfYwEmopigsPYSmW4X15A==} + turbo-darwin-arm64@2.5.3: + resolution: {integrity: sha512-5PefrwHd42UiZX7YA9m1LPW6x9YJBDErXmsegCkVp+GjmWrADfEOxpFrGQNonH3ZMj77WZB2PVE5Aw3gA+IOhg==} cpu: [arm64] os: [darwin] - turbo-linux-64@2.3.3: - resolution: {integrity: sha512-eHj9OIB0dFaP6BxB88jSuaCLsOQSYWBgmhy2ErCu6D2GG6xW3b6e2UWHl/1Ho9FsTg4uVgo4DB9wGsKa5erjUA==} + turbo-linux-64@2.5.3: + resolution: {integrity: sha512-M9xigFgawn5ofTmRzvjjLj3Lqc05O8VHKuOlWNUlnHPUltFquyEeSkpQNkE/vpPdOR14AzxqHbhhxtfS4qvb1w==} cpu: [x64] os: [linux] - turbo-linux-arm64@2.3.3: - resolution: {integrity: sha512-NmDE/NjZoDj1UWBhMtOPmqFLEBKhzGS61KObfrDEbXvU3lekwHeoPvAMfcovzswzch+kN2DrtbNIlz+/rp8OCg==} + turbo-linux-arm64@2.5.3: + resolution: {integrity: sha512-auJRbYZ8SGJVqvzTikpg1bsRAsiI9Tk0/SDkA5Xgg0GdiHDH/BOzv1ZjDE2mjmlrO/obr19Dw+39OlMhwLffrw==} cpu: [arm64] os: [linux] - turbo-windows-64@2.3.3: - resolution: {integrity: sha512-O2+BS4QqjK3dOERscXqv7N2GXNcqHr9hXumkMxDj/oGx9oCatIwnnwx34UmzodloSnJpgSqjl8iRWiY65SmYoQ==} + turbo-windows-64@2.5.3: + resolution: {integrity: sha512-arLQYohuHtIEKkmQSCU9vtrKUg+/1TTstWB9VYRSsz+khvg81eX6LYHtXJfH/dK7Ho6ck+JaEh5G+QrE1jEmCQ==} cpu: [x64] os: [win32] - turbo-windows-arm64@2.3.3: - resolution: {integrity: sha512-dW4ZK1r6XLPNYLIKjC4o87HxYidtRRcBeo/hZ9Wng2XM/MqqYkAyzJXJGgRMsc0MMEN9z4+ZIfnSNBrA0b08ag==} + turbo-windows-arm64@2.5.3: + resolution: {integrity: sha512-3JPn66HAynJ0gtr6H+hjY4VHpu1RPKcEwGATvGUTmLmYSYBQieVlnGDRMMoYN066YfyPqnNGCfhYbXfH92Cm0g==} cpu: [arm64] os: [win32] - turbo@2.3.3: - resolution: {integrity: sha512-DUHWQAcC8BTiUZDRzAYGvpSpGLiaOQPfYXlCieQbwUvmml/LRGIe3raKdrOPOoiX0DYlzxs2nH6BoWJoZrj8hA==} + turbo@2.5.3: + resolution: {integrity: sha512-iHuaNcq5GZZnr3XDZNuu2LSyCzAOPwDuo5Qt+q64DfsTP1i3T2bKfxJhni2ZQxsvAoxRbuUK5QetJki4qc5aYA==} hasBin: true tween-functions@1.2.0: @@ -16610,7 +16610,7 @@ snapshots: '@gar/promisify@1.1.3': optional: true - '@getzep/zep-cloud@1.0.12(@langchain/core@0.3.39(openai@4.78.1(encoding@0.1.13)(zod@3.24.1)))(encoding@0.1.13)(langchain@0.3.11(d71d1df33a22803bba4e47303d410a51))': + '@getzep/zep-cloud@1.0.12(@langchain/core@0.3.39(openai@4.78.1(encoding@0.1.13)(zod@3.24.1)))(encoding@0.1.13)(langchain@0.3.11(a9c8f655d0ec4bd70e0f938ef73f85da))': dependencies: form-data: 4.0.0 node-fetch: 2.7.0(encoding@0.1.13) @@ -16619,7 +16619,7 @@ snapshots: zod: 3.24.1 optionalDependencies: '@langchain/core': 0.3.39(openai@4.78.1(encoding@0.1.13)(zod@3.24.1)) - langchain: 0.3.11(d71d1df33a22803bba4e47303d410a51) + langchain: 0.3.11(a9c8f655d0ec4bd70e0f938ef73f85da) transitivePeerDependencies: - encoding @@ -17141,7 +17141,7 @@ snapshots: - aws-crt - encoding - '@langchain/community@0.3.24(9b62d544268bf8b21f905847a18f95f1)': + '@langchain/community@0.3.24(90a94ff6ee073f963f6e65dd73dfbb27)': dependencies: '@browserbasehq/stagehand': 1.9.0(@playwright/test@1.49.1)(deepmerge@4.3.1)(dotenv@16.5.0)(encoding@0.1.13)(openai@4.78.1(encoding@0.1.13)(zod@3.24.1))(zod@3.24.1) '@ibm-cloud/watsonx-ai': 1.1.2 @@ -17152,7 +17152,7 @@ snapshots: flat: 5.0.2 ibm-cloud-sdk-core: 5.3.2 js-yaml: 4.1.0 - langchain: 0.3.11(d71d1df33a22803bba4e47303d410a51) + langchain: 0.3.11(a9c8f655d0ec4bd70e0f938ef73f85da) langsmith: 0.2.15(openai@4.78.1(encoding@0.1.13)(zod@3.24.1)) openai: 4.78.1(encoding@0.1.13)(zod@3.24.1) uuid: 10.0.0 @@ -17167,7 +17167,7 @@ snapshots: '@aws-sdk/credential-provider-node': 3.808.0 '@azure/storage-blob': 12.26.0 '@browserbasehq/sdk': 2.6.0(encoding@0.1.13) - '@getzep/zep-cloud': 1.0.12(@langchain/core@0.3.39(openai@4.78.1(encoding@0.1.13)(zod@3.24.1)))(encoding@0.1.13)(langchain@0.3.11(d71d1df33a22803bba4e47303d410a51)) + '@getzep/zep-cloud': 1.0.12(@langchain/core@0.3.39(openai@4.78.1(encoding@0.1.13)(zod@3.24.1)))(encoding@0.1.13)(langchain@0.3.11(a9c8f655d0ec4bd70e0f938ef73f85da)) '@getzep/zep-js': 0.9.0 '@google-ai/generativelanguage': 2.6.0(encoding@0.1.13) '@google-cloud/storage': 7.12.1(encoding@0.1.13) @@ -22257,7 +22257,7 @@ snapshots: eslint-import-resolver-node@0.3.9: dependencies: - debug: 3.2.7(supports-color@5.5.0) + debug: 3.2.7(supports-color@8.1.1) is-core-module: 2.13.1 resolve: 1.22.8 transitivePeerDependencies: @@ -22282,7 +22282,7 @@ snapshots: eslint-module-utils@2.8.0(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.8.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0): dependencies: - debug: 3.2.7(supports-color@5.5.0) + debug: 3.2.7(supports-color@8.1.1) optionalDependencies: '@typescript-eslint/parser': 7.2.0(eslint@8.57.0)(typescript@5.8.2) eslint: 8.57.0 @@ -22302,7 +22302,7 @@ snapshots: array.prototype.findlastindex: 1.2.3 array.prototype.flat: 1.3.2 array.prototype.flatmap: 1.3.2 - debug: 3.2.7(supports-color@5.5.0) + debug: 3.2.7(supports-color@8.1.1) doctrine: 2.1.0 eslint: 8.57.0 eslint-import-resolver-node: 0.3.9 @@ -23137,7 +23137,7 @@ snapshots: array-parallel: 0.1.3 array-series: 0.1.5 cross-spawn: 7.0.6 - debug: 3.2.7(supports-color@5.5.0) + debug: 3.2.7(supports-color@8.1.1) transitivePeerDependencies: - supports-color @@ -23441,7 +23441,7 @@ snapshots: isstream: 0.1.2 jsonwebtoken: 9.0.2 mime-types: 2.1.35 - retry-axios: 2.6.0(axios@1.9.0) + retry-axios: 2.6.0(axios@1.9.0(debug@4.4.1)) tough-cookie: 4.1.4 transitivePeerDependencies: - supports-color @@ -24466,7 +24466,7 @@ snapshots: kuler@2.0.0: {} - langchain@0.3.11(d71d1df33a22803bba4e47303d410a51): + langchain@0.3.11(a9c8f655d0ec4bd70e0f938ef73f85da): dependencies: '@langchain/core': 0.3.39(openai@4.78.1(encoding@0.1.13)(zod@3.24.1)) '@langchain/openai': 0.3.17(@langchain/core@0.3.39(openai@4.78.1(encoding@0.1.13)(zod@3.24.1)))(encoding@0.1.13) @@ -26048,7 +26048,7 @@ snapshots: pdf-parse@1.1.1: dependencies: - debug: 3.2.7(supports-color@5.5.0) + debug: 3.2.7(supports-color@8.1.1) node-ensure: 0.0.0 transitivePeerDependencies: - supports-color @@ -26838,7 +26838,7 @@ snapshots: onetime: 5.1.2 signal-exit: 3.0.7 - retry-axios@2.6.0(axios@1.9.0): + retry-axios@2.6.0(axios@1.9.0(debug@4.4.1)): dependencies: axios: 1.9.0 @@ -26865,7 +26865,7 @@ snapshots: rhea@1.0.24: dependencies: - debug: 3.2.7(supports-color@5.5.0) + debug: 3.2.7(supports-color@8.1.1) transitivePeerDependencies: - supports-color @@ -28118,32 +28118,32 @@ snapshots: dependencies: safe-buffer: 5.2.1 - turbo-darwin-64@2.3.3: + turbo-darwin-64@2.5.3: optional: true - turbo-darwin-arm64@2.3.3: + turbo-darwin-arm64@2.5.3: optional: true - turbo-linux-64@2.3.3: + turbo-linux-64@2.5.3: optional: true - turbo-linux-arm64@2.3.3: + turbo-linux-arm64@2.5.3: optional: true - turbo-windows-64@2.3.3: + turbo-windows-64@2.5.3: optional: true - turbo-windows-arm64@2.3.3: + turbo-windows-arm64@2.5.3: optional: true - turbo@2.3.3: + turbo@2.5.3: optionalDependencies: - turbo-darwin-64: 2.3.3 - turbo-darwin-arm64: 2.3.3 - turbo-linux-64: 2.3.3 - turbo-linux-arm64: 2.3.3 - turbo-windows-64: 2.3.3 - turbo-windows-arm64: 2.3.3 + turbo-darwin-64: 2.5.3 + turbo-darwin-arm64: 2.5.3 + turbo-linux-64: 2.5.3 + turbo-linux-arm64: 2.5.3 + turbo-windows-64: 2.5.3 + turbo-windows-arm64: 2.5.3 tween-functions@1.2.0: {} diff --git a/turbo.json b/turbo.json index f3a3b4a589..671f9def19 100644 --- a/turbo.json +++ b/turbo.json @@ -1,6 +1,11 @@ { "$schema": "https://turbo.build/schema.json", "ui": "stream", + "remoteCache": { + "enabled": true, + "timeout": 90, + "uploadTimeout": 90 + }, "globalEnv": ["CI", "COVERAGE_ENABLED"], "tasks": { "clean": { @@ -26,6 +31,7 @@ "format:check": {}, "lint:backend": { "dependsOn": [ + "^build", "@n8n/api-types#lint", "@n8n/config#lint", "@n8n/decorators#lint", @@ -46,6 +52,7 @@ }, "lint:frontend": { "dependsOn": [ + "^build", "@n8n/chat#lint", "@n8n/codemirror-lang#lint", "@n8n/storybook#lint", @@ -57,12 +64,15 @@ }, "lint:nodes": { "dependsOn": [ + "^build", "n8n-nodes-base#lint", "@n8n/n8n-nodes-langchain#lint", "@n8n/json-schema-to-zod#lint" ] }, - "lint": {}, + "lint": { + "dependsOn": ["^build"] + }, "lintfix": {}, "test:backend": { "dependsOn": [ @@ -79,8 +89,8 @@ "n8n-core#test", "n8n#test" ], - "outputs": ["coverage/**"], - "inputs": ["jest.config.js"] + "outputs": ["coverage/**", "junit.xml", "cobertura-coverage.xml"], + "inputs": ["jest.config.*", "package.json", "pnpm-lock.yaml"] }, "test:frontend": { "dependsOn": [ @@ -90,8 +100,8 @@ "@n8n/design-system#test", "n8n-editor-ui#test" ], - "outputs": ["coverage/**"], - "inputs": ["jest.config.js"] + "outputs": ["coverage/**", "junit.xml", "cobertura-coverage.xml"], + "inputs": ["jest.config.*", "package.json", "pnpm-lock.yaml"] }, "test:nodes": { "dependsOn": [ @@ -99,10 +109,12 @@ "@n8n/n8n-nodes-langchain#test", "@n8n/json-schema-to-zod#test" ], - "outputs": ["coverage/**"], - "inputs": ["jest.config.js"] + "outputs": ["coverage/**", "junit.xml", "cobertura-coverage.xml"], + "inputs": ["jest.config.*", "package.json", "pnpm-lock.yaml"] + }, + "test": { + "outputs": ["coverage/**", "junit.xml", "cobertura-coverage.xml"] }, - "test": {}, "watch": { "cache": false, "persistent": true