ci: Support adding custom JS and Python dependencies in the runners image (#19220)

2025-12-16 09:36:44 +00:00 · 2025-09-08 09:16:10 +03:00
parent efc3a2d664
commit 509ab95d5d
6 changed files with 141 additions and 11 deletions
--- a/docker/images/runners/Dockerfile
+++ b/docker/images/runners/Dockerfile
@@ -4,9 +4,22 @@ ARG PYTHON_VERSION=3.13
 # ==============================================================================
 # STAGE 1: JavaScript runner (@n8n/task-runner) artifact from CI
 # ==============================================================================
-FROM alpine:3.22.1 AS app-artifact-processor
+FROM node:${NODE_VERSION}-alpine AS javascript-runner-builder
 COPY ./dist/task-runner-javascript /app/task-runner-javascript

+WORKDIR /app/task-runner-javascript
+
+RUN corepack enable pnpm
+
+# Install extra runtime-only npm packages. Allow usage in the Code node via
+# 'NODE_FUNCTION_ALLOW_EXTERNAL' env variable on n8n-task-runners.json.
+RUN rm -f node_modules/.modules.yaml
+RUN mv package.json package.json.bak
+COPY docker/images/runners/package.json /app/task-runner-javascript/package.json
+RUN pnpm install --prod --no-lockfile --silent
+RUN mv package.json extras.json
+RUN mv package.json.bak package.json
+
 # ==============================================================================
 # STAGE 2: Python runner build (@n8n/task-runner-python) with uv
 # Produces a relocatable venv tied to the python version used
@@ -48,6 +61,11 @@ RUN uv sync \
      --frozen \
      --no-editable

+# Install extra runtime-only Python packages. Allow usage in the Code node via
+# 'N8N_RUNNERS_EXTERNAL_ALLOW' env variable on n8n-task-runners.json.
+COPY docker/images/runners/extras.txt /app/task-runner-python/extras.txt
+RUN uv pip install -r /app/task-runner-python/extras.txt
+
 # ==============================================================================
 # STAGE 3: Task Runner Launcher download
 # ==============================================================================
@@ -99,7 +117,7 @@ RUN addgroup -g 1000 -S runner \
 && adduser -u 1000 -S -G runner -h /home/runner -D runner
 WORKDIR /home/runner

-COPY --from=app-artifact-processor /app/task-runner-javascript /opt/runners/task-runner-javascript
+COPY --from=javascript-runner-builder /app/task-runner-javascript /opt/runners/task-runner-javascript
 COPY --from=python-runner-builder /app/task-runner-python /opt/runners/task-runner-python
 COPY --from=launcher-downloader /launcher-bin/* /usr/local/bin/

--- a/docker/images/runners/README.md
+++ b/docker/images/runners/README.md
@@ -12,22 +12,22 @@ in the [Code Node](https://docs.n8n.io/integrations/builtin/core-nodes/n8n-nodes

 ## Testing locally

-1. Make a production build of n8n
+### 1) Make a production build of n8n

 ```
 pnpm run build:n8n
 ```

-2. Build the task runners image
+### 2) Build the task runners image

 ```
-docker buildx build --no-cache \
+docker buildx build \
  -f docker/images/runners/Dockerfile \
  -t n8nio/runners \
  .
 ```

-3. Start n8n on your host machine with Task Broker enabled
+### 3) Start n8n on your host machine with Task Broker enabled

 ```
 N8N_RUNNERS_ENABLED=true \
@@ -38,7 +38,7 @@ pnpm start
 ```


-4. Start the task runner container
+### 4) Start the task runner container

 ```
 docker run --rm -it \
@@ -48,3 +48,106 @@ docker run --rm -it \
 -p 5680:5680 \
 n8nio/runners
 ```
+
+## Adding extra dependencies (custom image)
+
+To make additional packages available on the Code node you can bake extra packages into your custom runners image at build time.
+
+* **JavaScript** — edit `docker/images/runners/package.json`
+  (package.json manifest used to install runtime-only deps into the JS runner)
+* **Python (Native)** — edit `docker/images/runners/extras.txt`
+  (requirements.txt-style list installed into the Python runner venv)
+
+> Important: for security, any external libraries must be explicitly allowed for Code node use. Update `n8n-task-runners.json` to allowlist what you add.
+
+### 1) JavaScript packages
+
+Edit the runtime extras manifest `docker/images/runners/package.json`:
+
+```json
+{
+  "name": "task-runner-runtime-extras",
+  "description": "Runtime-only deps for the JS task-runner image, installed at image build.",
+  "private": true,
+  "dependencies": {
+    "moment": "2.30.1"
+  }
+}
+```
+
+Add any packages you want under `"dependencies"` (pin them for reproducibility), e.g.:
+
+```json
+"dependencies": {
+  "moment": "2.30.1",
+  "uuid": "9.0.0"
+}
+```
+
+### 2) Python packages
+
+Edit the requirements file `docker/images/runners/extras.txt`:
+
+```
+# Runtime-only extras for the Python task runner (installed at image build)
+numpy==2.3.2
+# add more, one per line, e.g.:
+# pandas==2.2.2
+```
+
+> Tip: pin versions (e.g., `==2.3.2`) for deterministic builds.
+
+### 3) Allowlist packages for the Code node
+
+Open `docker/images/runners/n8n-task-runners.json` and add your packages to the env overrides:
+
+```json
+{
+  "task-runners": [
+    {
+      "runner-type": "javascript",
+      "env-overrides": {
+        "NODE_FUNCTION_ALLOW_BUILTIN": "crypto",
+        "NODE_FUNCTION_ALLOW_EXTERNAL": "moment,uuid",   // <-- add JS packages here
+      }
+    },
+    {
+      "runner-type": "python",
+      "env-overrides": {
+        "PYTHONPATH": "/opt/runners/task-runner-python",
+        "N8N_RUNNERS_STDLIB_ALLOW": "json",
+        "N8N_RUNNERS_EXTERNAL_ALLOW": "numpy,pandas"     // <-- add Python packages here
+      }
+    }
+  ]
+}
+```
+
+* `NODE_FUNCTION_ALLOW_BUILTIN` — comma-separated list of allowed node builtin modules.
+* `NODE_FUNCTION_ALLOW_EXTERNAL` — comma-separated list of allowed JS packages.
+* `N8N_RUNNERS_STDLIB_ALLOW` — comma-separated list of allowed Python standard library packages.
+* `N8N_RUNNERS_EXTERNAL_ALLOW` — comma-separated list of allowed Python packages.
+
+### 4) Build your custom image
+
+From the repo root:
+
+```bash
+docker buildx build \
+  -f docker/images/runners/Dockerfile \
+  -t n8nio/runners:custom \
+  .
+```
+
+### 5) Run it
+
+Same as before, but use your custom image's tag:
+
+```bash
+docker run --rm -it \
+  -e N8N_RUNNERS_AUTH_TOKEN=test \
+  -e N8N_RUNNERS_LAUNCHER_LOG_LEVEL=debug \
+  -e N8N_RUNNERS_TASK_BROKER_URI=http://host.docker.internal:5679 \
+  -p 5680:5680 \
+  n8nio/runners:custom
+```
--- a/docker/images/runners/extras.txt
+++ b/docker/images/runners/extras.txt
@@ -0,0 +1,5 @@
+# Runtime-only extra Requirements File for installing dependencies in the Python task runner image.
+# Installed at Docker image build time. Allow usage in the Code node
+# via 'N8N_RUNNERS_EXTERNAL_ALLOW' env variable on n8n-task-runners.json.
+
+# numpy==2.3.2
--- a/docker/images/runners/package.json
+++ b/docker/images/runners/package.json
@@ -0,0 +1,8 @@
+{
+  "name": "task-runner-runtime-extras",
+  "description": "Runtime-only extra dependencies for installing packages in the JavaScript task runner image. Installed at Docker image build time. Allow usage in the Code node via 'NODE_FUNCTION_ALLOW_EXTERNAL' env variable on n8n-task-runners.json.",
+  "private": true,
+  "dependencies": {
+    "moment": "2.30.1"
+  }
+}
--- a/packages/@n8n/task-runner/package.json
+++ b/packages/@n8n/task-runner/package.json
@@ -42,7 +42,6 @@
    "acorn-walk": "8.3.4",
    "lodash": "catalog:",
    "luxon": "catalog:",
-    "moment": "2.30.1",
    "n8n-core": "workspace:*",
    "n8n-workflow": "workspace:*",
    "nanoid": "catalog:",
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -1372,9 +1372,6 @@ importers:
      luxon:
        specifier: 'catalog:'
        version: 3.4.4
-      moment:
-        specifier: 2.30.1
-        version: 2.30.1
      n8n-core:
        specifier: workspace:*
        version: link:../../core