feat(core): Add auth and offers flow to native Python runner (no-changelog) (#18354)

.gitignore

@@ -39,3 +39,4 @@ packages/cli/src/modules/my-feature
 packages/testing/**/.cursor/rules/
 .venv
 .ruff_cache
+__pycache__

packages/@n8n/task-runner-python/.editorconfig

@@ -0,0 +1,3 @@
+[*.py]
+indent_style = space
+indent_size = 4

packages/@n8n/task-runner-python/justfile

@@ -1,5 +1,5 @@
 run:
-    uv run python src/main.py
+    uv run python -m src.main
 
 sync:
     uv sync

packages/@n8n/task-runner-python/pyproject.toml

@@ -4,7 +4,10 @@ version = "0.1.0"
 description = "Native Python task runner for n8n"
 readme = "README.md"
 requires-python = ">=3.13"
-dependencies = []
+dependencies = [
+    "nanoid>=2.0.0",
+    "websockets>=15.0.1",
+]
 
 [dependency-groups]
 dev = [

packages/@n8n/task-runner-python/pyrightconfig.json

@@ -0,0 +1,4 @@
+{
+	"venvPath": ".",
+	"venv": ".venv"
+}

packages/@n8n/task-runner-python/src/constants.py

@@ -0,0 +1,26 @@
+# Message Types
+BROKER_INFO_REQUEST = "broker:inforequest"
+BROKER_RUNNER_REGISTERED = "broker:runnerregistered"
+BROKER_TASK_OFFER_ACCEPT = "broker:taskofferaccept"
+
+RUNNER_INFO = "runner:info"
+RUNNER_TASK_OFFER = "runner:taskoffer"
+RUNNER_TASK_ACCEPTED = "runner:taskaccepted"
+RUNNER_TASK_REJECTED = "runner:taskrejected"
+
+# Task Runner Defaults
+TASK_TYPE_PYTHON = "python"
+DEFAULT_MAX_CONCURRENCY = 5
+DEFAULT_MAX_PAYLOAD_SIZE = 1024 * 1024 * 1024  # 1 GiB
+OFFER_INTERVAL = 0.25  # 250ms
+OFFER_VALIDITY = 5000  # ms
+OFFER_VALIDITY_MAX_JITTER = 500  # ms
+OFFER_VALIDITY_LATENCY_BUFFER = 0.1  # 100ms
+DEFAULT_TASK_BROKER_URI = "http://127.0.0.1:5679"
+
+# Environment Variables
+ENV_TASK_BROKER_URI = "N8N_RUNNERS_TASK_BROKER_URI"
+ENV_GRANT_TOKEN = "N8N_RUNNERS_GRANT_TOKEN"
+
+# WebSocket Paths
+WS_RUNNERS_PATH = "/runners/_ws"

packages/@n8n/task-runner-python/src/main.py

@@ -1,6 +1,38 @@
-def main():
-    print("Coming soon")
+import asyncio
+import logging
+import os
+import sys
+
+from .constants import ENV_TASK_BROKER_URI, ENV_GRANT_TOKEN, DEFAULT_TASK_BROKER_URI
+from .task_runner import TaskRunner
+
+logging.basicConfig(
+    level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+)
+
+logger = logging.getLogger(__name__)
+
+
+async def main():
+    task_broker_uri = os.getenv(ENV_TASK_BROKER_URI, DEFAULT_TASK_BROKER_URI)
+    grant_token = os.getenv(ENV_GRANT_TOKEN, "")
+
+    if not grant_token:
+        logger.error(f"{ENV_GRANT_TOKEN} environment variable is required")
+        sys.exit(1)
+
+    runner = TaskRunner(
+        task_broker_uri=task_broker_uri,
+        grant_token=grant_token,
+    )
+
+    try:
+        await runner.start()
+    except KeyboardInterrupt:
+        logger.info("Shutting down...")
+    finally:
+        await runner.stop()
 
 
 if __name__ == "__main__":
-    main()
+    asyncio.run(main())

packages/@n8n/task-runner-python/src/message_serde.py

@@ -0,0 +1,49 @@
+import json
+from dataclasses import asdict
+from .constants import (
+    BROKER_INFO_REQUEST,
+    BROKER_RUNNER_REGISTERED,
+    BROKER_TASK_OFFER_ACCEPT,
+)
+from .message_types import (
+    BrokerMessage,
+    RunnerMessage,
+    BrokerInfoRequest,
+    BrokerRunnerRegistered,
+    BrokerTaskOfferAccept,
+)
+
+
+class MessageSerde:
+    """Handles serialization and deserialization of broker and runner messages."""
+
+    MESSAGE_TYPE_MAP = {
+        BROKER_INFO_REQUEST: lambda _: BrokerInfoRequest(),
+        BROKER_RUNNER_REGISTERED: lambda _: BrokerRunnerRegistered(),
+        BROKER_TASK_OFFER_ACCEPT: lambda d: BrokerTaskOfferAccept(
+            task_id=d["taskId"], offer_id=d["offerId"]
+        ),
+    }
+
+    @staticmethod
+    def deserialize_broker_message(data: str) -> BrokerMessage:
+        message_dict = json.loads(data)
+        message_type = message_dict.get("type")
+
+        if message_type in MessageSerde.MESSAGE_TYPE_MAP:
+            return MessageSerde.MESSAGE_TYPE_MAP[message_type](message_dict)
+        else:
+            raise ValueError(f"Unknown message type: {message_type}")
+
+    @staticmethod
+    def serialize_runner_message(message: RunnerMessage) -> str:
+        data = asdict(message)
+        camel_case_data = {
+            MessageSerde._snake_to_camel_case(k): v for k, v in data.items()
+        }
+        return json.dumps(camel_case_data)
+
+    @staticmethod
+    def _snake_to_camel_case(snake_case_str: str) -> str:
+        parts = snake_case_str.split("_")
+        return parts[0] + "".join(word.capitalize() for word in parts[1:])

packages/@n8n/task-runner-python/src/message_types/__init__.py

@@ -0,0 +1,25 @@
+from .broker import (
+    BrokerMessage,
+    BrokerInfoRequest,
+    BrokerRunnerRegistered,
+    BrokerTaskOfferAccept,
+)
+from .runner import (
+    RunnerMessage,
+    RunnerInfo,
+    RunnerTaskOffer,
+    RunnerTaskAccepted,
+    RunnerTaskRejected,
+)
+
+__all__ = [
+    "BrokerMessage",
+    "BrokerInfoRequest",
+    "BrokerRunnerRegistered",
+    "BrokerTaskOfferAccept",
+    "RunnerMessage",
+    "RunnerInfo",
+    "RunnerTaskOffer",
+    "RunnerTaskAccepted",
+    "RunnerTaskRejected",
+]

packages/@n8n/task-runner-python/src/message_types/broker.py

@@ -0,0 +1,26 @@
+from dataclasses import dataclass
+from typing import Literal, Union
+
+
+@dataclass
+class BrokerInfoRequest:
+    type: Literal["broker:inforequest"] = "broker:inforequest"
+
+
+@dataclass
+class BrokerRunnerRegistered:
+    type: Literal["broker:runnerregistered"] = "broker:runnerregistered"
+
+
+@dataclass
+class BrokerTaskOfferAccept:
+    task_id: str
+    offer_id: str
+    type: Literal["broker:taskofferaccept"] = "broker:taskofferaccept"
+
+
+BrokerMessage = Union[
+    BrokerInfoRequest,
+    BrokerRunnerRegistered,
+    BrokerTaskOfferAccept,
+]

packages/@n8n/task-runner-python/src/message_types/runner.py

@@ -0,0 +1,38 @@
+from dataclasses import dataclass
+from typing import List, Literal, Union
+
+
+@dataclass
+class RunnerInfo:
+    name: str
+    types: List[str]
+    type: Literal["runner:info"] = "runner:info"
+
+
+@dataclass
+class RunnerTaskOffer:
+    offer_id: str
+    task_type: str
+    valid_for: int
+    type: Literal["runner:taskoffer"] = "runner:taskoffer"
+
+
+@dataclass
+class RunnerTaskAccepted:
+    task_id: str
+    type: Literal["runner:taskaccepted"] = "runner:taskaccepted"
+
+
+@dataclass
+class RunnerTaskRejected:
+    task_id: str
+    reason: str
+    type: Literal["runner:taskrejected"] = "runner:taskrejected"
+
+
+RunnerMessage = Union[
+    RunnerInfo,
+    RunnerTaskOffer,
+    RunnerTaskAccepted,
+    RunnerTaskRejected,
+]

packages/@n8n/task-runner-python/src/task_runner.py

@@ -0,0 +1,213 @@
+import asyncio
+import logging
+import time
+from typing import Dict, Optional
+from urllib.parse import urlparse
+from typing import Any
+import websockets
+import random
+
+from nanoid import generate as nanoid
+
+from .constants import (
+    TASK_TYPE_PYTHON,
+    DEFAULT_MAX_CONCURRENCY,
+    DEFAULT_MAX_PAYLOAD_SIZE,
+    OFFER_INTERVAL,
+    OFFER_VALIDITY,
+    OFFER_VALIDITY_MAX_JITTER,
+    OFFER_VALIDITY_LATENCY_BUFFER,
+    WS_RUNNERS_PATH,
+)
+from .message_types import (
+    BrokerMessage,
+    RunnerMessage,
+    BrokerInfoRequest,
+    BrokerRunnerRegistered,
+    BrokerTaskOfferAccept,
+    RunnerInfo,
+    RunnerTaskOffer,
+    RunnerTaskAccepted,
+    RunnerTaskRejected,
+)
+from .message_serde import MessageSerde
+
+logging.basicConfig(
+    level=logging.DEBUG, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+)
+logger = logging.getLogger(__name__)
+
+
+class TaskOffer:
+    def __init__(self, offer_id: str, valid_until: float):
+        self.offer_id = offer_id
+        self.valid_until = valid_until
+
+    @property
+    def has_expired(self) -> bool:
+        return time.time() > self.valid_until
+
+
+class TaskRunner:
+    def __init__(
+        self,
+        task_broker_uri: str = "http://127.0.0.1:5679",
+        grant_token: str = "",
+    ):
+        self.runner_id = nanoid()
+
+        self.task_broker_uri = task_broker_uri
+        self.grant_token = grant_token
+        self.name = "Python Task Runner"
+        self.max_concurrency = DEFAULT_MAX_CONCURRENCY
+        self.max_payload_size = DEFAULT_MAX_PAYLOAD_SIZE
+
+        self.websocket: Optional[Any] = None
+        self.can_send_offers = False
+
+        self.open_offers: Dict[str, TaskOffer] = {}  # offer_id -> TaskOffer
+        self.running_tasks: Dict[str, str] = {}  # task_id -> offer_id
+
+        self.offers_coroutine: Optional[asyncio.Task] = None
+
+        ws_host = urlparse(task_broker_uri).netloc
+        self.ws_url = f"ws://{ws_host}{WS_RUNNERS_PATH}?id={self.runner_id}"
+
+    async def start(self) -> None:
+        logger.info("Starting Python task runner...")
+
+        headers = {"Authorization": f"Bearer {self.grant_token}"}
+
+        try:
+            self.websocket = await websockets.connect(
+                self.ws_url,
+                additional_headers=headers,
+                max_size=self.max_payload_size,
+            )
+
+            logger.info(f"Connected to task broker at {self.ws_url}")
+
+            await self._listen_for_messages()
+
+        except Exception as e:
+            logger.error(f"Failed to connect to task broker: {e}")
+            raise
+
+    async def stop(self) -> None:
+        logger.info("Stopping Python task runner...")
+
+        if self.offers_coroutine:
+            self.offers_coroutine.cancel()
+
+        if self.websocket:
+            await self.websocket.close()
+
+    # ========== Messages ==========
+
+    async def _listen_for_messages(self) -> None:
+        if self.websocket is None:
+            raise RuntimeError("WebSocket not connected")
+
+        async for raw_message in self.websocket:
+            try:
+                message = MessageSerde.deserialize_broker_message(raw_message)
+                await self._handle_message(message)
+            except Exception as e:
+                logger.error(f"Error handling message: {e}")
+
+    async def _handle_message(self, message: BrokerMessage) -> None:
+        if isinstance(message, BrokerInfoRequest):
+            await self._handle_info_request()
+        elif isinstance(message, BrokerRunnerRegistered):
+            await self._handle_runner_registered()
+        elif isinstance(message, BrokerTaskOfferAccept):
+            await self._handle_task_offer_accept(message)
+        else:
+            logger.warning(f"Unhandled message type: {type(message)}")
+
+    async def _handle_info_request(self) -> None:
+        response = RunnerInfo(name=self.name, types=[TASK_TYPE_PYTHON])
+        await self._send_message(response)
+
+    async def _handle_runner_registered(self) -> None:
+        self.can_send_offers = True
+        self.offers_coroutine = asyncio.create_task(self._send_offers_loop())
+
+    async def _handle_task_offer_accept(self, message: BrokerTaskOfferAccept) -> None:
+        offer = self.open_offers.get(message.offer_id)
+
+        if not offer or offer.has_expired:
+            response = RunnerTaskRejected(
+                task_id=message.task_id,
+                reason="Offer expired - not accepted within validity window",
+            )
+            await self._send_message(response)
+            return
+
+        if len(self.running_tasks) >= self.max_concurrency:
+            response = RunnerTaskRejected(
+                task_id=message.task_id,
+                reason="No open task slots - runner already at capacity",
+            )
+            await self._send_message(response)
+            return
+
+        del self.open_offers[message.offer_id]
+        self.running_tasks[message.task_id] = message.offer_id
+
+        response = RunnerTaskAccepted(task_id=message.task_id)
+        await self._send_message(response)
+
+    async def _send_message(self, message: RunnerMessage) -> None:
+        if not self.websocket:
+            raise RuntimeError("WebSocket not connected")
+
+        serialized = MessageSerde.serialize_runner_message(message)
+        await self.websocket.send(serialized)
+
+    # ========== Offers ==========
+
+    async def _send_offers_loop(self) -> None:
+        while self.can_send_offers:
+            try:
+                await self._send_offers()
+                await asyncio.sleep(OFFER_INTERVAL)
+            except asyncio.CancelledError:
+                break
+            except Exception as e:
+                logger.error(f"Error sending offers: {e}")
+
+    async def _send_offers(self) -> None:
+        if not self.can_send_offers:
+            return
+
+        expired_offer_ids = [
+            offer_id
+            for offer_id, offer in self.open_offers.items()
+            if offer.has_expired
+        ]
+
+        for offer_id in expired_offer_ids:
+            del self.open_offers[offer_id]
+
+        offers_to_send = self.max_concurrency - (
+            len(self.open_offers) + len(self.running_tasks)
+        )
+
+        for _ in range(offers_to_send):
+            offer_id = nanoid()
+
+            valid_for_ms = OFFER_VALIDITY + random.randint(0, OFFER_VALIDITY_MAX_JITTER)
+
+            valid_until = (
+                time.time() + (valid_for_ms / 1000) + OFFER_VALIDITY_LATENCY_BUFFER
+            )
+
+            offer = TaskOffer(offer_id, valid_until)
+            self.open_offers[offer_id] = offer
+
+            message = RunnerTaskOffer(
+                offer_id=offer_id, task_type=TASK_TYPE_PYTHON, valid_for=valid_for_ms
+            )
+
+            await self._send_message(message)

packages/@n8n/task-runner-python/uv.lock

@@ -2,6 +2,15 @@ version = 1
 revision = 3
 requires-python = ">=3.13"
 
+[[package]]
+name = "nanoid"
+version = "2.0.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/b7/9d/0250bf5935d88e214df469d35eccc0f6ff7e9db046fc8a9aeb4b2a192775/nanoid-2.0.0.tar.gz", hash = "sha256:5a80cad5e9c6e9ae3a41fa2fb34ae189f7cb420b2a5d8f82bd9d23466e4efa68", size = 3290, upload-time = "2018-11-20T14:45:51.578Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/2e/0d/8630f13998638dc01e187fadd2e5c6d42d127d08aeb4943d231664d6e539/nanoid-2.0.0-py3-none-any.whl", hash = "sha256:90aefa650e328cffb0893bbd4c236cfd44c48bc1f2d0b525ecc53c3187b653bb", size = 5844, upload-time = "2018-11-20T14:45:50.165Z" },
+]
+
 [[package]]
 name = "ruff"
 version = "0.12.8"
@@ -31,6 +40,10 @@ wheels = [
 name = "task-runner-python"
 version = "0.1.0"
 source = { virtual = "." }
+dependencies = [
+    { name = "nanoid" },
+    { name = "websockets" },
+]
 
 [package.dev-dependencies]
 dev = [
@@ -39,6 +52,10 @@ dev = [
 ]
 
 [package.metadata]
+requires-dist = [
+    { name = "nanoid", specifier = ">=2.0.0" },
+    { name = "websockets", specifier = ">=15.0.1" },
+]
 
 [package.metadata.requires-dev]
 dev = [
@@ -70,3 +87,23 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/f5/94/08e2b3f6bc0af97abcd3fcc8ea28797a627296613256ae37e98043c871ca/ty-0.0.1a17-py3-none-win_amd64.whl", hash = "sha256:7d00b569ebd4635c58840d2ed9e1d2d8b36f496619c0bc0c8d1777767786b508", size = 8230727, upload-time = "2025-08-06T12:13:50.705Z" },
     { url = "https://files.pythonhosted.org/packages/98/c6/207bbc2f3bb71df4b1aeabe8e9c31a1cd22c72aff0ab9c1a832b9ae54f6e/ty-0.0.1a17-py3-none-win_arm64.whl", hash = "sha256:636eacc1dceaf09325415a70a03cd57eae53e5c7f281813aaa943a698a45cddb", size = 7782847, upload-time = "2025-08-06T12:13:54.243Z" },
 ]
+
+[[package]]
+name = "websockets"
+version = "15.0.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/21/e6/26d09fab466b7ca9c7737474c52be4f76a40301b08362eb2dbc19dcc16c1/websockets-15.0.1.tar.gz", hash = "sha256:82544de02076bafba038ce055ee6412d68da13ab47f0c60cab827346de828dee", size = 177016, upload-time = "2025-03-05T20:03:41.606Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/cb/9f/51f0cf64471a9d2b4d0fc6c534f323b664e7095640c34562f5182e5a7195/websockets-15.0.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ee443ef070bb3b6ed74514f5efaa37a252af57c90eb33b956d35c8e9c10a1931", size = 175440, upload-time = "2025-03-05T20:02:36.695Z" },
+    { url = "https://files.pythonhosted.org/packages/8a/05/aa116ec9943c718905997412c5989f7ed671bc0188ee2ba89520e8765d7b/websockets-15.0.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5a939de6b7b4e18ca683218320fc67ea886038265fd1ed30173f5ce3f8e85675", size = 173098, upload-time = "2025-03-05T20:02:37.985Z" },
+    { url = "https://files.pythonhosted.org/packages/ff/0b/33cef55ff24f2d92924923c99926dcce78e7bd922d649467f0eda8368923/websockets-15.0.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:746ee8dba912cd6fc889a8147168991d50ed70447bf18bcda7039f7d2e3d9151", size = 173329, upload-time = "2025-03-05T20:02:39.298Z" },
+    { url = "https://files.pythonhosted.org/packages/31/1d/063b25dcc01faa8fada1469bdf769de3768b7044eac9d41f734fd7b6ad6d/websockets-15.0.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:595b6c3969023ecf9041b2936ac3827e4623bfa3ccf007575f04c5a6aa318c22", size = 183111, upload-time = "2025-03-05T20:02:40.595Z" },
+    { url = "https://files.pythonhosted.org/packages/93/53/9a87ee494a51bf63e4ec9241c1ccc4f7c2f45fff85d5bde2ff74fcb68b9e/websockets-15.0.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3c714d2fc58b5ca3e285461a4cc0c9a66bd0e24c5da9911e30158286c9b5be7f", size = 182054, upload-time = "2025-03-05T20:02:41.926Z" },
+    { url = "https://files.pythonhosted.org/packages/ff/b2/83a6ddf56cdcbad4e3d841fcc55d6ba7d19aeb89c50f24dd7e859ec0805f/websockets-15.0.1-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0f3c1e2ab208db911594ae5b4f79addeb3501604a165019dd221c0bdcabe4db8", size = 182496, upload-time = "2025-03-05T20:02:43.304Z" },
+    { url = "https://files.pythonhosted.org/packages/98/41/e7038944ed0abf34c45aa4635ba28136f06052e08fc2168520bb8b25149f/websockets-15.0.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:229cf1d3ca6c1804400b0a9790dc66528e08a6a1feec0d5040e8b9eb14422375", size = 182829, upload-time = "2025-03-05T20:02:48.812Z" },
+    { url = "https://files.pythonhosted.org/packages/e0/17/de15b6158680c7623c6ef0db361da965ab25d813ae54fcfeae2e5b9ef910/websockets-15.0.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:756c56e867a90fb00177d530dca4b097dd753cde348448a1012ed6c5131f8b7d", size = 182217, upload-time = "2025-03-05T20:02:50.14Z" },
+    { url = "https://files.pythonhosted.org/packages/33/2b/1f168cb6041853eef0362fb9554c3824367c5560cbdaad89ac40f8c2edfc/websockets-15.0.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:558d023b3df0bffe50a04e710bc87742de35060580a293c2a984299ed83bc4e4", size = 182195, upload-time = "2025-03-05T20:02:51.561Z" },
+    { url = "https://files.pythonhosted.org/packages/86/eb/20b6cdf273913d0ad05a6a14aed4b9a85591c18a987a3d47f20fa13dcc47/websockets-15.0.1-cp313-cp313-win32.whl", hash = "sha256:ba9e56e8ceeeedb2e080147ba85ffcd5cd0711b89576b83784d8605a7df455fa", size = 176393, upload-time = "2025-03-05T20:02:53.814Z" },
+    { url = "https://files.pythonhosted.org/packages/1b/6c/c65773d6cab416a64d191d6ee8a8b1c68a09970ea6909d16965d26bfed1e/websockets-15.0.1-cp313-cp313-win_amd64.whl", hash = "sha256:e09473f095a819042ecb2ab9465aee615bd9c2028e4ef7d933600a8401c79561", size = 176837, upload-time = "2025-03-05T20:02:55.237Z" },
+    { url = "https://files.pythonhosted.org/packages/fa/a8/5b41e0da817d64113292ab1f8247140aac61cbf6cfd085d6a0fa77f4984f/websockets-15.0.1-py3-none-any.whl", hash = "sha256:f7a866fbc1e97b5c617ee4116daaa09b722101d4a3c170c787450ba409f9736f", size = 169743, upload-time = "2025-03-05T20:03:39.41Z" },
+]