mirror of
https://github.com/Abdulazizzn/n8n-enterprise-unlocked.git
synced 2025-12-17 01:56:46 +00:00
chore: Add Sentry to native Python runner (#19082)
This commit is contained in:
2
.github/workflows/ci-python.yml
vendored
2
.github/workflows/ci-python.yml
vendored
@@ -29,7 +29,7 @@ jobs:
|
|||||||
run: uv python install 3.13
|
run: uv python install 3.13
|
||||||
|
|
||||||
- name: Install project dependencies
|
- name: Install project dependencies
|
||||||
run: uv sync
|
run: uv sync --all-extras
|
||||||
|
|
||||||
- name: Format check
|
- name: Format check
|
||||||
run: uv run ruff format --check
|
run: uv run ruff format --check
|
||||||
|
|||||||
@@ -12,8 +12,9 @@ Install:
|
|||||||
|
|
||||||
Set up dependencies:
|
Set up dependencies:
|
||||||
|
|
||||||
```
|
```sh
|
||||||
just sync
|
just sync # or
|
||||||
|
just sync-all
|
||||||
```
|
```
|
||||||
|
|
||||||
See `justfile` for available commands.
|
See `justfile` for available commands.
|
||||||
|
|||||||
@@ -9,6 +9,9 @@ run:
|
|||||||
sync:
|
sync:
|
||||||
uv sync
|
uv sync
|
||||||
|
|
||||||
|
sync-all:
|
||||||
|
uv sync --all-extras
|
||||||
|
|
||||||
lint:
|
lint:
|
||||||
uv run ruff check
|
uv run ruff check
|
||||||
|
|
||||||
|
|||||||
@@ -8,6 +8,9 @@ dependencies = [
|
|||||||
"websockets>=15.0.1",
|
"websockets>=15.0.1",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[project.optional-dependencies]
|
||||||
|
sentry = ["sentry-sdk>=2.35.2"]
|
||||||
|
|
||||||
[dependency-groups]
|
[dependency-groups]
|
||||||
dev = [
|
dev = [
|
||||||
"ruff>=0.12.8",
|
"ruff>=0.12.8",
|
||||||
|
|||||||
@@ -0,0 +1,35 @@
|
|||||||
|
import os
|
||||||
|
from dataclasses import dataclass
|
||||||
|
|
||||||
|
from src.constants import (
|
||||||
|
DEFAULT_HEALTH_CHECK_SERVER_HOST,
|
||||||
|
DEFAULT_HEALTH_CHECK_SERVER_PORT,
|
||||||
|
ENV_HEALTH_CHECK_SERVER_ENABLED,
|
||||||
|
ENV_HEALTH_CHECK_SERVER_HOST,
|
||||||
|
ENV_HEALTH_CHECK_SERVER_PORT,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class HealthCheckConfig:
|
||||||
|
enabled: bool
|
||||||
|
host: str
|
||||||
|
port: int
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_env(cls):
|
||||||
|
port_str = os.getenv(
|
||||||
|
ENV_HEALTH_CHECK_SERVER_PORT, str(DEFAULT_HEALTH_CHECK_SERVER_PORT)
|
||||||
|
)
|
||||||
|
port = int(port_str)
|
||||||
|
if port < 1 or port > 65535:
|
||||||
|
raise ValueError(f"Port must be between 1 and 65535, got {port}")
|
||||||
|
|
||||||
|
return cls(
|
||||||
|
enabled=os.getenv(ENV_HEALTH_CHECK_SERVER_ENABLED, "false").lower()
|
||||||
|
== "true",
|
||||||
|
host=os.getenv(
|
||||||
|
ENV_HEALTH_CHECK_SERVER_HOST, DEFAULT_HEALTH_CHECK_SERVER_HOST
|
||||||
|
),
|
||||||
|
port=port,
|
||||||
|
)
|
||||||
30
packages/@n8n/task-runner-python/src/config/sentry_config.py
Normal file
30
packages/@n8n/task-runner-python/src/config/sentry_config.py
Normal file
@@ -0,0 +1,30 @@
|
|||||||
|
import os
|
||||||
|
from dataclasses import dataclass
|
||||||
|
|
||||||
|
from src.constants import (
|
||||||
|
ENV_DEPLOYMENT_NAME,
|
||||||
|
ENV_ENVIRONMENT,
|
||||||
|
ENV_N8N_VERSION,
|
||||||
|
ENV_SENTRY_DSN,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class SentryConfig:
|
||||||
|
dsn: str
|
||||||
|
n8n_version: str
|
||||||
|
environment: str
|
||||||
|
deployment_name: str
|
||||||
|
|
||||||
|
@property
|
||||||
|
def enabled(self) -> bool:
|
||||||
|
return bool(self.dsn)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_env(cls):
|
||||||
|
return cls(
|
||||||
|
dsn=os.getenv(ENV_SENTRY_DSN, ""),
|
||||||
|
n8n_version=os.getenv(ENV_N8N_VERSION, ""),
|
||||||
|
environment=os.getenv(ENV_ENVIRONMENT, ""),
|
||||||
|
deployment_name=os.getenv(ENV_DEPLOYMENT_NAME, ""),
|
||||||
|
)
|
||||||
@@ -0,0 +1,84 @@
|
|||||||
|
import os
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from typing import Set
|
||||||
|
|
||||||
|
from src.constants import (
|
||||||
|
BUILTINS_DENY_DEFAULT,
|
||||||
|
DEFAULT_MAX_CONCURRENCY,
|
||||||
|
DEFAULT_MAX_PAYLOAD_SIZE,
|
||||||
|
DEFAULT_TASK_BROKER_URI,
|
||||||
|
DEFAULT_TASK_TIMEOUT,
|
||||||
|
ENV_BUILTINS_DENY,
|
||||||
|
ENV_EXTERNAL_ALLOW,
|
||||||
|
ENV_GRANT_TOKEN,
|
||||||
|
ENV_MAX_CONCURRENCY,
|
||||||
|
ENV_MAX_PAYLOAD_SIZE,
|
||||||
|
ENV_STDLIB_ALLOW,
|
||||||
|
ENV_TASK_BROKER_URI,
|
||||||
|
ENV_TASK_TIMEOUT,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def parse_allowlist(allowlist_str: str, list_name: str) -> Set[str]:
|
||||||
|
if not allowlist_str:
|
||||||
|
return set()
|
||||||
|
|
||||||
|
modules = {
|
||||||
|
module
|
||||||
|
for raw_module in allowlist_str.split(",")
|
||||||
|
if (module := raw_module.strip())
|
||||||
|
}
|
||||||
|
|
||||||
|
if "*" in modules and len(modules) > 1:
|
||||||
|
raise ValueError(
|
||||||
|
f"Wildcard '*' in {list_name} must be used alone, not with other modules. "
|
||||||
|
f"Got: {', '.join(sorted(modules))}"
|
||||||
|
)
|
||||||
|
|
||||||
|
return modules
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class TaskRunnerConfig:
|
||||||
|
grant_token: str
|
||||||
|
task_broker_uri: str
|
||||||
|
max_concurrency: int
|
||||||
|
max_payload_size: int
|
||||||
|
task_timeout: int
|
||||||
|
stdlib_allow: Set[str]
|
||||||
|
external_allow: Set[str]
|
||||||
|
builtins_deny: Set[str]
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_env(cls):
|
||||||
|
grant_token = os.getenv(ENV_GRANT_TOKEN, "")
|
||||||
|
if not grant_token:
|
||||||
|
raise ValueError("Environment variable N8N_RUNNERS_GRANT_TOKEN is required")
|
||||||
|
|
||||||
|
task_timeout = int(os.getenv(ENV_TASK_TIMEOUT, str(DEFAULT_TASK_TIMEOUT)))
|
||||||
|
if task_timeout <= 0:
|
||||||
|
raise ValueError(f"Task timeout must be positive, got {task_timeout}")
|
||||||
|
|
||||||
|
return cls(
|
||||||
|
grant_token=grant_token,
|
||||||
|
task_broker_uri=os.getenv(ENV_TASK_BROKER_URI, DEFAULT_TASK_BROKER_URI),
|
||||||
|
max_concurrency=int(
|
||||||
|
os.getenv(ENV_MAX_CONCURRENCY, str(DEFAULT_MAX_CONCURRENCY))
|
||||||
|
),
|
||||||
|
max_payload_size=int(
|
||||||
|
os.getenv(ENV_MAX_PAYLOAD_SIZE, str(DEFAULT_MAX_PAYLOAD_SIZE))
|
||||||
|
),
|
||||||
|
task_timeout=task_timeout,
|
||||||
|
stdlib_allow=parse_allowlist(
|
||||||
|
os.getenv(ENV_STDLIB_ALLOW, ""), ENV_STDLIB_ALLOW
|
||||||
|
),
|
||||||
|
external_allow=parse_allowlist(
|
||||||
|
os.getenv(ENV_EXTERNAL_ALLOW, ""), ENV_EXTERNAL_ALLOW
|
||||||
|
),
|
||||||
|
builtins_deny=set(
|
||||||
|
module.strip()
|
||||||
|
for module in os.getenv(ENV_BUILTINS_DENY, BUILTINS_DENY_DEFAULT).split(
|
||||||
|
","
|
||||||
|
)
|
||||||
|
),
|
||||||
|
)
|
||||||
@@ -28,6 +28,9 @@ MAX_VALIDATION_CACHE_SIZE = 500 # cached validation results
|
|||||||
# Executor
|
# Executor
|
||||||
EXECUTOR_USER_OUTPUT_KEY = "__n8n_internal_user_output__"
|
EXECUTOR_USER_OUTPUT_KEY = "__n8n_internal_user_output__"
|
||||||
EXECUTOR_CIRCULAR_REFERENCE_KEY = "__n8n_internal_circular_ref__"
|
EXECUTOR_CIRCULAR_REFERENCE_KEY = "__n8n_internal_circular_ref__"
|
||||||
|
EXECUTOR_ALL_ITEMS_FILENAME = "<all_items_task_execution>"
|
||||||
|
EXECUTOR_PER_ITEM_FILENAME = "<per_item_task_execution>"
|
||||||
|
EXECUTOR_FILENAMES = {EXECUTOR_ALL_ITEMS_FILENAME, EXECUTOR_PER_ITEM_FILENAME}
|
||||||
|
|
||||||
# Broker
|
# Broker
|
||||||
DEFAULT_TASK_BROKER_URI = "http://127.0.0.1:5679"
|
DEFAULT_TASK_BROKER_URI = "http://127.0.0.1:5679"
|
||||||
@@ -49,6 +52,14 @@ ENV_BUILTINS_DENY = "N8N_RUNNERS_BUILTINS_DENY"
|
|||||||
ENV_HEALTH_CHECK_SERVER_ENABLED = "N8N_RUNNERS_HEALTH_CHECK_SERVER_ENABLED"
|
ENV_HEALTH_CHECK_SERVER_ENABLED = "N8N_RUNNERS_HEALTH_CHECK_SERVER_ENABLED"
|
||||||
ENV_HEALTH_CHECK_SERVER_HOST = "N8N_RUNNERS_HEALTH_CHECK_SERVER_HOST"
|
ENV_HEALTH_CHECK_SERVER_HOST = "N8N_RUNNERS_HEALTH_CHECK_SERVER_HOST"
|
||||||
ENV_HEALTH_CHECK_SERVER_PORT = "N8N_RUNNERS_HEALTH_CHECK_SERVER_PORT"
|
ENV_HEALTH_CHECK_SERVER_PORT = "N8N_RUNNERS_HEALTH_CHECK_SERVER_PORT"
|
||||||
|
ENV_SENTRY_DSN = "N8N_SENTRY_DSN"
|
||||||
|
ENV_N8N_VERSION = "N8N_VERSION"
|
||||||
|
ENV_ENVIRONMENT = "ENVIRONMENT"
|
||||||
|
ENV_DEPLOYMENT_NAME = "DEPLOYMENT_NAME"
|
||||||
|
|
||||||
|
# Sentry
|
||||||
|
SENTRY_TAG_SERVER_TYPE = "server_type"
|
||||||
|
SENTRY_TAG_SERVER_TYPE_VALUE = "task_runner_python"
|
||||||
|
|
||||||
# Logging
|
# Logging
|
||||||
LOG_FORMAT = "%(asctime)s.%(msecs)03d\t%(levelname)s\t%(message)s"
|
LOG_FORMAT = "%(asctime)s.%(msecs)03d\t%(levelname)s\t%(message)s"
|
||||||
@@ -59,6 +70,7 @@ LOG_TASK_CANCEL_UNKNOWN = (
|
|||||||
"Received cancel for unknown task: {task_id}. Discarding message."
|
"Received cancel for unknown task: {task_id}. Discarding message."
|
||||||
)
|
)
|
||||||
LOG_TASK_CANCEL_WAITING = "Cancelled task {task_id} (waiting for settings)"
|
LOG_TASK_CANCEL_WAITING = "Cancelled task {task_id} (waiting for settings)"
|
||||||
|
LOG_SENTRY_MISSING = "Sentry is enabled but sentry-sdk is not installed. Install with: uv sync --all-extras"
|
||||||
|
|
||||||
# RPC
|
# RPC
|
||||||
RPC_BROWSER_CONSOLE_LOG_METHOD = "logNodeOutput"
|
RPC_BROWSER_CONSOLE_LOG_METHOD = "logNodeOutput"
|
||||||
|
|||||||
@@ -1,100 +0,0 @@
|
|||||||
import os
|
|
||||||
from dataclasses import dataclass
|
|
||||||
from typing import Set, Tuple
|
|
||||||
|
|
||||||
from src.constants import (
|
|
||||||
DEFAULT_MAX_CONCURRENCY,
|
|
||||||
DEFAULT_TASK_TIMEOUT,
|
|
||||||
DEFAULT_TASK_BROKER_URI,
|
|
||||||
DEFAULT_MAX_PAYLOAD_SIZE,
|
|
||||||
DEFAULT_HEALTH_CHECK_SERVER_HOST,
|
|
||||||
DEFAULT_HEALTH_CHECK_SERVER_PORT,
|
|
||||||
BUILTINS_DENY_DEFAULT,
|
|
||||||
ENV_MAX_CONCURRENCY,
|
|
||||||
ENV_MAX_PAYLOAD_SIZE,
|
|
||||||
ENV_TASK_BROKER_URI,
|
|
||||||
ENV_GRANT_TOKEN,
|
|
||||||
ENV_TASK_TIMEOUT,
|
|
||||||
ENV_BUILTINS_DENY,
|
|
||||||
ENV_STDLIB_ALLOW,
|
|
||||||
ENV_EXTERNAL_ALLOW,
|
|
||||||
ENV_HEALTH_CHECK_SERVER_ENABLED,
|
|
||||||
ENV_HEALTH_CHECK_SERVER_HOST,
|
|
||||||
ENV_HEALTH_CHECK_SERVER_PORT,
|
|
||||||
)
|
|
||||||
from src.task_runner import TaskRunnerOpts
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
|
||||||
class HealthCheckOpts:
|
|
||||||
enabled: bool
|
|
||||||
host: str
|
|
||||||
port: int
|
|
||||||
|
|
||||||
|
|
||||||
def parse_allowlist(allowlist_str: str, list_name: str) -> Set[str]:
|
|
||||||
if not allowlist_str:
|
|
||||||
return set()
|
|
||||||
|
|
||||||
modules = {
|
|
||||||
module
|
|
||||||
for raw_module in allowlist_str.split(",")
|
|
||||||
if (module := raw_module.strip())
|
|
||||||
}
|
|
||||||
|
|
||||||
if "*" in modules and len(modules) > 1:
|
|
||||||
raise ValueError(
|
|
||||||
f"Wildcard '*' in {list_name} must be used alone, not with other modules. "
|
|
||||||
f"Got: {', '.join(sorted(modules))}"
|
|
||||||
)
|
|
||||||
|
|
||||||
return modules
|
|
||||||
|
|
||||||
|
|
||||||
def parse_denylist(denylist_str: str) -> Set[str]:
|
|
||||||
if not denylist_str:
|
|
||||||
return set()
|
|
||||||
|
|
||||||
return {name for raw_name in denylist_str.split(",") if (name := raw_name.strip())}
|
|
||||||
|
|
||||||
|
|
||||||
def parse_env_vars() -> Tuple[TaskRunnerOpts, HealthCheckOpts]:
|
|
||||||
grant_token = os.getenv(ENV_GRANT_TOKEN, "")
|
|
||||||
|
|
||||||
if not grant_token:
|
|
||||||
raise ValueError(f"{ENV_GRANT_TOKEN} environment variable is required")
|
|
||||||
|
|
||||||
builtins_deny_str = os.getenv(ENV_BUILTINS_DENY, BUILTINS_DENY_DEFAULT)
|
|
||||||
builtins_deny = parse_denylist(builtins_deny_str)
|
|
||||||
|
|
||||||
stdlib_allow_str = os.getenv(ENV_STDLIB_ALLOW, "")
|
|
||||||
stdlib_allow = parse_allowlist(stdlib_allow_str, "stdlib allowlist")
|
|
||||||
|
|
||||||
external_allow_str = os.getenv(ENV_EXTERNAL_ALLOW, "")
|
|
||||||
external_allow = parse_allowlist(external_allow_str, "external allowlist")
|
|
||||||
|
|
||||||
task_runner_opts = TaskRunnerOpts(
|
|
||||||
grant_token=grant_token,
|
|
||||||
task_broker_uri=os.getenv(ENV_TASK_BROKER_URI, DEFAULT_TASK_BROKER_URI),
|
|
||||||
max_concurrency=int(
|
|
||||||
os.getenv(ENV_MAX_CONCURRENCY) or str(DEFAULT_MAX_CONCURRENCY)
|
|
||||||
),
|
|
||||||
max_payload_size=int(
|
|
||||||
os.getenv(ENV_MAX_PAYLOAD_SIZE) or str(DEFAULT_MAX_PAYLOAD_SIZE)
|
|
||||||
),
|
|
||||||
task_timeout=int(os.getenv(ENV_TASK_TIMEOUT) or str(DEFAULT_TASK_TIMEOUT)),
|
|
||||||
stdlib_allow=stdlib_allow,
|
|
||||||
external_allow=external_allow,
|
|
||||||
builtins_deny=builtins_deny,
|
|
||||||
)
|
|
||||||
|
|
||||||
health_check_opts = HealthCheckOpts(
|
|
||||||
enabled=os.getenv(ENV_HEALTH_CHECK_SERVER_ENABLED, "") == "true",
|
|
||||||
host=os.getenv(ENV_HEALTH_CHECK_SERVER_HOST, DEFAULT_HEALTH_CHECK_SERVER_HOST),
|
|
||||||
port=int(
|
|
||||||
os.getenv(ENV_HEALTH_CHECK_SERVER_PORT)
|
|
||||||
or str(DEFAULT_HEALTH_CHECK_SERVER_PORT)
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
return task_runner_opts, health_check_opts
|
|
||||||
@@ -3,6 +3,8 @@ import errno
|
|||||||
import logging
|
import logging
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
|
|
||||||
|
from src.config.health_check_config import HealthCheckConfig
|
||||||
|
|
||||||
HEALTH_CHECK_RESPONSE = (
|
HEALTH_CHECK_RESPONSE = (
|
||||||
b"HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 2\r\n\r\nOK"
|
b"HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 2\r\n\r\nOK"
|
||||||
)
|
)
|
||||||
@@ -13,13 +15,17 @@ class HealthCheckServer:
|
|||||||
self.server: Optional[asyncio.Server] = None
|
self.server: Optional[asyncio.Server] = None
|
||||||
self.logger = logging.getLogger(__name__)
|
self.logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
async def start(self, host: str, port: int) -> None:
|
async def start(self, config: HealthCheckConfig) -> None:
|
||||||
try:
|
try:
|
||||||
self.server = await asyncio.start_server(self._handle_request, host, port)
|
self.server = await asyncio.start_server(
|
||||||
self.logger.info(f"Health check server listening on {host}, port {port}")
|
self._handle_request, config.host, config.port
|
||||||
|
)
|
||||||
|
self.logger.info(
|
||||||
|
f"Health check server listening on {config.host}, port {config.port}"
|
||||||
|
)
|
||||||
except OSError as e:
|
except OSError as e:
|
||||||
if e.errno == errno.EADDRINUSE:
|
if e.errno == errno.EADDRINUSE:
|
||||||
raise OSError(f"Port {port} is already in use") from e
|
raise OSError(f"Port {config.port} is already in use") from e
|
||||||
else:
|
else:
|
||||||
raise
|
raise
|
||||||
|
|
||||||
@@ -3,7 +3,9 @@ import logging
|
|||||||
import sys
|
import sys
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
|
|
||||||
from src.env import parse_env_vars
|
from src.config.health_check_config import HealthCheckConfig
|
||||||
|
from src.config.sentry_config import SentryConfig
|
||||||
|
from src.config.task_runner_config import TaskRunnerConfig
|
||||||
from src.logs import setup_logging
|
from src.logs import setup_logging
|
||||||
from src.task_runner import TaskRunner
|
from src.task_runner import TaskRunner
|
||||||
|
|
||||||
@@ -12,39 +14,56 @@ async def main():
|
|||||||
setup_logging()
|
setup_logging()
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
logger.info("Starting runner...")
|
sentry = None
|
||||||
|
sentry_config = SentryConfig.from_env()
|
||||||
|
|
||||||
|
if sentry_config.enabled:
|
||||||
|
from src.sentry import setup_sentry
|
||||||
|
|
||||||
|
sentry = setup_sentry(sentry_config)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
task_runner_opts, health_check_opts = parse_env_vars()
|
health_check_config = HealthCheckConfig.from_env()
|
||||||
except ValueError as e:
|
except ValueError as e:
|
||||||
logger.error(str(e))
|
logger.error(f"Invalid health check configuration: {e}")
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
task_runner = TaskRunner(task_runner_opts)
|
|
||||||
health_check_server: Optional["HealthCheckServer"] = None
|
health_check_server: Optional["HealthCheckServer"] = None
|
||||||
|
if health_check_config.enabled:
|
||||||
if health_check_opts.enabled:
|
from src.health_check_server import HealthCheckServer
|
||||||
from src.health import HealthCheckServer
|
|
||||||
|
|
||||||
health_check_server = HealthCheckServer()
|
health_check_server = HealthCheckServer()
|
||||||
try:
|
try:
|
||||||
await health_check_server.start(
|
await health_check_server.start(health_check_config)
|
||||||
health_check_opts.host, health_check_opts.port
|
|
||||||
)
|
|
||||||
except OSError as e:
|
except OSError as e:
|
||||||
logger.error(f"Failed to start health check server: {e}")
|
logger.error(f"Failed to start health check server: {e}")
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
|
try:
|
||||||
|
task_runner_config = TaskRunnerConfig.from_env()
|
||||||
|
except ValueError as e:
|
||||||
|
logger.error(str(e))
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
task_runner = TaskRunner(task_runner_config)
|
||||||
|
logger.info("Starting runner...")
|
||||||
|
|
||||||
try:
|
try:
|
||||||
await task_runner.start()
|
await task_runner.start()
|
||||||
except (KeyboardInterrupt, asyncio.CancelledError):
|
except (KeyboardInterrupt, asyncio.CancelledError):
|
||||||
logger.info("Shutting down runner...")
|
logger.info("Shutting down runner...")
|
||||||
|
except Exception:
|
||||||
|
logger.error("Unexpected error", exc_info=True)
|
||||||
|
raise
|
||||||
finally:
|
finally:
|
||||||
await task_runner.stop()
|
await task_runner.stop()
|
||||||
|
|
||||||
if health_check_server:
|
if health_check_server:
|
||||||
await health_check_server.stop()
|
await health_check_server.stop()
|
||||||
|
|
||||||
|
if sentry:
|
||||||
|
sentry.shutdown()
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
asyncio.run(main())
|
||||||
|
|||||||
78
packages/@n8n/task-runner-python/src/sentry.py
Normal file
78
packages/@n8n/task-runner-python/src/sentry.py
Normal file
@@ -0,0 +1,78 @@
|
|||||||
|
import logging
|
||||||
|
from typing import Any, Optional
|
||||||
|
|
||||||
|
from src.errors.task_runtime_error import TaskRuntimeError
|
||||||
|
from src.config.sentry_config import SentryConfig
|
||||||
|
from src.constants import (
|
||||||
|
EXECUTOR_FILENAMES,
|
||||||
|
LOG_SENTRY_MISSING,
|
||||||
|
SENTRY_TAG_SERVER_TYPE,
|
||||||
|
SENTRY_TAG_SERVER_TYPE_VALUE,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class TaskRunnerSentry:
|
||||||
|
def __init__(self, config: SentryConfig):
|
||||||
|
self.config = config
|
||||||
|
self.logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
def init(self) -> None:
|
||||||
|
import sentry_sdk
|
||||||
|
from sentry_sdk.integrations.logging import LoggingIntegration
|
||||||
|
|
||||||
|
sentry_sdk.init(
|
||||||
|
dsn=self.config.dsn,
|
||||||
|
release=f"n8n@{self.config.n8n_version}",
|
||||||
|
environment=self.config.environment,
|
||||||
|
server_name=self.config.deployment_name,
|
||||||
|
before_send=self._filter_out_user_code_errors,
|
||||||
|
attach_stacktrace=True,
|
||||||
|
send_default_pii=False,
|
||||||
|
auto_enabling_integrations=False,
|
||||||
|
default_integrations=True,
|
||||||
|
integrations=[LoggingIntegration(level=logging.ERROR)],
|
||||||
|
)
|
||||||
|
sentry_sdk.set_tag(SENTRY_TAG_SERVER_TYPE, SENTRY_TAG_SERVER_TYPE_VALUE)
|
||||||
|
self.logger.info("Sentry ready")
|
||||||
|
|
||||||
|
def shutdown(self) -> None:
|
||||||
|
import sentry_sdk
|
||||||
|
|
||||||
|
sentry_sdk.flush(timeout=2.0)
|
||||||
|
self.logger.info("Sentry stopped")
|
||||||
|
|
||||||
|
def _filter_out_user_code_errors(self, event: Any, hint: Any) -> Optional[Any]:
|
||||||
|
if "exc_info" in hint:
|
||||||
|
exc_type, _, _ = hint["exc_info"]
|
||||||
|
if exc_type is TaskRuntimeError:
|
||||||
|
return None
|
||||||
|
|
||||||
|
for exception in event.get("exception", {}).get("values", []):
|
||||||
|
if self._is_from_user_code(exception):
|
||||||
|
return None
|
||||||
|
|
||||||
|
return event
|
||||||
|
|
||||||
|
def _is_from_user_code(self, exception: dict[str, Any]):
|
||||||
|
for frame in exception.get("stacktrace", {}).get("frames", []):
|
||||||
|
if frame.get("filename", "") in EXECUTOR_FILENAMES:
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def setup_sentry(sentry_config: SentryConfig) -> Optional[TaskRunnerSentry]:
|
||||||
|
if not sentry_config.enabled:
|
||||||
|
return None
|
||||||
|
|
||||||
|
try:
|
||||||
|
sentry = TaskRunnerSentry(sentry_config)
|
||||||
|
sentry.init()
|
||||||
|
return sentry
|
||||||
|
except ImportError:
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
logger.warning(LOG_SENTRY_MISSING)
|
||||||
|
return None
|
||||||
|
except Exception as e:
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
logger.warning(f"Failed to initialize Sentry: {e}")
|
||||||
|
return None
|
||||||
@@ -14,7 +14,12 @@ from src.errors import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
from src.message_types.broker import NodeMode, Items
|
from src.message_types.broker import NodeMode, Items
|
||||||
from src.constants import EXECUTOR_CIRCULAR_REFERENCE_KEY, EXECUTOR_USER_OUTPUT_KEY
|
from src.constants import (
|
||||||
|
EXECUTOR_CIRCULAR_REFERENCE_KEY,
|
||||||
|
EXECUTOR_USER_OUTPUT_KEY,
|
||||||
|
EXECUTOR_ALL_ITEMS_FILENAME,
|
||||||
|
EXECUTOR_PER_ITEM_FILENAME,
|
||||||
|
)
|
||||||
from typing import Any, Set
|
from typing import Any, Set
|
||||||
|
|
||||||
from multiprocessing.context import SpawnProcess
|
from multiprocessing.context import SpawnProcess
|
||||||
@@ -134,7 +139,8 @@ class TaskExecutor:
|
|||||||
print_args: PrintArgs = []
|
print_args: PrintArgs = []
|
||||||
|
|
||||||
try:
|
try:
|
||||||
code = TaskExecutor._wrap_code(raw_code)
|
wrapped_code = TaskExecutor._wrap_code(raw_code)
|
||||||
|
compiled_code = compile(wrapped_code, EXECUTOR_ALL_ITEMS_FILENAME, "exec")
|
||||||
|
|
||||||
globals = {
|
globals = {
|
||||||
"__builtins__": TaskExecutor._filter_builtins(builtins_deny),
|
"__builtins__": TaskExecutor._filter_builtins(builtins_deny),
|
||||||
@@ -144,7 +150,7 @@ class TaskExecutor:
|
|||||||
else print,
|
else print,
|
||||||
}
|
}
|
||||||
|
|
||||||
exec(code, globals)
|
exec(compiled_code, globals)
|
||||||
|
|
||||||
queue.put(
|
queue.put(
|
||||||
{"result": globals[EXECUTOR_USER_OUTPUT_KEY], "print_args": print_args}
|
{"result": globals[EXECUTOR_USER_OUTPUT_KEY], "print_args": print_args}
|
||||||
@@ -173,7 +179,7 @@ class TaskExecutor:
|
|||||||
|
|
||||||
try:
|
try:
|
||||||
wrapped_code = TaskExecutor._wrap_code(raw_code)
|
wrapped_code = TaskExecutor._wrap_code(raw_code)
|
||||||
compiled_code = compile(wrapped_code, "<per_item_task_execution>", "exec")
|
compiled_code = compile(wrapped_code, EXECUTOR_PER_ITEM_FILENAME, "exec")
|
||||||
|
|
||||||
result = []
|
result = []
|
||||||
for index, item in enumerate(items):
|
for index, item in enumerate(items):
|
||||||
|
|||||||
@@ -1,13 +1,13 @@
|
|||||||
import asyncio
|
import asyncio
|
||||||
from dataclasses import dataclass
|
|
||||||
import logging
|
import logging
|
||||||
import time
|
import time
|
||||||
from typing import Dict, Optional, Any, Set
|
from typing import Dict, Optional, Any
|
||||||
from urllib.parse import urlparse
|
from urllib.parse import urlparse
|
||||||
import websockets
|
import websockets
|
||||||
import random
|
import random
|
||||||
|
|
||||||
|
|
||||||
|
from src.config.task_runner_config import TaskRunnerConfig
|
||||||
from src.errors import (
|
from src.errors import (
|
||||||
WebsocketConnectionError,
|
WebsocketConnectionError,
|
||||||
TaskMissingError,
|
TaskMissingError,
|
||||||
@@ -64,28 +64,14 @@ class TaskOffer:
|
|||||||
return time.time() > self.valid_until
|
return time.time() > self.valid_until
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
|
||||||
class TaskRunnerOpts:
|
|
||||||
grant_token: str
|
|
||||||
task_broker_uri: str
|
|
||||||
max_concurrency: int
|
|
||||||
max_payload_size: int
|
|
||||||
task_timeout: int
|
|
||||||
stdlib_allow: Set[str]
|
|
||||||
external_allow: Set[str]
|
|
||||||
builtins_deny: Set[str]
|
|
||||||
|
|
||||||
|
|
||||||
class TaskRunner:
|
class TaskRunner:
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
opts: TaskRunnerOpts,
|
config: TaskRunnerConfig,
|
||||||
):
|
):
|
||||||
self.runner_id = nanoid()
|
self.runner_id = nanoid()
|
||||||
self.name = RUNNER_NAME
|
self.name = RUNNER_NAME
|
||||||
|
self.config = config
|
||||||
self.grant_token = opts.grant_token
|
|
||||||
self.opts = opts
|
|
||||||
|
|
||||||
self.websocket_connection: Optional[Any] = None
|
self.websocket_connection: Optional[Any] = None
|
||||||
self.can_send_offers = False
|
self.can_send_offers = False
|
||||||
@@ -96,23 +82,23 @@ class TaskRunner:
|
|||||||
self.offers_coroutine: Optional[asyncio.Task] = None
|
self.offers_coroutine: Optional[asyncio.Task] = None
|
||||||
self.serde = MessageSerde()
|
self.serde = MessageSerde()
|
||||||
self.executor = TaskExecutor()
|
self.executor = TaskExecutor()
|
||||||
self.analyzer = TaskAnalyzer(opts.stdlib_allow, opts.external_allow)
|
self.analyzer = TaskAnalyzer(config.stdlib_allow, config.external_allow)
|
||||||
self.logger = logging.getLogger(__name__)
|
self.logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
self.task_broker_uri = opts.task_broker_uri
|
self.task_broker_uri = config.task_broker_uri
|
||||||
websocket_host = urlparse(opts.task_broker_uri).netloc
|
websocket_host = urlparse(config.task_broker_uri).netloc
|
||||||
self.websocket_url = (
|
self.websocket_url = (
|
||||||
f"ws://{websocket_host}{TASK_BROKER_WS_PATH}?id={self.runner_id}"
|
f"ws://{websocket_host}{TASK_BROKER_WS_PATH}?id={self.runner_id}"
|
||||||
)
|
)
|
||||||
|
|
||||||
async def start(self) -> None:
|
async def start(self) -> None:
|
||||||
headers = {"Authorization": f"Bearer {self.grant_token}"}
|
headers = {"Authorization": f"Bearer {self.config.grant_token}"}
|
||||||
|
|
||||||
try:
|
try:
|
||||||
self.websocket_connection = await websockets.connect(
|
self.websocket_connection = await websockets.connect(
|
||||||
self.websocket_url,
|
self.websocket_url,
|
||||||
additional_headers=headers,
|
additional_headers=headers,
|
||||||
max_size=self.opts.max_payload_size,
|
max_size=self.config.max_payload_size,
|
||||||
)
|
)
|
||||||
|
|
||||||
self.logger.info("Connected to broker")
|
self.logger.info("Connected to broker")
|
||||||
@@ -182,7 +168,7 @@ class TaskRunner:
|
|||||||
await self._send_message(response)
|
await self._send_message(response)
|
||||||
return
|
return
|
||||||
|
|
||||||
if len(self.running_tasks) >= self.opts.max_concurrency:
|
if len(self.running_tasks) >= self.config.max_concurrency:
|
||||||
response = RunnerTaskRejected(
|
response = RunnerTaskRejected(
|
||||||
task_id=message.task_id,
|
task_id=message.task_id,
|
||||||
reason=TASK_REJECTED_REASON_AT_CAPACITY,
|
reason=TASK_REJECTED_REASON_AT_CAPACITY,
|
||||||
@@ -234,9 +220,9 @@ class TaskRunner:
|
|||||||
code=task_settings.code,
|
code=task_settings.code,
|
||||||
node_mode=task_settings.node_mode,
|
node_mode=task_settings.node_mode,
|
||||||
items=task_settings.items,
|
items=task_settings.items,
|
||||||
stdlib_allow=self.opts.stdlib_allow,
|
stdlib_allow=self.config.stdlib_allow,
|
||||||
external_allow=self.opts.external_allow,
|
external_allow=self.config.external_allow,
|
||||||
builtins_deny=self.opts.builtins_deny,
|
builtins_deny=self.config.builtins_deny,
|
||||||
can_log=task_settings.can_log,
|
can_log=task_settings.can_log,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -245,7 +231,7 @@ class TaskRunner:
|
|||||||
result, print_args = self.executor.execute_process(
|
result, print_args = self.executor.execute_process(
|
||||||
process=process,
|
process=process,
|
||||||
queue=queue,
|
queue=queue,
|
||||||
task_timeout=self.opts.task_timeout,
|
task_timeout=self.config.task_timeout,
|
||||||
continue_on_fail=task_settings.continue_on_fail,
|
continue_on_fail=task_settings.continue_on_fail,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -266,6 +252,7 @@ class TaskRunner:
|
|||||||
)
|
)
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
self.logger.error(f"Task {task_id} failed", exc_info=True)
|
||||||
response = RunnerTaskError(task_id=task_id, error={"message": str(e)})
|
response = RunnerTaskError(task_id=task_id, error={"message": str(e)})
|
||||||
await self._send_message(response)
|
await self._send_message(response)
|
||||||
|
|
||||||
@@ -344,7 +331,7 @@ class TaskRunner:
|
|||||||
for offer_id in expired_offer_ids:
|
for offer_id in expired_offer_ids:
|
||||||
self.open_offers.pop(offer_id, None)
|
self.open_offers.pop(offer_id, None)
|
||||||
|
|
||||||
offers_to_send = self.opts.max_concurrency - (
|
offers_to_send = self.config.max_concurrency - (
|
||||||
len(self.open_offers) + len(self.running_tasks)
|
len(self.open_offers) + len(self.running_tasks)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
42
packages/@n8n/task-runner-python/uv.lock
generated
42
packages/@n8n/task-runner-python/uv.lock
generated
@@ -2,6 +2,15 @@ version = 1
|
|||||||
revision = 3
|
revision = 3
|
||||||
requires-python = ">=3.13"
|
requires-python = ">=3.13"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "certifi"
|
||||||
|
version = "2025.8.3"
|
||||||
|
source = { registry = "https://pypi.org/simple" }
|
||||||
|
sdist = { url = "https://files.pythonhosted.org/packages/dc/67/960ebe6bf230a96cda2e0abcf73af550ec4f090005363542f0765df162e0/certifi-2025.8.3.tar.gz", hash = "sha256:e564105f78ded564e3ae7c923924435e1daa7463faeab5bb932bc53ffae63407", size = 162386, upload-time = "2025-08-03T03:07:47.08Z" }
|
||||||
|
wheels = [
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/e5/48/1549795ba7742c948d2ad169c1c8cdbae65bc450d6cd753d124b17c8cd32/certifi-2025.8.3-py3-none-any.whl", hash = "sha256:f6c12493cfb1b06ba2ff328595af9350c65d6644968e5d3a2ffd78699af217a5", size = 161216, upload-time = "2025-08-03T03:07:45.777Z" },
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "ruff"
|
name = "ruff"
|
||||||
version = "0.12.8"
|
version = "0.12.8"
|
||||||
@@ -27,6 +36,19 @@ wheels = [
|
|||||||
{ url = "https://files.pythonhosted.org/packages/cb/5c/799a1efb8b5abab56e8a9f2a0b72d12bd64bb55815e9476c7d0a2887d2f7/ruff-0.12.8-py3-none-win_arm64.whl", hash = "sha256:c90e1a334683ce41b0e7a04f41790c429bf5073b62c1ae701c9dc5b3d14f0749", size = 11884718, upload-time = "2025-08-07T19:05:42.866Z" },
|
{ url = "https://files.pythonhosted.org/packages/cb/5c/799a1efb8b5abab56e8a9f2a0b72d12bd64bb55815e9476c7d0a2887d2f7/ruff-0.12.8-py3-none-win_arm64.whl", hash = "sha256:c90e1a334683ce41b0e7a04f41790c429bf5073b62c1ae701c9dc5b3d14f0749", size = 11884718, upload-time = "2025-08-07T19:05:42.866Z" },
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "sentry-sdk"
|
||||||
|
version = "2.35.2"
|
||||||
|
source = { registry = "https://pypi.org/simple" }
|
||||||
|
dependencies = [
|
||||||
|
{ name = "certifi" },
|
||||||
|
{ name = "urllib3" },
|
||||||
|
]
|
||||||
|
sdist = { url = "https://files.pythonhosted.org/packages/bd/79/0ecb942f3f1ad26c40c27f81ff82392d85c01d26a45e3c72c2b37807e680/sentry_sdk-2.35.2.tar.gz", hash = "sha256:e9e8f3c795044beb59f2c8f4c6b9b0f9779e5e604099882df05eec525e782cc6", size = 343377, upload-time = "2025-09-01T11:00:58.633Z" }
|
||||||
|
wheels = [
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/c0/91/a43308dc82a0e32d80cd0dfdcfca401ecbd0f431ab45f24e48bb97b7800d/sentry_sdk-2.35.2-py2.py3-none-any.whl", hash = "sha256:38c98e3cbb620dd3dd80a8d6e39c753d453dd41f8a9df581b0584c19a52bc926", size = 363975, upload-time = "2025-09-01T11:00:56.574Z" },
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "task-runner-python"
|
name = "task-runner-python"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
@@ -35,6 +57,11 @@ dependencies = [
|
|||||||
{ name = "websockets" },
|
{ name = "websockets" },
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[package.optional-dependencies]
|
||||||
|
sentry = [
|
||||||
|
{ name = "sentry-sdk" },
|
||||||
|
]
|
||||||
|
|
||||||
[package.dev-dependencies]
|
[package.dev-dependencies]
|
||||||
dev = [
|
dev = [
|
||||||
{ name = "ruff" },
|
{ name = "ruff" },
|
||||||
@@ -42,7 +69,11 @@ dev = [
|
|||||||
]
|
]
|
||||||
|
|
||||||
[package.metadata]
|
[package.metadata]
|
||||||
requires-dist = [{ name = "websockets", specifier = ">=15.0.1" }]
|
requires-dist = [
|
||||||
|
{ name = "sentry-sdk", marker = "extra == 'sentry'", specifier = ">=2.35.2" },
|
||||||
|
{ name = "websockets", specifier = ">=15.0.1" },
|
||||||
|
]
|
||||||
|
provides-extras = ["sentry"]
|
||||||
|
|
||||||
[package.metadata.requires-dev]
|
[package.metadata.requires-dev]
|
||||||
dev = [
|
dev = [
|
||||||
@@ -75,6 +106,15 @@ wheels = [
|
|||||||
{ url = "https://files.pythonhosted.org/packages/98/c6/207bbc2f3bb71df4b1aeabe8e9c31a1cd22c72aff0ab9c1a832b9ae54f6e/ty-0.0.1a17-py3-none-win_arm64.whl", hash = "sha256:636eacc1dceaf09325415a70a03cd57eae53e5c7f281813aaa943a698a45cddb", size = 7782847, upload-time = "2025-08-06T12:13:54.243Z" },
|
{ url = "https://files.pythonhosted.org/packages/98/c6/207bbc2f3bb71df4b1aeabe8e9c31a1cd22c72aff0ab9c1a832b9ae54f6e/ty-0.0.1a17-py3-none-win_arm64.whl", hash = "sha256:636eacc1dceaf09325415a70a03cd57eae53e5c7f281813aaa943a698a45cddb", size = 7782847, upload-time = "2025-08-06T12:13:54.243Z" },
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "urllib3"
|
||||||
|
version = "2.5.0"
|
||||||
|
source = { registry = "https://pypi.org/simple" }
|
||||||
|
sdist = { url = "https://files.pythonhosted.org/packages/15/22/9ee70a2574a4f4599c47dd506532914ce044817c7752a79b6a51286319bc/urllib3-2.5.0.tar.gz", hash = "sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760", size = 393185, upload-time = "2025-06-18T14:07:41.644Z" }
|
||||||
|
wheels = [
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/a7/c2/fe1e52489ae3122415c51f387e221dd0773709bad6c6cdaa599e8a2c5185/urllib3-2.5.0-py3-none-any.whl", hash = "sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc", size = 129795, upload-time = "2025-06-18T14:07:40.39Z" },
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "websockets"
|
name = "websockets"
|
||||||
version = "15.0.1"
|
version = "15.0.1"
|
||||||
|
|||||||
Reference in New Issue
Block a user