nexus-queue 0.1.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,86 @@
1
+ """nexus-queue — portable taskiq + Redis Streams worker runtime.
2
+
3
+ Public API is intentionally small and stable: the config, the ports, the
4
+ envelope, and the error taxonomy. The runtime pieces (broker, middleware,
5
+ publisher, kicker) are added on top of these and re-exported as they land.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from nexus_queue.app import WorkerApp, create_worker
11
+ from nexus_queue.broker import create_broker
12
+ from nexus_queue.config import RuntimeConfig
13
+ from nexus_queue.delayed import DelayedRetryPoller
14
+ from nexus_queue.envelope import (
15
+ Envelope,
16
+ missing_required_labels,
17
+ require_supported_version,
18
+ )
19
+ from nexus_queue.exceptions import (
20
+ NexusPermanentError,
21
+ NexusQueueError,
22
+ NexusRetryableError,
23
+ )
24
+ from nexus_queue.handlers import HandlerSpec, register
25
+ from nexus_queue.kicker import create_kicker
26
+ from nexus_queue.lifecycle import (
27
+ IdempotencyStore,
28
+ configure_logging,
29
+ register_lifecycle,
30
+ )
31
+ from nexus_queue.naming import (
32
+ NQ_VERSION,
33
+ SINGLE_TENANT,
34
+ consumer_group,
35
+ delayed_set,
36
+ dlq_stream,
37
+ status_stream,
38
+ work_stream,
39
+ )
40
+ from nexus_queue.pipeline import PipelineRouter
41
+ from nexus_queue.ports import (
42
+ BlobStorePort,
43
+ IndexPort,
44
+ JobStatePort,
45
+ JobStatus,
46
+ StatusEvent,
47
+ StatusEventPort,
48
+ )
49
+ from nexus_queue.publisher import Publisher
50
+ from nexus_queue.tracing import configure_tracing
51
+
52
+ __all__ = [
53
+ "NQ_VERSION",
54
+ "SINGLE_TENANT",
55
+ "BlobStorePort",
56
+ "DelayedRetryPoller",
57
+ "Envelope",
58
+ "HandlerSpec",
59
+ "IdempotencyStore",
60
+ "IndexPort",
61
+ "JobStatePort",
62
+ "JobStatus",
63
+ "NexusPermanentError",
64
+ "NexusQueueError",
65
+ "NexusRetryableError",
66
+ "PipelineRouter",
67
+ "Publisher",
68
+ "RuntimeConfig",
69
+ "StatusEvent",
70
+ "StatusEventPort",
71
+ "WorkerApp",
72
+ "configure_logging",
73
+ "configure_tracing",
74
+ "consumer_group",
75
+ "create_broker",
76
+ "create_kicker",
77
+ "create_worker",
78
+ "delayed_set",
79
+ "dlq_stream",
80
+ "missing_required_labels",
81
+ "register",
82
+ "register_lifecycle",
83
+ "require_supported_version",
84
+ "status_stream",
85
+ "work_stream",
86
+ ]
nexus_queue/app.py ADDED
@@ -0,0 +1,47 @@
1
+ """Top-level factory. Consumers call ``create_worker(config, adapters, handlers)``
2
+ and expose the returned ``broker`` to the taskiq CLI and ``app`` to uvicorn.
3
+ """
4
+
5
+ from __future__ import annotations
6
+
7
+ from collections.abc import Iterator, Sequence
8
+ from dataclasses import dataclass
9
+
10
+ from fastapi import FastAPI
11
+ from taskiq import AsyncBroker
12
+
13
+ from nexus_queue.broker import create_broker
14
+ from nexus_queue.config import RuntimeConfig
15
+ from nexus_queue.handlers import HandlerSpec, register
16
+ from nexus_queue.kicker import create_kicker
17
+ from nexus_queue.lifecycle import configure_logging, register_lifecycle
18
+ from nexus_queue.tracing import configure_tracing
19
+
20
+
21
+ @dataclass(slots=True, frozen=True)
22
+ class WorkerApp:
23
+ """Bundle returned by :func:`create_worker`: ``app, broker = create_worker(...)``."""
24
+
25
+ app: FastAPI
26
+ broker: AsyncBroker
27
+
28
+ def __iter__(self) -> Iterator[FastAPI | AsyncBroker]:
29
+ yield self.app
30
+ yield self.broker
31
+
32
+
33
+ def create_worker(
34
+ config: RuntimeConfig,
35
+ adapters: object,
36
+ handlers: Sequence[HandlerSpec],
37
+ ) -> WorkerApp:
38
+ """Build the broker (namespaced + middleware), register lifecycle + handlers,
39
+ and wrap a FastAPI kicker."""
40
+ configure_logging(config)
41
+ configure_tracing(config)
42
+ broker = create_broker(config)
43
+ register_lifecycle(broker, config, adapters)
44
+ for spec in handlers:
45
+ register(broker, spec, config)
46
+ app = create_kicker(broker, config)
47
+ return WorkerApp(app=app, broker=broker)
nexus_queue/broker.py ADDED
@@ -0,0 +1,29 @@
1
+ """Broker factory — the one place that builds a namespaced, middleware-wrapped broker.
2
+
3
+ Consumers never instantiate ``RedisStreamBroker`` directly: that is how ZP and
4
+ nixon ended up on the default global ``"taskiq"`` stream (which collides across
5
+ projects). Here the stream and consumer group are always
6
+ ``nq:{project}:{queue}`` / ``…:cg``.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ from taskiq import AsyncBroker
12
+ from taskiq_redis import RedisStreamBroker
13
+
14
+ from nexus_queue.config import RuntimeConfig
15
+ from nexus_queue.middleware.metrics import MetricsMiddleware
16
+ from nexus_queue.middleware.retry_dlq import RetryDlqMiddleware
17
+
18
+
19
+ def create_broker(config: RuntimeConfig) -> AsyncBroker:
20
+ """Build the standard broker: namespaced streams + the Nexus-Queue middleware stack."""
21
+ broker: AsyncBroker = RedisStreamBroker(
22
+ url=config.redis_url,
23
+ queue_name=config.work_stream,
24
+ consumer_group_name=config.consumer_group,
25
+ )
26
+ return broker.with_middlewares(
27
+ MetricsMiddleware(config),
28
+ RetryDlqMiddleware(config),
29
+ )
nexus_queue/config.py ADDED
@@ -0,0 +1,98 @@
1
+ """Runtime configuration consumed by the worker runtime.
2
+
3
+ A single pydantic model filled in by the consumer. No env loading inside the
4
+ library (mirrors `agno-agent-builder` / `payload-documents-worker-builder`) so a
5
+ multi-tenant deploy can build several `RuntimeConfig` instances from one env.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from pydantic import BaseModel, Field, SecretStr, ValidationInfo, field_validator
11
+
12
+ from nexus_queue import naming
13
+
14
+
15
+ class RuntimeConfig(BaseModel):
16
+ """Everything the runtime needs that is not a port adapter or a handler."""
17
+
18
+ app_name: str = Field(
19
+ description="FastAPI title and structlog identity; shows up in logs and /health.",
20
+ )
21
+
22
+ # ── Identity / namespacing ─────────────────────────────────────────────
23
+ project: str = Field(
24
+ description="Stable short project slug used to namespace streams, e.g. 'zp', 'nixon'.",
25
+ )
26
+ queue: str = Field(
27
+ description="Logical queue/pipeline name, e.g. 'documents', 'jobs'.",
28
+ )
29
+
30
+ # ── Broker ─────────────────────────────────────────────────────────────
31
+ redis_url: str = Field(
32
+ description="Redis URL for the taskiq-redis stream broker (e.g. redis://redis:6379).",
33
+ )
34
+
35
+ # ── HTTP kicker ────────────────────────────────────────────────────────
36
+ internal_secret: SecretStr = Field(
37
+ default=SecretStr(""),
38
+ description="Shared secret required by the kicker (X-Nexus-Secret). Empty disables the kicker auth gate.",
39
+ )
40
+ public_paths: tuple[str, ...] = Field(
41
+ default=("/health", "/ready", "/metrics", "/docs", "/openapi.json"),
42
+ description="Kicker paths served without the secret.",
43
+ )
44
+
45
+ # ── Retry / DLQ / idempotency ──────────────────────────────────────────
46
+ max_retries: int = Field(
47
+ default=3,
48
+ ge=0,
49
+ description="Retry attempts before a message is dead-lettered.",
50
+ )
51
+ retry_base_delay_s: float = Field(
52
+ default=2.0,
53
+ gt=0,
54
+ description="Base delay for the exponential backoff applied between retries.",
55
+ )
56
+ retry_poll_interval_s: float = Field(
57
+ default=1.0,
58
+ gt=0,
59
+ description="How often the delayed-retry poller drains due retries back onto the work stream.",
60
+ )
61
+ idempotency_ttl_s: int = Field(
62
+ default=86_400,
63
+ ge=0,
64
+ description="TTL of the dedup key; 0 disables the idempotency middleware.",
65
+ )
66
+ dlq_maxlen: int = Field(
67
+ default=100_000,
68
+ ge=0,
69
+ description="Approx MAXLEN for the dead-letter stream; 0 = unbounded.",
70
+ )
71
+
72
+ # ── Logging ────────────────────────────────────────────────────────────
73
+ log_level: str = Field(default="INFO")
74
+
75
+ @field_validator("project", "queue")
76
+ @classmethod
77
+ def _validate_slug(cls, value: str, info: ValidationInfo) -> str:
78
+ return naming.validate_slug(value, kind=info.field_name or "slug")
79
+
80
+ @property
81
+ def work_stream(self) -> str:
82
+ return naming.work_stream(self.project, self.queue)
83
+
84
+ @property
85
+ def consumer_group(self) -> str:
86
+ return naming.consumer_group(self.project, self.queue)
87
+
88
+ @property
89
+ def dlq_stream(self) -> str:
90
+ return naming.dlq_stream(self.project, self.queue)
91
+
92
+ @property
93
+ def delayed_set(self) -> str:
94
+ return naming.delayed_set(self.project, self.queue)
95
+
96
+ @property
97
+ def status_stream(self) -> str:
98
+ return naming.status_stream(self.project)
nexus_queue/delayed.py ADDED
@@ -0,0 +1,91 @@
1
+ """Delayed-retry queue: hold a failed message until its backoff elapses, then
2
+ re-enqueue it.
3
+
4
+ Redis Streams deliver immediately, so a backed-off retry can't ride the work
5
+ stream. :class:`~nexus_queue.middleware.retry_dlq.RetryDlqMiddleware` parks the
6
+ message in a sorted set scored by its due time; the :class:`DelayedRetryPoller`
7
+ (started by the worker lifecycle) drains due messages back onto the broker. The
8
+ atomic ``ZREM`` makes the move safe across worker replicas — only the one that
9
+ removes a member re-enqueues it.
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ import asyncio
15
+ import contextlib
16
+ import json
17
+ import time
18
+ from typing import Any
19
+
20
+ import redis.asyncio as aioredis
21
+ import structlog
22
+ from taskiq import AsyncBroker
23
+ from taskiq.kicker import AsyncKicker
24
+ from taskiq.message import TaskiqMessage
25
+
26
+ from nexus_queue.config import RuntimeConfig
27
+
28
+ logger = structlog.get_logger("nexus_queue.delayed")
29
+
30
+
31
+ def pack_retry(message: TaskiqMessage, labels: dict[str, Any]) -> str:
32
+ """Serialize a message (with its updated labels) for the delayed-retry set."""
33
+ return json.dumps(
34
+ {
35
+ "task_name": message.task_name,
36
+ "task_id": message.task_id,
37
+ "labels": labels,
38
+ "args": message.args,
39
+ "kwargs": message.kwargs,
40
+ }
41
+ )
42
+
43
+
44
+ class DelayedRetryPoller:
45
+ """Moves due retries from the delayed sorted set back onto the broker."""
46
+
47
+ def __init__(self, broker: AsyncBroker, config: RuntimeConfig) -> None:
48
+ self._broker = broker
49
+ self._config = config
50
+ self._redis: aioredis.Redis | None = None
51
+ self._task: asyncio.Task[None] | None = None
52
+
53
+ async def startup(self) -> None:
54
+ self._redis = aioredis.from_url(self._config.redis_url)
55
+ self._task = asyncio.create_task(self._run())
56
+
57
+ async def shutdown(self) -> None:
58
+ if self._task is not None:
59
+ self._task.cancel()
60
+ with contextlib.suppress(asyncio.CancelledError):
61
+ await self._task
62
+ if self._redis is not None:
63
+ await self._redis.aclose()
64
+
65
+ async def _run(self) -> None:
66
+ while True:
67
+ try:
68
+ await self._drain_due()
69
+ except asyncio.CancelledError:
70
+ raise
71
+ except Exception:
72
+ logger.exception("delayed-poll-failed")
73
+ await asyncio.sleep(self._config.retry_poll_interval_s)
74
+
75
+ async def _drain_due(self) -> None:
76
+ if self._redis is None:
77
+ return
78
+ due = await self._redis.zrangebyscore(self._config.delayed_set, "-inf", time.time())
79
+ for record in due:
80
+ # Atomic claim: only the replica that removes the member re-enqueues it.
81
+ if await self._redis.zrem(self._config.delayed_set, record) == 1:
82
+ await self._reenqueue(json.loads(record))
83
+
84
+ async def _reenqueue(self, data: dict[str, Any]) -> None:
85
+ kicker: AsyncKicker[Any, Any] = AsyncKicker(
86
+ task_name=data["task_name"],
87
+ broker=self._broker,
88
+ labels=data["labels"],
89
+ ).with_task_id(data["task_id"])
90
+ await kicker.kiq(*data["args"], **data["kwargs"])
91
+ logger.info("retry-reenqueued", task=data["task_name"])
@@ -0,0 +1,56 @@
1
+ """The Nexus-Queue envelope: standard labels stamped on every message.
2
+
3
+ Producers build an :class:`Envelope` and turn it into the ``labels`` dict that
4
+ rides on the taskiq message; consumers validate the version on receipt.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ from collections.abc import Mapping
10
+ from dataclasses import dataclass
11
+ from datetime import UTC, datetime
12
+ from typing import Any
13
+
14
+ from nexus_queue import naming
15
+ from nexus_queue.exceptions import NexusPermanentError
16
+
17
+
18
+ @dataclass(slots=True)
19
+ class Envelope:
20
+ """Routing/metadata for a single enqueue. Carried in message labels, not args."""
21
+
22
+ task: str
23
+ tenant: str = naming.SINGLE_TENANT
24
+ idempotency_key: str | None = None
25
+ trace: str | None = None
26
+ priority: str = "default"
27
+
28
+ def to_labels(self) -> dict[str, str]:
29
+ """Render the standard labels. ``nq_enqueued_at`` is stamped now (UTC)."""
30
+ labels: dict[str, str] = {
31
+ naming.LABEL_VERSION: naming.NQ_VERSION,
32
+ naming.LABEL_TASK: self.task,
33
+ naming.LABEL_TENANT: self.tenant,
34
+ naming.LABEL_ENQUEUED_AT: datetime.now(UTC).isoformat(),
35
+ naming.LABEL_PRIORITY: self.priority,
36
+ }
37
+ if self.idempotency_key:
38
+ labels[naming.LABEL_IDEM] = self.idempotency_key
39
+ if self.trace:
40
+ labels[naming.LABEL_TRACE] = self.trace
41
+ return labels
42
+
43
+
44
+ def require_supported_version(labels: Mapping[str, Any]) -> None:
45
+ """Raise :class:`NexusPermanent` (→ DLQ) if the message is a version we
46
+ don't speak. Keeps a future ``nq_v=2`` from being silently mishandled."""
47
+ version = str(labels.get(naming.LABEL_VERSION, ""))
48
+ if version != naming.NQ_VERSION:
49
+ raise NexusPermanentError(
50
+ f"Unsupported nq_v={version!r}; this worker speaks {naming.NQ_VERSION!r}"
51
+ )
52
+
53
+
54
+ def missing_required_labels(labels: Mapping[str, Any]) -> tuple[str, ...]:
55
+ """Return the required labels absent from ``labels`` (empty tuple if valid)."""
56
+ return tuple(key for key in naming.REQUIRED_LABELS if not labels.get(key))
@@ -0,0 +1,15 @@
1
+ """Error taxonomy that drives the runtime's retry vs dead-letter decision."""
2
+
3
+ from __future__ import annotations
4
+
5
+
6
+ class NexusQueueError(Exception):
7
+ """Base class for every error raised by nexus-queue."""
8
+
9
+
10
+ class NexusRetryableError(NexusQueueError):
11
+ """Transient failure — retry until attempts are exhausted, then dead-letter."""
12
+
13
+
14
+ class NexusPermanentError(NexusQueueError):
15
+ """Non-retryable failure — route straight to the DLQ, skip remaining retries."""
@@ -0,0 +1,80 @@
1
+ """Handler registration with the cross-cutting concerns that need an ``around``
2
+ scope (which taskiq middleware hooks can't give): idempotency, tracing, latency.
3
+
4
+ A handler is ``async def h(payload, deps) -> None`` — it depends only on its
5
+ typed payload and the project's :class:`NexusAdapters`. ``register`` wraps it
6
+ so the same handler runs unchanged in any project.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ from collections.abc import Awaitable, Callable
12
+ from dataclasses import dataclass
13
+ from typing import Any
14
+
15
+ import structlog
16
+ from opentelemetry import trace
17
+ from opentelemetry.propagate import extract
18
+ from pydantic import BaseModel
19
+ from taskiq import AsyncBroker, Context, TaskiqDepends
20
+
21
+ from nexus_queue.config import RuntimeConfig
22
+ from nexus_queue.envelope import require_supported_version
23
+ from nexus_queue.lifecycle import IdempotencyStore
24
+ from nexus_queue.middleware.metrics import CONSUME_SECONDS
25
+ from nexus_queue.naming import LABEL_IDEM, LABEL_TENANT, LABEL_TRACE, SINGLE_TENANT
26
+
27
+ logger = structlog.get_logger("nexus_queue.handlers")
28
+ _tracer = trace.get_tracer("nexus_queue")
29
+
30
+ # deps is the project's adapter container; the handler types it to a Protocol
31
+ # of the ports it actually uses.
32
+ HandlerFn = Callable[[Any, Any], Awaitable[None]]
33
+
34
+
35
+ @dataclass(slots=True)
36
+ class HandlerSpec:
37
+ """Binds a wire task name to a handler and its payload model."""
38
+
39
+ task_name: str
40
+ handler: HandlerFn
41
+ payload_model: type[BaseModel]
42
+ idempotent: bool = True
43
+
44
+
45
+ def register(broker: AsyncBroker, spec: HandlerSpec, config: RuntimeConfig) -> None:
46
+ """Register ``spec.handler`` under its wire task name, wrapped with version
47
+ check, idempotency dedup, an OTel consume span, and a latency histogram."""
48
+
49
+ async def _run(context: Context = TaskiqDepends(), **kwargs: Any) -> None:
50
+ labels = context.message.labels
51
+ require_supported_version(labels)
52
+ state = context.state
53
+
54
+ raw_idem = labels.get(LABEL_IDEM) if spec.idempotent else None
55
+ idem = str(raw_idem) if raw_idem else None
56
+ store: IdempotencyStore | None = state.nexus_idempotency if idem else None
57
+ if store is not None and idem and await store.already_processed(idem):
58
+ logger.info("duplicate-skipped", task=spec.task_name, idem=idem)
59
+ return
60
+
61
+ adapters = state.nexus_adapters
62
+ payload = spec.payload_model(**kwargs)
63
+
64
+ traceparent = labels.get(LABEL_TRACE)
65
+ parent = extract({"traceparent": str(traceparent)}) if traceparent else None
66
+ with _tracer.start_as_current_span(
67
+ f"nexus_queue.consume {spec.task_name}",
68
+ context=parent,
69
+ ) as span:
70
+ span.set_attribute("nq.task", spec.task_name)
71
+ span.set_attribute("nq.tenant", str(labels.get(LABEL_TENANT, SINGLE_TENANT)))
72
+ with CONSUME_SECONDS.labels(config.project, config.queue, spec.task_name).time():
73
+ await spec.handler(payload, adapters)
74
+
75
+ # Record dedup only after success: a failed attempt must stay retryable.
76
+ if store is not None and idem:
77
+ await store.mark_processed(idem)
78
+
79
+ _run.__name__ = "nexus_run_" + spec.task_name.replace(".", "_").replace(":", "_")
80
+ broker.task(task_name=spec.task_name)(_run)
nexus_queue/kicker.py ADDED
@@ -0,0 +1,98 @@
1
+ """Generic HTTP kicker for producers that can't speak Redis directly
2
+ (e.g. a TypeScript/Payload caller). One standard contract for every project:
3
+ ``POST /enqueue/{task}`` gated by ``X-Nexus-Secret``, plus ``/health``/``/ready``.
4
+ """
5
+
6
+ from __future__ import annotations
7
+
8
+ import hmac
9
+ from collections.abc import AsyncIterator
10
+ from contextlib import asynccontextmanager
11
+ from typing import Any
12
+
13
+ import structlog
14
+ from fastapi import FastAPI, Request, Response, status
15
+ from fastapi.responses import JSONResponse
16
+ from prometheus_client import CONTENT_TYPE_LATEST, generate_latest
17
+ from pydantic import BaseModel, Field
18
+ from taskiq import AsyncBroker
19
+
20
+ from nexus_queue.config import RuntimeConfig
21
+ from nexus_queue.naming import SINGLE_TENANT
22
+ from nexus_queue.publisher import Publisher
23
+
24
+ logger = structlog.get_logger("nexus_queue.kicker")
25
+
26
+
27
+ class EnqueueRequest(BaseModel):
28
+ payload: dict[str, Any] = Field(default_factory=dict)
29
+ tenant: str = SINGLE_TENANT
30
+ idempotency_key: str | None = None
31
+ priority: str = "default"
32
+ trace: str | None = None
33
+
34
+
35
+ def create_kicker(broker: AsyncBroker, config: RuntimeConfig) -> FastAPI:
36
+ """Build the FastAPI kicker the consumer hands to uvicorn."""
37
+
38
+ @asynccontextmanager
39
+ async def lifespan(_: FastAPI) -> AsyncIterator[None]:
40
+ # Fail-open is allowed (some deploys front the kicker with a mesh), but it
41
+ # must never be silent: an empty secret leaves /enqueue unauthenticated.
42
+ if not config.internal_secret.get_secret_value():
43
+ logger.warning(
44
+ "kicker-auth-disabled",
45
+ detail="internal_secret is empty; /enqueue is unauthenticated",
46
+ )
47
+ # The kicker connects the broker; the worker process owns its own lifecycle.
48
+ if not broker.is_worker_process:
49
+ await broker.startup()
50
+ yield
51
+ if not broker.is_worker_process:
52
+ await broker.shutdown()
53
+
54
+ app = FastAPI(title=config.app_name, lifespan=lifespan)
55
+ publisher = Publisher(broker, config)
56
+
57
+ @app.middleware("http")
58
+ async def _auth(request: Request, call_next: Any) -> Any: # pyright: ignore[reportUnusedFunction]
59
+ if request.url.path in config.public_paths:
60
+ return await call_next(request)
61
+ secret = config.internal_secret.get_secret_value()
62
+ if secret:
63
+ provided = request.headers.get("x-nexus-secret", "")
64
+ if not hmac.compare_digest(provided, secret):
65
+ return JSONResponse(
66
+ {"error": "Forbidden"},
67
+ status_code=status.HTTP_403_FORBIDDEN,
68
+ )
69
+ return await call_next(request)
70
+
71
+ @app.get("/health")
72
+ async def health() -> dict[str, str]: # pyright: ignore[reportUnusedFunction]
73
+ return {"status": "ok"}
74
+
75
+ @app.get("/ready")
76
+ async def ready() -> dict[str, str]: # pyright: ignore[reportUnusedFunction]
77
+ return {"status": "ok"}
78
+
79
+ @app.get("/metrics")
80
+ async def metrics() -> Response: # pyright: ignore[reportUnusedFunction]
81
+ return Response(generate_latest(), media_type=CONTENT_TYPE_LATEST)
82
+
83
+ @app.post("/enqueue/{task}", status_code=status.HTTP_202_ACCEPTED)
84
+ async def enqueue( # pyright: ignore[reportUnusedFunction]
85
+ task: str,
86
+ body: EnqueueRequest,
87
+ ) -> dict[str, str]:
88
+ task_id = await publisher.enqueue_raw(
89
+ task,
90
+ body.payload,
91
+ tenant=body.tenant,
92
+ idempotency_key=body.idempotency_key,
93
+ priority=body.priority,
94
+ trace=body.trace,
95
+ )
96
+ return {"status": "queued", "task": task, "task_id": task_id}
97
+
98
+ return app
@@ -0,0 +1,104 @@
1
+ """Worker lifecycle: logging, dependency injection, and the idempotency store.
2
+
3
+ ``register_lifecycle`` wires the project's adapters and a Redis-backed
4
+ idempotency store into the worker ``state`` on startup, where the handler
5
+ wrapper (see :mod:`nexus_queue.handlers`) reads them.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import logging
11
+
12
+ import redis.asyncio as aioredis
13
+ import structlog
14
+ from taskiq import AsyncBroker, TaskiqEvents, TaskiqState
15
+
16
+ from nexus_queue.config import RuntimeConfig
17
+ from nexus_queue.delayed import DelayedRetryPoller
18
+ from nexus_queue.naming import idempotency_redis_key
19
+
20
+
21
+ class IdempotencyStore:
22
+ """Redis mark-done dedup keyed by the message's ``nq_idem`` label.
23
+
24
+ The key is recorded only *after* a handler succeeds, so a failed attempt
25
+ leaves no marker and stays eligible for retry/DLQ. Claiming the key up
26
+ front would make a re-enqueued retry skip itself as a phantom duplicate."""
27
+
28
+ def __init__(self, config: RuntimeConfig) -> None:
29
+ self._config = config
30
+ self._redis: aioredis.Redis | None = None
31
+
32
+ async def startup(self) -> None:
33
+ self._redis = aioredis.from_url(self._config.redis_url)
34
+
35
+ async def shutdown(self) -> None:
36
+ if self._redis is not None:
37
+ await self._redis.aclose()
38
+
39
+ async def already_processed(self, idem: str) -> bool:
40
+ """True if a message with this key already completed within the TTL."""
41
+ if self._config.idempotency_ttl_s <= 0 or self._redis is None:
42
+ return False
43
+ return bool(await self._redis.exists(idempotency_redis_key(idem)))
44
+
45
+ async def mark_processed(self, idem: str) -> None:
46
+ """Record completion so later duplicates are skipped within the TTL."""
47
+ if self._config.idempotency_ttl_s <= 0 or self._redis is None:
48
+ return
49
+ await self._redis.set(
50
+ idempotency_redis_key(idem),
51
+ "1",
52
+ ex=self._config.idempotency_ttl_s,
53
+ )
54
+
55
+
56
+ def configure_logging(config: RuntimeConfig) -> None:
57
+ """Idempotent structlog setup so taskiq + FastAPI share one JSON sink."""
58
+ level = getattr(logging, config.log_level.upper(), logging.INFO)
59
+ logging.basicConfig(level=level, format="%(message)s")
60
+ structlog.configure(
61
+ processors=[
62
+ structlog.contextvars.merge_contextvars,
63
+ structlog.processors.add_log_level,
64
+ structlog.processors.TimeStamper(fmt="iso"),
65
+ structlog.processors.StackInfoRenderer(),
66
+ structlog.processors.format_exc_info,
67
+ structlog.processors.JSONRenderer(),
68
+ ],
69
+ wrapper_class=structlog.make_filtering_bound_logger(level),
70
+ cache_logger_on_first_use=True,
71
+ )
72
+
73
+
74
+ def register_lifecycle(
75
+ broker: AsyncBroker,
76
+ config: RuntimeConfig,
77
+ adapters: object,
78
+ ) -> None:
79
+ """Register startup/shutdown hooks that expose config, adapters and the
80
+ idempotency store on the worker ``state``.
81
+
82
+ ``adapters`` is whatever container the project chooses — the runtime only
83
+ stashes it on ``state`` for the handler wrapper to hand back as ``deps``.
84
+ The package contributes the *ports* (see :mod:`nexus_queue.ports`); the
85
+ *container* is the project's, so a handler can depend on exactly the ports
86
+ it needs."""
87
+
88
+ @broker.on_event(TaskiqEvents.WORKER_STARTUP)
89
+ async def _startup(state: TaskiqState) -> None: # pyright: ignore[reportUnusedFunction]
90
+ state.nexus_config = config
91
+ state.nexus_adapters = adapters
92
+ store = IdempotencyStore(config)
93
+ await store.startup()
94
+ state.nexus_idempotency = store
95
+ poller = DelayedRetryPoller(broker, config)
96
+ await poller.startup()
97
+ state.nexus_delayed = poller
98
+
99
+ @broker.on_event(TaskiqEvents.WORKER_SHUTDOWN)
100
+ async def _shutdown(state: TaskiqState) -> None: # pyright: ignore[reportUnusedFunction]
101
+ store: IdempotencyStore = state.nexus_idempotency
102
+ await store.shutdown()
103
+ poller: DelayedRetryPoller = state.nexus_delayed
104
+ await poller.shutdown()
@@ -0,0 +1,8 @@
1
+ """Broker-level middleware for the Nexus-Queue runtime."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from nexus_queue.middleware.metrics import MetricsMiddleware
6
+ from nexus_queue.middleware.retry_dlq import RetryDlqMiddleware
7
+
8
+ __all__ = ["MetricsMiddleware", "RetryDlqMiddleware"]
@@ -0,0 +1,59 @@
1
+ """Prometheus counters for queue throughput.
2
+
3
+ Latency is measured in the handler wrapper (it needs an ``around`` scope that
4
+ middleware hooks can't give); here we keep the stateless counters.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ from typing import Any
10
+
11
+ from prometheus_client import Counter, Histogram
12
+ from taskiq.abc.middleware import TaskiqMiddleware
13
+ from taskiq.message import TaskiqMessage
14
+ from taskiq.result import TaskiqResult
15
+
16
+ from nexus_queue.config import RuntimeConfig
17
+ from nexus_queue.naming import LABEL_TASK
18
+
19
+ _LABELNAMES = ("project", "queue", "task")
20
+
21
+ RECEIVED = Counter(
22
+ "nexus_queue_received_total",
23
+ "Messages pulled from the stream for execution.",
24
+ _LABELNAMES,
25
+ )
26
+ COMPLETED = Counter(
27
+ "nexus_queue_completed_total",
28
+ "Messages whose handler returned successfully.",
29
+ _LABELNAMES,
30
+ )
31
+ FAILED = Counter(
32
+ "nexus_queue_failed_total",
33
+ "Messages whose handler raised (before retry/DLQ resolution).",
34
+ _LABELNAMES,
35
+ )
36
+ CONSUME_SECONDS = Histogram(
37
+ "nexus_queue_consume_seconds",
38
+ "Handler execution wall-time (measured in the handler wrapper).",
39
+ _LABELNAMES,
40
+ )
41
+
42
+
43
+ class MetricsMiddleware(TaskiqMiddleware):
44
+ """Increment throughput counters around execution."""
45
+
46
+ def __init__(self, config: RuntimeConfig) -> None:
47
+ super().__init__()
48
+ self._config = config
49
+
50
+ def _task(self, message: TaskiqMessage) -> str:
51
+ return str(message.labels.get(LABEL_TASK, message.task_name))
52
+
53
+ def pre_execute(self, message: TaskiqMessage) -> TaskiqMessage:
54
+ RECEIVED.labels(self._config.project, self._config.queue, self._task(message)).inc()
55
+ return message
56
+
57
+ def post_execute(self, message: TaskiqMessage, result: TaskiqResult[Any]) -> None:
58
+ counter = FAILED if result.is_err else COMPLETED
59
+ counter.labels(self._config.project, self._config.queue, self._task(message)).inc()
@@ -0,0 +1,125 @@
1
+ """Retry + dead-letter middleware.
2
+
3
+ Unifies retry and DLQ in one place so the two decisions stay consistent:
4
+
5
+ * transient error and attempts remain → park the message in the delayed-retry
6
+ sorted set scored by its due time (``now + exponential backoff``). Redis
7
+ Streams deliver immediately, so the backoff can't ride the work stream; the
8
+ :class:`~nexus_queue.delayed.DelayedRetryPoller` moves the message back onto
9
+ the broker once its delay elapses.
10
+ * attempts exhausted, or a :class:`NexusPermanentError` → ``XADD`` to the
11
+ ``nq:{project}:{queue}:dlq`` stream with failure metadata, instead of the
12
+ silent ack-and-drop that both reference projects do today.
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ import json
18
+ import random
19
+ import time
20
+ from datetime import UTC, datetime
21
+ from typing import Any
22
+
23
+ import redis.asyncio as aioredis
24
+ import structlog
25
+ from taskiq.abc.middleware import TaskiqMiddleware
26
+ from taskiq.exceptions import NoResultError
27
+ from taskiq.message import TaskiqMessage
28
+ from taskiq.result import TaskiqResult
29
+
30
+ from nexus_queue.config import RuntimeConfig
31
+ from nexus_queue.delayed import pack_retry
32
+ from nexus_queue.exceptions import NexusPermanentError
33
+
34
+ logger = structlog.get_logger("nexus_queue.retry_dlq")
35
+
36
+ # SystemRandom keeps ruff's bandit check (S311) happy without a noqa: jitter is
37
+ # not security-sensitive, but a CSPRNG is a fine source for it.
38
+ _jitter = random.SystemRandom()
39
+
40
+ _MAX_BACKOFF_S = 60.0
41
+
42
+
43
+ class RetryDlqMiddleware(TaskiqMiddleware):
44
+ """Re-enqueue transient failures with backoff; dead-letter the rest."""
45
+
46
+ def __init__(self, config: RuntimeConfig) -> None:
47
+ super().__init__()
48
+ self._config = config
49
+ self._redis: aioredis.Redis | None = None
50
+
51
+ async def startup(self) -> None:
52
+ self._redis = aioredis.from_url(self._config.redis_url)
53
+
54
+ async def shutdown(self) -> None:
55
+ if self._redis is not None:
56
+ await self._redis.aclose()
57
+
58
+ def _backoff_delay(self, attempt: int) -> float:
59
+ base = self._config.retry_base_delay_s * (2.0 ** max(attempt - 1, 0))
60
+ return min(base, _MAX_BACKOFF_S) + _jitter.random()
61
+
62
+ async def on_error(
63
+ self,
64
+ message: TaskiqMessage,
65
+ result: TaskiqResult[Any],
66
+ exception: BaseException,
67
+ ) -> None:
68
+ if isinstance(exception, NoResultError):
69
+ return
70
+
71
+ permanent = isinstance(exception, NexusPermanentError)
72
+ attempt = int(message.labels.get("_retries", 0)) + 1
73
+ max_retries = int(message.labels.get("max_retries", self._config.max_retries))
74
+
75
+ if not permanent and attempt < max_retries:
76
+ delay = self._backoff_delay(attempt)
77
+ labels = {**message.labels, "_retries": attempt}
78
+ record = pack_retry(message, labels)
79
+ if self._redis is not None:
80
+ await self._redis.zadd(self._config.delayed_set, {record: time.time() + delay})
81
+ result.error = NoResultError()
82
+ logger.info(
83
+ "retry-scheduled",
84
+ task=message.task_name,
85
+ attempt=attempt,
86
+ max_retries=max_retries,
87
+ delay_s=round(delay, 2),
88
+ )
89
+ return
90
+
91
+ await self._dead_letter(message, exception, attempts=attempt, permanent=permanent)
92
+
93
+ async def _dead_letter(
94
+ self,
95
+ message: TaskiqMessage,
96
+ exception: BaseException,
97
+ *,
98
+ attempts: int,
99
+ permanent: bool,
100
+ ) -> None:
101
+ record: dict[str, Any] = {
102
+ "task_id": message.task_id,
103
+ "task_name": message.task_name,
104
+ "labels": message.labels,
105
+ "args": message.args,
106
+ "kwargs": message.kwargs,
107
+ "error": repr(exception),
108
+ "permanent": permanent,
109
+ "attempts": attempts,
110
+ "failed_at": datetime.now(UTC).isoformat(),
111
+ }
112
+ if self._redis is not None:
113
+ await self._redis.xadd(
114
+ self._config.dlq_stream,
115
+ {"data": json.dumps(record, default=str)},
116
+ maxlen=self._config.dlq_maxlen or None,
117
+ approximate=True,
118
+ )
119
+ logger.warning(
120
+ "dead-letter",
121
+ task=message.task_name,
122
+ stream=self._config.dlq_stream,
123
+ permanent=permanent,
124
+ attempts=attempts,
125
+ )
nexus_queue/naming.py ADDED
@@ -0,0 +1,72 @@
1
+ """Stream/group naming and envelope label keys for the Nexus-Queue wire contract.
2
+
3
+ Single source of truth for the strings that travel on the wire. Producers and
4
+ consumers in any language must agree on these; the TypeScript client mirrors
5
+ the same constants.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import re
11
+
12
+ #: Wire-contract version. Bumped only on incompatible envelope changes.
13
+ NQ_VERSION = "1"
14
+
15
+ # ── Envelope label keys (carried in TaskiqMessage.labels) ──────────────────
16
+ LABEL_VERSION = "nq_v"
17
+ LABEL_TASK = "nq_task"
18
+ LABEL_TENANT = "nq_tenant"
19
+ LABEL_IDEM = "nq_idem"
20
+ LABEL_TRACE = "nq_trace"
21
+ LABEL_ENQUEUED_AT = "nq_enqueued_at"
22
+ LABEL_PRIORITY = "nq_priority"
23
+
24
+ #: Labels a conformant message MUST carry.
25
+ REQUIRED_LABELS: tuple[str, ...] = (
26
+ LABEL_VERSION,
27
+ LABEL_TASK,
28
+ LABEL_TENANT,
29
+ LABEL_ENQUEUED_AT,
30
+ )
31
+
32
+ #: Sentinel tenant for single-tenant deployments.
33
+ SINGLE_TENANT = "_"
34
+
35
+ _SLUG = re.compile(r"^[a-z0-9][a-z0-9-]*$")
36
+
37
+
38
+ def validate_slug(part: str, *, kind: str) -> str:
39
+ """Reject slugs that would break the ``nq:{project}:{queue}`` scheme."""
40
+ if not _SLUG.match(part):
41
+ raise ValueError(f"Invalid {kind} {part!r}: must match [a-z0-9][a-z0-9-]*")
42
+ return part
43
+
44
+
45
+ def work_stream(project: str, queue: str) -> str:
46
+ """Redis stream key that carries work for a queue."""
47
+ return f"nq:{project}:{queue}"
48
+
49
+
50
+ def consumer_group(project: str, queue: str) -> str:
51
+ """Consumer group name for a queue's workers."""
52
+ return f"nq:{project}:{queue}:cg"
53
+
54
+
55
+ def dlq_stream(project: str, queue: str) -> str:
56
+ """Dead-letter stream key for a queue."""
57
+ return f"nq:{project}:{queue}:dlq"
58
+
59
+
60
+ def delayed_set(project: str, queue: str) -> str:
61
+ """Sorted-set key holding retries until their backoff elapses."""
62
+ return f"nq:{project}:{queue}:delayed"
63
+
64
+
65
+ def status_stream(project: str) -> str:
66
+ """Per-project status-event stream key."""
67
+ return f"nq:{project}:status"
68
+
69
+
70
+ def idempotency_redis_key(idem: str) -> str:
71
+ """Redis key used by the idempotency middleware to dedup a message."""
72
+ return f"nq:idem:{idem}"
@@ -0,0 +1,49 @@
1
+ """Pipeline routing: enqueue the next stage of a multi-step job, propagating
2
+ tenant + trace from the current message so the whole pipeline is one trace.
3
+
4
+ Single-stage workers (e.g. ZP documents) don't need this; nixon's split
5
+ pipeline does.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from typing import Any
11
+
12
+ from pydantic import BaseModel
13
+ from taskiq import AsyncBroker, TaskiqMessage
14
+ from taskiq.kicker import AsyncKicker
15
+
16
+ from nexus_queue.config import RuntimeConfig
17
+ from nexus_queue.envelope import Envelope
18
+ from nexus_queue.naming import LABEL_PRIORITY, LABEL_TENANT, LABEL_TRACE, SINGLE_TENANT
19
+
20
+
21
+ class PipelineRouter:
22
+ """Forwards a job to its next stage, carrying the envelope context over."""
23
+
24
+ def __init__(self, broker: AsyncBroker, config: RuntimeConfig) -> None:
25
+ self._broker = broker
26
+ self._config = config
27
+
28
+ async def forward(
29
+ self,
30
+ next_task: str,
31
+ payload: BaseModel,
32
+ *,
33
+ source: TaskiqMessage,
34
+ ) -> str:
35
+ """Enqueue ``next_task`` propagating tenant/trace/priority from ``source``."""
36
+ labels = source.labels
37
+ envelope = Envelope(
38
+ task=next_task,
39
+ tenant=str(labels.get(LABEL_TENANT, SINGLE_TENANT)),
40
+ trace=str(labels[LABEL_TRACE]) if labels.get(LABEL_TRACE) else None,
41
+ priority=str(labels.get(LABEL_PRIORITY, "default")),
42
+ )
43
+ kicker: AsyncKicker[Any, Any] = AsyncKicker(
44
+ task_name=next_task,
45
+ broker=self._broker,
46
+ labels=envelope.to_labels(),
47
+ )
48
+ task_obj = await kicker.kiq(**payload.model_dump())
49
+ return str(task_obj.task_id)
nexus_queue/ports.py ADDED
@@ -0,0 +1,81 @@
1
+ """Ports — the contract that makes workers movable across projects.
2
+
3
+ A handler depends only on these Protocols; each project provides the adapters
4
+ (e.g. ZP maps `JobStatePort` to a Payload document's ``parse_*`` fields, nixon
5
+ to its Postgres job state machine). Handlers never import a project's domain.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from collections.abc import Mapping, Sequence
11
+ from enum import StrEnum
12
+ from typing import Any, Protocol, runtime_checkable
13
+
14
+ from pydantic import BaseModel, Field
15
+
16
+ from nexus_queue.naming import NQ_VERSION, SINGLE_TENANT
17
+
18
+
19
+ class JobStatus(StrEnum):
20
+ """Minimal common job lifecycle. Projects may track richer states in their
21
+ own store; this is the subset the ports speak."""
22
+
23
+ CREATED = "created"
24
+ PROCESSING = "processing"
25
+ COMPLETED = "completed"
26
+ FAILED = "failed"
27
+
28
+
29
+ @runtime_checkable
30
+ class JobStatePort(Protocol):
31
+ """Transitions the job's state in the project's system-of-record.
32
+
33
+ The producer creates the job before enqueueing; the handler drives it from
34
+ here. Implementations must be idempotent (``complete`` of an already
35
+ completed job is a no-op)."""
36
+
37
+ async def processing(self, job_id: str, *, meta: Mapping[str, Any] | None = None) -> None: ...
38
+
39
+ async def complete(self, job_id: str, *, result: Mapping[str, Any]) -> None: ...
40
+
41
+ async def fail(self, job_id: str, *, error: str, permanent: bool) -> None: ...
42
+
43
+ async def get(self, job_id: str) -> JobStatus: ...
44
+
45
+
46
+ @runtime_checkable
47
+ class BlobStorePort(Protocol):
48
+ """Binary storage behind an opaque reference (MinIO, Payload media, S3…)."""
49
+
50
+ async def get(self, ref: str) -> bytes: ...
51
+
52
+ async def put(self, ref: str, data: bytes, *, content_type: str) -> str: ...
53
+
54
+
55
+ @runtime_checkable
56
+ class IndexPort(Protocol):
57
+ """Search index upserts/deletes (Typesense in both reference projects)."""
58
+
59
+ async def upsert(self, collection: str, docs: Sequence[Mapping[str, Any]]) -> None: ...
60
+
61
+ async def delete(self, collection: str, ids: Sequence[str]) -> None: ...
62
+
63
+
64
+ class StatusEvent(BaseModel):
65
+ """Versioned status event published to ``nq:{project}:status``."""
66
+
67
+ nq_v: str = Field(default=NQ_VERSION)
68
+ job_id: str
69
+ task: str
70
+ tenant: str = Field(default=SINGLE_TENANT)
71
+ state: JobStatus
72
+ ts: str
73
+ trace: str | None = None
74
+ error: str | None = None
75
+
76
+
77
+ @runtime_checkable
78
+ class StatusEventPort(Protocol):
79
+ """Emits a status event (push-status; generalizes nixon's domain-event stream)."""
80
+
81
+ async def emit(self, event: StatusEvent) -> None: ...
@@ -0,0 +1,78 @@
1
+ """Python producer. Stamps the standard envelope labels (and the current OTel
2
+ traceparent for end-to-end tracing) and enqueues onto the namespaced stream.
3
+
4
+ The TypeScript producer (`@zetesis/nexus-queue`) emits the same shape.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ from typing import Any
10
+
11
+ from opentelemetry.propagate import inject
12
+ from pydantic import BaseModel
13
+ from taskiq import AsyncBroker
14
+ from taskiq.kicker import AsyncKicker
15
+
16
+ from nexus_queue.config import RuntimeConfig
17
+ from nexus_queue.envelope import Envelope
18
+ from nexus_queue.naming import SINGLE_TENANT
19
+
20
+
21
+ def _current_traceparent() -> str | None:
22
+ carrier: dict[str, str] = {}
23
+ inject(carrier)
24
+ return carrier.get("traceparent")
25
+
26
+
27
+ class Publisher:
28
+ """Enqueues tasks by name with the Nexus-Queue envelope."""
29
+
30
+ def __init__(self, broker: AsyncBroker, config: RuntimeConfig) -> None:
31
+ self._broker = broker
32
+ self._config = config
33
+
34
+ async def enqueue(
35
+ self,
36
+ task: str,
37
+ payload: BaseModel,
38
+ *,
39
+ tenant: str = SINGLE_TENANT,
40
+ idempotency_key: str | None = None,
41
+ priority: str = "default",
42
+ trace: str | None = None,
43
+ ) -> str:
44
+ """Enqueue a typed payload; returns the taskiq task id."""
45
+ return await self.enqueue_raw(
46
+ task,
47
+ payload.model_dump(),
48
+ tenant=tenant,
49
+ idempotency_key=idempotency_key,
50
+ priority=priority,
51
+ trace=trace,
52
+ )
53
+
54
+ async def enqueue_raw(
55
+ self,
56
+ task: str,
57
+ payload: dict[str, Any],
58
+ *,
59
+ tenant: str = SINGLE_TENANT,
60
+ idempotency_key: str | None = None,
61
+ priority: str = "default",
62
+ trace: str | None = None,
63
+ ) -> str:
64
+ """Enqueue a raw kwargs payload (used by the HTTP kicker)."""
65
+ envelope = Envelope(
66
+ task=task,
67
+ tenant=tenant,
68
+ idempotency_key=idempotency_key,
69
+ trace=trace or _current_traceparent(),
70
+ priority=priority,
71
+ )
72
+ kicker: AsyncKicker[Any, Any] = AsyncKicker(
73
+ task_name=task,
74
+ broker=self._broker,
75
+ labels=envelope.to_labels(),
76
+ )
77
+ task_obj = await kicker.kiq(**payload)
78
+ return str(task_obj.task_id)
nexus_queue/py.typed ADDED
File without changes
nexus_queue/tracing.py ADDED
@@ -0,0 +1,38 @@
1
+ """Optional OpenTelemetry tracing setup for the worker.
2
+
3
+ The runtime instruments spans through the OTel *API* (see :mod:`nexus_queue.handlers`),
4
+ but emitting them needs an SDK + exporter. A deployment opts in by setting the
5
+ standard ``OTEL_EXPORTER_OTLP_ENDPOINT``; with it unset the spans stay no-op, so
6
+ the same image runs untraced where no collector exists.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import os
12
+
13
+ import structlog
14
+ from opentelemetry import trace
15
+ from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
16
+ from opentelemetry.sdk.resources import Resource
17
+ from opentelemetry.sdk.trace import TracerProvider
18
+ from opentelemetry.sdk.trace.export import BatchSpanProcessor
19
+
20
+ from nexus_queue.config import RuntimeConfig
21
+
22
+ logger = structlog.get_logger("nexus_queue.tracing")
23
+
24
+ OTLP_ENDPOINT_ENV = "OTEL_EXPORTER_OTLP_ENDPOINT"
25
+
26
+
27
+ def configure_tracing(config: RuntimeConfig) -> None:
28
+ """Wire an OTLP span exporter when a collector endpoint is configured.
29
+
30
+ No-op unless ``OTEL_EXPORTER_OTLP_ENDPOINT`` is set. The exporter reads the
31
+ endpoint (and any other ``OTEL_*`` options) from the environment itself."""
32
+ endpoint = os.environ.get(OTLP_ENDPOINT_ENV)
33
+ if not endpoint:
34
+ return
35
+ provider = TracerProvider(resource=Resource.create({"service.name": config.app_name}))
36
+ provider.add_span_processor(BatchSpanProcessor(OTLPSpanExporter()))
37
+ trace.set_tracer_provider(provider)
38
+ logger.info("tracing-configured", endpoint=endpoint, service=config.app_name)
@@ -0,0 +1,61 @@
1
+ Metadata-Version: 2.4
2
+ Name: nexus-queue
3
+ Version: 0.1.1
4
+ Summary: Nexus-Queue — portable taskiq + Redis Streams worker runtime: namespaced streams, versioned envelope, ports-and-adapters, retry/DLQ/idempotency/tracing/metrics middleware.
5
+ Project-URL: Homepage, https://github.com/Zetesis-Labs/PayloadAgents
6
+ Project-URL: Repository, https://github.com/Zetesis-Labs/PayloadAgents
7
+ Project-URL: Issues, https://github.com/Zetesis-Labs/PayloadAgents/issues
8
+ Author: Zetesis Labs
9
+ License: MIT
10
+ Keywords: dlq,ports-and-adapters,queue,redis,taskiq,worker
11
+ Classifier: Development Status :: 4 - Beta
12
+ Classifier: Framework :: FastAPI
13
+ Classifier: Intended Audience :: Developers
14
+ Classifier: License :: OSI Approved :: MIT License
15
+ Classifier: Programming Language :: Python :: 3.12
16
+ Classifier: Programming Language :: Python :: 3.13
17
+ Classifier: Topic :: Software Development :: Libraries :: Python Modules
18
+ Classifier: Typing :: Typed
19
+ Requires-Python: >=3.12
20
+ Requires-Dist: fastapi>=0.115
21
+ Requires-Dist: opentelemetry-api>=1.35
22
+ Requires-Dist: opentelemetry-exporter-otlp-proto-http>=1.35
23
+ Requires-Dist: opentelemetry-sdk>=1.35
24
+ Requires-Dist: prometheus-client>=0.20
25
+ Requires-Dist: pydantic>=2.9
26
+ Requires-Dist: redis>=5.0
27
+ Requires-Dist: structlog>=24.1
28
+ Requires-Dist: taskiq-redis>=1.1.1
29
+ Requires-Dist: taskiq>=0.11.18
30
+ Requires-Dist: uvicorn[standard]>=0.34
31
+ Description-Content-Type: text/markdown
32
+
33
+ # nexus-queue
34
+
35
+ Portable worker runtime for the **Nexus-Queue** standard: taskiq + Redis Streams,
36
+ domain-agnostic, ports-and-adapters. One project builds queues the same way as
37
+ the next, and a worker's handlers move between projects unchanged.
38
+
39
+ What this package owns (the parts that are the *same* across projects):
40
+
41
+ - **Namespaced streams** — `nq:{project}:{queue}` (+ `:cg`, `:dlq`) and
42
+ `nq:{project}:status`, so multiple projects/queues coexist on one Redis
43
+ (the default global `"taskiq"` stream is never used).
44
+ - **Versioned envelope** — standard labels (`nq_v`, `nq_task`, `nq_tenant`,
45
+ `nq_idem`, `nq_trace`, `nq_enqueued_at`, `nq_priority`) on top of taskiq's
46
+ message, plus typed pydantic payloads.
47
+ - **Ports** — `JobStatePort`, `BlobStorePort`, `IndexPort`, `StatusEventPort`.
48
+ Handlers depend only on these; each project supplies the adapters
49
+ (e.g. Payload vs Postgres/MinIO).
50
+ - **Middleware stack** (broker-level): idempotency (dedup on `nq_idem`),
51
+ DLQ (dead-letter on retry-exhaustion instead of silent drop), retries with
52
+ exponential backoff + jitter, OTel tracing (`nq_trace` propagation), and
53
+ Prometheus metrics.
54
+ - **Producer + kicker** — a Python `Publisher` and a generic HTTP kicker for
55
+ non-Python producers. The TypeScript producer client ships as
56
+ `@zetesis/nexus-queue`.
57
+
58
+ Spec: `nexus-queue-spec.md`.
59
+
60
+ Released via release-please on every conventional commit to `main`
61
+ (scope `nexus-queue`).
@@ -0,0 +1,22 @@
1
+ nexus_queue/__init__.py,sha256=vZC0Kh4XgC9PJAQUu_bjDHmp7I94L97jkgJVroF_rNg,2121
2
+ nexus_queue/app.py,sha256=xJ58WaTLGrpor0GiSoxfiagqIvzfUN_CjvWmoSM98lY,1484
3
+ nexus_queue/broker.py,sha256=zqUQFOMnZ3XDs0gegWC75wcL1F02514_0WjXu8QZ1hI,1066
4
+ nexus_queue/config.py,sha256=34GIQsRmbR7xb1Xy2iSumZVHtkLrH0xqwqkTCFX2fGU,3979
5
+ nexus_queue/delayed.py,sha256=FCDc88C1GujhH3yB9eEWAaLlv3JDgvgFBJRpM4iHRXU,3227
6
+ nexus_queue/envelope.py,sha256=7k81IrrPBijoDXp8uE60Fh4J57Fpetwc8t-3TRI6XOI,2069
7
+ nexus_queue/exceptions.py,sha256=ceB2nn6jHft_y32Qmmoq40hAgJ7mto3jWILNf2MogJE,475
8
+ nexus_queue/handlers.py,sha256=VWR3FRkalbEPzHbvg6JqaIjA3vjM2uokq5ONr_gGsgs,3254
9
+ nexus_queue/kicker.py,sha256=hwbCfd-vyxFsDPghaF8dBg5lVqYkwHWQftnDngKGhVQ,3648
10
+ nexus_queue/lifecycle.py,sha256=-H9Z-VTYab1CeLGz0iMuQBBxjyNr5I-1xd6G0gaoTEs,4042
11
+ nexus_queue/naming.py,sha256=uAkfFHT-DZMtPL3uKVt5gSSfZZ0yVxBDXXc-rzWdN3U,2108
12
+ nexus_queue/pipeline.py,sha256=L6ytmQhbCSEIRCe_VUqyMa8v_kfJrJNQxlONHRu0JLw,1647
13
+ nexus_queue/ports.py,sha256=wnRavsKSS4JReHhc5MfOTL-dtOxW08U5mQYn8CISjzA,2545
14
+ nexus_queue/publisher.py,sha256=zDgSVvwI4XIJtsEZrcCufEReOtOUOmHmb8zs_hCt6oM,2283
15
+ nexus_queue/tracing.py,sha256=lNSxUTAOO7VJBkOKP7EX0ns0DBWP-8nXdKqrieiF2A4,1520
16
+ nexus_queue/middleware/__init__.py,sha256=7dAc7nzCLrfOsZQOXYLOhxHSW7HUXbQ7c4zpZ2w8Q8M,276
17
+ nexus_queue/middleware/metrics.py,sha256=Dah5GrUSs2VJWNK5P7sSV1dyh4esEfr510tDASO3i64,1875
18
+ nexus_queue/middleware/retry_dlq.py,sha256=zAzdrMhzJjYyUsdntTtL3Jc6LmcrkzfI-U9Gcj8HPWM,4361
19
+ nexus_queue/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
20
+ nexus_queue-0.1.1.dist-info/METADATA,sha256=nuIq6JILH-c3mCN93ND1TBfqb4QK62I6KFf6a_syLqA,2821
21
+ nexus_queue-0.1.1.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
22
+ nexus_queue-0.1.1.dist-info/RECORD,,
@@ -0,0 +1,4 @@
1
+ Wheel-Version: 1.0
2
+ Generator: hatchling 1.30.1
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any